chore: format test files with ruff (enable formatter on tests/) (#2074)
Tests were excluded from both ruff lint and format via the top-level [tool.ruff].exclude in hindsight-api-slim, hindsight-embed and the shared ruff.toml. As a result test files drifted from the formatter's style and every PR that touched a test (or ran format-on-save) carried large formatting-only churn. Move the tests exclude into [tool.ruff.lint].exclude (and [lint].exclude in ruff.toml) so the formatter now covers tests while lint rules — too noisy for test code (unused imports/vars, import ordering) — stay excluded. Then run ruff format across all test directories. Note: lint.exclude is a post-traversal path filter, so it needs the glob form 'tests/**' rather than the directory form 'tests/' used by top-level exclude.
This commit is contained in:
@@ -187,12 +187,14 @@ dev = [
|
||||
[tool.ruff]
|
||||
line-length = 120
|
||||
target-version = "py311"
|
||||
exclude = [
|
||||
"tests/",
|
||||
"**/tests/",
|
||||
]
|
||||
|
||||
[tool.ruff.lint]
|
||||
# Tests are formatted (via `ruff format`) but excluded from lint rules, which
|
||||
# are too noisy for test code (unused imports/vars, import ordering).
|
||||
exclude = [
|
||||
"tests/**",
|
||||
"**/tests/**",
|
||||
]
|
||||
select = [
|
||||
"E", # pycodestyle errors
|
||||
"W", # pycodestyle warnings
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Pytest configuration and shared fixtures.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
@@ -76,6 +77,7 @@ def pg0_db_url(db_url, tmp_path_factory, worker_id):
|
||||
if db_url and not _parse_pg0_url(db_url)[0]:
|
||||
# Plain postgresql:// URL - use it directly but still run migrations
|
||||
from hindsight_api.migrations import run_migrations
|
||||
|
||||
run_migrations(db_url)
|
||||
return db_url
|
||||
|
||||
@@ -127,6 +129,7 @@ def pg0_db_url(db_url, tmp_path_factory, worker_id):
|
||||
# Run migrations - uses PostgreSQL advisory lock internally,
|
||||
# so safe to call from multiple workers (only one will actually run migrations)
|
||||
from hindsight_api.migrations import run_migrations
|
||||
|
||||
run_migrations(url)
|
||||
|
||||
# Clean up stale test data from previous sessions. Per-bank vector indexes
|
||||
@@ -157,8 +160,7 @@ def _cleanup_stale_test_data(db_url: str) -> None:
|
||||
conn = await asyncpg.connect(db_url)
|
||||
try:
|
||||
idx_rows = await conn.fetch(
|
||||
"SELECT indexname FROM pg_indexes "
|
||||
"WHERE schemaname = 'public' AND indexname LIKE 'idx_mu_emb_%'"
|
||||
"SELECT indexname FROM pg_indexes WHERE schemaname = 'public' AND indexname LIKE 'idx_mu_emb_%'"
|
||||
)
|
||||
if idx_rows:
|
||||
for row in idx_rows:
|
||||
@@ -166,10 +168,20 @@ def _cleanup_stale_test_data(db_url: str) -> None:
|
||||
|
||||
# Truncate test data in dependency order
|
||||
for table in [
|
||||
"entity_cooccurrences", "unit_entities", "memory_links",
|
||||
"entities", "memory_units", "chunks", "documents",
|
||||
"mental_models", "directives", "async_operations",
|
||||
"audit_log", "webhooks", "file_storage", "banks",
|
||||
"entity_cooccurrences",
|
||||
"unit_entities",
|
||||
"memory_links",
|
||||
"entities",
|
||||
"memory_units",
|
||||
"chunks",
|
||||
"documents",
|
||||
"mental_models",
|
||||
"directives",
|
||||
"async_operations",
|
||||
"audit_log",
|
||||
"webhooks",
|
||||
"file_storage",
|
||||
"banks",
|
||||
]:
|
||||
try:
|
||||
await conn.execute(f"TRUNCATE {table} CASCADE")
|
||||
@@ -252,8 +264,7 @@ def oracle_db_url(_oracle_admin_dsn):
|
||||
# Create test user (idempotent — skip if already exists)
|
||||
try:
|
||||
cursor.execute(
|
||||
f'CREATE USER {test_user} IDENTIFIED BY "{test_pass}" '
|
||||
f"DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"
|
||||
f'CREATE USER {test_user} IDENTIFIED BY "{test_pass}" DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS'
|
||||
)
|
||||
except oracledb.DatabaseError as e:
|
||||
if hasattr(e.args[0], "code") and e.args[0].code == 1920:
|
||||
@@ -420,13 +431,12 @@ def cross_encoder(tmp_path_factory, worker_id):
|
||||
|
||||
return ce
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def query_analyzer():
|
||||
return DateparserQueryAnalyzer()
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope="function")
|
||||
async def memory(pg0_db_url, embeddings, cross_encoder, query_analyzer):
|
||||
"""
|
||||
|
||||
@@ -23,6 +23,7 @@ from urllib.parse import urlparse
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _log(step: int, total: int, msg: str) -> None:
|
||||
print(f" [{step}/{total}] {msg}")
|
||||
|
||||
@@ -64,8 +65,7 @@ def _bootstrap_test_user(admin_dsn: dict[str, str]) -> str:
|
||||
# Create user (skip if already exists - ORA-01920)
|
||||
try:
|
||||
cursor.execute(
|
||||
f'CREATE USER {test_user} IDENTIFIED BY "{test_pass}" '
|
||||
f"DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"
|
||||
f'CREATE USER {test_user} IDENTIFIED BY "{test_pass}" DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS'
|
||||
)
|
||||
except oracledb.DatabaseError as e:
|
||||
if hasattr(e.args[0], "code") and e.args[0].code == 1920:
|
||||
@@ -100,6 +100,7 @@ def _bootstrap_test_user(admin_dsn: dict[str, str]) -> str:
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _run() -> None:
|
||||
total_steps = 8
|
||||
|
||||
@@ -290,6 +291,7 @@ def main() -> int:
|
||||
except Exception as exc:
|
||||
print(f"\nFAILED: {exc}", file=sys.stderr)
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ async def _judge_once(
|
||||
"content": (
|
||||
"You are a test evaluation judge. Given a response and evaluation criteria, "
|
||||
"determine whether the response meets the criteria. "
|
||||
"Respond with JSON: {\"meets_criteria\": true/false, \"reasoning\": \"brief explanation\"}"
|
||||
'Respond with JSON: {"meets_criteria": true/false, "reasoning": "brief explanation"}'
|
||||
),
|
||||
},
|
||||
{
|
||||
@@ -163,9 +163,7 @@ async def evaluate(
|
||||
|
||||
if met > not_met:
|
||||
agreeing = next(v for v in verdicts if v.meets_criteria)
|
||||
logger.info(
|
||||
f"Judge: primary 'not met' overruled by majority ({met}/{len(verdicts)} met). Criteria: {criteria}"
|
||||
)
|
||||
logger.info(f"Judge: primary 'not met' overruled by majority ({met}/{len(verdicts)} met). Criteria: {criteria}")
|
||||
return JudgeVerdict(
|
||||
meets_criteria=True,
|
||||
reasoning=f"Majority of {len(verdicts)} judges met criteria (primary verdict overruled as noise). {agreeing.reasoning}",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Tests for agent management API (profile, disposition).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import uuid
|
||||
from hindsight_api import MemoryEngine, RequestContext
|
||||
@@ -17,9 +18,7 @@ class TestAgentProfile:
|
||||
"""Tests for agent profile management."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_bank_profile_no_auto_create_returns_none(
|
||||
self, memory: MemoryEngine, request_context
|
||||
):
|
||||
async def test_get_bank_profile_no_auto_create_returns_none(self, memory: MemoryEngine, request_context):
|
||||
"""When create_if_missing=False is passed, a missing bank returns None
|
||||
rather than being silently auto-created. This is what read-only
|
||||
endpoints (HTTP GET, polling, etc.) must use to avoid creating banks
|
||||
@@ -27,28 +26,20 @@ class TestAgentProfile:
|
||||
bank_id = unique_agent_id("test_no_auto_create")
|
||||
|
||||
# First call with create_if_missing=False on a non-existent bank
|
||||
result = await memory.get_bank_profile(
|
||||
bank_id, request_context=request_context, create_if_missing=False
|
||||
)
|
||||
result = await memory.get_bank_profile(bank_id, request_context=request_context, create_if_missing=False)
|
||||
assert result is None, "Expected None for missing bank with create_if_missing=False"
|
||||
|
||||
# Verify the bank was NOT created as a side effect
|
||||
result_again = await memory.get_bank_profile(
|
||||
bank_id, request_context=request_context, create_if_missing=False
|
||||
)
|
||||
result_again = await memory.get_bank_profile(bank_id, request_context=request_context, create_if_missing=False)
|
||||
assert result_again is None, "Bank must not exist after read-only call"
|
||||
|
||||
# And explicit auto-create still works
|
||||
created = await memory.get_bank_profile(
|
||||
bank_id, request_context=request_context, create_if_missing=True
|
||||
)
|
||||
created = await memory.get_bank_profile(bank_id, request_context=request_context, create_if_missing=True)
|
||||
assert created is not None
|
||||
assert created["disposition"]["skepticism"] == 3
|
||||
|
||||
# Now read-only call sees it
|
||||
seen = await memory.get_bank_profile(
|
||||
bank_id, request_context=request_context, create_if_missing=False
|
||||
)
|
||||
seen = await memory.get_bank_profile(bank_id, request_context=request_context, create_if_missing=False)
|
||||
assert seen is not None
|
||||
assert seen["disposition"]["skepticism"] == 3
|
||||
|
||||
@@ -122,11 +113,7 @@ class TestAgentEndpoint:
|
||||
bank_id = unique_agent_id("test_put_create")
|
||||
|
||||
request = CreateBankRequest(
|
||||
disposition=DispositionTraits(
|
||||
skepticism=4,
|
||||
literalism=5,
|
||||
empathy=2
|
||||
),
|
||||
disposition=DispositionTraits(skepticism=4, literalism=5, empathy=2),
|
||||
)
|
||||
|
||||
profile = await memory.get_bank_profile(bank_id, request_context=request_context)
|
||||
@@ -155,7 +142,7 @@ class TestAgentDispositionIntegration:
|
||||
disposition = {
|
||||
"skepticism": 5, # Very skeptical
|
||||
"literalism": 4, # High literalism
|
||||
"empathy": 2, # Low empathy
|
||||
"empathy": 2, # Low empathy
|
||||
}
|
||||
await memory.update_bank_disposition(bank_id, disposition, request_context=request_context)
|
||||
|
||||
@@ -163,7 +150,7 @@ class TestAgentDispositionIntegration:
|
||||
bank_id=bank_id,
|
||||
contents=[
|
||||
{"content": "Traditional painting techniques have been used for centuries"},
|
||||
{"content": "Modern digital art is changing the art world"}
|
||||
{"content": "Modern digital art is changing the art world"},
|
||||
],
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
@@ -643,9 +643,7 @@ class TestDefaultBankTemplateEnvVar:
|
||||
yield default_template
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_template_applied_on_new_bank(
|
||||
self, api_client, bank_id, _patched_default_template
|
||||
):
|
||||
async def test_default_template_applied_on_new_bank(self, api_client, bank_id, _patched_default_template):
|
||||
"""Creating a new bank applies the default template (config + mental models + directives)."""
|
||||
# Trigger bank auto-creation via GET profile
|
||||
resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
|
||||
@@ -730,9 +728,7 @@ class TestDefaultBankTemplateEnvVar:
|
||||
assert config_resp.json()["overrides"] == {}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_template_malformed_is_swallowed(
|
||||
self, api_client, bank_id, monkeypatch
|
||||
):
|
||||
async def test_default_template_malformed_is_swallowed(self, api_client, bank_id, monkeypatch):
|
||||
"""A malformed default template is logged and ignored — bank creation still succeeds."""
|
||||
from hindsight_api.config import _get_raw_config
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ Integration test for API base path support.
|
||||
Tests that the API works correctly when deployed with a base path (e.g., /hindsight)
|
||||
for reverse proxy deployments.
|
||||
"""
|
||||
|
||||
import os
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
@@ -27,10 +28,7 @@ async def api_client_with_base_path(memory):
|
||||
|
||||
# Use base_url with base path
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url=f"http://test{base_path}"
|
||||
) as client:
|
||||
async with httpx.AsyncClient(transport=transport, base_url=f"http://test{base_path}") as client:
|
||||
yield client
|
||||
|
||||
# Cleanup: unset base path
|
||||
@@ -122,10 +120,10 @@ async def test_base_path_full_workflow(api_client_with_base_path):
|
||||
"items": [
|
||||
{
|
||||
"content": "The API supports base path deployment for reverse proxy use cases.",
|
||||
"context": "testing base path feature"
|
||||
"context": "testing base path feature",
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
result = response.json()
|
||||
@@ -133,10 +131,7 @@ async def test_base_path_full_workflow(api_client_with_base_path):
|
||||
|
||||
# 3. Recall the memory
|
||||
response = await api_client_with_base_path.post(
|
||||
f"/v1/default/banks/{bank_id}/memories/recall",
|
||||
json={
|
||||
"query": "base path support"
|
||||
}
|
||||
f"/v1/default/banks/{bank_id}/memories/recall", json={"query": "base path support"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
recall_result = response.json()
|
||||
|
||||
@@ -7,6 +7,7 @@ Tests cover:
|
||||
- Hard error when provider doesn't support the batch API (no silent fallback)
|
||||
- Worker recovery on restart
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
@@ -105,19 +106,21 @@ async def test_batch_api_normal_flow(mock_llm_config, test_contents, hindsight_c
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": json.dumps({
|
||||
"facts": [
|
||||
{
|
||||
"what": "Alice is a senior software engineer at TechCorp",
|
||||
"when": "present",
|
||||
"where": "TechCorp",
|
||||
"who": "Alice",
|
||||
"why": "Professional background information",
|
||||
"fact_type": "world",
|
||||
"fact_kind": "conversation",
|
||||
}
|
||||
]
|
||||
})
|
||||
"content": json.dumps(
|
||||
{
|
||||
"facts": [
|
||||
{
|
||||
"what": "Alice is a senior software engineer at TechCorp",
|
||||
"when": "present",
|
||||
"where": "TechCorp",
|
||||
"who": "Alice",
|
||||
"why": "Professional background information",
|
||||
"fact_type": "world",
|
||||
"fact_kind": "conversation",
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
],
|
||||
@@ -132,19 +135,21 @@ async def test_batch_api_normal_flow(mock_llm_config, test_contents, hindsight_c
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": json.dumps({
|
||||
"facts": [
|
||||
{
|
||||
"what": "Bob joined the team last month as a junior developer",
|
||||
"when": "last month",
|
||||
"where": "team",
|
||||
"who": "Bob",
|
||||
"why": "New team member information",
|
||||
"fact_type": "world",
|
||||
"fact_kind": "conversation",
|
||||
}
|
||||
]
|
||||
})
|
||||
"content": json.dumps(
|
||||
{
|
||||
"facts": [
|
||||
{
|
||||
"what": "Bob joined the team last month as a junior developer",
|
||||
"when": "last month",
|
||||
"where": "team",
|
||||
"who": "Bob",
|
||||
"why": "New team member information",
|
||||
"fact_type": "world",
|
||||
"fact_kind": "conversation",
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
],
|
||||
@@ -213,6 +218,7 @@ async def test_batch_api_crash_recovery(mock_llm_config, test_contents, hindsigh
|
||||
schema = request_context.tenant_id
|
||||
|
||||
from hindsight_api.engine.task_backend import fq_table
|
||||
|
||||
table = fq_table("async_operations", schema)
|
||||
|
||||
# Create operation with batch_id already stored
|
||||
@@ -223,11 +229,13 @@ async def test_batch_api_crash_recovery(mock_llm_config, test_contents, hindsigh
|
||||
""",
|
||||
operation_id,
|
||||
bank_id,
|
||||
json.dumps({
|
||||
"batch_id": batch_id,
|
||||
"batch_provider": "openai",
|
||||
"chunk_count": 2,
|
||||
}),
|
||||
json.dumps(
|
||||
{
|
||||
"batch_id": batch_id,
|
||||
"batch_provider": "openai",
|
||||
"chunk_count": 2,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
# Mock batch API responses for resume scenario
|
||||
@@ -250,19 +258,21 @@ async def test_batch_api_crash_recovery(mock_llm_config, test_contents, hindsigh
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": json.dumps({
|
||||
"facts": [
|
||||
{
|
||||
"what": "Alice is a senior software engineer",
|
||||
"when": "present",
|
||||
"where": "TechCorp",
|
||||
"who": "Alice",
|
||||
"why": "Background",
|
||||
"fact_type": "world",
|
||||
"fact_kind": "conversation",
|
||||
}
|
||||
]
|
||||
})
|
||||
"content": json.dumps(
|
||||
{
|
||||
"facts": [
|
||||
{
|
||||
"what": "Alice is a senior software engineer",
|
||||
"when": "present",
|
||||
"where": "TechCorp",
|
||||
"who": "Alice",
|
||||
"why": "Background",
|
||||
"fact_type": "world",
|
||||
"fact_kind": "conversation",
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
],
|
||||
@@ -277,19 +287,21 @@ async def test_batch_api_crash_recovery(mock_llm_config, test_contents, hindsigh
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": json.dumps({
|
||||
"facts": [
|
||||
{
|
||||
"what": "Bob is a junior developer",
|
||||
"when": "last month",
|
||||
"where": "team",
|
||||
"who": "Bob",
|
||||
"why": "New member",
|
||||
"fact_type": "world",
|
||||
"fact_kind": "conversation",
|
||||
}
|
||||
]
|
||||
})
|
||||
"content": json.dumps(
|
||||
{
|
||||
"facts": [
|
||||
{
|
||||
"what": "Bob is a junior developer",
|
||||
"when": "last month",
|
||||
"where": "team",
|
||||
"who": "Bob",
|
||||
"why": "New member",
|
||||
"fact_type": "world",
|
||||
"fact_kind": "conversation",
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
],
|
||||
@@ -377,19 +389,21 @@ async def test_batch_api_records_non_fatal_extraction_errors(
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": json.dumps({
|
||||
"facts": [
|
||||
{
|
||||
"what": "Alice is a senior software engineer",
|
||||
"when": "present",
|
||||
"where": "TechCorp",
|
||||
"who": "Alice",
|
||||
"why": "Background",
|
||||
"fact_type": "world",
|
||||
"fact_kind": "conversation",
|
||||
}
|
||||
]
|
||||
})
|
||||
"content": json.dumps(
|
||||
{
|
||||
"facts": [
|
||||
{
|
||||
"what": "Alice is a senior software engineer",
|
||||
"when": "present",
|
||||
"where": "TechCorp",
|
||||
"who": "Alice",
|
||||
"why": "Background",
|
||||
"fact_type": "world",
|
||||
"fact_kind": "conversation",
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
],
|
||||
@@ -416,7 +430,9 @@ async def test_batch_api_records_non_fatal_extraction_errors(
|
||||
assert usage.total_tokens == 150
|
||||
|
||||
row = await pool.fetchrow(f"SELECT result_metadata FROM {table} WHERE operation_id = $1", operation_id)
|
||||
metadata = json.loads(row["result_metadata"]) if isinstance(row["result_metadata"], str) else row["result_metadata"]
|
||||
metadata = (
|
||||
json.loads(row["result_metadata"]) if isinstance(row["result_metadata"], str) else row["result_metadata"]
|
||||
)
|
||||
assert metadata["batch_id"] == batch_id
|
||||
assert metadata["extraction_errors_count"] == 1
|
||||
assert metadata["extraction_errors_sample"] == ["chunk_1: missing batch result"]
|
||||
@@ -469,6 +485,7 @@ async def test_worker_batch_recovery(memory, request_context):
|
||||
schema = request_context.tenant_id
|
||||
|
||||
from hindsight_api.engine.task_backend import fq_table
|
||||
|
||||
table = fq_table("async_operations", schema)
|
||||
|
||||
# Create orphaned batch operation (simulates worker crash during polling)
|
||||
@@ -486,16 +503,19 @@ async def test_worker_batch_recovery(memory, request_context):
|
||||
""",
|
||||
operation_id,
|
||||
bank_id,
|
||||
json.dumps({
|
||||
"batch_id": batch_id,
|
||||
"batch_provider": "openai",
|
||||
"chunk_count": 1,
|
||||
}),
|
||||
json.dumps(
|
||||
{
|
||||
"batch_id": batch_id,
|
||||
"batch_provider": "openai",
|
||||
"chunk_count": 1,
|
||||
}
|
||||
),
|
||||
json.dumps(task_payload),
|
||||
)
|
||||
|
||||
# Create WorkerPoller
|
||||
from hindsight_api.extensions.builtin.tenant import DefaultTenantExtension
|
||||
|
||||
tenant_extension = DefaultTenantExtension(config={"schema": schema} if schema else {})
|
||||
|
||||
poller = WorkerPoller(
|
||||
@@ -559,13 +579,7 @@ async def test_batch_api_via_extract_facts_from_contents(
|
||||
"custom_id": "chunk_0",
|
||||
"response": {
|
||||
"body": {
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": json.dumps({"facts": []})
|
||||
}
|
||||
}
|
||||
],
|
||||
"choices": [{"message": {"content": json.dumps({"facts": []})}}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
},
|
||||
|
||||
@@ -10,6 +10,7 @@ To run:
|
||||
To skip in CI:
|
||||
Add @pytest.mark.skip at the test level
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import os
|
||||
import asyncio
|
||||
@@ -115,7 +116,9 @@ def integration_config():
|
||||
return config
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Real API test - takes minutes and costs money. Run manually with: pytest tests/test_batch_api_integration.py::test_real_openai_batch_api -v -s")
|
||||
@pytest.mark.skip(
|
||||
reason="Real API test - takes minutes and costs money. Run manually with: pytest tests/test_batch_api_integration.py::test_real_openai_batch_api -v -s"
|
||||
)
|
||||
@pytest.mark.integration # Mark as integration test
|
||||
@pytest.mark.slow # Mark as slow test
|
||||
@pytest.mark.asyncio
|
||||
@@ -174,17 +177,21 @@ async def test_real_openai_batch_api(real_llm_config, test_contents_real, integr
|
||||
logger.info("\n" + "=" * 80)
|
||||
logger.info("✅ BATCH COMPLETED SUCCESSFULLY")
|
||||
logger.info("=" * 80)
|
||||
logger.info(f"Total duration: {total_duration:.1f} seconds ({total_duration/60:.1f} minutes)")
|
||||
logger.info(f"Total duration: {total_duration:.1f} seconds ({total_duration / 60:.1f} minutes)")
|
||||
logger.info(f"Facts extracted: {len(facts)}")
|
||||
logger.info(f"Chunks processed: {len(chunks)}")
|
||||
logger.info(f"Token usage: {usage.input_tokens} input + {usage.output_tokens} output = {usage.total_tokens} total")
|
||||
logger.info(f"Estimated cost: ${(usage.input_tokens * 0.00015 / 1000 + usage.output_tokens * 0.0006 / 1000):.4f}")
|
||||
logger.info(
|
||||
f"Token usage: {usage.input_tokens} input + {usage.output_tokens} output = {usage.total_tokens} total"
|
||||
)
|
||||
logger.info(
|
||||
f"Estimated cost: ${(usage.input_tokens * 0.00015 / 1000 + usage.output_tokens * 0.0006 / 1000):.4f}"
|
||||
)
|
||||
logger.info("=" * 80)
|
||||
|
||||
# Log sample facts
|
||||
logger.info("\n📋 Sample extracted facts:")
|
||||
for i, fact in enumerate(facts[:5]): # Show first 5 facts
|
||||
logger.info(f"\nFact {i+1}:")
|
||||
logger.info(f"\nFact {i + 1}:")
|
||||
logger.info(f" Type: {fact.fact_type}")
|
||||
logger.info(f" Text: {fact.fact_text[:100]}...")
|
||||
logger.info(f" Entities: {fact.entities}")
|
||||
@@ -212,11 +219,13 @@ async def test_real_openai_batch_api(real_llm_config, test_contents_real, integr
|
||||
f.write(f"Contents: {len(test_contents_real)} items\n")
|
||||
f.write(f"Poll Interval: {integration_config.retain_batch_poll_interval_seconds}s\n\n")
|
||||
f.write(f"Results:\n")
|
||||
f.write(f" Total Duration: {total_duration:.1f}s ({total_duration/60:.1f} min)\n")
|
||||
f.write(f" Total Duration: {total_duration:.1f}s ({total_duration / 60:.1f} min)\n")
|
||||
f.write(f" Facts Extracted: {len(facts)}\n")
|
||||
f.write(f" Chunks Processed: {len(chunks)}\n")
|
||||
f.write(f" Token Usage: {usage.total_tokens} ({usage.input_tokens} in + {usage.output_tokens} out)\n")
|
||||
f.write(f" Estimated Cost: ${(usage.input_tokens * 0.00015 / 1000 + usage.output_tokens * 0.0006 / 1000):.4f}\n")
|
||||
f.write(
|
||||
f" Estimated Cost: ${(usage.input_tokens * 0.00015 / 1000 + usage.output_tokens * 0.0006 / 1000):.4f}\n"
|
||||
)
|
||||
|
||||
logger.info(f"\n📄 Timing report written to: {report_path}")
|
||||
|
||||
|
||||
@@ -174,9 +174,7 @@ def test_async_children_packs_small_items_by_budget():
|
||||
num_items = max(4, (tokens_per_batch // max(item_tokens, 1)) * 3)
|
||||
contents = [{"content": item_text, "document_id": f"doc-{i}"} for i in range(num_items)]
|
||||
total = sum(count_tokens(c["content"]) for c in contents)
|
||||
assert total > tokens_per_batch, (
|
||||
f"Test setup error: {total} tokens does not exceed budget {tokens_per_batch}"
|
||||
)
|
||||
assert total > tokens_per_batch, f"Test setup error: {total} tokens does not exceed budget {tokens_per_batch}"
|
||||
|
||||
children = _split_contents_into_async_children(contents, tokens_per_batch)
|
||||
|
||||
|
||||
@@ -104,8 +104,7 @@ class TestCausalRelationsValidation:
|
||||
for rel in facts[0].causal_relations:
|
||||
# This should never happen due to validation
|
||||
assert False, (
|
||||
f"First fact should not have causal relations, "
|
||||
f"but found: target_index={rel.target_fact_index}"
|
||||
f"First fact should not have causal relations, but found: target_index={rel.target_fact_index}"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -139,11 +138,13 @@ class TestCausalRelationsValidation:
|
||||
for i, fact in enumerate(facts):
|
||||
if fact.causal_relations:
|
||||
for rel in fact.causal_relations:
|
||||
all_relations.append({
|
||||
"from_fact": i,
|
||||
"to_fact": rel.target_fact_index,
|
||||
"type": rel.relation_type,
|
||||
})
|
||||
all_relations.append(
|
||||
{
|
||||
"from_fact": i,
|
||||
"to_fact": rel.target_fact_index,
|
||||
"type": rel.relation_type,
|
||||
}
|
||||
)
|
||||
|
||||
# If causal relations were extracted, verify they form a valid chain
|
||||
if all_relations:
|
||||
@@ -226,6 +227,5 @@ class TestCausalRelationsValidation:
|
||||
if fact.causal_relations:
|
||||
for rel in fact.causal_relations:
|
||||
assert rel.relation_type in valid_types, (
|
||||
f"Invalid relation_type '{rel.relation_type}'. "
|
||||
f"Must be one of: {valid_types}"
|
||||
f"Invalid relation_type '{rel.relation_type}'. Must be one of: {valid_types}"
|
||||
)
|
||||
|
||||
@@ -40,7 +40,11 @@ After searching for weeks, I finally found a cheaper apartment in Brooklyn.
|
||||
llm_config = LLMConfig.from_env()
|
||||
|
||||
facts, _, _ = await extract_facts_from_text(
|
||||
text=text, event_date=datetime(2024, 3, 15), context=context, llm_config=llm_config, agent_name="TestUser",
|
||||
text=text,
|
||||
event_date=datetime(2024, 3, 15),
|
||||
context=context,
|
||||
llm_config=llm_config,
|
||||
agent_name="TestUser",
|
||||
config=_get_raw_config(),
|
||||
)
|
||||
|
||||
@@ -109,7 +113,11 @@ The renovation took three months and cost $15,000.
|
||||
llm_config = LLMConfig.from_env()
|
||||
|
||||
facts, _, _ = await extract_facts_from_text(
|
||||
text=text, event_date=datetime(2024, 6, 1), context=context, llm_config=llm_config, agent_name="TestUser",
|
||||
text=text,
|
||||
event_date=datetime(2024, 6, 1),
|
||||
context=context,
|
||||
llm_config=llm_config,
|
||||
agent_name="TestUser",
|
||||
config=_get_raw_config(),
|
||||
)
|
||||
|
||||
@@ -140,7 +148,11 @@ Machine learning fascinated me so much that I changed my career to data science.
|
||||
llm_config = LLMConfig.from_env()
|
||||
|
||||
facts, _, _ = await extract_facts_from_text(
|
||||
text=text, event_date=datetime(2024, 1, 1), context=context, llm_config=llm_config, agent_name="TestUser",
|
||||
text=text,
|
||||
event_date=datetime(2024, 1, 1),
|
||||
context=context,
|
||||
llm_config=llm_config,
|
||||
agent_name="TestUser",
|
||||
config=_get_raw_config(),
|
||||
)
|
||||
|
||||
@@ -168,7 +180,11 @@ The new role enabled me to lead a team of engineers.
|
||||
llm_config = LLMConfig.from_env()
|
||||
|
||||
facts, _, _ = await extract_facts_from_text(
|
||||
text=text, event_date=datetime(2024, 2, 15), context=context, llm_config=llm_config, agent_name="TestUser",
|
||||
text=text,
|
||||
event_date=datetime(2024, 2, 15),
|
||||
context=context,
|
||||
llm_config=llm_config,
|
||||
agent_name="TestUser",
|
||||
config=_get_raw_config(),
|
||||
)
|
||||
|
||||
@@ -180,4 +196,3 @@ The new role enabled me to lead a team of engineers.
|
||||
f"Invalid target_fact_index {rel.target_fact_index} in fact {i}. "
|
||||
f"Must reference previous facts only (valid range: 0 to {i - 1})"
|
||||
)
|
||||
|
||||
|
||||
@@ -130,8 +130,7 @@ async def test_store_chunks_batch_second_call_with_identical_payload(memory):
|
||||
await _seed_bank_and_document(conn, bank_id, document_id)
|
||||
|
||||
chunks = [
|
||||
ChunkMetadata(chunk_text=f"chunk-{i}", fact_count=1, content_index=0, chunk_index=i)
|
||||
for i in range(5)
|
||||
ChunkMetadata(chunk_text=f"chunk-{i}", fact_count=1, content_index=0, chunk_index=i) for i in range(5)
|
||||
]
|
||||
|
||||
await chunk_storage.store_chunks_batch(conn, bank_id, document_id, chunks, ops=ops)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Test chunking functionality for large documents.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from hindsight_api.engine.retain.fact_extraction import chunk_text
|
||||
|
||||
@@ -53,4 +54,3 @@ def test_chunk_text_64k():
|
||||
# Verify we didn't lose content
|
||||
combined_length = sum(len(chunk) for chunk in chunks)
|
||||
assert combined_length >= len(text) * 0.95, "Lost too much content during chunking"
|
||||
|
||||
|
||||
@@ -344,4 +344,7 @@ class TestFactoryFunction:
|
||||
assert isinstance(encoder, CohereCrossEncoder)
|
||||
assert encoder.api_key == "test_key"
|
||||
assert encoder.model == "cohere-rerank-v3-english"
|
||||
assert encoder.base_url == "https://my-endpoint.inference.ai.azure.com/models/cohere-rerank-v3-english/invoke"
|
||||
assert (
|
||||
encoder.base_url
|
||||
== "https://my-endpoint.inference.ai.azure.com/models/cohere-rerank-v3-english/invoke"
|
||||
)
|
||||
|
||||
@@ -371,9 +371,7 @@ class TestRecoverConsolidation:
|
||||
mem_id,
|
||||
)
|
||||
|
||||
result = await memory_no_llm_verify.retry_failed_consolidation(
|
||||
bank_id, request_context=request_context
|
||||
)
|
||||
result = await memory_no_llm_verify.retry_failed_consolidation(bank_id, request_context=request_context)
|
||||
|
||||
assert result["retried_count"] == 2
|
||||
|
||||
@@ -394,9 +392,7 @@ class TestRecoverConsolidation:
|
||||
bank_id = f"test-recover-zero-{uuid.uuid4().hex[:8]}"
|
||||
await memory_no_llm_verify.get_bank_profile(bank_id=bank_id, request_context=request_context)
|
||||
|
||||
result = await memory_no_llm_verify.retry_failed_consolidation(
|
||||
bank_id, request_context=request_context
|
||||
)
|
||||
result = await memory_no_llm_verify.retry_failed_consolidation(bank_id, request_context=request_context)
|
||||
|
||||
assert result["retried_count"] == 0
|
||||
|
||||
@@ -410,14 +406,10 @@ class TestRecoverConsolidation:
|
||||
|
||||
async with memory_no_llm_verify._pool.acquire() as conn:
|
||||
(mem_id,) = await _insert_memories(conn, bank_id, ["Grace is an expert rock climber."])
|
||||
await conn.execute(
|
||||
"UPDATE memory_units SET consolidation_failed_at = NOW() WHERE id = $1", mem_id
|
||||
)
|
||||
await conn.execute("UPDATE memory_units SET consolidation_failed_at = NOW() WHERE id = $1", mem_id)
|
||||
|
||||
# Recover
|
||||
recover_result = await memory_no_llm_verify.retry_failed_consolidation(
|
||||
bank_id, request_context=request_context
|
||||
)
|
||||
recover_result = await memory_no_llm_verify.retry_failed_consolidation(bank_id, request_context=request_context)
|
||||
assert recover_result["retried_count"] == 1
|
||||
|
||||
# Now consolidate with a healthy LLM
|
||||
@@ -460,9 +452,7 @@ class TestRecoverConsolidation:
|
||||
["Henry is a professional chef.", "Henry trained at Le Cordon Bleu."],
|
||||
)
|
||||
for mem_id in ids:
|
||||
await conn.execute(
|
||||
"UPDATE memory_units SET consolidation_failed_at = NOW() WHERE id = $1", mem_id
|
||||
)
|
||||
await conn.execute("UPDATE memory_units SET consolidation_failed_at = NOW() WHERE id = $1", mem_id)
|
||||
|
||||
app = create_app(memory_no_llm_verify, initialize_memory=False)
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
|
||||
@@ -80,9 +80,7 @@ async def _pending_consolidation_ops(memory, bank_id: str) -> list[str]:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_round_limited_consolidation_leaves_followup_pending_op(
|
||||
memory: MemoryEngine, request_context
|
||||
):
|
||||
async def test_round_limited_consolidation_leaves_followup_pending_op(memory: MemoryEngine, request_context):
|
||||
"""A round-limited consolidation must leave a new ``pending`` consolidation
|
||||
op in ``async_operations`` for the same bank so the worker poller can
|
||||
drain the backlog without external intervention."""
|
||||
@@ -147,9 +145,7 @@ async def test_round_limited_consolidation_leaves_followup_pending_op(
|
||||
op_id,
|
||||
)
|
||||
assert row is not None
|
||||
assert row["status"] == "completed", (
|
||||
f"first consolidation op should be marked completed, got {row['status']}"
|
||||
)
|
||||
assert row["status"] == "completed", f"first consolidation op should be marked completed, got {row['status']}"
|
||||
|
||||
# 4. Backlog must remain (round limit kept one round under the total)
|
||||
unconsolidated_after = await _count_unconsolidated(memory, bank_id)
|
||||
@@ -169,8 +165,6 @@ async def test_round_limited_consolidation_leaves_followup_pending_op(
|
||||
f"backlog. Found {len(pending_ops)} pending ops; backlog still has "
|
||||
f"{unconsolidated_after} unconsolidated memory_units."
|
||||
)
|
||||
assert pending_ops[0] != str(op_id), (
|
||||
"The pending op must be a NEW row, not the original op we just executed."
|
||||
)
|
||||
assert pending_ops[0] != str(op_id), "The pending op must be a NEW row, not the original op we just executed."
|
||||
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
|
||||
@@ -230,8 +230,7 @@ async def test_backoff_matches_schedule_by_retry_count(memory, retry_count):
|
||||
|
||||
delta = (excinfo.value.retry_at - before).total_seconds()
|
||||
assert expected_backoff <= delta <= expected_backoff + 10, (
|
||||
f"retry_count={retry_count}: expected backoff ~{expected_backoff}s, "
|
||||
f"got delta={delta:.2f}s"
|
||||
f"retry_count={retry_count}: expected backoff ~{expected_backoff}s, got delta={delta:.2f}s"
|
||||
)
|
||||
|
||||
await _cleanup(pool, bank_id, op_id)
|
||||
@@ -266,8 +265,6 @@ async def test_retry_is_indefinite(memory):
|
||||
|
||||
delta = (excinfo.value.retry_at - before).total_seconds()
|
||||
cap = _CONSOLIDATION_RETRY_BACKOFF_MAX_SECONDS
|
||||
assert cap <= delta <= cap + 10, (
|
||||
f"At retry_count=100 expected backoff at cap (~{cap}s), got {delta:.2f}s"
|
||||
)
|
||||
assert cap <= delta <= cap + 10, f"At retry_count=100 expected backoff at cap (~{cap}s), got {delta:.2f}s"
|
||||
|
||||
await _cleanup(pool, bank_id, op_id)
|
||||
|
||||
@@ -77,9 +77,7 @@ async def test_round_limit_caps_processed_memories(memory: MemoryEngine, request
|
||||
assert result["memories_processed"] <= round_limit
|
||||
|
||||
# Must have re-queued consolidation for remaining work
|
||||
mock_requeue.assert_called_once_with(
|
||||
bank_id=bank_id, request_context=request_context, observation_scopes=None
|
||||
)
|
||||
mock_requeue.assert_called_once_with(bank_id=bank_id, request_context=request_context, observation_scopes=None)
|
||||
|
||||
# Mental model refresh should be skipped on intermediate round
|
||||
assert result.get("mental_models_refreshed", 0) == 0
|
||||
|
||||
@@ -113,10 +113,7 @@ def _mock_llm_one_obs_per_fact():
|
||||
# example UUIDs in its OUTPUT samples — read user only.
|
||||
prompt = "\n".join(m.get("content", "") for m in messages if m.get("role") == "user")
|
||||
fact_ids = re.findall(r"\[([0-9a-f-]{36})\]", prompt)
|
||||
creates = [
|
||||
_CreateAction(text=f"Observation about fact {fid[:8]}", source_fact_ids=[fid])
|
||||
for fid in fact_ids
|
||||
]
|
||||
creates = [_CreateAction(text=f"Observation about fact {fid[:8]}", source_fact_ids=[fid]) for fid in fact_ids]
|
||||
return _ConsolidationBatchResponse(creates=creates)
|
||||
|
||||
mock_llm.set_response_callback(callback)
|
||||
@@ -170,11 +167,13 @@ async def test_combined_mode_parallel_writes_to_memory_tag_set(memory: MemoryEng
|
||||
|
||||
assert result["status"] == "completed"
|
||||
tag_sets = _ag_sorted(await _fetch_observation_tag_sets(memory, bank_id))
|
||||
assert tag_sets == _ag_sorted([
|
||||
frozenset({"user:alice"}),
|
||||
frozenset({"user:bob"}),
|
||||
frozenset({"user:carol"}),
|
||||
])
|
||||
assert tag_sets == _ag_sorted(
|
||||
[
|
||||
frozenset({"user:alice"}),
|
||||
frozenset({"user:bob"}),
|
||||
frozenset({"user:carol"}),
|
||||
]
|
||||
)
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
|
||||
@@ -373,8 +372,7 @@ async def test_overlapping_scopes_serialise_under_parallelism(memory: MemoryEngi
|
||||
# The whole point: lock invariant per scope.
|
||||
for scope, peak in max_concurrent.items():
|
||||
assert peak <= 1, (
|
||||
f"scope {set(scope) or '<untagged>'} had {peak} concurrent in-flight recalls; "
|
||||
"lock invariant violated"
|
||||
f"scope {set(scope) or '<untagged>'} had {peak} concurrent in-flight recalls; lock invariant violated"
|
||||
)
|
||||
# Sanity: we DID see recalls for the shared scope, so the test wasn't trivial.
|
||||
assert frozenset({"a"}) in max_concurrent
|
||||
@@ -421,9 +419,7 @@ async def test_per_batch_log_line_attributes_only_own_work(memory: MemoryEngine,
|
||||
patch.object(memory, "submit_async_consolidation"),
|
||||
caplog.at_level(logging.INFO, logger="hindsight_api.engine.consolidation.consolidator"),
|
||||
):
|
||||
await run_consolidation_job(
|
||||
memory_engine=memory, bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
await run_consolidation_job(memory_engine=memory, bank_id=bank_id, request_context=request_context)
|
||||
finally:
|
||||
memory._consolidation_llm_config = original_llm
|
||||
|
||||
@@ -452,9 +448,7 @@ async def test_per_batch_log_line_attributes_only_own_work(memory: MemoryEngine,
|
||||
assert processed_values == sorted(processed_values), (
|
||||
f"processed counter must be monotonic, got {processed_values}"
|
||||
)
|
||||
assert max(processed_values) == 3, (
|
||||
f"final cumulative processed should be 3, got {max(processed_values)}"
|
||||
)
|
||||
assert max(processed_values) == 3, f"final cumulative processed should be 3, got {max(processed_values)}"
|
||||
assert set(processed_values) == {1, 2, 3}, (
|
||||
f"each batch should bump the counter by exactly 1, got {processed_values}"
|
||||
)
|
||||
@@ -466,9 +460,7 @@ async def test_per_batch_log_line_attributes_only_own_work(memory: MemoryEngine,
|
||||
assert m_llm_time, f"expected llm=Xs timing, got: {line}"
|
||||
# Sanity: a single mock-LLM call is fast — under a second easily.
|
||||
# If snapshot leaked, this would catch concurrent batches' LLM time too.
|
||||
assert float(m_llm_time.group(1)) < 5.0, (
|
||||
f"llm timing implausibly large for a single mock-LLM call: {line}"
|
||||
)
|
||||
assert float(m_llm_time.group(1)) < 5.0, f"llm timing implausibly large for a single mock-LLM call: {line}"
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
|
||||
|
||||
@@ -71,9 +71,7 @@ async def _count_unconsolidated(memory, bank_id: str) -> int:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_requeue_failure_propagates_to_worker_retry(
|
||||
memory: MemoryEngine, request_context
|
||||
):
|
||||
async def test_requeue_failure_propagates_to_worker_retry(memory: MemoryEngine, request_context):
|
||||
"""When the in-task ``submit_async_consolidation`` call raises, the op
|
||||
must NOT be silently completed. The consolidator's work for this round
|
||||
is durably committed (memories marked consolidated_at in their own
|
||||
@@ -183,8 +181,6 @@ async def test_requeue_failure_propagates_to_worker_retry(
|
||||
f"unconsolidated_remaining={unconsolidated_after}"
|
||||
)
|
||||
|
||||
assert call_count["n"] == 1, (
|
||||
f"only one in-task submit_async_consolidation call expected, got {call_count['n']}"
|
||||
)
|
||||
assert call_count["n"] == 1, f"only one in-task submit_async_consolidation call expected, got {call_count['n']}"
|
||||
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
|
||||
@@ -63,10 +63,7 @@ async def test_concurrent_submits_leave_one_pending(memory, request_context, no_
|
||||
await _ensure_bank(pool, bank_id)
|
||||
try:
|
||||
results = await asyncio.gather(
|
||||
*(
|
||||
memory.submit_async_consolidation(bank_id=bank_id, request_context=request_context)
|
||||
for _ in range(5)
|
||||
)
|
||||
*(memory.submit_async_consolidation(bank_id=bank_id, request_context=request_context) for _ in range(5))
|
||||
)
|
||||
assert await _count_pending(pool, bank_id) == 1
|
||||
op_ids = {r["operation_id"] for r in results}
|
||||
|
||||
@@ -287,7 +287,9 @@ class TestEmbeddingDimension:
|
||||
# Try to change dimension - should raise RuntimeError.
|
||||
# Retry on transient OID errors from concurrent xdist schema drops.
|
||||
_assert_raises_runtime_error_with_retry(
|
||||
db_url, 768, schema,
|
||||
db_url,
|
||||
768,
|
||||
schema,
|
||||
expected_messages=["Cannot change embedding dimension", "1 rows with embeddings"],
|
||||
)
|
||||
|
||||
@@ -332,7 +334,9 @@ class TestEmbeddingDimension:
|
||||
# Try to change dimension - should raise RuntimeError.
|
||||
# Retry on transient OID errors from concurrent xdist schema drops.
|
||||
_assert_raises_runtime_error_with_retry(
|
||||
db_url, 768, schema,
|
||||
db_url,
|
||||
768,
|
||||
schema,
|
||||
expected_messages=["Cannot change embedding dimension", "mental_models"],
|
||||
)
|
||||
|
||||
|
||||
@@ -193,8 +193,13 @@ class TestPostgreSQLDialect:
|
||||
|
||||
def test_build_semantic_arm(self, d):
|
||||
arm = d.build_semantic_arm(
|
||||
table="schema.memory_units", cols="id, text", fact_type="world",
|
||||
embedding_param="$1", bank_id_param="$2", fetch_limit=100, min_similarity=0.58,
|
||||
table="schema.memory_units",
|
||||
cols="id, text",
|
||||
fact_type="world",
|
||||
embedding_param="$1",
|
||||
bank_id_param="$2",
|
||||
fetch_limit=100,
|
||||
min_similarity=0.58,
|
||||
)
|
||||
assert "1 - (embedding <=> $1::vector)" in arm
|
||||
assert ">= 0.58" in arm
|
||||
@@ -204,8 +209,12 @@ class TestPostgreSQLDialect:
|
||||
|
||||
def test_build_bm25_arm_native(self, d):
|
||||
arm = d.build_bm25_arm(
|
||||
table="schema.memory_units", cols="id, text", fact_type="world",
|
||||
bank_id_param="$2", limit_param="$3", text_param="$4",
|
||||
table="schema.memory_units",
|
||||
cols="id, text",
|
||||
fact_type="world",
|
||||
bank_id_param="$2",
|
||||
limit_param="$3",
|
||||
text_param="$4",
|
||||
)
|
||||
assert "ts_rank_cd" in arm
|
||||
assert "to_tsquery" in arm
|
||||
@@ -216,8 +225,12 @@ class TestPostgreSQLDialect:
|
||||
|
||||
def test_build_bm25_arm_native_uses_configured_language(self, d):
|
||||
arm = d.build_bm25_arm(
|
||||
table="schema.memory_units", cols="id, text", fact_type="world",
|
||||
bank_id_param="$2", limit_param="$3", text_param="$4",
|
||||
table="schema.memory_units",
|
||||
cols="id, text",
|
||||
fact_type="world",
|
||||
bank_id_param="$2",
|
||||
limit_param="$3",
|
||||
text_param="$4",
|
||||
bm25_language="french",
|
||||
)
|
||||
# Both the score and the WHERE filter must use the configured dictionary
|
||||
@@ -226,8 +239,12 @@ class TestPostgreSQLDialect:
|
||||
|
||||
def test_build_bm25_arm_vchord(self, d):
|
||||
arm = d.build_bm25_arm(
|
||||
table="t", cols="id", fact_type="world",
|
||||
bank_id_param="$2", limit_param="$3", text_param="$4",
|
||||
table="t",
|
||||
cols="id",
|
||||
fact_type="world",
|
||||
bank_id_param="$2",
|
||||
limit_param="$3",
|
||||
text_param="$4",
|
||||
text_search_extension="vchord",
|
||||
)
|
||||
assert "to_bm25query" in arm
|
||||
@@ -240,16 +257,26 @@ class TestPostgreSQLDialect:
|
||||
rows with a genuine query-term match, mirroring native tsvector's `@@`.
|
||||
"""
|
||||
arm = d.build_bm25_arm(
|
||||
table="t", cols="id", fact_type="world",
|
||||
bank_id_param="$2", limit_param="$3", text_param="$4",
|
||||
table="t",
|
||||
cols="id",
|
||||
fact_type="world",
|
||||
bank_id_param="$2",
|
||||
limit_param="$3",
|
||||
text_param="$4",
|
||||
text_search_extension="vchord",
|
||||
)
|
||||
assert "-(search_vector <&> to_bm25query('idx_memory_units_text_search', tokenize($4, 'llmlingua2'))) > 0" in arm
|
||||
assert (
|
||||
"-(search_vector <&> to_bm25query('idx_memory_units_text_search', tokenize($4, 'llmlingua2'))) > 0" in arm
|
||||
)
|
||||
|
||||
def test_build_bm25_arm_vchord_honors_custom_min_score(self, d):
|
||||
arm = d.build_bm25_arm(
|
||||
table="t", cols="id", fact_type="world",
|
||||
bank_id_param="$2", limit_param="$3", text_param="$4",
|
||||
table="t",
|
||||
cols="id",
|
||||
fact_type="world",
|
||||
bank_id_param="$2",
|
||||
limit_param="$3",
|
||||
text_param="$4",
|
||||
text_search_extension="vchord",
|
||||
bm25_min_score=2.5,
|
||||
)
|
||||
@@ -257,8 +284,12 @@ class TestPostgreSQLDialect:
|
||||
|
||||
def test_build_bm25_arm_pgroonga(self, d):
|
||||
arm = d.build_bm25_arm(
|
||||
table="schema.memory_units", cols="id, text", fact_type="world",
|
||||
bank_id_param="$2", limit_param="$3", text_param="$4",
|
||||
table="schema.memory_units",
|
||||
cols="id, text",
|
||||
fact_type="world",
|
||||
bank_id_param="$2",
|
||||
limit_param="$3",
|
||||
text_param="$4",
|
||||
text_search_extension="pgroonga",
|
||||
)
|
||||
# pgroonga uses the &@~ operator + pgroonga_score for ranking. Escape
|
||||
@@ -272,8 +303,12 @@ class TestPostgreSQLDialect:
|
||||
def test_build_bm25_arm_pgroonga_ignores_bm25_language(self, d):
|
||||
"""pgroonga's tokenizer is fixed at index creation; bm25_language must not leak in."""
|
||||
arm = d.build_bm25_arm(
|
||||
table="t", cols="id", fact_type="world",
|
||||
bank_id_param="$2", limit_param="$3", text_param="$4",
|
||||
table="t",
|
||||
cols="id",
|
||||
fact_type="world",
|
||||
bank_id_param="$2",
|
||||
limit_param="$3",
|
||||
text_param="$4",
|
||||
text_search_extension="pgroonga",
|
||||
bm25_language="french",
|
||||
)
|
||||
@@ -281,8 +316,12 @@ class TestPostgreSQLDialect:
|
||||
|
||||
def test_build_bm25_arm_pg_search(self, d):
|
||||
arm = d.build_bm25_arm(
|
||||
table="schema.memory_units", cols="id, text", fact_type="world",
|
||||
bank_id_param="$2", limit_param="$3", text_param="$4",
|
||||
table="schema.memory_units",
|
||||
cols="id, text",
|
||||
fact_type="world",
|
||||
bank_id_param="$2",
|
||||
limit_param="$3",
|
||||
text_param="$4",
|
||||
text_search_extension="pg_search",
|
||||
)
|
||||
assert "paradedb.score(id)" in arm
|
||||
@@ -360,8 +399,13 @@ class TestOracleDialect:
|
||||
|
||||
def test_build_semantic_arm(self, d):
|
||||
arm = d.build_semantic_arm(
|
||||
table="memory_units", cols="id, text", fact_type="world",
|
||||
embedding_param=":1", bank_id_param=":2", fetch_limit=100, min_similarity=0.58,
|
||||
table="memory_units",
|
||||
cols="id, text",
|
||||
fact_type="world",
|
||||
embedding_param=":1",
|
||||
bank_id_param=":2",
|
||||
fetch_limit=100,
|
||||
min_similarity=0.58,
|
||||
)
|
||||
assert "VECTOR_DISTANCE" in arm
|
||||
assert ">= 0.58" in arm
|
||||
@@ -371,8 +415,12 @@ class TestOracleDialect:
|
||||
|
||||
def test_build_bm25_arm(self, d):
|
||||
arm = d.build_bm25_arm(
|
||||
table="memory_units", cols="id, text", fact_type="world",
|
||||
bank_id_param=":2", limit_param=":3", text_param=":4",
|
||||
table="memory_units",
|
||||
cols="id, text",
|
||||
fact_type="world",
|
||||
bank_id_param=":2",
|
||||
limit_param=":3",
|
||||
text_param=":4",
|
||||
arm_index=0,
|
||||
)
|
||||
assert "CONTAINS" in arm
|
||||
@@ -383,12 +431,22 @@ class TestOracleDialect:
|
||||
def test_build_bm25_arm_unique_labels(self, d):
|
||||
"""Each arm_index produces a unique SCORE label to avoid conflicts in UNION ALL."""
|
||||
arm0 = d.build_bm25_arm(
|
||||
table="t", cols="id", fact_type="world",
|
||||
bank_id_param=":2", limit_param=":3", text_param=":4", arm_index=0,
|
||||
table="t",
|
||||
cols="id",
|
||||
fact_type="world",
|
||||
bank_id_param=":2",
|
||||
limit_param=":3",
|
||||
text_param=":4",
|
||||
arm_index=0,
|
||||
)
|
||||
arm1 = d.build_bm25_arm(
|
||||
table="t", cols="id", fact_type="experience",
|
||||
bank_id_param=":2", limit_param=":3", text_param=":4", arm_index=1,
|
||||
table="t",
|
||||
cols="id",
|
||||
fact_type="experience",
|
||||
bank_id_param=":2",
|
||||
limit_param=":3",
|
||||
text_param=":4",
|
||||
arm_index=1,
|
||||
)
|
||||
assert "SCORE(10)" in arm0
|
||||
assert "SCORE(11)" in arm1
|
||||
@@ -474,9 +532,7 @@ class TestOracleQueryRewriter:
|
||||
"""Verify JSONB ->> boolean comparison is rewritten to JSON_VALUE."""
|
||||
from hindsight_api.engine.db.oracle import _rewrite_pg_to_oracle
|
||||
|
||||
query, _, _ = _rewrite_pg_to_oracle(
|
||||
"WHERE (trigger->>'refresh_after_consolidation')::boolean = true"
|
||||
)
|
||||
query, _, _ = _rewrite_pg_to_oracle("WHERE (trigger->>'refresh_after_consolidation')::boolean = true")
|
||||
assert "JSON_VALUE" in query
|
||||
assert "'true'" in query
|
||||
assert "->>" not in query
|
||||
@@ -493,9 +549,7 @@ class TestOracleQueryRewriter:
|
||||
"""Verify ->> works with quoted column names."""
|
||||
from hindsight_api.engine.db.oracle import _rewrite_pg_to_oracle
|
||||
|
||||
query, _, _ = _rewrite_pg_to_oracle(
|
||||
"ORDER BY (result_metadata->>'sub_batch_index')::int"
|
||||
)
|
||||
query, _, _ = _rewrite_pg_to_oracle("ORDER BY (result_metadata->>'sub_batch_index')::int")
|
||||
assert "JSON_VALUE" in query
|
||||
assert "->>" not in query
|
||||
|
||||
@@ -681,9 +735,7 @@ class TestOracleOpsInsertFactsBatch:
|
||||
@pytest.mark.asyncio
|
||||
async def test_tags_json_decoded_to_list(self, ops, mock_conn):
|
||||
"""Tags JSON strings must be decoded to Python lists, not passed as strings."""
|
||||
await ops.insert_facts_batch(
|
||||
conn=mock_conn, **{**self._make_batch(1), "tags_list": ['["tag1", "tag2"]']}
|
||||
)
|
||||
await ops.insert_facts_batch(conn=mock_conn, **{**self._make_batch(1), "tags_list": ['["tag1", "tag2"]']})
|
||||
_, rows_data = mock_conn.executemany.call_args.args
|
||||
assert rows_data[0][13] == ["tag1", "tag2"]
|
||||
assert isinstance(rows_data[0][13], list)
|
||||
@@ -691,9 +743,7 @@ class TestOracleOpsInsertFactsBatch:
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_tags_becomes_empty_list(self, ops, mock_conn):
|
||||
"""Empty/falsy tags string must become [], not crash or pass empty string."""
|
||||
await ops.insert_facts_batch(
|
||||
conn=mock_conn, **{**self._make_batch(1), "tags_list": [""]}
|
||||
)
|
||||
await ops.insert_facts_batch(conn=mock_conn, **{**self._make_batch(1), "tags_list": [""]})
|
||||
_, rows_data = mock_conn.executemany.call_args.args
|
||||
assert rows_data[0][13] == []
|
||||
|
||||
|
||||
@@ -36,16 +36,10 @@ class TestPassthrough:
|
||||
|
||||
class TestSchemeNormalization:
|
||||
def test_asyncpg_scheme_stripped(self) -> None:
|
||||
assert (
|
||||
to_libpq_url("postgresql+asyncpg://user:pass@host:5432/db")
|
||||
== "postgresql://user:pass@host:5432/db"
|
||||
)
|
||||
assert to_libpq_url("postgresql+asyncpg://user:pass@host:5432/db") == "postgresql://user:pass@host:5432/db"
|
||||
|
||||
def test_postgres_asyncpg_scheme_normalized(self) -> None:
|
||||
assert (
|
||||
to_libpq_url("postgres+asyncpg://user:pass@host/db")
|
||||
== "postgresql://user:pass@host/db"
|
||||
)
|
||||
assert to_libpq_url("postgres+asyncpg://user:pass@host/db") == "postgresql://user:pass@host/db"
|
||||
|
||||
def test_bare_postgres_scheme_normalized_to_postgresql(self) -> None:
|
||||
assert to_libpq_url("postgres://user:pass@host/db") == "postgresql://user:pass@host/db"
|
||||
@@ -68,9 +62,7 @@ class TestSslParamRename:
|
||||
assert to_libpq_url("postgresql://h/d?ssl=require") == "postgresql://h/d?sslmode=require"
|
||||
|
||||
def test_ssl_param_preserved_among_other_params(self) -> None:
|
||||
result = to_libpq_url(
|
||||
"postgresql+asyncpg://h/d?ssl=require&application_name=hindsight&connect_timeout=10"
|
||||
)
|
||||
result = to_libpq_url("postgresql+asyncpg://h/d?ssl=require&application_name=hindsight&connect_timeout=10")
|
||||
assert result.startswith("postgresql://h/d?")
|
||||
# Query order should be preserved; ssl renamed, others untouched.
|
||||
assert "sslmode=require" in result
|
||||
@@ -80,10 +72,7 @@ class TestSslParamRename:
|
||||
|
||||
def test_sslmode_not_double_renamed(self) -> None:
|
||||
"""An already-correct sslmode= param must not be altered."""
|
||||
assert (
|
||||
to_libpq_url("postgresql+asyncpg://h/d?sslmode=require")
|
||||
== "postgresql://h/d?sslmode=require"
|
||||
)
|
||||
assert to_libpq_url("postgresql+asyncpg://h/d?sslmode=require") == "postgresql://h/d?sslmode=require"
|
||||
|
||||
|
||||
class TestProductionConfigs:
|
||||
@@ -132,10 +121,7 @@ class TestEdgeCases:
|
||||
assert result == "postgresql://user:my%2Basyncpgpass@host/db"
|
||||
|
||||
def test_url_without_query_string(self) -> None:
|
||||
assert (
|
||||
to_libpq_url("postgresql+asyncpg://user:pass@host/db")
|
||||
== "postgresql://user:pass@host/db"
|
||||
)
|
||||
assert to_libpq_url("postgresql+asyncpg://user:pass@host/db") == "postgresql://user:pass@host/db"
|
||||
|
||||
def test_url_with_port_and_path_only(self) -> None:
|
||||
assert to_libpq_url("postgresql+asyncpg://host:5432/db") == "postgresql://host:5432/db"
|
||||
|
||||
@@ -126,22 +126,30 @@ class TestDeltaEditorialFusion:
|
||||
|
||||
# Phase 1: Ingest SEO best practices
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id, content=SEO_BEST_PRACTICES,
|
||||
document_id="seo-best-practices", request_context=request_context,
|
||||
bank_id=bank_id,
|
||||
content=SEO_BEST_PRACTICES,
|
||||
document_id="seo-best-practices",
|
||||
request_context=request_context,
|
||||
)
|
||||
mm_after_seo = await memory.refresh_mental_model(
|
||||
bank_id=bank_id, mental_model_id=mm_id, request_context=request_context,
|
||||
bank_id=bank_id,
|
||||
mental_model_id=mm_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
seo_content = mm_after_seo["content"]
|
||||
assert len(seo_content) > 100, f"First refresh produced too little content: {len(seo_content)} chars"
|
||||
|
||||
# Phase 2: Ingest brand voice -> delta refresh
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id, content=BRAND_VOICE,
|
||||
document_id="brand-voice", request_context=request_context,
|
||||
bank_id=bank_id,
|
||||
content=BRAND_VOICE,
|
||||
document_id="brand-voice",
|
||||
request_context=request_context,
|
||||
)
|
||||
mm_after_brand = await memory.refresh_mental_model(
|
||||
bank_id=bank_id, mental_model_id=mm_id, request_context=request_context,
|
||||
bank_id=bank_id,
|
||||
mental_model_id=mm_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
fused = mm_after_brand["content"]
|
||||
rr = mm_after_brand.get("reflect_response") or {}
|
||||
@@ -156,8 +164,7 @@ class TestDeltaEditorialFusion:
|
||||
"vocabulary rules": ["jargon", "leverage", "empower", "forbidden"],
|
||||
}.items():
|
||||
assert any(s in fused_lower for s in signals), (
|
||||
f"Brand voice concept '{concept}' missing (looked for {signals}).\n"
|
||||
f"Fused content:\n{fused[:500]}"
|
||||
f"Brand voice concept '{concept}' missing (looked for {signals}).\nFused content:\n{fused[:500]}"
|
||||
)
|
||||
|
||||
# SEO concepts still present (not wiped by delta)
|
||||
@@ -167,8 +174,7 @@ class TestDeltaEditorialFusion:
|
||||
"seo": ["meta", "e-e-a-t", "seo", "search"],
|
||||
}.items():
|
||||
assert any(s in fused_lower for s in signals), (
|
||||
f"SEO concept '{concept}' missing (looked for {signals}).\n"
|
||||
f"Fused content:\n{fused[:500]}"
|
||||
f"SEO concept '{concept}' missing (looked for {signals}).\nFused content:\n{fused[:500]}"
|
||||
)
|
||||
|
||||
# Brand voice overrides generic tone
|
||||
@@ -177,15 +183,9 @@ class TestDeltaEditorialFusion:
|
||||
)
|
||||
|
||||
# No duplicate paragraphs
|
||||
lines = [
|
||||
ln.strip() for ln in fused.split("\n")
|
||||
if ln.strip() and not ln.strip().startswith("#")
|
||||
]
|
||||
lines = [ln.strip() for ln in fused.split("\n") if ln.strip() and not ln.strip().startswith("#")]
|
||||
dupes = {line: cnt for line, cnt in Counter(lines).items() if cnt > 1}
|
||||
assert not dupes, (
|
||||
"Duplicate paragraphs:\n" +
|
||||
"\n".join(f" [{c}x] {t[:80]}" for t, c in dupes.items())
|
||||
)
|
||||
assert not dupes, "Duplicate paragraphs:\n" + "\n".join(f" [{c}x] {t[:80]}" for t, c in dupes.items())
|
||||
|
||||
# based_on accumulates from both docs
|
||||
obs_count = len(rr.get("based_on", {}).get("observation", []))
|
||||
|
||||
@@ -146,7 +146,10 @@ async def test_delta_retain_appended_content(memory, request_context):
|
||||
|
||||
# Second version — original content + new content appended
|
||||
# This should preserve facts from the first chunk and add new ones
|
||||
v2_content = v1_content + "\n\nBob joined Google as a product manager in 2024. He previously worked at Meta on AR/VR products."
|
||||
v2_content = (
|
||||
v1_content
|
||||
+ "\n\nBob joined Google as a product manager in 2024. He previously worked at Meta on AR/VR products."
|
||||
)
|
||||
|
||||
v2_units = await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
@@ -387,9 +390,7 @@ async def test_delta_retain_links_preserved_for_unchanged_chunks(memory, request
|
||||
document_id,
|
||||
)
|
||||
|
||||
assert v2_link_count == v1_link_count, (
|
||||
f"Links should be preserved: v1={v1_link_count}, v2={v2_link_count}"
|
||||
)
|
||||
assert v2_link_count == v1_link_count, f"Links should be preserved: v1={v1_link_count}, v2={v2_link_count}"
|
||||
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
@@ -456,11 +457,13 @@ async def test_delta_retain_tags_propagated_to_existing_units(memory, request_co
|
||||
# v1 with tag "team-a"
|
||||
await memory.retain_batch_async(
|
||||
bank_id=bank_id,
|
||||
contents=[{
|
||||
"content": content,
|
||||
"document_id": document_id,
|
||||
"tags": ["team-a"],
|
||||
}],
|
||||
contents=[
|
||||
{
|
||||
"content": content,
|
||||
"document_id": document_id,
|
||||
"tags": ["team-a"],
|
||||
}
|
||||
],
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
@@ -476,11 +479,13 @@ async def test_delta_retain_tags_propagated_to_existing_units(memory, request_co
|
||||
# v2 with same content but different tags
|
||||
await memory.retain_batch_async(
|
||||
bank_id=bank_id,
|
||||
contents=[{
|
||||
"content": content,
|
||||
"document_id": document_id,
|
||||
"tags": ["team-b", "important"],
|
||||
}],
|
||||
contents=[
|
||||
{
|
||||
"content": content,
|
||||
"document_id": document_id,
|
||||
"tags": ["team-b", "important"],
|
||||
}
|
||||
],
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
@@ -692,7 +697,9 @@ async def test_delta_retain_empty_to_content(memory, request_context):
|
||||
|
||||
doc_v2 = await memory.get_document(document_id, bank_id, request_context=request_context)
|
||||
assert doc_v2 is not None
|
||||
assert doc_v2["memory_unit_count"] > 0 or len(v2_units) > 0, "Should have facts after updating with real content"
|
||||
assert doc_v2["memory_unit_count"] > 0 or len(v2_units) > 0, (
|
||||
"Should have facts after updating with real content"
|
||||
)
|
||||
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
@@ -775,11 +782,13 @@ async def test_delta_retain_with_user_entities(memory, request_context):
|
||||
# v1 with user entities
|
||||
await memory.retain_batch_async(
|
||||
bank_id=bank_id,
|
||||
contents=[{
|
||||
"content": content,
|
||||
"document_id": document_id,
|
||||
"entities": [{"text": "Project Alpha", "type": "PROJECT"}],
|
||||
}],
|
||||
contents=[
|
||||
{
|
||||
"content": content,
|
||||
"document_id": document_id,
|
||||
"entities": [{"text": "Project Alpha", "type": "PROJECT"}],
|
||||
}
|
||||
],
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
@@ -797,14 +806,16 @@ async def test_delta_retain_with_user_entities(memory, request_context):
|
||||
v2_content = content + "\n\nThe timeline is on track for Q2 delivery."
|
||||
await memory.retain_batch_async(
|
||||
bank_id=bank_id,
|
||||
contents=[{
|
||||
"content": v2_content,
|
||||
"document_id": document_id,
|
||||
"entities": [
|
||||
{"text": "Project Alpha", "type": "PROJECT"},
|
||||
{"text": "Q2 Deadline", "type": "MILESTONE"},
|
||||
],
|
||||
}],
|
||||
contents=[
|
||||
{
|
||||
"content": v2_content,
|
||||
"document_id": document_id,
|
||||
"entities": [
|
||||
{"text": "Project Alpha", "type": "PROJECT"},
|
||||
{"text": "Q2 Deadline", "type": "MILESTONE"},
|
||||
],
|
||||
}
|
||||
],
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
@@ -867,9 +878,7 @@ async def test_delta_retain_recall_with_chunks(memory, request_context):
|
||||
facts_with_chunks = [r for r in result.results if r.chunk_id]
|
||||
if facts_with_chunks and result.chunks:
|
||||
for fact in facts_with_chunks:
|
||||
assert fact.chunk_id in result.chunks, (
|
||||
f"Chunk {fact.chunk_id} should be in returned chunks"
|
||||
)
|
||||
assert fact.chunk_id in result.chunks, f"Chunk {fact.chunk_id} should be in returned chunks"
|
||||
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
@@ -1024,8 +1033,7 @@ async def test_processed_content_tokens_appended_reports_delta(memory, request_c
|
||||
return
|
||||
assert second > 0, "Partial-delta retain should report a positive token count"
|
||||
assert second < submitted_tokens, (
|
||||
"Partial-delta retain should report fewer processed tokens "
|
||||
"than the full submitted payload"
|
||||
"Partial-delta retain should report fewer processed tokens than the full submitted payload"
|
||||
)
|
||||
finally:
|
||||
memory._operation_validator = None
|
||||
|
||||
@@ -143,9 +143,7 @@ async def test_delta_detects_unchanged_after_first_retain(memory, request_contex
|
||||
bank_id,
|
||||
document_id,
|
||||
)
|
||||
assert v2_count == v1_count, (
|
||||
f"Memory unit count changed on same-content upsert: {v1_count} -> {v2_count}"
|
||||
)
|
||||
assert v2_count == v1_count, f"Memory unit count changed on same-content upsert: {v1_count} -> {v2_count}"
|
||||
|
||||
# Third retain — verify stability
|
||||
v3_units = await memory.retain_async(
|
||||
@@ -163,9 +161,7 @@ async def test_delta_detects_unchanged_after_first_retain(memory, request_contex
|
||||
bank_id,
|
||||
document_id,
|
||||
)
|
||||
assert v3_count == v1_count, (
|
||||
f"Memory unit count changed on third upsert: {v1_count} -> {v3_count}"
|
||||
)
|
||||
assert v3_count == v1_count, f"Memory unit count changed on third upsert: {v1_count} -> {v3_count}"
|
||||
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
@@ -371,9 +367,7 @@ async def test_concurrent_upserts_no_duplicates(memory_no_llm, request_context):
|
||||
# splitter may cut mid-text, so later chunks might not start with the prefix.
|
||||
winning_person = f"Person_{winning_version}"
|
||||
wrong_version_units = [
|
||||
(r["text"], r["chunk_id"], r["unit_id"])
|
||||
for r in units
|
||||
if winning_person not in r["text"]
|
||||
(r["text"], r["chunk_id"], r["unit_id"]) for r in units if winning_person not in r["text"]
|
||||
]
|
||||
assert not wrong_version_units, (
|
||||
f"Found {len(wrong_version_units)} memory units NOT from winning version "
|
||||
@@ -397,8 +391,7 @@ async def test_concurrent_upserts_no_duplicates(memory_no_llm, request_context):
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Concurrent test passed: version {winning_version} won with "
|
||||
f"{len(unit_texts)} memory units, no duplicates"
|
||||
f"Concurrent test passed: version {winning_version} won with {len(unit_texts)} memory units, no duplicates"
|
||||
)
|
||||
|
||||
finally:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Tests for document chunks API, reprocess, nodes_by_fact_type, and graph document/chunk filtering.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
@@ -223,9 +224,7 @@ async def test_graph_chunk_id_filter(api_client, bank_id):
|
||||
await _retain(api_client, bank_id, "doc-chunk-test", "Alice works at Google. " * 20)
|
||||
|
||||
# First get chunks to find a valid chunk_id
|
||||
response = await api_client.get(
|
||||
f"/v1/default/banks/{bank_id}/documents/doc-chunk-test/chunks"
|
||||
)
|
||||
response = await api_client.get(f"/v1/default/banks/{bank_id}/documents/doc-chunk-test/chunks")
|
||||
assert response.status_code == 200
|
||||
chunks_data = response.json()
|
||||
if chunks_data["total"] == 0:
|
||||
@@ -251,11 +250,14 @@ async def test_graph_chunk_id_filter(api_client, bank_id):
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_list_document_chunks(api_client, bank_id):
|
||||
"""HTTP GET .../documents/{id}/chunks returns chunks."""
|
||||
await _retain(api_client, bank_id, "doc-http-chunks", "Alice works at Google on AI research. Bob works at Meta on VR systems. " * 20)
|
||||
|
||||
response = await api_client.get(
|
||||
f"/v1/default/banks/{bank_id}/documents/doc-http-chunks/chunks"
|
||||
await _retain(
|
||||
api_client,
|
||||
bank_id,
|
||||
"doc-http-chunks",
|
||||
"Alice works at Google on AI research. Bob works at Meta on VR systems. " * 20,
|
||||
)
|
||||
|
||||
response = await api_client.get(f"/v1/default/banks/{bank_id}/documents/doc-http-chunks/chunks")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "items" in data
|
||||
@@ -266,9 +268,7 @@ async def test_http_list_document_chunks(api_client, bank_id):
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_list_document_chunks_not_found(api_client, bank_id):
|
||||
"""HTTP GET .../documents/{id}/chunks returns 404 for non-existent document."""
|
||||
response = await api_client.get(
|
||||
f"/v1/default/banks/{bank_id}/documents/nonexistent/chunks"
|
||||
)
|
||||
response = await api_client.get(f"/v1/default/banks/{bank_id}/documents/nonexistent/chunks")
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
@@ -277,9 +277,7 @@ async def test_http_reprocess_document(api_client, bank_id):
|
||||
"""HTTP POST .../documents/{id}/reprocess returns success with operation_id."""
|
||||
await _retain(api_client, bank_id, "doc-http-reprocess", "Alice works at Google.")
|
||||
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{bank_id}/documents/doc-http-reprocess/reprocess"
|
||||
)
|
||||
response = await api_client.post(f"/v1/default/banks/{bank_id}/documents/doc-http-reprocess/reprocess")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["success"] is True
|
||||
@@ -289,9 +287,7 @@ async def test_http_reprocess_document(api_client, bank_id):
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_reprocess_document_not_found(api_client, bank_id):
|
||||
"""HTTP POST .../documents/{id}/reprocess returns 404 for non-existent document."""
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{bank_id}/documents/nonexistent/reprocess"
|
||||
)
|
||||
response = await api_client.post(f"/v1/default/banks/{bank_id}/documents/nonexistent/reprocess")
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
@@ -300,9 +296,7 @@ async def test_http_get_document_includes_nodes_by_fact_type(api_client, bank_id
|
||||
"""HTTP GET .../documents/{id} includes nodes_by_fact_type."""
|
||||
await _retain(api_client, bank_id, "doc-http-comp", "Alice works at Google on AI research.")
|
||||
|
||||
response = await api_client.get(
|
||||
f"/v1/default/banks/{bank_id}/documents/doc-http-comp"
|
||||
)
|
||||
response = await api_client.get(f"/v1/default/banks/{bank_id}/documents/doc-http-comp")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "nodes_by_fact_type" in data
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Tests for document tracking and upsert functionality.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import patch
|
||||
@@ -357,9 +358,7 @@ async def test_document_persisted_with_zero_facts_async_submit(memory_real_llm,
|
||||
elapsed += wait_interval
|
||||
|
||||
# Check if document exists
|
||||
doc = await memory.get_document(
|
||||
"doc-async-zero-facts", bank_id, request_context=request_context
|
||||
)
|
||||
doc = await memory.get_document("doc-async-zero-facts", bank_id, request_context=request_context)
|
||||
if doc is not None:
|
||||
break
|
||||
|
||||
|
||||
@@ -86,9 +86,7 @@ async def _import(memory, bank_id, archive, request_context, on_conflict="skip")
|
||||
inline and is already completed when submit returns.
|
||||
"""
|
||||
submission = await memory.import_documents_async(bank_id, archive, request_context, on_conflict)
|
||||
status = await memory.get_operation_status(
|
||||
bank_id, submission["operation_id"], request_context=request_context
|
||||
)
|
||||
status = await memory.get_operation_status(bank_id, submission["operation_id"], request_context=request_context)
|
||||
assert status["status"] == "completed", status
|
||||
return status["result_metadata"]
|
||||
|
||||
@@ -340,12 +338,8 @@ async def test_bank_roundtrip_carries_mental_model_history(memory, request_conte
|
||||
mental_model_id="mm-1",
|
||||
request_context=request_context,
|
||||
)
|
||||
await memory.update_mental_model(
|
||||
bank, mental_model_id="mm-1", content="v2", request_context=request_context
|
||||
)
|
||||
await memory.update_mental_model(
|
||||
bank, mental_model_id="mm-1", content="v3", request_context=request_context
|
||||
)
|
||||
await memory.update_mental_model(bank, mental_model_id="mm-1", content="v2", request_context=request_context)
|
||||
await memory.update_mental_model(bank, mental_model_id="mm-1", content="v3", request_context=request_context)
|
||||
# Two refreshes → two snapshots (previous content v1 then v2), newest-first.
|
||||
before = await memory.get_mental_model_history(bank, "mm-1", request_context=request_context)
|
||||
assert [h["previous_content"] for h in before] == ["v2", "v1"]
|
||||
@@ -475,13 +469,11 @@ async def _bank_snapshot(memory, bank_id):
|
||||
backend = await memory._get_backend()
|
||||
async with acquire_with_retry(backend) as conn:
|
||||
docs = await conn.fetch(
|
||||
f"SELECT id, COALESCE(length(original_text), 0) AS len FROM {fq_table('documents')} "
|
||||
f"WHERE bank_id = $1",
|
||||
f"SELECT id, COALESCE(length(original_text), 0) AS len FROM {fq_table('documents')} WHERE bank_id = $1",
|
||||
bank_id,
|
||||
)
|
||||
chunks = await conn.fetch(
|
||||
f"SELECT document_id, chunk_index, length(chunk_text) AS len FROM {fq_table('chunks')} "
|
||||
f"WHERE bank_id = $1",
|
||||
f"SELECT document_id, chunk_index, length(chunk_text) AS len FROM {fq_table('chunks')} WHERE bank_id = $1",
|
||||
bank_id,
|
||||
)
|
||||
ftypes = await conn.fetch(
|
||||
@@ -591,9 +583,7 @@ async def test_export_import_observations(memory, request_context):
|
||||
backend = await memory._get_backend()
|
||||
async with acquire_with_retry(backend) as conn:
|
||||
async with conn.transaction():
|
||||
await _create_observation_directly(
|
||||
conn, memory, src, source_ids, "Alice and Bob are colleagues."
|
||||
)
|
||||
await _create_observation_directly(conn, memory, src, source_ids, "Alice and Bob are colleagues.")
|
||||
|
||||
# Export WITHOUT observations -> none in the archive (the bank may also
|
||||
# contain auto-consolidation observations; the flag is what gates them).
|
||||
@@ -762,9 +752,7 @@ async def test_include_observations_requires_whole_bank_export(memory, request_c
|
||||
await _retain(memory, src, "Alice works at Google.", request_context, "doc-1")
|
||||
# Subset export (document_ids set) + observations must be rejected.
|
||||
with pytest.raises(ValueError, match="whole bank"):
|
||||
await memory.export_documents_async(
|
||||
src, request_context, ["doc-1"], include_observations=True
|
||||
)
|
||||
await memory.export_documents_async(src, request_context, ["doc-1"], include_observations=True)
|
||||
# Whole-bank export with observations is fine; subset without observations is fine.
|
||||
await memory.export_documents_async(src, request_context, include_observations=True)
|
||||
await memory.export_documents_async(src, request_context, ["doc-1"])
|
||||
|
||||
@@ -80,11 +80,7 @@ def test_parse_entity_labels_dict_format():
|
||||
|
||||
def test_parse_entity_labels_dict_format_defaults():
|
||||
"""Dict format parses attributes correctly."""
|
||||
raw = {
|
||||
"attributes": [
|
||||
{"key": "topic", "values": [{"value": "math", "description": "Mathematics"}]}
|
||||
]
|
||||
}
|
||||
raw = {"attributes": [{"key": "topic", "values": [{"value": "math", "description": "Mathematics"}]}]}
|
||||
result = parse_entity_labels(raw)
|
||||
assert result is not None
|
||||
assert len(result.attributes) == 1
|
||||
@@ -178,9 +174,7 @@ def test_build_labels_model_free_values_optional():
|
||||
"""type='text', optional=True → str | None field."""
|
||||
from hindsight_api.engine.retain.entity_labels import build_labels_model
|
||||
|
||||
labels_cfg = EntityLabelsConfig(
|
||||
attributes=[LabelGroup(key="topic", type="text", optional=True, values=[])]
|
||||
)
|
||||
labels_cfg = EntityLabelsConfig(attributes=[LabelGroup(key="topic", type="text", optional=True, values=[])])
|
||||
Model = build_labels_model(labels_cfg)
|
||||
assert Model is not None
|
||||
schema = Model.model_json_schema()
|
||||
@@ -194,9 +188,7 @@ def test_build_labels_model_free_values_always_optional():
|
||||
"""type='text' with optional=False is still treated as str | None — always optional."""
|
||||
from hindsight_api.engine.retain.entity_labels import build_labels_model
|
||||
|
||||
labels_cfg = EntityLabelsConfig(
|
||||
attributes=[LabelGroup(key="topic", type="text", optional=False, values=[])]
|
||||
)
|
||||
labels_cfg = EntityLabelsConfig(attributes=[LabelGroup(key="topic", type="text", optional=False, values=[])])
|
||||
Model = build_labels_model(labels_cfg)
|
||||
assert Model is not None
|
||||
schema = Model.model_json_schema()
|
||||
@@ -210,9 +202,7 @@ def test_build_labels_model_free_values_multi_still_optional():
|
||||
"""type='text' is always str | None — multi-values only applies to enum types."""
|
||||
from hindsight_api.engine.retain.entity_labels import build_labels_model
|
||||
|
||||
labels_cfg = EntityLabelsConfig(
|
||||
attributes=[LabelGroup(key="tags", type="text", values=[])]
|
||||
)
|
||||
labels_cfg = EntityLabelsConfig(attributes=[LabelGroup(key="tags", type="text", values=[])])
|
||||
Model = build_labels_model(labels_cfg)
|
||||
assert Model is not None
|
||||
schema = Model.model_json_schema()
|
||||
@@ -226,9 +216,7 @@ def test_build_labels_model_free_values_no_values_still_creates_field():
|
||||
"""type='text' group with no values still creates a field (description holds examples)."""
|
||||
from hindsight_api.engine.retain.entity_labels import build_labels_model
|
||||
|
||||
labels_cfg = EntityLabelsConfig(
|
||||
attributes=[LabelGroup(key="mood", type="text", values=[])]
|
||||
)
|
||||
labels_cfg = EntityLabelsConfig(attributes=[LabelGroup(key="mood", type="text", values=[])])
|
||||
Model = build_labels_model(labels_cfg)
|
||||
assert Model is not None
|
||||
assert "mood" in Model.model_json_schema()["properties"]
|
||||
@@ -549,9 +537,7 @@ def test_label_entity_post_processing_invalid_value_ignored():
|
||||
from hindsight_api.engine.retain.entity_labels import build_labels_lookup, parse_entity_labels
|
||||
from hindsight_api.engine.retain.fact_extraction import Entity
|
||||
|
||||
labels_cfg = parse_entity_labels(
|
||||
[{"key": "pedagogy", "values": [{"value": "scaffolding", "description": ""}]}]
|
||||
)
|
||||
labels_cfg = parse_entity_labels([{"key": "pedagogy", "values": [{"value": "scaffolding", "description": ""}]}])
|
||||
labels_lookup = build_labels_lookup(labels_cfg)
|
||||
|
||||
labels_data = {"pedagogy": "unknown_value"}
|
||||
@@ -665,9 +651,7 @@ def test_free_values_label_is_single_value():
|
||||
"""type='text' groups are always single-value (str | None)."""
|
||||
from hindsight_api.engine.retain.entity_labels import build_labels_model, parse_entity_labels
|
||||
|
||||
labels_cfg = parse_entity_labels(
|
||||
[{"key": "topic", "type": "text", "values": []}]
|
||||
)
|
||||
labels_cfg = parse_entity_labels([{"key": "topic", "type": "text", "values": []}])
|
||||
Model = build_labels_model(labels_cfg)
|
||||
assert Model is not None
|
||||
schema = Model.model_json_schema()
|
||||
@@ -681,9 +665,7 @@ def test_free_values_label_not_in_lookup():
|
||||
"""type='text' group values do NOT appear in the lookup set (no fixed vocabulary)."""
|
||||
from hindsight_api.engine.retain.entity_labels import build_labels_lookup, parse_entity_labels
|
||||
|
||||
labels_cfg = parse_entity_labels(
|
||||
[{"key": "topic", "type": "text", "values": [{"value": "algebra"}]}]
|
||||
)
|
||||
labels_cfg = parse_entity_labels([{"key": "topic", "type": "text", "values": [{"value": "algebra"}]}])
|
||||
lookup = build_labels_lookup(labels_cfg)
|
||||
assert "topic:algebra" not in lookup # example hints not added to lookup
|
||||
assert len(lookup) == 0
|
||||
@@ -725,9 +707,7 @@ def test_optional_label_string_none_produces_no_entity():
|
||||
|
||||
# LLM returned the string "None" instead of JSON null — must not be stored
|
||||
entity_texts = _run_label_post_processing(labels_cfg, {"engagement": "None"})
|
||||
assert entity_texts == set(), (
|
||||
f"String 'None' must not produce engagement:None entity, got: {entity_texts}"
|
||||
)
|
||||
assert entity_texts == set(), f"String 'None' must not produce engagement:None entity, got: {entity_texts}"
|
||||
|
||||
|
||||
def test_optional_label_null_does_not_affect_other_labels():
|
||||
@@ -744,9 +724,7 @@ def test_optional_label_null_does_not_affect_other_labels():
|
||||
# engagement is null, but topic is set
|
||||
entity_texts = _run_label_post_processing(labels_cfg, {"engagement": None, "topic": "math"})
|
||||
assert "topic:math" in entity_texts, f"Expected topic:math entity, got: {entity_texts}"
|
||||
assert not any("engagement" in t for t in entity_texts), (
|
||||
f"engagement should not appear, got: {entity_texts}"
|
||||
)
|
||||
assert not any("engagement" in t for t in entity_texts), f"engagement should not appear, got: {entity_texts}"
|
||||
|
||||
|
||||
def test_free_form_entities_false_clears_entities():
|
||||
@@ -982,9 +960,7 @@ async def test_retain_extracts_single_value_label(memory_real_llm, request_conte
|
||||
)
|
||||
|
||||
entity_names = {r["canonical_name"].lower() for r in rows}
|
||||
assert "engagement:active" in entity_names, (
|
||||
f"Expected 'engagement:active' label entity. Got: {entity_names}"
|
||||
)
|
||||
assert "engagement:active" in entity_names, f"Expected 'engagement:active' label entity. Got: {entity_names}"
|
||||
# In labels-only mode, free-form entities like 'Maria' should be absent
|
||||
assert not any("maria" in n for n in entity_names), (
|
||||
f"Free-form entity 'Maria' should not appear in labels-only mode. Got: {entity_names}"
|
||||
@@ -1054,9 +1030,7 @@ async def test_retain_extracts_multi_value_label(memory_real_llm, request_contex
|
||||
entity_names = {r["canonical_name"].lower() for r in rows}
|
||||
# At least one pedagogy label should be assigned
|
||||
pedagogy_labels = {n for n in entity_names if n.startswith("pedagogy:")}
|
||||
assert len(pedagogy_labels) > 0, (
|
||||
f"Expected at least one pedagogy:* label entity. Got: {entity_names}"
|
||||
)
|
||||
assert len(pedagogy_labels) > 0, f"Expected at least one pedagogy:* label entity. Got: {entity_names}"
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
|
||||
@@ -1118,9 +1092,7 @@ async def test_retain_extracts_free_values_label(memory_real_llm, request_contex
|
||||
entity_names = {r["canonical_name"].lower() for r in rows}
|
||||
# A topic:* entity must exist — value is free-form so we only check the prefix
|
||||
topic_entities = {n for n in entity_names if n.startswith("topic:")}
|
||||
assert len(topic_entities) > 0, (
|
||||
f"Expected at least one topic:* free-value entity. Got: {entity_names}"
|
||||
)
|
||||
assert len(topic_entities) > 0, f"Expected at least one topic:* free-value entity. Got: {entity_names}"
|
||||
# The value must not be the literal string "none" or "null"
|
||||
assert not any(n in ("topic:none", "topic:null", "topic:n/a") for n in topic_entities), (
|
||||
f"topic entity should not be a null sentinel. Got: {topic_entities}"
|
||||
@@ -1188,18 +1160,14 @@ async def test_retain_extracts_map_type_entities(memory_real_llm, request_contex
|
||||
entity_names = {r["canonical_name"].lower() for r in rows}
|
||||
# Should have person:name:* entity
|
||||
name_entities = {n for n in entity_names if n.startswith("person:name:")}
|
||||
assert len(name_entities) > 0, (
|
||||
f"Expected at least one person:name:* entity. Got: {entity_names}"
|
||||
)
|
||||
assert len(name_entities) > 0, f"Expected at least one person:name:* entity. Got: {entity_names}"
|
||||
# Name should contain "alice" somewhere
|
||||
assert any("alice" in n for n in name_entities), (
|
||||
f"Expected person:name entity containing 'alice'. Got: {name_entities}"
|
||||
)
|
||||
# Should have person:organization:* entity mentioning google
|
||||
org_entities = {n for n in entity_names if n.startswith("person:organization:")}
|
||||
assert len(org_entities) > 0, (
|
||||
f"Expected at least one person:organization:* entity. Got: {entity_names}"
|
||||
)
|
||||
assert len(org_entities) > 0, f"Expected at least one person:organization:* entity. Got: {entity_names}"
|
||||
assert any("google" in n for n in org_entities), (
|
||||
f"Expected person:organization entity containing 'google'. Got: {org_entities}"
|
||||
)
|
||||
@@ -2036,9 +2004,7 @@ async def test_retain_multivalue_tag_entities_all_stored(memory_real_llm, reques
|
||||
|
||||
# The core assertion from GH-1558: tags and entities should match
|
||||
# Tags show both but entities only show a subset → BUG
|
||||
assert len(use_tags) >= 2, (
|
||||
f"Expected at least 2 use:* tags. Got: {use_tags}"
|
||||
)
|
||||
assert len(use_tags) >= 2, f"Expected at least 2 use:* tags. Got: {use_tags}"
|
||||
assert len(use_entities) >= 2, (
|
||||
f"GH-1558 BUG: Expected at least 2 use:* entities in unit_entities, "
|
||||
f"but only got {len(use_entities)}: {use_entities}. "
|
||||
@@ -2097,8 +2063,7 @@ async def test_retain_multivalue_tag_entities_second_retain(memory_real_llm, req
|
||||
await memory_real_llm.retain_async(
|
||||
bank_id=bank_id,
|
||||
content=(
|
||||
"## Authentication Flow (use-001)\n\n"
|
||||
"The authentication flow use-001 handles user login via OAuth2."
|
||||
"## Authentication Flow (use-001)\n\nThe authentication flow use-001 handles user login via OAuth2."
|
||||
),
|
||||
request_context=request_context,
|
||||
)
|
||||
@@ -2145,9 +2110,7 @@ async def test_retain_multivalue_tag_entities_second_retain(memory_real_llm, req
|
||||
use_entities = {n for n in entity_names if n.startswith("use:")}
|
||||
use_tags = {t for t in all_tags if t.startswith("use:")}
|
||||
|
||||
assert len(use_tags) >= 2, (
|
||||
f"Expected at least 2 use:* tags on second retain. Got: {use_tags}"
|
||||
)
|
||||
assert len(use_tags) >= 2, f"Expected at least 2 use:* tags on second retain. Got: {use_tags}"
|
||||
assert len(use_entities) >= 2, (
|
||||
f"GH-1558 BUG: On second retain, expected at least 2 use:* entities "
|
||||
f"but only got {len(use_entities)}: {use_entities}. "
|
||||
@@ -2155,9 +2118,7 @@ async def test_retain_multivalue_tag_entities_second_retain(memory_real_llm, req
|
||||
f"Entity resolution may be merging similar names."
|
||||
)
|
||||
missing = use_tags - use_entities
|
||||
assert len(missing) == 0, (
|
||||
f"GH-1558 BUG: Tags present but entities missing after second retain: {missing}"
|
||||
)
|
||||
assert len(missing) == 0, f"GH-1558 BUG: Tags present but entities missing after second retain: {missing}"
|
||||
finally:
|
||||
await memory_real_llm.delete_bank(bank_id, request_context=request_context)
|
||||
|
||||
@@ -2243,9 +2204,7 @@ async def test_entity_resolution_does_not_merge_distinct_label_values(memory, re
|
||||
)
|
||||
|
||||
# We should get 2 DISTINCT entity IDs, not the same ID twice
|
||||
assert len(resolved_entity_ids) == 2, (
|
||||
f"Expected 2 resolved entity IDs, got {len(resolved_entity_ids)}"
|
||||
)
|
||||
assert len(resolved_entity_ids) == 2, f"Expected 2 resolved entity IDs, got {len(resolved_entity_ids)}"
|
||||
unique_ids = set(resolved_entity_ids)
|
||||
assert len(unique_ids) == 2, (
|
||||
f"GH-1558 BUG: Entity resolution merged 'use:use-001' and 'use:use-002' "
|
||||
|
||||
@@ -305,9 +305,7 @@ class TestOracleFuzzyEntityResolution:
|
||||
conn = AsyncMock()
|
||||
conn.backend_type = "oracle"
|
||||
conn.fetch = AsyncMock(return_value=[])
|
||||
entities_data = [
|
||||
{"text": f"Entity {idx}", "nearby_entities": [], "event_date": None} for idx in range(5)
|
||||
]
|
||||
entities_data = [{"text": f"Entity {idx}", "nearby_entities": [], "event_date": None} for idx in range(5)]
|
||||
|
||||
with patch.object(resolver, "_resolve_from_candidates", new_callable=AsyncMock, return_value=[]):
|
||||
await resolver._resolve_entities_batch_oracle_fuzzy(
|
||||
|
||||
@@ -101,25 +101,19 @@ class RateLimitingValidator(OperationValidatorExtension):
|
||||
async def validate_retain(self, ctx: RetainContext) -> ValidationResult:
|
||||
self.retain_counts[ctx.bank_id] += 1
|
||||
if self.retain_counts[ctx.bank_id] > self.max_attempts:
|
||||
return ValidationResult.reject(
|
||||
f"Retain limit exceeded for bank {ctx.bank_id}"
|
||||
)
|
||||
return ValidationResult.reject(f"Retain limit exceeded for bank {ctx.bank_id}")
|
||||
return ValidationResult.accept()
|
||||
|
||||
async def validate_recall(self, ctx: RecallContext) -> ValidationResult:
|
||||
self.recall_counts[ctx.bank_id] += 1
|
||||
if self.recall_counts[ctx.bank_id] > self.max_attempts:
|
||||
return ValidationResult.reject(
|
||||
f"Recall limit exceeded for bank {ctx.bank_id}"
|
||||
)
|
||||
return ValidationResult.reject(f"Recall limit exceeded for bank {ctx.bank_id}")
|
||||
return ValidationResult.accept()
|
||||
|
||||
async def validate_reflect(self, ctx: ReflectContext) -> ValidationResult:
|
||||
self.reflect_counts[ctx.bank_id] += 1
|
||||
if self.reflect_counts[ctx.bank_id] > self.max_attempts:
|
||||
return ValidationResult.reject(
|
||||
f"Reflect limit exceeded for bank {ctx.bank_id}"
|
||||
)
|
||||
return ValidationResult.reject(f"Reflect limit exceeded for bank {ctx.bank_id}")
|
||||
return ValidationResult.accept()
|
||||
|
||||
|
||||
@@ -579,9 +573,7 @@ class TestMemoryEngineTenantAuth:
|
||||
"""Tests for tenant authentication in MemoryEngine."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retain_requires_tenant_request_when_extension_configured(
|
||||
self, memory_with_tenant
|
||||
):
|
||||
async def test_retain_requires_tenant_request_when_extension_configured(self, memory_with_tenant):
|
||||
"""Retain fails without RequestContext when tenant extension is configured."""
|
||||
memory = memory_with_tenant
|
||||
|
||||
@@ -621,9 +613,7 @@ class TestMemoryEngineTenantAuth:
|
||||
assert "Invalid API key" in str(exc_info.value)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recall_requires_tenant_request_when_extension_configured(
|
||||
self, memory_with_tenant
|
||||
):
|
||||
async def test_recall_requires_tenant_request_when_extension_configured(self, memory_with_tenant):
|
||||
"""Recall fails without RequestContext when tenant extension is configured."""
|
||||
memory = memory_with_tenant
|
||||
|
||||
@@ -861,8 +851,7 @@ class RecordingPrecheckValidator(OperationValidatorExtension):
|
||||
instantiable; the tests here only exercise precheck.
|
||||
"""
|
||||
|
||||
def __init__(self, *, reject: bool = False, status_code: int = 402,
|
||||
reason: str = "rejected by precheck") -> None:
|
||||
def __init__(self, *, reject: bool = False, status_code: int = 402, reason: str = "rejected by precheck") -> None:
|
||||
super().__init__(config={})
|
||||
self.reject = reject
|
||||
self.status_code = status_code
|
||||
@@ -1027,9 +1016,7 @@ class TestPrecheckHttpWiring:
|
||||
assert body_parses == ["retain"]
|
||||
|
||||
def test_precheck_rejection_returns_status_and_reason(self):
|
||||
validator = RecordingPrecheckValidator(
|
||||
reject=True, status_code=402, reason="Insufficient credits"
|
||||
)
|
||||
validator = RecordingPrecheckValidator(reject=True, status_code=402, reason="Insufficient credits")
|
||||
app, _ = self._build_app(validator)
|
||||
client = TestClient(app)
|
||||
|
||||
@@ -1045,9 +1032,7 @@ class TestPrecheckHttpWiring:
|
||||
deserialises the body. We send an oversized body and verify the
|
||||
body-parse counter never incremented.
|
||||
"""
|
||||
validator = RecordingPrecheckValidator(
|
||||
reject=True, status_code=402, reason="rejected by precheck"
|
||||
)
|
||||
validator = RecordingPrecheckValidator(reject=True, status_code=402, reason="rejected by precheck")
|
||||
app, body_parses = self._build_app(validator)
|
||||
client = TestClient(app)
|
||||
|
||||
@@ -1063,9 +1048,7 @@ class TestPrecheckHttpWiring:
|
||||
)
|
||||
|
||||
def test_precheck_rejection_skips_body_parse_for_recall(self):
|
||||
validator = RecordingPrecheckValidator(
|
||||
reject=True, status_code=402, reason="rejected"
|
||||
)
|
||||
validator = RecordingPrecheckValidator(reject=True, status_code=402, reason="rejected")
|
||||
app, body_parses = self._build_app(validator)
|
||||
client = TestClient(app)
|
||||
|
||||
@@ -1078,9 +1061,7 @@ class TestPrecheckHttpWiring:
|
||||
assert body_parses == []
|
||||
|
||||
def test_precheck_rejection_skips_body_parse_for_reflect(self):
|
||||
validator = RecordingPrecheckValidator(
|
||||
reject=True, status_code=402, reason="rejected"
|
||||
)
|
||||
validator = RecordingPrecheckValidator(reject=True, status_code=402, reason="rejected")
|
||||
app, body_parses = self._build_app(validator)
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Test to analyze fact extraction token usage and identify optimization opportunities.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
@@ -63,9 +64,9 @@ async def test_fact_extraction_basic_analysis(llm_config):
|
||||
|
||||
duration = time.time() - start_time
|
||||
|
||||
logger.info(f"\n{'='*60}")
|
||||
logger.info(f"\n{'=' * 60}")
|
||||
logger.info(f"EXTRACTION RESULTS")
|
||||
logger.info(f"{'='*60}")
|
||||
logger.info(f"{'=' * 60}")
|
||||
logger.info(f"Duration: {duration:.2f}s")
|
||||
logger.info(f"Chunks: {len(chunks)}")
|
||||
logger.info(f"Facts extracted: {len(facts)}")
|
||||
@@ -86,13 +87,13 @@ async def test_fact_extraction_basic_analysis(llm_config):
|
||||
# Show sample facts
|
||||
logger.info(f"\nSample facts (first 10):")
|
||||
for i, fact in enumerate(facts[:10]):
|
||||
logger.info(f"\n [{i+1}] {fact.fact_type}: {fact.fact[:150]}...")
|
||||
logger.info(f"\n [{i + 1}] {fact.fact_type}: {fact.fact[:150]}...")
|
||||
|
||||
# Show facts containing key terms
|
||||
key_terms = ["kubernetes", "k8s", "CKA", "certification", "Alice"]
|
||||
logger.info(f"\n{'='*60}")
|
||||
logger.info(f"\n{'=' * 60}")
|
||||
logger.info(f"FACTS CONTAINING KEY TERMS")
|
||||
logger.info(f"{'='*60}")
|
||||
logger.info(f"{'=' * 60}")
|
||||
|
||||
for term in key_terms:
|
||||
matching = [f for f in facts if term.lower() in f.fact.lower()]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Unit tests for metadata inclusion in fact extraction LLM prompt.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from hindsight_api.engine.retain.fact_extraction import _build_user_message
|
||||
|
||||
@@ -109,8 +109,7 @@ User: Perfect, I'll make a reservation for Saturday at 7pm.
|
||||
|
||||
# Output should not be more than 5x the input
|
||||
assert ratio < 5.0, (
|
||||
f"Output/input ratio {ratio:.2f} is too high! "
|
||||
f"Input: {input_length} chars, Output: {output_length} chars"
|
||||
f"Output/input ratio {ratio:.2f} is too high! Input: {input_length} chars, Output: {output_length} chars"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -168,16 +167,12 @@ I edited about 20 photos from my recent trip to the mountains.
|
||||
# Output should not be more than 4x the input for longer texts
|
||||
# (ratio should decrease as input grows)
|
||||
assert ratio < 4.0, (
|
||||
f"Output/input ratio {ratio:.2f} is too high! "
|
||||
f"Input: {input_length} chars, Output: {output_length} chars"
|
||||
f"Output/input ratio {ratio:.2f} is too high! Input: {input_length} chars, Output: {output_length} chars"
|
||||
)
|
||||
|
||||
# Also check that individual facts aren't excessively long
|
||||
max_fact_length = max(len(f.fact) for f in facts) if facts else 0
|
||||
assert max_fact_length < 1000, (
|
||||
f"Individual fact too long: {max_fact_length} chars. "
|
||||
f"Facts should be concise."
|
||||
)
|
||||
assert max_fact_length < 1000, f"Individual fact too long: {max_fact_length} chars. Facts should be concise."
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_token_ratio_with_locomo_conversation(self):
|
||||
@@ -190,11 +185,7 @@ I edited about 20 photos from my recent trip to the mountains.
|
||||
import os
|
||||
|
||||
# Load locomo conversation
|
||||
fixture_path = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"fixtures",
|
||||
"locomo_conversation_sample.json"
|
||||
)
|
||||
fixture_path = os.path.join(os.path.dirname(__file__), "fixtures", "locomo_conversation_sample.json")
|
||||
with open(fixture_path, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
@@ -246,8 +237,7 @@ I edited about 20 photos from my recent trip to the mountains.
|
||||
max_expected_facts = num_turns * 2 # At most 2 facts per conversation turn
|
||||
|
||||
assert len(facts) <= max_expected_facts, (
|
||||
f"Too many facts: {len(facts)} for {num_turns} conversation turns. "
|
||||
f"Expected at most {max_expected_facts}."
|
||||
f"Too many facts: {len(facts)} for {num_turns} conversation turns. Expected at most {max_expected_facts}."
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -279,7 +269,7 @@ I'm planning to visit Japan next year.
|
||||
)
|
||||
|
||||
# Count approximate number of statements (sentences)
|
||||
num_statements = len([s for s in text.split('.') if s.strip()])
|
||||
num_statements = len([s for s in text.split(".") if s.strip()])
|
||||
|
||||
print(f"\nNumber of facts test:")
|
||||
print(f" Input statements: ~{num_statements}")
|
||||
|
||||
@@ -5,6 +5,7 @@ This ensures that when multiple facts are extracted from a long conversation,
|
||||
their relative order is preserved via time offsets, allowing retrieval to
|
||||
distinguish between things said earlier vs later.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from datetime import datetime, timezone
|
||||
from hindsight_api import MemoryEngine, RequestContext
|
||||
@@ -20,11 +21,9 @@ async def test_fact_ordering_within_conversation(memory, request_context):
|
||||
await memory.get_bank_profile(bank_id, request_context=request_context)
|
||||
|
||||
# Update disposition to match Marcus
|
||||
await memory.update_bank_disposition(bank_id, {
|
||||
"skepticism": 3,
|
||||
"literalism": 3,
|
||||
"empathy": 3
|
||||
}, request_context=request_context)
|
||||
await memory.update_bank_disposition(
|
||||
bank_id, {"skepticism": 3, "literalism": 3, "empathy": 3}, request_context=request_context
|
||||
)
|
||||
|
||||
# A conversation where Marcus changes his position
|
||||
conversation = """
|
||||
@@ -51,7 +50,7 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
|
||||
results = await memory.recall_async(
|
||||
bank_id=bank_id,
|
||||
query="Marcus prediction Rams",
|
||||
fact_type=['experience', 'world'],
|
||||
fact_type=["experience", "world"],
|
||||
budget=Budget.LOW,
|
||||
max_tokens=8192,
|
||||
request_context=request_context,
|
||||
@@ -59,37 +58,41 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
|
||||
|
||||
print(f"\n=== Retrieved {len(results.results)} facts ===")
|
||||
for i, result in enumerate(results.results):
|
||||
print(f"{i+1}. [{result.mentioned_at}] {result.text[:100]}")
|
||||
print(f"{i + 1}. [{result.mentioned_at}] {result.text[:100]}")
|
||||
|
||||
# Get all facts (Marcus's predictions/statements)
|
||||
agent_facts = results.results
|
||||
|
||||
print(f"\n=== Agent facts (Marcus's statements) ===")
|
||||
for i, fact in enumerate(agent_facts):
|
||||
print(f"{i+1}. [{fact.mentioned_at}] {fact.text}")
|
||||
print(f"{i + 1}. [{fact.mentioned_at}] {fact.text}")
|
||||
|
||||
# Check that agent facts have different timestamps
|
||||
if len(agent_facts) >= 2:
|
||||
# Parse timestamps
|
||||
timestamps = [datetime.fromisoformat(f.mentioned_at.replace('Z', '+00:00')) for f in agent_facts]
|
||||
timestamps = [datetime.fromisoformat(f.mentioned_at.replace("Z", "+00:00")) for f in agent_facts]
|
||||
|
||||
# Verify timestamps are different (have time offsets)
|
||||
unique_timestamps = set(timestamps)
|
||||
assert len(unique_timestamps) == len(timestamps), \
|
||||
assert len(unique_timestamps) == len(timestamps), (
|
||||
f"Expected unique timestamps for each fact, but got duplicates: {timestamps}"
|
||||
)
|
||||
|
||||
# Sort facts by timestamp for ordering check
|
||||
# Note: recall returns by relevance, not time order
|
||||
sorted_facts = sorted(agent_facts, key=lambda f: datetime.fromisoformat(f.mentioned_at.replace('Z', '+00:00')))
|
||||
sorted_timestamps = [datetime.fromisoformat(f.mentioned_at.replace('Z', '+00:00')) for f in sorted_facts]
|
||||
sorted_facts = sorted(agent_facts, key=lambda f: datetime.fromisoformat(f.mentioned_at.replace("Z", "+00:00")))
|
||||
sorted_timestamps = [datetime.fromisoformat(f.mentioned_at.replace("Z", "+00:00")) for f in sorted_facts]
|
||||
|
||||
# Verify sorted timestamps are in ascending order
|
||||
for i in range(len(sorted_timestamps) - 1):
|
||||
assert sorted_timestamps[i] < sorted_timestamps[i + 1], \
|
||||
f"Facts should have sequential timestamps. Fact {i} ({sorted_timestamps[i]}) >= Fact {i+1} ({sorted_timestamps[i+1]})"
|
||||
assert sorted_timestamps[i] < sorted_timestamps[i + 1], (
|
||||
f"Facts should have sequential timestamps. Fact {i} ({sorted_timestamps[i]}) >= Fact {i + 1} ({sorted_timestamps[i + 1]})"
|
||||
)
|
||||
|
||||
# Verify facts have distinct timestamps (ordering is preserved)
|
||||
time_diffs = [(sorted_timestamps[i+1] - sorted_timestamps[i]).total_seconds() for i in range(len(sorted_timestamps) - 1)]
|
||||
time_diffs = [
|
||||
(sorted_timestamps[i + 1] - sorted_timestamps[i]).total_seconds() for i in range(len(sorted_timestamps) - 1)
|
||||
]
|
||||
print(f"\n=== Time differences between facts: {time_diffs} seconds ===")
|
||||
|
||||
# Each fact should have a positive time difference (uniqueness already checked above)
|
||||
@@ -108,7 +111,7 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
|
||||
all_text = " ".join(agent_texts)
|
||||
|
||||
# Look for evidence of the predictions being captured (may be merged or separate)
|
||||
has_prediction_info = '27' in all_text or 'rams' in all_text or 'prediction' in all_text
|
||||
has_prediction_info = "27" in all_text or "rams" in all_text or "prediction" in all_text
|
||||
|
||||
assert has_prediction_info, "Facts should contain information about Marcus's predictions"
|
||||
print(f"\n✅ Facts capture prediction information")
|
||||
@@ -121,7 +124,6 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_documents_ordering(memory, request_context):
|
||||
|
||||
bank_id = "test_multi_doc_agent"
|
||||
|
||||
await memory.get_bank_profile(bank_id, request_context=request_context) # Auto-creates with defaults
|
||||
@@ -149,7 +151,7 @@ Alice: I reconsidered the team's experience level.
|
||||
bank_id=bank_id,
|
||||
contents=[
|
||||
{"content": conv1, "context": "project discussion 1", "event_date": time1},
|
||||
{"content": conv2, "context": "project discussion 2", "event_date": time2}
|
||||
{"content": conv2, "context": "project discussion 2", "event_date": time2},
|
||||
],
|
||||
request_context=request_context,
|
||||
)
|
||||
@@ -168,18 +170,21 @@ Alice: I reconsidered the team's experience level.
|
||||
agent_facts = results.results
|
||||
|
||||
for i, fact in enumerate(agent_facts):
|
||||
print(f"{i+1}. [{fact.mentioned_at}] {fact.text[:80]}")
|
||||
print(f"{i + 1}. [{fact.mentioned_at}] {fact.text[:80]}")
|
||||
|
||||
# Each conversation's facts should have different timestamps.
|
||||
# Filter out observations — they inherit their source fact's timestamp,
|
||||
# which can collapse the unique set. Also skip facts without timestamps.
|
||||
source_facts = [f for f in agent_facts if f.mentioned_at is not None and getattr(f, "fact_type", "") != "observation"]
|
||||
source_facts = [
|
||||
f for f in agent_facts if f.mentioned_at is not None and getattr(f, "fact_type", "") != "observation"
|
||||
]
|
||||
if len(source_facts) >= 2:
|
||||
timestamps = [datetime.fromisoformat(f.mentioned_at.replace('Z', '+00:00')) for f in source_facts]
|
||||
timestamps = [datetime.fromisoformat(f.mentioned_at.replace("Z", "+00:00")) for f in source_facts]
|
||||
unique_timestamps = set(timestamps)
|
||||
|
||||
assert len(unique_timestamps) >= 2, \
|
||||
assert len(unique_timestamps) >= 2, (
|
||||
f"Expected multiple unique timestamps across conversations, got: {len(unique_timestamps)}"
|
||||
)
|
||||
|
||||
print(f"\n✅ Facts from {len(source_facts)} statements have {len(unique_timestamps)} unique timestamps")
|
||||
|
||||
|
||||
@@ -216,7 +216,9 @@ async def test_file_retain_validation_errors(memory_no_llm_verify):
|
||||
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||
# Create bank
|
||||
bank_response = await client.put("/v1/default/banks/test-validation-bank", json={"name": "Test Validation Bank"})
|
||||
bank_response = await client.put(
|
||||
"/v1/default/banks/test-validation-bank", json={"name": "Test Validation Bank"}
|
||||
)
|
||||
assert bank_response.status_code in (200, 201)
|
||||
|
||||
# Test: metadata count mismatch
|
||||
|
||||
@@ -98,9 +98,7 @@ def seaweedfs_container():
|
||||
DockerContainer(image="chrislusf/seaweedfs:latest")
|
||||
.with_exposed_ports(SEAWEEDFS_S3_PORT)
|
||||
.with_volume_mapping(s3_config_file.name, "/etc/seaweedfs/s3.json", "ro")
|
||||
.with_command(
|
||||
f"server -s3 -s3.port={SEAWEEDFS_S3_PORT} -s3.config=/etc/seaweedfs/s3.json -ip.bind=0.0.0.0"
|
||||
)
|
||||
.with_command(f"server -s3 -s3.port={SEAWEEDFS_S3_PORT} -s3.config=/etc/seaweedfs/s3.json -ip.bind=0.0.0.0")
|
||||
)
|
||||
|
||||
container.start()
|
||||
|
||||
@@ -309,9 +309,7 @@ async def test_api_errors_surface_the_response_body():
|
||||
llm = _make_fireworks(http_client=client)
|
||||
|
||||
with pytest.raises(httpx.HTTPStatusError, match="invalid field 'userUploaded'"):
|
||||
await llm.submit_batch(
|
||||
[{"custom_id": "c0", "method": "POST", "url": "/v1/chat/completions", "body": {}}]
|
||||
)
|
||||
await llm.submit_batch([{"custom_id": "c0", "method": "POST", "url": "/v1/chat/completions", "body": {}}])
|
||||
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@@ -33,9 +33,7 @@ def _make_client(create_side_effect=None):
|
||||
elif callable(create_side_effect):
|
||||
create_mock.side_effect = create_side_effect
|
||||
else:
|
||||
create_mock.return_value = SimpleNamespace(
|
||||
name="cachedContents/test-cache-name-001"
|
||||
)
|
||||
create_mock.return_value = SimpleNamespace(name="cachedContents/test-cache-name-001")
|
||||
|
||||
client = MagicMock()
|
||||
client.aio = MagicMock()
|
||||
@@ -127,18 +125,12 @@ async def test_first_call_creates_subsequent_reuses():
|
||||
@pytest.mark.asyncio
|
||||
async def test_different_prefixes_create_separately():
|
||||
client, create_mock = _make_client(
|
||||
create_side_effect=lambda *a, **kw: SimpleNamespace(
|
||||
name=f"cachedContents/created-{create_mock.call_count}"
|
||||
)
|
||||
create_side_effect=lambda *a, **kw: SimpleNamespace(name=f"cachedContents/created-{create_mock.call_count}")
|
||||
)
|
||||
mgr = GeminiCacheManager(client)
|
||||
|
||||
name_a = await mgr.get_or_create(
|
||||
model="m", system_instruction="A", response_schema=None
|
||||
)
|
||||
name_b = await mgr.get_or_create(
|
||||
model="m", system_instruction="B", response_schema=None
|
||||
)
|
||||
name_a = await mgr.get_or_create(model="m", system_instruction="A", response_schema=None)
|
||||
name_b = await mgr.get_or_create(model="m", system_instruction="B", response_schema=None)
|
||||
assert name_a != name_b
|
||||
assert create_mock.call_count == 2
|
||||
|
||||
@@ -155,9 +147,7 @@ async def test_minimum_token_count_error_returns_none():
|
||||
client, _ = _make_client(create_side_effect=err)
|
||||
mgr = GeminiCacheManager(client)
|
||||
|
||||
result = await mgr.get_or_create(
|
||||
model="m", system_instruction="tiny", response_schema=None
|
||||
)
|
||||
result = await mgr.get_or_create(model="m", system_instruction="tiny", response_schema=None)
|
||||
assert result is None
|
||||
|
||||
|
||||
@@ -169,9 +159,7 @@ async def test_other_sdk_errors_also_return_none():
|
||||
client, _ = _make_client(create_side_effect=err)
|
||||
mgr = GeminiCacheManager(client)
|
||||
|
||||
result = await mgr.get_or_create(
|
||||
model="m", system_instruction="ok-sized prefix", response_schema=None
|
||||
)
|
||||
result = await mgr.get_or_create(model="m", system_instruction="ok-sized prefix", response_schema=None)
|
||||
assert result is None
|
||||
|
||||
|
||||
@@ -194,12 +182,8 @@ async def test_failed_create_does_not_poison_cache():
|
||||
|
||||
mgr = GeminiCacheManager(client)
|
||||
|
||||
first = await mgr.get_or_create(
|
||||
model="m", system_instruction="prefix", response_schema=None
|
||||
)
|
||||
second = await mgr.get_or_create(
|
||||
model="m", system_instruction="prefix", response_schema=None
|
||||
)
|
||||
first = await mgr.get_or_create(model="m", system_instruction="prefix", response_schema=None)
|
||||
second = await mgr.get_or_create(model="m", system_instruction="prefix", response_schema=None)
|
||||
|
||||
assert first is None
|
||||
assert second == "cachedContents/recovered"
|
||||
@@ -214,9 +198,7 @@ async def test_refreshes_after_ttl_margin(monkeypatch):
|
||||
"""An entry created at t=0 with ttl=10 and margin=2 should be
|
||||
treated as stale at t>=8 and trigger a recreate."""
|
||||
client, create_mock = _make_client(
|
||||
create_side_effect=lambda *a, **kw: SimpleNamespace(
|
||||
name=f"cachedContents/v{create_mock.call_count}"
|
||||
)
|
||||
create_side_effect=lambda *a, **kw: SimpleNamespace(name=f"cachedContents/v{create_mock.call_count}")
|
||||
)
|
||||
mgr = GeminiCacheManager(client, ttl_seconds=10, refresh_margin_seconds=2)
|
||||
|
||||
@@ -226,24 +208,18 @@ async def test_refreshes_after_ttl_margin(monkeypatch):
|
||||
lambda: fake_now["t"],
|
||||
)
|
||||
|
||||
first = await mgr.get_or_create(
|
||||
model="m", system_instruction="p", response_schema=None
|
||||
)
|
||||
first = await mgr.get_or_create(model="m", system_instruction="p", response_schema=None)
|
||||
assert first == "cachedContents/v1"
|
||||
|
||||
# Advance to just before the refresh boundary — should reuse.
|
||||
fake_now["t"] = 1000.0 + 7.0
|
||||
again = await mgr.get_or_create(
|
||||
model="m", system_instruction="p", response_schema=None
|
||||
)
|
||||
again = await mgr.get_or_create(model="m", system_instruction="p", response_schema=None)
|
||||
assert again == "cachedContents/v1"
|
||||
assert create_mock.call_count == 1
|
||||
|
||||
# Advance past the refresh boundary — should recreate.
|
||||
fake_now["t"] = 1000.0 + 9.0
|
||||
refreshed = await mgr.get_or_create(
|
||||
model="m", system_instruction="p", response_schema=None
|
||||
)
|
||||
refreshed = await mgr.get_or_create(model="m", system_instruction="p", response_schema=None)
|
||||
assert refreshed == "cachedContents/v2"
|
||||
assert create_mock.call_count == 2
|
||||
|
||||
@@ -295,9 +271,7 @@ async def test_gemini_llm_uses_cache_when_enabled(monkeypatch):
|
||||
# Replace the SDK-shaped client with a fake whose caches.create returns
|
||||
# a predictable name. The lazy import inside get_or_create_cached_prefix
|
||||
# picks up the patched module-level GeminiCacheManager naturally.
|
||||
fake_create = AsyncMock(
|
||||
return_value=SimpleNamespace(name="cachedContents/from-llm-test")
|
||||
)
|
||||
fake_create = AsyncMock(return_value=SimpleNamespace(name="cachedContents/from-llm-test"))
|
||||
llm._client = MagicMock()
|
||||
llm._client.aio = MagicMock()
|
||||
llm._client.aio.caches = MagicMock()
|
||||
@@ -333,7 +307,9 @@ async def test_call_falls_back_to_uncached_when_cache_400s():
|
||||
from hindsight_api.engine.providers.gemini_cache import GeminiCacheManager, _CacheEntry
|
||||
from hindsight_api.engine.providers.gemini_llm import GeminiLLM
|
||||
|
||||
llm = GeminiLLM(provider="gemini", api_key="not-real-key", base_url="", model="gemini-test", prompt_cache_enabled=True)
|
||||
llm = GeminiLLM(
|
||||
provider="gemini", api_key="not-real-key", base_url="", model="gemini-test", prompt_cache_enabled=True
|
||||
)
|
||||
|
||||
# Seed a cache manager entry that maps to the (now invalid) cache name.
|
||||
mgr = GeminiCacheManager(client=MagicMock())
|
||||
@@ -413,9 +389,7 @@ def test_fingerprint_changes_with_tools():
|
||||
"""Two prefixes that differ ONLY in tools must hash differently —
|
||||
otherwise a loop that adds a tool would silently reuse a stale
|
||||
cache that doesn't know about it."""
|
||||
tools_a = [
|
||||
{"type": "function", "function": {"name": "search", "description": "search", "parameters": {}}}
|
||||
]
|
||||
tools_a = [{"type": "function", "function": {"name": "search", "description": "search", "parameters": {}}}]
|
||||
tools_b = [
|
||||
{"type": "function", "function": {"name": "search", "description": "search", "parameters": {}}},
|
||||
{"type": "function", "function": {"name": "fetch", "description": "fetch", "parameters": {}}},
|
||||
@@ -455,7 +429,10 @@ async def test_get_or_create_passes_tools_to_create():
|
||||
|
||||
mgr = GeminiCacheManager(client)
|
||||
tools = [
|
||||
{"type": "function", "function": {"name": "search", "description": "do a search", "parameters": {"type": "object"}}}
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "search", "description": "do a search", "parameters": {"type": "object"}},
|
||||
}
|
||||
]
|
||||
name = await mgr.get_or_create(
|
||||
model="gemini-3.1-flash-lite",
|
||||
|
||||
@@ -34,9 +34,7 @@ from hindsight_api.engine.consolidation.consolidator import run_consolidation_jo
|
||||
from hindsight_api.engine.llm_trace import LLMRequestEntry
|
||||
from hindsight_api.engine.llm_wrapper import LLMConfig
|
||||
|
||||
_GEMINI_API_KEY = (
|
||||
os.getenv("HINDSIGHT_GEMINI_API_KEY") or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")
|
||||
)
|
||||
_GEMINI_API_KEY = os.getenv("HINDSIGHT_GEMINI_API_KEY") or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")
|
||||
_RUN = os.getenv("HINDSIGHT_RUN_GEMINI_EVALS") == "1" and bool(_GEMINI_API_KEY)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
|
||||
@@ -133,13 +133,17 @@ async def test_call_applies_safety_settings():
|
||||
assert hasattr(config_arg, "safety_settings"), "Config should have safety_settings"
|
||||
assert config_arg.safety_settings is not None
|
||||
|
||||
categories = [s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings]
|
||||
categories = [
|
||||
s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings
|
||||
]
|
||||
assert "HARM_CATEGORY_HARASSMENT" in categories
|
||||
assert "HARM_CATEGORY_HATE_SPEECH" in categories
|
||||
assert "HARM_CATEGORY_SEXUALLY_EXPLICIT" in categories
|
||||
assert "HARM_CATEGORY_DANGEROUS_CONTENT" in categories
|
||||
|
||||
thresholds = [s.threshold.value if hasattr(s.threshold, "value") else str(s.threshold) for s in config_arg.safety_settings]
|
||||
thresholds = [
|
||||
s.threshold.value if hasattr(s.threshold, "value") else str(s.threshold) for s in config_arg.safety_settings
|
||||
]
|
||||
assert all(t == "BLOCK_NONE" for t in thresholds)
|
||||
|
||||
|
||||
@@ -212,7 +216,9 @@ async def test_call_with_tools_applies_safety_settings():
|
||||
|
||||
assert config_arg is not None
|
||||
assert config_arg.safety_settings is not None
|
||||
categories = [s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings]
|
||||
categories = [
|
||||
s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings
|
||||
]
|
||||
assert "HARM_CATEGORY_HARASSMENT" in categories
|
||||
|
||||
|
||||
@@ -266,7 +272,9 @@ async def test_with_config_overrides_instance_settings():
|
||||
|
||||
config_arg = provider._provider_impl._client.aio.models.generate_content.call_args.kwargs.get("config")
|
||||
assert config_arg is not None
|
||||
categories = [s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings]
|
||||
categories = [
|
||||
s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings
|
||||
]
|
||||
# Should use override_settings (HATE_SPEECH), not instance_settings (HARASSMENT)
|
||||
assert "HARM_CATEGORY_HATE_SPEECH" in categories
|
||||
assert "HARM_CATEGORY_HARASSMENT" not in categories
|
||||
@@ -285,7 +293,9 @@ async def test_with_config_none_falls_back_to_instance():
|
||||
|
||||
config_arg = provider._provider_impl._client.aio.models.generate_content.call_args.kwargs.get("config")
|
||||
assert config_arg is not None
|
||||
categories = [s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings]
|
||||
categories = [
|
||||
s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings
|
||||
]
|
||||
assert "HARM_CATEGORY_HARASSMENT" in categories
|
||||
|
||||
|
||||
|
||||
@@ -97,19 +97,23 @@ class TestGoogleCrossEncoder:
|
||||
async def test_predict_single_query(self):
|
||||
"""Test prediction with a single query and multiple documents."""
|
||||
mock_creds = _make_mock_credentials()
|
||||
mock_client = _make_mock_httpx_client([
|
||||
_make_rank_response([("1", 0.95), ("0", 0.30)]),
|
||||
])
|
||||
mock_client = _make_mock_httpx_client(
|
||||
[
|
||||
_make_rank_response([("1", 0.95), ("0", 0.30)]),
|
||||
]
|
||||
)
|
||||
|
||||
encoder = GoogleCrossEncoder(project_id="test-project")
|
||||
with patch("google.auth.default", return_value=(mock_creds, "p")):
|
||||
await encoder.initialize()
|
||||
encoder._client = mock_client
|
||||
|
||||
scores = await encoder.predict([
|
||||
("What is AI?", "AI is artificial intelligence"),
|
||||
("What is AI?", "The sky is blue"),
|
||||
])
|
||||
scores = await encoder.predict(
|
||||
[
|
||||
("What is AI?", "AI is artificial intelligence"),
|
||||
("What is AI?", "The sky is blue"),
|
||||
]
|
||||
)
|
||||
|
||||
assert len(scores) == 2
|
||||
assert scores[0] == 0.30 # id="0" -> index 0
|
||||
@@ -119,21 +123,25 @@ class TestGoogleCrossEncoder:
|
||||
async def test_predict_multiple_queries(self):
|
||||
"""Test prediction with multiple distinct queries."""
|
||||
mock_creds = _make_mock_credentials()
|
||||
mock_client = _make_mock_httpx_client([
|
||||
_make_rank_response([("0", 0.9), ("1", 0.1)]),
|
||||
_make_rank_response([("0", 0.8)]),
|
||||
])
|
||||
mock_client = _make_mock_httpx_client(
|
||||
[
|
||||
_make_rank_response([("0", 0.9), ("1", 0.1)]),
|
||||
_make_rank_response([("0", 0.8)]),
|
||||
]
|
||||
)
|
||||
|
||||
encoder = GoogleCrossEncoder(project_id="test-project")
|
||||
with patch("google.auth.default", return_value=(mock_creds, "p")):
|
||||
await encoder.initialize()
|
||||
encoder._client = mock_client
|
||||
|
||||
scores = await encoder.predict([
|
||||
("Query A", "Doc A1"),
|
||||
("Query A", "Doc A2"),
|
||||
("Query B", "Doc B1"),
|
||||
])
|
||||
scores = await encoder.predict(
|
||||
[
|
||||
("Query A", "Doc A1"),
|
||||
("Query A", "Doc A2"),
|
||||
("Query B", "Doc B1"),
|
||||
]
|
||||
)
|
||||
|
||||
assert len(scores) == 3
|
||||
assert scores[0] == 0.9
|
||||
@@ -161,10 +169,12 @@ class TestGoogleCrossEncoder:
|
||||
async def test_predict_batching(self):
|
||||
"""Test that >200 records are split into batches."""
|
||||
mock_creds = _make_mock_credentials()
|
||||
mock_client = _make_mock_httpx_client([
|
||||
_make_rank_response([(str(i), 0.5) for i in range(200)]),
|
||||
_make_rank_response([(str(i), 0.3) for i in range(50)]),
|
||||
])
|
||||
mock_client = _make_mock_httpx_client(
|
||||
[
|
||||
_make_rank_response([(str(i), 0.5) for i in range(200)]),
|
||||
_make_rank_response([(str(i), 0.3) for i in range(50)]),
|
||||
]
|
||||
)
|
||||
|
||||
encoder = GoogleCrossEncoder(project_id="test-project")
|
||||
with patch("google.auth.default", return_value=(mock_creds, "p")):
|
||||
@@ -181,9 +191,11 @@ class TestGoogleCrossEncoder:
|
||||
"""Test that Authorization header is sent with requests."""
|
||||
mock_creds = _make_mock_credentials()
|
||||
mock_creds.token = "test-bearer-token"
|
||||
mock_client = _make_mock_httpx_client([
|
||||
_make_rank_response([("0", 0.9)]),
|
||||
])
|
||||
mock_client = _make_mock_httpx_client(
|
||||
[
|
||||
_make_rank_response([("0", 0.9)]),
|
||||
]
|
||||
)
|
||||
|
||||
encoder = GoogleCrossEncoder(project_id="test-project")
|
||||
with patch("google.auth.default", return_value=(mock_creds, "p")):
|
||||
|
||||
@@ -167,9 +167,7 @@ class TestEnqueueRelinkVictims:
|
||||
assert await _queue_unit_ids(conn, bank_id) == [str(survivor)]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_excludes_deleted_units_themselves(
|
||||
self, memory: MemoryEngine, request_context: RequestContext
|
||||
):
|
||||
async def test_excludes_deleted_units_themselves(self, memory: MemoryEngine, request_context: RequestContext):
|
||||
"""A unit being deleted that linked TO another deleted unit must not enqueue itself."""
|
||||
bank_id = f"test-gm-self-{uuid.uuid4().hex[:8]}"
|
||||
await _ensure_bank(memory, bank_id, request_context)
|
||||
@@ -270,9 +268,7 @@ class TestDeleteDocumentEnqueue:
|
||||
|
||||
class TestRelinkPass:
|
||||
@pytest.mark.asyncio
|
||||
async def test_drains_empty_queue_cleanly(
|
||||
self, memory: MemoryEngine, request_context: RequestContext
|
||||
):
|
||||
async def test_drains_empty_queue_cleanly(self, memory: MemoryEngine, request_context: RequestContext):
|
||||
bank_id = f"test-gm-empty-{uuid.uuid4().hex[:8]}"
|
||||
await _ensure_bank(memory, bank_id, request_context)
|
||||
|
||||
@@ -285,9 +281,7 @@ class TestRelinkPass:
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_skips_missing_unit_silently(
|
||||
self, memory: MemoryEngine, request_context: RequestContext
|
||||
):
|
||||
async def test_skips_missing_unit_silently(self, memory: MemoryEngine, request_context: RequestContext):
|
||||
"""Unit deleted between enqueue and drain: worker dequeues and no-ops."""
|
||||
bank_id = f"test-gm-miss-{uuid.uuid4().hex[:8]}"
|
||||
await _ensure_bank(memory, bank_id, request_context)
|
||||
@@ -309,9 +303,7 @@ class TestRelinkPass:
|
||||
assert await _queue_unit_ids(conn, bank_id) == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tops_up_temporal_when_under_cap(
|
||||
self, memory: MemoryEngine, request_context: RequestContext
|
||||
):
|
||||
async def test_tops_up_temporal_when_under_cap(self, memory: MemoryEngine, request_context: RequestContext):
|
||||
"""A victim under the temporal cap gets new outgoing links to neighbours
|
||||
that were never linked at retain time."""
|
||||
bank_id = f"test-gm-topup-{uuid.uuid4().hex[:8]}"
|
||||
@@ -365,9 +357,7 @@ class TestRelinkPass:
|
||||
assert await _queue_unit_ids(conn, bank_id) == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_topup_when_victim_at_cap(
|
||||
self, memory: MemoryEngine, request_context: RequestContext
|
||||
):
|
||||
async def test_no_topup_when_victim_at_cap(self, memory: MemoryEngine, request_context: RequestContext):
|
||||
"""If the victim already has cap links, probing is skipped."""
|
||||
bank_id = f"test-gm-atcap-{uuid.uuid4().hex[:8]}"
|
||||
await _ensure_bank(memory, bank_id, request_context)
|
||||
@@ -414,9 +404,7 @@ class TestRelinkPass:
|
||||
|
||||
class TestOrphanEntityPrune:
|
||||
@pytest.mark.asyncio
|
||||
async def test_prunes_entities_with_no_unit_references(
|
||||
self, memory: MemoryEngine, request_context: RequestContext
|
||||
):
|
||||
async def test_prunes_entities_with_no_unit_references(self, memory: MemoryEngine, request_context: RequestContext):
|
||||
"""An entity with zero unit_entities rows is an orphan and should be
|
||||
deleted by the sweep."""
|
||||
bank_id = f"test-gm-orphan-{uuid.uuid4().hex[:8]}"
|
||||
@@ -435,9 +423,7 @@ class TestOrphanEntityPrune:
|
||||
assert result["orphan_entities_pruned"] == 2
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
survivors = await conn.fetch(
|
||||
"SELECT id FROM entities WHERE bank_id = $1 ORDER BY id", bank_id
|
||||
)
|
||||
survivors = await conn.fetch("SELECT id FROM entities WHERE bank_id = $1 ORDER BY id", bank_id)
|
||||
survivor_ids = {str(r["id"]) for r in survivors}
|
||||
assert survivor_ids == {str(referenced)}
|
||||
# Confirm orphans are gone.
|
||||
@@ -445,9 +431,7 @@ class TestOrphanEntityPrune:
|
||||
assert orphan not in survivor_ids
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_touch_other_banks(
|
||||
self, memory: MemoryEngine, request_context: RequestContext
|
||||
):
|
||||
async def test_does_not_touch_other_banks(self, memory: MemoryEngine, request_context: RequestContext):
|
||||
"""The sweep is scoped by bank — orphan entities in OTHER banks
|
||||
must not be touched."""
|
||||
bank_a = f"test-gm-scopea-{uuid.uuid4().hex[:8]}"
|
||||
@@ -478,9 +462,7 @@ class TestOrphanEntityPrune:
|
||||
|
||||
class TestStaleCooccurrencePrune:
|
||||
@pytest.mark.asyncio
|
||||
async def test_prunes_cooccurrence_with_no_shared_unit(
|
||||
self, memory: MemoryEngine, request_context: RequestContext
|
||||
):
|
||||
async def test_prunes_cooccurrence_with_no_shared_unit(self, memory: MemoryEngine, request_context: RequestContext):
|
||||
"""Both entities still exist but no unit references both of them — the
|
||||
cooccurrence row is stale and should be pruned."""
|
||||
bank_id = f"test-gm-cocc-{uuid.uuid4().hex[:8]}"
|
||||
@@ -515,9 +497,7 @@ class TestStaleCooccurrencePrune:
|
||||
assert remaining == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_keeps_cooccurrence_with_shared_unit(
|
||||
self, memory: MemoryEngine, request_context: RequestContext
|
||||
):
|
||||
async def test_keeps_cooccurrence_with_shared_unit(self, memory: MemoryEngine, request_context: RequestContext):
|
||||
"""If at least one unit still references both entities, the cooccurrence
|
||||
row stays."""
|
||||
bank_id = f"test-gm-keep-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
@@ -7,6 +7,7 @@ Covers:
|
||||
- Per-bank vector indexes dropped on bank deletion
|
||||
- retrieve_semantic_bm25_combined groups results correctly by fact_type and source
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
@@ -152,10 +153,7 @@ async def test_retrieve_semantic_bm25_grouped_by_fact_type(memory, request_conte
|
||||
try:
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content=(
|
||||
"Alice is a software engineer at TechCorp. "
|
||||
"She visited Paris in 2023 for a conference."
|
||||
),
|
||||
content=("Alice is a software engineer at TechCorp. She visited Paris in 2023 for a conference."),
|
||||
context="background",
|
||||
event_date=datetime(2023, 6, 1, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
|
||||
@@ -87,9 +87,7 @@ async def test_unique_violation_marks_failed_without_retry(memory):
|
||||
try:
|
||||
await memory.execute_task(task_dict)
|
||||
except RetryTaskAt as exc:
|
||||
pytest.fail(
|
||||
f"IntegrityConstraintViolationError must not be retried, but execute_task raised {exc!r}"
|
||||
)
|
||||
pytest.fail(f"IntegrityConstraintViolationError must not be retried, but execute_task raised {exc!r}")
|
||||
|
||||
# The operation must be marked 'failed' (not left pending / retrying).
|
||||
row = await pool.fetchrow(
|
||||
@@ -97,9 +95,7 @@ async def test_unique_violation_marks_failed_without_retry(memory):
|
||||
operation_id,
|
||||
)
|
||||
assert row is not None, "Operation row disappeared"
|
||||
assert row["status"] == "failed", (
|
||||
f"Expected status='failed' after integrity violation, got {row['status']!r}"
|
||||
)
|
||||
assert row["status"] == "failed", f"Expected status='failed' after integrity violation, got {row['status']!r}"
|
||||
assert row["error_message"] is not None
|
||||
assert "pk_chunks" in row["error_message"]
|
||||
|
||||
@@ -123,7 +119,7 @@ async def test_foreign_key_violation_also_not_retried(memory):
|
||||
await _create_pending_operation(pool, bank_id, operation_id)
|
||||
|
||||
fk_violation = asyncpg.exceptions.ForeignKeyViolationError(
|
||||
"insert or update on table \"memory_units\" violates foreign key constraint \"fk_bank\""
|
||||
'insert or update on table "memory_units" violates foreign key constraint "fk_bank"'
|
||||
)
|
||||
|
||||
task_dict = {
|
||||
@@ -137,9 +133,7 @@ async def test_foreign_key_violation_also_not_retried(memory):
|
||||
try:
|
||||
await memory.execute_task(task_dict)
|
||||
except RetryTaskAt as exc:
|
||||
pytest.fail(
|
||||
f"ForeignKeyViolationError must not be retried, but execute_task raised {exc!r}"
|
||||
)
|
||||
pytest.fail(f"ForeignKeyViolationError must not be retried, but execute_task raised {exc!r}")
|
||||
|
||||
row = await pool.fetchrow(
|
||||
"SELECT status FROM async_operations WHERE operation_id = $1",
|
||||
|
||||
@@ -68,5 +68,3 @@ async def test_iris_parser_converts_pdf(iris_parser: IrisParser):
|
||||
async def test_iris_parser_name(iris_parser: IrisParser):
|
||||
"""IrisParser.name() should return 'iris'."""
|
||||
assert iris_parser.name() == "iris"
|
||||
|
||||
|
||||
|
||||
@@ -48,9 +48,7 @@ def _make_replacement_body() -> str:
|
||||
than one sub-batch.
|
||||
"""
|
||||
lines = [
|
||||
f"[role: user] turn {i}: alpha bravo charlie delta echo "
|
||||
f"foxtrot golf hotel india juliet"
|
||||
for i in range(20)
|
||||
f"[role: user] turn {i}: alpha bravo charlie delta echo foxtrot golf hotel india juliet" for i in range(20)
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -79,9 +77,7 @@ async def test_large_same_id_replacement_preserves_full_body(memory, request_con
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
doc_initial = await memory.get_document(
|
||||
document_id, bank_id, request_context=request_context
|
||||
)
|
||||
doc_initial = await memory.get_document(document_id, bank_id, request_context=request_context)
|
||||
assert doc_initial is not None
|
||||
assert doc_initial["original_text"] == initial_body
|
||||
|
||||
@@ -95,9 +91,7 @@ async def test_large_same_id_replacement_preserves_full_body(memory, request_con
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
doc_replaced = await memory.get_document(
|
||||
document_id, bank_id, request_context=request_context
|
||||
)
|
||||
doc_replaced = await memory.get_document(document_id, bank_id, request_context=request_context)
|
||||
assert doc_replaced is not None
|
||||
|
||||
stored = doc_replaced["original_text"]
|
||||
@@ -105,10 +99,7 @@ async def test_large_same_id_replacement_preserves_full_body(memory, request_con
|
||||
f"stored body length {len(stored)} != submitted length "
|
||||
f"{len(replacement_body)} — partial replacement persisted"
|
||||
)
|
||||
assert stored == replacement_body, (
|
||||
"stored original_text does not exactly match the submitted "
|
||||
"replacement body"
|
||||
)
|
||||
assert stored == replacement_body, "stored original_text does not exactly match the submitted replacement body"
|
||||
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
@@ -143,9 +134,7 @@ async def test_repeated_large_same_id_replacement_is_idempotent(memory, request_
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
doc = await memory.get_document(
|
||||
document_id, bank_id, request_context=request_context
|
||||
)
|
||||
doc = await memory.get_document(document_id, bank_id, request_context=request_context)
|
||||
assert doc is not None, f"attempt {attempt}: document missing after retain"
|
||||
assert doc["original_text"] == replacement_body, (
|
||||
f"attempt {attempt}: stored body diverged from submitted body "
|
||||
|
||||
@@ -133,7 +133,9 @@ async def test_link_expansion_observation_graph_retrieval(memory_real_llm, reque
|
||||
|
||||
assert obs_result is not None and obs_result.results is not None, "Should have observations after consolidation"
|
||||
# We should have observations from consolidation
|
||||
assert len(obs_result.results) >= 1, f"Should have at least 1 observation about Python, got {len(obs_result.results)}"
|
||||
assert len(obs_result.results) >= 1, (
|
||||
f"Should have at least 1 observation about Python, got {len(obs_result.results)}"
|
||||
)
|
||||
|
||||
# Now test graph retrieval specifically
|
||||
# Query for Alice - should find Bob via shared "Python" entity
|
||||
@@ -175,9 +177,7 @@ async def test_link_expansion_observation_graph_retrieval(memory_real_llm, reque
|
||||
|
||||
assert world_result.trace is not None, "Should have trace data for world facts"
|
||||
world_retrieval_results = world_result.trace.get("retrieval_results", [])
|
||||
world_graph_results = [
|
||||
r for r in world_retrieval_results if r.get("method_name") == "graph"
|
||||
]
|
||||
world_graph_results = [r for r in world_retrieval_results if r.get("method_name") == "graph"]
|
||||
|
||||
if world_graph_results:
|
||||
world_graph_result = [r for r in world_graph_results if r.get("fact_type") == "world"][0]
|
||||
@@ -192,7 +192,9 @@ async def test_link_expansion_observation_graph_retrieval(memory_real_llm, reque
|
||||
print(" Found Bob's world fact via shared 'Python' entity!")
|
||||
|
||||
print("\n✓ Link expansion observation test passed!")
|
||||
print(" Entity traversal path verified (observations -> sources -> entities -> connected sources -> observations)")
|
||||
print(
|
||||
" Entity traversal path verified (observations -> sources -> entities -> connected sources -> observations)"
|
||||
)
|
||||
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
@@ -259,15 +261,11 @@ async def test_link_expansion_world_fact_graph_retrieval(memory, request_context
|
||||
# Verify graph retrieval ran (it may or may not find new results depending
|
||||
# on whether semantic search already found everything)
|
||||
retrieval_results = result.trace.get("retrieval_results", [])
|
||||
graph_results = [
|
||||
r for r in retrieval_results if r.get("method_name") == "graph"
|
||||
]
|
||||
graph_results = [r for r in retrieval_results if r.get("method_name") == "graph"]
|
||||
assert len(graph_results) > 0, "Should have graph retrieval results in trace"
|
||||
|
||||
# The important thing is that recall works and returns relevant results
|
||||
assert result.results is not None and len(result.results) > 0, (
|
||||
"Should return results for 'Alice' query"
|
||||
)
|
||||
assert result.results is not None and len(result.results) > 0, "Should return results for 'Alice' query"
|
||||
|
||||
# Alice's result should be at or near the top
|
||||
result_texts = [r.text for r in result.results]
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for link_utils datetime handling, temporal link computation, and semantic link splitting."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from datetime import datetime, timezone, timedelta
|
||||
@@ -374,6 +375,7 @@ class TestComputeSemanticLinksWithinBatch:
|
||||
links = compute_semantic_links_within_batch(unit_ids, embs, top_k=3, threshold=0.5)
|
||||
# Each unit should have at most 3 outgoing links
|
||||
from collections import Counter
|
||||
|
||||
from_counts = Counter(lnk[0] for lnk in links)
|
||||
for count in from_counts.values():
|
||||
assert count <= 3
|
||||
@@ -532,8 +534,6 @@ class TestComputeSemanticLinksAnnPgBouncerSafety:
|
||||
tuning_statements = [s for s in executed_sql if guc in s]
|
||||
assert tuning_statements, f"{guc} must be tuned for retain ANN under ext={ext}"
|
||||
for stmt in tuning_statements:
|
||||
assert stmt.strip().startswith("SET LOCAL"), (
|
||||
f"{guc} must use SET LOCAL, got: {stmt}"
|
||||
)
|
||||
assert stmt.strip().startswith("SET LOCAL"), f"{guc} must use SET LOCAL, got: {stmt}"
|
||||
# And there must not be a RESET — SET LOCAL handles it at commit.
|
||||
assert not any(f"RESET {guc}" in s for s in executed_sql)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Tests for list_documents pagination and tags filtering.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
@@ -28,25 +29,19 @@ async def test_list_documents_offset_pagination(memory, request_context):
|
||||
await _retain_doc(memory, bank_id, f"doc-{i:02d}", [], request_context)
|
||||
|
||||
# All documents, ordered by created_at DESC → doc-03, doc-02, doc-01, doc-00
|
||||
all_docs = await memory.list_documents(
|
||||
bank_id=bank_id, limit=10, offset=0, request_context=request_context
|
||||
)
|
||||
all_docs = await memory.list_documents(bank_id=bank_id, limit=10, offset=0, request_context=request_context)
|
||||
assert all_docs["total"] == 4
|
||||
assert len(all_docs["items"]) == 4
|
||||
all_ids = [d["id"] for d in all_docs["items"]]
|
||||
|
||||
# offset=2 should skip the first two and return the remaining two
|
||||
page2 = await memory.list_documents(
|
||||
bank_id=bank_id, limit=10, offset=2, request_context=request_context
|
||||
)
|
||||
page2 = await memory.list_documents(bank_id=bank_id, limit=10, offset=2, request_context=request_context)
|
||||
assert page2["total"] == 4 # total is always the full count
|
||||
assert len(page2["items"]) == 2
|
||||
assert [d["id"] for d in page2["items"]] == all_ids[2:]
|
||||
|
||||
# offset beyond total returns empty items but correct total
|
||||
beyond = await memory.list_documents(
|
||||
bank_id=bank_id, limit=10, offset=10, request_context=request_context
|
||||
)
|
||||
beyond = await memory.list_documents(bank_id=bank_id, limit=10, offset=10, request_context=request_context)
|
||||
assert beyond["total"] == 4
|
||||
assert beyond["items"] == []
|
||||
|
||||
|
||||
@@ -57,7 +57,10 @@ class TestLiteLLMSDKEmbeddings:
|
||||
|
||||
async def test_initialization_success(self, mock_litellm):
|
||||
"""Test successful initialization."""
|
||||
with patch("builtins.__import__", side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args)):
|
||||
with patch(
|
||||
"builtins.__import__",
|
||||
side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args),
|
||||
):
|
||||
emb = LiteLLMSDKEmbeddings(
|
||||
api_key="test_key",
|
||||
model="cohere/embed-english-v3.0",
|
||||
@@ -84,7 +87,10 @@ class TestLiteLLMSDKEmbeddings:
|
||||
|
||||
async def test_initialization_without_api_key(self, mock_litellm):
|
||||
"""Test initialization without api_key (e.g. AWS Bedrock with IAM auth)."""
|
||||
with patch("builtins.__import__", side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args)):
|
||||
with patch(
|
||||
"builtins.__import__",
|
||||
side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args),
|
||||
):
|
||||
emb = LiteLLMSDKEmbeddings(
|
||||
model="bedrock/amazon.titan-embed-text-v2:0",
|
||||
batch_size=100,
|
||||
@@ -119,6 +125,7 @@ class TestLiteLLMSDKEmbeddings:
|
||||
|
||||
async def test_initialization_missing_package(self):
|
||||
"""Test initialization fails gracefully when litellm is not installed."""
|
||||
|
||||
def mock_import(name, *args):
|
||||
if name == "litellm":
|
||||
raise ImportError("No module named 'litellm'")
|
||||
@@ -208,9 +215,7 @@ class TestLiteLLMSDKEmbeddings:
|
||||
# Mock responses for each batch
|
||||
def mock_embedding_side_effect(model, input, **kwargs):
|
||||
mock_response = MagicMock()
|
||||
mock_response.data = [
|
||||
{"embedding": [float(i)] * 768, "index": i} for i in range(len(input))
|
||||
]
|
||||
mock_response.data = [{"embedding": [float(i)] * 768, "index": i} for i in range(len(input))]
|
||||
return mock_response
|
||||
|
||||
mock_litellm.embedding.side_effect = mock_embedding_side_effect
|
||||
@@ -279,7 +284,10 @@ class TestLiteLLMSDKEmbeddings:
|
||||
|
||||
async def test_custom_api_base(self, mock_litellm):
|
||||
"""Test custom API base URL is passed to embedding calls."""
|
||||
with patch("builtins.__import__", side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args)):
|
||||
with patch(
|
||||
"builtins.__import__",
|
||||
side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args),
|
||||
):
|
||||
emb = LiteLLMSDKEmbeddings(
|
||||
api_key="test_key",
|
||||
model="cohere/embed-english-v3.0",
|
||||
|
||||
@@ -230,9 +230,7 @@ async def test_litellm_explicit_param_wins_over_extra_body():
|
||||
provider._acompletion = AsyncMock(return_value=_fake_litellm_response())
|
||||
|
||||
with patch("hindsight_api.engine.providers.litellm_llm.get_metrics_collector"):
|
||||
await provider.call(
|
||||
messages=[{"role": "user", "content": "hi"}], temperature=0.9, scope="test", max_retries=0
|
||||
)
|
||||
await provider.call(messages=[{"role": "user", "content": "hi"}], temperature=0.9, scope="test", max_retries=0)
|
||||
|
||||
assert provider._acompletion.call_args.kwargs.get("temperature") == 0.9
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ per-operation semaphores when `HINDSIGHT_API_{RETAIN,REFLECT,CONSOLIDATION}_LLM_
|
||||
is set. They patch the module-level semaphore registry so they can run without
|
||||
needing to re-import the module with custom env vars.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from contextlib import AsyncExitStack
|
||||
from unittest.mock import patch
|
||||
@@ -79,9 +80,7 @@ class TestSemaphoresForScope:
|
||||
"consolidation": consolidation_sem,
|
||||
},
|
||||
):
|
||||
assert _semaphores_for_scope("mental_model_delta_ops") == [
|
||||
llm_wrapper._global_llm_semaphore
|
||||
]
|
||||
assert _semaphores_for_scope("mental_model_delta_ops") == [llm_wrapper._global_llm_semaphore]
|
||||
assert _semaphores_for_scope("memory_think") == [llm_wrapper._global_llm_semaphore]
|
||||
assert _semaphores_for_scope("verification") == [llm_wrapper._global_llm_semaphore]
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ pytestmark = pytest.mark.hs_llm_mat
|
||||
_PROVIDER = os.environ.get("HINDSIGHT_API_LLM_PROVIDER", "")
|
||||
_MODEL = os.environ.get("HINDSIGHT_API_LLM_MODEL", "")
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
"""Get API key from HINDSIGHT_API_LLM_API_KEY (CI) or provider-specific env var."""
|
||||
key = os.environ.get("HINDSIGHT_API_LLM_API_KEY", "")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Test that LLM calls record token metrics via the metrics collector.
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
import pytest
|
||||
@@ -31,7 +32,9 @@ async def test_llm_metrics_recorded_for_groq():
|
||||
mock_collector = MagicMock(spec=MetricsCollector)
|
||||
|
||||
# Patch the provider module where get_metrics_collector is actually called
|
||||
with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector", return_value=mock_collector):
|
||||
with patch(
|
||||
"hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector", return_value=mock_collector
|
||||
):
|
||||
llm = LLMProvider(
|
||||
provider="groq",
|
||||
api_key=api_key,
|
||||
@@ -43,7 +46,7 @@ async def test_llm_metrics_recorded_for_groq():
|
||||
response = await llm.call(
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant. Always respond."},
|
||||
{"role": "user", "content": "What is 2+2? Reply with just the number."}
|
||||
{"role": "user", "content": "What is 2+2? Reply with just the number."},
|
||||
],
|
||||
max_completion_tokens=50,
|
||||
scope="test_metrics",
|
||||
@@ -92,7 +95,9 @@ async def test_llm_metrics_recorded_for_structured_output():
|
||||
mock_collector = MagicMock(spec=MetricsCollector)
|
||||
|
||||
# Patch the provider module where get_metrics_collector is actually called
|
||||
with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector", return_value=mock_collector):
|
||||
with patch(
|
||||
"hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector", return_value=mock_collector
|
||||
):
|
||||
llm = LLMProvider(
|
||||
provider="groq",
|
||||
api_key=api_key,
|
||||
@@ -180,7 +185,7 @@ async def test_return_usage_returns_tuple():
|
||||
result, usage = await llm.call(
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "What is 2+2? Reply with just the number."}
|
||||
{"role": "user", "content": "What is 2+2? Reply with just the number."},
|
||||
],
|
||||
max_completion_tokens=50,
|
||||
return_usage=True,
|
||||
|
||||
@@ -51,9 +51,11 @@ class TestMockToolCalling:
|
||||
llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
|
||||
|
||||
# Set mock response to return tool calls
|
||||
llm.set_mock_response([
|
||||
{"name": "get_weather", "arguments": {"location": "Paris", "unit": "celsius"}},
|
||||
])
|
||||
llm.set_mock_response(
|
||||
[
|
||||
{"name": "get_weather", "arguments": {"location": "Paris", "unit": "celsius"}},
|
||||
]
|
||||
)
|
||||
|
||||
result = await llm.call_with_tools(
|
||||
messages=[{"role": "user", "content": "What's the weather in Paris?"}],
|
||||
@@ -105,10 +107,12 @@ class TestMockToolCalling:
|
||||
"""Test handling multiple tool calls in one response."""
|
||||
llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
|
||||
|
||||
llm.set_mock_response([
|
||||
{"name": "get_weather", "arguments": {"location": "Paris"}},
|
||||
{"name": "search", "arguments": {"query": "weather forecast"}},
|
||||
])
|
||||
llm.set_mock_response(
|
||||
[
|
||||
{"name": "get_weather", "arguments": {"location": "Paris"}},
|
||||
{"name": "search", "arguments": {"query": "weather forecast"}},
|
||||
]
|
||||
)
|
||||
|
||||
result = await llm.call_with_tools(
|
||||
messages=[{"role": "user", "content": "Weather in Paris and search for forecasts"}],
|
||||
|
||||
@@ -307,9 +307,7 @@ async def test_retain_creates_trace_rows_with_tokens(trace_api_client, bank_id):
|
||||
|
||||
# Filtering by a trace_id returns only that operation run's calls.
|
||||
a_trace = entry["trace_id"]
|
||||
resp = await trace_api_client.get(
|
||||
f"/v1/default/banks/{bank_id}/llm-requests", params={"trace_id": a_trace}
|
||||
)
|
||||
resp = await trace_api_client.get(f"/v1/default/banks/{bank_id}/llm-requests", params={"trace_id": a_trace})
|
||||
assert resp.status_code == 200
|
||||
filtered = resp.json()
|
||||
assert filtered["total"] >= 1
|
||||
@@ -402,9 +400,7 @@ async def test_memory_ids_mapped_to_retain_and_consolidation(trace_api_client, b
|
||||
# the retain that produced it (memory_ids) and any consolidation that consumed
|
||||
# it as a source (source_memory_ids).
|
||||
by_mem = (
|
||||
await trace_api_client.get(
|
||||
f"/v1/default/banks/{bank_id}/llm-requests", params={"memory_id": created[0]}
|
||||
)
|
||||
await trace_api_client.get(f"/v1/default/banks/{bank_id}/llm-requests", params={"memory_id": created[0]})
|
||||
).json()
|
||||
assert by_mem["total"] >= 1
|
||||
for it in by_mem["items"]:
|
||||
@@ -433,9 +429,7 @@ async def test_filter_by_status_and_operation(trace_api_client, bank_id):
|
||||
assert item["status"] == "success"
|
||||
assert item["operation"] == "retain"
|
||||
|
||||
response = await trace_api_client.get(
|
||||
f"/v1/default/banks/{bank_id}/llm-requests", params={"status": "error"}
|
||||
)
|
||||
response = await trace_api_client.get(f"/v1/default/banks/{bank_id}/llm-requests", params={"status": "error"})
|
||||
assert response.json()["total"] == 0
|
||||
|
||||
|
||||
@@ -448,9 +442,7 @@ async def test_stats_endpoint_includes_tokens(trace_api_client, bank_id):
|
||||
)
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
response = await trace_api_client.get(
|
||||
f"/v1/default/banks/{bank_id}/llm-requests/stats", params={"period": "1d"}
|
||||
)
|
||||
response = await trace_api_client.get(f"/v1/default/banks/{bank_id}/llm-requests/stats", params={"period": "1d"})
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["trunc"] == "day"
|
||||
|
||||
@@ -72,21 +72,23 @@ def create_mock_facts_from_content(content: str, ratio: float = 1.5, max_facts:
|
||||
If content has N sentences, return approximately N * ratio facts (capped at max_facts).
|
||||
"""
|
||||
# Estimate sentences by splitting on periods
|
||||
sentences = [s.strip() for s in content.split('.') if s.strip()]
|
||||
sentences = [s.strip() for s in content.split(".") if s.strip()]
|
||||
num_facts = min(max(1, int(len(sentences) * ratio)), max_facts)
|
||||
|
||||
facts = []
|
||||
for i in range(num_facts):
|
||||
facts.append({
|
||||
"what": f"Mock fact {i}: Something happened based on the content",
|
||||
"when": "2024-06-15",
|
||||
"where": "San Francisco",
|
||||
"who": "John, Sarah",
|
||||
"why": "Business reasons",
|
||||
"fact_type": "world",
|
||||
"entities": [{"text": "John", "type": "PERSON"}],
|
||||
"causal_relations": [],
|
||||
})
|
||||
facts.append(
|
||||
{
|
||||
"what": f"Mock fact {i}: Something happened based on the content",
|
||||
"when": "2024-06-15",
|
||||
"where": "San Francisco",
|
||||
"who": "John, Sarah",
|
||||
"why": "Business reasons",
|
||||
"fact_type": "world",
|
||||
"entities": [{"text": "John", "type": "PERSON"}],
|
||||
"causal_relations": [],
|
||||
}
|
||||
)
|
||||
|
||||
return facts
|
||||
|
||||
@@ -122,6 +124,7 @@ class TestLargeBatchRetain:
|
||||
@pytest.fixture
|
||||
def disable_observations(self):
|
||||
from hindsight_api.config import _get_raw_config
|
||||
|
||||
config = _get_raw_config()
|
||||
original = config.enable_observations
|
||||
config.enable_observations = False
|
||||
@@ -147,11 +150,13 @@ class TestLargeBatchRetain:
|
||||
contents = []
|
||||
for i in range(num_items):
|
||||
content_text = generate_content(chars_per_item)
|
||||
contents.append({
|
||||
"content": content_text,
|
||||
"context": f"Test content item {i + 1} of {num_items}",
|
||||
"event_date": datetime.now(UTC),
|
||||
})
|
||||
contents.append(
|
||||
{
|
||||
"content": content_text,
|
||||
"context": f"Test content item {i + 1} of {num_items}",
|
||||
"event_date": datetime.now(UTC),
|
||||
}
|
||||
)
|
||||
|
||||
actual_total_chars = sum(len(c["content"]) for c in contents)
|
||||
logger.info(f"Created {num_items} content items with {actual_total_chars:,} total chars")
|
||||
@@ -191,7 +196,7 @@ class TestLargeBatchRetain:
|
||||
return response_dict
|
||||
|
||||
# Patch LLMProvider.call at the class level
|
||||
with patch('hindsight_api.engine.llm_wrapper.LLMProvider.call', new=mock_llm_call):
|
||||
with patch("hindsight_api.engine.llm_wrapper.LLMProvider.call", new=mock_llm_call):
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
@@ -247,11 +252,13 @@ class TestLargeBatchRetain:
|
||||
|
||||
contents = []
|
||||
for i in range(num_items):
|
||||
contents.append({
|
||||
"content": generate_content(chars_per_item),
|
||||
"context": f"Chunk test item {i + 1}",
|
||||
"event_date": datetime.now(UTC),
|
||||
})
|
||||
contents.append(
|
||||
{
|
||||
"content": generate_content(chars_per_item),
|
||||
"context": f"Chunk test item {i + 1}",
|
||||
"event_date": datetime.now(UTC),
|
||||
}
|
||||
)
|
||||
|
||||
actual_total_chars = sum(len(c["content"]) for c in contents)
|
||||
logger.info(f"Created {num_items} items with {actual_total_chars:,} chars (should trigger chunking)")
|
||||
@@ -275,7 +282,7 @@ class TestLargeBatchRetain:
|
||||
return response_dict, TokenUsage(input_tokens=100, output_tokens=50)
|
||||
return response_dict
|
||||
|
||||
with patch('hindsight_api.engine.llm_wrapper.LLMProvider.call', new=mock_llm_call):
|
||||
with patch("hindsight_api.engine.llm_wrapper.LLMProvider.call", new=mock_llm_call):
|
||||
start_time = time.time()
|
||||
|
||||
result = await memory.retain_batch_async(
|
||||
@@ -305,9 +312,18 @@ class TestLargeBatchRetain:
|
||||
async def mock_llm_call(*args, **kwargs):
|
||||
# Small delay to simulate real LLM latency
|
||||
await asyncio.sleep(0.01)
|
||||
mock_facts = [{"what": "Test fact", "when": "now", "where": "here",
|
||||
"who": "someone", "why": "testing", "fact_type": "world",
|
||||
"entities": [], "causal_relations": []}]
|
||||
mock_facts = [
|
||||
{
|
||||
"what": "Test fact",
|
||||
"when": "now",
|
||||
"where": "here",
|
||||
"who": "someone",
|
||||
"why": "testing",
|
||||
"fact_type": "world",
|
||||
"entities": [],
|
||||
"causal_relations": [],
|
||||
}
|
||||
]
|
||||
response_dict = {"facts": mock_facts}
|
||||
|
||||
return_usage = kwargs.get("return_usage", False)
|
||||
@@ -315,16 +331,18 @@ class TestLargeBatchRetain:
|
||||
return response_dict, TokenUsage(input_tokens=10, output_tokens=10)
|
||||
return response_dict
|
||||
|
||||
with patch('hindsight_api.engine.llm_wrapper.LLMProvider.call', new=mock_llm_call):
|
||||
with patch("hindsight_api.engine.llm_wrapper.LLMProvider.call", new=mock_llm_call):
|
||||
# Run 10 concurrent retain operations
|
||||
tasks = []
|
||||
for i in range(10):
|
||||
bank_id = f"pool-test-{uuid.uuid4().hex[:8]}"
|
||||
contents = [{
|
||||
"content": f"Test content for concurrent operation {i}. " * 50,
|
||||
"context": f"Pool test {i}",
|
||||
"event_date": datetime.now(UTC),
|
||||
}]
|
||||
contents = [
|
||||
{
|
||||
"content": f"Test content for concurrent operation {i}. " * 50,
|
||||
"context": f"Pool test {i}",
|
||||
"event_date": datetime.now(UTC),
|
||||
}
|
||||
]
|
||||
tasks.append(
|
||||
memory.retain_batch_async(bank_id=bank_id, contents=contents, request_context=request_context)
|
||||
)
|
||||
|
||||
@@ -43,14 +43,15 @@ class TestMainModuleExtensionLoading:
|
||||
loaded_extensions[name] = result
|
||||
return result
|
||||
|
||||
with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
|
||||
patch("hindsight_api.main.create_app") as mock_create_app, \
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
|
||||
patch("hindsight_api.main.load_extension", side_effect=tracking_load_extension), \
|
||||
patch("hindsight_api.main.DefaultExtensionContext"), \
|
||||
patch("hindsight_api.main.print_banner"), \
|
||||
patch("uvicorn.run"): # Don't actually start uvicorn
|
||||
|
||||
with (
|
||||
patch("hindsight_api.main.MemoryEngine") as mock_engine,
|
||||
patch("hindsight_api.main.create_app") as mock_create_app,
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config,
|
||||
patch("hindsight_api.main.load_extension", side_effect=tracking_load_extension),
|
||||
patch("hindsight_api.main.DefaultExtensionContext"),
|
||||
patch("hindsight_api.main.print_banner"),
|
||||
patch("uvicorn.run"),
|
||||
): # Don't actually start uvicorn
|
||||
mock_config = MagicMock()
|
||||
mock_config.host = "0.0.0.0"
|
||||
mock_config.port = 8888
|
||||
@@ -63,17 +64,21 @@ class TestMainModuleExtensionLoading:
|
||||
mock_create_app.return_value = MagicMock()
|
||||
|
||||
# Mock sys.argv to simulate CLI invocation
|
||||
with patch.object(sys, 'argv', ['hindsight-api']):
|
||||
with patch.object(sys, "argv", ["hindsight-api"]):
|
||||
from hindsight_api.main import main
|
||||
|
||||
main()
|
||||
|
||||
# Verify TENANT extension was loaded
|
||||
assert "TENANT" in loaded_extensions, \
|
||||
assert "TENANT" in loaded_extensions, (
|
||||
"main.py did not call load_extension('TENANT', ...) - extensions not loaded!"
|
||||
assert loaded_extensions["TENANT"] is not None, \
|
||||
)
|
||||
assert loaded_extensions["TENANT"] is not None, (
|
||||
"load_extension('TENANT', ...) returned None despite env var being set"
|
||||
assert isinstance(loaded_extensions["TENANT"], MockTenantExtension), \
|
||||
)
|
||||
assert isinstance(loaded_extensions["TENANT"], MockTenantExtension), (
|
||||
f"Expected MockTenantExtension, got {type(loaded_extensions['TENANT'])}"
|
||||
)
|
||||
|
||||
def test_main_loads_operation_validator_when_configured(self, monkeypatch):
|
||||
"""
|
||||
@@ -94,14 +99,15 @@ class TestMainModuleExtensionLoading:
|
||||
loaded_extensions[name] = result
|
||||
return result
|
||||
|
||||
with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
|
||||
patch("hindsight_api.main.create_app") as mock_create_app, \
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
|
||||
patch("hindsight_api.main.load_extension", side_effect=tracking_load_extension), \
|
||||
patch("hindsight_api.main.DefaultExtensionContext"), \
|
||||
patch("hindsight_api.main.print_banner"), \
|
||||
patch("uvicorn.run"):
|
||||
|
||||
with (
|
||||
patch("hindsight_api.main.MemoryEngine") as mock_engine,
|
||||
patch("hindsight_api.main.create_app") as mock_create_app,
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config,
|
||||
patch("hindsight_api.main.load_extension", side_effect=tracking_load_extension),
|
||||
patch("hindsight_api.main.DefaultExtensionContext"),
|
||||
patch("hindsight_api.main.print_banner"),
|
||||
patch("uvicorn.run"),
|
||||
):
|
||||
mock_config = MagicMock()
|
||||
mock_config.host = "0.0.0.0"
|
||||
mock_config.port = 8888
|
||||
@@ -113,12 +119,14 @@ class TestMainModuleExtensionLoading:
|
||||
mock_engine.return_value = MagicMock()
|
||||
mock_create_app.return_value = MagicMock()
|
||||
|
||||
with patch.object(sys, 'argv', ['hindsight-api']):
|
||||
with patch.object(sys, "argv", ["hindsight-api"]):
|
||||
from hindsight_api.main import main
|
||||
|
||||
main()
|
||||
|
||||
assert "OPERATION_VALIDATOR" in loaded_extensions, \
|
||||
assert "OPERATION_VALIDATOR" in loaded_extensions, (
|
||||
"main.py did not call load_extension('OPERATION_VALIDATOR', ...)"
|
||||
)
|
||||
assert loaded_extensions["OPERATION_VALIDATOR"] is not None
|
||||
assert isinstance(loaded_extensions["OPERATION_VALIDATOR"], MockOperationValidator)
|
||||
|
||||
@@ -141,13 +149,14 @@ class TestMainModuleExtensionLoading:
|
||||
memory_engine_calls.append({"args": args, "kwargs": kwargs})
|
||||
return MagicMock()
|
||||
|
||||
with patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine), \
|
||||
patch("hindsight_api.main.create_app") as mock_create_app, \
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
|
||||
patch("hindsight_api.main.DefaultExtensionContext"), \
|
||||
patch("hindsight_api.main.print_banner"), \
|
||||
patch("uvicorn.run"):
|
||||
|
||||
with (
|
||||
patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine),
|
||||
patch("hindsight_api.main.create_app") as mock_create_app,
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config,
|
||||
patch("hindsight_api.main.DefaultExtensionContext"),
|
||||
patch("hindsight_api.main.print_banner"),
|
||||
patch("uvicorn.run"),
|
||||
):
|
||||
mock_config = MagicMock()
|
||||
mock_config.host = "0.0.0.0"
|
||||
mock_config.port = 8888
|
||||
@@ -158,8 +167,9 @@ class TestMainModuleExtensionLoading:
|
||||
mock_get_config.return_value = mock_config
|
||||
mock_create_app.return_value = MagicMock()
|
||||
|
||||
with patch.object(sys, 'argv', ['hindsight-api']):
|
||||
with patch.object(sys, "argv", ["hindsight-api"]):
|
||||
from hindsight_api.main import main
|
||||
|
||||
main()
|
||||
|
||||
# Verify MemoryEngine was called
|
||||
@@ -168,10 +178,10 @@ class TestMainModuleExtensionLoading:
|
||||
call_kwargs = memory_engine_calls[0]["kwargs"]
|
||||
|
||||
# THE CRITICAL ASSERTION: tenant_extension must be passed and not None
|
||||
assert "tenant_extension" in call_kwargs, \
|
||||
"MemoryEngine was not called with tenant_extension parameter!"
|
||||
assert call_kwargs["tenant_extension"] is not None, \
|
||||
assert "tenant_extension" in call_kwargs, "MemoryEngine was not called with tenant_extension parameter!"
|
||||
assert call_kwargs["tenant_extension"] is not None, (
|
||||
"tenant_extension was None - main.py did not pass loaded extension to MemoryEngine!"
|
||||
)
|
||||
|
||||
def test_main_sets_extension_context_on_tenant_extension(self, monkeypatch):
|
||||
"""
|
||||
@@ -198,13 +208,14 @@ class TestMainModuleExtensionLoading:
|
||||
context_created.append(ctx)
|
||||
return ctx
|
||||
|
||||
with patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine), \
|
||||
patch("hindsight_api.main.create_app") as mock_create_app, \
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
|
||||
patch("hindsight_api.main.DefaultExtensionContext", side_effect=capture_context), \
|
||||
patch("hindsight_api.main.print_banner"), \
|
||||
patch("uvicorn.run"):
|
||||
|
||||
with (
|
||||
patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine),
|
||||
patch("hindsight_api.main.create_app") as mock_create_app,
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config,
|
||||
patch("hindsight_api.main.DefaultExtensionContext", side_effect=capture_context),
|
||||
patch("hindsight_api.main.print_banner"),
|
||||
patch("uvicorn.run"),
|
||||
):
|
||||
mock_config = MagicMock()
|
||||
mock_config.host = "0.0.0.0"
|
||||
mock_config.port = 8888
|
||||
@@ -215,15 +226,15 @@ class TestMainModuleExtensionLoading:
|
||||
mock_get_config.return_value = mock_config
|
||||
mock_create_app.return_value = MagicMock()
|
||||
|
||||
with patch.object(sys, 'argv', ['hindsight-api']):
|
||||
with patch.object(sys, "argv", ["hindsight-api"]):
|
||||
from hindsight_api.main import main
|
||||
|
||||
main()
|
||||
|
||||
# Verify context was created and set
|
||||
assert len(context_created) == 1, "DefaultExtensionContext should be created"
|
||||
assert captured_tenant_ext[0] is not None, "Tenant extension should be captured"
|
||||
assert captured_tenant_ext[0]._context_set, \
|
||||
"set_context was not called on tenant extension"
|
||||
assert captured_tenant_ext[0]._context_set, "set_context was not called on tenant extension"
|
||||
|
||||
def test_main_works_without_extensions(self, monkeypatch):
|
||||
"""
|
||||
@@ -240,12 +251,13 @@ class TestMainModuleExtensionLoading:
|
||||
memory_engine_calls.append({"args": args, "kwargs": kwargs})
|
||||
return MagicMock()
|
||||
|
||||
with patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine), \
|
||||
patch("hindsight_api.main.create_app") as mock_create_app, \
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
|
||||
patch("hindsight_api.main.print_banner"), \
|
||||
patch("uvicorn.run"):
|
||||
|
||||
with (
|
||||
patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine),
|
||||
patch("hindsight_api.main.create_app") as mock_create_app,
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config,
|
||||
patch("hindsight_api.main.print_banner"),
|
||||
patch("uvicorn.run"),
|
||||
):
|
||||
mock_config = MagicMock()
|
||||
mock_config.host = "0.0.0.0"
|
||||
mock_config.port = 8888
|
||||
@@ -256,8 +268,9 @@ class TestMainModuleExtensionLoading:
|
||||
mock_get_config.return_value = mock_config
|
||||
mock_create_app.return_value = MagicMock()
|
||||
|
||||
with patch.object(sys, 'argv', ['hindsight-api']):
|
||||
with patch.object(sys, "argv", ["hindsight-api"]):
|
||||
from hindsight_api.main import main
|
||||
|
||||
main()
|
||||
|
||||
# Should work without extensions
|
||||
@@ -285,12 +298,13 @@ class TestMainModuleExtensionLoading:
|
||||
|
||||
mock_app = MagicMock()
|
||||
|
||||
with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
|
||||
patch("hindsight_api.main.create_app", return_value=mock_app), \
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
|
||||
patch("hindsight_api.main.print_banner"), \
|
||||
patch("uvicorn.run", side_effect=capture_uvicorn_run):
|
||||
|
||||
with (
|
||||
patch("hindsight_api.main.MemoryEngine") as mock_engine,
|
||||
patch("hindsight_api.main.create_app", return_value=mock_app),
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config,
|
||||
patch("hindsight_api.main.print_banner"),
|
||||
patch("uvicorn.run", side_effect=capture_uvicorn_run),
|
||||
):
|
||||
mock_config = MagicMock()
|
||||
mock_config.host = "0.0.0.0"
|
||||
mock_config.port = 8888
|
||||
@@ -301,14 +315,14 @@ class TestMainModuleExtensionLoading:
|
||||
mock_get_config.return_value = mock_config
|
||||
mock_engine.return_value = MagicMock()
|
||||
|
||||
with patch.object(sys, 'argv', ['hindsight-api', '--workers', '1']):
|
||||
with patch.object(sys, "argv", ["hindsight-api", "--workers", "1"]):
|
||||
from hindsight_api.main import main
|
||||
|
||||
main()
|
||||
|
||||
assert len(uvicorn_calls) == 1
|
||||
# With workers=1, should pass app object, not import string
|
||||
assert uvicorn_calls[0]["app"] is mock_app, \
|
||||
"main.py should pass app object (not import string) when workers=1"
|
||||
assert uvicorn_calls[0]["app"] is mock_app, "main.py should pass app object (not import string) when workers=1"
|
||||
|
||||
def test_main_uses_import_string_for_multiple_workers(self, monkeypatch):
|
||||
"""
|
||||
@@ -325,12 +339,13 @@ class TestMainModuleExtensionLoading:
|
||||
def capture_uvicorn_run(**kwargs):
|
||||
uvicorn_calls.append(kwargs)
|
||||
|
||||
with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
|
||||
patch("hindsight_api.main.create_app") as mock_create_app, \
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
|
||||
patch("hindsight_api.main.print_banner"), \
|
||||
patch("uvicorn.run", side_effect=capture_uvicorn_run):
|
||||
|
||||
with (
|
||||
patch("hindsight_api.main.MemoryEngine") as mock_engine,
|
||||
patch("hindsight_api.main.create_app") as mock_create_app,
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config,
|
||||
patch("hindsight_api.main.print_banner"),
|
||||
patch("uvicorn.run", side_effect=capture_uvicorn_run),
|
||||
):
|
||||
mock_config = MagicMock()
|
||||
mock_config.host = "0.0.0.0"
|
||||
mock_config.port = 8888
|
||||
@@ -342,14 +357,16 @@ class TestMainModuleExtensionLoading:
|
||||
mock_engine.return_value = MagicMock()
|
||||
mock_create_app.return_value = MagicMock()
|
||||
|
||||
with patch.object(sys, 'argv', ['hindsight-api', '--workers', '2']):
|
||||
with patch.object(sys, "argv", ["hindsight-api", "--workers", "2"]):
|
||||
from hindsight_api.main import main
|
||||
|
||||
main()
|
||||
|
||||
assert len(uvicorn_calls) == 1
|
||||
# With workers > 1, should use import string
|
||||
assert uvicorn_calls[0]["app"] == "hindsight_api.server:app", \
|
||||
assert uvicorn_calls[0]["app"] == "hindsight_api.server:app", (
|
||||
"main.py should use import string when workers > 1"
|
||||
)
|
||||
assert uvicorn_calls[0]["workers"] == 2
|
||||
|
||||
def test_main_sets_keepalive_timeout(self, monkeypatch):
|
||||
@@ -366,12 +383,13 @@ class TestMainModuleExtensionLoading:
|
||||
def capture_uvicorn_run(**kwargs):
|
||||
uvicorn_calls.append(kwargs)
|
||||
|
||||
with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
|
||||
patch("hindsight_api.main.create_app") as mock_create_app, \
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
|
||||
patch("hindsight_api.main.print_banner"), \
|
||||
patch("uvicorn.run", side_effect=capture_uvicorn_run):
|
||||
|
||||
with (
|
||||
patch("hindsight_api.main.MemoryEngine") as mock_engine,
|
||||
patch("hindsight_api.main.create_app") as mock_create_app,
|
||||
patch("hindsight_api.main._get_raw_config") as mock_get_config,
|
||||
patch("hindsight_api.main.print_banner"),
|
||||
patch("uvicorn.run", side_effect=capture_uvicorn_run),
|
||||
):
|
||||
mock_config = MagicMock()
|
||||
mock_config.host = "0.0.0.0"
|
||||
mock_config.port = 8888
|
||||
@@ -383,15 +401,16 @@ class TestMainModuleExtensionLoading:
|
||||
mock_engine.return_value = MagicMock()
|
||||
mock_create_app.return_value = MagicMock()
|
||||
|
||||
with patch.object(sys, 'argv', ['hindsight-api']):
|
||||
with patch.object(sys, "argv", ["hindsight-api"]):
|
||||
from hindsight_api.main import main
|
||||
|
||||
main()
|
||||
|
||||
assert len(uvicorn_calls) == 1
|
||||
assert "timeout_keep_alive" in uvicorn_calls[0], \
|
||||
"uvicorn config must set timeout_keep_alive"
|
||||
assert uvicorn_calls[0]["timeout_keep_alive"] > 15, \
|
||||
assert "timeout_keep_alive" in uvicorn_calls[0], "uvicorn config must set timeout_keep_alive"
|
||||
assert uvicorn_calls[0]["timeout_keep_alive"] > 15, (
|
||||
"timeout_keep_alive must exceed aiohttp's 15s client default"
|
||||
)
|
||||
|
||||
|
||||
# Mock extensions for testing
|
||||
|
||||
@@ -50,7 +50,9 @@ async def _insert_fact(conn, bank_id: str) -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconcile_submits_eligible_skips_disabled_and_in_flight(memory: MemoryEngine, request_context, monkeypatch):
|
||||
async def test_reconcile_submits_eligible_skips_disabled_and_in_flight(
|
||||
memory: MemoryEngine, request_context, monkeypatch
|
||||
):
|
||||
"""Reconcile enqueues consolidation for eligible banks and skips banks that
|
||||
disabled auto-consolidation or already have an in-flight consolidation."""
|
||||
eligible = await _make_bank(
|
||||
|
||||
@@ -111,7 +111,7 @@ async def test_maintenance_loop_targets_only_affected_tenants(
|
||||
)
|
||||
if i % 2 == 0:
|
||||
await conn.execute(
|
||||
f"INSERT INTO \"{s}\".audit_log (action, transport, started_at) "
|
||||
f'INSERT INTO "{s}".audit_log (action, transport, started_at) '
|
||||
f"VALUES ('OLD', 'system', now() - INTERVAL '10 days')"
|
||||
)
|
||||
audit_with_old.add(s)
|
||||
@@ -119,7 +119,7 @@ async def test_maintenance_loop_targets_only_affected_tenants(
|
||||
await conn.execute(f"INSERT INTO \"{s}\".llm_requests (status, started_at) VALUES ('success', now())")
|
||||
if i % 3 == 0:
|
||||
await conn.execute(
|
||||
f"INSERT INTO \"{s}\".llm_requests (status, started_at) "
|
||||
f'INSERT INTO "{s}".llm_requests (status, started_at) '
|
||||
f"VALUES ('success', now() - INTERVAL '3 days')"
|
||||
)
|
||||
llm_with_old.add(s)
|
||||
|
||||
@@ -53,7 +53,9 @@ async def _make_bank(memory: MemoryEngine, request_context, suffix: str) -> str:
|
||||
return bank_id
|
||||
|
||||
|
||||
async def _insert_fact(conn, bank_id: str, *, fact_type: str = "experience", consolidated: bool = False, failed: bool = False) -> None:
|
||||
async def _insert_fact(
|
||||
conn, bank_id: str, *, fact_type: str = "experience", consolidated: bool = False, failed: bool = False
|
||||
) -> None:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO memory_units (id, bank_id, text, fact_type, created_at, consolidated_at, consolidation_failed_at)
|
||||
@@ -141,9 +143,7 @@ async def test_schemas_with_expired_rows(memory: MemoryEngine):
|
||||
)
|
||||
|
||||
# 7-day cutoff: the 10-day-old row makes 'public' expired.
|
||||
expired_7 = await conn.fetch(
|
||||
"SELECT * FROM public.schemas_with_expired_rows('audit_log', 'started_at', 7)"
|
||||
)
|
||||
expired_7 = await conn.fetch("SELECT * FROM public.schemas_with_expired_rows('audit_log', 'started_at', 7)")
|
||||
assert "public" in {r[0] for r in expired_7}
|
||||
|
||||
# 100-year cutoff: nothing is that old.
|
||||
|
||||
@@ -35,16 +35,12 @@ class TestCollectCoercibleTypes:
|
||||
|
||||
def test_anyof_nullable_array(self):
|
||||
"""list[str] | None → anyOf with array and null."""
|
||||
arrays, objects = self._run(
|
||||
{"anyOf": [{"type": "array", "items": {"type": "string"}}, {"type": "null"}]}
|
||||
)
|
||||
arrays, objects = self._run({"anyOf": [{"type": "array", "items": {"type": "string"}}, {"type": "null"}]})
|
||||
assert "p" in arrays
|
||||
|
||||
def test_oneof_nullable_array(self):
|
||||
"""oneOf variant."""
|
||||
arrays, objects = self._run(
|
||||
{"oneOf": [{"type": "array", "items": {"type": "string"}}, {"type": "null"}]}
|
||||
)
|
||||
arrays, objects = self._run({"oneOf": [{"type": "array", "items": {"type": "string"}}, {"type": "null"}]})
|
||||
assert "p" in arrays
|
||||
|
||||
# --- object types ---
|
||||
@@ -136,9 +132,7 @@ class TestCoerceStringJson:
|
||||
assert result["metadata"] == {}
|
||||
|
||||
def test_native_dict_passthrough(self):
|
||||
result = _coerce_string_json(
|
||||
{"metadata": {"key": "value"}}, array_params=set(), object_params={"metadata"}
|
||||
)
|
||||
result = _coerce_string_json({"metadata": {"key": "value"}}, array_params=set(), object_params={"metadata"})
|
||||
assert result["metadata"] == {"key": "value"}
|
||||
|
||||
# --- non-coercible values left untouched ---
|
||||
@@ -153,16 +147,12 @@ class TestCoerceStringJson:
|
||||
|
||||
def test_wrong_json_type_not_coerced_list(self):
|
||||
"""String that parses to a dict should NOT be coerced for an array param."""
|
||||
result = _coerce_string_json(
|
||||
{"tags": '{"key": "value"}'}, array_params={"tags"}, object_params=set()
|
||||
)
|
||||
result = _coerce_string_json({"tags": '{"key": "value"}'}, array_params={"tags"}, object_params=set())
|
||||
assert result["tags"] == '{"key": "value"}'
|
||||
|
||||
def test_wrong_json_type_not_coerced_dict(self):
|
||||
"""String that parses to a list should NOT be coerced for an object param."""
|
||||
result = _coerce_string_json(
|
||||
{"metadata": '["a", "b"]'}, array_params=set(), object_params={"metadata"}
|
||||
)
|
||||
result = _coerce_string_json({"metadata": '["a", "b"]'}, array_params=set(), object_params={"metadata"})
|
||||
assert result["metadata"] == '["a", "b"]'
|
||||
|
||||
def test_string_param_not_touched(self):
|
||||
@@ -175,15 +165,11 @@ class TestCoerceStringJson:
|
||||
assert result["query"] == '["looks", "like", "json"]'
|
||||
|
||||
def test_integer_param_not_touched(self):
|
||||
result = _coerce_string_json(
|
||||
{"max_tokens": 4096}, array_params=set(), object_params=set()
|
||||
)
|
||||
result = _coerce_string_json({"max_tokens": 4096}, array_params=set(), object_params=set())
|
||||
assert result["max_tokens"] == 4096
|
||||
|
||||
def test_boolean_param_not_touched(self):
|
||||
result = _coerce_string_json(
|
||||
{"verbose": True}, array_params=set(), object_params=set()
|
||||
)
|
||||
result = _coerce_string_json({"verbose": True}, array_params=set(), object_params=set())
|
||||
assert result["verbose"] is True
|
||||
|
||||
def test_missing_param_no_error(self):
|
||||
@@ -277,13 +263,15 @@ class TestMakeToolsTolerantIntegration:
|
||||
mcp, captured = self._create_mcp_with_tool()
|
||||
_make_tools_tolerant(mcp)
|
||||
tool = _get_mcp_tools(mcp)["test_tool"]
|
||||
await tool.run({
|
||||
"query": "hi",
|
||||
"max_tokens": 200,
|
||||
"verbose": True,
|
||||
"tags": ["x"],
|
||||
"metadata": {"a": "b"},
|
||||
})
|
||||
await tool.run(
|
||||
{
|
||||
"query": "hi",
|
||||
"max_tokens": 200,
|
||||
"verbose": True,
|
||||
"tags": ["x"],
|
||||
"metadata": {"a": "b"},
|
||||
}
|
||||
)
|
||||
assert captured["query"] == "hi"
|
||||
assert captured["max_tokens"] == 200
|
||||
assert captured["verbose"] is True
|
||||
@@ -296,11 +284,13 @@ class TestMakeToolsTolerantIntegration:
|
||||
mcp, captured = self._create_mcp_with_tool()
|
||||
_make_tools_tolerant(mcp)
|
||||
tool = _get_mcp_tools(mcp)["test_tool"]
|
||||
await tool.run({
|
||||
"query": "hi",
|
||||
"tags": '["x"]',
|
||||
"explanation": "LLM added this",
|
||||
})
|
||||
await tool.run(
|
||||
{
|
||||
"query": "hi",
|
||||
"tags": '["x"]',
|
||||
"explanation": "LLM added this",
|
||||
}
|
||||
)
|
||||
assert captured["tags"] == ["x"]
|
||||
assert "explanation" not in captured
|
||||
|
||||
|
||||
@@ -78,10 +78,13 @@ async def test_filter_mcp_tools_returns_empty_set():
|
||||
class DenyAllValidator(OperationValidatorExtension):
|
||||
async def validate_retain(self, ctx):
|
||||
return ValidationResult.accept()
|
||||
|
||||
async def validate_recall(self, ctx):
|
||||
return ValidationResult.accept()
|
||||
|
||||
async def validate_reflect(self, ctx):
|
||||
return ValidationResult.accept()
|
||||
|
||||
async def filter_mcp_tools(self, bank_id, request_context, tools):
|
||||
return frozenset()
|
||||
|
||||
@@ -189,10 +192,13 @@ async def test_validator_cannot_add_tools_beyond_bank_config():
|
||||
class PermissiveValidator(OperationValidatorExtension):
|
||||
async def validate_retain(self, ctx):
|
||||
return ValidationResult.accept()
|
||||
|
||||
async def validate_recall(self, ctx):
|
||||
return ValidationResult.accept()
|
||||
|
||||
async def validate_reflect(self, ctx):
|
||||
return ValidationResult.accept()
|
||||
|
||||
async def filter_mcp_tools(self, bank_id, request_context, tools):
|
||||
return tools | {"retain", "delete_bank"}
|
||||
|
||||
@@ -241,15 +247,19 @@ async def test_validator_cannot_add_tools_beyond_bank_config():
|
||||
async def test_validator_exception_fails_open(caplog):
|
||||
"""If filter_mcp_tools raises, all tools remain visible and warning is logged."""
|
||||
import logging
|
||||
|
||||
caplog.set_level(logging.WARNING)
|
||||
|
||||
class BrokenValidator(OperationValidatorExtension):
|
||||
async def validate_retain(self, ctx):
|
||||
return ValidationResult.accept()
|
||||
|
||||
async def validate_recall(self, ctx):
|
||||
return ValidationResult.accept()
|
||||
|
||||
async def validate_reflect(self, ctx):
|
||||
return ValidationResult.accept()
|
||||
|
||||
async def filter_mcp_tools(self, bank_id, request_context, tools):
|
||||
raise RuntimeError("Policy backend unreachable")
|
||||
|
||||
|
||||
@@ -250,9 +250,7 @@ class TestDeltaRefreshPlumbing:
|
||||
# First refresh: establishes last_refreshed_source_query.
|
||||
patch_reflect(memory, text="# Team\n\nFirst pass.")
|
||||
patch_llm_call(memory, returns="unused-first")
|
||||
await memory.refresh_mental_model(
|
||||
bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
|
||||
)
|
||||
await memory.refresh_mental_model(bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context)
|
||||
|
||||
# Now change the source_query — a genuine topic shift.
|
||||
await memory.update_mental_model(
|
||||
@@ -289,15 +287,7 @@ class TestDeltaRefreshPlumbing:
|
||||
bank_id = f"test-delta-apply-{uuid.uuid4().hex[:8]}"
|
||||
await memory.get_bank_profile(bank_id, request_context=request_context)
|
||||
|
||||
existing = (
|
||||
"# Team\n"
|
||||
"\n"
|
||||
"Alice is the lead.\n"
|
||||
"\n"
|
||||
"## Members\n"
|
||||
"\n"
|
||||
"- Alice — lead\n"
|
||||
)
|
||||
existing = "# Team\n\nAlice is the lead.\n\n## Members\n\n- Alice — lead\n"
|
||||
mm = await memory.create_mental_model(
|
||||
bank_id=bank_id,
|
||||
name="Team Info",
|
||||
@@ -311,9 +301,7 @@ class TestDeltaRefreshPlumbing:
|
||||
# render of the parsed existing content. This also seeds the tracking column.
|
||||
patch_reflect(memory, text="ignored — full mode candidate")
|
||||
patch_llm_call(memory, returns=[]) # zero ops
|
||||
await memory.refresh_mental_model(
|
||||
bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
|
||||
)
|
||||
await memory.refresh_mental_model(bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context)
|
||||
|
||||
# Second refresh: a new fact arrives; LLM returns one append_block op.
|
||||
candidate = "# Team\n\nAlice is the lead. Bob joined as junior engineer."
|
||||
@@ -386,15 +374,7 @@ class TestDeltaRefreshPlumbing:
|
||||
bank_id = f"test-delta-noop-{uuid.uuid4().hex[:8]}"
|
||||
await memory.get_bank_profile(bank_id, request_context=request_context)
|
||||
|
||||
existing = (
|
||||
"# Team\n"
|
||||
"\n"
|
||||
"Alice is the lead.\n"
|
||||
"\n"
|
||||
"## Members\n"
|
||||
"\n"
|
||||
"- Alice\n"
|
||||
)
|
||||
existing = "# Team\n\nAlice is the lead.\n\n## Members\n\n- Alice\n"
|
||||
mm = await memory.create_mental_model(
|
||||
bank_id=bank_id,
|
||||
name="Team Info",
|
||||
@@ -461,9 +441,7 @@ class TestDeltaRefreshPlumbing:
|
||||
return DeltaOperationList()
|
||||
|
||||
monkeypatch.setattr(memory._reflect_llm_config, "call", ok_call)
|
||||
await memory.refresh_mental_model(
|
||||
bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
|
||||
)
|
||||
await memory.refresh_mental_model(bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context)
|
||||
|
||||
# Now the second refresh: LLM raises. Refresh must not crash; it should
|
||||
# store the candidate markdown.
|
||||
@@ -512,15 +490,7 @@ class TestDeltaRefreshPlumbing:
|
||||
bank_id = f"test-empty-reflect-{uuid.uuid4().hex[:8]}"
|
||||
await memory.get_bank_profile(bank_id, request_context=request_context)
|
||||
|
||||
existing = (
|
||||
"# Team\n"
|
||||
"\n"
|
||||
"Alice is the lead.\n"
|
||||
"\n"
|
||||
"## Members\n"
|
||||
"\n"
|
||||
"- Alice\n"
|
||||
)
|
||||
existing = "# Team\n\nAlice is the lead.\n\n## Members\n\n- Alice\n"
|
||||
mm = await memory.create_mental_model(
|
||||
bank_id=bank_id,
|
||||
name="Team Info",
|
||||
@@ -576,15 +546,9 @@ class TestDeltaRefreshPlumbing:
|
||||
# Real-Gemini evaluation tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_GEMINI_API_KEY = (
|
||||
os.getenv("HINDSIGHT_GEMINI_API_KEY")
|
||||
or os.getenv("GEMINI_API_KEY")
|
||||
or os.getenv("GOOGLE_API_KEY")
|
||||
)
|
||||
_GEMINI_API_KEY = os.getenv("HINDSIGHT_GEMINI_API_KEY") or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")
|
||||
_OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
|
||||
_RUN_LLM_EVAL = os.getenv("HINDSIGHT_RUN_GEMINI_EVALS") == "1" and (
|
||||
bool(_GEMINI_API_KEY) or bool(_OPENAI_API_KEY)
|
||||
)
|
||||
_RUN_LLM_EVAL = os.getenv("HINDSIGHT_RUN_GEMINI_EVALS") == "1" and (bool(_GEMINI_API_KEY) or bool(_OPENAI_API_KEY))
|
||||
|
||||
|
||||
pytestmark_gemini = pytest.mark.skipif(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for metrics instrumentation."""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -79,8 +80,10 @@ class TestMetricsCollector:
|
||||
"""Create a MetricsCollector with a mock meter."""
|
||||
mock_config = MagicMock()
|
||||
mock_config.metrics_include_bank_id = False
|
||||
with patch("hindsight_api.metrics.get_meter", return_value=mock_meter), \
|
||||
patch("hindsight_api.config.get_config", return_value=mock_config):
|
||||
with (
|
||||
patch("hindsight_api.metrics.get_meter", return_value=mock_meter),
|
||||
patch("hindsight_api.config.get_config", return_value=mock_config),
|
||||
):
|
||||
return MetricsCollector()
|
||||
|
||||
def test_record_operation_records_duration(self, collector):
|
||||
@@ -174,8 +177,10 @@ class TestMetricsCollector:
|
||||
"""Test that bank_id is included in attributes when metrics_include_bank_id is enabled."""
|
||||
mock_config = MagicMock()
|
||||
mock_config.metrics_include_bank_id = True
|
||||
with patch("hindsight_api.metrics.get_meter") as mock_get_meter, \
|
||||
patch("hindsight_api.config.get_config", return_value=mock_config):
|
||||
with (
|
||||
patch("hindsight_api.metrics.get_meter") as mock_get_meter,
|
||||
patch("hindsight_api.config.get_config", return_value=mock_config),
|
||||
):
|
||||
mock_get_meter.return_value = MagicMock()
|
||||
collector = MetricsCollector()
|
||||
|
||||
@@ -193,6 +198,7 @@ class TestGetMetricsCollector:
|
||||
"""Test that get_metrics_collector returns NoOpMetricsCollector by default."""
|
||||
# Reset global state
|
||||
import hindsight_api.metrics as metrics_module
|
||||
|
||||
original_collector = metrics_module._metrics_collector
|
||||
|
||||
try:
|
||||
@@ -208,6 +214,7 @@ class TestMetricsCollectorBase:
|
||||
|
||||
def test_is_abstract(self):
|
||||
"""Test that MetricsCollectorBase methods are abstract."""
|
||||
|
||||
# Create a class that inherits but doesn't implement
|
||||
class IncompleteCollector(MetricsCollectorBase):
|
||||
pass
|
||||
@@ -291,8 +298,10 @@ class TestLLMMetrics:
|
||||
"""Create a MetricsCollector with a mock meter."""
|
||||
mock_config = MagicMock()
|
||||
mock_config.metrics_include_bank_id = False
|
||||
with patch("hindsight_api.metrics.get_meter", return_value=mock_meter), \
|
||||
patch("hindsight_api.config.get_config", return_value=mock_config):
|
||||
with (
|
||||
patch("hindsight_api.metrics.get_meter", return_value=mock_meter),
|
||||
patch("hindsight_api.config.get_config", return_value=mock_config),
|
||||
):
|
||||
return MetricsCollector()
|
||||
|
||||
def test_record_llm_call_records_duration(self, collector):
|
||||
|
||||
@@ -41,6 +41,7 @@ def _downgrade(db_url: str, revision: str) -> None:
|
||||
# Fixture: fresh database at the revision just before the backsweep
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def pre_backsweep_db_url():
|
||||
"""
|
||||
@@ -73,6 +74,7 @@ def pre_backsweep_db_url():
|
||||
# The test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_backsweep_removes_orphans_and_preserves_legit_rows(pre_backsweep_db_url):
|
||||
"""
|
||||
Seed four kinds of rows then apply the backsweep migration and verify:
|
||||
@@ -100,12 +102,12 @@ def test_backsweep_removes_orphans_and_preserves_legit_rows(pre_backsweep_db_url
|
||||
ghost_bank = f"bank_{uuid.uuid4().hex[:8]}" # never inserted into banks
|
||||
|
||||
# UUIDs for memory units
|
||||
id_pass1_world = uuid.uuid4() # A: world unit, ghost bank
|
||||
id_pass1_obs = uuid.uuid4() # A: observation, ghost bank
|
||||
id_pass2_obs = uuid.uuid4() # B: observation, all sources gone
|
||||
id_keep_obs = uuid.uuid4() # C: observation with one live source
|
||||
id_keep_world = uuid.uuid4() # D: world unit, alive bank
|
||||
id_live_source = uuid.uuid4() # live source for C
|
||||
id_pass1_world = uuid.uuid4() # A: world unit, ghost bank
|
||||
id_pass1_obs = uuid.uuid4() # A: observation, ghost bank
|
||||
id_pass2_obs = uuid.uuid4() # B: observation, all sources gone
|
||||
id_keep_obs = uuid.uuid4() # C: observation with one live source
|
||||
id_keep_world = uuid.uuid4() # D: world unit, alive bank
|
||||
id_live_source = uuid.uuid4() # live source for C
|
||||
|
||||
with engine.connect() as conn:
|
||||
# --- banks ---
|
||||
@@ -147,10 +149,9 @@ def test_backsweep_removes_orphans_and_preserves_legit_rows(pre_backsweep_db_url
|
||||
|
||||
# --- verify ---
|
||||
with engine.connect() as conn:
|
||||
|
||||
def exists(uid):
|
||||
return conn.execute(
|
||||
text("SELECT 1 FROM memory_units WHERE id = :id"), {"id": uid}
|
||||
).fetchone() is not None
|
||||
return conn.execute(text("SELECT 1 FROM memory_units WHERE id = :id"), {"id": uid}).fetchone() is not None
|
||||
|
||||
# Must be gone
|
||||
assert not exists(id_pass1_world), "Pass 1: world unit with ghost bank should be deleted"
|
||||
|
||||
@@ -85,10 +85,7 @@ async def test_retain_chinese_content(memory_real_llm, request_context):
|
||||
for fact in result.results:
|
||||
logger.info(f"Fact: {fact.text[:100]}...")
|
||||
# Check for common Chinese characters or the name
|
||||
if any(
|
||||
char in fact.text
|
||||
for char in ["张", "伟", "腾讯", "软件", "工程师", "分布式", "系统", "代码"]
|
||||
):
|
||||
if any(char in fact.text for char in ["张", "伟", "腾讯", "软件", "工程师", "分布式", "系统", "代码"]):
|
||||
chinese_facts_found += 1
|
||||
|
||||
logger.info(f"Found {chinese_facts_found} facts with Chinese content")
|
||||
@@ -185,7 +182,7 @@ async def test_reflect_chinese_content(memory_real_llm, request_context):
|
||||
expected_names = set()
|
||||
for fact in result.based_on.get("world", []):
|
||||
# Extract Chinese entity names from the fact
|
||||
for entity in (fact.entities or []):
|
||||
for entity in fact.entities or []:
|
||||
# Check if entity contains Chinese characters
|
||||
if any("\u4e00" <= char <= "\u9fff" for char in entity):
|
||||
expected_names.add(entity)
|
||||
@@ -277,8 +274,7 @@ async def test_retain_japanese_content(memory_real_llm, request_context):
|
||||
japanese_facts_found += 1
|
||||
|
||||
assert japanese_facts_found > 0, (
|
||||
f"Expected facts to contain Japanese characters. "
|
||||
f"Facts: {[f.text for f in result.results]}"
|
||||
f"Expected facts to contain Japanese characters. Facts: {[f.text for f in result.results]}"
|
||||
)
|
||||
|
||||
logger.info("Japanese retain test passed - facts preserved in Japanese")
|
||||
@@ -350,8 +346,7 @@ async def test_english_content_stays_english(memory_real_llm, request_context):
|
||||
|
||||
# Count Japanese characters (hiragana, katakana)
|
||||
japanese_chars = sum(
|
||||
1 for char in fact.text
|
||||
if ("\u3040" <= char <= "\u309f") or ("\u30a0" <= char <= "\u30ff")
|
||||
1 for char in fact.text if ("\u3040" <= char <= "\u309f") or ("\u30a0" <= char <= "\u30ff")
|
||||
)
|
||||
|
||||
# Count Chinese/CJK characters (excluding those also used in Japanese)
|
||||
@@ -426,8 +421,7 @@ async def test_italian_content_stays_italian(memory_real_llm, request_context):
|
||||
# Count CJK characters
|
||||
cjk_chars = sum(1 for char in fact.text if "\u4e00" <= char <= "\u9fff")
|
||||
japanese_chars = sum(
|
||||
1 for char in fact.text
|
||||
if ("\u3040" <= char <= "\u309f") or ("\u30a0" <= char <= "\u30ff")
|
||||
1 for char in fact.text if ("\u3040" <= char <= "\u309f") or ("\u30a0" <= char <= "\u30ff")
|
||||
)
|
||||
|
||||
total_chars = len(fact.text)
|
||||
@@ -503,13 +497,9 @@ async def test_mixed_language_entities(memory_real_llm, request_context):
|
||||
|
||||
# Should contain Chinese name and/or English company names
|
||||
has_chinese_name = "王芳" in all_text
|
||||
has_english_company = any(
|
||||
company in all_text for company in ["Google", "Microsoft", "Amazon", "YouTube"]
|
||||
)
|
||||
has_english_company = any(company in all_text for company in ["Google", "Microsoft", "Amazon", "YouTube"])
|
||||
|
||||
assert has_chinese_name or has_english_company, (
|
||||
f"Expected mixed language entities. Facts: {all_text}"
|
||||
)
|
||||
assert has_chinese_name or has_english_company, f"Expected mixed language entities. Facts: {all_text}"
|
||||
|
||||
logger.info("Mixed language entity test passed")
|
||||
|
||||
|
||||
@@ -66,13 +66,9 @@ class TestObservationHistory:
|
||||
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
|
||||
async def test_returns_none_for_missing_observation(
|
||||
self, memory: MemoryEngine, request_context: Any
|
||||
) -> None:
|
||||
async def test_returns_none_for_missing_observation(self, memory: MemoryEngine, request_context: Any) -> None:
|
||||
bank_id = f"test-obs-hist-{uuid.uuid4().hex[:8]}"
|
||||
await memory.get_bank_profile(bank_id, request_context=request_context)
|
||||
result = await memory.get_observation_history(
|
||||
bank_id, str(uuid.uuid4()), request_context=request_context
|
||||
)
|
||||
result = await memory.get_observation_history(bank_id, str(uuid.uuid4()), request_context=request_context)
|
||||
assert result is None
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
|
||||
@@ -267,6 +267,7 @@ class TestDeleteDocumentObservationCleanup:
|
||||
# Tests: document upsert via retain pipeline (regression for orphan observations)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDocumentUpsertObservationCleanup:
|
||||
"""Regression: re-ingesting a document via the retain pipeline must clean up
|
||||
observations derived from the outgoing memory_units, the same way the
|
||||
@@ -360,9 +361,7 @@ class TestDocumentUpsertObservationCleanup:
|
||||
# be reset for re-consolidation since one of its observations was
|
||||
# invalidated by the upsert.
|
||||
consolidated_at = await _get_consolidated_at(conn, standalone_mem)
|
||||
assert consolidated_at is None, (
|
||||
"Surviving co-source memory should be reset for re-consolidation"
|
||||
)
|
||||
assert consolidated_at is None, "Surviving co-source memory should be reset for re-consolidation"
|
||||
|
||||
# The two doc-scoped memories are gone via FK cascade.
|
||||
doc_mem_count = await conn.fetchval(
|
||||
|
||||
@@ -5,6 +5,7 @@ NOTE: Observations are now stored as summaries on the entities table,
|
||||
not as separate memory_units. The observations list in EntityState is
|
||||
populated from the summary for backwards compatibility.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from hindsight_api.engine.memory_engine import Budget
|
||||
from hindsight_api import RequestContext
|
||||
@@ -64,7 +65,7 @@ async def test_entity_extraction_on_retain(memory, request_context):
|
||||
WHERE bank_id = $1 AND LOWER(canonical_name) LIKE '%john%'
|
||||
LIMIT 1
|
||||
""",
|
||||
bank_id
|
||||
bank_id,
|
||||
)
|
||||
|
||||
# Check the fact count for this entity
|
||||
@@ -73,7 +74,7 @@ async def test_entity_extraction_on_retain(memory, request_context):
|
||||
"""
|
||||
SELECT COUNT(*) FROM unit_entities WHERE entity_id = $1
|
||||
""",
|
||||
entity_row['id']
|
||||
entity_row["id"],
|
||||
)
|
||||
print(f"\n=== Entity Facts ===")
|
||||
print(f"Entity: {entity_row['canonical_name']} has {fact_count} linked facts")
|
||||
@@ -175,7 +176,7 @@ async def test_observation_fact_type_in_database(memory, request_context, disabl
|
||||
FROM memory_units
|
||||
WHERE bank_id = $1 AND fact_type = 'observation'
|
||||
""",
|
||||
bank_id
|
||||
bank_id,
|
||||
)
|
||||
|
||||
print(f"\n=== Observation Records in memory_units ===")
|
||||
@@ -243,7 +244,7 @@ async def test_entity_mention_counts(memory, request_context):
|
||||
WHERE e.bank_id = $1
|
||||
ORDER BY e.mention_count DESC
|
||||
""",
|
||||
bank_id
|
||||
bank_id,
|
||||
)
|
||||
|
||||
print(f"\n=== Entity Mention Counts Test ===")
|
||||
@@ -253,8 +254,8 @@ async def test_entity_mention_counts(memory, request_context):
|
||||
low_mention_entity = None
|
||||
|
||||
for entity in entities:
|
||||
name = entity['canonical_name'].lower()
|
||||
mention_count = entity['mention_count']
|
||||
name = entity["canonical_name"].lower()
|
||||
mention_count = entity["mention_count"]
|
||||
|
||||
print(f" {entity['canonical_name']}: mentions={mention_count}")
|
||||
|
||||
@@ -268,8 +269,9 @@ async def test_entity_mention_counts(memory, request_context):
|
||||
assert low_mention_entity is not None, "Trivex entity should exist"
|
||||
|
||||
# Nexora (10 mentions) must rank higher than Trivex (1 mention)
|
||||
assert high_mention_entity['mention_count'] > low_mention_entity['mention_count'], \
|
||||
assert high_mention_entity["mention_count"] > low_mention_entity["mention_count"], (
|
||||
f"Nexora ({high_mention_entity['mention_count']} mentions) should have more than Trivex ({low_mention_entity['mention_count']} mentions)"
|
||||
)
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
@@ -297,7 +299,7 @@ async def test_entity_mention_ranking(memory, request_context):
|
||||
for i in range(6):
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content=f"Alice is mentioned here in fact {i+1}.",
|
||||
content=f"Alice is mentioned here in fact {i + 1}.",
|
||||
context="test",
|
||||
event_date=datetime(2024, 1, 1 + i, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
@@ -310,7 +312,7 @@ async def test_entity_mention_ranking(memory, request_context):
|
||||
for mention in range(10):
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content=f"{entity_name} is a very important entity, mention {mention+1}.",
|
||||
content=f"{entity_name} is a very important entity, mention {mention + 1}.",
|
||||
context="test",
|
||||
event_date=datetime(2024, 2, 1 + mention, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
@@ -329,7 +331,7 @@ async def test_entity_mention_ranking(memory, request_context):
|
||||
WHERE bank_id = $1
|
||||
ORDER BY mention_count DESC
|
||||
""",
|
||||
bank_id
|
||||
bank_id,
|
||||
)
|
||||
|
||||
print(f"\nAll entities by mention count:")
|
||||
@@ -337,15 +339,15 @@ async def test_entity_mention_ranking(memory, request_context):
|
||||
print(f" {e['canonical_name']}: mentions={e['mention_count']}")
|
||||
|
||||
# Verify high-mention entities rank higher than Alice (6 mentions)
|
||||
alice = next((e for e in all_entities if 'alice' in e['canonical_name'].lower()), None)
|
||||
high_mention = [e for e in all_entities if e['canonical_name'].lower() in ('bruno', 'carlos', 'diana')]
|
||||
alice = next((e for e in all_entities if "alice" in e["canonical_name"].lower()), None)
|
||||
high_mention = [e for e in all_entities if e["canonical_name"].lower() in ("bruno", "carlos", "diana")]
|
||||
|
||||
assert alice is not None, "Alice entity should exist"
|
||||
assert len(high_mention) > 0, "High-mention entities should exist"
|
||||
|
||||
# Entities with 10 mentions each should rank higher than Alice (6 mentions)
|
||||
for entity in high_mention:
|
||||
assert entity['mention_count'] > alice['mention_count'], (
|
||||
assert entity["mention_count"] > alice["mention_count"], (
|
||||
f"{entity['canonical_name']} ({entity['mention_count']} mentions) "
|
||||
f"should rank higher than Alice ({alice['mention_count']} mentions)"
|
||||
)
|
||||
@@ -407,7 +409,7 @@ async def test_user_entity_extraction(memory_real_llm, request_context):
|
||||
AND LOWER(e.canonical_name) LIKE '%user%'
|
||||
LIMIT 1
|
||||
""",
|
||||
bank_id
|
||||
bank_id,
|
||||
)
|
||||
|
||||
# Get all entities with their fact counts
|
||||
@@ -421,7 +423,7 @@ async def test_user_entity_extraction(memory_real_llm, request_context):
|
||||
WHERE e.bank_id = $1
|
||||
ORDER BY fact_count DESC
|
||||
""",
|
||||
bank_id
|
||||
bank_id,
|
||||
)
|
||||
|
||||
print(f"\n=== Entities by Mention Count ===")
|
||||
|
||||
@@ -139,7 +139,9 @@ async def test_onnx_embeddings_dimension_mismatch_raises_value_error():
|
||||
tokenizer_name_or_path="/models/e5",
|
||||
dimensions=3,
|
||||
)
|
||||
fake_transformers = SimpleNamespace(AutoTokenizer=SimpleNamespace(from_pretrained=MagicMock(return_value=FakeTokenizer())))
|
||||
fake_transformers = SimpleNamespace(
|
||||
AutoTokenizer=SimpleNamespace(from_pretrained=MagicMock(return_value=FakeTokenizer()))
|
||||
)
|
||||
fake_onnxruntime = SimpleNamespace(InferenceSession=MagicMock(return_value=FakeOnnxSession()))
|
||||
|
||||
with patch.dict(sys.modules, {"transformers": fake_transformers, "onnxruntime": fake_onnxruntime}):
|
||||
@@ -153,7 +155,9 @@ async def test_onnx_embeddings_downloads_external_data_sidecar_when_needed():
|
||||
download = MagicMock(return_value="/hf/bge-m3")
|
||||
session = MagicMock(return_value=FakeOnnxSession())
|
||||
fake_hf = SimpleNamespace(snapshot_download=download)
|
||||
fake_transformers = SimpleNamespace(AutoTokenizer=SimpleNamespace(from_pretrained=MagicMock(return_value=FakeTokenizer())))
|
||||
fake_transformers = SimpleNamespace(
|
||||
AutoTokenizer=SimpleNamespace(from_pretrained=MagicMock(return_value=FakeTokenizer()))
|
||||
)
|
||||
fake_onnxruntime = SimpleNamespace(InferenceSession=session)
|
||||
|
||||
with patch.dict(
|
||||
|
||||
@@ -190,9 +190,7 @@ class TestMarkOperationGracefulOnMissingRow:
|
||||
await memory._mark_operation_failed(missing_id, "some error", "traceback here") # no exception
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mark_completed_and_fire_webhook_does_not_raise_when_row_missing(
|
||||
self, memory: MemoryEngine
|
||||
):
|
||||
async def test_mark_completed_and_fire_webhook_does_not_raise_when_row_missing(self, memory: MemoryEngine):
|
||||
missing_id = str(uuid.uuid4())
|
||||
await memory._mark_operation_completed_and_fire_webhook(
|
||||
operation_id=missing_id,
|
||||
@@ -265,9 +263,7 @@ class TestConsolidationCheckpoint:
|
||||
|
||||
class TestRetainCheckpoint:
|
||||
@pytest.mark.asyncio
|
||||
async def test_retain_stops_between_sub_batches_when_cancelled(
|
||||
self, memory: MemoryEngine, request_context
|
||||
):
|
||||
async def test_retain_stops_between_sub_batches_when_cancelled(self, memory: MemoryEngine, request_context):
|
||||
"""retain_batch_async returns partial results if _check_op_alive is False between sub-batches."""
|
||||
from hindsight_api.config import _get_raw_config
|
||||
|
||||
@@ -290,9 +286,7 @@ class TestRetainCheckpoint:
|
||||
# Cancel after the first sub-batch completes
|
||||
return check_calls <= 1
|
||||
|
||||
contents = [
|
||||
{"content": f"Memory item {i} about something interesting."} for i in range(4)
|
||||
]
|
||||
contents = [{"content": f"Memory item {i} about something interesting."} for i in range(4)]
|
||||
|
||||
with patch.object(memory, "_check_op_alive", side_effect=_fake_check):
|
||||
result = await memory.retain_batch_async(
|
||||
|
||||
@@ -7,6 +7,7 @@ Regression tests:
|
||||
- Cancel used to delete the operation row; now it sets status to 'cancelled'.
|
||||
- Retry now accepts both 'failed' and 'cancelled' operations.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
@@ -124,9 +125,7 @@ async def test_get_operation_returns_processing_status(api_client, memory, test_
|
||||
|
||||
processing_id = await _insert_operation(pool, test_bank_id, "processing")
|
||||
|
||||
response = await api_client.get(
|
||||
f"/v1/default/banks/{test_bank_id}/operations/{processing_id}"
|
||||
)
|
||||
response = await api_client.get(f"/v1/default/banks/{test_bank_id}/operations/{processing_id}")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "processing"
|
||||
@@ -154,9 +153,7 @@ async def test_all_statuses_returned_correctly(api_client, memory, test_bank_id)
|
||||
|
||||
# Verify get endpoint for each
|
||||
for status, op_id in ids.items():
|
||||
response = await api_client.get(
|
||||
f"/v1/default/banks/{test_bank_id}/operations/{op_id}"
|
||||
)
|
||||
response = await api_client.get(f"/v1/default/banks/{test_bank_id}/operations/{op_id}")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == status, f"Get: expected {status} for {op_id}"
|
||||
|
||||
@@ -170,16 +167,12 @@ async def test_cancel_sets_cancelled_status(api_client, memory, test_bank_id):
|
||||
op_id = await _insert_operation(pool, test_bank_id, "pending")
|
||||
|
||||
# Cancel the operation
|
||||
response = await api_client.delete(
|
||||
f"/v1/default/banks/{test_bank_id}/operations/{op_id}"
|
||||
)
|
||||
response = await api_client.delete(f"/v1/default/banks/{test_bank_id}/operations/{op_id}")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["success"] is True
|
||||
|
||||
# Verify the operation still exists with 'cancelled' status
|
||||
response = await api_client.get(
|
||||
f"/v1/default/banks/{test_bank_id}/operations/{op_id}"
|
||||
)
|
||||
response = await api_client.get(f"/v1/default/banks/{test_bank_id}/operations/{op_id}")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "cancelled"
|
||||
|
||||
@@ -203,16 +196,12 @@ async def test_retry_cancelled_operation(api_client, memory, test_bank_id):
|
||||
op_id = await _insert_operation(pool, test_bank_id, "cancelled")
|
||||
|
||||
# Retry the cancelled operation
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{test_bank_id}/operations/{op_id}/retry"
|
||||
)
|
||||
response = await api_client.post(f"/v1/default/banks/{test_bank_id}/operations/{op_id}/retry")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["success"] is True
|
||||
|
||||
# Verify the operation is now pending
|
||||
response = await api_client.get(
|
||||
f"/v1/default/banks/{test_bank_id}/operations/{op_id}"
|
||||
)
|
||||
response = await api_client.get(f"/v1/default/banks/{test_bank_id}/operations/{op_id}")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "pending"
|
||||
|
||||
@@ -225,9 +214,7 @@ async def test_retry_rejects_non_retriable_statuses(api_client, memory, test_ban
|
||||
|
||||
for status in ("pending", "processing", "completed"):
|
||||
op_id = await _insert_operation(pool, test_bank_id, status)
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{test_bank_id}/operations/{op_id}/retry"
|
||||
)
|
||||
response = await api_client.post(f"/v1/default/banks/{test_bank_id}/operations/{op_id}/retry")
|
||||
assert response.status_code == 409, f"Expected 409 for {status}, got {response.status_code}"
|
||||
|
||||
|
||||
@@ -239,7 +226,5 @@ async def test_cancel_rejects_non_pending_operations(api_client, memory, test_ba
|
||||
|
||||
for status in ("processing", "completed", "failed"):
|
||||
op_id = await _insert_operation(pool, test_bank_id, status)
|
||||
response = await api_client.delete(
|
||||
f"/v1/default/banks/{test_bank_id}/operations/{op_id}"
|
||||
)
|
||||
response = await api_client.delete(f"/v1/default/banks/{test_bank_id}/operations/{op_id}")
|
||||
assert response.status_code == 409, f"Expected 409 for {status}, got {response.status_code}"
|
||||
|
||||
@@ -27,6 +27,7 @@ logger = logging.getLogger(__name__)
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _bank_id(prefix: str = "http") -> str:
|
||||
return f"test-{prefix}-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
@@ -188,11 +189,7 @@ class TestOracleHTTP:
|
||||
# Retain
|
||||
resp = await api_client.post(
|
||||
f"/v1/default/banks/{bank_id}/memories",
|
||||
json={
|
||||
"items": [
|
||||
{"content": "Memory CRUD via HTTP on Oracle.", "context": "test"}
|
||||
]
|
||||
},
|
||||
json={"items": [{"content": "Memory CRUD via HTTP on Oracle.", "context": "test"}]},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
@@ -268,9 +265,7 @@ class TestOracleHTTP:
|
||||
|
||||
# Delete
|
||||
if directive_id:
|
||||
resp = await api_client.delete(
|
||||
f"/v1/default/banks/{bank_id}/directives/{directive_id}"
|
||||
)
|
||||
resp = await api_client.delete(f"/v1/default/banks/{bank_id}/directives/{directive_id}")
|
||||
assert resp.status_code == 200
|
||||
finally:
|
||||
await _safe_http_cleanup(api_client, bank_id)
|
||||
@@ -306,11 +301,7 @@ class TestOracleHTTP:
|
||||
try:
|
||||
await api_client.post(
|
||||
f"/v1/default/banks/{bank_id}/memories",
|
||||
json={
|
||||
"items": [
|
||||
{"content": "Operations tracking test.", "context": "test"}
|
||||
]
|
||||
},
|
||||
json={"items": [{"content": "Operations tracking test.", "context": "test"}]},
|
||||
)
|
||||
resp = await api_client.get(f"/v1/default/banks/{bank_id}/operations")
|
||||
assert resp.status_code == 200
|
||||
@@ -445,9 +436,7 @@ class TestOracleEndToEnd:
|
||||
|
||||
# --- 5. Verify the operation completed (not stuck as 'pending') ---
|
||||
if operation_id:
|
||||
resp = await api_client.get(
|
||||
f"/v1/default/banks/{bank_id}/operations/{operation_id}"
|
||||
)
|
||||
resp = await api_client.get(f"/v1/default/banks/{bank_id}/operations/{operation_id}")
|
||||
if resp.status_code == 200:
|
||||
op = resp.json()
|
||||
# SyncTaskBackend should have completed the refresh inline
|
||||
@@ -456,9 +445,7 @@ class TestOracleEndToEnd:
|
||||
)
|
||||
|
||||
# --- 6. Verify the mental model has real content (not placeholder) ---
|
||||
resp = await api_client.get(
|
||||
f"/v1/default/banks/{bank_id}/mental-models/{mental_model_id}"
|
||||
)
|
||||
resp = await api_client.get(f"/v1/default/banks/{bank_id}/mental-models/{mental_model_id}")
|
||||
assert resp.status_code == 200, f"Get mental model failed: {resp.text}"
|
||||
mm = resp.json()
|
||||
content = mm.get("content", "")
|
||||
|
||||
@@ -28,6 +28,7 @@ logger = logging.getLogger(__name__)
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _bank_id(prefix: str = "oracle") -> str:
|
||||
return f"test-{prefix}-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
@@ -136,21 +137,25 @@ class TestCoreCRUD:
|
||||
try:
|
||||
await oracle_memory.retain_batch_async(
|
||||
bank_id=bank_id,
|
||||
contents=[{
|
||||
"content": "Dan is an expert in distributed systems.",
|
||||
"context": "engineering",
|
||||
"event_date": datetime(2024, 5, 1, tzinfo=timezone.utc),
|
||||
}],
|
||||
contents=[
|
||||
{
|
||||
"content": "Dan is an expert in distributed systems.",
|
||||
"context": "engineering",
|
||||
"event_date": datetime(2024, 5, 1, tzinfo=timezone.utc),
|
||||
}
|
||||
],
|
||||
document_tags=["backend"],
|
||||
request_context=request_context,
|
||||
)
|
||||
await oracle_memory.retain_batch_async(
|
||||
bank_id=bank_id,
|
||||
contents=[{
|
||||
"content": "Eve designs beautiful user interfaces.",
|
||||
"context": "design",
|
||||
"event_date": datetime(2024, 5, 2, tzinfo=timezone.utc),
|
||||
}],
|
||||
contents=[
|
||||
{
|
||||
"content": "Eve designs beautiful user interfaces.",
|
||||
"context": "design",
|
||||
"event_date": datetime(2024, 5, 2, tzinfo=timezone.utc),
|
||||
}
|
||||
],
|
||||
document_tags=["frontend"],
|
||||
request_context=request_context,
|
||||
)
|
||||
@@ -236,9 +241,7 @@ class TestCoreCRUD:
|
||||
memory_id = unit_ids[0]
|
||||
|
||||
# Delete the memory
|
||||
await oracle_memory.delete_memory_unit(
|
||||
str(memory_id), request_context=request_context
|
||||
)
|
||||
await oracle_memory.delete_memory_unit(str(memory_id), request_context=request_context)
|
||||
|
||||
# Verify deletion
|
||||
mem = await oracle_memory.get_memory_unit(
|
||||
@@ -306,9 +309,7 @@ class TestCoreCRUD:
|
||||
event_date=datetime(2024, 6, i + 1, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
)
|
||||
memories = await oracle_memory.list_memory_units(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
memories = await oracle_memory.list_memory_units(bank_id=bank_id, request_context=request_context)
|
||||
# Each retain may extract multiple facts
|
||||
assert len(memories) >= 3
|
||||
finally:
|
||||
@@ -361,7 +362,7 @@ class TestRetainPipeline:
|
||||
"Leonardo da Vinci painted the Mona Lisa between 1503 and 1519, and it now hangs in the Louvre Museum in Paris.",
|
||||
"The International Space Station orbits Earth at an altitude of approximately 250 miles at a speed of 17,500 mph.",
|
||||
]
|
||||
long_content = " ".join(f"Section {i+1}: {fact} " * 3 for i, fact in enumerate(facts))
|
||||
long_content = " ".join(f"Section {i + 1}: {fact} " * 3 for i, fact in enumerate(facts))
|
||||
unit_ids = await oracle_memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content=long_content,
|
||||
@@ -514,11 +515,13 @@ class TestRetainPipeline:
|
||||
try:
|
||||
await oracle_memory.retain_batch_async(
|
||||
bank_id=bank_id,
|
||||
contents=[{
|
||||
"content": "Tagged memory about machine learning models.",
|
||||
"context": "ml",
|
||||
"event_date": datetime(2024, 6, 1, tzinfo=timezone.utc),
|
||||
}],
|
||||
contents=[
|
||||
{
|
||||
"content": "Tagged memory about machine learning models.",
|
||||
"context": "ml",
|
||||
"event_date": datetime(2024, 6, 1, tzinfo=timezone.utc),
|
||||
}
|
||||
],
|
||||
document_tags=["ml", "models"],
|
||||
request_context=request_context,
|
||||
)
|
||||
@@ -654,7 +657,7 @@ class TestSearchRetrieval:
|
||||
for i in range(5):
|
||||
await oracle_memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content=f"Fact {i}: Machine learning model {i} achieved {90+i}% accuracy.",
|
||||
content=f"Fact {i}: Machine learning model {i} achieved {90 + i}% accuracy.",
|
||||
context="ml",
|
||||
event_date=datetime(2024, 1, i + 1, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
@@ -818,9 +821,7 @@ class TestAdvancedFeatures:
|
||||
assert model["id"] is not None
|
||||
|
||||
# List
|
||||
models = await oracle_memory.list_mental_models(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
models = await oracle_memory.list_mental_models(bank_id=bank_id, request_context=request_context)
|
||||
assert len(models) > 0
|
||||
|
||||
# Get
|
||||
@@ -899,16 +900,12 @@ class TestAdvancedFeatures:
|
||||
)
|
||||
# Verify facts were stored (consolidation is async and may run inline
|
||||
# via SyncTaskBackend, but the key assertion is that all 5 retains persisted)
|
||||
memories = await oracle_memory.list_memory_units(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
memories = await oracle_memory.list_memory_units(bank_id=bank_id, request_context=request_context)
|
||||
items = memories.get("items", memories) if isinstance(memories, dict) else memories
|
||||
assert len(items) >= 5, f"Expected at least 5 stored memories, got {len(items)}"
|
||||
# Verify facts contain expected content
|
||||
texts = [item.get("text", "") for item in items]
|
||||
assert any("dark mode" in t for t in texts), (
|
||||
f"Expected 'dark mode' in stored facts, got: {texts[:3]}"
|
||||
)
|
||||
assert any("dark mode" in t for t in texts), f"Expected 'dark mode' in stored facts, got: {texts[:3]}"
|
||||
finally:
|
||||
await _safe_cleanup(oracle_memory, bank_id, request_context)
|
||||
|
||||
@@ -923,9 +920,7 @@ class TestAdvancedFeatures:
|
||||
event_date=datetime(2024, 6, 1, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
)
|
||||
ops = await oracle_memory.list_operations(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
ops = await oracle_memory.list_operations(bank_id=bank_id, request_context=request_context)
|
||||
# Retain creates async operations (consolidation at minimum)
|
||||
assert ops is not None
|
||||
items = ops.get("items", ops) if isinstance(ops, dict) else ops
|
||||
@@ -948,9 +943,7 @@ class TestAdvancedFeatures:
|
||||
)
|
||||
assert directive is not None
|
||||
|
||||
directives = await oracle_memory.list_directives(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
directives = await oracle_memory.list_directives(bank_id=bank_id, request_context=request_context)
|
||||
assert len(directives) > 0
|
||||
|
||||
await oracle_memory.delete_directive(
|
||||
@@ -967,17 +960,17 @@ class TestAdvancedFeatures:
|
||||
try:
|
||||
await oracle_memory.retain_batch_async(
|
||||
bank_id=bank_id,
|
||||
contents=[{
|
||||
"content": "Tag listing test.",
|
||||
"context": "test",
|
||||
"event_date": datetime(2024, 6, 1, tzinfo=timezone.utc),
|
||||
}],
|
||||
contents=[
|
||||
{
|
||||
"content": "Tag listing test.",
|
||||
"context": "test",
|
||||
"event_date": datetime(2024, 6, 1, tzinfo=timezone.utc),
|
||||
}
|
||||
],
|
||||
document_tags=["alpha", "beta"],
|
||||
request_context=request_context,
|
||||
)
|
||||
tags = await oracle_memory.list_tags(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
tags = await oracle_memory.list_tags(bank_id=bank_id, request_context=request_context)
|
||||
assert len(tags) > 0
|
||||
finally:
|
||||
await _safe_cleanup(oracle_memory, bank_id, request_context)
|
||||
@@ -994,9 +987,7 @@ class TestAdvancedFeatures:
|
||||
event_date=datetime(2024, 6, 1, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
)
|
||||
ops = await oracle_memory.list_operations(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
ops = await oracle_memory.list_operations(bank_id=bank_id, request_context=request_context)
|
||||
assert ops is not None
|
||||
items = ops.get("items", ops) if isinstance(ops, dict) else ops
|
||||
assert len(items) > 0, "Retain should enqueue at least one task"
|
||||
@@ -1098,9 +1089,7 @@ class TestOracleSpecific:
|
||||
mission="Test JSON CLOB storage",
|
||||
request_context=request_context,
|
||||
)
|
||||
profile = await oracle_memory.get_bank_profile(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
profile = await oracle_memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
|
||||
assert profile is not None
|
||||
assert profile["name"] == "JSON Test Bank"
|
||||
assert profile["mission"] == "Test JSON CLOB storage"
|
||||
@@ -1167,9 +1156,7 @@ class TestEdgeCases:
|
||||
event_date=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
)
|
||||
memories = await oracle_memory.list_memory_units(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
memories = await oracle_memory.list_memory_units(bank_id=bank_id, request_context=request_context)
|
||||
items = memories.get("items", memories) if isinstance(memories, dict) else memories
|
||||
assert len(items) >= 1
|
||||
finally:
|
||||
@@ -1197,19 +1184,13 @@ class TestEdgeCases:
|
||||
event_date=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
)
|
||||
memories = await oracle_memory.list_memory_units(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
memories = await oracle_memory.list_memory_units(bank_id=bank_id, request_context=request_context)
|
||||
items = memories.get("items", memories) if isinstance(memories, dict) else memories
|
||||
# Large content (~10KB, 50 paragraphs) should produce multiple memory units
|
||||
# from LLM fact extraction. At minimum we expect several facts.
|
||||
assert len(items) >= 3, (
|
||||
f"Expected large content to produce at least 3 memory units, got {len(items)}"
|
||||
)
|
||||
assert len(items) >= 3, f"Expected large content to produce at least 3 memory units, got {len(items)}"
|
||||
# Verify operations completed without errors (catches background datetime issues etc.)
|
||||
ops = await oracle_memory.list_operations(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
ops = await oracle_memory.list_operations(bank_id=bank_id, request_context=request_context)
|
||||
if ops:
|
||||
op_list = ops.get("items", ops) if isinstance(ops, dict) else ops
|
||||
failed = [o for o in op_list if isinstance(o, dict) and o.get("status") == "failed"]
|
||||
@@ -1272,19 +1253,13 @@ class TestEdgeCases:
|
||||
# not a code bug. Allow up to 1 deadlock failure.
|
||||
deadlocks = [r for r in results if isinstance(r, Exception) and "ORA-00060" in str(r)]
|
||||
other_failures = [r for r in results if isinstance(r, Exception) and "ORA-00060" not in str(r)]
|
||||
assert len(other_failures) == 0, (
|
||||
f"Non-deadlock failures: {[str(e)[:100] for e in other_failures]}"
|
||||
)
|
||||
assert len(other_failures) == 0, f"Non-deadlock failures: {[str(e)[:100] for e in other_failures]}"
|
||||
successes = len(results) - len(deadlocks)
|
||||
assert successes >= 2, f"Expected at least 2 successful retains, got {successes}"
|
||||
|
||||
memories = await oracle_memory.list_memory_units(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
memories = await oracle_memory.list_memory_units(bank_id=bank_id, request_context=request_context)
|
||||
items = memories.get("items", memories) if isinstance(memories, dict) else memories
|
||||
assert len(items) >= successes, (
|
||||
f"Expected at least {successes} memories, got {len(items)}"
|
||||
)
|
||||
assert len(items) >= successes, f"Expected at least {successes} memories, got {len(items)}"
|
||||
finally:
|
||||
await _safe_cleanup(oracle_memory, bank_id, request_context)
|
||||
|
||||
@@ -1311,9 +1286,7 @@ class TestEdgeCases:
|
||||
async def test_delete_nonexistent_bank(self, oracle_memory: MemoryEngine, request_context: RequestContext):
|
||||
"""Verify deleting a non-existent bank doesn't raise."""
|
||||
# Should not raise an exception
|
||||
await oracle_memory.delete_bank(
|
||||
f"nonexistent-{uuid.uuid4().hex[:8]}", request_context=request_context
|
||||
)
|
||||
await oracle_memory.delete_bank(f"nonexistent-{uuid.uuid4().hex[:8]}", request_context=request_context)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retain_and_delete_cycle(self, oracle_memory: MemoryEngine, request_context: RequestContext):
|
||||
@@ -1369,19 +1342,13 @@ class TestEdgeCases:
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
docs = await oracle_memory.list_documents(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
docs = await oracle_memory.list_documents(bank_id=bank_id, request_context=request_context)
|
||||
items = docs.get("items", docs.get("documents", []))
|
||||
assert len(items) >= 3
|
||||
|
||||
# Delete one document, verify others remain
|
||||
await oracle_memory.delete_document(
|
||||
bank_id=bank_id, document_id="doc-1", request_context=request_context
|
||||
)
|
||||
docs_after = await oracle_memory.list_documents(
|
||||
bank_id=bank_id, request_context=request_context
|
||||
)
|
||||
await oracle_memory.delete_document(bank_id=bank_id, document_id="doc-1", request_context=request_context)
|
||||
docs_after = await oracle_memory.list_documents(bank_id=bank_id, request_context=request_context)
|
||||
items_after = docs_after.get("items", docs_after.get("documents", []))
|
||||
assert len(items_after) >= 2
|
||||
finally:
|
||||
|
||||
@@ -63,9 +63,7 @@ class TestConsolidationBraceSafety:
|
||||
)
|
||||
|
||||
note = "Use shape {limit, used}"
|
||||
prompt = build_batch_consolidation_prompt(
|
||||
observations_mission="m", observation_capacity_note=note
|
||||
)
|
||||
prompt = build_batch_consolidation_prompt(observations_mission="m", observation_capacity_note=note)
|
||||
rendered = prompt.format(facts_text="<facts>", observations_text="<obs>")
|
||||
assert "{limit, used}" in rendered
|
||||
|
||||
|
||||
@@ -24,7 +24,9 @@ def test_provider_default_models():
|
||||
|
||||
config = HindsightConfig.from_env()
|
||||
assert config.llm_provider == provider, f"Provider mismatch for {provider}"
|
||||
assert config.llm_model == expected_model, f"Expected {expected_model} for {provider}, got {config.llm_model}"
|
||||
assert config.llm_model == expected_model, (
|
||||
f"Expected {expected_model} for {provider}, got {config.llm_model}"
|
||||
)
|
||||
|
||||
finally:
|
||||
# Restore original env vars
|
||||
@@ -96,9 +98,9 @@ def test_per_operation_provider_default_model():
|
||||
assert config.llm_model == "gpt-4o-mini", f"Expected gpt-4o-mini, got {config.llm_model}"
|
||||
|
||||
# Retain should use Anthropic default
|
||||
assert (
|
||||
config.retain_llm_model == "claude-haiku-4-5"
|
||||
), f"Expected claude-haiku-4-5, got {config.retain_llm_model}"
|
||||
assert config.retain_llm_model == "claude-haiku-4-5", (
|
||||
f"Expected claude-haiku-4-5, got {config.retain_llm_model}"
|
||||
)
|
||||
|
||||
finally:
|
||||
clear_config_cache()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Test query analyzer for temporal extraction.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
from hindsight_api.engine.query_analyzer import DateparserQueryAnalyzer, QueryAnalysis
|
||||
@@ -312,8 +313,4 @@ def test_query_analyzer_dateparser_crash_returns_no_constraint(query_analyzer, m
|
||||
assert analysis.temporal_constraint is None, (
|
||||
"dateparser failures should be treated as no temporal constraint, not propagated"
|
||||
)
|
||||
assert any("dateparser" in rec.message for rec in caplog.records), (
|
||||
"Should log a warning when dateparser fails"
|
||||
)
|
||||
|
||||
|
||||
assert any("dateparser" in rec.message for rec in caplog.records), "Should log a warning when dateparser fails"
|
||||
|
||||
@@ -26,16 +26,18 @@ async def test_recall_chunks_independent_of_max_tokens(memory, request_context):
|
||||
bank_id = "test-chunks-independence"
|
||||
|
||||
try:
|
||||
|
||||
# Retain some test content with substantial size to generate chunks
|
||||
test_content = """
|
||||
test_content = (
|
||||
"""
|
||||
The quantum computing research team at MIT has made significant breakthroughs.
|
||||
Dr. Sarah Chen leads the team and focuses on quantum error correction.
|
||||
The team published three papers in Nature Physics this year.
|
||||
Their work on topological qubits shows promise for scalable quantum computers.
|
||||
Collaborators include IBM Research and Google Quantum AI.
|
||||
The research is funded by a $5M NSF grant running through 2026.
|
||||
""" * 10 # Repeat to ensure we get multiple chunks
|
||||
"""
|
||||
* 10
|
||||
) # Repeat to ensure we get multiple chunks
|
||||
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
@@ -98,7 +100,6 @@ async def test_recall_chunks_batching_with_varying_sizes(memory, request_context
|
||||
bank_id = "test-chunks-batching"
|
||||
|
||||
try:
|
||||
|
||||
# Retain multiple documents with different content sizes
|
||||
# Document 1: Short content (small chunks)
|
||||
await memory.retain_async(
|
||||
@@ -109,12 +110,15 @@ async def test_recall_chunks_batching_with_varying_sizes(memory, request_context
|
||||
)
|
||||
|
||||
# Document 2: Medium content
|
||||
content_bob = """
|
||||
content_bob = (
|
||||
"""
|
||||
Bob works as a data scientist at a tech startup in San Francisco.
|
||||
He has expertise in natural language processing and computer vision.
|
||||
Bob completed his PhD at Stanford University in 2020.
|
||||
He leads a team of five engineers working on AI-powered recommendation systems.
|
||||
""" * 5
|
||||
"""
|
||||
* 5
|
||||
)
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content=content_bob,
|
||||
@@ -123,14 +127,17 @@ async def test_recall_chunks_batching_with_varying_sizes(memory, request_context
|
||||
)
|
||||
|
||||
# Document 3: Long content (large chunks)
|
||||
content_charlie = """
|
||||
content_charlie = (
|
||||
"""
|
||||
Charlie is the CTO of a growing AI company focused on healthcare applications.
|
||||
He has over 15 years of experience in software architecture and distributed systems.
|
||||
Charlie's team builds machine learning models for medical image analysis and diagnosis.
|
||||
The company recently raised $50 million in Series B funding.
|
||||
They have partnerships with major hospitals in the United States and Europe.
|
||||
Charlie holds several patents in medical imaging and deep learning.
|
||||
""" * 20
|
||||
"""
|
||||
* 20
|
||||
)
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content=content_charlie,
|
||||
@@ -178,7 +185,6 @@ async def test_recall_chunks_ordering_by_relevance(memory, request_context):
|
||||
bank_id = "test-chunks-ordering"
|
||||
|
||||
try:
|
||||
|
||||
# Retain content with different relevance to query
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
@@ -223,8 +229,9 @@ async def test_recall_chunks_ordering_by_relevance(memory, request_context):
|
||||
all_chunk_text = " ".join(chunk.chunk_text for chunk in result.chunks.values())
|
||||
# At least some chunks should mention Python (higher relevance)
|
||||
# This is a soft check since exact ordering depends on scoring
|
||||
assert "Python" in all_chunk_text or "python" in all_chunk_text.lower(), \
|
||||
assert "Python" in all_chunk_text or "python" in all_chunk_text.lower(), (
|
||||
"Chunks should include content about Python (relevant to query)"
|
||||
)
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
@@ -243,13 +250,16 @@ async def test_recall_chunks_for_observations(memory, request_context):
|
||||
|
||||
try:
|
||||
# Retain content that will generate observations via consolidation
|
||||
test_content = """
|
||||
test_content = (
|
||||
"""
|
||||
Alice is a senior software engineer at a large technology company.
|
||||
She specializes in distributed systems and has 10 years of experience.
|
||||
Alice leads a team of 8 engineers working on cloud infrastructure.
|
||||
She holds a PhD in computer science from Stanford University.
|
||||
Alice has published several papers on fault-tolerant distributed systems.
|
||||
""" * 8
|
||||
"""
|
||||
* 8
|
||||
)
|
||||
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
@@ -296,7 +306,6 @@ async def test_recall_chunks_without_include_flag(memory, request_context):
|
||||
bank_id = "test-chunks-no-include"
|
||||
|
||||
try:
|
||||
|
||||
# Retain content
|
||||
test_content = """
|
||||
Sarah is a product manager at a fintech company in New York.
|
||||
@@ -321,8 +330,7 @@ async def test_recall_chunks_without_include_flag(memory, request_context):
|
||||
|
||||
# Should have facts but no chunks
|
||||
assert len(result.results) > 0, "Should return facts"
|
||||
assert result.chunks is None or len(result.chunks) == 0, \
|
||||
"Should NOT return chunks when include_chunks=False"
|
||||
assert result.chunks is None or len(result.chunks) == 0, "Should NOT return chunks when include_chunks=False"
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
|
||||
@@ -54,9 +54,7 @@ class TestToolRecallIncludeChunks:
|
||||
async def test_max_chunk_tokens_propagates(self, mock_request_context):
|
||||
engine = _make_mock_engine()
|
||||
|
||||
await tool_recall(
|
||||
engine, "bank-1", "q", mock_request_context, max_chunk_tokens=2500, max_tokens=512
|
||||
)
|
||||
await tool_recall(engine, "bank-1", "q", mock_request_context, max_chunk_tokens=2500, max_tokens=512)
|
||||
|
||||
kwargs = engine.recall_async.call_args.kwargs
|
||||
assert kwargs["max_chunk_tokens"] == 2500
|
||||
|
||||
@@ -45,9 +45,7 @@ async def test_recall_async_error_preserves_original(memory_no_llm_verify: Memor
|
||||
# the symptom in #1384 was an empty trailer like "Failed to search memories: ".
|
||||
message = str(excinfo.value)
|
||||
assert "Failed to search memories" in message
|
||||
assert "_SilentError" in message, (
|
||||
f"wrapper message dropped the original exception class: {message!r}"
|
||||
)
|
||||
assert "_SilentError" in message, f"wrapper message dropped the original exception class: {message!r}"
|
||||
|
||||
# `from e` chain must be preserved so worker logs / debuggers can walk
|
||||
# back to the real cause.
|
||||
|
||||
@@ -32,14 +32,14 @@ class TestCleanAnswerText:
|
||||
|
||||
def test_clean_text_with_done_call(self):
|
||||
"""Text ending with done() call should have it stripped."""
|
||||
text = '''The team's OKRs focus on performance.done({"answer":"The team's OKRs","memory_ids":[]})'''
|
||||
text = """The team's OKRs focus on performance.done({"answer":"The team's OKRs","memory_ids":[]})"""
|
||||
cleaned = _clean_answer_text(text)
|
||||
assert cleaned == "The team's OKRs focus on performance."
|
||||
assert "done(" not in cleaned
|
||||
|
||||
def test_clean_text_with_done_call_and_whitespace(self):
|
||||
"""done() call with whitespace should be stripped."""
|
||||
text = '''Answer text here. done( {"answer": "short", "memory_ids": []} )'''
|
||||
text = """Answer text here. done( {"answer": "short", "memory_ids": []} )"""
|
||||
cleaned = _clean_answer_text(text)
|
||||
assert cleaned == "Answer text here."
|
||||
|
||||
@@ -61,10 +61,10 @@ class TestCleanAnswerText:
|
||||
|
||||
def test_clean_text_multiline_done(self):
|
||||
"""done() call spanning multiple lines should be stripped."""
|
||||
text = '''Summary of findings.done({
|
||||
text = """Summary of findings.done({
|
||||
"answer": "Summary",
|
||||
"memory_ids": ["id1", "id2"]
|
||||
})'''
|
||||
})"""
|
||||
cleaned = _clean_answer_text(text)
|
||||
assert cleaned == "Summary of findings."
|
||||
|
||||
@@ -74,22 +74,22 @@ class TestCleanDoneAnswer:
|
||||
|
||||
def test_clean_answer_with_leaked_json_code_block(self):
|
||||
"""Answer with leaked JSON code block at the end should be cleaned."""
|
||||
text = '''The user's favorite color is blue.
|
||||
text = """The user's favorite color is blue.
|
||||
|
||||
```json
|
||||
{"observation_ids": ["obs-1", "obs-2"]}
|
||||
```'''
|
||||
```"""
|
||||
cleaned = _clean_done_answer(text)
|
||||
assert cleaned == "The user's favorite color is blue."
|
||||
assert "observation_ids" not in cleaned
|
||||
|
||||
def test_clean_answer_with_memory_ids_code_block(self):
|
||||
"""Answer with leaked memory_ids JSON code block should be cleaned."""
|
||||
text = '''Here is the answer.
|
||||
text = """Here is the answer.
|
||||
|
||||
```json
|
||||
{"memory_ids": ["mem-1"]}
|
||||
```'''
|
||||
```"""
|
||||
cleaned = _clean_done_answer(text)
|
||||
assert cleaned == "Here is the answer."
|
||||
|
||||
@@ -101,13 +101,13 @@ class TestCleanDoneAnswer:
|
||||
|
||||
def test_clean_answer_with_trailing_ids_pattern(self):
|
||||
"""Answer with 'observation_ids: [...]' pattern at the end should be cleaned."""
|
||||
text = "This is the answer.\n\nobservation_ids: [\"obs-1\", \"obs-2\"]"
|
||||
text = 'This is the answer.\n\nobservation_ids: ["obs-1", "obs-2"]'
|
||||
cleaned = _clean_done_answer(text)
|
||||
assert cleaned == "This is the answer."
|
||||
|
||||
def test_clean_answer_with_memory_ids_equals(self):
|
||||
"""Answer with 'memory_ids = [...]' pattern at the end should be cleaned."""
|
||||
text = "Answer text here.\nmemory_ids = [\"mem-1\"]"
|
||||
text = 'Answer text here.\nmemory_ids = ["mem-1"]'
|
||||
cleaned = _clean_done_answer(text)
|
||||
assert cleaned == "Answer text here."
|
||||
|
||||
@@ -129,13 +129,13 @@ class TestCleanDoneAnswer:
|
||||
|
||||
def test_clean_answer_multiline_with_markdown(self):
|
||||
"""Answer with markdown and leaked JSON at end should clean only the leak."""
|
||||
text = '''Summary:
|
||||
text = """Summary:
|
||||
- Point 1
|
||||
- Point 2
|
||||
|
||||
```json
|
||||
{"mental_model_ids": ["mm-1"]}
|
||||
```'''
|
||||
```"""
|
||||
cleaned = _clean_done_answer(text)
|
||||
assert "Point 1" in cleaned
|
||||
assert "Point 2" in cleaned
|
||||
@@ -244,7 +244,10 @@ class TestReflectAgentMocked:
|
||||
llm.call_with_tools = AsyncMock()
|
||||
# Also mock call() for final iteration fallback - returns (response, usage) tuple
|
||||
llm.call = AsyncMock(
|
||||
return_value=("Fallback answer from final iteration", TokenUsage(input_tokens=100, output_tokens=50, total_tokens=150))
|
||||
return_value=(
|
||||
"Fallback answer from final iteration",
|
||||
TokenUsage(input_tokens=100, output_tokens=50, total_tokens=150),
|
||||
)
|
||||
)
|
||||
return llm
|
||||
|
||||
@@ -284,9 +287,7 @@ class TestReflectAgentMocked:
|
||||
self._mm_call(),
|
||||
LLMToolCallResult(
|
||||
tool_calls=[
|
||||
LLMToolCall(
|
||||
id="2", name="done", arguments={"answer": "Be concise.", "mental_model_ids": ["mm-1"]}
|
||||
)
|
||||
LLMToolCall(id="2", name="done", arguments={"answer": "Be concise.", "mental_model_ids": ["mm-1"]})
|
||||
],
|
||||
finish_reason="tool_calls",
|
||||
),
|
||||
@@ -326,7 +327,9 @@ class TestReflectAgentMocked:
|
||||
self._mm_call(),
|
||||
LLMToolCallResult(
|
||||
tool_calls=[
|
||||
LLMToolCall(id="2", name="recall", arguments={"reason": "verify", "query": "launch completion proof"})
|
||||
LLMToolCall(
|
||||
id="2", name="recall", arguments={"reason": "verify", "query": "launch completion proof"}
|
||||
)
|
||||
],
|
||||
finish_reason="tool_calls",
|
||||
),
|
||||
@@ -472,9 +475,7 @@ class TestReflectAgentMocked:
|
||||
finish_reason="tool_calls",
|
||||
),
|
||||
LLMToolCallResult(
|
||||
tool_calls=[
|
||||
LLMToolCall(id="4", name="done", arguments={"answer": "Done.", "memory_ids": ["mem-1"]})
|
||||
],
|
||||
tool_calls=[LLMToolCall(id="4", name="done", arguments={"answer": "Done.", "memory_ids": ["mem-1"]})],
|
||||
finish_reason="tool_calls",
|
||||
),
|
||||
]
|
||||
@@ -679,10 +680,7 @@ class TestReflectAgentMocked:
|
||||
"""
|
||||
# Build a long response that's well over the cap in cl100k_base tokens.
|
||||
long_answer = " ".join(
|
||||
[
|
||||
"This is a detailed paragraph about the team, their roles, and their recurring meetings."
|
||||
]
|
||||
* 80
|
||||
["This is a detailed paragraph about the team, their roles, and their recurring meetings."] * 80
|
||||
)
|
||||
# The short-circuit path: tool_calls empty, content populated.
|
||||
mock_llm.call_with_tools.return_value = LLMToolCallResult(
|
||||
@@ -715,8 +713,7 @@ class TestReflectAgentMocked:
|
||||
)
|
||||
rewrite_kwargs = mock_llm.call.await_args.kwargs
|
||||
assert rewrite_kwargs.get("max_completion_tokens") == cap, (
|
||||
f"rewrite call should use max_completion_tokens={cap}, "
|
||||
f"got {rewrite_kwargs.get('max_completion_tokens')}"
|
||||
f"rewrite call should use max_completion_tokens={cap}, got {rewrite_kwargs.get('max_completion_tokens')}"
|
||||
)
|
||||
|
||||
# The final answer is the rewritten text, not the oversized original.
|
||||
@@ -824,7 +821,14 @@ class TestContextOverflowHelpers:
|
||||
"role": "tool",
|
||||
"tool_call_id": "x",
|
||||
"name": "recall",
|
||||
"content": '{"memories": [' + ', '.join([f'{{"id": "m{i}", "content": "A long memory fact about some topic that goes on and on."}}' for i in range(50)]) + ']}',
|
||||
"content": '{"memories": ['
|
||||
+ ", ".join(
|
||||
[
|
||||
f'{{"id": "m{i}", "content": "A long memory fact about some topic that goes on and on."}}'
|
||||
for i in range(50)
|
||||
]
|
||||
)
|
||||
+ "]}",
|
||||
},
|
||||
]
|
||||
small = _count_messages_tokens(small_messages)
|
||||
@@ -833,7 +837,11 @@ class TestContextOverflowHelpers:
|
||||
|
||||
def test_is_context_overflow_error_openai(self):
|
||||
assert _is_context_overflow_error(Exception("context_length_exceeded: too many tokens"))
|
||||
assert _is_context_overflow_error(Exception("This model's maximum context length is 128000 tokens. However, your messages resulted in 142164 tokens."))
|
||||
assert _is_context_overflow_error(
|
||||
Exception(
|
||||
"This model's maximum context length is 128000 tokens. However, your messages resulted in 142164 tokens."
|
||||
)
|
||||
)
|
||||
|
||||
def test_is_context_overflow_error_anthropic(self):
|
||||
assert _is_context_overflow_error(Exception("prompt_too_long"))
|
||||
@@ -860,17 +868,17 @@ class TestContextOverflowBehavior:
|
||||
llm = MagicMock()
|
||||
llm.call_with_tools = AsyncMock()
|
||||
llm.call = AsyncMock(
|
||||
return_value=("Synthesized answer from gathered evidence.", TokenUsage(input_tokens=50, output_tokens=20, total_tokens=70))
|
||||
return_value=(
|
||||
"Synthesized answer from gathered evidence.",
|
||||
TokenUsage(input_tokens=50, output_tokens=20, total_tokens=70),
|
||||
)
|
||||
)
|
||||
return llm
|
||||
|
||||
@pytest.fixture
|
||||
def mock_functions_with_large_output(self):
|
||||
"""Mock functions that return a large enough payload to exceed a tiny token budget."""
|
||||
large_memories = [
|
||||
{"id": f"mem-{i}", "content": f"Memory fact number {i}: " + "A" * 200}
|
||||
for i in range(20)
|
||||
]
|
||||
large_memories = [{"id": f"mem-{i}", "content": f"Memory fact number {i}: " + "A" * 200} for i in range(20)]
|
||||
return {
|
||||
"search_mental_models_fn": AsyncMock(return_value={"mental_models": []}),
|
||||
"search_observations_fn": AsyncMock(return_value={"observations": []}),
|
||||
@@ -910,9 +918,7 @@ class TestContextOverflowBehavior:
|
||||
async def test_context_overflow_error_skips_retry(self, mock_llm, mock_functions_with_large_output):
|
||||
"""A context_length_exceeded error from the LLM should NOT be retried —
|
||||
it should immediately fall back to final synthesis."""
|
||||
mock_llm.call_with_tools.side_effect = Exception(
|
||||
"context_length_exceeded: messages resulted in 150000 tokens."
|
||||
)
|
||||
mock_llm.call_with_tools.side_effect = Exception("context_length_exceeded: messages resulted in 150000 tokens.")
|
||||
|
||||
result = await run_reflect_agent(
|
||||
llm_config=mock_llm,
|
||||
@@ -969,8 +975,7 @@ class TestDirectiveLeakageOnEmptyBank:
|
||||
|
||||
# The directive content must NOT leak into the answer.
|
||||
assert directive_text not in result.text, (
|
||||
f"Directive content leaked into the answer verbatim. "
|
||||
f"Got: {result.text!r}"
|
||||
f"Directive content leaked into the answer verbatim. Got: {result.text!r}"
|
||||
)
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
@@ -1023,6 +1028,7 @@ class TestContextOverflowIntegration:
|
||||
class _TinyContextProxy:
|
||||
"""Forwards all attribute access to the real config proxy except
|
||||
reflect_max_context_tokens which is forced to 1."""
|
||||
|
||||
_real = _real_get_config()
|
||||
|
||||
def __getattr__(self, name: str):
|
||||
|
||||
@@ -34,8 +34,8 @@ async def test_reflect_with_no_memories_empty_bank(api_client):
|
||||
"budget": "low",
|
||||
"include": {
|
||||
"facts": {} # Request facts but bank is empty
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -43,10 +43,11 @@ async def test_reflect_with_no_memories_empty_bank(api_client):
|
||||
|
||||
# DEBUG: Print what the API actually returned
|
||||
import json
|
||||
print("\n" + "="*80)
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("API Response:")
|
||||
print(json.dumps(data, indent=2))
|
||||
print("="*80 + "\n")
|
||||
print("=" * 80 + "\n")
|
||||
|
||||
# Verify response structure
|
||||
assert "text" in data
|
||||
@@ -87,9 +88,9 @@ async def test_reflect_without_include_facts(api_client):
|
||||
f"/v1/default/banks/{bank_id}/reflect",
|
||||
json={
|
||||
"query": "Hello world",
|
||||
"budget": "low"
|
||||
"budget": "low",
|
||||
# No include.facts
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -30,9 +30,7 @@ class _FakeRequestContext:
|
||||
|
||||
def _mock_engine():
|
||||
engine = MagicMock()
|
||||
engine.recall_async = AsyncMock(
|
||||
return_value=RecallResult(results=[], source_facts={})
|
||||
)
|
||||
engine.recall_async = AsyncMock(return_value=RecallResult(results=[], source_facts={}))
|
||||
return engine
|
||||
|
||||
|
||||
|
||||
@@ -386,9 +386,7 @@ class TestBudgetBranches:
|
||||
include_observations=False,
|
||||
budget="low",
|
||||
)
|
||||
assert actual == _assemble(
|
||||
_RETRIEVAL_RECALL_ONLY, _WORKFLOW_RECALL_ONLY, budget=_BUDGET_LOW
|
||||
)
|
||||
assert actual == _assemble(_RETRIEVAL_RECALL_ONLY, _WORKFLOW_RECALL_ONLY, budget=_BUDGET_LOW)
|
||||
|
||||
def test_budget_mid_inserts_moderate_block(self):
|
||||
actual = build_system_prompt_for_tools(
|
||||
@@ -397,9 +395,7 @@ class TestBudgetBranches:
|
||||
include_observations=False,
|
||||
budget="mid",
|
||||
)
|
||||
assert actual == _assemble(
|
||||
_RETRIEVAL_RECALL_ONLY, _WORKFLOW_RECALL_ONLY, budget=_BUDGET_MID
|
||||
)
|
||||
assert actual == _assemble(_RETRIEVAL_RECALL_ONLY, _WORKFLOW_RECALL_ONLY, budget=_BUDGET_MID)
|
||||
|
||||
def test_budget_high_inserts_deep_block(self):
|
||||
actual = build_system_prompt_for_tools(
|
||||
@@ -408,9 +404,7 @@ class TestBudgetBranches:
|
||||
include_observations=False,
|
||||
budget="high",
|
||||
)
|
||||
assert actual == _assemble(
|
||||
_RETRIEVAL_RECALL_ONLY, _WORKFLOW_RECALL_ONLY, budget=_BUDGET_HIGH
|
||||
)
|
||||
assert actual == _assemble(_RETRIEVAL_RECALL_ONLY, _WORKFLOW_RECALL_ONLY, budget=_BUDGET_HIGH)
|
||||
|
||||
def test_unknown_budget_inserts_nothing(self):
|
||||
# The builder only recognises low/mid/high; any other value is a no-op.
|
||||
@@ -544,7 +538,5 @@ def test_include_observations_defaults_to_true():
|
||||
"""Callers that don't pass ``include_observations`` get the original
|
||||
observations-enabled prompt — this guards the API default so reflect
|
||||
paths that don't gate the flag aren't silently changed."""
|
||||
actual = build_system_prompt_for_tools(
|
||||
bank_profile=BANK, has_mental_models=False
|
||||
)
|
||||
actual = build_system_prompt_for_tools(bank_profile=BANK, has_mental_models=False)
|
||||
assert actual == _assemble(_RETRIEVAL_OBS_ONLY, _WORKFLOW_OBS_ONLY)
|
||||
|
||||
@@ -40,9 +40,7 @@ class TestSearchObservationsSourceFacts:
|
||||
"""Default source_facts_max_tokens=-1 should disable source facts."""
|
||||
engine = _make_mock_engine()
|
||||
|
||||
await tool_search_observations(
|
||||
engine, "bank-1", "test query", mock_request_context
|
||||
)
|
||||
await tool_search_observations(engine, "bank-1", "test query", mock_request_context)
|
||||
|
||||
engine.recall_async.assert_called_once()
|
||||
call_kwargs = engine.recall_async.call_args.kwargs
|
||||
@@ -55,7 +53,10 @@ class TestSearchObservationsSourceFacts:
|
||||
engine = _make_mock_engine()
|
||||
|
||||
await tool_search_observations(
|
||||
engine, "bank-1", "test query", mock_request_context,
|
||||
engine,
|
||||
"bank-1",
|
||||
"test query",
|
||||
mock_request_context,
|
||||
source_facts_max_tokens=0,
|
||||
)
|
||||
|
||||
@@ -70,7 +71,10 @@ class TestSearchObservationsSourceFacts:
|
||||
engine = _make_mock_engine()
|
||||
|
||||
await tool_search_observations(
|
||||
engine, "bank-1", "test query", mock_request_context,
|
||||
engine,
|
||||
"bank-1",
|
||||
"test query",
|
||||
mock_request_context,
|
||||
source_facts_max_tokens=5000,
|
||||
)
|
||||
|
||||
@@ -85,7 +89,10 @@ class TestSearchObservationsSourceFacts:
|
||||
engine = _make_mock_engine()
|
||||
|
||||
await tool_search_observations(
|
||||
engine, "bank-1", "test query", mock_request_context,
|
||||
engine,
|
||||
"bank-1",
|
||||
"test query",
|
||||
mock_request_context,
|
||||
source_facts_max_tokens=-1,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Test to verify reflect operation creates proper span hierarchy.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -11,11 +12,7 @@ async def test_reflect_creates_child_spans(memory, request_context):
|
||||
from hindsight_api.tracing import initialize_tracing, get_span_recorder, create_span_recorder
|
||||
|
||||
# Initialize tracing with a mock endpoint
|
||||
initialize_tracing(
|
||||
service_name="test-hindsight",
|
||||
endpoint="http://localhost:4318",
|
||||
deployment_environment="test"
|
||||
)
|
||||
initialize_tracing(service_name="test-hindsight", endpoint="http://localhost:4318", deployment_environment="test")
|
||||
|
||||
# Create span recorder
|
||||
recorder = create_span_recorder()
|
||||
|
||||
@@ -23,6 +23,7 @@ from hindsight_api.engine.search.types import MergedCandidate, RetrievalResult
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_candidates(n: int) -> list[MergedCandidate]:
|
||||
"""Create *n* minimal MergedCandidate objects."""
|
||||
candidates = []
|
||||
@@ -34,9 +35,7 @@ def _make_candidates(n: int) -> list[MergedCandidate]:
|
||||
occurred_start=None,
|
||||
occurred_end=None,
|
||||
)
|
||||
candidates.append(
|
||||
MergedCandidate(retrieval=retrieval, rrf_score=1.0 / (i + 1))
|
||||
)
|
||||
candidates.append(MergedCandidate(retrieval=retrieval, rrf_score=1.0 / (i + 1)))
|
||||
return candidates
|
||||
|
||||
|
||||
@@ -53,6 +52,7 @@ def _make_cross_encoder(predict_return: list[float]):
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_passthrough_for_0_1_scores():
|
||||
"""Scores already in [0, 1] should be passed through as-is."""
|
||||
|
||||
@@ -11,6 +11,7 @@ from hindsight_api.engine.search.reranking import apply_combined_scoring
|
||||
|
||||
UTC = timezone.utc
|
||||
|
||||
|
||||
def create_mock_scored_result(proof_count: int | None = None, ce_score: float = 0.8) -> ScoredResult:
|
||||
"""Helper to create a minimal ScoredResult suitable for scoring tests."""
|
||||
retrieval = RetrievalResult(
|
||||
@@ -22,7 +23,7 @@ def create_mock_scored_result(proof_count: int | None = None, ce_score: float =
|
||||
proof_count=proof_count,
|
||||
# Use None for neutral recency so only proof_count changes score
|
||||
occurred_start=None,
|
||||
occurred_end=None
|
||||
occurred_end=None,
|
||||
)
|
||||
candidate = MergedCandidate(
|
||||
retrieval=retrieval,
|
||||
@@ -35,57 +36,60 @@ def create_mock_scored_result(proof_count: int | None = None, ce_score: float =
|
||||
weight=ce_score,
|
||||
)
|
||||
|
||||
|
||||
def test_proof_count_neutral_when_none():
|
||||
"""Test that when proof_count is None (e.g. non-observation), it gets neutral 0.5 norm."""
|
||||
sr = create_mock_scored_result(proof_count=None, ce_score=0.8)
|
||||
now = datetime.now(UTC)
|
||||
|
||||
|
||||
apply_combined_scoring([sr], now, proof_count_alpha=0.1)
|
||||
|
||||
|
||||
# Neutral multiplier means score shouldn't be boosted by proof_count
|
||||
# Since recency is neutral (just created) and temporal is neutral, score should remain unchanged
|
||||
assert sr.combined_score == pytest.approx(0.8, rel=1e-3)
|
||||
|
||||
|
||||
def test_proof_count_neutral_at_one():
|
||||
"""Test that proof_count=1 gives neutral multiplier."""
|
||||
sr = create_mock_scored_result(proof_count=1, ce_score=0.8)
|
||||
now = datetime.now(UTC)
|
||||
|
||||
|
||||
apply_combined_scoring([sr], now, proof_count_alpha=0.1)
|
||||
|
||||
|
||||
# proof_count=1 -> math.log(1) = 0 -> 0.5 + 0/10 = 0.5 (neutral) -> multiplier 1.0
|
||||
assert sr.combined_score == pytest.approx(0.8, rel=1e-3)
|
||||
|
||||
|
||||
def test_proof_count_increases_with_higher_counts():
|
||||
"""Test that higher proof counts yield strictly higher scores."""
|
||||
now = datetime.now(UTC)
|
||||
|
||||
|
||||
# Create results with increasing proof counts
|
||||
sr_5 = create_mock_scored_result(proof_count=5, ce_score=0.8)
|
||||
sr_50 = create_mock_scored_result(proof_count=50, ce_score=0.8)
|
||||
sr_100 = create_mock_scored_result(proof_count=100, ce_score=0.8)
|
||||
|
||||
|
||||
# Process them
|
||||
apply_combined_scoring([sr_5, sr_50, sr_100], now, proof_count_alpha=0.1)
|
||||
|
||||
|
||||
# Assure scores strictly increase
|
||||
assert sr_5.combined_score > 0.8
|
||||
assert sr_50.combined_score > sr_5.combined_score
|
||||
assert sr_100.combined_score > sr_50.combined_score
|
||||
|
||||
|
||||
def test_proof_count_no_hardcoded_cap_at_100():
|
||||
"""Test that proof_count continues to scale within the clamped [0, 1] range."""
|
||||
now = datetime.now(UTC)
|
||||
|
||||
|
||||
# Use values that stay below the clamp ceiling (proof_norm < 1.0)
|
||||
# log(5)/10=0.16, log(20)/10=0.30, log(100)/10=0.46 → all below 0.5 headroom
|
||||
sr_5 = create_mock_scored_result(proof_count=5, ce_score=0.8)
|
||||
sr_20 = create_mock_scored_result(proof_count=20, ce_score=0.8)
|
||||
sr_100 = create_mock_scored_result(proof_count=100, ce_score=0.8)
|
||||
|
||||
|
||||
apply_combined_scoring([sr_5, sr_20, sr_100], now, proof_count_alpha=0.1)
|
||||
|
||||
|
||||
# Must strictly increase within the valid range
|
||||
assert sr_20.combined_score > sr_5.combined_score
|
||||
assert sr_100.combined_score > sr_20.combined_score
|
||||
|
||||
|
||||
@@ -107,10 +107,12 @@ class TestSchemaIsolation:
|
||||
await conn.close()
|
||||
|
||||
# Configure tenant extension that provisions schemas via run_migrations
|
||||
tenant_ext = MultiSchemaTestTenantExtension({
|
||||
"db_url": pg0_db_url,
|
||||
"valid_schemas": set(schemas),
|
||||
})
|
||||
tenant_ext = MultiSchemaTestTenantExtension(
|
||||
{
|
||||
"db_url": pg0_db_url,
|
||||
"valid_schemas": set(schemas),
|
||||
}
|
||||
)
|
||||
memory._tenant_extension = tenant_ext
|
||||
|
||||
# Define concurrent insert tasks for each tenant
|
||||
@@ -129,7 +131,7 @@ class TestSchemaIsolation:
|
||||
for i in range(3):
|
||||
await conn.execute(
|
||||
f"""
|
||||
INSERT INTO {fq_table('memory_units')} (bank_id, text, event_date, fact_type)
|
||||
INSERT INTO {fq_table("memory_units")} (bank_id, text, event_date, fact_type)
|
||||
VALUES ($1, $2, now(), 'world')
|
||||
""",
|
||||
bank_id,
|
||||
@@ -155,9 +157,7 @@ class TestSchemaIsolation:
|
||||
|
||||
# All texts should contain the schema's marker
|
||||
for text in texts:
|
||||
assert f"MARKER_{prefix}" in text, (
|
||||
f"Memory in {schema} missing its marker: {text}"
|
||||
)
|
||||
assert f"MARKER_{prefix}" in text, f"Memory in {schema} missing its marker: {text}"
|
||||
|
||||
# Should NOT contain other tenants' markers
|
||||
other_prefixes = ["ALPHA", "BETA", "GAMMA"]
|
||||
@@ -265,10 +265,12 @@ class TestSchemaIsolation:
|
||||
await conn.close()
|
||||
|
||||
# Configure tenant extension
|
||||
tenant_ext = MultiSchemaTestTenantExtension({
|
||||
"db_url": pg0_db_url,
|
||||
"valid_schemas": set(schemas),
|
||||
})
|
||||
tenant_ext = MultiSchemaTestTenantExtension(
|
||||
{
|
||||
"db_url": pg0_db_url,
|
||||
"valid_schemas": set(schemas),
|
||||
}
|
||||
)
|
||||
memory._tenant_extension = tenant_ext
|
||||
|
||||
try:
|
||||
@@ -333,10 +335,12 @@ class TestSchemaIsolation:
|
||||
run_migrations(pg0_db_url, schema=schema)
|
||||
|
||||
# Configure tenant extension (schemas already provisioned)
|
||||
tenant_ext = MultiSchemaTestTenantExtension({
|
||||
"db_url": pg0_db_url,
|
||||
"valid_schemas": set(schemas),
|
||||
})
|
||||
tenant_ext = MultiSchemaTestTenantExtension(
|
||||
{
|
||||
"db_url": pg0_db_url,
|
||||
"valid_schemas": set(schemas),
|
||||
}
|
||||
)
|
||||
# Mark schemas as already provisioned so extension doesn't re-run migrations
|
||||
tenant_ext._provisioned = set(schemas)
|
||||
memory._tenant_extension = tenant_ext
|
||||
@@ -357,7 +361,7 @@ class TestSchemaIsolation:
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
await conn.execute(
|
||||
f"""
|
||||
INSERT INTO {fq_table('memory_units')} (bank_id, text, event_date, fact_type)
|
||||
INSERT INTO {fq_table("memory_units")} (bank_id, text, event_date, fact_type)
|
||||
VALUES ($1, $2, now(), 'world')
|
||||
""",
|
||||
bank_id,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Test search tracing functionality.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
@@ -33,7 +34,6 @@ async def test_search_with_trace(memory, request_context):
|
||||
bank_id = f"test_trace_{datetime.now(timezone.utc).timestamp()}"
|
||||
|
||||
try:
|
||||
|
||||
# Store some test memories
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
@@ -59,7 +59,7 @@ async def test_search_with_trace(memory, request_context):
|
||||
bank_id=bank_id,
|
||||
query="Who works at Google?",
|
||||
fact_type=["world"],
|
||||
budget=Budget.LOW, # 20,
|
||||
budget=Budget.LOW, # 20,
|
||||
max_tokens=512,
|
||||
enable_trace=True,
|
||||
request_context=request_context,
|
||||
@@ -134,7 +134,6 @@ async def test_search_without_trace(memory, request_context):
|
||||
bank_id = f"test_no_trace_{datetime.now(timezone.utc).timestamp()}"
|
||||
|
||||
try:
|
||||
|
||||
# Store a test memory
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
@@ -148,7 +147,7 @@ async def test_search_without_trace(memory, request_context):
|
||||
bank_id=bank_id,
|
||||
query="test",
|
||||
fact_type=["world"],
|
||||
budget=Budget.LOW, # 10,
|
||||
budget=Budget.LOW, # 10,
|
||||
max_tokens=512,
|
||||
enable_trace=False,
|
||||
request_context=request_context,
|
||||
|
||||
@@ -53,12 +53,13 @@ class TestServerModuleExtensionLoading:
|
||||
|
||||
# Patch at source level BEFORE importing server
|
||||
# Note: We patch the entire hindsight_api module namespace
|
||||
with patch("hindsight_api.MemoryEngine") as mock_engine, \
|
||||
patch("hindsight_api.api.create_app") as mock_create_app, \
|
||||
patch("hindsight_api.config.get_config") as mock_get_config, \
|
||||
patch("hindsight_api.extensions.load_extension", side_effect=tracking_load_extension), \
|
||||
patch("hindsight_api.extensions.DefaultExtensionContext"):
|
||||
|
||||
with (
|
||||
patch("hindsight_api.MemoryEngine") as mock_engine,
|
||||
patch("hindsight_api.api.create_app") as mock_create_app,
|
||||
patch("hindsight_api.config.get_config") as mock_get_config,
|
||||
patch("hindsight_api.extensions.load_extension", side_effect=tracking_load_extension),
|
||||
patch("hindsight_api.extensions.DefaultExtensionContext"),
|
||||
):
|
||||
mock_config = MagicMock()
|
||||
mock_config.mcp_enabled = False
|
||||
mock_config.run_migrations_on_startup = False
|
||||
@@ -71,12 +72,15 @@ class TestServerModuleExtensionLoading:
|
||||
import hindsight_api.server
|
||||
|
||||
# Verify TENANT extension was loaded
|
||||
assert "TENANT" in loaded_extensions, \
|
||||
assert "TENANT" in loaded_extensions, (
|
||||
"server.py did not call load_extension('TENANT', ...) - extensions not loaded!"
|
||||
assert loaded_extensions["TENANT"] is not None, \
|
||||
)
|
||||
assert loaded_extensions["TENANT"] is not None, (
|
||||
"load_extension('TENANT', ...) returned None despite env var being set"
|
||||
assert isinstance(loaded_extensions["TENANT"], MockTenantExtension), \
|
||||
)
|
||||
assert isinstance(loaded_extensions["TENANT"], MockTenantExtension), (
|
||||
f"Expected MockTenantExtension, got {type(loaded_extensions['TENANT'])}"
|
||||
)
|
||||
|
||||
def test_server_loads_operation_validator_when_configured(self, monkeypatch):
|
||||
"""
|
||||
@@ -98,12 +102,13 @@ class TestServerModuleExtensionLoading:
|
||||
loaded_extensions[name] = result
|
||||
return result
|
||||
|
||||
with patch("hindsight_api.MemoryEngine") as mock_engine, \
|
||||
patch("hindsight_api.api.create_app") as mock_create_app, \
|
||||
patch("hindsight_api.config.get_config") as mock_get_config, \
|
||||
patch("hindsight_api.extensions.load_extension", side_effect=tracking_load_extension), \
|
||||
patch("hindsight_api.extensions.DefaultExtensionContext"):
|
||||
|
||||
with (
|
||||
patch("hindsight_api.MemoryEngine") as mock_engine,
|
||||
patch("hindsight_api.api.create_app") as mock_create_app,
|
||||
patch("hindsight_api.config.get_config") as mock_get_config,
|
||||
patch("hindsight_api.extensions.load_extension", side_effect=tracking_load_extension),
|
||||
patch("hindsight_api.extensions.DefaultExtensionContext"),
|
||||
):
|
||||
mock_config = MagicMock()
|
||||
mock_config.mcp_enabled = False
|
||||
mock_config.run_migrations_on_startup = False
|
||||
@@ -114,8 +119,9 @@ class TestServerModuleExtensionLoading:
|
||||
|
||||
import hindsight_api.server
|
||||
|
||||
assert "OPERATION_VALIDATOR" in loaded_extensions, \
|
||||
assert "OPERATION_VALIDATOR" in loaded_extensions, (
|
||||
"server.py did not call load_extension('OPERATION_VALIDATOR', ...)"
|
||||
)
|
||||
assert loaded_extensions["OPERATION_VALIDATOR"] is not None
|
||||
assert isinstance(loaded_extensions["OPERATION_VALIDATOR"], MockOperationValidator)
|
||||
|
||||
@@ -139,11 +145,12 @@ class TestServerModuleExtensionLoading:
|
||||
memory_engine_calls.append({"args": args, "kwargs": kwargs})
|
||||
return MagicMock()
|
||||
|
||||
with patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine), \
|
||||
patch("hindsight_api.api.create_app") as mock_create_app, \
|
||||
patch("hindsight_api.config.get_config") as mock_get_config, \
|
||||
patch("hindsight_api.extensions.DefaultExtensionContext"):
|
||||
|
||||
with (
|
||||
patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine),
|
||||
patch("hindsight_api.api.create_app") as mock_create_app,
|
||||
patch("hindsight_api.config.get_config") as mock_get_config,
|
||||
patch("hindsight_api.extensions.DefaultExtensionContext"),
|
||||
):
|
||||
mock_config = MagicMock()
|
||||
mock_config.mcp_enabled = False
|
||||
mock_config.run_migrations_on_startup = False
|
||||
@@ -159,10 +166,10 @@ class TestServerModuleExtensionLoading:
|
||||
call_kwargs = memory_engine_calls[0]["kwargs"]
|
||||
|
||||
# THE CRITICAL ASSERTION: tenant_extension must be passed and not None
|
||||
assert "tenant_extension" in call_kwargs, \
|
||||
"MemoryEngine was not called with tenant_extension parameter!"
|
||||
assert call_kwargs["tenant_extension"] is not None, \
|
||||
assert "tenant_extension" in call_kwargs, "MemoryEngine was not called with tenant_extension parameter!"
|
||||
assert call_kwargs["tenant_extension"] is not None, (
|
||||
"tenant_extension was None - server.py did not pass loaded extension to MemoryEngine!"
|
||||
)
|
||||
|
||||
def test_server_sets_extension_context_on_tenant_extension(self, monkeypatch):
|
||||
"""
|
||||
@@ -189,11 +196,12 @@ class TestServerModuleExtensionLoading:
|
||||
context_set_calls.append(ctx)
|
||||
return ctx
|
||||
|
||||
with patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine), \
|
||||
patch("hindsight_api.api.create_app") as mock_create_app, \
|
||||
patch("hindsight_api.config.get_config") as mock_get_config, \
|
||||
patch("hindsight_api.extensions.DefaultExtensionContext", side_effect=capture_context):
|
||||
|
||||
with (
|
||||
patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine),
|
||||
patch("hindsight_api.api.create_app") as mock_create_app,
|
||||
patch("hindsight_api.config.get_config") as mock_get_config,
|
||||
patch("hindsight_api.extensions.DefaultExtensionContext", side_effect=capture_context),
|
||||
):
|
||||
mock_config = MagicMock()
|
||||
mock_config.mcp_enabled = False
|
||||
mock_config.run_migrations_on_startup = False
|
||||
@@ -206,8 +214,7 @@ class TestServerModuleExtensionLoading:
|
||||
# Verify context was created and set
|
||||
assert len(context_set_calls) == 1, "DefaultExtensionContext should be created"
|
||||
assert captured_tenant_ext[0] is not None, "Tenant extension should be captured"
|
||||
assert captured_tenant_ext[0]._context_set, \
|
||||
"set_context was not called on tenant extension"
|
||||
assert captured_tenant_ext[0]._context_set, "set_context was not called on tenant extension"
|
||||
|
||||
def test_server_works_without_extensions(self, monkeypatch):
|
||||
"""
|
||||
@@ -225,10 +232,11 @@ class TestServerModuleExtensionLoading:
|
||||
memory_engine_calls.append({"args": args, "kwargs": kwargs})
|
||||
return MagicMock()
|
||||
|
||||
with patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine), \
|
||||
patch("hindsight_api.api.create_app") as mock_create_app, \
|
||||
patch("hindsight_api.config.get_config") as mock_get_config:
|
||||
|
||||
with (
|
||||
patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine),
|
||||
patch("hindsight_api.api.create_app") as mock_create_app,
|
||||
patch("hindsight_api.config.get_config") as mock_get_config,
|
||||
):
|
||||
mock_config = MagicMock()
|
||||
mock_config.mcp_enabled = False
|
||||
mock_config.run_migrations_on_startup = False
|
||||
|
||||
@@ -96,7 +96,7 @@ class TestRecallSourceFactsPerObservationCap:
|
||||
fact_type=["observation"],
|
||||
max_tokens=4096,
|
||||
include_source_facts=True,
|
||||
max_source_facts_tokens=4096, # large global budget
|
||||
max_source_facts_tokens=4096, # large global budget
|
||||
max_source_facts_tokens_per_observation=512, # reasonable per-obs limit
|
||||
budget=Budget.MID,
|
||||
request_context=request_context,
|
||||
|
||||
@@ -93,9 +93,7 @@ def find_unqualified_table_refs(content: str, filename: str) -> list[tuple[int,
|
||||
qualified_pattern = rf"\.\s*{table}(?:\s|$|,|\))"
|
||||
fq_table_pattern = rf'fq_table\s*\(\s*["\']?{table}'
|
||||
|
||||
if not re.search(qualified_pattern, line) and not re.search(
|
||||
fq_table_pattern, line
|
||||
):
|
||||
if not re.search(qualified_pattern, line) and not re.search(fq_table_pattern, line):
|
||||
# Additional check: line must have SQL indicators
|
||||
# This avoids false positives in docstrings like "split into chunks"
|
||||
if sql_indicators.search(line):
|
||||
@@ -117,8 +115,7 @@ class TestSQLSchemaSafety:
|
||||
|
||||
for line_num, table, line in violations:
|
||||
all_violations.append(
|
||||
f"{py_file.relative_to(py_file.parent.parent)}:{line_num} - "
|
||||
f"unqualified '{table}': {line[:80]}..."
|
||||
f"{py_file.relative_to(py_file.parent.parent)}:{line_num} - unqualified '{table}': {line[:80]}..."
|
||||
)
|
||||
|
||||
if all_violations:
|
||||
|
||||
@@ -127,9 +127,7 @@ class TestRenderer:
|
||||
assert render_block(block) == "```\nraw text\n```"
|
||||
|
||||
def test_section_heading_level(self):
|
||||
section = Section(
|
||||
id="purpose", heading="Purpose", level=3, blocks=[ParagraphBlock(text="hi")]
|
||||
)
|
||||
section = Section(id="purpose", heading="Purpose", level=3, blocks=[ParagraphBlock(text="hi")])
|
||||
assert render_section(section).startswith("### Purpose\n\nhi")
|
||||
|
||||
def test_document_round_trip_is_stable(self):
|
||||
@@ -157,18 +155,7 @@ class TestRenderer:
|
||||
class TestParser:
|
||||
def test_simple_document(self):
|
||||
markdown = (
|
||||
"# Team Overview\n"
|
||||
"\n"
|
||||
"Quick summary.\n"
|
||||
"\n"
|
||||
"## Members\n"
|
||||
"\n"
|
||||
"- Alice\n"
|
||||
"- Bob\n"
|
||||
"\n"
|
||||
"## Cadence\n"
|
||||
"\n"
|
||||
"Standups daily.\n"
|
||||
"# Team Overview\n\nQuick summary.\n\n## Members\n\n- Alice\n- Bob\n\n## Cadence\n\nStandups daily.\n"
|
||||
)
|
||||
doc = parse_markdown(markdown)
|
||||
assert [s.id for s in doc.sections] == ["team-overview", "members", "cadence"]
|
||||
@@ -182,11 +169,7 @@ class TestParser:
|
||||
doc = parse_markdown(markdown)
|
||||
assert [s.id for s in doc.sections] == ["rules", "stop"]
|
||||
# Horizontal rule must NOT become a paragraph.
|
||||
assert all(
|
||||
not (isinstance(b, ParagraphBlock) and "---" in b.text)
|
||||
for s in doc.sections
|
||||
for b in s.blocks
|
||||
)
|
||||
assert all(not (isinstance(b, ParagraphBlock) and "---" in b.text) for s in doc.sections for b in s.blocks)
|
||||
|
||||
def test_ordered_list(self):
|
||||
markdown = "## Steps\n\n1. one\n2. two\n3. three\n"
|
||||
@@ -275,9 +258,7 @@ class TestApplyOperations:
|
||||
|
||||
def test_insert_block_out_of_range_skipped(self):
|
||||
doc = _team_overview_doc()
|
||||
op = InsertBlockOp(
|
||||
section_id="members", index=99, block=ParagraphBlock(text="x")
|
||||
)
|
||||
op = InsertBlockOp(section_id="members", index=99, block=ParagraphBlock(text="x"))
|
||||
result = apply_operations(doc, [op])
|
||||
assert result.applied == []
|
||||
assert "index out of range" in result.skipped[0]["reason"]
|
||||
@@ -381,13 +362,9 @@ class TestApplyOperations:
|
||||
)
|
||||
result = apply_operations(doc, [op])
|
||||
before_overview = render_section(doc.section_by_id("team-overview"))
|
||||
after_overview = render_section(
|
||||
result.document.section_by_id("team-overview")
|
||||
)
|
||||
after_overview = render_section(result.document.section_by_id("team-overview"))
|
||||
before_cadence = render_section(doc.section_by_id("cadence"))
|
||||
after_cadence = render_section(
|
||||
result.document.section_by_id("cadence")
|
||||
)
|
||||
after_cadence = render_section(result.document.section_by_id("cadence"))
|
||||
assert before_overview == after_overview
|
||||
assert before_cadence == after_cadence
|
||||
|
||||
@@ -418,9 +395,7 @@ class TestDeltaOperationListSchema:
|
||||
|
||||
def test_invalid_op_field_rejected(self):
|
||||
with pytest.raises(Exception): # pydantic ValidationError
|
||||
DeltaOperationList.model_validate(
|
||||
{"operations": [{"op": "not_a_real_op", "section_id": "x"}]}
|
||||
)
|
||||
DeltaOperationList.model_validate({"operations": [{"op": "not_a_real_op", "section_id": "x"}]})
|
||||
|
||||
def test_extra_field_rejected(self):
|
||||
with pytest.raises(Exception):
|
||||
|
||||
@@ -9,6 +9,7 @@ Use cases:
|
||||
|
||||
The tags use OR-based matching: a memory matches if ANY of its tags overlap with the request tags.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import httpx
|
||||
@@ -329,10 +330,12 @@ class TestBuildTagGroupsWhereClause:
|
||||
"""AND of two leaves generates AND-joined clause."""
|
||||
groups = [
|
||||
TagGroupAnd.model_validate(
|
||||
{"and": [
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
{"tags": ["user:ep_42"], "match": "all_strict"},
|
||||
]}
|
||||
{
|
||||
"and": [
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
{"tags": ["user:ep_42"], "match": "all_strict"},
|
||||
]
|
||||
}
|
||||
)
|
||||
]
|
||||
clause, params, next_offset = build_tag_groups_where_clause(groups, 3)
|
||||
@@ -348,10 +351,12 @@ class TestBuildTagGroupsWhereClause:
|
||||
"""OR of two leaves generates OR-joined clause."""
|
||||
groups = [
|
||||
TagGroupOr.model_validate(
|
||||
{"or": [
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
{"tags": ["priority:high"], "match": "all_strict"},
|
||||
]}
|
||||
{
|
||||
"or": [
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
{"tags": ["priority:high"], "match": "all_strict"},
|
||||
]
|
||||
}
|
||||
)
|
||||
]
|
||||
clause, params, next_offset = build_tag_groups_where_clause(groups, 1)
|
||||
@@ -363,11 +368,7 @@ class TestBuildTagGroupsWhereClause:
|
||||
|
||||
def test_not_wraps_with_not(self):
|
||||
"""NOT group wraps child clause with NOT."""
|
||||
groups = [
|
||||
TagGroupNot.model_validate(
|
||||
{"not": {"tags": ["archived"], "match": "any_strict"}}
|
||||
)
|
||||
]
|
||||
groups = [TagGroupNot.model_validate({"not": {"tags": ["archived"], "match": "any_strict"}})]
|
||||
clause, params, next_offset = build_tag_groups_where_clause(groups, 2)
|
||||
assert "NOT" in clause
|
||||
assert "$2" in clause
|
||||
@@ -378,13 +379,17 @@ class TestBuildTagGroupsWhereClause:
|
||||
"""AND containing an OR generates correct nested SQL."""
|
||||
groups = [
|
||||
TagGroupAnd.model_validate(
|
||||
{"and": [
|
||||
{"tags": ["user:alice"], "match": "all_strict"},
|
||||
{"or": [
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
{"tags": ["priority:high"], "match": "all_strict"},
|
||||
]},
|
||||
]}
|
||||
{
|
||||
"and": [
|
||||
{"tags": ["user:alice"], "match": "all_strict"},
|
||||
{
|
||||
"or": [
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
{"tags": ["priority:high"], "match": "all_strict"},
|
||||
]
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
]
|
||||
clause, params, next_offset = build_tag_groups_where_clause(groups, 1)
|
||||
@@ -397,11 +402,13 @@ class TestBuildTagGroupsWhereClause:
|
||||
"""Params are numbered sequentially starting from param_offset."""
|
||||
groups = [
|
||||
TagGroupAnd.model_validate(
|
||||
{"and": [
|
||||
{"tags": ["a"], "match": "any_strict"},
|
||||
{"tags": ["b"], "match": "any_strict"},
|
||||
{"tags": ["c"], "match": "any_strict"},
|
||||
]}
|
||||
{
|
||||
"and": [
|
||||
{"tags": ["a"], "match": "any_strict"},
|
||||
{"tags": ["b"], "match": "any_strict"},
|
||||
{"tags": ["c"], "match": "any_strict"},
|
||||
]
|
||||
}
|
||||
)
|
||||
]
|
||||
clause, params, next_offset = build_tag_groups_where_clause(groups, 5)
|
||||
@@ -421,10 +428,12 @@ class TestBuildTagGroupsWhereClause:
|
||||
"""Table alias propagates to nested leaves (each leaf uses the alias)."""
|
||||
groups = [
|
||||
TagGroupAnd.model_validate(
|
||||
{"and": [
|
||||
{"tags": ["a"], "match": "any_strict"},
|
||||
{"tags": ["b"], "match": "any_strict"},
|
||||
]}
|
||||
{
|
||||
"and": [
|
||||
{"tags": ["a"], "match": "any_strict"},
|
||||
{"tags": ["b"], "match": "any_strict"},
|
||||
]
|
||||
}
|
||||
)
|
||||
]
|
||||
clause, params, next_offset = build_tag_groups_where_clause(groups, 1, table_alias="mu.")
|
||||
@@ -433,6 +442,7 @@ class TestBuildTagGroupsWhereClause:
|
||||
assert "mu.tags" in clause
|
||||
# No bare 'tags' keyword without the alias prefix (other than inside the alias itself)
|
||||
import re
|
||||
|
||||
bare_tags = re.findall(r"(?<!\.)tags", clause)
|
||||
assert len(bare_tags) == 0, f"Found bare 'tags' references without alias: {bare_tags}"
|
||||
|
||||
@@ -496,16 +506,18 @@ class TestFilterResultsByTagGroups:
|
||||
"""AND group: both leaf conditions must match."""
|
||||
groups = [
|
||||
TagGroupAnd.model_validate(
|
||||
{"and": [
|
||||
{"tags": ["user:alice"], "match": "all_strict"},
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
]}
|
||||
{
|
||||
"and": [
|
||||
{"tags": ["user:alice"], "match": "all_strict"},
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
]
|
||||
}
|
||||
)
|
||||
]
|
||||
results = [
|
||||
MockResult(["user:alice", "step:5"]), # matches both
|
||||
MockResult(["user:alice"]), # only matches first
|
||||
MockResult(["step:5"]), # only matches second
|
||||
MockResult(["user:alice"]), # only matches first
|
||||
MockResult(["step:5"]), # only matches second
|
||||
MockResult(None),
|
||||
]
|
||||
filtered = filter_results_by_tag_groups(results, groups)
|
||||
@@ -516,10 +528,12 @@ class TestFilterResultsByTagGroups:
|
||||
"""OR group: either condition matching is sufficient."""
|
||||
groups = [
|
||||
TagGroupOr.model_validate(
|
||||
{"or": [
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
{"tags": ["priority:high"], "match": "all_strict"},
|
||||
]}
|
||||
{
|
||||
"or": [
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
{"tags": ["priority:high"], "match": "all_strict"},
|
||||
]
|
||||
}
|
||||
)
|
||||
]
|
||||
results = [
|
||||
@@ -535,11 +549,7 @@ class TestFilterResultsByTagGroups:
|
||||
|
||||
def test_not_negation(self):
|
||||
"""NOT group: inverts the child match."""
|
||||
groups = [
|
||||
TagGroupNot.model_validate(
|
||||
{"not": {"tags": ["archived"], "match": "any_strict"}}
|
||||
)
|
||||
]
|
||||
groups = [TagGroupNot.model_validate({"not": {"tags": ["archived"], "match": "any_strict"}})]
|
||||
results = [
|
||||
MockResult(["archived"]),
|
||||
MockResult(["active"]),
|
||||
@@ -558,20 +568,24 @@ class TestFilterResultsByTagGroups:
|
||||
"""AND containing OR: nested boolean logic works correctly."""
|
||||
groups = [
|
||||
TagGroupAnd.model_validate(
|
||||
{"and": [
|
||||
{"tags": ["user:alice"], "match": "all_strict"},
|
||||
{"or": [
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
{"tags": ["priority:high"], "match": "any_strict"},
|
||||
]},
|
||||
]}
|
||||
{
|
||||
"and": [
|
||||
{"tags": ["user:alice"], "match": "all_strict"},
|
||||
{
|
||||
"or": [
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
{"tags": ["priority:high"], "match": "any_strict"},
|
||||
]
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
]
|
||||
results = [
|
||||
MockResult(["user:alice", "step:5"]), # user:alice AND (step:5 OR ...)
|
||||
MockResult(["user:alice", "priority:high"]), # user:alice AND (... OR priority:high)
|
||||
MockResult(["user:alice"]), # user:alice but neither step nor priority
|
||||
MockResult(["step:5"]), # step:5 but not user:alice
|
||||
MockResult(["user:alice", "step:5"]), # user:alice AND (step:5 OR ...)
|
||||
MockResult(["user:alice", "priority:high"]), # user:alice AND (... OR priority:high)
|
||||
MockResult(["user:alice"]), # user:alice but neither step nor priority
|
||||
MockResult(["step:5"]), # step:5 but not user:alice
|
||||
MockResult(None),
|
||||
]
|
||||
filtered = filter_results_by_tag_groups(results, groups)
|
||||
@@ -585,8 +599,8 @@ class TestFilterResultsByTagGroups:
|
||||
]
|
||||
results = [
|
||||
MockResult(["user:alice", "step:5"]), # both match
|
||||
MockResult(["user:alice"]), # only first
|
||||
MockResult(["step:5"]), # only second
|
||||
MockResult(["user:alice"]), # only first
|
||||
MockResult(["step:5"]), # only second
|
||||
]
|
||||
filtered = filter_results_by_tag_groups(results, groups)
|
||||
assert len(filtered) == 1
|
||||
@@ -619,14 +633,7 @@ async def test_retain_with_tags(api_client, test_bank_id):
|
||||
# Store memory with tags
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{test_bank_id}/memories",
|
||||
json={
|
||||
"items": [
|
||||
{
|
||||
"content": "Alice loves hiking in the mountains.",
|
||||
"tags": ["user_alice"]
|
||||
}
|
||||
]
|
||||
}
|
||||
json={"items": [{"content": "Alice loves hiking in the mountains.", "tags": ["user_alice"]}]},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
result = response.json()
|
||||
@@ -644,9 +651,9 @@ async def test_retain_with_document_tags(api_client, test_bank_id):
|
||||
"document_tags": ["session_123"],
|
||||
"items": [
|
||||
{"content": "Bob discussed the quarterly report."},
|
||||
{"content": "Charlie mentioned the new product launch."}
|
||||
]
|
||||
}
|
||||
{"content": "Charlie mentioned the new product launch."},
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
result = response.json()
|
||||
@@ -662,13 +669,8 @@ async def test_retain_merges_document_and_item_tags(api_client, test_bank_id):
|
||||
f"/v1/default/banks/{test_bank_id}/memories",
|
||||
json={
|
||||
"document_tags": ["session_abc"],
|
||||
"items": [
|
||||
{
|
||||
"content": "Dave talked about machine learning.",
|
||||
"tags": ["user_dave"]
|
||||
}
|
||||
]
|
||||
}
|
||||
"items": [{"content": "Dave talked about machine learning.", "tags": ["user_dave"]}],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
result = response.json()
|
||||
@@ -686,14 +688,13 @@ async def test_recall_without_tags_returns_all_memories(api_client, test_bank_id
|
||||
{"content": "Eve works on natural language processing.", "tags": ["user_eve"]},
|
||||
{"content": "Frank specializes in computer vision.", "tags": ["user_frank"]},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Recall without tags - should return all
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{test_bank_id}/memories/recall",
|
||||
json={"query": "Who works on what?", "budget": "low"}
|
||||
f"/v1/default/banks/{test_bank_id}/memories/recall", json={"query": "Who works on what?", "budget": "low"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
results = response.json()["results"]
|
||||
@@ -715,14 +716,14 @@ async def test_recall_with_tags_filters_memories(api_client, test_bank_id):
|
||||
{"content": "Grace is a data scientist at Google.", "tags": ["user_grace"]},
|
||||
{"content": "Henry is a software engineer at Meta.", "tags": ["user_henry"]},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Recall with user_grace tag - should only return Grace's memory
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{test_bank_id}/memories/recall",
|
||||
json={"query": "Who works at which company?", "budget": "low", "tags": ["user_grace"]}
|
||||
json={"query": "Who works at which company?", "budget": "low", "tags": ["user_grace"]},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
results = response.json()["results"]
|
||||
@@ -746,14 +747,14 @@ async def test_recall_with_multiple_tags_uses_or_matching(api_client, test_bank_
|
||||
{"content": "Julia manages the design team.", "tags": ["user_julia"]},
|
||||
{"content": "Karl oversees the marketing team.", "tags": ["user_karl"]},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Recall with user_ivan OR user_julia - should return both Ivan and Julia, but not Karl
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{test_bank_id}/memories/recall",
|
||||
json={"query": "Who leads which team?", "budget": "low", "tags": ["user_ivan", "user_julia"]}
|
||||
json={"query": "Who leads which team?", "budget": "low", "tags": ["user_ivan", "user_julia"]},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
results = response.json()["results"]
|
||||
@@ -774,18 +775,18 @@ async def test_recall_returns_memories_with_any_overlapping_tag(api_client, test
|
||||
"items": [
|
||||
{
|
||||
"content": "Lisa and Mike discussed the budget in a group chat.",
|
||||
"tags": ["user_lisa", "user_mike"] # Memory visible to both
|
||||
"tags": ["user_lisa", "user_mike"], # Memory visible to both
|
||||
},
|
||||
{"content": "Nancy reviewed the budget alone.", "tags": ["user_nancy"]},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Recall with user_lisa - should return the group chat memory
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{test_bank_id}/memories/recall",
|
||||
json={"query": "What was discussed about the budget?", "budget": "low", "tags": ["user_lisa"]}
|
||||
json={"query": "What was discussed about the budget?", "budget": "low", "tags": ["user_lisa"]},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
results = response.json()["results"]
|
||||
@@ -806,7 +807,7 @@ async def test_reflect_with_tags_filters_memories(api_client, test_bank_id):
|
||||
{"content": "Oscar's favorite color is blue.", "tags": ["user_oscar"]},
|
||||
{"content": "Peter's favorite color is red.", "tags": ["user_peter"]},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -817,8 +818,8 @@ async def test_reflect_with_tags_filters_memories(api_client, test_bank_id):
|
||||
"query": "What is the favorite color?",
|
||||
"budget": "low",
|
||||
"tags": ["user_oscar"],
|
||||
"include": {"facts": {}} # Request facts to verify what was used
|
||||
}
|
||||
"include": {"facts": {}}, # Request facts to verify what was used
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
result = response.json()
|
||||
@@ -845,14 +846,14 @@ async def test_recall_with_empty_tags_returns_all(api_client, test_bank_id):
|
||||
{"content": "Quinn studies mathematics.", "tags": ["user_quinn"]},
|
||||
{"content": "Rachel studies physics.", "tags": ["user_rachel"]},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Recall with empty tags list - should return all
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{test_bank_id}/memories/recall",
|
||||
json={"query": "Who studies what?", "budget": "low", "tags": []}
|
||||
json={"query": "Who studies what?", "budget": "low", "tags": []},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
results = response.json()["results"]
|
||||
@@ -889,14 +890,14 @@ async def test_multi_user_agent_visibility(api_client):
|
||||
# Room 3: Group chat with both users
|
||||
{"content": "In the group meeting, they agreed to meet at noon.", "tags": ["user_a", "user_b"]},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# User A queries - should see their private chat and group chat
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{bank_id}/memories/recall",
|
||||
json={"query": "What meeting time preferences were discussed?", "budget": "low", "tags": ["user_a"]}
|
||||
json={"query": "What meeting time preferences were discussed?", "budget": "low", "tags": ["user_a"]},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
user_a_results = response.json()["results"]
|
||||
@@ -909,7 +910,7 @@ async def test_multi_user_agent_visibility(api_client):
|
||||
# User B queries - should see their private chat and group chat
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{bank_id}/memories/recall",
|
||||
json={"query": "What meeting time preferences were discussed?", "budget": "low", "tags": ["user_b"]}
|
||||
json={"query": "What meeting time preferences were discussed?", "budget": "low", "tags": ["user_b"]},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
user_b_results = response.json()["results"]
|
||||
@@ -922,7 +923,7 @@ async def test_multi_user_agent_visibility(api_client):
|
||||
# Agent queries (no filter) - should see everything
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{bank_id}/memories/recall",
|
||||
json={"query": "What meeting time preferences were discussed?", "budget": "low"} # No tags
|
||||
json={"query": "What meeting time preferences were discussed?", "budget": "low"}, # No tags
|
||||
)
|
||||
assert response.status_code == 200
|
||||
agent_results = response.json()["results"]
|
||||
@@ -955,14 +956,14 @@ async def test_student_tracking_visibility(api_client):
|
||||
{"content": "Student B struggled with geometry concepts.", "tags": ["student_b"]},
|
||||
{"content": "Student A participated actively in class discussion.", "tags": ["student_a"]},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Student A queries - should only see their own data
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{bank_id}/memories/recall",
|
||||
json={"query": "How am I doing in class?", "budget": "low", "tags": ["student_a"]}
|
||||
json={"query": "How am I doing in class?", "budget": "low", "tags": ["student_a"]},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
student_a_results = response.json()["results"]
|
||||
@@ -970,12 +971,14 @@ async def test_student_tracking_visibility(api_client):
|
||||
|
||||
assert any("algebra" in t for t in student_a_texts), "Student A should see their algebra progress"
|
||||
assert any("participated" in t for t in student_a_texts), "Student A should see their participation"
|
||||
assert not any("Student B" in t or "geometry" in t for t in student_a_texts), "Student A should NOT see Student B's data"
|
||||
assert not any("Student B" in t or "geometry" in t for t in student_a_texts), (
|
||||
"Student A should NOT see Student B's data"
|
||||
)
|
||||
|
||||
# Teacher queries (no filter) - should see all students
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{bank_id}/memories/recall",
|
||||
json={"query": "Which students need help?", "budget": "low"} # No tags
|
||||
json={"query": "Which students need help?", "budget": "low"}, # No tags
|
||||
)
|
||||
assert response.status_code == 200
|
||||
teacher_results = response.json()["results"]
|
||||
@@ -1011,7 +1014,7 @@ async def test_list_tags_returns_all_tags(api_client):
|
||||
{"content": "Memory 4 in session 123.", "tags": ["session:123"]},
|
||||
{"content": "Memory 5 for alice in session 456.", "tags": ["user:alice", "session:456"]},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -1057,7 +1060,7 @@ async def test_list_tags_with_wildcard_prefix(api_client):
|
||||
{"content": "Session memory about the meeting.", "tags": ["session:abc"]},
|
||||
{"content": "Room memory for conference room.", "tags": ["room:123"]},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -1091,7 +1094,7 @@ async def test_list_tags_with_wildcard_suffix(api_client):
|
||||
{"content": "Mike is a standard role-user who can only view content.", "tags": ["role-user"]},
|
||||
{"content": "Alice is a role-guest visitor with limited read access.", "tags": ["role-guest"]},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -1119,12 +1122,24 @@ async def test_list_tags_with_wildcard_middle(api_client):
|
||||
f"/v1/default/banks/{bank_id}/memories",
|
||||
json={
|
||||
"items": [
|
||||
{"content": "The production environment is configured with high availability and uses AWS infrastructure.", "tags": ["env-prod"]},
|
||||
{"content": "The enterprise environment for production runs on dedicated servers with 24/7 monitoring.", "tags": ["environment-prod"]},
|
||||
{"content": "The staging environment mirrors production but uses smaller instance sizes.", "tags": ["env-staging"]},
|
||||
{"content": "The development environment allows developers to test their code locally.", "tags": ["env-dev"]},
|
||||
{
|
||||
"content": "The production environment is configured with high availability and uses AWS infrastructure.",
|
||||
"tags": ["env-prod"],
|
||||
},
|
||||
{
|
||||
"content": "The enterprise environment for production runs on dedicated servers with 24/7 monitoring.",
|
||||
"tags": ["environment-prod"],
|
||||
},
|
||||
{
|
||||
"content": "The staging environment mirrors production but uses smaller instance sizes.",
|
||||
"tags": ["env-staging"],
|
||||
},
|
||||
{
|
||||
"content": "The development environment allows developers to test their code locally.",
|
||||
"tags": ["env-dev"],
|
||||
},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -1152,11 +1167,17 @@ async def test_list_tags_case_insensitive(api_client):
|
||||
f"/v1/default/banks/{bank_id}/memories",
|
||||
json={
|
||||
"items": [
|
||||
{"content": "Alice is a software engineer who specializes in machine learning algorithms.", "tags": ["User:Alice"]},
|
||||
{
|
||||
"content": "Alice is a software engineer who specializes in machine learning algorithms.",
|
||||
"tags": ["User:Alice"],
|
||||
},
|
||||
{"content": "Bob works as a data scientist at a large technology company.", "tags": ["user:bob"]},
|
||||
{"content": "Charlie is the lead designer responsible for the user interface.", "tags": ["USER:CHARLIE"]},
|
||||
{
|
||||
"content": "Charlie is the lead designer responsible for the user interface.",
|
||||
"tags": ["USER:CHARLIE"],
|
||||
},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -1182,10 +1203,7 @@ async def test_list_tags_pagination(api_client):
|
||||
{"content": f"{name} works as a software engineer at company {i}.", "tags": [f"tag:{i:03d}"]}
|
||||
for i, name in enumerate(names)
|
||||
]
|
||||
response = await api_client.post(
|
||||
f"/v1/default/banks/{bank_id}/memories",
|
||||
json={"items": items}
|
||||
)
|
||||
response = await api_client.post(f"/v1/default/banks/{bank_id}/memories", json={"items": items})
|
||||
assert response.status_code == 200
|
||||
|
||||
# Get first page (limit 3)
|
||||
@@ -1236,7 +1254,7 @@ async def test_list_tags_ordered_by_count(api_client):
|
||||
{"content": "Eve is a data scientist at Amazon.", "tags": ["medium"]},
|
||||
{"content": "Frank handles customer support at Meta.", "tags": ["medium"]},
|
||||
]
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -1283,9 +1301,7 @@ async def test_list_memories_includes_tags(api_client, test_bank_id):
|
||||
memory_item = next((item for item in result["items"] if "Alice" in item["text"]), None)
|
||||
assert memory_item is not None, "Should find the stored memory"
|
||||
assert "tags" in memory_item, "Memory item must include a 'tags' field"
|
||||
assert set(memory_item["tags"]) == set(tags), (
|
||||
f"All {len(tags)} tags should be returned, got: {memory_item['tags']}"
|
||||
)
|
||||
assert set(memory_item["tags"]) == set(tags), f"All {len(tags)} tags should be returned, got: {memory_item['tags']}"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
@@ -1388,10 +1404,12 @@ async def test_tag_groups_or_compound(api_client):
|
||||
"query": "what are the engineers working on",
|
||||
"budget": "mid",
|
||||
"tag_groups": [
|
||||
{"or": [
|
||||
{"tags": ["user:alice"], "match": "any_strict"},
|
||||
{"tags": ["user:bob"], "match": "any_strict"},
|
||||
]},
|
||||
{
|
||||
"or": [
|
||||
{"tags": ["user:alice"], "match": "any_strict"},
|
||||
{"tags": ["user:bob"], "match": "any_strict"},
|
||||
]
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
@@ -1484,13 +1502,17 @@ async def test_tag_groups_nested_and_containing_or(api_client):
|
||||
"query": "verification step completion",
|
||||
"budget": "mid",
|
||||
"tag_groups": [
|
||||
{"and": [
|
||||
{"tags": ["user:alice"], "match": "all_strict"},
|
||||
{"or": [
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
{"tags": ["step:8"], "match": "any_strict"},
|
||||
]},
|
||||
]},
|
||||
{
|
||||
"and": [
|
||||
{"tags": ["user:alice"], "match": "all_strict"},
|
||||
{
|
||||
"or": [
|
||||
{"tags": ["step:5"], "match": "any_strict"},
|
||||
{"tags": ["step:8"], "match": "any_strict"},
|
||||
]
|
||||
},
|
||||
]
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
@@ -1564,9 +1586,7 @@ async def test_list_mental_model_tags_with_wildcard(memory, request_context):
|
||||
memory, bank_id=bank_id, name="MM 3", tags=["session:abc"], request_context=request_context
|
||||
)
|
||||
|
||||
result = await memory.list_mental_model_tags(
|
||||
bank_id=bank_id, pattern="topic:*", request_context=request_context
|
||||
)
|
||||
result = await memory.list_mental_model_tags(bank_id=bank_id, pattern="topic:*", request_context=request_context)
|
||||
|
||||
returned = sorted(item["tag"] for item in result["items"])
|
||||
assert returned == ["topic:alpha", "topic:beta"]
|
||||
@@ -1585,9 +1605,7 @@ async def test_list_tags_endpoint_with_source_mental_models(memory, request_cont
|
||||
app = create_app(memory, initialize_memory=False)
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get(
|
||||
f"/v1/default/banks/{bank_id}/tags", params={"source": "mental_models"}
|
||||
)
|
||||
response = await client.get(f"/v1/default/banks/{bank_id}/tags", params={"source": "mental_models"})
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
assert {item["tag"] for item in body["items"]} == {"alpha"}
|
||||
@@ -1667,9 +1685,7 @@ async def test_reflect_with_tag_groups_propagates_to_internal_recall(memory, req
|
||||
)
|
||||
if n == 2:
|
||||
return LLMToolCallResult(
|
||||
tool_calls=[
|
||||
LLMToolCall(id="s1", name="search_observations", arguments={"query": "hardware"})
|
||||
],
|
||||
tool_calls=[LLMToolCall(id="s1", name="search_observations", arguments={"query": "hardware"})],
|
||||
finish_reason="tool_calls",
|
||||
)
|
||||
return LLMToolCallResult(
|
||||
@@ -1703,20 +1719,13 @@ async def test_reflect_with_tag_groups_propagates_to_internal_recall(memory, req
|
||||
f"got {len(internal_calls)}: {internal_calls}"
|
||||
)
|
||||
for call in internal_calls:
|
||||
assert call["tag_groups"] == tag_groups, (
|
||||
f"Internal recall_async lost tag_groups; got {call!r}"
|
||||
)
|
||||
assert call["tag_groups"] == tag_groups, f"Internal recall_async lost tag_groups; got {call!r}"
|
||||
|
||||
# End-to-end: the tool result messages the LLM saw must reference only the
|
||||
# tagged ("Strix Halo") memory, never the untagged ("MacBook Pro") one.
|
||||
# This catches any future regression where tag_groups is silently ignored
|
||||
# at the SQL layer even though kwargs propagation looks correct.
|
||||
tool_messages = [
|
||||
msg
|
||||
for call in mock_llm.get_mock_calls()
|
||||
for msg in call["messages"]
|
||||
if msg.get("role") == "tool"
|
||||
]
|
||||
tool_messages = [msg for call in mock_llm.get_mock_calls() for msg in call["messages"] if msg.get("role") == "tool"]
|
||||
tool_payload = "\n".join(msg.get("content", "") for msg in tool_messages)
|
||||
assert "Strix Halo" in tool_payload, (
|
||||
f"Tagged 'Strix Halo' memory must appear in the agent's tool results; got: {tool_payload[:1000]!r}"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user