chore: format test files with ruff (enable formatter on tests/) (#2074)

Tests were excluded from both ruff lint and format via the top-level
[tool.ruff].exclude in hindsight-api-slim, hindsight-embed and the shared
ruff.toml. As a result test files drifted from the formatter's style and
every PR that touched a test (or ran format-on-save) carried large
formatting-only churn.

Move the tests exclude into [tool.ruff.lint].exclude (and [lint].exclude in
ruff.toml) so the formatter now covers tests while lint rules — too noisy for
test code (unused imports/vars, import ordering) — stay excluded. Then run
ruff format across all test directories.

Note: lint.exclude is a post-traversal path filter, so it needs the glob form
'tests/**' rather than the directory form 'tests/' used by top-level exclude.
This commit is contained in:
Nicolò Boschi
2026-06-09 13:23:11 +02:00
committed by GitHub
parent 9ea1ef164a
commit 6ba4aeaf03
138 changed files with 1614 additions and 2117 deletions
+6 -4
View File
@@ -187,12 +187,14 @@ dev = [
[tool.ruff]
line-length = 120
target-version = "py311"
exclude = [
"tests/",
"**/tests/",
]
[tool.ruff.lint]
# Tests are formatted (via `ruff format`) but excluded from lint rules, which
# are too noisy for test code (unused imports/vars, import ordering).
exclude = [
"tests/**",
"**/tests/**",
]
select = [
"E", # pycodestyle errors
"W", # pycodestyle warnings
+20 -10
View File
@@ -1,6 +1,7 @@
"""
Pytest configuration and shared fixtures.
"""
import asyncio
import os
from pathlib import Path
@@ -76,6 +77,7 @@ def pg0_db_url(db_url, tmp_path_factory, worker_id):
if db_url and not _parse_pg0_url(db_url)[0]:
# Plain postgresql:// URL - use it directly but still run migrations
from hindsight_api.migrations import run_migrations
run_migrations(db_url)
return db_url
@@ -127,6 +129,7 @@ def pg0_db_url(db_url, tmp_path_factory, worker_id):
# Run migrations - uses PostgreSQL advisory lock internally,
# so safe to call from multiple workers (only one will actually run migrations)
from hindsight_api.migrations import run_migrations
run_migrations(url)
# Clean up stale test data from previous sessions. Per-bank vector indexes
@@ -157,8 +160,7 @@ def _cleanup_stale_test_data(db_url: str) -> None:
conn = await asyncpg.connect(db_url)
try:
idx_rows = await conn.fetch(
"SELECT indexname FROM pg_indexes "
"WHERE schemaname = 'public' AND indexname LIKE 'idx_mu_emb_%'"
"SELECT indexname FROM pg_indexes WHERE schemaname = 'public' AND indexname LIKE 'idx_mu_emb_%'"
)
if idx_rows:
for row in idx_rows:
@@ -166,10 +168,20 @@ def _cleanup_stale_test_data(db_url: str) -> None:
# Truncate test data in dependency order
for table in [
"entity_cooccurrences", "unit_entities", "memory_links",
"entities", "memory_units", "chunks", "documents",
"mental_models", "directives", "async_operations",
"audit_log", "webhooks", "file_storage", "banks",
"entity_cooccurrences",
"unit_entities",
"memory_links",
"entities",
"memory_units",
"chunks",
"documents",
"mental_models",
"directives",
"async_operations",
"audit_log",
"webhooks",
"file_storage",
"banks",
]:
try:
await conn.execute(f"TRUNCATE {table} CASCADE")
@@ -252,8 +264,7 @@ def oracle_db_url(_oracle_admin_dsn):
# Create test user (idempotent — skip if already exists)
try:
cursor.execute(
f'CREATE USER {test_user} IDENTIFIED BY "{test_pass}" '
f"DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"
f'CREATE USER {test_user} IDENTIFIED BY "{test_pass}" DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS'
)
except oracledb.DatabaseError as e:
if hasattr(e.args[0], "code") and e.args[0].code == 1920:
@@ -420,13 +431,12 @@ def cross_encoder(tmp_path_factory, worker_id):
return ce
@pytest.fixture(scope="session")
def query_analyzer():
return DateparserQueryAnalyzer()
@pytest_asyncio.fixture(scope="function")
async def memory(pg0_db_url, embeddings, cross_encoder, query_analyzer):
"""
+4 -2
View File
@@ -23,6 +23,7 @@ from urllib.parse import urlparse
# Helpers
# ---------------------------------------------------------------------------
def _log(step: int, total: int, msg: str) -> None:
print(f" [{step}/{total}] {msg}")
@@ -64,8 +65,7 @@ def _bootstrap_test_user(admin_dsn: dict[str, str]) -> str:
# Create user (skip if already exists - ORA-01920)
try:
cursor.execute(
f'CREATE USER {test_user} IDENTIFIED BY "{test_pass}" '
f"DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"
f'CREATE USER {test_user} IDENTIFIED BY "{test_pass}" DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS'
)
except oracledb.DatabaseError as e:
if hasattr(e.args[0], "code") and e.args[0].code == 1920:
@@ -100,6 +100,7 @@ def _bootstrap_test_user(admin_dsn: dict[str, str]) -> str:
# Main
# ---------------------------------------------------------------------------
async def _run() -> None:
total_steps = 8
@@ -290,6 +291,7 @@ def main() -> int:
except Exception as exc:
print(f"\nFAILED: {exc}", file=sys.stderr)
import traceback
traceback.print_exc()
return 1
+2 -4
View File
@@ -87,7 +87,7 @@ async def _judge_once(
"content": (
"You are a test evaluation judge. Given a response and evaluation criteria, "
"determine whether the response meets the criteria. "
"Respond with JSON: {\"meets_criteria\": true/false, \"reasoning\": \"brief explanation\"}"
'Respond with JSON: {"meets_criteria": true/false, "reasoning": "brief explanation"}'
),
},
{
@@ -163,9 +163,7 @@ async def evaluate(
if met > not_met:
agreeing = next(v for v in verdicts if v.meets_criteria)
logger.info(
f"Judge: primary 'not met' overruled by majority ({met}/{len(verdicts)} met). Criteria: {criteria}"
)
logger.info(f"Judge: primary 'not met' overruled by majority ({met}/{len(verdicts)} met). Criteria: {criteria}")
return JudgeVerdict(
meets_criteria=True,
reasoning=f"Majority of {len(verdicts)} judges met criteria (primary verdict overruled as noise). {agreeing.reasoning}",
+9 -22
View File
@@ -1,6 +1,7 @@
"""
Tests for agent management API (profile, disposition).
"""
import pytest
import uuid
from hindsight_api import MemoryEngine, RequestContext
@@ -17,9 +18,7 @@ class TestAgentProfile:
"""Tests for agent profile management."""
@pytest.mark.asyncio
async def test_get_bank_profile_no_auto_create_returns_none(
self, memory: MemoryEngine, request_context
):
async def test_get_bank_profile_no_auto_create_returns_none(self, memory: MemoryEngine, request_context):
"""When create_if_missing=False is passed, a missing bank returns None
rather than being silently auto-created. This is what read-only
endpoints (HTTP GET, polling, etc.) must use to avoid creating banks
@@ -27,28 +26,20 @@ class TestAgentProfile:
bank_id = unique_agent_id("test_no_auto_create")
# First call with create_if_missing=False on a non-existent bank
result = await memory.get_bank_profile(
bank_id, request_context=request_context, create_if_missing=False
)
result = await memory.get_bank_profile(bank_id, request_context=request_context, create_if_missing=False)
assert result is None, "Expected None for missing bank with create_if_missing=False"
# Verify the bank was NOT created as a side effect
result_again = await memory.get_bank_profile(
bank_id, request_context=request_context, create_if_missing=False
)
result_again = await memory.get_bank_profile(bank_id, request_context=request_context, create_if_missing=False)
assert result_again is None, "Bank must not exist after read-only call"
# And explicit auto-create still works
created = await memory.get_bank_profile(
bank_id, request_context=request_context, create_if_missing=True
)
created = await memory.get_bank_profile(bank_id, request_context=request_context, create_if_missing=True)
assert created is not None
assert created["disposition"]["skepticism"] == 3
# Now read-only call sees it
seen = await memory.get_bank_profile(
bank_id, request_context=request_context, create_if_missing=False
)
seen = await memory.get_bank_profile(bank_id, request_context=request_context, create_if_missing=False)
assert seen is not None
assert seen["disposition"]["skepticism"] == 3
@@ -122,11 +113,7 @@ class TestAgentEndpoint:
bank_id = unique_agent_id("test_put_create")
request = CreateBankRequest(
disposition=DispositionTraits(
skepticism=4,
literalism=5,
empathy=2
),
disposition=DispositionTraits(skepticism=4, literalism=5, empathy=2),
)
profile = await memory.get_bank_profile(bank_id, request_context=request_context)
@@ -155,7 +142,7 @@ class TestAgentDispositionIntegration:
disposition = {
"skepticism": 5, # Very skeptical
"literalism": 4, # High literalism
"empathy": 2, # Low empathy
"empathy": 2, # Low empathy
}
await memory.update_bank_disposition(bank_id, disposition, request_context=request_context)
@@ -163,7 +150,7 @@ class TestAgentDispositionIntegration:
bank_id=bank_id,
contents=[
{"content": "Traditional painting techniques have been used for centuries"},
{"content": "Modern digital art is changing the art world"}
{"content": "Modern digital art is changing the art world"},
],
request_context=request_context,
)
@@ -643,9 +643,7 @@ class TestDefaultBankTemplateEnvVar:
yield default_template
@pytest.mark.asyncio
async def test_default_template_applied_on_new_bank(
self, api_client, bank_id, _patched_default_template
):
async def test_default_template_applied_on_new_bank(self, api_client, bank_id, _patched_default_template):
"""Creating a new bank applies the default template (config + mental models + directives)."""
# Trigger bank auto-creation via GET profile
resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
@@ -730,9 +728,7 @@ class TestDefaultBankTemplateEnvVar:
assert config_resp.json()["overrides"] == {}
@pytest.mark.asyncio
async def test_default_template_malformed_is_swallowed(
self, api_client, bank_id, monkeypatch
):
async def test_default_template_malformed_is_swallowed(self, api_client, bank_id, monkeypatch):
"""A malformed default template is logged and ignored — bank creation still succeeds."""
from hindsight_api.config import _get_raw_config
+5 -10
View File
@@ -4,6 +4,7 @@ Integration test for API base path support.
Tests that the API works correctly when deployed with a base path (e.g., /hindsight)
for reverse proxy deployments.
"""
import os
import pytest
import pytest_asyncio
@@ -27,10 +28,7 @@ async def api_client_with_base_path(memory):
# Use base_url with base path
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(
transport=transport,
base_url=f"http://test{base_path}"
) as client:
async with httpx.AsyncClient(transport=transport, base_url=f"http://test{base_path}") as client:
yield client
# Cleanup: unset base path
@@ -122,10 +120,10 @@ async def test_base_path_full_workflow(api_client_with_base_path):
"items": [
{
"content": "The API supports base path deployment for reverse proxy use cases.",
"context": "testing base path feature"
"context": "testing base path feature",
}
]
}
},
)
assert response.status_code == 200
result = response.json()
@@ -133,10 +131,7 @@ async def test_base_path_full_workflow(api_client_with_base_path):
# 3. Recall the memory
response = await api_client_with_base_path.post(
f"/v1/default/banks/{bank_id}/memories/recall",
json={
"query": "base path support"
}
f"/v1/default/banks/{bank_id}/memories/recall", json={"query": "base path support"}
)
assert response.status_code == 200
recall_result = response.json()
+97 -83
View File
@@ -7,6 +7,7 @@ Tests cover:
- Hard error when provider doesn't support the batch API (no silent fallback)
- Worker recovery on restart
"""
import asyncio
import json
import logging
@@ -105,19 +106,21 @@ async def test_batch_api_normal_flow(mock_llm_config, test_contents, hindsight_c
"choices": [
{
"message": {
"content": json.dumps({
"facts": [
{
"what": "Alice is a senior software engineer at TechCorp",
"when": "present",
"where": "TechCorp",
"who": "Alice",
"why": "Professional background information",
"fact_type": "world",
"fact_kind": "conversation",
}
]
})
"content": json.dumps(
{
"facts": [
{
"what": "Alice is a senior software engineer at TechCorp",
"when": "present",
"where": "TechCorp",
"who": "Alice",
"why": "Professional background information",
"fact_type": "world",
"fact_kind": "conversation",
}
]
}
)
}
}
],
@@ -132,19 +135,21 @@ async def test_batch_api_normal_flow(mock_llm_config, test_contents, hindsight_c
"choices": [
{
"message": {
"content": json.dumps({
"facts": [
{
"what": "Bob joined the team last month as a junior developer",
"when": "last month",
"where": "team",
"who": "Bob",
"why": "New team member information",
"fact_type": "world",
"fact_kind": "conversation",
}
]
})
"content": json.dumps(
{
"facts": [
{
"what": "Bob joined the team last month as a junior developer",
"when": "last month",
"where": "team",
"who": "Bob",
"why": "New team member information",
"fact_type": "world",
"fact_kind": "conversation",
}
]
}
)
}
}
],
@@ -213,6 +218,7 @@ async def test_batch_api_crash_recovery(mock_llm_config, test_contents, hindsigh
schema = request_context.tenant_id
from hindsight_api.engine.task_backend import fq_table
table = fq_table("async_operations", schema)
# Create operation with batch_id already stored
@@ -223,11 +229,13 @@ async def test_batch_api_crash_recovery(mock_llm_config, test_contents, hindsigh
""",
operation_id,
bank_id,
json.dumps({
"batch_id": batch_id,
"batch_provider": "openai",
"chunk_count": 2,
}),
json.dumps(
{
"batch_id": batch_id,
"batch_provider": "openai",
"chunk_count": 2,
}
),
)
# Mock batch API responses for resume scenario
@@ -250,19 +258,21 @@ async def test_batch_api_crash_recovery(mock_llm_config, test_contents, hindsigh
"choices": [
{
"message": {
"content": json.dumps({
"facts": [
{
"what": "Alice is a senior software engineer",
"when": "present",
"where": "TechCorp",
"who": "Alice",
"why": "Background",
"fact_type": "world",
"fact_kind": "conversation",
}
]
})
"content": json.dumps(
{
"facts": [
{
"what": "Alice is a senior software engineer",
"when": "present",
"where": "TechCorp",
"who": "Alice",
"why": "Background",
"fact_type": "world",
"fact_kind": "conversation",
}
]
}
)
}
}
],
@@ -277,19 +287,21 @@ async def test_batch_api_crash_recovery(mock_llm_config, test_contents, hindsigh
"choices": [
{
"message": {
"content": json.dumps({
"facts": [
{
"what": "Bob is a junior developer",
"when": "last month",
"where": "team",
"who": "Bob",
"why": "New member",
"fact_type": "world",
"fact_kind": "conversation",
}
]
})
"content": json.dumps(
{
"facts": [
{
"what": "Bob is a junior developer",
"when": "last month",
"where": "team",
"who": "Bob",
"why": "New member",
"fact_type": "world",
"fact_kind": "conversation",
}
]
}
)
}
}
],
@@ -377,19 +389,21 @@ async def test_batch_api_records_non_fatal_extraction_errors(
"choices": [
{
"message": {
"content": json.dumps({
"facts": [
{
"what": "Alice is a senior software engineer",
"when": "present",
"where": "TechCorp",
"who": "Alice",
"why": "Background",
"fact_type": "world",
"fact_kind": "conversation",
}
]
})
"content": json.dumps(
{
"facts": [
{
"what": "Alice is a senior software engineer",
"when": "present",
"where": "TechCorp",
"who": "Alice",
"why": "Background",
"fact_type": "world",
"fact_kind": "conversation",
}
]
}
)
}
}
],
@@ -416,7 +430,9 @@ async def test_batch_api_records_non_fatal_extraction_errors(
assert usage.total_tokens == 150
row = await pool.fetchrow(f"SELECT result_metadata FROM {table} WHERE operation_id = $1", operation_id)
metadata = json.loads(row["result_metadata"]) if isinstance(row["result_metadata"], str) else row["result_metadata"]
metadata = (
json.loads(row["result_metadata"]) if isinstance(row["result_metadata"], str) else row["result_metadata"]
)
assert metadata["batch_id"] == batch_id
assert metadata["extraction_errors_count"] == 1
assert metadata["extraction_errors_sample"] == ["chunk_1: missing batch result"]
@@ -469,6 +485,7 @@ async def test_worker_batch_recovery(memory, request_context):
schema = request_context.tenant_id
from hindsight_api.engine.task_backend import fq_table
table = fq_table("async_operations", schema)
# Create orphaned batch operation (simulates worker crash during polling)
@@ -486,16 +503,19 @@ async def test_worker_batch_recovery(memory, request_context):
""",
operation_id,
bank_id,
json.dumps({
"batch_id": batch_id,
"batch_provider": "openai",
"chunk_count": 1,
}),
json.dumps(
{
"batch_id": batch_id,
"batch_provider": "openai",
"chunk_count": 1,
}
),
json.dumps(task_payload),
)
# Create WorkerPoller
from hindsight_api.extensions.builtin.tenant import DefaultTenantExtension
tenant_extension = DefaultTenantExtension(config={"schema": schema} if schema else {})
poller = WorkerPoller(
@@ -559,13 +579,7 @@ async def test_batch_api_via_extract_facts_from_contents(
"custom_id": "chunk_0",
"response": {
"body": {
"choices": [
{
"message": {
"content": json.dumps({"facts": []})
}
}
],
"choices": [{"message": {"content": json.dumps({"facts": []})}}],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
}
},
@@ -10,6 +10,7 @@ To run:
To skip in CI:
Add @pytest.mark.skip at the test level
"""
import pytest
import os
import asyncio
@@ -115,7 +116,9 @@ def integration_config():
return config
@pytest.mark.skip(reason="Real API test - takes minutes and costs money. Run manually with: pytest tests/test_batch_api_integration.py::test_real_openai_batch_api -v -s")
@pytest.mark.skip(
reason="Real API test - takes minutes and costs money. Run manually with: pytest tests/test_batch_api_integration.py::test_real_openai_batch_api -v -s"
)
@pytest.mark.integration # Mark as integration test
@pytest.mark.slow # Mark as slow test
@pytest.mark.asyncio
@@ -174,17 +177,21 @@ async def test_real_openai_batch_api(real_llm_config, test_contents_real, integr
logger.info("\n" + "=" * 80)
logger.info("✅ BATCH COMPLETED SUCCESSFULLY")
logger.info("=" * 80)
logger.info(f"Total duration: {total_duration:.1f} seconds ({total_duration/60:.1f} minutes)")
logger.info(f"Total duration: {total_duration:.1f} seconds ({total_duration / 60:.1f} minutes)")
logger.info(f"Facts extracted: {len(facts)}")
logger.info(f"Chunks processed: {len(chunks)}")
logger.info(f"Token usage: {usage.input_tokens} input + {usage.output_tokens} output = {usage.total_tokens} total")
logger.info(f"Estimated cost: ${(usage.input_tokens * 0.00015 / 1000 + usage.output_tokens * 0.0006 / 1000):.4f}")
logger.info(
f"Token usage: {usage.input_tokens} input + {usage.output_tokens} output = {usage.total_tokens} total"
)
logger.info(
f"Estimated cost: ${(usage.input_tokens * 0.00015 / 1000 + usage.output_tokens * 0.0006 / 1000):.4f}"
)
logger.info("=" * 80)
# Log sample facts
logger.info("\n📋 Sample extracted facts:")
for i, fact in enumerate(facts[:5]): # Show first 5 facts
logger.info(f"\nFact {i+1}:")
logger.info(f"\nFact {i + 1}:")
logger.info(f" Type: {fact.fact_type}")
logger.info(f" Text: {fact.fact_text[:100]}...")
logger.info(f" Entities: {fact.entities}")
@@ -212,11 +219,13 @@ async def test_real_openai_batch_api(real_llm_config, test_contents_real, integr
f.write(f"Contents: {len(test_contents_real)} items\n")
f.write(f"Poll Interval: {integration_config.retain_batch_poll_interval_seconds}s\n\n")
f.write(f"Results:\n")
f.write(f" Total Duration: {total_duration:.1f}s ({total_duration/60:.1f} min)\n")
f.write(f" Total Duration: {total_duration:.1f}s ({total_duration / 60:.1f} min)\n")
f.write(f" Facts Extracted: {len(facts)}\n")
f.write(f" Chunks Processed: {len(chunks)}\n")
f.write(f" Token Usage: {usage.total_tokens} ({usage.input_tokens} in + {usage.output_tokens} out)\n")
f.write(f" Estimated Cost: ${(usage.input_tokens * 0.00015 / 1000 + usage.output_tokens * 0.0006 / 1000):.4f}\n")
f.write(
f" Estimated Cost: ${(usage.input_tokens * 0.00015 / 1000 + usage.output_tokens * 0.0006 / 1000):.4f}\n"
)
logger.info(f"\n📄 Timing report written to: {report_path}")
@@ -174,9 +174,7 @@ def test_async_children_packs_small_items_by_budget():
num_items = max(4, (tokens_per_batch // max(item_tokens, 1)) * 3)
contents = [{"content": item_text, "document_id": f"doc-{i}"} for i in range(num_items)]
total = sum(count_tokens(c["content"]) for c in contents)
assert total > tokens_per_batch, (
f"Test setup error: {total} tokens does not exceed budget {tokens_per_batch}"
)
assert total > tokens_per_batch, f"Test setup error: {total} tokens does not exceed budget {tokens_per_batch}"
children = _split_contents_into_async_children(contents, tokens_per_batch)
@@ -104,8 +104,7 @@ class TestCausalRelationsValidation:
for rel in facts[0].causal_relations:
# This should never happen due to validation
assert False, (
f"First fact should not have causal relations, "
f"but found: target_index={rel.target_fact_index}"
f"First fact should not have causal relations, but found: target_index={rel.target_fact_index}"
)
@pytest.mark.asyncio
@@ -139,11 +138,13 @@ class TestCausalRelationsValidation:
for i, fact in enumerate(facts):
if fact.causal_relations:
for rel in fact.causal_relations:
all_relations.append({
"from_fact": i,
"to_fact": rel.target_fact_index,
"type": rel.relation_type,
})
all_relations.append(
{
"from_fact": i,
"to_fact": rel.target_fact_index,
"type": rel.relation_type,
}
)
# If causal relations were extracted, verify they form a valid chain
if all_relations:
@@ -226,6 +227,5 @@ class TestCausalRelationsValidation:
if fact.causal_relations:
for rel in fact.causal_relations:
assert rel.relation_type in valid_types, (
f"Invalid relation_type '{rel.relation_type}'. "
f"Must be one of: {valid_types}"
f"Invalid relation_type '{rel.relation_type}'. Must be one of: {valid_types}"
)
@@ -40,7 +40,11 @@ After searching for weeks, I finally found a cheaper apartment in Brooklyn.
llm_config = LLMConfig.from_env()
facts, _, _ = await extract_facts_from_text(
text=text, event_date=datetime(2024, 3, 15), context=context, llm_config=llm_config, agent_name="TestUser",
text=text,
event_date=datetime(2024, 3, 15),
context=context,
llm_config=llm_config,
agent_name="TestUser",
config=_get_raw_config(),
)
@@ -109,7 +113,11 @@ The renovation took three months and cost $15,000.
llm_config = LLMConfig.from_env()
facts, _, _ = await extract_facts_from_text(
text=text, event_date=datetime(2024, 6, 1), context=context, llm_config=llm_config, agent_name="TestUser",
text=text,
event_date=datetime(2024, 6, 1),
context=context,
llm_config=llm_config,
agent_name="TestUser",
config=_get_raw_config(),
)
@@ -140,7 +148,11 @@ Machine learning fascinated me so much that I changed my career to data science.
llm_config = LLMConfig.from_env()
facts, _, _ = await extract_facts_from_text(
text=text, event_date=datetime(2024, 1, 1), context=context, llm_config=llm_config, agent_name="TestUser",
text=text,
event_date=datetime(2024, 1, 1),
context=context,
llm_config=llm_config,
agent_name="TestUser",
config=_get_raw_config(),
)
@@ -168,7 +180,11 @@ The new role enabled me to lead a team of engineers.
llm_config = LLMConfig.from_env()
facts, _, _ = await extract_facts_from_text(
text=text, event_date=datetime(2024, 2, 15), context=context, llm_config=llm_config, agent_name="TestUser",
text=text,
event_date=datetime(2024, 2, 15),
context=context,
llm_config=llm_config,
agent_name="TestUser",
config=_get_raw_config(),
)
@@ -180,4 +196,3 @@ The new role enabled me to lead a team of engineers.
f"Invalid target_fact_index {rel.target_fact_index} in fact {i}. "
f"Must reference previous facts only (valid range: 0 to {i - 1})"
)
@@ -130,8 +130,7 @@ async def test_store_chunks_batch_second_call_with_identical_payload(memory):
await _seed_bank_and_document(conn, bank_id, document_id)
chunks = [
ChunkMetadata(chunk_text=f"chunk-{i}", fact_count=1, content_index=0, chunk_index=i)
for i in range(5)
ChunkMetadata(chunk_text=f"chunk-{i}", fact_count=1, content_index=0, chunk_index=i) for i in range(5)
]
await chunk_storage.store_chunks_batch(conn, bank_id, document_id, chunks, ops=ops)
+1 -1
View File
@@ -1,6 +1,7 @@
"""
Test chunking functionality for large documents.
"""
import pytest
from hindsight_api.engine.retain.fact_extraction import chunk_text
@@ -53,4 +54,3 @@ def test_chunk_text_64k():
# Verify we didn't lose content
combined_length = sum(len(chunk) for chunk in chunks)
assert combined_length >= len(text) * 0.95, "Lost too much content during chunking"
@@ -344,4 +344,7 @@ class TestFactoryFunction:
assert isinstance(encoder, CohereCrossEncoder)
assert encoder.api_key == "test_key"
assert encoder.model == "cohere-rerank-v3-english"
assert encoder.base_url == "https://my-endpoint.inference.ai.azure.com/models/cohere-rerank-v3-english/invoke"
assert (
encoder.base_url
== "https://my-endpoint.inference.ai.azure.com/models/cohere-rerank-v3-english/invoke"
)
@@ -371,9 +371,7 @@ class TestRecoverConsolidation:
mem_id,
)
result = await memory_no_llm_verify.retry_failed_consolidation(
bank_id, request_context=request_context
)
result = await memory_no_llm_verify.retry_failed_consolidation(bank_id, request_context=request_context)
assert result["retried_count"] == 2
@@ -394,9 +392,7 @@ class TestRecoverConsolidation:
bank_id = f"test-recover-zero-{uuid.uuid4().hex[:8]}"
await memory_no_llm_verify.get_bank_profile(bank_id=bank_id, request_context=request_context)
result = await memory_no_llm_verify.retry_failed_consolidation(
bank_id, request_context=request_context
)
result = await memory_no_llm_verify.retry_failed_consolidation(bank_id, request_context=request_context)
assert result["retried_count"] == 0
@@ -410,14 +406,10 @@ class TestRecoverConsolidation:
async with memory_no_llm_verify._pool.acquire() as conn:
(mem_id,) = await _insert_memories(conn, bank_id, ["Grace is an expert rock climber."])
await conn.execute(
"UPDATE memory_units SET consolidation_failed_at = NOW() WHERE id = $1", mem_id
)
await conn.execute("UPDATE memory_units SET consolidation_failed_at = NOW() WHERE id = $1", mem_id)
# Recover
recover_result = await memory_no_llm_verify.retry_failed_consolidation(
bank_id, request_context=request_context
)
recover_result = await memory_no_llm_verify.retry_failed_consolidation(bank_id, request_context=request_context)
assert recover_result["retried_count"] == 1
# Now consolidate with a healthy LLM
@@ -460,9 +452,7 @@ class TestRecoverConsolidation:
["Henry is a professional chef.", "Henry trained at Le Cordon Bleu."],
)
for mem_id in ids:
await conn.execute(
"UPDATE memory_units SET consolidation_failed_at = NOW() WHERE id = $1", mem_id
)
await conn.execute("UPDATE memory_units SET consolidation_failed_at = NOW() WHERE id = $1", mem_id)
app = create_app(memory_no_llm_verify, initialize_memory=False)
transport = httpx.ASGITransport(app=app)
@@ -80,9 +80,7 @@ async def _pending_consolidation_ops(memory, bank_id: str) -> list[str]:
@pytest.mark.asyncio
async def test_round_limited_consolidation_leaves_followup_pending_op(
memory: MemoryEngine, request_context
):
async def test_round_limited_consolidation_leaves_followup_pending_op(memory: MemoryEngine, request_context):
"""A round-limited consolidation must leave a new ``pending`` consolidation
op in ``async_operations`` for the same bank so the worker poller can
drain the backlog without external intervention."""
@@ -147,9 +145,7 @@ async def test_round_limited_consolidation_leaves_followup_pending_op(
op_id,
)
assert row is not None
assert row["status"] == "completed", (
f"first consolidation op should be marked completed, got {row['status']}"
)
assert row["status"] == "completed", f"first consolidation op should be marked completed, got {row['status']}"
# 4. Backlog must remain (round limit kept one round under the total)
unconsolidated_after = await _count_unconsolidated(memory, bank_id)
@@ -169,8 +165,6 @@ async def test_round_limited_consolidation_leaves_followup_pending_op(
f"backlog. Found {len(pending_ops)} pending ops; backlog still has "
f"{unconsolidated_after} unconsolidated memory_units."
)
assert pending_ops[0] != str(op_id), (
"The pending op must be a NEW row, not the original op we just executed."
)
assert pending_ops[0] != str(op_id), "The pending op must be a NEW row, not the original op we just executed."
await memory.delete_bank(bank_id, request_context=request_context)
@@ -230,8 +230,7 @@ async def test_backoff_matches_schedule_by_retry_count(memory, retry_count):
delta = (excinfo.value.retry_at - before).total_seconds()
assert expected_backoff <= delta <= expected_backoff + 10, (
f"retry_count={retry_count}: expected backoff ~{expected_backoff}s, "
f"got delta={delta:.2f}s"
f"retry_count={retry_count}: expected backoff ~{expected_backoff}s, got delta={delta:.2f}s"
)
await _cleanup(pool, bank_id, op_id)
@@ -266,8 +265,6 @@ async def test_retry_is_indefinite(memory):
delta = (excinfo.value.retry_at - before).total_seconds()
cap = _CONSOLIDATION_RETRY_BACKOFF_MAX_SECONDS
assert cap <= delta <= cap + 10, (
f"At retry_count=100 expected backoff at cap (~{cap}s), got {delta:.2f}s"
)
assert cap <= delta <= cap + 10, f"At retry_count=100 expected backoff at cap (~{cap}s), got {delta:.2f}s"
await _cleanup(pool, bank_id, op_id)
@@ -77,9 +77,7 @@ async def test_round_limit_caps_processed_memories(memory: MemoryEngine, request
assert result["memories_processed"] <= round_limit
# Must have re-queued consolidation for remaining work
mock_requeue.assert_called_once_with(
bank_id=bank_id, request_context=request_context, observation_scopes=None
)
mock_requeue.assert_called_once_with(bank_id=bank_id, request_context=request_context, observation_scopes=None)
# Mental model refresh should be skipped on intermediate round
assert result.get("mental_models_refreshed", 0) == 0
@@ -113,10 +113,7 @@ def _mock_llm_one_obs_per_fact():
# example UUIDs in its OUTPUT samples — read user only.
prompt = "\n".join(m.get("content", "") for m in messages if m.get("role") == "user")
fact_ids = re.findall(r"\[([0-9a-f-]{36})\]", prompt)
creates = [
_CreateAction(text=f"Observation about fact {fid[:8]}", source_fact_ids=[fid])
for fid in fact_ids
]
creates = [_CreateAction(text=f"Observation about fact {fid[:8]}", source_fact_ids=[fid]) for fid in fact_ids]
return _ConsolidationBatchResponse(creates=creates)
mock_llm.set_response_callback(callback)
@@ -170,11 +167,13 @@ async def test_combined_mode_parallel_writes_to_memory_tag_set(memory: MemoryEng
assert result["status"] == "completed"
tag_sets = _ag_sorted(await _fetch_observation_tag_sets(memory, bank_id))
assert tag_sets == _ag_sorted([
frozenset({"user:alice"}),
frozenset({"user:bob"}),
frozenset({"user:carol"}),
])
assert tag_sets == _ag_sorted(
[
frozenset({"user:alice"}),
frozenset({"user:bob"}),
frozenset({"user:carol"}),
]
)
finally:
await memory.delete_bank(bank_id, request_context=request_context)
@@ -373,8 +372,7 @@ async def test_overlapping_scopes_serialise_under_parallelism(memory: MemoryEngi
# The whole point: lock invariant per scope.
for scope, peak in max_concurrent.items():
assert peak <= 1, (
f"scope {set(scope) or '<untagged>'} had {peak} concurrent in-flight recalls; "
"lock invariant violated"
f"scope {set(scope) or '<untagged>'} had {peak} concurrent in-flight recalls; lock invariant violated"
)
# Sanity: we DID see recalls for the shared scope, so the test wasn't trivial.
assert frozenset({"a"}) in max_concurrent
@@ -421,9 +419,7 @@ async def test_per_batch_log_line_attributes_only_own_work(memory: MemoryEngine,
patch.object(memory, "submit_async_consolidation"),
caplog.at_level(logging.INFO, logger="hindsight_api.engine.consolidation.consolidator"),
):
await run_consolidation_job(
memory_engine=memory, bank_id=bank_id, request_context=request_context
)
await run_consolidation_job(memory_engine=memory, bank_id=bank_id, request_context=request_context)
finally:
memory._consolidation_llm_config = original_llm
@@ -452,9 +448,7 @@ async def test_per_batch_log_line_attributes_only_own_work(memory: MemoryEngine,
assert processed_values == sorted(processed_values), (
f"processed counter must be monotonic, got {processed_values}"
)
assert max(processed_values) == 3, (
f"final cumulative processed should be 3, got {max(processed_values)}"
)
assert max(processed_values) == 3, f"final cumulative processed should be 3, got {max(processed_values)}"
assert set(processed_values) == {1, 2, 3}, (
f"each batch should bump the counter by exactly 1, got {processed_values}"
)
@@ -466,9 +460,7 @@ async def test_per_batch_log_line_attributes_only_own_work(memory: MemoryEngine,
assert m_llm_time, f"expected llm=Xs timing, got: {line}"
# Sanity: a single mock-LLM call is fast — under a second easily.
# If snapshot leaked, this would catch concurrent batches' LLM time too.
assert float(m_llm_time.group(1)) < 5.0, (
f"llm timing implausibly large for a single mock-LLM call: {line}"
)
assert float(m_llm_time.group(1)) < 5.0, f"llm timing implausibly large for a single mock-LLM call: {line}"
finally:
await memory.delete_bank(bank_id, request_context=request_context)
@@ -71,9 +71,7 @@ async def _count_unconsolidated(memory, bank_id: str) -> int:
@pytest.mark.asyncio
async def test_requeue_failure_propagates_to_worker_retry(
memory: MemoryEngine, request_context
):
async def test_requeue_failure_propagates_to_worker_retry(memory: MemoryEngine, request_context):
"""When the in-task ``submit_async_consolidation`` call raises, the op
must NOT be silently completed. The consolidator's work for this round
is durably committed (memories marked consolidated_at in their own
@@ -183,8 +181,6 @@ async def test_requeue_failure_propagates_to_worker_retry(
f"unconsolidated_remaining={unconsolidated_after}"
)
assert call_count["n"] == 1, (
f"only one in-task submit_async_consolidation call expected, got {call_count['n']}"
)
assert call_count["n"] == 1, f"only one in-task submit_async_consolidation call expected, got {call_count['n']}"
await memory.delete_bank(bank_id, request_context=request_context)
@@ -63,10 +63,7 @@ async def test_concurrent_submits_leave_one_pending(memory, request_context, no_
await _ensure_bank(pool, bank_id)
try:
results = await asyncio.gather(
*(
memory.submit_async_consolidation(bank_id=bank_id, request_context=request_context)
for _ in range(5)
)
*(memory.submit_async_consolidation(bank_id=bank_id, request_context=request_context) for _ in range(5))
)
assert await _count_pending(pool, bank_id) == 1
op_ids = {r["operation_id"] for r in results}
@@ -287,7 +287,9 @@ class TestEmbeddingDimension:
# Try to change dimension - should raise RuntimeError.
# Retry on transient OID errors from concurrent xdist schema drops.
_assert_raises_runtime_error_with_retry(
db_url, 768, schema,
db_url,
768,
schema,
expected_messages=["Cannot change embedding dimension", "1 rows with embeddings"],
)
@@ -332,7 +334,9 @@ class TestEmbeddingDimension:
# Try to change dimension - should raise RuntimeError.
# Retry on transient OID errors from concurrent xdist schema drops.
_assert_raises_runtime_error_with_retry(
db_url, 768, schema,
db_url,
768,
schema,
expected_messages=["Cannot change embedding dimension", "mental_models"],
)
+89 -39
View File
@@ -193,8 +193,13 @@ class TestPostgreSQLDialect:
def test_build_semantic_arm(self, d):
arm = d.build_semantic_arm(
table="schema.memory_units", cols="id, text", fact_type="world",
embedding_param="$1", bank_id_param="$2", fetch_limit=100, min_similarity=0.58,
table="schema.memory_units",
cols="id, text",
fact_type="world",
embedding_param="$1",
bank_id_param="$2",
fetch_limit=100,
min_similarity=0.58,
)
assert "1 - (embedding <=> $1::vector)" in arm
assert ">= 0.58" in arm
@@ -204,8 +209,12 @@ class TestPostgreSQLDialect:
def test_build_bm25_arm_native(self, d):
arm = d.build_bm25_arm(
table="schema.memory_units", cols="id, text", fact_type="world",
bank_id_param="$2", limit_param="$3", text_param="$4",
table="schema.memory_units",
cols="id, text",
fact_type="world",
bank_id_param="$2",
limit_param="$3",
text_param="$4",
)
assert "ts_rank_cd" in arm
assert "to_tsquery" in arm
@@ -216,8 +225,12 @@ class TestPostgreSQLDialect:
def test_build_bm25_arm_native_uses_configured_language(self, d):
arm = d.build_bm25_arm(
table="schema.memory_units", cols="id, text", fact_type="world",
bank_id_param="$2", limit_param="$3", text_param="$4",
table="schema.memory_units",
cols="id, text",
fact_type="world",
bank_id_param="$2",
limit_param="$3",
text_param="$4",
bm25_language="french",
)
# Both the score and the WHERE filter must use the configured dictionary
@@ -226,8 +239,12 @@ class TestPostgreSQLDialect:
def test_build_bm25_arm_vchord(self, d):
arm = d.build_bm25_arm(
table="t", cols="id", fact_type="world",
bank_id_param="$2", limit_param="$3", text_param="$4",
table="t",
cols="id",
fact_type="world",
bank_id_param="$2",
limit_param="$3",
text_param="$4",
text_search_extension="vchord",
)
assert "to_bm25query" in arm
@@ -240,16 +257,26 @@ class TestPostgreSQLDialect:
rows with a genuine query-term match, mirroring native tsvector's `@@`.
"""
arm = d.build_bm25_arm(
table="t", cols="id", fact_type="world",
bank_id_param="$2", limit_param="$3", text_param="$4",
table="t",
cols="id",
fact_type="world",
bank_id_param="$2",
limit_param="$3",
text_param="$4",
text_search_extension="vchord",
)
assert "-(search_vector <&> to_bm25query('idx_memory_units_text_search', tokenize($4, 'llmlingua2'))) > 0" in arm
assert (
"-(search_vector <&> to_bm25query('idx_memory_units_text_search', tokenize($4, 'llmlingua2'))) > 0" in arm
)
def test_build_bm25_arm_vchord_honors_custom_min_score(self, d):
arm = d.build_bm25_arm(
table="t", cols="id", fact_type="world",
bank_id_param="$2", limit_param="$3", text_param="$4",
table="t",
cols="id",
fact_type="world",
bank_id_param="$2",
limit_param="$3",
text_param="$4",
text_search_extension="vchord",
bm25_min_score=2.5,
)
@@ -257,8 +284,12 @@ class TestPostgreSQLDialect:
def test_build_bm25_arm_pgroonga(self, d):
arm = d.build_bm25_arm(
table="schema.memory_units", cols="id, text", fact_type="world",
bank_id_param="$2", limit_param="$3", text_param="$4",
table="schema.memory_units",
cols="id, text",
fact_type="world",
bank_id_param="$2",
limit_param="$3",
text_param="$4",
text_search_extension="pgroonga",
)
# pgroonga uses the &@~ operator + pgroonga_score for ranking. Escape
@@ -272,8 +303,12 @@ class TestPostgreSQLDialect:
def test_build_bm25_arm_pgroonga_ignores_bm25_language(self, d):
"""pgroonga's tokenizer is fixed at index creation; bm25_language must not leak in."""
arm = d.build_bm25_arm(
table="t", cols="id", fact_type="world",
bank_id_param="$2", limit_param="$3", text_param="$4",
table="t",
cols="id",
fact_type="world",
bank_id_param="$2",
limit_param="$3",
text_param="$4",
text_search_extension="pgroonga",
bm25_language="french",
)
@@ -281,8 +316,12 @@ class TestPostgreSQLDialect:
def test_build_bm25_arm_pg_search(self, d):
arm = d.build_bm25_arm(
table="schema.memory_units", cols="id, text", fact_type="world",
bank_id_param="$2", limit_param="$3", text_param="$4",
table="schema.memory_units",
cols="id, text",
fact_type="world",
bank_id_param="$2",
limit_param="$3",
text_param="$4",
text_search_extension="pg_search",
)
assert "paradedb.score(id)" in arm
@@ -360,8 +399,13 @@ class TestOracleDialect:
def test_build_semantic_arm(self, d):
arm = d.build_semantic_arm(
table="memory_units", cols="id, text", fact_type="world",
embedding_param=":1", bank_id_param=":2", fetch_limit=100, min_similarity=0.58,
table="memory_units",
cols="id, text",
fact_type="world",
embedding_param=":1",
bank_id_param=":2",
fetch_limit=100,
min_similarity=0.58,
)
assert "VECTOR_DISTANCE" in arm
assert ">= 0.58" in arm
@@ -371,8 +415,12 @@ class TestOracleDialect:
def test_build_bm25_arm(self, d):
arm = d.build_bm25_arm(
table="memory_units", cols="id, text", fact_type="world",
bank_id_param=":2", limit_param=":3", text_param=":4",
table="memory_units",
cols="id, text",
fact_type="world",
bank_id_param=":2",
limit_param=":3",
text_param=":4",
arm_index=0,
)
assert "CONTAINS" in arm
@@ -383,12 +431,22 @@ class TestOracleDialect:
def test_build_bm25_arm_unique_labels(self, d):
"""Each arm_index produces a unique SCORE label to avoid conflicts in UNION ALL."""
arm0 = d.build_bm25_arm(
table="t", cols="id", fact_type="world",
bank_id_param=":2", limit_param=":3", text_param=":4", arm_index=0,
table="t",
cols="id",
fact_type="world",
bank_id_param=":2",
limit_param=":3",
text_param=":4",
arm_index=0,
)
arm1 = d.build_bm25_arm(
table="t", cols="id", fact_type="experience",
bank_id_param=":2", limit_param=":3", text_param=":4", arm_index=1,
table="t",
cols="id",
fact_type="experience",
bank_id_param=":2",
limit_param=":3",
text_param=":4",
arm_index=1,
)
assert "SCORE(10)" in arm0
assert "SCORE(11)" in arm1
@@ -474,9 +532,7 @@ class TestOracleQueryRewriter:
"""Verify JSONB ->> boolean comparison is rewritten to JSON_VALUE."""
from hindsight_api.engine.db.oracle import _rewrite_pg_to_oracle
query, _, _ = _rewrite_pg_to_oracle(
"WHERE (trigger->>'refresh_after_consolidation')::boolean = true"
)
query, _, _ = _rewrite_pg_to_oracle("WHERE (trigger->>'refresh_after_consolidation')::boolean = true")
assert "JSON_VALUE" in query
assert "'true'" in query
assert "->>" not in query
@@ -493,9 +549,7 @@ class TestOracleQueryRewriter:
"""Verify ->> works with quoted column names."""
from hindsight_api.engine.db.oracle import _rewrite_pg_to_oracle
query, _, _ = _rewrite_pg_to_oracle(
"ORDER BY (result_metadata->>'sub_batch_index')::int"
)
query, _, _ = _rewrite_pg_to_oracle("ORDER BY (result_metadata->>'sub_batch_index')::int")
assert "JSON_VALUE" in query
assert "->>" not in query
@@ -681,9 +735,7 @@ class TestOracleOpsInsertFactsBatch:
@pytest.mark.asyncio
async def test_tags_json_decoded_to_list(self, ops, mock_conn):
"""Tags JSON strings must be decoded to Python lists, not passed as strings."""
await ops.insert_facts_batch(
conn=mock_conn, **{**self._make_batch(1), "tags_list": ['["tag1", "tag2"]']}
)
await ops.insert_facts_batch(conn=mock_conn, **{**self._make_batch(1), "tags_list": ['["tag1", "tag2"]']})
_, rows_data = mock_conn.executemany.call_args.args
assert rows_data[0][13] == ["tag1", "tag2"]
assert isinstance(rows_data[0][13], list)
@@ -691,9 +743,7 @@ class TestOracleOpsInsertFactsBatch:
@pytest.mark.asyncio
async def test_empty_tags_becomes_empty_list(self, ops, mock_conn):
"""Empty/falsy tags string must become [], not crash or pass empty string."""
await ops.insert_facts_batch(
conn=mock_conn, **{**self._make_batch(1), "tags_list": [""]}
)
await ops.insert_facts_batch(conn=mock_conn, **{**self._make_batch(1), "tags_list": [""]})
_, rows_data = mock_conn.executemany.call_args.args
assert rows_data[0][13] == []
+5 -19
View File
@@ -36,16 +36,10 @@ class TestPassthrough:
class TestSchemeNormalization:
def test_asyncpg_scheme_stripped(self) -> None:
assert (
to_libpq_url("postgresql+asyncpg://user:pass@host:5432/db")
== "postgresql://user:pass@host:5432/db"
)
assert to_libpq_url("postgresql+asyncpg://user:pass@host:5432/db") == "postgresql://user:pass@host:5432/db"
def test_postgres_asyncpg_scheme_normalized(self) -> None:
assert (
to_libpq_url("postgres+asyncpg://user:pass@host/db")
== "postgresql://user:pass@host/db"
)
assert to_libpq_url("postgres+asyncpg://user:pass@host/db") == "postgresql://user:pass@host/db"
def test_bare_postgres_scheme_normalized_to_postgresql(self) -> None:
assert to_libpq_url("postgres://user:pass@host/db") == "postgresql://user:pass@host/db"
@@ -68,9 +62,7 @@ class TestSslParamRename:
assert to_libpq_url("postgresql://h/d?ssl=require") == "postgresql://h/d?sslmode=require"
def test_ssl_param_preserved_among_other_params(self) -> None:
result = to_libpq_url(
"postgresql+asyncpg://h/d?ssl=require&application_name=hindsight&connect_timeout=10"
)
result = to_libpq_url("postgresql+asyncpg://h/d?ssl=require&application_name=hindsight&connect_timeout=10")
assert result.startswith("postgresql://h/d?")
# Query order should be preserved; ssl renamed, others untouched.
assert "sslmode=require" in result
@@ -80,10 +72,7 @@ class TestSslParamRename:
def test_sslmode_not_double_renamed(self) -> None:
"""An already-correct sslmode= param must not be altered."""
assert (
to_libpq_url("postgresql+asyncpg://h/d?sslmode=require")
== "postgresql://h/d?sslmode=require"
)
assert to_libpq_url("postgresql+asyncpg://h/d?sslmode=require") == "postgresql://h/d?sslmode=require"
class TestProductionConfigs:
@@ -132,10 +121,7 @@ class TestEdgeCases:
assert result == "postgresql://user:my%2Basyncpgpass@host/db"
def test_url_without_query_string(self) -> None:
assert (
to_libpq_url("postgresql+asyncpg://user:pass@host/db")
== "postgresql://user:pass@host/db"
)
assert to_libpq_url("postgresql+asyncpg://user:pass@host/db") == "postgresql://user:pass@host/db"
def test_url_with_port_and_path_only(self) -> None:
assert to_libpq_url("postgresql+asyncpg://host:5432/db") == "postgresql://host:5432/db"
@@ -126,22 +126,30 @@ class TestDeltaEditorialFusion:
# Phase 1: Ingest SEO best practices
await memory.retain_async(
bank_id=bank_id, content=SEO_BEST_PRACTICES,
document_id="seo-best-practices", request_context=request_context,
bank_id=bank_id,
content=SEO_BEST_PRACTICES,
document_id="seo-best-practices",
request_context=request_context,
)
mm_after_seo = await memory.refresh_mental_model(
bank_id=bank_id, mental_model_id=mm_id, request_context=request_context,
bank_id=bank_id,
mental_model_id=mm_id,
request_context=request_context,
)
seo_content = mm_after_seo["content"]
assert len(seo_content) > 100, f"First refresh produced too little content: {len(seo_content)} chars"
# Phase 2: Ingest brand voice -> delta refresh
await memory.retain_async(
bank_id=bank_id, content=BRAND_VOICE,
document_id="brand-voice", request_context=request_context,
bank_id=bank_id,
content=BRAND_VOICE,
document_id="brand-voice",
request_context=request_context,
)
mm_after_brand = await memory.refresh_mental_model(
bank_id=bank_id, mental_model_id=mm_id, request_context=request_context,
bank_id=bank_id,
mental_model_id=mm_id,
request_context=request_context,
)
fused = mm_after_brand["content"]
rr = mm_after_brand.get("reflect_response") or {}
@@ -156,8 +164,7 @@ class TestDeltaEditorialFusion:
"vocabulary rules": ["jargon", "leverage", "empower", "forbidden"],
}.items():
assert any(s in fused_lower for s in signals), (
f"Brand voice concept '{concept}' missing (looked for {signals}).\n"
f"Fused content:\n{fused[:500]}"
f"Brand voice concept '{concept}' missing (looked for {signals}).\nFused content:\n{fused[:500]}"
)
# SEO concepts still present (not wiped by delta)
@@ -167,8 +174,7 @@ class TestDeltaEditorialFusion:
"seo": ["meta", "e-e-a-t", "seo", "search"],
}.items():
assert any(s in fused_lower for s in signals), (
f"SEO concept '{concept}' missing (looked for {signals}).\n"
f"Fused content:\n{fused[:500]}"
f"SEO concept '{concept}' missing (looked for {signals}).\nFused content:\n{fused[:500]}"
)
# Brand voice overrides generic tone
@@ -177,15 +183,9 @@ class TestDeltaEditorialFusion:
)
# No duplicate paragraphs
lines = [
ln.strip() for ln in fused.split("\n")
if ln.strip() and not ln.strip().startswith("#")
]
lines = [ln.strip() for ln in fused.split("\n") if ln.strip() and not ln.strip().startswith("#")]
dupes = {line: cnt for line, cnt in Counter(lines).items() if cnt > 1}
assert not dupes, (
"Duplicate paragraphs:\n" +
"\n".join(f" [{c}x] {t[:80]}" for t, c in dupes.items())
)
assert not dupes, "Duplicate paragraphs:\n" + "\n".join(f" [{c}x] {t[:80]}" for t, c in dupes.items())
# based_on accumulates from both docs
obs_count = len(rr.get("based_on", {}).get("observation", []))
+41 -33
View File
@@ -146,7 +146,10 @@ async def test_delta_retain_appended_content(memory, request_context):
# Second version — original content + new content appended
# This should preserve facts from the first chunk and add new ones
v2_content = v1_content + "\n\nBob joined Google as a product manager in 2024. He previously worked at Meta on AR/VR products."
v2_content = (
v1_content
+ "\n\nBob joined Google as a product manager in 2024. He previously worked at Meta on AR/VR products."
)
v2_units = await memory.retain_async(
bank_id=bank_id,
@@ -387,9 +390,7 @@ async def test_delta_retain_links_preserved_for_unchanged_chunks(memory, request
document_id,
)
assert v2_link_count == v1_link_count, (
f"Links should be preserved: v1={v1_link_count}, v2={v2_link_count}"
)
assert v2_link_count == v1_link_count, f"Links should be preserved: v1={v1_link_count}, v2={v2_link_count}"
finally:
await memory.delete_bank(bank_id, request_context=request_context)
@@ -456,11 +457,13 @@ async def test_delta_retain_tags_propagated_to_existing_units(memory, request_co
# v1 with tag "team-a"
await memory.retain_batch_async(
bank_id=bank_id,
contents=[{
"content": content,
"document_id": document_id,
"tags": ["team-a"],
}],
contents=[
{
"content": content,
"document_id": document_id,
"tags": ["team-a"],
}
],
request_context=request_context,
)
@@ -476,11 +479,13 @@ async def test_delta_retain_tags_propagated_to_existing_units(memory, request_co
# v2 with same content but different tags
await memory.retain_batch_async(
bank_id=bank_id,
contents=[{
"content": content,
"document_id": document_id,
"tags": ["team-b", "important"],
}],
contents=[
{
"content": content,
"document_id": document_id,
"tags": ["team-b", "important"],
}
],
request_context=request_context,
)
@@ -692,7 +697,9 @@ async def test_delta_retain_empty_to_content(memory, request_context):
doc_v2 = await memory.get_document(document_id, bank_id, request_context=request_context)
assert doc_v2 is not None
assert doc_v2["memory_unit_count"] > 0 or len(v2_units) > 0, "Should have facts after updating with real content"
assert doc_v2["memory_unit_count"] > 0 or len(v2_units) > 0, (
"Should have facts after updating with real content"
)
finally:
await memory.delete_bank(bank_id, request_context=request_context)
@@ -775,11 +782,13 @@ async def test_delta_retain_with_user_entities(memory, request_context):
# v1 with user entities
await memory.retain_batch_async(
bank_id=bank_id,
contents=[{
"content": content,
"document_id": document_id,
"entities": [{"text": "Project Alpha", "type": "PROJECT"}],
}],
contents=[
{
"content": content,
"document_id": document_id,
"entities": [{"text": "Project Alpha", "type": "PROJECT"}],
}
],
request_context=request_context,
)
@@ -797,14 +806,16 @@ async def test_delta_retain_with_user_entities(memory, request_context):
v2_content = content + "\n\nThe timeline is on track for Q2 delivery."
await memory.retain_batch_async(
bank_id=bank_id,
contents=[{
"content": v2_content,
"document_id": document_id,
"entities": [
{"text": "Project Alpha", "type": "PROJECT"},
{"text": "Q2 Deadline", "type": "MILESTONE"},
],
}],
contents=[
{
"content": v2_content,
"document_id": document_id,
"entities": [
{"text": "Project Alpha", "type": "PROJECT"},
{"text": "Q2 Deadline", "type": "MILESTONE"},
],
}
],
request_context=request_context,
)
@@ -867,9 +878,7 @@ async def test_delta_retain_recall_with_chunks(memory, request_context):
facts_with_chunks = [r for r in result.results if r.chunk_id]
if facts_with_chunks and result.chunks:
for fact in facts_with_chunks:
assert fact.chunk_id in result.chunks, (
f"Chunk {fact.chunk_id} should be in returned chunks"
)
assert fact.chunk_id in result.chunks, f"Chunk {fact.chunk_id} should be in returned chunks"
finally:
await memory.delete_bank(bank_id, request_context=request_context)
@@ -1024,8 +1033,7 @@ async def test_processed_content_tokens_appended_reports_delta(memory, request_c
return
assert second > 0, "Partial-delta retain should report a positive token count"
assert second < submitted_tokens, (
"Partial-delta retain should report fewer processed tokens "
"than the full submitted payload"
"Partial-delta retain should report fewer processed tokens than the full submitted payload"
)
finally:
memory._operation_validator = None
@@ -143,9 +143,7 @@ async def test_delta_detects_unchanged_after_first_retain(memory, request_contex
bank_id,
document_id,
)
assert v2_count == v1_count, (
f"Memory unit count changed on same-content upsert: {v1_count} -> {v2_count}"
)
assert v2_count == v1_count, f"Memory unit count changed on same-content upsert: {v1_count} -> {v2_count}"
# Third retain — verify stability
v3_units = await memory.retain_async(
@@ -163,9 +161,7 @@ async def test_delta_detects_unchanged_after_first_retain(memory, request_contex
bank_id,
document_id,
)
assert v3_count == v1_count, (
f"Memory unit count changed on third upsert: {v1_count} -> {v3_count}"
)
assert v3_count == v1_count, f"Memory unit count changed on third upsert: {v1_count} -> {v3_count}"
finally:
await memory.delete_bank(bank_id, request_context=request_context)
@@ -371,9 +367,7 @@ async def test_concurrent_upserts_no_duplicates(memory_no_llm, request_context):
# splitter may cut mid-text, so later chunks might not start with the prefix.
winning_person = f"Person_{winning_version}"
wrong_version_units = [
(r["text"], r["chunk_id"], r["unit_id"])
for r in units
if winning_person not in r["text"]
(r["text"], r["chunk_id"], r["unit_id"]) for r in units if winning_person not in r["text"]
]
assert not wrong_version_units, (
f"Found {len(wrong_version_units)} memory units NOT from winning version "
@@ -397,8 +391,7 @@ async def test_concurrent_upserts_no_duplicates(memory_no_llm, request_context):
)
logger.info(
f"Concurrent test passed: version {winning_version} won with "
f"{len(unit_texts)} memory units, no duplicates"
f"Concurrent test passed: version {winning_version} won with {len(unit_texts)} memory units, no duplicates"
)
finally:
@@ -1,6 +1,7 @@
"""
Tests for document chunks API, reprocess, nodes_by_fact_type, and graph document/chunk filtering.
"""
from datetime import datetime, timezone
import httpx
@@ -223,9 +224,7 @@ async def test_graph_chunk_id_filter(api_client, bank_id):
await _retain(api_client, bank_id, "doc-chunk-test", "Alice works at Google. " * 20)
# First get chunks to find a valid chunk_id
response = await api_client.get(
f"/v1/default/banks/{bank_id}/documents/doc-chunk-test/chunks"
)
response = await api_client.get(f"/v1/default/banks/{bank_id}/documents/doc-chunk-test/chunks")
assert response.status_code == 200
chunks_data = response.json()
if chunks_data["total"] == 0:
@@ -251,11 +250,14 @@ async def test_graph_chunk_id_filter(api_client, bank_id):
@pytest.mark.asyncio
async def test_http_list_document_chunks(api_client, bank_id):
"""HTTP GET .../documents/{id}/chunks returns chunks."""
await _retain(api_client, bank_id, "doc-http-chunks", "Alice works at Google on AI research. Bob works at Meta on VR systems. " * 20)
response = await api_client.get(
f"/v1/default/banks/{bank_id}/documents/doc-http-chunks/chunks"
await _retain(
api_client,
bank_id,
"doc-http-chunks",
"Alice works at Google on AI research. Bob works at Meta on VR systems. " * 20,
)
response = await api_client.get(f"/v1/default/banks/{bank_id}/documents/doc-http-chunks/chunks")
assert response.status_code == 200
data = response.json()
assert "items" in data
@@ -266,9 +268,7 @@ async def test_http_list_document_chunks(api_client, bank_id):
@pytest.mark.asyncio
async def test_http_list_document_chunks_not_found(api_client, bank_id):
"""HTTP GET .../documents/{id}/chunks returns 404 for non-existent document."""
response = await api_client.get(
f"/v1/default/banks/{bank_id}/documents/nonexistent/chunks"
)
response = await api_client.get(f"/v1/default/banks/{bank_id}/documents/nonexistent/chunks")
assert response.status_code == 404
@@ -277,9 +277,7 @@ async def test_http_reprocess_document(api_client, bank_id):
"""HTTP POST .../documents/{id}/reprocess returns success with operation_id."""
await _retain(api_client, bank_id, "doc-http-reprocess", "Alice works at Google.")
response = await api_client.post(
f"/v1/default/banks/{bank_id}/documents/doc-http-reprocess/reprocess"
)
response = await api_client.post(f"/v1/default/banks/{bank_id}/documents/doc-http-reprocess/reprocess")
assert response.status_code == 200
data = response.json()
assert data["success"] is True
@@ -289,9 +287,7 @@ async def test_http_reprocess_document(api_client, bank_id):
@pytest.mark.asyncio
async def test_http_reprocess_document_not_found(api_client, bank_id):
"""HTTP POST .../documents/{id}/reprocess returns 404 for non-existent document."""
response = await api_client.post(
f"/v1/default/banks/{bank_id}/documents/nonexistent/reprocess"
)
response = await api_client.post(f"/v1/default/banks/{bank_id}/documents/nonexistent/reprocess")
assert response.status_code == 404
@@ -300,9 +296,7 @@ async def test_http_get_document_includes_nodes_by_fact_type(api_client, bank_id
"""HTTP GET .../documents/{id} includes nodes_by_fact_type."""
await _retain(api_client, bank_id, "doc-http-comp", "Alice works at Google on AI research.")
response = await api_client.get(
f"/v1/default/banks/{bank_id}/documents/doc-http-comp"
)
response = await api_client.get(f"/v1/default/banks/{bank_id}/documents/doc-http-comp")
assert response.status_code == 200
data = response.json()
assert "nodes_by_fact_type" in data
@@ -1,6 +1,7 @@
"""
Tests for document tracking and upsert functionality.
"""
import logging
from datetime import datetime, timezone
from unittest.mock import patch
@@ -357,9 +358,7 @@ async def test_document_persisted_with_zero_facts_async_submit(memory_real_llm,
elapsed += wait_interval
# Check if document exists
doc = await memory.get_document(
"doc-async-zero-facts", bank_id, request_context=request_context
)
doc = await memory.get_document("doc-async-zero-facts", bank_id, request_context=request_context)
if doc is not None:
break
@@ -86,9 +86,7 @@ async def _import(memory, bank_id, archive, request_context, on_conflict="skip")
inline and is already completed when submit returns.
"""
submission = await memory.import_documents_async(bank_id, archive, request_context, on_conflict)
status = await memory.get_operation_status(
bank_id, submission["operation_id"], request_context=request_context
)
status = await memory.get_operation_status(bank_id, submission["operation_id"], request_context=request_context)
assert status["status"] == "completed", status
return status["result_metadata"]
@@ -340,12 +338,8 @@ async def test_bank_roundtrip_carries_mental_model_history(memory, request_conte
mental_model_id="mm-1",
request_context=request_context,
)
await memory.update_mental_model(
bank, mental_model_id="mm-1", content="v2", request_context=request_context
)
await memory.update_mental_model(
bank, mental_model_id="mm-1", content="v3", request_context=request_context
)
await memory.update_mental_model(bank, mental_model_id="mm-1", content="v2", request_context=request_context)
await memory.update_mental_model(bank, mental_model_id="mm-1", content="v3", request_context=request_context)
# Two refreshes → two snapshots (previous content v1 then v2), newest-first.
before = await memory.get_mental_model_history(bank, "mm-1", request_context=request_context)
assert [h["previous_content"] for h in before] == ["v2", "v1"]
@@ -475,13 +469,11 @@ async def _bank_snapshot(memory, bank_id):
backend = await memory._get_backend()
async with acquire_with_retry(backend) as conn:
docs = await conn.fetch(
f"SELECT id, COALESCE(length(original_text), 0) AS len FROM {fq_table('documents')} "
f"WHERE bank_id = $1",
f"SELECT id, COALESCE(length(original_text), 0) AS len FROM {fq_table('documents')} WHERE bank_id = $1",
bank_id,
)
chunks = await conn.fetch(
f"SELECT document_id, chunk_index, length(chunk_text) AS len FROM {fq_table('chunks')} "
f"WHERE bank_id = $1",
f"SELECT document_id, chunk_index, length(chunk_text) AS len FROM {fq_table('chunks')} WHERE bank_id = $1",
bank_id,
)
ftypes = await conn.fetch(
@@ -591,9 +583,7 @@ async def test_export_import_observations(memory, request_context):
backend = await memory._get_backend()
async with acquire_with_retry(backend) as conn:
async with conn.transaction():
await _create_observation_directly(
conn, memory, src, source_ids, "Alice and Bob are colleagues."
)
await _create_observation_directly(conn, memory, src, source_ids, "Alice and Bob are colleagues.")
# Export WITHOUT observations -> none in the archive (the bank may also
# contain auto-consolidation observations; the flag is what gates them).
@@ -762,9 +752,7 @@ async def test_include_observations_requires_whole_bank_export(memory, request_c
await _retain(memory, src, "Alice works at Google.", request_context, "doc-1")
# Subset export (document_ids set) + observations must be rejected.
with pytest.raises(ValueError, match="whole bank"):
await memory.export_documents_async(
src, request_context, ["doc-1"], include_observations=True
)
await memory.export_documents_async(src, request_context, ["doc-1"], include_observations=True)
# Whole-bank export with observations is fine; subset without observations is fine.
await memory.export_documents_async(src, request_context, include_observations=True)
await memory.export_documents_async(src, request_context, ["doc-1"])
+20 -61
View File
@@ -80,11 +80,7 @@ def test_parse_entity_labels_dict_format():
def test_parse_entity_labels_dict_format_defaults():
"""Dict format parses attributes correctly."""
raw = {
"attributes": [
{"key": "topic", "values": [{"value": "math", "description": "Mathematics"}]}
]
}
raw = {"attributes": [{"key": "topic", "values": [{"value": "math", "description": "Mathematics"}]}]}
result = parse_entity_labels(raw)
assert result is not None
assert len(result.attributes) == 1
@@ -178,9 +174,7 @@ def test_build_labels_model_free_values_optional():
"""type='text', optional=True → str | None field."""
from hindsight_api.engine.retain.entity_labels import build_labels_model
labels_cfg = EntityLabelsConfig(
attributes=[LabelGroup(key="topic", type="text", optional=True, values=[])]
)
labels_cfg = EntityLabelsConfig(attributes=[LabelGroup(key="topic", type="text", optional=True, values=[])])
Model = build_labels_model(labels_cfg)
assert Model is not None
schema = Model.model_json_schema()
@@ -194,9 +188,7 @@ def test_build_labels_model_free_values_always_optional():
"""type='text' with optional=False is still treated as str | None — always optional."""
from hindsight_api.engine.retain.entity_labels import build_labels_model
labels_cfg = EntityLabelsConfig(
attributes=[LabelGroup(key="topic", type="text", optional=False, values=[])]
)
labels_cfg = EntityLabelsConfig(attributes=[LabelGroup(key="topic", type="text", optional=False, values=[])])
Model = build_labels_model(labels_cfg)
assert Model is not None
schema = Model.model_json_schema()
@@ -210,9 +202,7 @@ def test_build_labels_model_free_values_multi_still_optional():
"""type='text' is always str | None — multi-values only applies to enum types."""
from hindsight_api.engine.retain.entity_labels import build_labels_model
labels_cfg = EntityLabelsConfig(
attributes=[LabelGroup(key="tags", type="text", values=[])]
)
labels_cfg = EntityLabelsConfig(attributes=[LabelGroup(key="tags", type="text", values=[])])
Model = build_labels_model(labels_cfg)
assert Model is not None
schema = Model.model_json_schema()
@@ -226,9 +216,7 @@ def test_build_labels_model_free_values_no_values_still_creates_field():
"""type='text' group with no values still creates a field (description holds examples)."""
from hindsight_api.engine.retain.entity_labels import build_labels_model
labels_cfg = EntityLabelsConfig(
attributes=[LabelGroup(key="mood", type="text", values=[])]
)
labels_cfg = EntityLabelsConfig(attributes=[LabelGroup(key="mood", type="text", values=[])])
Model = build_labels_model(labels_cfg)
assert Model is not None
assert "mood" in Model.model_json_schema()["properties"]
@@ -549,9 +537,7 @@ def test_label_entity_post_processing_invalid_value_ignored():
from hindsight_api.engine.retain.entity_labels import build_labels_lookup, parse_entity_labels
from hindsight_api.engine.retain.fact_extraction import Entity
labels_cfg = parse_entity_labels(
[{"key": "pedagogy", "values": [{"value": "scaffolding", "description": ""}]}]
)
labels_cfg = parse_entity_labels([{"key": "pedagogy", "values": [{"value": "scaffolding", "description": ""}]}])
labels_lookup = build_labels_lookup(labels_cfg)
labels_data = {"pedagogy": "unknown_value"}
@@ -665,9 +651,7 @@ def test_free_values_label_is_single_value():
"""type='text' groups are always single-value (str | None)."""
from hindsight_api.engine.retain.entity_labels import build_labels_model, parse_entity_labels
labels_cfg = parse_entity_labels(
[{"key": "topic", "type": "text", "values": []}]
)
labels_cfg = parse_entity_labels([{"key": "topic", "type": "text", "values": []}])
Model = build_labels_model(labels_cfg)
assert Model is not None
schema = Model.model_json_schema()
@@ -681,9 +665,7 @@ def test_free_values_label_not_in_lookup():
"""type='text' group values do NOT appear in the lookup set (no fixed vocabulary)."""
from hindsight_api.engine.retain.entity_labels import build_labels_lookup, parse_entity_labels
labels_cfg = parse_entity_labels(
[{"key": "topic", "type": "text", "values": [{"value": "algebra"}]}]
)
labels_cfg = parse_entity_labels([{"key": "topic", "type": "text", "values": [{"value": "algebra"}]}])
lookup = build_labels_lookup(labels_cfg)
assert "topic:algebra" not in lookup # example hints not added to lookup
assert len(lookup) == 0
@@ -725,9 +707,7 @@ def test_optional_label_string_none_produces_no_entity():
# LLM returned the string "None" instead of JSON null — must not be stored
entity_texts = _run_label_post_processing(labels_cfg, {"engagement": "None"})
assert entity_texts == set(), (
f"String 'None' must not produce engagement:None entity, got: {entity_texts}"
)
assert entity_texts == set(), f"String 'None' must not produce engagement:None entity, got: {entity_texts}"
def test_optional_label_null_does_not_affect_other_labels():
@@ -744,9 +724,7 @@ def test_optional_label_null_does_not_affect_other_labels():
# engagement is null, but topic is set
entity_texts = _run_label_post_processing(labels_cfg, {"engagement": None, "topic": "math"})
assert "topic:math" in entity_texts, f"Expected topic:math entity, got: {entity_texts}"
assert not any("engagement" in t for t in entity_texts), (
f"engagement should not appear, got: {entity_texts}"
)
assert not any("engagement" in t for t in entity_texts), f"engagement should not appear, got: {entity_texts}"
def test_free_form_entities_false_clears_entities():
@@ -982,9 +960,7 @@ async def test_retain_extracts_single_value_label(memory_real_llm, request_conte
)
entity_names = {r["canonical_name"].lower() for r in rows}
assert "engagement:active" in entity_names, (
f"Expected 'engagement:active' label entity. Got: {entity_names}"
)
assert "engagement:active" in entity_names, f"Expected 'engagement:active' label entity. Got: {entity_names}"
# In labels-only mode, free-form entities like 'Maria' should be absent
assert not any("maria" in n for n in entity_names), (
f"Free-form entity 'Maria' should not appear in labels-only mode. Got: {entity_names}"
@@ -1054,9 +1030,7 @@ async def test_retain_extracts_multi_value_label(memory_real_llm, request_contex
entity_names = {r["canonical_name"].lower() for r in rows}
# At least one pedagogy label should be assigned
pedagogy_labels = {n for n in entity_names if n.startswith("pedagogy:")}
assert len(pedagogy_labels) > 0, (
f"Expected at least one pedagogy:* label entity. Got: {entity_names}"
)
assert len(pedagogy_labels) > 0, f"Expected at least one pedagogy:* label entity. Got: {entity_names}"
finally:
await memory.delete_bank(bank_id, request_context=request_context)
@@ -1118,9 +1092,7 @@ async def test_retain_extracts_free_values_label(memory_real_llm, request_contex
entity_names = {r["canonical_name"].lower() for r in rows}
# A topic:* entity must exist — value is free-form so we only check the prefix
topic_entities = {n for n in entity_names if n.startswith("topic:")}
assert len(topic_entities) > 0, (
f"Expected at least one topic:* free-value entity. Got: {entity_names}"
)
assert len(topic_entities) > 0, f"Expected at least one topic:* free-value entity. Got: {entity_names}"
# The value must not be the literal string "none" or "null"
assert not any(n in ("topic:none", "topic:null", "topic:n/a") for n in topic_entities), (
f"topic entity should not be a null sentinel. Got: {topic_entities}"
@@ -1188,18 +1160,14 @@ async def test_retain_extracts_map_type_entities(memory_real_llm, request_contex
entity_names = {r["canonical_name"].lower() for r in rows}
# Should have person:name:* entity
name_entities = {n for n in entity_names if n.startswith("person:name:")}
assert len(name_entities) > 0, (
f"Expected at least one person:name:* entity. Got: {entity_names}"
)
assert len(name_entities) > 0, f"Expected at least one person:name:* entity. Got: {entity_names}"
# Name should contain "alice" somewhere
assert any("alice" in n for n in name_entities), (
f"Expected person:name entity containing 'alice'. Got: {name_entities}"
)
# Should have person:organization:* entity mentioning google
org_entities = {n for n in entity_names if n.startswith("person:organization:")}
assert len(org_entities) > 0, (
f"Expected at least one person:organization:* entity. Got: {entity_names}"
)
assert len(org_entities) > 0, f"Expected at least one person:organization:* entity. Got: {entity_names}"
assert any("google" in n for n in org_entities), (
f"Expected person:organization entity containing 'google'. Got: {org_entities}"
)
@@ -2036,9 +2004,7 @@ async def test_retain_multivalue_tag_entities_all_stored(memory_real_llm, reques
# The core assertion from GH-1558: tags and entities should match
# Tags show both but entities only show a subset → BUG
assert len(use_tags) >= 2, (
f"Expected at least 2 use:* tags. Got: {use_tags}"
)
assert len(use_tags) >= 2, f"Expected at least 2 use:* tags. Got: {use_tags}"
assert len(use_entities) >= 2, (
f"GH-1558 BUG: Expected at least 2 use:* entities in unit_entities, "
f"but only got {len(use_entities)}: {use_entities}. "
@@ -2097,8 +2063,7 @@ async def test_retain_multivalue_tag_entities_second_retain(memory_real_llm, req
await memory_real_llm.retain_async(
bank_id=bank_id,
content=(
"## Authentication Flow (use-001)\n\n"
"The authentication flow use-001 handles user login via OAuth2."
"## Authentication Flow (use-001)\n\nThe authentication flow use-001 handles user login via OAuth2."
),
request_context=request_context,
)
@@ -2145,9 +2110,7 @@ async def test_retain_multivalue_tag_entities_second_retain(memory_real_llm, req
use_entities = {n for n in entity_names if n.startswith("use:")}
use_tags = {t for t in all_tags if t.startswith("use:")}
assert len(use_tags) >= 2, (
f"Expected at least 2 use:* tags on second retain. Got: {use_tags}"
)
assert len(use_tags) >= 2, f"Expected at least 2 use:* tags on second retain. Got: {use_tags}"
assert len(use_entities) >= 2, (
f"GH-1558 BUG: On second retain, expected at least 2 use:* entities "
f"but only got {len(use_entities)}: {use_entities}. "
@@ -2155,9 +2118,7 @@ async def test_retain_multivalue_tag_entities_second_retain(memory_real_llm, req
f"Entity resolution may be merging similar names."
)
missing = use_tags - use_entities
assert len(missing) == 0, (
f"GH-1558 BUG: Tags present but entities missing after second retain: {missing}"
)
assert len(missing) == 0, f"GH-1558 BUG: Tags present but entities missing after second retain: {missing}"
finally:
await memory_real_llm.delete_bank(bank_id, request_context=request_context)
@@ -2243,9 +2204,7 @@ async def test_entity_resolution_does_not_merge_distinct_label_values(memory, re
)
# We should get 2 DISTINCT entity IDs, not the same ID twice
assert len(resolved_entity_ids) == 2, (
f"Expected 2 resolved entity IDs, got {len(resolved_entity_ids)}"
)
assert len(resolved_entity_ids) == 2, f"Expected 2 resolved entity IDs, got {len(resolved_entity_ids)}"
unique_ids = set(resolved_entity_ids)
assert len(unique_ids) == 2, (
f"GH-1558 BUG: Entity resolution merged 'use:use-001' and 'use:use-002' "
@@ -305,9 +305,7 @@ class TestOracleFuzzyEntityResolution:
conn = AsyncMock()
conn.backend_type = "oracle"
conn.fetch = AsyncMock(return_value=[])
entities_data = [
{"text": f"Entity {idx}", "nearby_entities": [], "event_date": None} for idx in range(5)
]
entities_data = [{"text": f"Entity {idx}", "nearby_entities": [], "event_date": None} for idx in range(5)]
with patch.object(resolver, "_resolve_from_candidates", new_callable=AsyncMock, return_value=[]):
await resolver._resolve_entities_batch_oracle_fuzzy(
+10 -29
View File
@@ -101,25 +101,19 @@ class RateLimitingValidator(OperationValidatorExtension):
async def validate_retain(self, ctx: RetainContext) -> ValidationResult:
self.retain_counts[ctx.bank_id] += 1
if self.retain_counts[ctx.bank_id] > self.max_attempts:
return ValidationResult.reject(
f"Retain limit exceeded for bank {ctx.bank_id}"
)
return ValidationResult.reject(f"Retain limit exceeded for bank {ctx.bank_id}")
return ValidationResult.accept()
async def validate_recall(self, ctx: RecallContext) -> ValidationResult:
self.recall_counts[ctx.bank_id] += 1
if self.recall_counts[ctx.bank_id] > self.max_attempts:
return ValidationResult.reject(
f"Recall limit exceeded for bank {ctx.bank_id}"
)
return ValidationResult.reject(f"Recall limit exceeded for bank {ctx.bank_id}")
return ValidationResult.accept()
async def validate_reflect(self, ctx: ReflectContext) -> ValidationResult:
self.reflect_counts[ctx.bank_id] += 1
if self.reflect_counts[ctx.bank_id] > self.max_attempts:
return ValidationResult.reject(
f"Reflect limit exceeded for bank {ctx.bank_id}"
)
return ValidationResult.reject(f"Reflect limit exceeded for bank {ctx.bank_id}")
return ValidationResult.accept()
@@ -579,9 +573,7 @@ class TestMemoryEngineTenantAuth:
"""Tests for tenant authentication in MemoryEngine."""
@pytest.mark.asyncio
async def test_retain_requires_tenant_request_when_extension_configured(
self, memory_with_tenant
):
async def test_retain_requires_tenant_request_when_extension_configured(self, memory_with_tenant):
"""Retain fails without RequestContext when tenant extension is configured."""
memory = memory_with_tenant
@@ -621,9 +613,7 @@ class TestMemoryEngineTenantAuth:
assert "Invalid API key" in str(exc_info.value)
@pytest.mark.asyncio
async def test_recall_requires_tenant_request_when_extension_configured(
self, memory_with_tenant
):
async def test_recall_requires_tenant_request_when_extension_configured(self, memory_with_tenant):
"""Recall fails without RequestContext when tenant extension is configured."""
memory = memory_with_tenant
@@ -861,8 +851,7 @@ class RecordingPrecheckValidator(OperationValidatorExtension):
instantiable; the tests here only exercise precheck.
"""
def __init__(self, *, reject: bool = False, status_code: int = 402,
reason: str = "rejected by precheck") -> None:
def __init__(self, *, reject: bool = False, status_code: int = 402, reason: str = "rejected by precheck") -> None:
super().__init__(config={})
self.reject = reject
self.status_code = status_code
@@ -1027,9 +1016,7 @@ class TestPrecheckHttpWiring:
assert body_parses == ["retain"]
def test_precheck_rejection_returns_status_and_reason(self):
validator = RecordingPrecheckValidator(
reject=True, status_code=402, reason="Insufficient credits"
)
validator = RecordingPrecheckValidator(reject=True, status_code=402, reason="Insufficient credits")
app, _ = self._build_app(validator)
client = TestClient(app)
@@ -1045,9 +1032,7 @@ class TestPrecheckHttpWiring:
deserialises the body. We send an oversized body and verify the
body-parse counter never incremented.
"""
validator = RecordingPrecheckValidator(
reject=True, status_code=402, reason="rejected by precheck"
)
validator = RecordingPrecheckValidator(reject=True, status_code=402, reason="rejected by precheck")
app, body_parses = self._build_app(validator)
client = TestClient(app)
@@ -1063,9 +1048,7 @@ class TestPrecheckHttpWiring:
)
def test_precheck_rejection_skips_body_parse_for_recall(self):
validator = RecordingPrecheckValidator(
reject=True, status_code=402, reason="rejected"
)
validator = RecordingPrecheckValidator(reject=True, status_code=402, reason="rejected")
app, body_parses = self._build_app(validator)
client = TestClient(app)
@@ -1078,9 +1061,7 @@ class TestPrecheckHttpWiring:
assert body_parses == []
def test_precheck_rejection_skips_body_parse_for_reflect(self):
validator = RecordingPrecheckValidator(
reject=True, status_code=402, reason="rejected"
)
validator = RecordingPrecheckValidator(reject=True, status_code=402, reason="rejected")
app, body_parses = self._build_app(validator)
client = TestClient(app)
@@ -1,6 +1,7 @@
"""
Test to analyze fact extraction token usage and identify optimization opportunities.
"""
import asyncio
import logging
import time
@@ -63,9 +64,9 @@ async def test_fact_extraction_basic_analysis(llm_config):
duration = time.time() - start_time
logger.info(f"\n{'='*60}")
logger.info(f"\n{'=' * 60}")
logger.info(f"EXTRACTION RESULTS")
logger.info(f"{'='*60}")
logger.info(f"{'=' * 60}")
logger.info(f"Duration: {duration:.2f}s")
logger.info(f"Chunks: {len(chunks)}")
logger.info(f"Facts extracted: {len(facts)}")
@@ -86,13 +87,13 @@ async def test_fact_extraction_basic_analysis(llm_config):
# Show sample facts
logger.info(f"\nSample facts (first 10):")
for i, fact in enumerate(facts[:10]):
logger.info(f"\n [{i+1}] {fact.fact_type}: {fact.fact[:150]}...")
logger.info(f"\n [{i + 1}] {fact.fact_type}: {fact.fact[:150]}...")
# Show facts containing key terms
key_terms = ["kubernetes", "k8s", "CKA", "certification", "Alice"]
logger.info(f"\n{'='*60}")
logger.info(f"\n{'=' * 60}")
logger.info(f"FACTS CONTAINING KEY TERMS")
logger.info(f"{'='*60}")
logger.info(f"{'=' * 60}")
for term in key_terms:
matching = [f for f in facts if term.lower() in f.fact.lower()]
@@ -1,6 +1,7 @@
"""
Unit tests for metadata inclusion in fact extraction LLM prompt.
"""
from datetime import datetime
from hindsight_api.engine.retain.fact_extraction import _build_user_message
@@ -109,8 +109,7 @@ User: Perfect, I'll make a reservation for Saturday at 7pm.
# Output should not be more than 5x the input
assert ratio < 5.0, (
f"Output/input ratio {ratio:.2f} is too high! "
f"Input: {input_length} chars, Output: {output_length} chars"
f"Output/input ratio {ratio:.2f} is too high! Input: {input_length} chars, Output: {output_length} chars"
)
@pytest.mark.asyncio
@@ -168,16 +167,12 @@ I edited about 20 photos from my recent trip to the mountains.
# Output should not be more than 4x the input for longer texts
# (ratio should decrease as input grows)
assert ratio < 4.0, (
f"Output/input ratio {ratio:.2f} is too high! "
f"Input: {input_length} chars, Output: {output_length} chars"
f"Output/input ratio {ratio:.2f} is too high! Input: {input_length} chars, Output: {output_length} chars"
)
# Also check that individual facts aren't excessively long
max_fact_length = max(len(f.fact) for f in facts) if facts else 0
assert max_fact_length < 1000, (
f"Individual fact too long: {max_fact_length} chars. "
f"Facts should be concise."
)
assert max_fact_length < 1000, f"Individual fact too long: {max_fact_length} chars. Facts should be concise."
@pytest.mark.asyncio
async def test_token_ratio_with_locomo_conversation(self):
@@ -190,11 +185,7 @@ I edited about 20 photos from my recent trip to the mountains.
import os
# Load locomo conversation
fixture_path = os.path.join(
os.path.dirname(__file__),
"fixtures",
"locomo_conversation_sample.json"
)
fixture_path = os.path.join(os.path.dirname(__file__), "fixtures", "locomo_conversation_sample.json")
with open(fixture_path, "r") as f:
data = json.load(f)
@@ -246,8 +237,7 @@ I edited about 20 photos from my recent trip to the mountains.
max_expected_facts = num_turns * 2 # At most 2 facts per conversation turn
assert len(facts) <= max_expected_facts, (
f"Too many facts: {len(facts)} for {num_turns} conversation turns. "
f"Expected at most {max_expected_facts}."
f"Too many facts: {len(facts)} for {num_turns} conversation turns. Expected at most {max_expected_facts}."
)
@pytest.mark.asyncio
@@ -279,7 +269,7 @@ I'm planning to visit Japan next year.
)
# Count approximate number of statements (sentences)
num_statements = len([s for s in text.split('.') if s.strip()])
num_statements = len([s for s in text.split(".") if s.strip()])
print(f"\nNumber of facts test:")
print(f" Input statements: ~{num_statements}")
+27 -22
View File
@@ -5,6 +5,7 @@ This ensures that when multiple facts are extracted from a long conversation,
their relative order is preserved via time offsets, allowing retrieval to
distinguish between things said earlier vs later.
"""
import pytest
from datetime import datetime, timezone
from hindsight_api import MemoryEngine, RequestContext
@@ -20,11 +21,9 @@ async def test_fact_ordering_within_conversation(memory, request_context):
await memory.get_bank_profile(bank_id, request_context=request_context)
# Update disposition to match Marcus
await memory.update_bank_disposition(bank_id, {
"skepticism": 3,
"literalism": 3,
"empathy": 3
}, request_context=request_context)
await memory.update_bank_disposition(
bank_id, {"skepticism": 3, "literalism": 3, "empathy": 3}, request_context=request_context
)
# A conversation where Marcus changes his position
conversation = """
@@ -51,7 +50,7 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
results = await memory.recall_async(
bank_id=bank_id,
query="Marcus prediction Rams",
fact_type=['experience', 'world'],
fact_type=["experience", "world"],
budget=Budget.LOW,
max_tokens=8192,
request_context=request_context,
@@ -59,37 +58,41 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
print(f"\n=== Retrieved {len(results.results)} facts ===")
for i, result in enumerate(results.results):
print(f"{i+1}. [{result.mentioned_at}] {result.text[:100]}")
print(f"{i + 1}. [{result.mentioned_at}] {result.text[:100]}")
# Get all facts (Marcus's predictions/statements)
agent_facts = results.results
print(f"\n=== Agent facts (Marcus's statements) ===")
for i, fact in enumerate(agent_facts):
print(f"{i+1}. [{fact.mentioned_at}] {fact.text}")
print(f"{i + 1}. [{fact.mentioned_at}] {fact.text}")
# Check that agent facts have different timestamps
if len(agent_facts) >= 2:
# Parse timestamps
timestamps = [datetime.fromisoformat(f.mentioned_at.replace('Z', '+00:00')) for f in agent_facts]
timestamps = [datetime.fromisoformat(f.mentioned_at.replace("Z", "+00:00")) for f in agent_facts]
# Verify timestamps are different (have time offsets)
unique_timestamps = set(timestamps)
assert len(unique_timestamps) == len(timestamps), \
assert len(unique_timestamps) == len(timestamps), (
f"Expected unique timestamps for each fact, but got duplicates: {timestamps}"
)
# Sort facts by timestamp for ordering check
# Note: recall returns by relevance, not time order
sorted_facts = sorted(agent_facts, key=lambda f: datetime.fromisoformat(f.mentioned_at.replace('Z', '+00:00')))
sorted_timestamps = [datetime.fromisoformat(f.mentioned_at.replace('Z', '+00:00')) for f in sorted_facts]
sorted_facts = sorted(agent_facts, key=lambda f: datetime.fromisoformat(f.mentioned_at.replace("Z", "+00:00")))
sorted_timestamps = [datetime.fromisoformat(f.mentioned_at.replace("Z", "+00:00")) for f in sorted_facts]
# Verify sorted timestamps are in ascending order
for i in range(len(sorted_timestamps) - 1):
assert sorted_timestamps[i] < sorted_timestamps[i + 1], \
f"Facts should have sequential timestamps. Fact {i} ({sorted_timestamps[i]}) >= Fact {i+1} ({sorted_timestamps[i+1]})"
assert sorted_timestamps[i] < sorted_timestamps[i + 1], (
f"Facts should have sequential timestamps. Fact {i} ({sorted_timestamps[i]}) >= Fact {i + 1} ({sorted_timestamps[i + 1]})"
)
# Verify facts have distinct timestamps (ordering is preserved)
time_diffs = [(sorted_timestamps[i+1] - sorted_timestamps[i]).total_seconds() for i in range(len(sorted_timestamps) - 1)]
time_diffs = [
(sorted_timestamps[i + 1] - sorted_timestamps[i]).total_seconds() for i in range(len(sorted_timestamps) - 1)
]
print(f"\n=== Time differences between facts: {time_diffs} seconds ===")
# Each fact should have a positive time difference (uniqueness already checked above)
@@ -108,7 +111,7 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
all_text = " ".join(agent_texts)
# Look for evidence of the predictions being captured (may be merged or separate)
has_prediction_info = '27' in all_text or 'rams' in all_text or 'prediction' in all_text
has_prediction_info = "27" in all_text or "rams" in all_text or "prediction" in all_text
assert has_prediction_info, "Facts should contain information about Marcus's predictions"
print(f"\n✅ Facts capture prediction information")
@@ -121,7 +124,6 @@ Marcus: Yeah, I realized I was being too optimistic about their defense.
@pytest.mark.asyncio
async def test_multiple_documents_ordering(memory, request_context):
bank_id = "test_multi_doc_agent"
await memory.get_bank_profile(bank_id, request_context=request_context) # Auto-creates with defaults
@@ -149,7 +151,7 @@ Alice: I reconsidered the team's experience level.
bank_id=bank_id,
contents=[
{"content": conv1, "context": "project discussion 1", "event_date": time1},
{"content": conv2, "context": "project discussion 2", "event_date": time2}
{"content": conv2, "context": "project discussion 2", "event_date": time2},
],
request_context=request_context,
)
@@ -168,18 +170,21 @@ Alice: I reconsidered the team's experience level.
agent_facts = results.results
for i, fact in enumerate(agent_facts):
print(f"{i+1}. [{fact.mentioned_at}] {fact.text[:80]}")
print(f"{i + 1}. [{fact.mentioned_at}] {fact.text[:80]}")
# Each conversation's facts should have different timestamps.
# Filter out observations — they inherit their source fact's timestamp,
# which can collapse the unique set. Also skip facts without timestamps.
source_facts = [f for f in agent_facts if f.mentioned_at is not None and getattr(f, "fact_type", "") != "observation"]
source_facts = [
f for f in agent_facts if f.mentioned_at is not None and getattr(f, "fact_type", "") != "observation"
]
if len(source_facts) >= 2:
timestamps = [datetime.fromisoformat(f.mentioned_at.replace('Z', '+00:00')) for f in source_facts]
timestamps = [datetime.fromisoformat(f.mentioned_at.replace("Z", "+00:00")) for f in source_facts]
unique_timestamps = set(timestamps)
assert len(unique_timestamps) >= 2, \
assert len(unique_timestamps) >= 2, (
f"Expected multiple unique timestamps across conversations, got: {len(unique_timestamps)}"
)
print(f"\n✅ Facts from {len(source_facts)} statements have {len(unique_timestamps)} unique timestamps")
+3 -1
View File
@@ -216,7 +216,9 @@ async def test_file_retain_validation_errors(memory_no_llm_verify):
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
# Create bank
bank_response = await client.put("/v1/default/banks/test-validation-bank", json={"name": "Test Validation Bank"})
bank_response = await client.put(
"/v1/default/banks/test-validation-bank", json={"name": "Test Validation Bank"}
)
assert bank_response.status_code in (200, 201)
# Test: metadata count mismatch
@@ -98,9 +98,7 @@ def seaweedfs_container():
DockerContainer(image="chrislusf/seaweedfs:latest")
.with_exposed_ports(SEAWEEDFS_S3_PORT)
.with_volume_mapping(s3_config_file.name, "/etc/seaweedfs/s3.json", "ro")
.with_command(
f"server -s3 -s3.port={SEAWEEDFS_S3_PORT} -s3.config=/etc/seaweedfs/s3.json -ip.bind=0.0.0.0"
)
.with_command(f"server -s3 -s3.port={SEAWEEDFS_S3_PORT} -s3.config=/etc/seaweedfs/s3.json -ip.bind=0.0.0.0")
)
container.start()
@@ -309,9 +309,7 @@ async def test_api_errors_surface_the_response_body():
llm = _make_fireworks(http_client=client)
with pytest.raises(httpx.HTTPStatusError, match="invalid field 'userUploaded'"):
await llm.submit_batch(
[{"custom_id": "c0", "method": "POST", "url": "/v1/chat/completions", "body": {}}]
)
await llm.submit_batch([{"custom_id": "c0", "method": "POST", "url": "/v1/chat/completions", "body": {}}])
await client.aclose()
+21 -44
View File
@@ -33,9 +33,7 @@ def _make_client(create_side_effect=None):
elif callable(create_side_effect):
create_mock.side_effect = create_side_effect
else:
create_mock.return_value = SimpleNamespace(
name="cachedContents/test-cache-name-001"
)
create_mock.return_value = SimpleNamespace(name="cachedContents/test-cache-name-001")
client = MagicMock()
client.aio = MagicMock()
@@ -127,18 +125,12 @@ async def test_first_call_creates_subsequent_reuses():
@pytest.mark.asyncio
async def test_different_prefixes_create_separately():
client, create_mock = _make_client(
create_side_effect=lambda *a, **kw: SimpleNamespace(
name=f"cachedContents/created-{create_mock.call_count}"
)
create_side_effect=lambda *a, **kw: SimpleNamespace(name=f"cachedContents/created-{create_mock.call_count}")
)
mgr = GeminiCacheManager(client)
name_a = await mgr.get_or_create(
model="m", system_instruction="A", response_schema=None
)
name_b = await mgr.get_or_create(
model="m", system_instruction="B", response_schema=None
)
name_a = await mgr.get_or_create(model="m", system_instruction="A", response_schema=None)
name_b = await mgr.get_or_create(model="m", system_instruction="B", response_schema=None)
assert name_a != name_b
assert create_mock.call_count == 2
@@ -155,9 +147,7 @@ async def test_minimum_token_count_error_returns_none():
client, _ = _make_client(create_side_effect=err)
mgr = GeminiCacheManager(client)
result = await mgr.get_or_create(
model="m", system_instruction="tiny", response_schema=None
)
result = await mgr.get_or_create(model="m", system_instruction="tiny", response_schema=None)
assert result is None
@@ -169,9 +159,7 @@ async def test_other_sdk_errors_also_return_none():
client, _ = _make_client(create_side_effect=err)
mgr = GeminiCacheManager(client)
result = await mgr.get_or_create(
model="m", system_instruction="ok-sized prefix", response_schema=None
)
result = await mgr.get_or_create(model="m", system_instruction="ok-sized prefix", response_schema=None)
assert result is None
@@ -194,12 +182,8 @@ async def test_failed_create_does_not_poison_cache():
mgr = GeminiCacheManager(client)
first = await mgr.get_or_create(
model="m", system_instruction="prefix", response_schema=None
)
second = await mgr.get_or_create(
model="m", system_instruction="prefix", response_schema=None
)
first = await mgr.get_or_create(model="m", system_instruction="prefix", response_schema=None)
second = await mgr.get_or_create(model="m", system_instruction="prefix", response_schema=None)
assert first is None
assert second == "cachedContents/recovered"
@@ -214,9 +198,7 @@ async def test_refreshes_after_ttl_margin(monkeypatch):
"""An entry created at t=0 with ttl=10 and margin=2 should be
treated as stale at t>=8 and trigger a recreate."""
client, create_mock = _make_client(
create_side_effect=lambda *a, **kw: SimpleNamespace(
name=f"cachedContents/v{create_mock.call_count}"
)
create_side_effect=lambda *a, **kw: SimpleNamespace(name=f"cachedContents/v{create_mock.call_count}")
)
mgr = GeminiCacheManager(client, ttl_seconds=10, refresh_margin_seconds=2)
@@ -226,24 +208,18 @@ async def test_refreshes_after_ttl_margin(monkeypatch):
lambda: fake_now["t"],
)
first = await mgr.get_or_create(
model="m", system_instruction="p", response_schema=None
)
first = await mgr.get_or_create(model="m", system_instruction="p", response_schema=None)
assert first == "cachedContents/v1"
# Advance to just before the refresh boundary — should reuse.
fake_now["t"] = 1000.0 + 7.0
again = await mgr.get_or_create(
model="m", system_instruction="p", response_schema=None
)
again = await mgr.get_or_create(model="m", system_instruction="p", response_schema=None)
assert again == "cachedContents/v1"
assert create_mock.call_count == 1
# Advance past the refresh boundary — should recreate.
fake_now["t"] = 1000.0 + 9.0
refreshed = await mgr.get_or_create(
model="m", system_instruction="p", response_schema=None
)
refreshed = await mgr.get_or_create(model="m", system_instruction="p", response_schema=None)
assert refreshed == "cachedContents/v2"
assert create_mock.call_count == 2
@@ -295,9 +271,7 @@ async def test_gemini_llm_uses_cache_when_enabled(monkeypatch):
# Replace the SDK-shaped client with a fake whose caches.create returns
# a predictable name. The lazy import inside get_or_create_cached_prefix
# picks up the patched module-level GeminiCacheManager naturally.
fake_create = AsyncMock(
return_value=SimpleNamespace(name="cachedContents/from-llm-test")
)
fake_create = AsyncMock(return_value=SimpleNamespace(name="cachedContents/from-llm-test"))
llm._client = MagicMock()
llm._client.aio = MagicMock()
llm._client.aio.caches = MagicMock()
@@ -333,7 +307,9 @@ async def test_call_falls_back_to_uncached_when_cache_400s():
from hindsight_api.engine.providers.gemini_cache import GeminiCacheManager, _CacheEntry
from hindsight_api.engine.providers.gemini_llm import GeminiLLM
llm = GeminiLLM(provider="gemini", api_key="not-real-key", base_url="", model="gemini-test", prompt_cache_enabled=True)
llm = GeminiLLM(
provider="gemini", api_key="not-real-key", base_url="", model="gemini-test", prompt_cache_enabled=True
)
# Seed a cache manager entry that maps to the (now invalid) cache name.
mgr = GeminiCacheManager(client=MagicMock())
@@ -413,9 +389,7 @@ def test_fingerprint_changes_with_tools():
"""Two prefixes that differ ONLY in tools must hash differently —
otherwise a loop that adds a tool would silently reuse a stale
cache that doesn't know about it."""
tools_a = [
{"type": "function", "function": {"name": "search", "description": "search", "parameters": {}}}
]
tools_a = [{"type": "function", "function": {"name": "search", "description": "search", "parameters": {}}}]
tools_b = [
{"type": "function", "function": {"name": "search", "description": "search", "parameters": {}}},
{"type": "function", "function": {"name": "fetch", "description": "fetch", "parameters": {}}},
@@ -455,7 +429,10 @@ async def test_get_or_create_passes_tools_to_create():
mgr = GeminiCacheManager(client)
tools = [
{"type": "function", "function": {"name": "search", "description": "do a search", "parameters": {"type": "object"}}}
{
"type": "function",
"function": {"name": "search", "description": "do a search", "parameters": {"type": "object"}},
}
]
name = await mgr.get_or_create(
model="gemini-3.1-flash-lite",
@@ -34,9 +34,7 @@ from hindsight_api.engine.consolidation.consolidator import run_consolidation_jo
from hindsight_api.engine.llm_trace import LLMRequestEntry
from hindsight_api.engine.llm_wrapper import LLMConfig
_GEMINI_API_KEY = (
os.getenv("HINDSIGHT_GEMINI_API_KEY") or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")
)
_GEMINI_API_KEY = os.getenv("HINDSIGHT_GEMINI_API_KEY") or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")
_RUN = os.getenv("HINDSIGHT_RUN_GEMINI_EVALS") == "1" and bool(_GEMINI_API_KEY)
pytestmark = pytest.mark.skipif(
@@ -133,13 +133,17 @@ async def test_call_applies_safety_settings():
assert hasattr(config_arg, "safety_settings"), "Config should have safety_settings"
assert config_arg.safety_settings is not None
categories = [s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings]
categories = [
s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings
]
assert "HARM_CATEGORY_HARASSMENT" in categories
assert "HARM_CATEGORY_HATE_SPEECH" in categories
assert "HARM_CATEGORY_SEXUALLY_EXPLICIT" in categories
assert "HARM_CATEGORY_DANGEROUS_CONTENT" in categories
thresholds = [s.threshold.value if hasattr(s.threshold, "value") else str(s.threshold) for s in config_arg.safety_settings]
thresholds = [
s.threshold.value if hasattr(s.threshold, "value") else str(s.threshold) for s in config_arg.safety_settings
]
assert all(t == "BLOCK_NONE" for t in thresholds)
@@ -212,7 +216,9 @@ async def test_call_with_tools_applies_safety_settings():
assert config_arg is not None
assert config_arg.safety_settings is not None
categories = [s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings]
categories = [
s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings
]
assert "HARM_CATEGORY_HARASSMENT" in categories
@@ -266,7 +272,9 @@ async def test_with_config_overrides_instance_settings():
config_arg = provider._provider_impl._client.aio.models.generate_content.call_args.kwargs.get("config")
assert config_arg is not None
categories = [s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings]
categories = [
s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings
]
# Should use override_settings (HATE_SPEECH), not instance_settings (HARASSMENT)
assert "HARM_CATEGORY_HATE_SPEECH" in categories
assert "HARM_CATEGORY_HARASSMENT" not in categories
@@ -285,7 +293,9 @@ async def test_with_config_none_falls_back_to_instance():
config_arg = provider._provider_impl._client.aio.models.generate_content.call_args.kwargs.get("config")
assert config_arg is not None
categories = [s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings]
categories = [
s.category.value if hasattr(s.category, "value") else str(s.category) for s in config_arg.safety_settings
]
assert "HARM_CATEGORY_HARASSMENT" in categories
@@ -97,19 +97,23 @@ class TestGoogleCrossEncoder:
async def test_predict_single_query(self):
"""Test prediction with a single query and multiple documents."""
mock_creds = _make_mock_credentials()
mock_client = _make_mock_httpx_client([
_make_rank_response([("1", 0.95), ("0", 0.30)]),
])
mock_client = _make_mock_httpx_client(
[
_make_rank_response([("1", 0.95), ("0", 0.30)]),
]
)
encoder = GoogleCrossEncoder(project_id="test-project")
with patch("google.auth.default", return_value=(mock_creds, "p")):
await encoder.initialize()
encoder._client = mock_client
scores = await encoder.predict([
("What is AI?", "AI is artificial intelligence"),
("What is AI?", "The sky is blue"),
])
scores = await encoder.predict(
[
("What is AI?", "AI is artificial intelligence"),
("What is AI?", "The sky is blue"),
]
)
assert len(scores) == 2
assert scores[0] == 0.30 # id="0" -> index 0
@@ -119,21 +123,25 @@ class TestGoogleCrossEncoder:
async def test_predict_multiple_queries(self):
"""Test prediction with multiple distinct queries."""
mock_creds = _make_mock_credentials()
mock_client = _make_mock_httpx_client([
_make_rank_response([("0", 0.9), ("1", 0.1)]),
_make_rank_response([("0", 0.8)]),
])
mock_client = _make_mock_httpx_client(
[
_make_rank_response([("0", 0.9), ("1", 0.1)]),
_make_rank_response([("0", 0.8)]),
]
)
encoder = GoogleCrossEncoder(project_id="test-project")
with patch("google.auth.default", return_value=(mock_creds, "p")):
await encoder.initialize()
encoder._client = mock_client
scores = await encoder.predict([
("Query A", "Doc A1"),
("Query A", "Doc A2"),
("Query B", "Doc B1"),
])
scores = await encoder.predict(
[
("Query A", "Doc A1"),
("Query A", "Doc A2"),
("Query B", "Doc B1"),
]
)
assert len(scores) == 3
assert scores[0] == 0.9
@@ -161,10 +169,12 @@ class TestGoogleCrossEncoder:
async def test_predict_batching(self):
"""Test that >200 records are split into batches."""
mock_creds = _make_mock_credentials()
mock_client = _make_mock_httpx_client([
_make_rank_response([(str(i), 0.5) for i in range(200)]),
_make_rank_response([(str(i), 0.3) for i in range(50)]),
])
mock_client = _make_mock_httpx_client(
[
_make_rank_response([(str(i), 0.5) for i in range(200)]),
_make_rank_response([(str(i), 0.3) for i in range(50)]),
]
)
encoder = GoogleCrossEncoder(project_id="test-project")
with patch("google.auth.default", return_value=(mock_creds, "p")):
@@ -181,9 +191,11 @@ class TestGoogleCrossEncoder:
"""Test that Authorization header is sent with requests."""
mock_creds = _make_mock_credentials()
mock_creds.token = "test-bearer-token"
mock_client = _make_mock_httpx_client([
_make_rank_response([("0", 0.9)]),
])
mock_client = _make_mock_httpx_client(
[
_make_rank_response([("0", 0.9)]),
]
)
encoder = GoogleCrossEncoder(project_id="test-project")
with patch("google.auth.default", return_value=(mock_creds, "p")):
@@ -167,9 +167,7 @@ class TestEnqueueRelinkVictims:
assert await _queue_unit_ids(conn, bank_id) == [str(survivor)]
@pytest.mark.asyncio
async def test_excludes_deleted_units_themselves(
self, memory: MemoryEngine, request_context: RequestContext
):
async def test_excludes_deleted_units_themselves(self, memory: MemoryEngine, request_context: RequestContext):
"""A unit being deleted that linked TO another deleted unit must not enqueue itself."""
bank_id = f"test-gm-self-{uuid.uuid4().hex[:8]}"
await _ensure_bank(memory, bank_id, request_context)
@@ -270,9 +268,7 @@ class TestDeleteDocumentEnqueue:
class TestRelinkPass:
@pytest.mark.asyncio
async def test_drains_empty_queue_cleanly(
self, memory: MemoryEngine, request_context: RequestContext
):
async def test_drains_empty_queue_cleanly(self, memory: MemoryEngine, request_context: RequestContext):
bank_id = f"test-gm-empty-{uuid.uuid4().hex[:8]}"
await _ensure_bank(memory, bank_id, request_context)
@@ -285,9 +281,7 @@ class TestRelinkPass:
}
@pytest.mark.asyncio
async def test_skips_missing_unit_silently(
self, memory: MemoryEngine, request_context: RequestContext
):
async def test_skips_missing_unit_silently(self, memory: MemoryEngine, request_context: RequestContext):
"""Unit deleted between enqueue and drain: worker dequeues and no-ops."""
bank_id = f"test-gm-miss-{uuid.uuid4().hex[:8]}"
await _ensure_bank(memory, bank_id, request_context)
@@ -309,9 +303,7 @@ class TestRelinkPass:
assert await _queue_unit_ids(conn, bank_id) == []
@pytest.mark.asyncio
async def test_tops_up_temporal_when_under_cap(
self, memory: MemoryEngine, request_context: RequestContext
):
async def test_tops_up_temporal_when_under_cap(self, memory: MemoryEngine, request_context: RequestContext):
"""A victim under the temporal cap gets new outgoing links to neighbours
that were never linked at retain time."""
bank_id = f"test-gm-topup-{uuid.uuid4().hex[:8]}"
@@ -365,9 +357,7 @@ class TestRelinkPass:
assert await _queue_unit_ids(conn, bank_id) == []
@pytest.mark.asyncio
async def test_no_topup_when_victim_at_cap(
self, memory: MemoryEngine, request_context: RequestContext
):
async def test_no_topup_when_victim_at_cap(self, memory: MemoryEngine, request_context: RequestContext):
"""If the victim already has cap links, probing is skipped."""
bank_id = f"test-gm-atcap-{uuid.uuid4().hex[:8]}"
await _ensure_bank(memory, bank_id, request_context)
@@ -414,9 +404,7 @@ class TestRelinkPass:
class TestOrphanEntityPrune:
@pytest.mark.asyncio
async def test_prunes_entities_with_no_unit_references(
self, memory: MemoryEngine, request_context: RequestContext
):
async def test_prunes_entities_with_no_unit_references(self, memory: MemoryEngine, request_context: RequestContext):
"""An entity with zero unit_entities rows is an orphan and should be
deleted by the sweep."""
bank_id = f"test-gm-orphan-{uuid.uuid4().hex[:8]}"
@@ -435,9 +423,7 @@ class TestOrphanEntityPrune:
assert result["orphan_entities_pruned"] == 2
async with pool.acquire() as conn:
survivors = await conn.fetch(
"SELECT id FROM entities WHERE bank_id = $1 ORDER BY id", bank_id
)
survivors = await conn.fetch("SELECT id FROM entities WHERE bank_id = $1 ORDER BY id", bank_id)
survivor_ids = {str(r["id"]) for r in survivors}
assert survivor_ids == {str(referenced)}
# Confirm orphans are gone.
@@ -445,9 +431,7 @@ class TestOrphanEntityPrune:
assert orphan not in survivor_ids
@pytest.mark.asyncio
async def test_does_not_touch_other_banks(
self, memory: MemoryEngine, request_context: RequestContext
):
async def test_does_not_touch_other_banks(self, memory: MemoryEngine, request_context: RequestContext):
"""The sweep is scoped by bank — orphan entities in OTHER banks
must not be touched."""
bank_a = f"test-gm-scopea-{uuid.uuid4().hex[:8]}"
@@ -478,9 +462,7 @@ class TestOrphanEntityPrune:
class TestStaleCooccurrencePrune:
@pytest.mark.asyncio
async def test_prunes_cooccurrence_with_no_shared_unit(
self, memory: MemoryEngine, request_context: RequestContext
):
async def test_prunes_cooccurrence_with_no_shared_unit(self, memory: MemoryEngine, request_context: RequestContext):
"""Both entities still exist but no unit references both of them — the
cooccurrence row is stale and should be pruned."""
bank_id = f"test-gm-cocc-{uuid.uuid4().hex[:8]}"
@@ -515,9 +497,7 @@ class TestStaleCooccurrencePrune:
assert remaining == 0
@pytest.mark.asyncio
async def test_keeps_cooccurrence_with_shared_unit(
self, memory: MemoryEngine, request_context: RequestContext
):
async def test_keeps_cooccurrence_with_shared_unit(self, memory: MemoryEngine, request_context: RequestContext):
"""If at least one unit still references both entities, the cooccurrence
row stays."""
bank_id = f"test-gm-keep-{uuid.uuid4().hex[:8]}"
@@ -7,6 +7,7 @@ Covers:
- Per-bank vector indexes dropped on bank deletion
- retrieve_semantic_bm25_combined groups results correctly by fact_type and source
"""
import uuid
from datetime import datetime, timezone
@@ -152,10 +153,7 @@ async def test_retrieve_semantic_bm25_grouped_by_fact_type(memory, request_conte
try:
await memory.retain_async(
bank_id=bank_id,
content=(
"Alice is a software engineer at TechCorp. "
"She visited Paris in 2023 for a conference."
),
content=("Alice is a software engineer at TechCorp. She visited Paris in 2023 for a conference."),
context="background",
event_date=datetime(2023, 6, 1, tzinfo=timezone.utc),
request_context=request_context,
@@ -87,9 +87,7 @@ async def test_unique_violation_marks_failed_without_retry(memory):
try:
await memory.execute_task(task_dict)
except RetryTaskAt as exc:
pytest.fail(
f"IntegrityConstraintViolationError must not be retried, but execute_task raised {exc!r}"
)
pytest.fail(f"IntegrityConstraintViolationError must not be retried, but execute_task raised {exc!r}")
# The operation must be marked 'failed' (not left pending / retrying).
row = await pool.fetchrow(
@@ -97,9 +95,7 @@ async def test_unique_violation_marks_failed_without_retry(memory):
operation_id,
)
assert row is not None, "Operation row disappeared"
assert row["status"] == "failed", (
f"Expected status='failed' after integrity violation, got {row['status']!r}"
)
assert row["status"] == "failed", f"Expected status='failed' after integrity violation, got {row['status']!r}"
assert row["error_message"] is not None
assert "pk_chunks" in row["error_message"]
@@ -123,7 +119,7 @@ async def test_foreign_key_violation_also_not_retried(memory):
await _create_pending_operation(pool, bank_id, operation_id)
fk_violation = asyncpg.exceptions.ForeignKeyViolationError(
"insert or update on table \"memory_units\" violates foreign key constraint \"fk_bank\""
'insert or update on table "memory_units" violates foreign key constraint "fk_bank"'
)
task_dict = {
@@ -137,9 +133,7 @@ async def test_foreign_key_violation_also_not_retried(memory):
try:
await memory.execute_task(task_dict)
except RetryTaskAt as exc:
pytest.fail(
f"ForeignKeyViolationError must not be retried, but execute_task raised {exc!r}"
)
pytest.fail(f"ForeignKeyViolationError must not be retried, but execute_task raised {exc!r}")
row = await pool.fetchrow(
"SELECT status FROM async_operations WHERE operation_id = $1",
@@ -68,5 +68,3 @@ async def test_iris_parser_converts_pdf(iris_parser: IrisParser):
async def test_iris_parser_name(iris_parser: IrisParser):
"""IrisParser.name() should return 'iris'."""
assert iris_parser.name() == "iris"
@@ -48,9 +48,7 @@ def _make_replacement_body() -> str:
than one sub-batch.
"""
lines = [
f"[role: user] turn {i}: alpha bravo charlie delta echo "
f"foxtrot golf hotel india juliet"
for i in range(20)
f"[role: user] turn {i}: alpha bravo charlie delta echo foxtrot golf hotel india juliet" for i in range(20)
]
return "\n".join(lines)
@@ -79,9 +77,7 @@ async def test_large_same_id_replacement_preserves_full_body(memory, request_con
request_context=request_context,
)
doc_initial = await memory.get_document(
document_id, bank_id, request_context=request_context
)
doc_initial = await memory.get_document(document_id, bank_id, request_context=request_context)
assert doc_initial is not None
assert doc_initial["original_text"] == initial_body
@@ -95,9 +91,7 @@ async def test_large_same_id_replacement_preserves_full_body(memory, request_con
request_context=request_context,
)
doc_replaced = await memory.get_document(
document_id, bank_id, request_context=request_context
)
doc_replaced = await memory.get_document(document_id, bank_id, request_context=request_context)
assert doc_replaced is not None
stored = doc_replaced["original_text"]
@@ -105,10 +99,7 @@ async def test_large_same_id_replacement_preserves_full_body(memory, request_con
f"stored body length {len(stored)} != submitted length "
f"{len(replacement_body)} — partial replacement persisted"
)
assert stored == replacement_body, (
"stored original_text does not exactly match the submitted "
"replacement body"
)
assert stored == replacement_body, "stored original_text does not exactly match the submitted replacement body"
finally:
await memory.delete_bank(bank_id, request_context=request_context)
@@ -143,9 +134,7 @@ async def test_repeated_large_same_id_replacement_is_idempotent(memory, request_
request_context=request_context,
)
doc = await memory.get_document(
document_id, bank_id, request_context=request_context
)
doc = await memory.get_document(document_id, bank_id, request_context=request_context)
assert doc is not None, f"attempt {attempt}: document missing after retain"
assert doc["original_text"] == replacement_body, (
f"attempt {attempt}: stored body diverged from submitted body "
@@ -133,7 +133,9 @@ async def test_link_expansion_observation_graph_retrieval(memory_real_llm, reque
assert obs_result is not None and obs_result.results is not None, "Should have observations after consolidation"
# We should have observations from consolidation
assert len(obs_result.results) >= 1, f"Should have at least 1 observation about Python, got {len(obs_result.results)}"
assert len(obs_result.results) >= 1, (
f"Should have at least 1 observation about Python, got {len(obs_result.results)}"
)
# Now test graph retrieval specifically
# Query for Alice - should find Bob via shared "Python" entity
@@ -175,9 +177,7 @@ async def test_link_expansion_observation_graph_retrieval(memory_real_llm, reque
assert world_result.trace is not None, "Should have trace data for world facts"
world_retrieval_results = world_result.trace.get("retrieval_results", [])
world_graph_results = [
r for r in world_retrieval_results if r.get("method_name") == "graph"
]
world_graph_results = [r for r in world_retrieval_results if r.get("method_name") == "graph"]
if world_graph_results:
world_graph_result = [r for r in world_graph_results if r.get("fact_type") == "world"][0]
@@ -192,7 +192,9 @@ async def test_link_expansion_observation_graph_retrieval(memory_real_llm, reque
print(" Found Bob's world fact via shared 'Python' entity!")
print("\n✓ Link expansion observation test passed!")
print(" Entity traversal path verified (observations -> sources -> entities -> connected sources -> observations)")
print(
" Entity traversal path verified (observations -> sources -> entities -> connected sources -> observations)"
)
finally:
await memory.delete_bank(bank_id, request_context=request_context)
@@ -259,15 +261,11 @@ async def test_link_expansion_world_fact_graph_retrieval(memory, request_context
# Verify graph retrieval ran (it may or may not find new results depending
# on whether semantic search already found everything)
retrieval_results = result.trace.get("retrieval_results", [])
graph_results = [
r for r in retrieval_results if r.get("method_name") == "graph"
]
graph_results = [r for r in retrieval_results if r.get("method_name") == "graph"]
assert len(graph_results) > 0, "Should have graph retrieval results in trace"
# The important thing is that recall works and returns relevant results
assert result.results is not None and len(result.results) > 0, (
"Should return results for 'Alice' query"
)
assert result.results is not None and len(result.results) > 0, "Should return results for 'Alice' query"
# Alice's result should be at or near the top
result_texts = [r.text for r in result.results]
+3 -3
View File
@@ -1,4 +1,5 @@
"""Tests for link_utils datetime handling, temporal link computation, and semantic link splitting."""
import numpy as np
import pytest
from datetime import datetime, timezone, timedelta
@@ -374,6 +375,7 @@ class TestComputeSemanticLinksWithinBatch:
links = compute_semantic_links_within_batch(unit_ids, embs, top_k=3, threshold=0.5)
# Each unit should have at most 3 outgoing links
from collections import Counter
from_counts = Counter(lnk[0] for lnk in links)
for count in from_counts.values():
assert count <= 3
@@ -532,8 +534,6 @@ class TestComputeSemanticLinksAnnPgBouncerSafety:
tuning_statements = [s for s in executed_sql if guc in s]
assert tuning_statements, f"{guc} must be tuned for retain ANN under ext={ext}"
for stmt in tuning_statements:
assert stmt.strip().startswith("SET LOCAL"), (
f"{guc} must use SET LOCAL, got: {stmt}"
)
assert stmt.strip().startswith("SET LOCAL"), f"{guc} must use SET LOCAL, got: {stmt}"
# And there must not be a RESET — SET LOCAL handles it at commit.
assert not any(f"RESET {guc}" in s for s in executed_sql)
@@ -1,6 +1,7 @@
"""
Tests for list_documents pagination and tags filtering.
"""
from datetime import datetime, timezone
import pytest
@@ -28,25 +29,19 @@ async def test_list_documents_offset_pagination(memory, request_context):
await _retain_doc(memory, bank_id, f"doc-{i:02d}", [], request_context)
# All documents, ordered by created_at DESC → doc-03, doc-02, doc-01, doc-00
all_docs = await memory.list_documents(
bank_id=bank_id, limit=10, offset=0, request_context=request_context
)
all_docs = await memory.list_documents(bank_id=bank_id, limit=10, offset=0, request_context=request_context)
assert all_docs["total"] == 4
assert len(all_docs["items"]) == 4
all_ids = [d["id"] for d in all_docs["items"]]
# offset=2 should skip the first two and return the remaining two
page2 = await memory.list_documents(
bank_id=bank_id, limit=10, offset=2, request_context=request_context
)
page2 = await memory.list_documents(bank_id=bank_id, limit=10, offset=2, request_context=request_context)
assert page2["total"] == 4 # total is always the full count
assert len(page2["items"]) == 2
assert [d["id"] for d in page2["items"]] == all_ids[2:]
# offset beyond total returns empty items but correct total
beyond = await memory.list_documents(
bank_id=bank_id, limit=10, offset=10, request_context=request_context
)
beyond = await memory.list_documents(bank_id=bank_id, limit=10, offset=10, request_context=request_context)
assert beyond["total"] == 4
assert beyond["items"] == []
@@ -57,7 +57,10 @@ class TestLiteLLMSDKEmbeddings:
async def test_initialization_success(self, mock_litellm):
"""Test successful initialization."""
with patch("builtins.__import__", side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args)):
with patch(
"builtins.__import__",
side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args),
):
emb = LiteLLMSDKEmbeddings(
api_key="test_key",
model="cohere/embed-english-v3.0",
@@ -84,7 +87,10 @@ class TestLiteLLMSDKEmbeddings:
async def test_initialization_without_api_key(self, mock_litellm):
"""Test initialization without api_key (e.g. AWS Bedrock with IAM auth)."""
with patch("builtins.__import__", side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args)):
with patch(
"builtins.__import__",
side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args),
):
emb = LiteLLMSDKEmbeddings(
model="bedrock/amazon.titan-embed-text-v2:0",
batch_size=100,
@@ -119,6 +125,7 @@ class TestLiteLLMSDKEmbeddings:
async def test_initialization_missing_package(self):
"""Test initialization fails gracefully when litellm is not installed."""
def mock_import(name, *args):
if name == "litellm":
raise ImportError("No module named 'litellm'")
@@ -208,9 +215,7 @@ class TestLiteLLMSDKEmbeddings:
# Mock responses for each batch
def mock_embedding_side_effect(model, input, **kwargs):
mock_response = MagicMock()
mock_response.data = [
{"embedding": [float(i)] * 768, "index": i} for i in range(len(input))
]
mock_response.data = [{"embedding": [float(i)] * 768, "index": i} for i in range(len(input))]
return mock_response
mock_litellm.embedding.side_effect = mock_embedding_side_effect
@@ -279,7 +284,10 @@ class TestLiteLLMSDKEmbeddings:
async def test_custom_api_base(self, mock_litellm):
"""Test custom API base URL is passed to embedding calls."""
with patch("builtins.__import__", side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args)):
with patch(
"builtins.__import__",
side_effect=lambda name, *args: mock_litellm if name == "litellm" else __import__(name, *args),
):
emb = LiteLLMSDKEmbeddings(
api_key="test_key",
model="cohere/embed-english-v3.0",
@@ -230,9 +230,7 @@ async def test_litellm_explicit_param_wins_over_extra_body():
provider._acompletion = AsyncMock(return_value=_fake_litellm_response())
with patch("hindsight_api.engine.providers.litellm_llm.get_metrics_collector"):
await provider.call(
messages=[{"role": "user", "content": "hi"}], temperature=0.9, scope="test", max_retries=0
)
await provider.call(messages=[{"role": "user", "content": "hi"}], temperature=0.9, scope="test", max_retries=0)
assert provider._acompletion.call_args.kwargs.get("temperature") == 0.9
@@ -5,6 +5,7 @@ per-operation semaphores when `HINDSIGHT_API_{RETAIN,REFLECT,CONSOLIDATION}_LLM_
is set. They patch the module-level semaphore registry so they can run without
needing to re-import the module with custom env vars.
"""
import asyncio
from contextlib import AsyncExitStack
from unittest.mock import patch
@@ -79,9 +80,7 @@ class TestSemaphoresForScope:
"consolidation": consolidation_sem,
},
):
assert _semaphores_for_scope("mental_model_delta_ops") == [
llm_wrapper._global_llm_semaphore
]
assert _semaphores_for_scope("mental_model_delta_ops") == [llm_wrapper._global_llm_semaphore]
assert _semaphores_for_scope("memory_think") == [llm_wrapper._global_llm_semaphore]
assert _semaphores_for_scope("verification") == [llm_wrapper._global_llm_semaphore]
@@ -26,6 +26,7 @@ pytestmark = pytest.mark.hs_llm_mat
_PROVIDER = os.environ.get("HINDSIGHT_API_LLM_PROVIDER", "")
_MODEL = os.environ.get("HINDSIGHT_API_LLM_MODEL", "")
def _get_api_key() -> str:
"""Get API key from HINDSIGHT_API_LLM_API_KEY (CI) or provider-specific env var."""
key = os.environ.get("HINDSIGHT_API_LLM_API_KEY", "")
@@ -1,6 +1,7 @@
"""
Test that LLM calls record token metrics via the metrics collector.
"""
import os
from unittest.mock import MagicMock, patch
import pytest
@@ -31,7 +32,9 @@ async def test_llm_metrics_recorded_for_groq():
mock_collector = MagicMock(spec=MetricsCollector)
# Patch the provider module where get_metrics_collector is actually called
with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector", return_value=mock_collector):
with patch(
"hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector", return_value=mock_collector
):
llm = LLMProvider(
provider="groq",
api_key=api_key,
@@ -43,7 +46,7 @@ async def test_llm_metrics_recorded_for_groq():
response = await llm.call(
messages=[
{"role": "system", "content": "You are a helpful assistant. Always respond."},
{"role": "user", "content": "What is 2+2? Reply with just the number."}
{"role": "user", "content": "What is 2+2? Reply with just the number."},
],
max_completion_tokens=50,
scope="test_metrics",
@@ -92,7 +95,9 @@ async def test_llm_metrics_recorded_for_structured_output():
mock_collector = MagicMock(spec=MetricsCollector)
# Patch the provider module where get_metrics_collector is actually called
with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector", return_value=mock_collector):
with patch(
"hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector", return_value=mock_collector
):
llm = LLMProvider(
provider="groq",
api_key=api_key,
@@ -180,7 +185,7 @@ async def test_return_usage_returns_tuple():
result, usage = await llm.call(
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What is 2+2? Reply with just the number."}
{"role": "user", "content": "What is 2+2? Reply with just the number."},
],
max_completion_tokens=50,
return_usage=True,
+11 -7
View File
@@ -51,9 +51,11 @@ class TestMockToolCalling:
llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
# Set mock response to return tool calls
llm.set_mock_response([
{"name": "get_weather", "arguments": {"location": "Paris", "unit": "celsius"}},
])
llm.set_mock_response(
[
{"name": "get_weather", "arguments": {"location": "Paris", "unit": "celsius"}},
]
)
result = await llm.call_with_tools(
messages=[{"role": "user", "content": "What's the weather in Paris?"}],
@@ -105,10 +107,12 @@ class TestMockToolCalling:
"""Test handling multiple tool calls in one response."""
llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
llm.set_mock_response([
{"name": "get_weather", "arguments": {"location": "Paris"}},
{"name": "search", "arguments": {"query": "weather forecast"}},
])
llm.set_mock_response(
[
{"name": "get_weather", "arguments": {"location": "Paris"}},
{"name": "search", "arguments": {"query": "weather forecast"}},
]
)
result = await llm.call_with_tools(
messages=[{"role": "user", "content": "Weather in Paris and search for forecasts"}],
+4 -12
View File
@@ -307,9 +307,7 @@ async def test_retain_creates_trace_rows_with_tokens(trace_api_client, bank_id):
# Filtering by a trace_id returns only that operation run's calls.
a_trace = entry["trace_id"]
resp = await trace_api_client.get(
f"/v1/default/banks/{bank_id}/llm-requests", params={"trace_id": a_trace}
)
resp = await trace_api_client.get(f"/v1/default/banks/{bank_id}/llm-requests", params={"trace_id": a_trace})
assert resp.status_code == 200
filtered = resp.json()
assert filtered["total"] >= 1
@@ -402,9 +400,7 @@ async def test_memory_ids_mapped_to_retain_and_consolidation(trace_api_client, b
# the retain that produced it (memory_ids) and any consolidation that consumed
# it as a source (source_memory_ids).
by_mem = (
await trace_api_client.get(
f"/v1/default/banks/{bank_id}/llm-requests", params={"memory_id": created[0]}
)
await trace_api_client.get(f"/v1/default/banks/{bank_id}/llm-requests", params={"memory_id": created[0]})
).json()
assert by_mem["total"] >= 1
for it in by_mem["items"]:
@@ -433,9 +429,7 @@ async def test_filter_by_status_and_operation(trace_api_client, bank_id):
assert item["status"] == "success"
assert item["operation"] == "retain"
response = await trace_api_client.get(
f"/v1/default/banks/{bank_id}/llm-requests", params={"status": "error"}
)
response = await trace_api_client.get(f"/v1/default/banks/{bank_id}/llm-requests", params={"status": "error"})
assert response.json()["total"] == 0
@@ -448,9 +442,7 @@ async def test_stats_endpoint_includes_tokens(trace_api_client, bank_id):
)
await asyncio.sleep(1.0)
response = await trace_api_client.get(
f"/v1/default/banks/{bank_id}/llm-requests/stats", params={"period": "1d"}
)
response = await trace_api_client.get(f"/v1/default/banks/{bank_id}/llm-requests/stats", params={"period": "1d"})
assert response.status_code == 200
data = response.json()
assert data["trunc"] == "day"
@@ -72,21 +72,23 @@ def create_mock_facts_from_content(content: str, ratio: float = 1.5, max_facts:
If content has N sentences, return approximately N * ratio facts (capped at max_facts).
"""
# Estimate sentences by splitting on periods
sentences = [s.strip() for s in content.split('.') if s.strip()]
sentences = [s.strip() for s in content.split(".") if s.strip()]
num_facts = min(max(1, int(len(sentences) * ratio)), max_facts)
facts = []
for i in range(num_facts):
facts.append({
"what": f"Mock fact {i}: Something happened based on the content",
"when": "2024-06-15",
"where": "San Francisco",
"who": "John, Sarah",
"why": "Business reasons",
"fact_type": "world",
"entities": [{"text": "John", "type": "PERSON"}],
"causal_relations": [],
})
facts.append(
{
"what": f"Mock fact {i}: Something happened based on the content",
"when": "2024-06-15",
"where": "San Francisco",
"who": "John, Sarah",
"why": "Business reasons",
"fact_type": "world",
"entities": [{"text": "John", "type": "PERSON"}],
"causal_relations": [],
}
)
return facts
@@ -122,6 +124,7 @@ class TestLargeBatchRetain:
@pytest.fixture
def disable_observations(self):
from hindsight_api.config import _get_raw_config
config = _get_raw_config()
original = config.enable_observations
config.enable_observations = False
@@ -147,11 +150,13 @@ class TestLargeBatchRetain:
contents = []
for i in range(num_items):
content_text = generate_content(chars_per_item)
contents.append({
"content": content_text,
"context": f"Test content item {i + 1} of {num_items}",
"event_date": datetime.now(UTC),
})
contents.append(
{
"content": content_text,
"context": f"Test content item {i + 1} of {num_items}",
"event_date": datetime.now(UTC),
}
)
actual_total_chars = sum(len(c["content"]) for c in contents)
logger.info(f"Created {num_items} content items with {actual_total_chars:,} total chars")
@@ -191,7 +196,7 @@ class TestLargeBatchRetain:
return response_dict
# Patch LLMProvider.call at the class level
with patch('hindsight_api.engine.llm_wrapper.LLMProvider.call', new=mock_llm_call):
with patch("hindsight_api.engine.llm_wrapper.LLMProvider.call", new=mock_llm_call):
start_time = time.time()
try:
@@ -247,11 +252,13 @@ class TestLargeBatchRetain:
contents = []
for i in range(num_items):
contents.append({
"content": generate_content(chars_per_item),
"context": f"Chunk test item {i + 1}",
"event_date": datetime.now(UTC),
})
contents.append(
{
"content": generate_content(chars_per_item),
"context": f"Chunk test item {i + 1}",
"event_date": datetime.now(UTC),
}
)
actual_total_chars = sum(len(c["content"]) for c in contents)
logger.info(f"Created {num_items} items with {actual_total_chars:,} chars (should trigger chunking)")
@@ -275,7 +282,7 @@ class TestLargeBatchRetain:
return response_dict, TokenUsage(input_tokens=100, output_tokens=50)
return response_dict
with patch('hindsight_api.engine.llm_wrapper.LLMProvider.call', new=mock_llm_call):
with patch("hindsight_api.engine.llm_wrapper.LLMProvider.call", new=mock_llm_call):
start_time = time.time()
result = await memory.retain_batch_async(
@@ -305,9 +312,18 @@ class TestLargeBatchRetain:
async def mock_llm_call(*args, **kwargs):
# Small delay to simulate real LLM latency
await asyncio.sleep(0.01)
mock_facts = [{"what": "Test fact", "when": "now", "where": "here",
"who": "someone", "why": "testing", "fact_type": "world",
"entities": [], "causal_relations": []}]
mock_facts = [
{
"what": "Test fact",
"when": "now",
"where": "here",
"who": "someone",
"why": "testing",
"fact_type": "world",
"entities": [],
"causal_relations": [],
}
]
response_dict = {"facts": mock_facts}
return_usage = kwargs.get("return_usage", False)
@@ -315,16 +331,18 @@ class TestLargeBatchRetain:
return response_dict, TokenUsage(input_tokens=10, output_tokens=10)
return response_dict
with patch('hindsight_api.engine.llm_wrapper.LLMProvider.call', new=mock_llm_call):
with patch("hindsight_api.engine.llm_wrapper.LLMProvider.call", new=mock_llm_call):
# Run 10 concurrent retain operations
tasks = []
for i in range(10):
bank_id = f"pool-test-{uuid.uuid4().hex[:8]}"
contents = [{
"content": f"Test content for concurrent operation {i}. " * 50,
"context": f"Pool test {i}",
"event_date": datetime.now(UTC),
}]
contents = [
{
"content": f"Test content for concurrent operation {i}. " * 50,
"context": f"Pool test {i}",
"event_date": datetime.now(UTC),
}
]
tasks.append(
memory.retain_batch_async(bank_id=bank_id, contents=contents, request_context=request_context)
)
+96 -77
View File
@@ -43,14 +43,15 @@ class TestMainModuleExtensionLoading:
loaded_extensions[name] = result
return result
with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
patch("hindsight_api.main.create_app") as mock_create_app, \
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
patch("hindsight_api.main.load_extension", side_effect=tracking_load_extension), \
patch("hindsight_api.main.DefaultExtensionContext"), \
patch("hindsight_api.main.print_banner"), \
patch("uvicorn.run"): # Don't actually start uvicorn
with (
patch("hindsight_api.main.MemoryEngine") as mock_engine,
patch("hindsight_api.main.create_app") as mock_create_app,
patch("hindsight_api.main._get_raw_config") as mock_get_config,
patch("hindsight_api.main.load_extension", side_effect=tracking_load_extension),
patch("hindsight_api.main.DefaultExtensionContext"),
patch("hindsight_api.main.print_banner"),
patch("uvicorn.run"),
): # Don't actually start uvicorn
mock_config = MagicMock()
mock_config.host = "0.0.0.0"
mock_config.port = 8888
@@ -63,17 +64,21 @@ class TestMainModuleExtensionLoading:
mock_create_app.return_value = MagicMock()
# Mock sys.argv to simulate CLI invocation
with patch.object(sys, 'argv', ['hindsight-api']):
with patch.object(sys, "argv", ["hindsight-api"]):
from hindsight_api.main import main
main()
# Verify TENANT extension was loaded
assert "TENANT" in loaded_extensions, \
assert "TENANT" in loaded_extensions, (
"main.py did not call load_extension('TENANT', ...) - extensions not loaded!"
assert loaded_extensions["TENANT"] is not None, \
)
assert loaded_extensions["TENANT"] is not None, (
"load_extension('TENANT', ...) returned None despite env var being set"
assert isinstance(loaded_extensions["TENANT"], MockTenantExtension), \
)
assert isinstance(loaded_extensions["TENANT"], MockTenantExtension), (
f"Expected MockTenantExtension, got {type(loaded_extensions['TENANT'])}"
)
def test_main_loads_operation_validator_when_configured(self, monkeypatch):
"""
@@ -94,14 +99,15 @@ class TestMainModuleExtensionLoading:
loaded_extensions[name] = result
return result
with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
patch("hindsight_api.main.create_app") as mock_create_app, \
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
patch("hindsight_api.main.load_extension", side_effect=tracking_load_extension), \
patch("hindsight_api.main.DefaultExtensionContext"), \
patch("hindsight_api.main.print_banner"), \
patch("uvicorn.run"):
with (
patch("hindsight_api.main.MemoryEngine") as mock_engine,
patch("hindsight_api.main.create_app") as mock_create_app,
patch("hindsight_api.main._get_raw_config") as mock_get_config,
patch("hindsight_api.main.load_extension", side_effect=tracking_load_extension),
patch("hindsight_api.main.DefaultExtensionContext"),
patch("hindsight_api.main.print_banner"),
patch("uvicorn.run"),
):
mock_config = MagicMock()
mock_config.host = "0.0.0.0"
mock_config.port = 8888
@@ -113,12 +119,14 @@ class TestMainModuleExtensionLoading:
mock_engine.return_value = MagicMock()
mock_create_app.return_value = MagicMock()
with patch.object(sys, 'argv', ['hindsight-api']):
with patch.object(sys, "argv", ["hindsight-api"]):
from hindsight_api.main import main
main()
assert "OPERATION_VALIDATOR" in loaded_extensions, \
assert "OPERATION_VALIDATOR" in loaded_extensions, (
"main.py did not call load_extension('OPERATION_VALIDATOR', ...)"
)
assert loaded_extensions["OPERATION_VALIDATOR"] is not None
assert isinstance(loaded_extensions["OPERATION_VALIDATOR"], MockOperationValidator)
@@ -141,13 +149,14 @@ class TestMainModuleExtensionLoading:
memory_engine_calls.append({"args": args, "kwargs": kwargs})
return MagicMock()
with patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine), \
patch("hindsight_api.main.create_app") as mock_create_app, \
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
patch("hindsight_api.main.DefaultExtensionContext"), \
patch("hindsight_api.main.print_banner"), \
patch("uvicorn.run"):
with (
patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine),
patch("hindsight_api.main.create_app") as mock_create_app,
patch("hindsight_api.main._get_raw_config") as mock_get_config,
patch("hindsight_api.main.DefaultExtensionContext"),
patch("hindsight_api.main.print_banner"),
patch("uvicorn.run"),
):
mock_config = MagicMock()
mock_config.host = "0.0.0.0"
mock_config.port = 8888
@@ -158,8 +167,9 @@ class TestMainModuleExtensionLoading:
mock_get_config.return_value = mock_config
mock_create_app.return_value = MagicMock()
with patch.object(sys, 'argv', ['hindsight-api']):
with patch.object(sys, "argv", ["hindsight-api"]):
from hindsight_api.main import main
main()
# Verify MemoryEngine was called
@@ -168,10 +178,10 @@ class TestMainModuleExtensionLoading:
call_kwargs = memory_engine_calls[0]["kwargs"]
# THE CRITICAL ASSERTION: tenant_extension must be passed and not None
assert "tenant_extension" in call_kwargs, \
"MemoryEngine was not called with tenant_extension parameter!"
assert call_kwargs["tenant_extension"] is not None, \
assert "tenant_extension" in call_kwargs, "MemoryEngine was not called with tenant_extension parameter!"
assert call_kwargs["tenant_extension"] is not None, (
"tenant_extension was None - main.py did not pass loaded extension to MemoryEngine!"
)
def test_main_sets_extension_context_on_tenant_extension(self, monkeypatch):
"""
@@ -198,13 +208,14 @@ class TestMainModuleExtensionLoading:
context_created.append(ctx)
return ctx
with patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine), \
patch("hindsight_api.main.create_app") as mock_create_app, \
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
patch("hindsight_api.main.DefaultExtensionContext", side_effect=capture_context), \
patch("hindsight_api.main.print_banner"), \
patch("uvicorn.run"):
with (
patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine),
patch("hindsight_api.main.create_app") as mock_create_app,
patch("hindsight_api.main._get_raw_config") as mock_get_config,
patch("hindsight_api.main.DefaultExtensionContext", side_effect=capture_context),
patch("hindsight_api.main.print_banner"),
patch("uvicorn.run"),
):
mock_config = MagicMock()
mock_config.host = "0.0.0.0"
mock_config.port = 8888
@@ -215,15 +226,15 @@ class TestMainModuleExtensionLoading:
mock_get_config.return_value = mock_config
mock_create_app.return_value = MagicMock()
with patch.object(sys, 'argv', ['hindsight-api']):
with patch.object(sys, "argv", ["hindsight-api"]):
from hindsight_api.main import main
main()
# Verify context was created and set
assert len(context_created) == 1, "DefaultExtensionContext should be created"
assert captured_tenant_ext[0] is not None, "Tenant extension should be captured"
assert captured_tenant_ext[0]._context_set, \
"set_context was not called on tenant extension"
assert captured_tenant_ext[0]._context_set, "set_context was not called on tenant extension"
def test_main_works_without_extensions(self, monkeypatch):
"""
@@ -240,12 +251,13 @@ class TestMainModuleExtensionLoading:
memory_engine_calls.append({"args": args, "kwargs": kwargs})
return MagicMock()
with patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine), \
patch("hindsight_api.main.create_app") as mock_create_app, \
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
patch("hindsight_api.main.print_banner"), \
patch("uvicorn.run"):
with (
patch("hindsight_api.main.MemoryEngine", side_effect=capture_memory_engine),
patch("hindsight_api.main.create_app") as mock_create_app,
patch("hindsight_api.main._get_raw_config") as mock_get_config,
patch("hindsight_api.main.print_banner"),
patch("uvicorn.run"),
):
mock_config = MagicMock()
mock_config.host = "0.0.0.0"
mock_config.port = 8888
@@ -256,8 +268,9 @@ class TestMainModuleExtensionLoading:
mock_get_config.return_value = mock_config
mock_create_app.return_value = MagicMock()
with patch.object(sys, 'argv', ['hindsight-api']):
with patch.object(sys, "argv", ["hindsight-api"]):
from hindsight_api.main import main
main()
# Should work without extensions
@@ -285,12 +298,13 @@ class TestMainModuleExtensionLoading:
mock_app = MagicMock()
with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
patch("hindsight_api.main.create_app", return_value=mock_app), \
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
patch("hindsight_api.main.print_banner"), \
patch("uvicorn.run", side_effect=capture_uvicorn_run):
with (
patch("hindsight_api.main.MemoryEngine") as mock_engine,
patch("hindsight_api.main.create_app", return_value=mock_app),
patch("hindsight_api.main._get_raw_config") as mock_get_config,
patch("hindsight_api.main.print_banner"),
patch("uvicorn.run", side_effect=capture_uvicorn_run),
):
mock_config = MagicMock()
mock_config.host = "0.0.0.0"
mock_config.port = 8888
@@ -301,14 +315,14 @@ class TestMainModuleExtensionLoading:
mock_get_config.return_value = mock_config
mock_engine.return_value = MagicMock()
with patch.object(sys, 'argv', ['hindsight-api', '--workers', '1']):
with patch.object(sys, "argv", ["hindsight-api", "--workers", "1"]):
from hindsight_api.main import main
main()
assert len(uvicorn_calls) == 1
# With workers=1, should pass app object, not import string
assert uvicorn_calls[0]["app"] is mock_app, \
"main.py should pass app object (not import string) when workers=1"
assert uvicorn_calls[0]["app"] is mock_app, "main.py should pass app object (not import string) when workers=1"
def test_main_uses_import_string_for_multiple_workers(self, monkeypatch):
"""
@@ -325,12 +339,13 @@ class TestMainModuleExtensionLoading:
def capture_uvicorn_run(**kwargs):
uvicorn_calls.append(kwargs)
with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
patch("hindsight_api.main.create_app") as mock_create_app, \
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
patch("hindsight_api.main.print_banner"), \
patch("uvicorn.run", side_effect=capture_uvicorn_run):
with (
patch("hindsight_api.main.MemoryEngine") as mock_engine,
patch("hindsight_api.main.create_app") as mock_create_app,
patch("hindsight_api.main._get_raw_config") as mock_get_config,
patch("hindsight_api.main.print_banner"),
patch("uvicorn.run", side_effect=capture_uvicorn_run),
):
mock_config = MagicMock()
mock_config.host = "0.0.0.0"
mock_config.port = 8888
@@ -342,14 +357,16 @@ class TestMainModuleExtensionLoading:
mock_engine.return_value = MagicMock()
mock_create_app.return_value = MagicMock()
with patch.object(sys, 'argv', ['hindsight-api', '--workers', '2']):
with patch.object(sys, "argv", ["hindsight-api", "--workers", "2"]):
from hindsight_api.main import main
main()
assert len(uvicorn_calls) == 1
# With workers > 1, should use import string
assert uvicorn_calls[0]["app"] == "hindsight_api.server:app", \
assert uvicorn_calls[0]["app"] == "hindsight_api.server:app", (
"main.py should use import string when workers > 1"
)
assert uvicorn_calls[0]["workers"] == 2
def test_main_sets_keepalive_timeout(self, monkeypatch):
@@ -366,12 +383,13 @@ class TestMainModuleExtensionLoading:
def capture_uvicorn_run(**kwargs):
uvicorn_calls.append(kwargs)
with patch("hindsight_api.main.MemoryEngine") as mock_engine, \
patch("hindsight_api.main.create_app") as mock_create_app, \
patch("hindsight_api.main._get_raw_config") as mock_get_config, \
patch("hindsight_api.main.print_banner"), \
patch("uvicorn.run", side_effect=capture_uvicorn_run):
with (
patch("hindsight_api.main.MemoryEngine") as mock_engine,
patch("hindsight_api.main.create_app") as mock_create_app,
patch("hindsight_api.main._get_raw_config") as mock_get_config,
patch("hindsight_api.main.print_banner"),
patch("uvicorn.run", side_effect=capture_uvicorn_run),
):
mock_config = MagicMock()
mock_config.host = "0.0.0.0"
mock_config.port = 8888
@@ -383,15 +401,16 @@ class TestMainModuleExtensionLoading:
mock_engine.return_value = MagicMock()
mock_create_app.return_value = MagicMock()
with patch.object(sys, 'argv', ['hindsight-api']):
with patch.object(sys, "argv", ["hindsight-api"]):
from hindsight_api.main import main
main()
assert len(uvicorn_calls) == 1
assert "timeout_keep_alive" in uvicorn_calls[0], \
"uvicorn config must set timeout_keep_alive"
assert uvicorn_calls[0]["timeout_keep_alive"] > 15, \
assert "timeout_keep_alive" in uvicorn_calls[0], "uvicorn config must set timeout_keep_alive"
assert uvicorn_calls[0]["timeout_keep_alive"] > 15, (
"timeout_keep_alive must exceed aiohttp's 15s client default"
)
# Mock extensions for testing
@@ -50,7 +50,9 @@ async def _insert_fact(conn, bank_id: str) -> None:
@pytest.mark.asyncio
async def test_reconcile_submits_eligible_skips_disabled_and_in_flight(memory: MemoryEngine, request_context, monkeypatch):
async def test_reconcile_submits_eligible_skips_disabled_and_in_flight(
memory: MemoryEngine, request_context, monkeypatch
):
"""Reconcile enqueues consolidation for eligible banks and skips banks that
disabled auto-consolidation or already have an in-flight consolidation."""
eligible = await _make_bank(
@@ -111,7 +111,7 @@ async def test_maintenance_loop_targets_only_affected_tenants(
)
if i % 2 == 0:
await conn.execute(
f"INSERT INTO \"{s}\".audit_log (action, transport, started_at) "
f'INSERT INTO "{s}".audit_log (action, transport, started_at) '
f"VALUES ('OLD', 'system', now() - INTERVAL '10 days')"
)
audit_with_old.add(s)
@@ -119,7 +119,7 @@ async def test_maintenance_loop_targets_only_affected_tenants(
await conn.execute(f"INSERT INTO \"{s}\".llm_requests (status, started_at) VALUES ('success', now())")
if i % 3 == 0:
await conn.execute(
f"INSERT INTO \"{s}\".llm_requests (status, started_at) "
f'INSERT INTO "{s}".llm_requests (status, started_at) '
f"VALUES ('success', now() - INTERVAL '3 days')"
)
llm_with_old.add(s)
@@ -53,7 +53,9 @@ async def _make_bank(memory: MemoryEngine, request_context, suffix: str) -> str:
return bank_id
async def _insert_fact(conn, bank_id: str, *, fact_type: str = "experience", consolidated: bool = False, failed: bool = False) -> None:
async def _insert_fact(
conn, bank_id: str, *, fact_type: str = "experience", consolidated: bool = False, failed: bool = False
) -> None:
await conn.execute(
"""
INSERT INTO memory_units (id, bank_id, text, fact_type, created_at, consolidated_at, consolidation_failed_at)
@@ -141,9 +143,7 @@ async def test_schemas_with_expired_rows(memory: MemoryEngine):
)
# 7-day cutoff: the 10-day-old row makes 'public' expired.
expired_7 = await conn.fetch(
"SELECT * FROM public.schemas_with_expired_rows('audit_log', 'started_at', 7)"
)
expired_7 = await conn.fetch("SELECT * FROM public.schemas_with_expired_rows('audit_log', 'started_at', 7)")
assert "public" in {r[0] for r in expired_7}
# 100-year cutoff: nothing is that old.
@@ -35,16 +35,12 @@ class TestCollectCoercibleTypes:
def test_anyof_nullable_array(self):
"""list[str] | None → anyOf with array and null."""
arrays, objects = self._run(
{"anyOf": [{"type": "array", "items": {"type": "string"}}, {"type": "null"}]}
)
arrays, objects = self._run({"anyOf": [{"type": "array", "items": {"type": "string"}}, {"type": "null"}]})
assert "p" in arrays
def test_oneof_nullable_array(self):
"""oneOf variant."""
arrays, objects = self._run(
{"oneOf": [{"type": "array", "items": {"type": "string"}}, {"type": "null"}]}
)
arrays, objects = self._run({"oneOf": [{"type": "array", "items": {"type": "string"}}, {"type": "null"}]})
assert "p" in arrays
# --- object types ---
@@ -136,9 +132,7 @@ class TestCoerceStringJson:
assert result["metadata"] == {}
def test_native_dict_passthrough(self):
result = _coerce_string_json(
{"metadata": {"key": "value"}}, array_params=set(), object_params={"metadata"}
)
result = _coerce_string_json({"metadata": {"key": "value"}}, array_params=set(), object_params={"metadata"})
assert result["metadata"] == {"key": "value"}
# --- non-coercible values left untouched ---
@@ -153,16 +147,12 @@ class TestCoerceStringJson:
def test_wrong_json_type_not_coerced_list(self):
"""String that parses to a dict should NOT be coerced for an array param."""
result = _coerce_string_json(
{"tags": '{"key": "value"}'}, array_params={"tags"}, object_params=set()
)
result = _coerce_string_json({"tags": '{"key": "value"}'}, array_params={"tags"}, object_params=set())
assert result["tags"] == '{"key": "value"}'
def test_wrong_json_type_not_coerced_dict(self):
"""String that parses to a list should NOT be coerced for an object param."""
result = _coerce_string_json(
{"metadata": '["a", "b"]'}, array_params=set(), object_params={"metadata"}
)
result = _coerce_string_json({"metadata": '["a", "b"]'}, array_params=set(), object_params={"metadata"})
assert result["metadata"] == '["a", "b"]'
def test_string_param_not_touched(self):
@@ -175,15 +165,11 @@ class TestCoerceStringJson:
assert result["query"] == '["looks", "like", "json"]'
def test_integer_param_not_touched(self):
result = _coerce_string_json(
{"max_tokens": 4096}, array_params=set(), object_params=set()
)
result = _coerce_string_json({"max_tokens": 4096}, array_params=set(), object_params=set())
assert result["max_tokens"] == 4096
def test_boolean_param_not_touched(self):
result = _coerce_string_json(
{"verbose": True}, array_params=set(), object_params=set()
)
result = _coerce_string_json({"verbose": True}, array_params=set(), object_params=set())
assert result["verbose"] is True
def test_missing_param_no_error(self):
@@ -277,13 +263,15 @@ class TestMakeToolsTolerantIntegration:
mcp, captured = self._create_mcp_with_tool()
_make_tools_tolerant(mcp)
tool = _get_mcp_tools(mcp)["test_tool"]
await tool.run({
"query": "hi",
"max_tokens": 200,
"verbose": True,
"tags": ["x"],
"metadata": {"a": "b"},
})
await tool.run(
{
"query": "hi",
"max_tokens": 200,
"verbose": True,
"tags": ["x"],
"metadata": {"a": "b"},
}
)
assert captured["query"] == "hi"
assert captured["max_tokens"] == 200
assert captured["verbose"] is True
@@ -296,11 +284,13 @@ class TestMakeToolsTolerantIntegration:
mcp, captured = self._create_mcp_with_tool()
_make_tools_tolerant(mcp)
tool = _get_mcp_tools(mcp)["test_tool"]
await tool.run({
"query": "hi",
"tags": '["x"]',
"explanation": "LLM added this",
})
await tool.run(
{
"query": "hi",
"tags": '["x"]',
"explanation": "LLM added this",
}
)
assert captured["tags"] == ["x"]
assert "explanation" not in captured
@@ -78,10 +78,13 @@ async def test_filter_mcp_tools_returns_empty_set():
class DenyAllValidator(OperationValidatorExtension):
async def validate_retain(self, ctx):
return ValidationResult.accept()
async def validate_recall(self, ctx):
return ValidationResult.accept()
async def validate_reflect(self, ctx):
return ValidationResult.accept()
async def filter_mcp_tools(self, bank_id, request_context, tools):
return frozenset()
@@ -189,10 +192,13 @@ async def test_validator_cannot_add_tools_beyond_bank_config():
class PermissiveValidator(OperationValidatorExtension):
async def validate_retain(self, ctx):
return ValidationResult.accept()
async def validate_recall(self, ctx):
return ValidationResult.accept()
async def validate_reflect(self, ctx):
return ValidationResult.accept()
async def filter_mcp_tools(self, bank_id, request_context, tools):
return tools | {"retain", "delete_bank"}
@@ -241,15 +247,19 @@ async def test_validator_cannot_add_tools_beyond_bank_config():
async def test_validator_exception_fails_open(caplog):
"""If filter_mcp_tools raises, all tools remain visible and warning is logged."""
import logging
caplog.set_level(logging.WARNING)
class BrokenValidator(OperationValidatorExtension):
async def validate_retain(self, ctx):
return ValidationResult.accept()
async def validate_recall(self, ctx):
return ValidationResult.accept()
async def validate_reflect(self, ctx):
return ValidationResult.accept()
async def filter_mcp_tools(self, bank_id, request_context, tools):
raise RuntimeError("Policy backend unreachable")
@@ -250,9 +250,7 @@ class TestDeltaRefreshPlumbing:
# First refresh: establishes last_refreshed_source_query.
patch_reflect(memory, text="# Team\n\nFirst pass.")
patch_llm_call(memory, returns="unused-first")
await memory.refresh_mental_model(
bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
)
await memory.refresh_mental_model(bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context)
# Now change the source_query — a genuine topic shift.
await memory.update_mental_model(
@@ -289,15 +287,7 @@ class TestDeltaRefreshPlumbing:
bank_id = f"test-delta-apply-{uuid.uuid4().hex[:8]}"
await memory.get_bank_profile(bank_id, request_context=request_context)
existing = (
"# Team\n"
"\n"
"Alice is the lead.\n"
"\n"
"## Members\n"
"\n"
"- Alice — lead\n"
)
existing = "# Team\n\nAlice is the lead.\n\n## Members\n\n- Alice — lead\n"
mm = await memory.create_mental_model(
bank_id=bank_id,
name="Team Info",
@@ -311,9 +301,7 @@ class TestDeltaRefreshPlumbing:
# render of the parsed existing content. This also seeds the tracking column.
patch_reflect(memory, text="ignored — full mode candidate")
patch_llm_call(memory, returns=[]) # zero ops
await memory.refresh_mental_model(
bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
)
await memory.refresh_mental_model(bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context)
# Second refresh: a new fact arrives; LLM returns one append_block op.
candidate = "# Team\n\nAlice is the lead. Bob joined as junior engineer."
@@ -386,15 +374,7 @@ class TestDeltaRefreshPlumbing:
bank_id = f"test-delta-noop-{uuid.uuid4().hex[:8]}"
await memory.get_bank_profile(bank_id, request_context=request_context)
existing = (
"# Team\n"
"\n"
"Alice is the lead.\n"
"\n"
"## Members\n"
"\n"
"- Alice\n"
)
existing = "# Team\n\nAlice is the lead.\n\n## Members\n\n- Alice\n"
mm = await memory.create_mental_model(
bank_id=bank_id,
name="Team Info",
@@ -461,9 +441,7 @@ class TestDeltaRefreshPlumbing:
return DeltaOperationList()
monkeypatch.setattr(memory._reflect_llm_config, "call", ok_call)
await memory.refresh_mental_model(
bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context
)
await memory.refresh_mental_model(bank_id=bank_id, mental_model_id=mm["id"], request_context=request_context)
# Now the second refresh: LLM raises. Refresh must not crash; it should
# store the candidate markdown.
@@ -512,15 +490,7 @@ class TestDeltaRefreshPlumbing:
bank_id = f"test-empty-reflect-{uuid.uuid4().hex[:8]}"
await memory.get_bank_profile(bank_id, request_context=request_context)
existing = (
"# Team\n"
"\n"
"Alice is the lead.\n"
"\n"
"## Members\n"
"\n"
"- Alice\n"
)
existing = "# Team\n\nAlice is the lead.\n\n## Members\n\n- Alice\n"
mm = await memory.create_mental_model(
bank_id=bank_id,
name="Team Info",
@@ -576,15 +546,9 @@ class TestDeltaRefreshPlumbing:
# Real-Gemini evaluation tests
# ---------------------------------------------------------------------------
_GEMINI_API_KEY = (
os.getenv("HINDSIGHT_GEMINI_API_KEY")
or os.getenv("GEMINI_API_KEY")
or os.getenv("GOOGLE_API_KEY")
)
_GEMINI_API_KEY = os.getenv("HINDSIGHT_GEMINI_API_KEY") or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")
_OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
_RUN_LLM_EVAL = os.getenv("HINDSIGHT_RUN_GEMINI_EVALS") == "1" and (
bool(_GEMINI_API_KEY) or bool(_OPENAI_API_KEY)
)
_RUN_LLM_EVAL = os.getenv("HINDSIGHT_RUN_GEMINI_EVALS") == "1" and (bool(_GEMINI_API_KEY) or bool(_OPENAI_API_KEY))
pytestmark_gemini = pytest.mark.skipif(
+15 -6
View File
@@ -1,4 +1,5 @@
"""Tests for metrics instrumentation."""
import pytest
from unittest.mock import MagicMock, patch
@@ -79,8 +80,10 @@ class TestMetricsCollector:
"""Create a MetricsCollector with a mock meter."""
mock_config = MagicMock()
mock_config.metrics_include_bank_id = False
with patch("hindsight_api.metrics.get_meter", return_value=mock_meter), \
patch("hindsight_api.config.get_config", return_value=mock_config):
with (
patch("hindsight_api.metrics.get_meter", return_value=mock_meter),
patch("hindsight_api.config.get_config", return_value=mock_config),
):
return MetricsCollector()
def test_record_operation_records_duration(self, collector):
@@ -174,8 +177,10 @@ class TestMetricsCollector:
"""Test that bank_id is included in attributes when metrics_include_bank_id is enabled."""
mock_config = MagicMock()
mock_config.metrics_include_bank_id = True
with patch("hindsight_api.metrics.get_meter") as mock_get_meter, \
patch("hindsight_api.config.get_config", return_value=mock_config):
with (
patch("hindsight_api.metrics.get_meter") as mock_get_meter,
patch("hindsight_api.config.get_config", return_value=mock_config),
):
mock_get_meter.return_value = MagicMock()
collector = MetricsCollector()
@@ -193,6 +198,7 @@ class TestGetMetricsCollector:
"""Test that get_metrics_collector returns NoOpMetricsCollector by default."""
# Reset global state
import hindsight_api.metrics as metrics_module
original_collector = metrics_module._metrics_collector
try:
@@ -208,6 +214,7 @@ class TestMetricsCollectorBase:
def test_is_abstract(self):
"""Test that MetricsCollectorBase methods are abstract."""
# Create a class that inherits but doesn't implement
class IncompleteCollector(MetricsCollectorBase):
pass
@@ -291,8 +298,10 @@ class TestLLMMetrics:
"""Create a MetricsCollector with a mock meter."""
mock_config = MagicMock()
mock_config.metrics_include_bank_id = False
with patch("hindsight_api.metrics.get_meter", return_value=mock_meter), \
patch("hindsight_api.config.get_config", return_value=mock_config):
with (
patch("hindsight_api.metrics.get_meter", return_value=mock_meter),
patch("hindsight_api.config.get_config", return_value=mock_config),
):
return MetricsCollector()
def test_record_llm_call_records_duration(self, collector):
@@ -41,6 +41,7 @@ def _downgrade(db_url: str, revision: str) -> None:
# Fixture: fresh database at the revision just before the backsweep
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def pre_backsweep_db_url():
"""
@@ -73,6 +74,7 @@ def pre_backsweep_db_url():
# The test
# ---------------------------------------------------------------------------
def test_backsweep_removes_orphans_and_preserves_legit_rows(pre_backsweep_db_url):
"""
Seed four kinds of rows then apply the backsweep migration and verify:
@@ -100,12 +102,12 @@ def test_backsweep_removes_orphans_and_preserves_legit_rows(pre_backsweep_db_url
ghost_bank = f"bank_{uuid.uuid4().hex[:8]}" # never inserted into banks
# UUIDs for memory units
id_pass1_world = uuid.uuid4() # A: world unit, ghost bank
id_pass1_obs = uuid.uuid4() # A: observation, ghost bank
id_pass2_obs = uuid.uuid4() # B: observation, all sources gone
id_keep_obs = uuid.uuid4() # C: observation with one live source
id_keep_world = uuid.uuid4() # D: world unit, alive bank
id_live_source = uuid.uuid4() # live source for C
id_pass1_world = uuid.uuid4() # A: world unit, ghost bank
id_pass1_obs = uuid.uuid4() # A: observation, ghost bank
id_pass2_obs = uuid.uuid4() # B: observation, all sources gone
id_keep_obs = uuid.uuid4() # C: observation with one live source
id_keep_world = uuid.uuid4() # D: world unit, alive bank
id_live_source = uuid.uuid4() # live source for C
with engine.connect() as conn:
# --- banks ---
@@ -147,10 +149,9 @@ def test_backsweep_removes_orphans_and_preserves_legit_rows(pre_backsweep_db_url
# --- verify ---
with engine.connect() as conn:
def exists(uid):
return conn.execute(
text("SELECT 1 FROM memory_units WHERE id = :id"), {"id": uid}
).fetchone() is not None
return conn.execute(text("SELECT 1 FROM memory_units WHERE id = :id"), {"id": uid}).fetchone() is not None
# Must be gone
assert not exists(id_pass1_world), "Pass 1: world unit with ghost bank should be deleted"
+7 -17
View File
@@ -85,10 +85,7 @@ async def test_retain_chinese_content(memory_real_llm, request_context):
for fact in result.results:
logger.info(f"Fact: {fact.text[:100]}...")
# Check for common Chinese characters or the name
if any(
char in fact.text
for char in ["", "", "腾讯", "软件", "工程师", "分布式", "系统", "代码"]
):
if any(char in fact.text for char in ["", "", "腾讯", "软件", "工程师", "分布式", "系统", "代码"]):
chinese_facts_found += 1
logger.info(f"Found {chinese_facts_found} facts with Chinese content")
@@ -185,7 +182,7 @@ async def test_reflect_chinese_content(memory_real_llm, request_context):
expected_names = set()
for fact in result.based_on.get("world", []):
# Extract Chinese entity names from the fact
for entity in (fact.entities or []):
for entity in fact.entities or []:
# Check if entity contains Chinese characters
if any("\u4e00" <= char <= "\u9fff" for char in entity):
expected_names.add(entity)
@@ -277,8 +274,7 @@ async def test_retain_japanese_content(memory_real_llm, request_context):
japanese_facts_found += 1
assert japanese_facts_found > 0, (
f"Expected facts to contain Japanese characters. "
f"Facts: {[f.text for f in result.results]}"
f"Expected facts to contain Japanese characters. Facts: {[f.text for f in result.results]}"
)
logger.info("Japanese retain test passed - facts preserved in Japanese")
@@ -350,8 +346,7 @@ async def test_english_content_stays_english(memory_real_llm, request_context):
# Count Japanese characters (hiragana, katakana)
japanese_chars = sum(
1 for char in fact.text
if ("\u3040" <= char <= "\u309f") or ("\u30a0" <= char <= "\u30ff")
1 for char in fact.text if ("\u3040" <= char <= "\u309f") or ("\u30a0" <= char <= "\u30ff")
)
# Count Chinese/CJK characters (excluding those also used in Japanese)
@@ -426,8 +421,7 @@ async def test_italian_content_stays_italian(memory_real_llm, request_context):
# Count CJK characters
cjk_chars = sum(1 for char in fact.text if "\u4e00" <= char <= "\u9fff")
japanese_chars = sum(
1 for char in fact.text
if ("\u3040" <= char <= "\u309f") or ("\u30a0" <= char <= "\u30ff")
1 for char in fact.text if ("\u3040" <= char <= "\u309f") or ("\u30a0" <= char <= "\u30ff")
)
total_chars = len(fact.text)
@@ -503,13 +497,9 @@ async def test_mixed_language_entities(memory_real_llm, request_context):
# Should contain Chinese name and/or English company names
has_chinese_name = "王芳" in all_text
has_english_company = any(
company in all_text for company in ["Google", "Microsoft", "Amazon", "YouTube"]
)
has_english_company = any(company in all_text for company in ["Google", "Microsoft", "Amazon", "YouTube"])
assert has_chinese_name or has_english_company, (
f"Expected mixed language entities. Facts: {all_text}"
)
assert has_chinese_name or has_english_company, f"Expected mixed language entities. Facts: {all_text}"
logger.info("Mixed language entity test passed")
@@ -66,13 +66,9 @@ class TestObservationHistory:
await memory.delete_bank(bank_id, request_context=request_context)
async def test_returns_none_for_missing_observation(
self, memory: MemoryEngine, request_context: Any
) -> None:
async def test_returns_none_for_missing_observation(self, memory: MemoryEngine, request_context: Any) -> None:
bank_id = f"test-obs-hist-{uuid.uuid4().hex[:8]}"
await memory.get_bank_profile(bank_id, request_context=request_context)
result = await memory.get_observation_history(
bank_id, str(uuid.uuid4()), request_context=request_context
)
result = await memory.get_observation_history(bank_id, str(uuid.uuid4()), request_context=request_context)
assert result is None
await memory.delete_bank(bank_id, request_context=request_context)
@@ -267,6 +267,7 @@ class TestDeleteDocumentObservationCleanup:
# Tests: document upsert via retain pipeline (regression for orphan observations)
# ---------------------------------------------------------------------------
class TestDocumentUpsertObservationCleanup:
"""Regression: re-ingesting a document via the retain pipeline must clean up
observations derived from the outgoing memory_units, the same way the
@@ -360,9 +361,7 @@ class TestDocumentUpsertObservationCleanup:
# be reset for re-consolidation since one of its observations was
# invalidated by the upsert.
consolidated_at = await _get_consolidated_at(conn, standalone_mem)
assert consolidated_at is None, (
"Surviving co-source memory should be reset for re-consolidation"
)
assert consolidated_at is None, "Surviving co-source memory should be reset for re-consolidation"
# The two doc-scoped memories are gone via FK cascade.
doc_mem_count = await conn.fetchval(
+17 -15
View File
@@ -5,6 +5,7 @@ NOTE: Observations are now stored as summaries on the entities table,
not as separate memory_units. The observations list in EntityState is
populated from the summary for backwards compatibility.
"""
import pytest
from hindsight_api.engine.memory_engine import Budget
from hindsight_api import RequestContext
@@ -64,7 +65,7 @@ async def test_entity_extraction_on_retain(memory, request_context):
WHERE bank_id = $1 AND LOWER(canonical_name) LIKE '%john%'
LIMIT 1
""",
bank_id
bank_id,
)
# Check the fact count for this entity
@@ -73,7 +74,7 @@ async def test_entity_extraction_on_retain(memory, request_context):
"""
SELECT COUNT(*) FROM unit_entities WHERE entity_id = $1
""",
entity_row['id']
entity_row["id"],
)
print(f"\n=== Entity Facts ===")
print(f"Entity: {entity_row['canonical_name']} has {fact_count} linked facts")
@@ -175,7 +176,7 @@ async def test_observation_fact_type_in_database(memory, request_context, disabl
FROM memory_units
WHERE bank_id = $1 AND fact_type = 'observation'
""",
bank_id
bank_id,
)
print(f"\n=== Observation Records in memory_units ===")
@@ -243,7 +244,7 @@ async def test_entity_mention_counts(memory, request_context):
WHERE e.bank_id = $1
ORDER BY e.mention_count DESC
""",
bank_id
bank_id,
)
print(f"\n=== Entity Mention Counts Test ===")
@@ -253,8 +254,8 @@ async def test_entity_mention_counts(memory, request_context):
low_mention_entity = None
for entity in entities:
name = entity['canonical_name'].lower()
mention_count = entity['mention_count']
name = entity["canonical_name"].lower()
mention_count = entity["mention_count"]
print(f" {entity['canonical_name']}: mentions={mention_count}")
@@ -268,8 +269,9 @@ async def test_entity_mention_counts(memory, request_context):
assert low_mention_entity is not None, "Trivex entity should exist"
# Nexora (10 mentions) must rank higher than Trivex (1 mention)
assert high_mention_entity['mention_count'] > low_mention_entity['mention_count'], \
assert high_mention_entity["mention_count"] > low_mention_entity["mention_count"], (
f"Nexora ({high_mention_entity['mention_count']} mentions) should have more than Trivex ({low_mention_entity['mention_count']} mentions)"
)
finally:
# Cleanup
@@ -297,7 +299,7 @@ async def test_entity_mention_ranking(memory, request_context):
for i in range(6):
await memory.retain_async(
bank_id=bank_id,
content=f"Alice is mentioned here in fact {i+1}.",
content=f"Alice is mentioned here in fact {i + 1}.",
context="test",
event_date=datetime(2024, 1, 1 + i, tzinfo=timezone.utc),
request_context=request_context,
@@ -310,7 +312,7 @@ async def test_entity_mention_ranking(memory, request_context):
for mention in range(10):
await memory.retain_async(
bank_id=bank_id,
content=f"{entity_name} is a very important entity, mention {mention+1}.",
content=f"{entity_name} is a very important entity, mention {mention + 1}.",
context="test",
event_date=datetime(2024, 2, 1 + mention, tzinfo=timezone.utc),
request_context=request_context,
@@ -329,7 +331,7 @@ async def test_entity_mention_ranking(memory, request_context):
WHERE bank_id = $1
ORDER BY mention_count DESC
""",
bank_id
bank_id,
)
print(f"\nAll entities by mention count:")
@@ -337,15 +339,15 @@ async def test_entity_mention_ranking(memory, request_context):
print(f" {e['canonical_name']}: mentions={e['mention_count']}")
# Verify high-mention entities rank higher than Alice (6 mentions)
alice = next((e for e in all_entities if 'alice' in e['canonical_name'].lower()), None)
high_mention = [e for e in all_entities if e['canonical_name'].lower() in ('bruno', 'carlos', 'diana')]
alice = next((e for e in all_entities if "alice" in e["canonical_name"].lower()), None)
high_mention = [e for e in all_entities if e["canonical_name"].lower() in ("bruno", "carlos", "diana")]
assert alice is not None, "Alice entity should exist"
assert len(high_mention) > 0, "High-mention entities should exist"
# Entities with 10 mentions each should rank higher than Alice (6 mentions)
for entity in high_mention:
assert entity['mention_count'] > alice['mention_count'], (
assert entity["mention_count"] > alice["mention_count"], (
f"{entity['canonical_name']} ({entity['mention_count']} mentions) "
f"should rank higher than Alice ({alice['mention_count']} mentions)"
)
@@ -407,7 +409,7 @@ async def test_user_entity_extraction(memory_real_llm, request_context):
AND LOWER(e.canonical_name) LIKE '%user%'
LIMIT 1
""",
bank_id
bank_id,
)
# Get all entities with their fact counts
@@ -421,7 +423,7 @@ async def test_user_entity_extraction(memory_real_llm, request_context):
WHERE e.bank_id = $1
ORDER BY fact_count DESC
""",
bank_id
bank_id,
)
print(f"\n=== Entities by Mention Count ===")
@@ -139,7 +139,9 @@ async def test_onnx_embeddings_dimension_mismatch_raises_value_error():
tokenizer_name_or_path="/models/e5",
dimensions=3,
)
fake_transformers = SimpleNamespace(AutoTokenizer=SimpleNamespace(from_pretrained=MagicMock(return_value=FakeTokenizer())))
fake_transformers = SimpleNamespace(
AutoTokenizer=SimpleNamespace(from_pretrained=MagicMock(return_value=FakeTokenizer()))
)
fake_onnxruntime = SimpleNamespace(InferenceSession=MagicMock(return_value=FakeOnnxSession()))
with patch.dict(sys.modules, {"transformers": fake_transformers, "onnxruntime": fake_onnxruntime}):
@@ -153,7 +155,9 @@ async def test_onnx_embeddings_downloads_external_data_sidecar_when_needed():
download = MagicMock(return_value="/hf/bge-m3")
session = MagicMock(return_value=FakeOnnxSession())
fake_hf = SimpleNamespace(snapshot_download=download)
fake_transformers = SimpleNamespace(AutoTokenizer=SimpleNamespace(from_pretrained=MagicMock(return_value=FakeTokenizer())))
fake_transformers = SimpleNamespace(
AutoTokenizer=SimpleNamespace(from_pretrained=MagicMock(return_value=FakeTokenizer()))
)
fake_onnxruntime = SimpleNamespace(InferenceSession=session)
with patch.dict(
@@ -190,9 +190,7 @@ class TestMarkOperationGracefulOnMissingRow:
await memory._mark_operation_failed(missing_id, "some error", "traceback here") # no exception
@pytest.mark.asyncio
async def test_mark_completed_and_fire_webhook_does_not_raise_when_row_missing(
self, memory: MemoryEngine
):
async def test_mark_completed_and_fire_webhook_does_not_raise_when_row_missing(self, memory: MemoryEngine):
missing_id = str(uuid.uuid4())
await memory._mark_operation_completed_and_fire_webhook(
operation_id=missing_id,
@@ -265,9 +263,7 @@ class TestConsolidationCheckpoint:
class TestRetainCheckpoint:
@pytest.mark.asyncio
async def test_retain_stops_between_sub_batches_when_cancelled(
self, memory: MemoryEngine, request_context
):
async def test_retain_stops_between_sub_batches_when_cancelled(self, memory: MemoryEngine, request_context):
"""retain_batch_async returns partial results if _check_op_alive is False between sub-batches."""
from hindsight_api.config import _get_raw_config
@@ -290,9 +286,7 @@ class TestRetainCheckpoint:
# Cancel after the first sub-batch completes
return check_calls <= 1
contents = [
{"content": f"Memory item {i} about something interesting."} for i in range(4)
]
contents = [{"content": f"Memory item {i} about something interesting."} for i in range(4)]
with patch.object(memory, "_check_op_alive", side_effect=_fake_check):
result = await memory.retain_batch_async(
@@ -7,6 +7,7 @@ Regression tests:
- Cancel used to delete the operation row; now it sets status to 'cancelled'.
- Retry now accepts both 'failed' and 'cancelled' operations.
"""
import uuid
from datetime import datetime
@@ -124,9 +125,7 @@ async def test_get_operation_returns_processing_status(api_client, memory, test_
processing_id = await _insert_operation(pool, test_bank_id, "processing")
response = await api_client.get(
f"/v1/default/banks/{test_bank_id}/operations/{processing_id}"
)
response = await api_client.get(f"/v1/default/banks/{test_bank_id}/operations/{processing_id}")
assert response.status_code == 200
data = response.json()
assert data["status"] == "processing"
@@ -154,9 +153,7 @@ async def test_all_statuses_returned_correctly(api_client, memory, test_bank_id)
# Verify get endpoint for each
for status, op_id in ids.items():
response = await api_client.get(
f"/v1/default/banks/{test_bank_id}/operations/{op_id}"
)
response = await api_client.get(f"/v1/default/banks/{test_bank_id}/operations/{op_id}")
assert response.status_code == 200
assert response.json()["status"] == status, f"Get: expected {status} for {op_id}"
@@ -170,16 +167,12 @@ async def test_cancel_sets_cancelled_status(api_client, memory, test_bank_id):
op_id = await _insert_operation(pool, test_bank_id, "pending")
# Cancel the operation
response = await api_client.delete(
f"/v1/default/banks/{test_bank_id}/operations/{op_id}"
)
response = await api_client.delete(f"/v1/default/banks/{test_bank_id}/operations/{op_id}")
assert response.status_code == 200
assert response.json()["success"] is True
# Verify the operation still exists with 'cancelled' status
response = await api_client.get(
f"/v1/default/banks/{test_bank_id}/operations/{op_id}"
)
response = await api_client.get(f"/v1/default/banks/{test_bank_id}/operations/{op_id}")
assert response.status_code == 200
assert response.json()["status"] == "cancelled"
@@ -203,16 +196,12 @@ async def test_retry_cancelled_operation(api_client, memory, test_bank_id):
op_id = await _insert_operation(pool, test_bank_id, "cancelled")
# Retry the cancelled operation
response = await api_client.post(
f"/v1/default/banks/{test_bank_id}/operations/{op_id}/retry"
)
response = await api_client.post(f"/v1/default/banks/{test_bank_id}/operations/{op_id}/retry")
assert response.status_code == 200
assert response.json()["success"] is True
# Verify the operation is now pending
response = await api_client.get(
f"/v1/default/banks/{test_bank_id}/operations/{op_id}"
)
response = await api_client.get(f"/v1/default/banks/{test_bank_id}/operations/{op_id}")
assert response.status_code == 200
assert response.json()["status"] == "pending"
@@ -225,9 +214,7 @@ async def test_retry_rejects_non_retriable_statuses(api_client, memory, test_ban
for status in ("pending", "processing", "completed"):
op_id = await _insert_operation(pool, test_bank_id, status)
response = await api_client.post(
f"/v1/default/banks/{test_bank_id}/operations/{op_id}/retry"
)
response = await api_client.post(f"/v1/default/banks/{test_bank_id}/operations/{op_id}/retry")
assert response.status_code == 409, f"Expected 409 for {status}, got {response.status_code}"
@@ -239,7 +226,5 @@ async def test_cancel_rejects_non_pending_operations(api_client, memory, test_ba
for status in ("processing", "completed", "failed"):
op_id = await _insert_operation(pool, test_bank_id, status)
response = await api_client.delete(
f"/v1/default/banks/{test_bank_id}/operations/{op_id}"
)
response = await api_client.delete(f"/v1/default/banks/{test_bank_id}/operations/{op_id}")
assert response.status_code == 409, f"Expected 409 for {status}, got {response.status_code}"
@@ -27,6 +27,7 @@ logger = logging.getLogger(__name__)
# Fixtures
# ---------------------------------------------------------------------------
def _bank_id(prefix: str = "http") -> str:
return f"test-{prefix}-{uuid.uuid4().hex[:8]}"
@@ -188,11 +189,7 @@ class TestOracleHTTP:
# Retain
resp = await api_client.post(
f"/v1/default/banks/{bank_id}/memories",
json={
"items": [
{"content": "Memory CRUD via HTTP on Oracle.", "context": "test"}
]
},
json={"items": [{"content": "Memory CRUD via HTTP on Oracle.", "context": "test"}]},
)
assert resp.status_code == 200
@@ -268,9 +265,7 @@ class TestOracleHTTP:
# Delete
if directive_id:
resp = await api_client.delete(
f"/v1/default/banks/{bank_id}/directives/{directive_id}"
)
resp = await api_client.delete(f"/v1/default/banks/{bank_id}/directives/{directive_id}")
assert resp.status_code == 200
finally:
await _safe_http_cleanup(api_client, bank_id)
@@ -306,11 +301,7 @@ class TestOracleHTTP:
try:
await api_client.post(
f"/v1/default/banks/{bank_id}/memories",
json={
"items": [
{"content": "Operations tracking test.", "context": "test"}
]
},
json={"items": [{"content": "Operations tracking test.", "context": "test"}]},
)
resp = await api_client.get(f"/v1/default/banks/{bank_id}/operations")
assert resp.status_code == 200
@@ -445,9 +436,7 @@ class TestOracleEndToEnd:
# --- 5. Verify the operation completed (not stuck as 'pending') ---
if operation_id:
resp = await api_client.get(
f"/v1/default/banks/{bank_id}/operations/{operation_id}"
)
resp = await api_client.get(f"/v1/default/banks/{bank_id}/operations/{operation_id}")
if resp.status_code == 200:
op = resp.json()
# SyncTaskBackend should have completed the refresh inline
@@ -456,9 +445,7 @@ class TestOracleEndToEnd:
)
# --- 6. Verify the mental model has real content (not placeholder) ---
resp = await api_client.get(
f"/v1/default/banks/{bank_id}/mental-models/{mental_model_id}"
)
resp = await api_client.get(f"/v1/default/banks/{bank_id}/mental-models/{mental_model_id}")
assert resp.status_code == 200, f"Get mental model failed: {resp.text}"
mm = resp.json()
content = mm.get("content", "")
@@ -28,6 +28,7 @@ logger = logging.getLogger(__name__)
# Helpers
# ---------------------------------------------------------------------------
def _bank_id(prefix: str = "oracle") -> str:
return f"test-{prefix}-{uuid.uuid4().hex[:8]}"
@@ -136,21 +137,25 @@ class TestCoreCRUD:
try:
await oracle_memory.retain_batch_async(
bank_id=bank_id,
contents=[{
"content": "Dan is an expert in distributed systems.",
"context": "engineering",
"event_date": datetime(2024, 5, 1, tzinfo=timezone.utc),
}],
contents=[
{
"content": "Dan is an expert in distributed systems.",
"context": "engineering",
"event_date": datetime(2024, 5, 1, tzinfo=timezone.utc),
}
],
document_tags=["backend"],
request_context=request_context,
)
await oracle_memory.retain_batch_async(
bank_id=bank_id,
contents=[{
"content": "Eve designs beautiful user interfaces.",
"context": "design",
"event_date": datetime(2024, 5, 2, tzinfo=timezone.utc),
}],
contents=[
{
"content": "Eve designs beautiful user interfaces.",
"context": "design",
"event_date": datetime(2024, 5, 2, tzinfo=timezone.utc),
}
],
document_tags=["frontend"],
request_context=request_context,
)
@@ -236,9 +241,7 @@ class TestCoreCRUD:
memory_id = unit_ids[0]
# Delete the memory
await oracle_memory.delete_memory_unit(
str(memory_id), request_context=request_context
)
await oracle_memory.delete_memory_unit(str(memory_id), request_context=request_context)
# Verify deletion
mem = await oracle_memory.get_memory_unit(
@@ -306,9 +309,7 @@ class TestCoreCRUD:
event_date=datetime(2024, 6, i + 1, tzinfo=timezone.utc),
request_context=request_context,
)
memories = await oracle_memory.list_memory_units(
bank_id=bank_id, request_context=request_context
)
memories = await oracle_memory.list_memory_units(bank_id=bank_id, request_context=request_context)
# Each retain may extract multiple facts
assert len(memories) >= 3
finally:
@@ -361,7 +362,7 @@ class TestRetainPipeline:
"Leonardo da Vinci painted the Mona Lisa between 1503 and 1519, and it now hangs in the Louvre Museum in Paris.",
"The International Space Station orbits Earth at an altitude of approximately 250 miles at a speed of 17,500 mph.",
]
long_content = " ".join(f"Section {i+1}: {fact} " * 3 for i, fact in enumerate(facts))
long_content = " ".join(f"Section {i + 1}: {fact} " * 3 for i, fact in enumerate(facts))
unit_ids = await oracle_memory.retain_async(
bank_id=bank_id,
content=long_content,
@@ -514,11 +515,13 @@ class TestRetainPipeline:
try:
await oracle_memory.retain_batch_async(
bank_id=bank_id,
contents=[{
"content": "Tagged memory about machine learning models.",
"context": "ml",
"event_date": datetime(2024, 6, 1, tzinfo=timezone.utc),
}],
contents=[
{
"content": "Tagged memory about machine learning models.",
"context": "ml",
"event_date": datetime(2024, 6, 1, tzinfo=timezone.utc),
}
],
document_tags=["ml", "models"],
request_context=request_context,
)
@@ -654,7 +657,7 @@ class TestSearchRetrieval:
for i in range(5):
await oracle_memory.retain_async(
bank_id=bank_id,
content=f"Fact {i}: Machine learning model {i} achieved {90+i}% accuracy.",
content=f"Fact {i}: Machine learning model {i} achieved {90 + i}% accuracy.",
context="ml",
event_date=datetime(2024, 1, i + 1, tzinfo=timezone.utc),
request_context=request_context,
@@ -818,9 +821,7 @@ class TestAdvancedFeatures:
assert model["id"] is not None
# List
models = await oracle_memory.list_mental_models(
bank_id=bank_id, request_context=request_context
)
models = await oracle_memory.list_mental_models(bank_id=bank_id, request_context=request_context)
assert len(models) > 0
# Get
@@ -899,16 +900,12 @@ class TestAdvancedFeatures:
)
# Verify facts were stored (consolidation is async and may run inline
# via SyncTaskBackend, but the key assertion is that all 5 retains persisted)
memories = await oracle_memory.list_memory_units(
bank_id=bank_id, request_context=request_context
)
memories = await oracle_memory.list_memory_units(bank_id=bank_id, request_context=request_context)
items = memories.get("items", memories) if isinstance(memories, dict) else memories
assert len(items) >= 5, f"Expected at least 5 stored memories, got {len(items)}"
# Verify facts contain expected content
texts = [item.get("text", "") for item in items]
assert any("dark mode" in t for t in texts), (
f"Expected 'dark mode' in stored facts, got: {texts[:3]}"
)
assert any("dark mode" in t for t in texts), f"Expected 'dark mode' in stored facts, got: {texts[:3]}"
finally:
await _safe_cleanup(oracle_memory, bank_id, request_context)
@@ -923,9 +920,7 @@ class TestAdvancedFeatures:
event_date=datetime(2024, 6, 1, tzinfo=timezone.utc),
request_context=request_context,
)
ops = await oracle_memory.list_operations(
bank_id=bank_id, request_context=request_context
)
ops = await oracle_memory.list_operations(bank_id=bank_id, request_context=request_context)
# Retain creates async operations (consolidation at minimum)
assert ops is not None
items = ops.get("items", ops) if isinstance(ops, dict) else ops
@@ -948,9 +943,7 @@ class TestAdvancedFeatures:
)
assert directive is not None
directives = await oracle_memory.list_directives(
bank_id=bank_id, request_context=request_context
)
directives = await oracle_memory.list_directives(bank_id=bank_id, request_context=request_context)
assert len(directives) > 0
await oracle_memory.delete_directive(
@@ -967,17 +960,17 @@ class TestAdvancedFeatures:
try:
await oracle_memory.retain_batch_async(
bank_id=bank_id,
contents=[{
"content": "Tag listing test.",
"context": "test",
"event_date": datetime(2024, 6, 1, tzinfo=timezone.utc),
}],
contents=[
{
"content": "Tag listing test.",
"context": "test",
"event_date": datetime(2024, 6, 1, tzinfo=timezone.utc),
}
],
document_tags=["alpha", "beta"],
request_context=request_context,
)
tags = await oracle_memory.list_tags(
bank_id=bank_id, request_context=request_context
)
tags = await oracle_memory.list_tags(bank_id=bank_id, request_context=request_context)
assert len(tags) > 0
finally:
await _safe_cleanup(oracle_memory, bank_id, request_context)
@@ -994,9 +987,7 @@ class TestAdvancedFeatures:
event_date=datetime(2024, 6, 1, tzinfo=timezone.utc),
request_context=request_context,
)
ops = await oracle_memory.list_operations(
bank_id=bank_id, request_context=request_context
)
ops = await oracle_memory.list_operations(bank_id=bank_id, request_context=request_context)
assert ops is not None
items = ops.get("items", ops) if isinstance(ops, dict) else ops
assert len(items) > 0, "Retain should enqueue at least one task"
@@ -1098,9 +1089,7 @@ class TestOracleSpecific:
mission="Test JSON CLOB storage",
request_context=request_context,
)
profile = await oracle_memory.get_bank_profile(
bank_id=bank_id, request_context=request_context
)
profile = await oracle_memory.get_bank_profile(bank_id=bank_id, request_context=request_context)
assert profile is not None
assert profile["name"] == "JSON Test Bank"
assert profile["mission"] == "Test JSON CLOB storage"
@@ -1167,9 +1156,7 @@ class TestEdgeCases:
event_date=datetime(2024, 1, 1, tzinfo=timezone.utc),
request_context=request_context,
)
memories = await oracle_memory.list_memory_units(
bank_id=bank_id, request_context=request_context
)
memories = await oracle_memory.list_memory_units(bank_id=bank_id, request_context=request_context)
items = memories.get("items", memories) if isinstance(memories, dict) else memories
assert len(items) >= 1
finally:
@@ -1197,19 +1184,13 @@ class TestEdgeCases:
event_date=datetime(2024, 1, 1, tzinfo=timezone.utc),
request_context=request_context,
)
memories = await oracle_memory.list_memory_units(
bank_id=bank_id, request_context=request_context
)
memories = await oracle_memory.list_memory_units(bank_id=bank_id, request_context=request_context)
items = memories.get("items", memories) if isinstance(memories, dict) else memories
# Large content (~10KB, 50 paragraphs) should produce multiple memory units
# from LLM fact extraction. At minimum we expect several facts.
assert len(items) >= 3, (
f"Expected large content to produce at least 3 memory units, got {len(items)}"
)
assert len(items) >= 3, f"Expected large content to produce at least 3 memory units, got {len(items)}"
# Verify operations completed without errors (catches background datetime issues etc.)
ops = await oracle_memory.list_operations(
bank_id=bank_id, request_context=request_context
)
ops = await oracle_memory.list_operations(bank_id=bank_id, request_context=request_context)
if ops:
op_list = ops.get("items", ops) if isinstance(ops, dict) else ops
failed = [o for o in op_list if isinstance(o, dict) and o.get("status") == "failed"]
@@ -1272,19 +1253,13 @@ class TestEdgeCases:
# not a code bug. Allow up to 1 deadlock failure.
deadlocks = [r for r in results if isinstance(r, Exception) and "ORA-00060" in str(r)]
other_failures = [r for r in results if isinstance(r, Exception) and "ORA-00060" not in str(r)]
assert len(other_failures) == 0, (
f"Non-deadlock failures: {[str(e)[:100] for e in other_failures]}"
)
assert len(other_failures) == 0, f"Non-deadlock failures: {[str(e)[:100] for e in other_failures]}"
successes = len(results) - len(deadlocks)
assert successes >= 2, f"Expected at least 2 successful retains, got {successes}"
memories = await oracle_memory.list_memory_units(
bank_id=bank_id, request_context=request_context
)
memories = await oracle_memory.list_memory_units(bank_id=bank_id, request_context=request_context)
items = memories.get("items", memories) if isinstance(memories, dict) else memories
assert len(items) >= successes, (
f"Expected at least {successes} memories, got {len(items)}"
)
assert len(items) >= successes, f"Expected at least {successes} memories, got {len(items)}"
finally:
await _safe_cleanup(oracle_memory, bank_id, request_context)
@@ -1311,9 +1286,7 @@ class TestEdgeCases:
async def test_delete_nonexistent_bank(self, oracle_memory: MemoryEngine, request_context: RequestContext):
"""Verify deleting a non-existent bank doesn't raise."""
# Should not raise an exception
await oracle_memory.delete_bank(
f"nonexistent-{uuid.uuid4().hex[:8]}", request_context=request_context
)
await oracle_memory.delete_bank(f"nonexistent-{uuid.uuid4().hex[:8]}", request_context=request_context)
@pytest.mark.asyncio
async def test_retain_and_delete_cycle(self, oracle_memory: MemoryEngine, request_context: RequestContext):
@@ -1369,19 +1342,13 @@ class TestEdgeCases:
request_context=request_context,
)
docs = await oracle_memory.list_documents(
bank_id=bank_id, request_context=request_context
)
docs = await oracle_memory.list_documents(bank_id=bank_id, request_context=request_context)
items = docs.get("items", docs.get("documents", []))
assert len(items) >= 3
# Delete one document, verify others remain
await oracle_memory.delete_document(
bank_id=bank_id, document_id="doc-1", request_context=request_context
)
docs_after = await oracle_memory.list_documents(
bank_id=bank_id, request_context=request_context
)
await oracle_memory.delete_document(bank_id=bank_id, document_id="doc-1", request_context=request_context)
docs_after = await oracle_memory.list_documents(bank_id=bank_id, request_context=request_context)
items_after = docs_after.get("items", docs_after.get("documents", []))
assert len(items_after) >= 2
finally:
@@ -63,9 +63,7 @@ class TestConsolidationBraceSafety:
)
note = "Use shape {limit, used}"
prompt = build_batch_consolidation_prompt(
observations_mission="m", observation_capacity_note=note
)
prompt = build_batch_consolidation_prompt(observations_mission="m", observation_capacity_note=note)
rendered = prompt.format(facts_text="<facts>", observations_text="<obs>")
assert "{limit, used}" in rendered
@@ -24,7 +24,9 @@ def test_provider_default_models():
config = HindsightConfig.from_env()
assert config.llm_provider == provider, f"Provider mismatch for {provider}"
assert config.llm_model == expected_model, f"Expected {expected_model} for {provider}, got {config.llm_model}"
assert config.llm_model == expected_model, (
f"Expected {expected_model} for {provider}, got {config.llm_model}"
)
finally:
# Restore original env vars
@@ -96,9 +98,9 @@ def test_per_operation_provider_default_model():
assert config.llm_model == "gpt-4o-mini", f"Expected gpt-4o-mini, got {config.llm_model}"
# Retain should use Anthropic default
assert (
config.retain_llm_model == "claude-haiku-4-5"
), f"Expected claude-haiku-4-5, got {config.retain_llm_model}"
assert config.retain_llm_model == "claude-haiku-4-5", (
f"Expected claude-haiku-4-5, got {config.retain_llm_model}"
)
finally:
clear_config_cache()
@@ -1,6 +1,7 @@
"""
Test query analyzer for temporal extraction.
"""
import pytest
from datetime import datetime
from hindsight_api.engine.query_analyzer import DateparserQueryAnalyzer, QueryAnalysis
@@ -312,8 +313,4 @@ def test_query_analyzer_dateparser_crash_returns_no_constraint(query_analyzer, m
assert analysis.temporal_constraint is None, (
"dateparser failures should be treated as no temporal constraint, not propagated"
)
assert any("dateparser" in rec.message for rec in caplog.records), (
"Should log a warning when dateparser fails"
)
assert any("dateparser" in rec.message for rec in caplog.records), "Should log a warning when dateparser fails"
@@ -26,16 +26,18 @@ async def test_recall_chunks_independent_of_max_tokens(memory, request_context):
bank_id = "test-chunks-independence"
try:
# Retain some test content with substantial size to generate chunks
test_content = """
test_content = (
"""
The quantum computing research team at MIT has made significant breakthroughs.
Dr. Sarah Chen leads the team and focuses on quantum error correction.
The team published three papers in Nature Physics this year.
Their work on topological qubits shows promise for scalable quantum computers.
Collaborators include IBM Research and Google Quantum AI.
The research is funded by a $5M NSF grant running through 2026.
""" * 10 # Repeat to ensure we get multiple chunks
"""
* 10
) # Repeat to ensure we get multiple chunks
await memory.retain_async(
bank_id=bank_id,
@@ -98,7 +100,6 @@ async def test_recall_chunks_batching_with_varying_sizes(memory, request_context
bank_id = "test-chunks-batching"
try:
# Retain multiple documents with different content sizes
# Document 1: Short content (small chunks)
await memory.retain_async(
@@ -109,12 +110,15 @@ async def test_recall_chunks_batching_with_varying_sizes(memory, request_context
)
# Document 2: Medium content
content_bob = """
content_bob = (
"""
Bob works as a data scientist at a tech startup in San Francisco.
He has expertise in natural language processing and computer vision.
Bob completed his PhD at Stanford University in 2020.
He leads a team of five engineers working on AI-powered recommendation systems.
""" * 5
"""
* 5
)
await memory.retain_async(
bank_id=bank_id,
content=content_bob,
@@ -123,14 +127,17 @@ async def test_recall_chunks_batching_with_varying_sizes(memory, request_context
)
# Document 3: Long content (large chunks)
content_charlie = """
content_charlie = (
"""
Charlie is the CTO of a growing AI company focused on healthcare applications.
He has over 15 years of experience in software architecture and distributed systems.
Charlie's team builds machine learning models for medical image analysis and diagnosis.
The company recently raised $50 million in Series B funding.
They have partnerships with major hospitals in the United States and Europe.
Charlie holds several patents in medical imaging and deep learning.
""" * 20
"""
* 20
)
await memory.retain_async(
bank_id=bank_id,
content=content_charlie,
@@ -178,7 +185,6 @@ async def test_recall_chunks_ordering_by_relevance(memory, request_context):
bank_id = "test-chunks-ordering"
try:
# Retain content with different relevance to query
await memory.retain_async(
bank_id=bank_id,
@@ -223,8 +229,9 @@ async def test_recall_chunks_ordering_by_relevance(memory, request_context):
all_chunk_text = " ".join(chunk.chunk_text for chunk in result.chunks.values())
# At least some chunks should mention Python (higher relevance)
# This is a soft check since exact ordering depends on scoring
assert "Python" in all_chunk_text or "python" in all_chunk_text.lower(), \
assert "Python" in all_chunk_text or "python" in all_chunk_text.lower(), (
"Chunks should include content about Python (relevant to query)"
)
finally:
# Cleanup
@@ -243,13 +250,16 @@ async def test_recall_chunks_for_observations(memory, request_context):
try:
# Retain content that will generate observations via consolidation
test_content = """
test_content = (
"""
Alice is a senior software engineer at a large technology company.
She specializes in distributed systems and has 10 years of experience.
Alice leads a team of 8 engineers working on cloud infrastructure.
She holds a PhD in computer science from Stanford University.
Alice has published several papers on fault-tolerant distributed systems.
""" * 8
"""
* 8
)
await memory.retain_async(
bank_id=bank_id,
@@ -296,7 +306,6 @@ async def test_recall_chunks_without_include_flag(memory, request_context):
bank_id = "test-chunks-no-include"
try:
# Retain content
test_content = """
Sarah is a product manager at a fintech company in New York.
@@ -321,8 +330,7 @@ async def test_recall_chunks_without_include_flag(memory, request_context):
# Should have facts but no chunks
assert len(result.results) > 0, "Should return facts"
assert result.chunks is None or len(result.chunks) == 0, \
"Should NOT return chunks when include_chunks=False"
assert result.chunks is None or len(result.chunks) == 0, "Should NOT return chunks when include_chunks=False"
finally:
# Cleanup
@@ -54,9 +54,7 @@ class TestToolRecallIncludeChunks:
async def test_max_chunk_tokens_propagates(self, mock_request_context):
engine = _make_mock_engine()
await tool_recall(
engine, "bank-1", "q", mock_request_context, max_chunk_tokens=2500, max_tokens=512
)
await tool_recall(engine, "bank-1", "q", mock_request_context, max_chunk_tokens=2500, max_tokens=512)
kwargs = engine.recall_async.call_args.kwargs
assert kwargs["max_chunk_tokens"] == 2500
@@ -45,9 +45,7 @@ async def test_recall_async_error_preserves_original(memory_no_llm_verify: Memor
# the symptom in #1384 was an empty trailer like "Failed to search memories: ".
message = str(excinfo.value)
assert "Failed to search memories" in message
assert "_SilentError" in message, (
f"wrapper message dropped the original exception class: {message!r}"
)
assert "_SilentError" in message, f"wrapper message dropped the original exception class: {message!r}"
# `from e` chain must be preserved so worker logs / debuggers can walk
# back to the real cause.
+44 -38
View File
@@ -32,14 +32,14 @@ class TestCleanAnswerText:
def test_clean_text_with_done_call(self):
"""Text ending with done() call should have it stripped."""
text = '''The team's OKRs focus on performance.done({"answer":"The team's OKRs","memory_ids":[]})'''
text = """The team's OKRs focus on performance.done({"answer":"The team's OKRs","memory_ids":[]})"""
cleaned = _clean_answer_text(text)
assert cleaned == "The team's OKRs focus on performance."
assert "done(" not in cleaned
def test_clean_text_with_done_call_and_whitespace(self):
"""done() call with whitespace should be stripped."""
text = '''Answer text here. done( {"answer": "short", "memory_ids": []} )'''
text = """Answer text here. done( {"answer": "short", "memory_ids": []} )"""
cleaned = _clean_answer_text(text)
assert cleaned == "Answer text here."
@@ -61,10 +61,10 @@ class TestCleanAnswerText:
def test_clean_text_multiline_done(self):
"""done() call spanning multiple lines should be stripped."""
text = '''Summary of findings.done({
text = """Summary of findings.done({
"answer": "Summary",
"memory_ids": ["id1", "id2"]
})'''
})"""
cleaned = _clean_answer_text(text)
assert cleaned == "Summary of findings."
@@ -74,22 +74,22 @@ class TestCleanDoneAnswer:
def test_clean_answer_with_leaked_json_code_block(self):
"""Answer with leaked JSON code block at the end should be cleaned."""
text = '''The user's favorite color is blue.
text = """The user's favorite color is blue.
```json
{"observation_ids": ["obs-1", "obs-2"]}
```'''
```"""
cleaned = _clean_done_answer(text)
assert cleaned == "The user's favorite color is blue."
assert "observation_ids" not in cleaned
def test_clean_answer_with_memory_ids_code_block(self):
"""Answer with leaked memory_ids JSON code block should be cleaned."""
text = '''Here is the answer.
text = """Here is the answer.
```json
{"memory_ids": ["mem-1"]}
```'''
```"""
cleaned = _clean_done_answer(text)
assert cleaned == "Here is the answer."
@@ -101,13 +101,13 @@ class TestCleanDoneAnswer:
def test_clean_answer_with_trailing_ids_pattern(self):
"""Answer with 'observation_ids: [...]' pattern at the end should be cleaned."""
text = "This is the answer.\n\nobservation_ids: [\"obs-1\", \"obs-2\"]"
text = 'This is the answer.\n\nobservation_ids: ["obs-1", "obs-2"]'
cleaned = _clean_done_answer(text)
assert cleaned == "This is the answer."
def test_clean_answer_with_memory_ids_equals(self):
"""Answer with 'memory_ids = [...]' pattern at the end should be cleaned."""
text = "Answer text here.\nmemory_ids = [\"mem-1\"]"
text = 'Answer text here.\nmemory_ids = ["mem-1"]'
cleaned = _clean_done_answer(text)
assert cleaned == "Answer text here."
@@ -129,13 +129,13 @@ class TestCleanDoneAnswer:
def test_clean_answer_multiline_with_markdown(self):
"""Answer with markdown and leaked JSON at end should clean only the leak."""
text = '''Summary:
text = """Summary:
- Point 1
- Point 2
```json
{"mental_model_ids": ["mm-1"]}
```'''
```"""
cleaned = _clean_done_answer(text)
assert "Point 1" in cleaned
assert "Point 2" in cleaned
@@ -244,7 +244,10 @@ class TestReflectAgentMocked:
llm.call_with_tools = AsyncMock()
# Also mock call() for final iteration fallback - returns (response, usage) tuple
llm.call = AsyncMock(
return_value=("Fallback answer from final iteration", TokenUsage(input_tokens=100, output_tokens=50, total_tokens=150))
return_value=(
"Fallback answer from final iteration",
TokenUsage(input_tokens=100, output_tokens=50, total_tokens=150),
)
)
return llm
@@ -284,9 +287,7 @@ class TestReflectAgentMocked:
self._mm_call(),
LLMToolCallResult(
tool_calls=[
LLMToolCall(
id="2", name="done", arguments={"answer": "Be concise.", "mental_model_ids": ["mm-1"]}
)
LLMToolCall(id="2", name="done", arguments={"answer": "Be concise.", "mental_model_ids": ["mm-1"]})
],
finish_reason="tool_calls",
),
@@ -326,7 +327,9 @@ class TestReflectAgentMocked:
self._mm_call(),
LLMToolCallResult(
tool_calls=[
LLMToolCall(id="2", name="recall", arguments={"reason": "verify", "query": "launch completion proof"})
LLMToolCall(
id="2", name="recall", arguments={"reason": "verify", "query": "launch completion proof"}
)
],
finish_reason="tool_calls",
),
@@ -472,9 +475,7 @@ class TestReflectAgentMocked:
finish_reason="tool_calls",
),
LLMToolCallResult(
tool_calls=[
LLMToolCall(id="4", name="done", arguments={"answer": "Done.", "memory_ids": ["mem-1"]})
],
tool_calls=[LLMToolCall(id="4", name="done", arguments={"answer": "Done.", "memory_ids": ["mem-1"]})],
finish_reason="tool_calls",
),
]
@@ -679,10 +680,7 @@ class TestReflectAgentMocked:
"""
# Build a long response that's well over the cap in cl100k_base tokens.
long_answer = " ".join(
[
"This is a detailed paragraph about the team, their roles, and their recurring meetings."
]
* 80
["This is a detailed paragraph about the team, their roles, and their recurring meetings."] * 80
)
# The short-circuit path: tool_calls empty, content populated.
mock_llm.call_with_tools.return_value = LLMToolCallResult(
@@ -715,8 +713,7 @@ class TestReflectAgentMocked:
)
rewrite_kwargs = mock_llm.call.await_args.kwargs
assert rewrite_kwargs.get("max_completion_tokens") == cap, (
f"rewrite call should use max_completion_tokens={cap}, "
f"got {rewrite_kwargs.get('max_completion_tokens')}"
f"rewrite call should use max_completion_tokens={cap}, got {rewrite_kwargs.get('max_completion_tokens')}"
)
# The final answer is the rewritten text, not the oversized original.
@@ -824,7 +821,14 @@ class TestContextOverflowHelpers:
"role": "tool",
"tool_call_id": "x",
"name": "recall",
"content": '{"memories": [' + ', '.join([f'{{"id": "m{i}", "content": "A long memory fact about some topic that goes on and on."}}' for i in range(50)]) + ']}',
"content": '{"memories": ['
+ ", ".join(
[
f'{{"id": "m{i}", "content": "A long memory fact about some topic that goes on and on."}}'
for i in range(50)
]
)
+ "]}",
},
]
small = _count_messages_tokens(small_messages)
@@ -833,7 +837,11 @@ class TestContextOverflowHelpers:
def test_is_context_overflow_error_openai(self):
assert _is_context_overflow_error(Exception("context_length_exceeded: too many tokens"))
assert _is_context_overflow_error(Exception("This model's maximum context length is 128000 tokens. However, your messages resulted in 142164 tokens."))
assert _is_context_overflow_error(
Exception(
"This model's maximum context length is 128000 tokens. However, your messages resulted in 142164 tokens."
)
)
def test_is_context_overflow_error_anthropic(self):
assert _is_context_overflow_error(Exception("prompt_too_long"))
@@ -860,17 +868,17 @@ class TestContextOverflowBehavior:
llm = MagicMock()
llm.call_with_tools = AsyncMock()
llm.call = AsyncMock(
return_value=("Synthesized answer from gathered evidence.", TokenUsage(input_tokens=50, output_tokens=20, total_tokens=70))
return_value=(
"Synthesized answer from gathered evidence.",
TokenUsage(input_tokens=50, output_tokens=20, total_tokens=70),
)
)
return llm
@pytest.fixture
def mock_functions_with_large_output(self):
"""Mock functions that return a large enough payload to exceed a tiny token budget."""
large_memories = [
{"id": f"mem-{i}", "content": f"Memory fact number {i}: " + "A" * 200}
for i in range(20)
]
large_memories = [{"id": f"mem-{i}", "content": f"Memory fact number {i}: " + "A" * 200} for i in range(20)]
return {
"search_mental_models_fn": AsyncMock(return_value={"mental_models": []}),
"search_observations_fn": AsyncMock(return_value={"observations": []}),
@@ -910,9 +918,7 @@ class TestContextOverflowBehavior:
async def test_context_overflow_error_skips_retry(self, mock_llm, mock_functions_with_large_output):
"""A context_length_exceeded error from the LLM should NOT be retried —
it should immediately fall back to final synthesis."""
mock_llm.call_with_tools.side_effect = Exception(
"context_length_exceeded: messages resulted in 150000 tokens."
)
mock_llm.call_with_tools.side_effect = Exception("context_length_exceeded: messages resulted in 150000 tokens.")
result = await run_reflect_agent(
llm_config=mock_llm,
@@ -969,8 +975,7 @@ class TestDirectiveLeakageOnEmptyBank:
# The directive content must NOT leak into the answer.
assert directive_text not in result.text, (
f"Directive content leaked into the answer verbatim. "
f"Got: {result.text!r}"
f"Directive content leaked into the answer verbatim. Got: {result.text!r}"
)
finally:
await memory.delete_bank(bank_id, request_context=request_context)
@@ -1023,6 +1028,7 @@ class TestContextOverflowIntegration:
class _TinyContextProxy:
"""Forwards all attribute access to the real config proxy except
reflect_max_context_tokens which is forced to 1."""
_real = _real_get_config()
def __getattr__(self, name: str):
@@ -34,8 +34,8 @@ async def test_reflect_with_no_memories_empty_bank(api_client):
"budget": "low",
"include": {
"facts": {} # Request facts but bank is empty
}
}
},
},
)
assert response.status_code == 200
@@ -43,10 +43,11 @@ async def test_reflect_with_no_memories_empty_bank(api_client):
# DEBUG: Print what the API actually returned
import json
print("\n" + "="*80)
print("\n" + "=" * 80)
print("API Response:")
print(json.dumps(data, indent=2))
print("="*80 + "\n")
print("=" * 80 + "\n")
# Verify response structure
assert "text" in data
@@ -87,9 +88,9 @@ async def test_reflect_without_include_facts(api_client):
f"/v1/default/banks/{bank_id}/reflect",
json={
"query": "Hello world",
"budget": "low"
"budget": "low",
# No include.facts
}
},
)
assert response.status_code == 200
@@ -30,9 +30,7 @@ class _FakeRequestContext:
def _mock_engine():
engine = MagicMock()
engine.recall_async = AsyncMock(
return_value=RecallResult(results=[], source_facts={})
)
engine.recall_async = AsyncMock(return_value=RecallResult(results=[], source_facts={}))
return engine
@@ -386,9 +386,7 @@ class TestBudgetBranches:
include_observations=False,
budget="low",
)
assert actual == _assemble(
_RETRIEVAL_RECALL_ONLY, _WORKFLOW_RECALL_ONLY, budget=_BUDGET_LOW
)
assert actual == _assemble(_RETRIEVAL_RECALL_ONLY, _WORKFLOW_RECALL_ONLY, budget=_BUDGET_LOW)
def test_budget_mid_inserts_moderate_block(self):
actual = build_system_prompt_for_tools(
@@ -397,9 +395,7 @@ class TestBudgetBranches:
include_observations=False,
budget="mid",
)
assert actual == _assemble(
_RETRIEVAL_RECALL_ONLY, _WORKFLOW_RECALL_ONLY, budget=_BUDGET_MID
)
assert actual == _assemble(_RETRIEVAL_RECALL_ONLY, _WORKFLOW_RECALL_ONLY, budget=_BUDGET_MID)
def test_budget_high_inserts_deep_block(self):
actual = build_system_prompt_for_tools(
@@ -408,9 +404,7 @@ class TestBudgetBranches:
include_observations=False,
budget="high",
)
assert actual == _assemble(
_RETRIEVAL_RECALL_ONLY, _WORKFLOW_RECALL_ONLY, budget=_BUDGET_HIGH
)
assert actual == _assemble(_RETRIEVAL_RECALL_ONLY, _WORKFLOW_RECALL_ONLY, budget=_BUDGET_HIGH)
def test_unknown_budget_inserts_nothing(self):
# The builder only recognises low/mid/high; any other value is a no-op.
@@ -544,7 +538,5 @@ def test_include_observations_defaults_to_true():
"""Callers that don't pass ``include_observations`` get the original
observations-enabled prompt this guards the API default so reflect
paths that don't gate the flag aren't silently changed."""
actual = build_system_prompt_for_tools(
bank_profile=BANK, has_mental_models=False
)
actual = build_system_prompt_for_tools(bank_profile=BANK, has_mental_models=False)
assert actual == _assemble(_RETRIEVAL_OBS_ONLY, _WORKFLOW_OBS_ONLY)
@@ -40,9 +40,7 @@ class TestSearchObservationsSourceFacts:
"""Default source_facts_max_tokens=-1 should disable source facts."""
engine = _make_mock_engine()
await tool_search_observations(
engine, "bank-1", "test query", mock_request_context
)
await tool_search_observations(engine, "bank-1", "test query", mock_request_context)
engine.recall_async.assert_called_once()
call_kwargs = engine.recall_async.call_args.kwargs
@@ -55,7 +53,10 @@ class TestSearchObservationsSourceFacts:
engine = _make_mock_engine()
await tool_search_observations(
engine, "bank-1", "test query", mock_request_context,
engine,
"bank-1",
"test query",
mock_request_context,
source_facts_max_tokens=0,
)
@@ -70,7 +71,10 @@ class TestSearchObservationsSourceFacts:
engine = _make_mock_engine()
await tool_search_observations(
engine, "bank-1", "test query", mock_request_context,
engine,
"bank-1",
"test query",
mock_request_context,
source_facts_max_tokens=5000,
)
@@ -85,7 +89,10 @@ class TestSearchObservationsSourceFacts:
engine = _make_mock_engine()
await tool_search_observations(
engine, "bank-1", "test query", mock_request_context,
engine,
"bank-1",
"test query",
mock_request_context,
source_facts_max_tokens=-1,
)
@@ -1,6 +1,7 @@
"""
Test to verify reflect operation creates proper span hierarchy.
"""
import pytest
@@ -11,11 +12,7 @@ async def test_reflect_creates_child_spans(memory, request_context):
from hindsight_api.tracing import initialize_tracing, get_span_recorder, create_span_recorder
# Initialize tracing with a mock endpoint
initialize_tracing(
service_name="test-hindsight",
endpoint="http://localhost:4318",
deployment_environment="test"
)
initialize_tracing(service_name="test-hindsight", endpoint="http://localhost:4318", deployment_environment="test")
# Create span recorder
recorder = create_span_recorder()
@@ -23,6 +23,7 @@ from hindsight_api.engine.search.types import MergedCandidate, RetrievalResult
# Helpers
# ---------------------------------------------------------------------------
def _make_candidates(n: int) -> list[MergedCandidate]:
"""Create *n* minimal MergedCandidate objects."""
candidates = []
@@ -34,9 +35,7 @@ def _make_candidates(n: int) -> list[MergedCandidate]:
occurred_start=None,
occurred_end=None,
)
candidates.append(
MergedCandidate(retrieval=retrieval, rrf_score=1.0 / (i + 1))
)
candidates.append(MergedCandidate(retrieval=retrieval, rrf_score=1.0 / (i + 1)))
return candidates
@@ -53,6 +52,7 @@ def _make_cross_encoder(predict_return: list[float]):
# Tests
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_passthrough_for_0_1_scores():
"""Scores already in [0, 1] should be passed through as-is."""
@@ -11,6 +11,7 @@ from hindsight_api.engine.search.reranking import apply_combined_scoring
UTC = timezone.utc
def create_mock_scored_result(proof_count: int | None = None, ce_score: float = 0.8) -> ScoredResult:
"""Helper to create a minimal ScoredResult suitable for scoring tests."""
retrieval = RetrievalResult(
@@ -22,7 +23,7 @@ def create_mock_scored_result(proof_count: int | None = None, ce_score: float =
proof_count=proof_count,
# Use None for neutral recency so only proof_count changes score
occurred_start=None,
occurred_end=None
occurred_end=None,
)
candidate = MergedCandidate(
retrieval=retrieval,
@@ -35,57 +36,60 @@ def create_mock_scored_result(proof_count: int | None = None, ce_score: float =
weight=ce_score,
)
def test_proof_count_neutral_when_none():
"""Test that when proof_count is None (e.g. non-observation), it gets neutral 0.5 norm."""
sr = create_mock_scored_result(proof_count=None, ce_score=0.8)
now = datetime.now(UTC)
apply_combined_scoring([sr], now, proof_count_alpha=0.1)
# Neutral multiplier means score shouldn't be boosted by proof_count
# Since recency is neutral (just created) and temporal is neutral, score should remain unchanged
assert sr.combined_score == pytest.approx(0.8, rel=1e-3)
def test_proof_count_neutral_at_one():
"""Test that proof_count=1 gives neutral multiplier."""
sr = create_mock_scored_result(proof_count=1, ce_score=0.8)
now = datetime.now(UTC)
apply_combined_scoring([sr], now, proof_count_alpha=0.1)
# proof_count=1 -> math.log(1) = 0 -> 0.5 + 0/10 = 0.5 (neutral) -> multiplier 1.0
assert sr.combined_score == pytest.approx(0.8, rel=1e-3)
def test_proof_count_increases_with_higher_counts():
"""Test that higher proof counts yield strictly higher scores."""
now = datetime.now(UTC)
# Create results with increasing proof counts
sr_5 = create_mock_scored_result(proof_count=5, ce_score=0.8)
sr_50 = create_mock_scored_result(proof_count=50, ce_score=0.8)
sr_100 = create_mock_scored_result(proof_count=100, ce_score=0.8)
# Process them
apply_combined_scoring([sr_5, sr_50, sr_100], now, proof_count_alpha=0.1)
# Assure scores strictly increase
assert sr_5.combined_score > 0.8
assert sr_50.combined_score > sr_5.combined_score
assert sr_100.combined_score > sr_50.combined_score
def test_proof_count_no_hardcoded_cap_at_100():
"""Test that proof_count continues to scale within the clamped [0, 1] range."""
now = datetime.now(UTC)
# Use values that stay below the clamp ceiling (proof_norm < 1.0)
# log(5)/10=0.16, log(20)/10=0.30, log(100)/10=0.46 → all below 0.5 headroom
sr_5 = create_mock_scored_result(proof_count=5, ce_score=0.8)
sr_20 = create_mock_scored_result(proof_count=20, ce_score=0.8)
sr_100 = create_mock_scored_result(proof_count=100, ce_score=0.8)
apply_combined_scoring([sr_5, sr_20, sr_100], now, proof_count_alpha=0.1)
# Must strictly increase within the valid range
assert sr_20.combined_score > sr_5.combined_score
assert sr_100.combined_score > sr_20.combined_score
@@ -107,10 +107,12 @@ class TestSchemaIsolation:
await conn.close()
# Configure tenant extension that provisions schemas via run_migrations
tenant_ext = MultiSchemaTestTenantExtension({
"db_url": pg0_db_url,
"valid_schemas": set(schemas),
})
tenant_ext = MultiSchemaTestTenantExtension(
{
"db_url": pg0_db_url,
"valid_schemas": set(schemas),
}
)
memory._tenant_extension = tenant_ext
# Define concurrent insert tasks for each tenant
@@ -129,7 +131,7 @@ class TestSchemaIsolation:
for i in range(3):
await conn.execute(
f"""
INSERT INTO {fq_table('memory_units')} (bank_id, text, event_date, fact_type)
INSERT INTO {fq_table("memory_units")} (bank_id, text, event_date, fact_type)
VALUES ($1, $2, now(), 'world')
""",
bank_id,
@@ -155,9 +157,7 @@ class TestSchemaIsolation:
# All texts should contain the schema's marker
for text in texts:
assert f"MARKER_{prefix}" in text, (
f"Memory in {schema} missing its marker: {text}"
)
assert f"MARKER_{prefix}" in text, f"Memory in {schema} missing its marker: {text}"
# Should NOT contain other tenants' markers
other_prefixes = ["ALPHA", "BETA", "GAMMA"]
@@ -265,10 +265,12 @@ class TestSchemaIsolation:
await conn.close()
# Configure tenant extension
tenant_ext = MultiSchemaTestTenantExtension({
"db_url": pg0_db_url,
"valid_schemas": set(schemas),
})
tenant_ext = MultiSchemaTestTenantExtension(
{
"db_url": pg0_db_url,
"valid_schemas": set(schemas),
}
)
memory._tenant_extension = tenant_ext
try:
@@ -333,10 +335,12 @@ class TestSchemaIsolation:
run_migrations(pg0_db_url, schema=schema)
# Configure tenant extension (schemas already provisioned)
tenant_ext = MultiSchemaTestTenantExtension({
"db_url": pg0_db_url,
"valid_schemas": set(schemas),
})
tenant_ext = MultiSchemaTestTenantExtension(
{
"db_url": pg0_db_url,
"valid_schemas": set(schemas),
}
)
# Mark schemas as already provisioned so extension doesn't re-run migrations
tenant_ext._provisioned = set(schemas)
memory._tenant_extension = tenant_ext
@@ -357,7 +361,7 @@ class TestSchemaIsolation:
async with acquire_with_retry(pool) as conn:
await conn.execute(
f"""
INSERT INTO {fq_table('memory_units')} (bank_id, text, event_date, fact_type)
INSERT INTO {fq_table("memory_units")} (bank_id, text, event_date, fact_type)
VALUES ($1, $2, now(), 'world')
""",
bank_id,
@@ -1,6 +1,7 @@
"""
Test search tracing functionality.
"""
from datetime import datetime, timezone
import pytest
@@ -33,7 +34,6 @@ async def test_search_with_trace(memory, request_context):
bank_id = f"test_trace_{datetime.now(timezone.utc).timestamp()}"
try:
# Store some test memories
await memory.retain_async(
bank_id=bank_id,
@@ -59,7 +59,7 @@ async def test_search_with_trace(memory, request_context):
bank_id=bank_id,
query="Who works at Google?",
fact_type=["world"],
budget=Budget.LOW, # 20,
budget=Budget.LOW, # 20,
max_tokens=512,
enable_trace=True,
request_context=request_context,
@@ -134,7 +134,6 @@ async def test_search_without_trace(memory, request_context):
bank_id = f"test_no_trace_{datetime.now(timezone.utc).timestamp()}"
try:
# Store a test memory
await memory.retain_async(
bank_id=bank_id,
@@ -148,7 +147,7 @@ async def test_search_without_trace(memory, request_context):
bank_id=bank_id,
query="test",
fact_type=["world"],
budget=Budget.LOW, # 10,
budget=Budget.LOW, # 10,
max_tokens=512,
enable_trace=False,
request_context=request_context,
+43 -35
View File
@@ -53,12 +53,13 @@ class TestServerModuleExtensionLoading:
# Patch at source level BEFORE importing server
# Note: We patch the entire hindsight_api module namespace
with patch("hindsight_api.MemoryEngine") as mock_engine, \
patch("hindsight_api.api.create_app") as mock_create_app, \
patch("hindsight_api.config.get_config") as mock_get_config, \
patch("hindsight_api.extensions.load_extension", side_effect=tracking_load_extension), \
patch("hindsight_api.extensions.DefaultExtensionContext"):
with (
patch("hindsight_api.MemoryEngine") as mock_engine,
patch("hindsight_api.api.create_app") as mock_create_app,
patch("hindsight_api.config.get_config") as mock_get_config,
patch("hindsight_api.extensions.load_extension", side_effect=tracking_load_extension),
patch("hindsight_api.extensions.DefaultExtensionContext"),
):
mock_config = MagicMock()
mock_config.mcp_enabled = False
mock_config.run_migrations_on_startup = False
@@ -71,12 +72,15 @@ class TestServerModuleExtensionLoading:
import hindsight_api.server
# Verify TENANT extension was loaded
assert "TENANT" in loaded_extensions, \
assert "TENANT" in loaded_extensions, (
"server.py did not call load_extension('TENANT', ...) - extensions not loaded!"
assert loaded_extensions["TENANT"] is not None, \
)
assert loaded_extensions["TENANT"] is not None, (
"load_extension('TENANT', ...) returned None despite env var being set"
assert isinstance(loaded_extensions["TENANT"], MockTenantExtension), \
)
assert isinstance(loaded_extensions["TENANT"], MockTenantExtension), (
f"Expected MockTenantExtension, got {type(loaded_extensions['TENANT'])}"
)
def test_server_loads_operation_validator_when_configured(self, monkeypatch):
"""
@@ -98,12 +102,13 @@ class TestServerModuleExtensionLoading:
loaded_extensions[name] = result
return result
with patch("hindsight_api.MemoryEngine") as mock_engine, \
patch("hindsight_api.api.create_app") as mock_create_app, \
patch("hindsight_api.config.get_config") as mock_get_config, \
patch("hindsight_api.extensions.load_extension", side_effect=tracking_load_extension), \
patch("hindsight_api.extensions.DefaultExtensionContext"):
with (
patch("hindsight_api.MemoryEngine") as mock_engine,
patch("hindsight_api.api.create_app") as mock_create_app,
patch("hindsight_api.config.get_config") as mock_get_config,
patch("hindsight_api.extensions.load_extension", side_effect=tracking_load_extension),
patch("hindsight_api.extensions.DefaultExtensionContext"),
):
mock_config = MagicMock()
mock_config.mcp_enabled = False
mock_config.run_migrations_on_startup = False
@@ -114,8 +119,9 @@ class TestServerModuleExtensionLoading:
import hindsight_api.server
assert "OPERATION_VALIDATOR" in loaded_extensions, \
assert "OPERATION_VALIDATOR" in loaded_extensions, (
"server.py did not call load_extension('OPERATION_VALIDATOR', ...)"
)
assert loaded_extensions["OPERATION_VALIDATOR"] is not None
assert isinstance(loaded_extensions["OPERATION_VALIDATOR"], MockOperationValidator)
@@ -139,11 +145,12 @@ class TestServerModuleExtensionLoading:
memory_engine_calls.append({"args": args, "kwargs": kwargs})
return MagicMock()
with patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine), \
patch("hindsight_api.api.create_app") as mock_create_app, \
patch("hindsight_api.config.get_config") as mock_get_config, \
patch("hindsight_api.extensions.DefaultExtensionContext"):
with (
patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine),
patch("hindsight_api.api.create_app") as mock_create_app,
patch("hindsight_api.config.get_config") as mock_get_config,
patch("hindsight_api.extensions.DefaultExtensionContext"),
):
mock_config = MagicMock()
mock_config.mcp_enabled = False
mock_config.run_migrations_on_startup = False
@@ -159,10 +166,10 @@ class TestServerModuleExtensionLoading:
call_kwargs = memory_engine_calls[0]["kwargs"]
# THE CRITICAL ASSERTION: tenant_extension must be passed and not None
assert "tenant_extension" in call_kwargs, \
"MemoryEngine was not called with tenant_extension parameter!"
assert call_kwargs["tenant_extension"] is not None, \
assert "tenant_extension" in call_kwargs, "MemoryEngine was not called with tenant_extension parameter!"
assert call_kwargs["tenant_extension"] is not None, (
"tenant_extension was None - server.py did not pass loaded extension to MemoryEngine!"
)
def test_server_sets_extension_context_on_tenant_extension(self, monkeypatch):
"""
@@ -189,11 +196,12 @@ class TestServerModuleExtensionLoading:
context_set_calls.append(ctx)
return ctx
with patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine), \
patch("hindsight_api.api.create_app") as mock_create_app, \
patch("hindsight_api.config.get_config") as mock_get_config, \
patch("hindsight_api.extensions.DefaultExtensionContext", side_effect=capture_context):
with (
patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine),
patch("hindsight_api.api.create_app") as mock_create_app,
patch("hindsight_api.config.get_config") as mock_get_config,
patch("hindsight_api.extensions.DefaultExtensionContext", side_effect=capture_context),
):
mock_config = MagicMock()
mock_config.mcp_enabled = False
mock_config.run_migrations_on_startup = False
@@ -206,8 +214,7 @@ class TestServerModuleExtensionLoading:
# Verify context was created and set
assert len(context_set_calls) == 1, "DefaultExtensionContext should be created"
assert captured_tenant_ext[0] is not None, "Tenant extension should be captured"
assert captured_tenant_ext[0]._context_set, \
"set_context was not called on tenant extension"
assert captured_tenant_ext[0]._context_set, "set_context was not called on tenant extension"
def test_server_works_without_extensions(self, monkeypatch):
"""
@@ -225,10 +232,11 @@ class TestServerModuleExtensionLoading:
memory_engine_calls.append({"args": args, "kwargs": kwargs})
return MagicMock()
with patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine), \
patch("hindsight_api.api.create_app") as mock_create_app, \
patch("hindsight_api.config.get_config") as mock_get_config:
with (
patch("hindsight_api.MemoryEngine", side_effect=capture_memory_engine),
patch("hindsight_api.api.create_app") as mock_create_app,
patch("hindsight_api.config.get_config") as mock_get_config,
):
mock_config = MagicMock()
mock_config.mcp_enabled = False
mock_config.run_migrations_on_startup = False
@@ -96,7 +96,7 @@ class TestRecallSourceFactsPerObservationCap:
fact_type=["observation"],
max_tokens=4096,
include_source_facts=True,
max_source_facts_tokens=4096, # large global budget
max_source_facts_tokens=4096, # large global budget
max_source_facts_tokens_per_observation=512, # reasonable per-obs limit
budget=Budget.MID,
request_context=request_context,
@@ -93,9 +93,7 @@ def find_unqualified_table_refs(content: str, filename: str) -> list[tuple[int,
qualified_pattern = rf"\.\s*{table}(?:\s|$|,|\))"
fq_table_pattern = rf'fq_table\s*\(\s*["\']?{table}'
if not re.search(qualified_pattern, line) and not re.search(
fq_table_pattern, line
):
if not re.search(qualified_pattern, line) and not re.search(fq_table_pattern, line):
# Additional check: line must have SQL indicators
# This avoids false positives in docstrings like "split into chunks"
if sql_indicators.search(line):
@@ -117,8 +115,7 @@ class TestSQLSchemaSafety:
for line_num, table, line in violations:
all_violations.append(
f"{py_file.relative_to(py_file.parent.parent)}:{line_num} - "
f"unqualified '{table}': {line[:80]}..."
f"{py_file.relative_to(py_file.parent.parent)}:{line_num} - unqualified '{table}': {line[:80]}..."
)
if all_violations:
@@ -127,9 +127,7 @@ class TestRenderer:
assert render_block(block) == "```\nraw text\n```"
def test_section_heading_level(self):
section = Section(
id="purpose", heading="Purpose", level=3, blocks=[ParagraphBlock(text="hi")]
)
section = Section(id="purpose", heading="Purpose", level=3, blocks=[ParagraphBlock(text="hi")])
assert render_section(section).startswith("### Purpose\n\nhi")
def test_document_round_trip_is_stable(self):
@@ -157,18 +155,7 @@ class TestRenderer:
class TestParser:
def test_simple_document(self):
markdown = (
"# Team Overview\n"
"\n"
"Quick summary.\n"
"\n"
"## Members\n"
"\n"
"- Alice\n"
"- Bob\n"
"\n"
"## Cadence\n"
"\n"
"Standups daily.\n"
"# Team Overview\n\nQuick summary.\n\n## Members\n\n- Alice\n- Bob\n\n## Cadence\n\nStandups daily.\n"
)
doc = parse_markdown(markdown)
assert [s.id for s in doc.sections] == ["team-overview", "members", "cadence"]
@@ -182,11 +169,7 @@ class TestParser:
doc = parse_markdown(markdown)
assert [s.id for s in doc.sections] == ["rules", "stop"]
# Horizontal rule must NOT become a paragraph.
assert all(
not (isinstance(b, ParagraphBlock) and "---" in b.text)
for s in doc.sections
for b in s.blocks
)
assert all(not (isinstance(b, ParagraphBlock) and "---" in b.text) for s in doc.sections for b in s.blocks)
def test_ordered_list(self):
markdown = "## Steps\n\n1. one\n2. two\n3. three\n"
@@ -275,9 +258,7 @@ class TestApplyOperations:
def test_insert_block_out_of_range_skipped(self):
doc = _team_overview_doc()
op = InsertBlockOp(
section_id="members", index=99, block=ParagraphBlock(text="x")
)
op = InsertBlockOp(section_id="members", index=99, block=ParagraphBlock(text="x"))
result = apply_operations(doc, [op])
assert result.applied == []
assert "index out of range" in result.skipped[0]["reason"]
@@ -381,13 +362,9 @@ class TestApplyOperations:
)
result = apply_operations(doc, [op])
before_overview = render_section(doc.section_by_id("team-overview"))
after_overview = render_section(
result.document.section_by_id("team-overview")
)
after_overview = render_section(result.document.section_by_id("team-overview"))
before_cadence = render_section(doc.section_by_id("cadence"))
after_cadence = render_section(
result.document.section_by_id("cadence")
)
after_cadence = render_section(result.document.section_by_id("cadence"))
assert before_overview == after_overview
assert before_cadence == after_cadence
@@ -418,9 +395,7 @@ class TestDeltaOperationListSchema:
def test_invalid_op_field_rejected(self):
with pytest.raises(Exception): # pydantic ValidationError
DeltaOperationList.model_validate(
{"operations": [{"op": "not_a_real_op", "section_id": "x"}]}
)
DeltaOperationList.model_validate({"operations": [{"op": "not_a_real_op", "section_id": "x"}]})
def test_extra_field_rejected(self):
with pytest.raises(Exception):
+155 -146
View File
@@ -9,6 +9,7 @@ Use cases:
The tags use OR-based matching: a memory matches if ANY of its tags overlap with the request tags.
"""
from datetime import datetime
import httpx
@@ -329,10 +330,12 @@ class TestBuildTagGroupsWhereClause:
"""AND of two leaves generates AND-joined clause."""
groups = [
TagGroupAnd.model_validate(
{"and": [
{"tags": ["step:5"], "match": "any_strict"},
{"tags": ["user:ep_42"], "match": "all_strict"},
]}
{
"and": [
{"tags": ["step:5"], "match": "any_strict"},
{"tags": ["user:ep_42"], "match": "all_strict"},
]
}
)
]
clause, params, next_offset = build_tag_groups_where_clause(groups, 3)
@@ -348,10 +351,12 @@ class TestBuildTagGroupsWhereClause:
"""OR of two leaves generates OR-joined clause."""
groups = [
TagGroupOr.model_validate(
{"or": [
{"tags": ["step:5"], "match": "any_strict"},
{"tags": ["priority:high"], "match": "all_strict"},
]}
{
"or": [
{"tags": ["step:5"], "match": "any_strict"},
{"tags": ["priority:high"], "match": "all_strict"},
]
}
)
]
clause, params, next_offset = build_tag_groups_where_clause(groups, 1)
@@ -363,11 +368,7 @@ class TestBuildTagGroupsWhereClause:
def test_not_wraps_with_not(self):
"""NOT group wraps child clause with NOT."""
groups = [
TagGroupNot.model_validate(
{"not": {"tags": ["archived"], "match": "any_strict"}}
)
]
groups = [TagGroupNot.model_validate({"not": {"tags": ["archived"], "match": "any_strict"}})]
clause, params, next_offset = build_tag_groups_where_clause(groups, 2)
assert "NOT" in clause
assert "$2" in clause
@@ -378,13 +379,17 @@ class TestBuildTagGroupsWhereClause:
"""AND containing an OR generates correct nested SQL."""
groups = [
TagGroupAnd.model_validate(
{"and": [
{"tags": ["user:alice"], "match": "all_strict"},
{"or": [
{"tags": ["step:5"], "match": "any_strict"},
{"tags": ["priority:high"], "match": "all_strict"},
]},
]}
{
"and": [
{"tags": ["user:alice"], "match": "all_strict"},
{
"or": [
{"tags": ["step:5"], "match": "any_strict"},
{"tags": ["priority:high"], "match": "all_strict"},
]
},
]
}
)
]
clause, params, next_offset = build_tag_groups_where_clause(groups, 1)
@@ -397,11 +402,13 @@ class TestBuildTagGroupsWhereClause:
"""Params are numbered sequentially starting from param_offset."""
groups = [
TagGroupAnd.model_validate(
{"and": [
{"tags": ["a"], "match": "any_strict"},
{"tags": ["b"], "match": "any_strict"},
{"tags": ["c"], "match": "any_strict"},
]}
{
"and": [
{"tags": ["a"], "match": "any_strict"},
{"tags": ["b"], "match": "any_strict"},
{"tags": ["c"], "match": "any_strict"},
]
}
)
]
clause, params, next_offset = build_tag_groups_where_clause(groups, 5)
@@ -421,10 +428,12 @@ class TestBuildTagGroupsWhereClause:
"""Table alias propagates to nested leaves (each leaf uses the alias)."""
groups = [
TagGroupAnd.model_validate(
{"and": [
{"tags": ["a"], "match": "any_strict"},
{"tags": ["b"], "match": "any_strict"},
]}
{
"and": [
{"tags": ["a"], "match": "any_strict"},
{"tags": ["b"], "match": "any_strict"},
]
}
)
]
clause, params, next_offset = build_tag_groups_where_clause(groups, 1, table_alias="mu.")
@@ -433,6 +442,7 @@ class TestBuildTagGroupsWhereClause:
assert "mu.tags" in clause
# No bare 'tags' keyword without the alias prefix (other than inside the alias itself)
import re
bare_tags = re.findall(r"(?<!\.)tags", clause)
assert len(bare_tags) == 0, f"Found bare 'tags' references without alias: {bare_tags}"
@@ -496,16 +506,18 @@ class TestFilterResultsByTagGroups:
"""AND group: both leaf conditions must match."""
groups = [
TagGroupAnd.model_validate(
{"and": [
{"tags": ["user:alice"], "match": "all_strict"},
{"tags": ["step:5"], "match": "any_strict"},
]}
{
"and": [
{"tags": ["user:alice"], "match": "all_strict"},
{"tags": ["step:5"], "match": "any_strict"},
]
}
)
]
results = [
MockResult(["user:alice", "step:5"]), # matches both
MockResult(["user:alice"]), # only matches first
MockResult(["step:5"]), # only matches second
MockResult(["user:alice"]), # only matches first
MockResult(["step:5"]), # only matches second
MockResult(None),
]
filtered = filter_results_by_tag_groups(results, groups)
@@ -516,10 +528,12 @@ class TestFilterResultsByTagGroups:
"""OR group: either condition matching is sufficient."""
groups = [
TagGroupOr.model_validate(
{"or": [
{"tags": ["step:5"], "match": "any_strict"},
{"tags": ["priority:high"], "match": "all_strict"},
]}
{
"or": [
{"tags": ["step:5"], "match": "any_strict"},
{"tags": ["priority:high"], "match": "all_strict"},
]
}
)
]
results = [
@@ -535,11 +549,7 @@ class TestFilterResultsByTagGroups:
def test_not_negation(self):
"""NOT group: inverts the child match."""
groups = [
TagGroupNot.model_validate(
{"not": {"tags": ["archived"], "match": "any_strict"}}
)
]
groups = [TagGroupNot.model_validate({"not": {"tags": ["archived"], "match": "any_strict"}})]
results = [
MockResult(["archived"]),
MockResult(["active"]),
@@ -558,20 +568,24 @@ class TestFilterResultsByTagGroups:
"""AND containing OR: nested boolean logic works correctly."""
groups = [
TagGroupAnd.model_validate(
{"and": [
{"tags": ["user:alice"], "match": "all_strict"},
{"or": [
{"tags": ["step:5"], "match": "any_strict"},
{"tags": ["priority:high"], "match": "any_strict"},
]},
]}
{
"and": [
{"tags": ["user:alice"], "match": "all_strict"},
{
"or": [
{"tags": ["step:5"], "match": "any_strict"},
{"tags": ["priority:high"], "match": "any_strict"},
]
},
]
}
)
]
results = [
MockResult(["user:alice", "step:5"]), # user:alice AND (step:5 OR ...)
MockResult(["user:alice", "priority:high"]), # user:alice AND (... OR priority:high)
MockResult(["user:alice"]), # user:alice but neither step nor priority
MockResult(["step:5"]), # step:5 but not user:alice
MockResult(["user:alice", "step:5"]), # user:alice AND (step:5 OR ...)
MockResult(["user:alice", "priority:high"]), # user:alice AND (... OR priority:high)
MockResult(["user:alice"]), # user:alice but neither step nor priority
MockResult(["step:5"]), # step:5 but not user:alice
MockResult(None),
]
filtered = filter_results_by_tag_groups(results, groups)
@@ -585,8 +599,8 @@ class TestFilterResultsByTagGroups:
]
results = [
MockResult(["user:alice", "step:5"]), # both match
MockResult(["user:alice"]), # only first
MockResult(["step:5"]), # only second
MockResult(["user:alice"]), # only first
MockResult(["step:5"]), # only second
]
filtered = filter_results_by_tag_groups(results, groups)
assert len(filtered) == 1
@@ -619,14 +633,7 @@ async def test_retain_with_tags(api_client, test_bank_id):
# Store memory with tags
response = await api_client.post(
f"/v1/default/banks/{test_bank_id}/memories",
json={
"items": [
{
"content": "Alice loves hiking in the mountains.",
"tags": ["user_alice"]
}
]
}
json={"items": [{"content": "Alice loves hiking in the mountains.", "tags": ["user_alice"]}]},
)
assert response.status_code == 200
result = response.json()
@@ -644,9 +651,9 @@ async def test_retain_with_document_tags(api_client, test_bank_id):
"document_tags": ["session_123"],
"items": [
{"content": "Bob discussed the quarterly report."},
{"content": "Charlie mentioned the new product launch."}
]
}
{"content": "Charlie mentioned the new product launch."},
],
},
)
assert response.status_code == 200
result = response.json()
@@ -662,13 +669,8 @@ async def test_retain_merges_document_and_item_tags(api_client, test_bank_id):
f"/v1/default/banks/{test_bank_id}/memories",
json={
"document_tags": ["session_abc"],
"items": [
{
"content": "Dave talked about machine learning.",
"tags": ["user_dave"]
}
]
}
"items": [{"content": "Dave talked about machine learning.", "tags": ["user_dave"]}],
},
)
assert response.status_code == 200
result = response.json()
@@ -686,14 +688,13 @@ async def test_recall_without_tags_returns_all_memories(api_client, test_bank_id
{"content": "Eve works on natural language processing.", "tags": ["user_eve"]},
{"content": "Frank specializes in computer vision.", "tags": ["user_frank"]},
]
}
},
)
assert response.status_code == 200
# Recall without tags - should return all
response = await api_client.post(
f"/v1/default/banks/{test_bank_id}/memories/recall",
json={"query": "Who works on what?", "budget": "low"}
f"/v1/default/banks/{test_bank_id}/memories/recall", json={"query": "Who works on what?", "budget": "low"}
)
assert response.status_code == 200
results = response.json()["results"]
@@ -715,14 +716,14 @@ async def test_recall_with_tags_filters_memories(api_client, test_bank_id):
{"content": "Grace is a data scientist at Google.", "tags": ["user_grace"]},
{"content": "Henry is a software engineer at Meta.", "tags": ["user_henry"]},
]
}
},
)
assert response.status_code == 200
# Recall with user_grace tag - should only return Grace's memory
response = await api_client.post(
f"/v1/default/banks/{test_bank_id}/memories/recall",
json={"query": "Who works at which company?", "budget": "low", "tags": ["user_grace"]}
json={"query": "Who works at which company?", "budget": "low", "tags": ["user_grace"]},
)
assert response.status_code == 200
results = response.json()["results"]
@@ -746,14 +747,14 @@ async def test_recall_with_multiple_tags_uses_or_matching(api_client, test_bank_
{"content": "Julia manages the design team.", "tags": ["user_julia"]},
{"content": "Karl oversees the marketing team.", "tags": ["user_karl"]},
]
}
},
)
assert response.status_code == 200
# Recall with user_ivan OR user_julia - should return both Ivan and Julia, but not Karl
response = await api_client.post(
f"/v1/default/banks/{test_bank_id}/memories/recall",
json={"query": "Who leads which team?", "budget": "low", "tags": ["user_ivan", "user_julia"]}
json={"query": "Who leads which team?", "budget": "low", "tags": ["user_ivan", "user_julia"]},
)
assert response.status_code == 200
results = response.json()["results"]
@@ -774,18 +775,18 @@ async def test_recall_returns_memories_with_any_overlapping_tag(api_client, test
"items": [
{
"content": "Lisa and Mike discussed the budget in a group chat.",
"tags": ["user_lisa", "user_mike"] # Memory visible to both
"tags": ["user_lisa", "user_mike"], # Memory visible to both
},
{"content": "Nancy reviewed the budget alone.", "tags": ["user_nancy"]},
]
}
},
)
assert response.status_code == 200
# Recall with user_lisa - should return the group chat memory
response = await api_client.post(
f"/v1/default/banks/{test_bank_id}/memories/recall",
json={"query": "What was discussed about the budget?", "budget": "low", "tags": ["user_lisa"]}
json={"query": "What was discussed about the budget?", "budget": "low", "tags": ["user_lisa"]},
)
assert response.status_code == 200
results = response.json()["results"]
@@ -806,7 +807,7 @@ async def test_reflect_with_tags_filters_memories(api_client, test_bank_id):
{"content": "Oscar's favorite color is blue.", "tags": ["user_oscar"]},
{"content": "Peter's favorite color is red.", "tags": ["user_peter"]},
]
}
},
)
assert response.status_code == 200
@@ -817,8 +818,8 @@ async def test_reflect_with_tags_filters_memories(api_client, test_bank_id):
"query": "What is the favorite color?",
"budget": "low",
"tags": ["user_oscar"],
"include": {"facts": {}} # Request facts to verify what was used
}
"include": {"facts": {}}, # Request facts to verify what was used
},
)
assert response.status_code == 200
result = response.json()
@@ -845,14 +846,14 @@ async def test_recall_with_empty_tags_returns_all(api_client, test_bank_id):
{"content": "Quinn studies mathematics.", "tags": ["user_quinn"]},
{"content": "Rachel studies physics.", "tags": ["user_rachel"]},
]
}
},
)
assert response.status_code == 200
# Recall with empty tags list - should return all
response = await api_client.post(
f"/v1/default/banks/{test_bank_id}/memories/recall",
json={"query": "Who studies what?", "budget": "low", "tags": []}
json={"query": "Who studies what?", "budget": "low", "tags": []},
)
assert response.status_code == 200
results = response.json()["results"]
@@ -889,14 +890,14 @@ async def test_multi_user_agent_visibility(api_client):
# Room 3: Group chat with both users
{"content": "In the group meeting, they agreed to meet at noon.", "tags": ["user_a", "user_b"]},
]
}
},
)
assert response.status_code == 200
# User A queries - should see their private chat and group chat
response = await api_client.post(
f"/v1/default/banks/{bank_id}/memories/recall",
json={"query": "What meeting time preferences were discussed?", "budget": "low", "tags": ["user_a"]}
json={"query": "What meeting time preferences were discussed?", "budget": "low", "tags": ["user_a"]},
)
assert response.status_code == 200
user_a_results = response.json()["results"]
@@ -909,7 +910,7 @@ async def test_multi_user_agent_visibility(api_client):
# User B queries - should see their private chat and group chat
response = await api_client.post(
f"/v1/default/banks/{bank_id}/memories/recall",
json={"query": "What meeting time preferences were discussed?", "budget": "low", "tags": ["user_b"]}
json={"query": "What meeting time preferences were discussed?", "budget": "low", "tags": ["user_b"]},
)
assert response.status_code == 200
user_b_results = response.json()["results"]
@@ -922,7 +923,7 @@ async def test_multi_user_agent_visibility(api_client):
# Agent queries (no filter) - should see everything
response = await api_client.post(
f"/v1/default/banks/{bank_id}/memories/recall",
json={"query": "What meeting time preferences were discussed?", "budget": "low"} # No tags
json={"query": "What meeting time preferences were discussed?", "budget": "low"}, # No tags
)
assert response.status_code == 200
agent_results = response.json()["results"]
@@ -955,14 +956,14 @@ async def test_student_tracking_visibility(api_client):
{"content": "Student B struggled with geometry concepts.", "tags": ["student_b"]},
{"content": "Student A participated actively in class discussion.", "tags": ["student_a"]},
]
}
},
)
assert response.status_code == 200
# Student A queries - should only see their own data
response = await api_client.post(
f"/v1/default/banks/{bank_id}/memories/recall",
json={"query": "How am I doing in class?", "budget": "low", "tags": ["student_a"]}
json={"query": "How am I doing in class?", "budget": "low", "tags": ["student_a"]},
)
assert response.status_code == 200
student_a_results = response.json()["results"]
@@ -970,12 +971,14 @@ async def test_student_tracking_visibility(api_client):
assert any("algebra" in t for t in student_a_texts), "Student A should see their algebra progress"
assert any("participated" in t for t in student_a_texts), "Student A should see their participation"
assert not any("Student B" in t or "geometry" in t for t in student_a_texts), "Student A should NOT see Student B's data"
assert not any("Student B" in t or "geometry" in t for t in student_a_texts), (
"Student A should NOT see Student B's data"
)
# Teacher queries (no filter) - should see all students
response = await api_client.post(
f"/v1/default/banks/{bank_id}/memories/recall",
json={"query": "Which students need help?", "budget": "low"} # No tags
json={"query": "Which students need help?", "budget": "low"}, # No tags
)
assert response.status_code == 200
teacher_results = response.json()["results"]
@@ -1011,7 +1014,7 @@ async def test_list_tags_returns_all_tags(api_client):
{"content": "Memory 4 in session 123.", "tags": ["session:123"]},
{"content": "Memory 5 for alice in session 456.", "tags": ["user:alice", "session:456"]},
]
}
},
)
assert response.status_code == 200
@@ -1057,7 +1060,7 @@ async def test_list_tags_with_wildcard_prefix(api_client):
{"content": "Session memory about the meeting.", "tags": ["session:abc"]},
{"content": "Room memory for conference room.", "tags": ["room:123"]},
]
}
},
)
assert response.status_code == 200
@@ -1091,7 +1094,7 @@ async def test_list_tags_with_wildcard_suffix(api_client):
{"content": "Mike is a standard role-user who can only view content.", "tags": ["role-user"]},
{"content": "Alice is a role-guest visitor with limited read access.", "tags": ["role-guest"]},
]
}
},
)
assert response.status_code == 200
@@ -1119,12 +1122,24 @@ async def test_list_tags_with_wildcard_middle(api_client):
f"/v1/default/banks/{bank_id}/memories",
json={
"items": [
{"content": "The production environment is configured with high availability and uses AWS infrastructure.", "tags": ["env-prod"]},
{"content": "The enterprise environment for production runs on dedicated servers with 24/7 monitoring.", "tags": ["environment-prod"]},
{"content": "The staging environment mirrors production but uses smaller instance sizes.", "tags": ["env-staging"]},
{"content": "The development environment allows developers to test their code locally.", "tags": ["env-dev"]},
{
"content": "The production environment is configured with high availability and uses AWS infrastructure.",
"tags": ["env-prod"],
},
{
"content": "The enterprise environment for production runs on dedicated servers with 24/7 monitoring.",
"tags": ["environment-prod"],
},
{
"content": "The staging environment mirrors production but uses smaller instance sizes.",
"tags": ["env-staging"],
},
{
"content": "The development environment allows developers to test their code locally.",
"tags": ["env-dev"],
},
]
}
},
)
assert response.status_code == 200
@@ -1152,11 +1167,17 @@ async def test_list_tags_case_insensitive(api_client):
f"/v1/default/banks/{bank_id}/memories",
json={
"items": [
{"content": "Alice is a software engineer who specializes in machine learning algorithms.", "tags": ["User:Alice"]},
{
"content": "Alice is a software engineer who specializes in machine learning algorithms.",
"tags": ["User:Alice"],
},
{"content": "Bob works as a data scientist at a large technology company.", "tags": ["user:bob"]},
{"content": "Charlie is the lead designer responsible for the user interface.", "tags": ["USER:CHARLIE"]},
{
"content": "Charlie is the lead designer responsible for the user interface.",
"tags": ["USER:CHARLIE"],
},
]
}
},
)
assert response.status_code == 200
@@ -1182,10 +1203,7 @@ async def test_list_tags_pagination(api_client):
{"content": f"{name} works as a software engineer at company {i}.", "tags": [f"tag:{i:03d}"]}
for i, name in enumerate(names)
]
response = await api_client.post(
f"/v1/default/banks/{bank_id}/memories",
json={"items": items}
)
response = await api_client.post(f"/v1/default/banks/{bank_id}/memories", json={"items": items})
assert response.status_code == 200
# Get first page (limit 3)
@@ -1236,7 +1254,7 @@ async def test_list_tags_ordered_by_count(api_client):
{"content": "Eve is a data scientist at Amazon.", "tags": ["medium"]},
{"content": "Frank handles customer support at Meta.", "tags": ["medium"]},
]
}
},
)
assert response.status_code == 200
@@ -1283,9 +1301,7 @@ async def test_list_memories_includes_tags(api_client, test_bank_id):
memory_item = next((item for item in result["items"] if "Alice" in item["text"]), None)
assert memory_item is not None, "Should find the stored memory"
assert "tags" in memory_item, "Memory item must include a 'tags' field"
assert set(memory_item["tags"]) == set(tags), (
f"All {len(tags)} tags should be returned, got: {memory_item['tags']}"
)
assert set(memory_item["tags"]) == set(tags), f"All {len(tags)} tags should be returned, got: {memory_item['tags']}"
# ============================================================================
@@ -1388,10 +1404,12 @@ async def test_tag_groups_or_compound(api_client):
"query": "what are the engineers working on",
"budget": "mid",
"tag_groups": [
{"or": [
{"tags": ["user:alice"], "match": "any_strict"},
{"tags": ["user:bob"], "match": "any_strict"},
]},
{
"or": [
{"tags": ["user:alice"], "match": "any_strict"},
{"tags": ["user:bob"], "match": "any_strict"},
]
},
],
},
)
@@ -1484,13 +1502,17 @@ async def test_tag_groups_nested_and_containing_or(api_client):
"query": "verification step completion",
"budget": "mid",
"tag_groups": [
{"and": [
{"tags": ["user:alice"], "match": "all_strict"},
{"or": [
{"tags": ["step:5"], "match": "any_strict"},
{"tags": ["step:8"], "match": "any_strict"},
]},
]},
{
"and": [
{"tags": ["user:alice"], "match": "all_strict"},
{
"or": [
{"tags": ["step:5"], "match": "any_strict"},
{"tags": ["step:8"], "match": "any_strict"},
]
},
]
},
],
},
)
@@ -1564,9 +1586,7 @@ async def test_list_mental_model_tags_with_wildcard(memory, request_context):
memory, bank_id=bank_id, name="MM 3", tags=["session:abc"], request_context=request_context
)
result = await memory.list_mental_model_tags(
bank_id=bank_id, pattern="topic:*", request_context=request_context
)
result = await memory.list_mental_model_tags(bank_id=bank_id, pattern="topic:*", request_context=request_context)
returned = sorted(item["tag"] for item in result["items"])
assert returned == ["topic:alpha", "topic:beta"]
@@ -1585,9 +1605,7 @@ async def test_list_tags_endpoint_with_source_mental_models(memory, request_cont
app = create_app(memory, initialize_memory=False)
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get(
f"/v1/default/banks/{bank_id}/tags", params={"source": "mental_models"}
)
response = await client.get(f"/v1/default/banks/{bank_id}/tags", params={"source": "mental_models"})
assert response.status_code == 200, response.text
body = response.json()
assert {item["tag"] for item in body["items"]} == {"alpha"}
@@ -1667,9 +1685,7 @@ async def test_reflect_with_tag_groups_propagates_to_internal_recall(memory, req
)
if n == 2:
return LLMToolCallResult(
tool_calls=[
LLMToolCall(id="s1", name="search_observations", arguments={"query": "hardware"})
],
tool_calls=[LLMToolCall(id="s1", name="search_observations", arguments={"query": "hardware"})],
finish_reason="tool_calls",
)
return LLMToolCallResult(
@@ -1703,20 +1719,13 @@ async def test_reflect_with_tag_groups_propagates_to_internal_recall(memory, req
f"got {len(internal_calls)}: {internal_calls}"
)
for call in internal_calls:
assert call["tag_groups"] == tag_groups, (
f"Internal recall_async lost tag_groups; got {call!r}"
)
assert call["tag_groups"] == tag_groups, f"Internal recall_async lost tag_groups; got {call!r}"
# End-to-end: the tool result messages the LLM saw must reference only the
# tagged ("Strix Halo") memory, never the untagged ("MacBook Pro") one.
# This catches any future regression where tag_groups is silently ignored
# at the SQL layer even though kwargs propagation looks correct.
tool_messages = [
msg
for call in mock_llm.get_mock_calls()
for msg in call["messages"]
if msg.get("role") == "tool"
]
tool_messages = [msg for call in mock_llm.get_mock_calls() for msg in call["messages"] if msg.get("role") == "tool"]
tool_payload = "\n".join(msg.get("content", "") for msg in tool_messages)
assert "Strix Halo" in tool_payload, (
f"Tagged 'Strix Halo' memory must appear in the agent's tool results; got: {tool_payload[:1000]!r}"

Some files were not shown because too many files have changed in this diff Show More