speed up batch writes

- Update version to 0.0.18 in all components
- Python packages: hindsight-api, hindsight-dev, hindsight-dev/benchmarks
- Python client: hindsight-clients/python
- TypeScript client: hindsight-clients/typescript
- Rust CLI: hindsight-cli
- Control Plane: hindsight-control-plane
- Helm chart

helm/hindsight/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: hindsight
 description: A Helm chart for Hindsight - temporal-semantic-entity memory system for AI agents
 type: application
-version: 0.0.17
-appVersion: "0.0.17"
+version: 0.0.18
+appVersion: "0.0.18"
 keywords:
   - ai
   - memory

hindsight-api/alembic/versions/5a366d414dce_initial_schema.py

@@ -24,7 +24,6 @@ def upgrade() -> None:
     """Upgrade schema - create all tables from scratch."""
 
     # Enable required extensions
-    op.execute('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"')
     op.execute('CREATE EXTENSION IF NOT EXISTS vector')
 
     # Create banks table
@@ -57,7 +56,7 @@ def upgrade() -> None:
     # Create async_operations table
     op.create_table(
         'async_operations',
-        sa.Column('operation_id', postgresql.UUID(as_uuid=True), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+        sa.Column('operation_id', postgresql.UUID(as_uuid=True), server_default=sa.text('gen_random_uuid()'), nullable=False),
         sa.Column('bank_id', sa.Text(), nullable=False),
         sa.Column('operation_type', sa.Text(), nullable=False),
         sa.Column('status', sa.Text(), server_default='pending', nullable=False),
@@ -76,7 +75,7 @@ def upgrade() -> None:
     # Create entities table
     op.create_table(
         'entities',
-        sa.Column('id', postgresql.UUID(as_uuid=True), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+        sa.Column('id', postgresql.UUID(as_uuid=True), server_default=sa.text('gen_random_uuid()'), nullable=False),
         sa.Column('canonical_name', sa.Text(), nullable=False),
         sa.Column('bank_id', sa.Text(), nullable=False),
         sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
@@ -94,7 +93,7 @@ def upgrade() -> None:
     # Create memory_units table
     op.create_table(
         'memory_units',
-        sa.Column('id', postgresql.UUID(as_uuid=True), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+        sa.Column('id', postgresql.UUID(as_uuid=True), server_default=sa.text('gen_random_uuid()'), nullable=False),
         sa.Column('bank_id', sa.Text(), nullable=False),
         sa.Column('document_id', sa.Text(), nullable=True),
         sa.Column('text', sa.Text(), nullable=False),

hindsight-api/hindsight_api/engine/retain/link_utils.py

@@ -529,25 +529,53 @@ async def create_semantic_links_batch(
         raise
 
 
-async def insert_entity_links_batch(conn, links: List[tuple]):
+async def insert_entity_links_batch(conn, links: List[tuple], chunk_size: int = 5000):
     """
-    Insert all entity links in a single batch.
+    Insert all entity links in bulk using unnest for efficiency.
+
+    Uses PostgreSQL unnest() to insert many rows in a single query,
+    which is much faster than executemany over high-latency connections.
 
     Args:
         conn: Database connection
         links: List of tuples (from_unit_id, to_unit_id, link_type, weight, entity_id)
+        chunk_size: Number of rows per batch (default 5000)
     """
     if not links:
         return
 
-    await conn.executemany(
-        """
-        INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id)
-        VALUES ($1, $2, $3, $4, $5)
-        ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
-        """,
-        links
-    )
+    import uuid as uuid_mod
+
+    # Process in chunks to avoid query size limits
+    for i in range(0, len(links), chunk_size):
+        chunk = links[i:i + chunk_size]
+
+        # Separate into arrays for unnest
+        from_ids = []
+        to_ids = []
+        link_types = []
+        weights = []
+        entity_ids = []
+
+        for from_id, to_id, link_type, weight, entity_id in chunk:
+            from_ids.append(uuid_mod.UUID(from_id) if isinstance(from_id, str) else from_id)
+            to_ids.append(uuid_mod.UUID(to_id) if isinstance(to_id, str) else to_id)
+            link_types.append(link_type)
+            weights.append(weight)
+            entity_ids.append(
+                uuid_mod.UUID(str(entity_id)) if entity_id and not isinstance(entity_id, uuid_mod.UUID)
+                else entity_id
+            )
+
+        # Use unnest to insert all rows in one query
+        await conn.execute(
+            """
+            INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id)
+            SELECT * FROM unnest($1::uuid[], $2::uuid[], $3::text[], $4::float[], $5::uuid[])
+            ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
+            """,
+            from_ids, to_ids, link_types, weights, entity_ids
+        )
 
 
 async def create_causal_links_batch(

hindsight-api/hindsight_api/models.py

@@ -59,7 +59,7 @@ class MemoryUnit(Base):
     __tablename__ = "memory_units"
 
     id: Mapped[PyUUID] = mapped_column(
-        UUID(as_uuid=True), primary_key=True, server_default=sql_text("uuid_generate_v4()")
+        UUID(as_uuid=True), primary_key=True, server_default=sql_text("gen_random_uuid()")
     )
     bank_id: Mapped[str] = mapped_column(Text, nullable=False)
     document_id: Mapped[Optional[str]] = mapped_column(Text)
@@ -155,7 +155,7 @@ class Entity(Base):
     __tablename__ = "entities"
 
     id: Mapped[PyUUID] = mapped_column(
-        UUID(as_uuid=True), primary_key=True, server_default=sql_text("uuid_generate_v4()")
+        UUID(as_uuid=True), primary_key=True, server_default=sql_text("gen_random_uuid()")
     )
     canonical_name: Mapped[str] = mapped_column(Text, nullable=False)
     bank_id: Mapped[str] = mapped_column(Text, nullable=False)

hindsight-api/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-api"
-version = "0.0.17"
+version = "0.0.18"
 description = "Temporal + Semantic + Entity Memory System for AI agents using PostgreSQL"
 readme = "README.md"
 requires-python = ">=3.11"

hindsight-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hindsight-cli"
-version = "0.0.17"
+version = "0.0.18"
 edition = "2021"
 authors = ["Hindsight Team"]
 description = "A beautiful CLI for Hindsight - semantic memory system"

hindsight-clients/python/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hindsight-client"
-version = "0.0.17"
+version = "0.0.18"
 description = "Python client for Hindsight - Semantic memory system with personality-driven thinking"
 authors = [
     {name = "Hindsight Team"}

hindsight-clients/typescript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vectorize-io/hindsight-client",
-  "version": "0.0.17",
+  "version": "0.0.18",
   "description": "TypeScript client for Hindsight - Semantic memory system with personality-driven thinking",
   "main": "./dist/src/index.js",
   "types": "./dist/src/index.d.ts",

hindsight-control-plane/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hindsight-control-plane",
-  "version": "0.0.17",
+  "version": "0.0.18",
   "private": true,
   "scripts": {
     "dev": "next dev",

hindsight-dev/benchmarks/locomo/results/results_table.md

@@ -1,16 +0,0 @@
-# LoComo Benchmark Results
-
-**Overall Accuracy**: 79.61% (1226/1540)
-
-| Sample ID | Sessions | Questions | Correct | Accuracy | Multi-hop | Single-hop | Temporal | Open-domain |
-|-----------|----------|-----------|---------|----------|-----------|------------|----------|-------------|
-| conv-49 | 25 | 156 | 128 | 82.05% | N/A | N/A | N/A | N/A |
-| conv-48 | 30 | 191 | 153 | 80.10% | N/A | N/A | N/A | N/A |
-| conv-41 | 32 | 152 | 136 | 89.47% | N/A | N/A | N/A | N/A |
-| conv-30 | 19 | 81 | 62 | 76.54% | N/A | N/A | N/A | N/A |
-| conv-47 | 31 | 150 | 120 | 80.00% | N/A | N/A | N/A | N/A |
-| conv-26 | 19 | 152 | 114 | 75.00% | N/A | N/A | N/A | N/A |
-| conv-44 | 28 | 123 | 97 | 78.86% | N/A | N/A | N/A | N/A |
-| conv-43 | 29 | 178 | 136 | 76.40% | N/A | N/A | N/A | N/A |
-| conv-50 | 30 | 158 | 127 | 80.38% | N/A | N/A | N/A | N/A |
-| conv-42 | 29 | 199 | 153 | 76.88% | N/A | N/A | N/A | N/A |

hindsight-dev/benchmarks/locomo/results/results_table_think.md

@@ -1,16 +0,0 @@
-# LoComo Benchmark Results (Think Mode)
-
-**Overall Accuracy**: 77.24% (1191/1542)
-
-| Sample ID | Sessions | Questions | Correct | Accuracy | Multi-hop | Single-hop | Temporal | Open-domain |
-|-----------|----------|-----------|---------|----------|-----------|------------|----------|-------------|
-| conv-26 | 19 | 154 | 120 | 77.92% | N/A | N/A | N/A | N/A |
-| conv-30 | 19 | 81 | 64 | 79.01% | N/A | N/A | N/A | N/A |
-| conv-41 | 32 | 152 | 122 | 80.26% | N/A | N/A | N/A | N/A |
-| conv-42 | 29 | 199 | 152 | 76.38% | N/A | N/A | N/A | N/A |
-| conv-43 | 29 | 178 | 131 | 73.60% | N/A | N/A | N/A | N/A |
-| conv-44 | 28 | 123 | 93 | 75.61% | N/A | N/A | N/A | N/A |
-| conv-47 | 31 | 150 | 117 | 78.00% | N/A | N/A | N/A | N/A |
-| conv-48 | 30 | 191 | 146 | 76.44% | N/A | N/A | N/A | N/A |
-| conv-49 | 25 | 156 | 123 | 78.85% | N/A | N/A | N/A | N/A |
-| conv-50 | 30 | 158 | 123 | 77.85% | N/A | N/A | N/A | N/A |

hindsight-dev/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hindsight-dev"
-version = "0.0.17"
+version = "0.0.18"
 description = "Development utilities for Hindsight"
 requires-python = ">=3.11"
 dependencies = [

uv.lock

@@ -1141,7 +1141,7 @@ provides-extras = ["test"]
 
 [[package]]
 name = "hindsight-api"
-version = "0.0.16"
+version = "0.0.18"
 source = { editable = "hindsight-api" }
 dependencies = [
     { name = "alembic" },
@@ -1243,7 +1243,7 @@ dev = [
 
 [[package]]
 name = "hindsight-client"
-version = "0.0.16"
+version = "0.0.18"
 source = { editable = "hindsight-clients/python" }
 dependencies = [
     { name = "aiohttp" },
@@ -1275,7 +1275,7 @@ provides-extras = ["test"]
 
 [[package]]
 name = "hindsight-dev"
-version = "0.0.16"
+version = "0.0.18"
 source = { editable = "hindsight-dev" }
 dependencies = [
     { name = "hindsight-api" },