fix: ui shows only 1000 memories

hindsight-api/hindsight_api/api/http.py

@@ -647,6 +647,7 @@ class GraphDataResponse(BaseModel):
                     }
                 ],
                 "total_units": 2,
+                "limit": 1000,
             }
         }
     )
@@ -655,6 +656,7 @@ class GraphDataResponse(BaseModel):
     edges: list[dict[str, Any]]
     table_rows: list[dict[str, Any]]
     total_units: int
+    limit: int
 
 
 class ListMemoryUnitsResponse(BaseModel):
@@ -1066,16 +1068,19 @@ def _register_routes(app: FastAPI):
         "/v1/default/banks/{bank_id}/graph",
         response_model=GraphDataResponse,
         summary="Get memory graph data",
-        description="Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). Limited to 1000 most recent items.",
+        description="Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion).",
         operation_id="get_graph",
         tags=["Memory"],
     )
     async def api_graph(
-        bank_id: str, type: str | None = None, request_context: RequestContext = Depends(get_request_context)
+        bank_id: str,
+        type: str | None = None,
+        limit: int = 1000,
+        request_context: RequestContext = Depends(get_request_context),
     ):
         """Get graph data from database, filtered by bank_id and optionally by type."""
         try:
-            data = await app.state.memory.get_graph_data(bank_id, type, request_context=request_context)
+            data = await app.state.memory.get_graph_data(bank_id, type, limit=limit, request_context=request_context)
             return data
         except (AuthenticationError, HTTPException):
             raise

hindsight-api/hindsight_api/engine/interface.py

@@ -289,6 +289,7 @@ class MemoryEngineInterface(ABC):
         bank_id: str,
         *,
         fact_type: str | None = None,
+        limit: int = 1000,
         request_context: "RequestContext",
     ) -> dict[str, Any]:
         """
@@ -297,10 +298,11 @@ class MemoryEngineInterface(ABC):
         Args:
             bank_id: The memory bank ID.
             fact_type: Filter by fact type.
+            limit: Maximum number of items to return (default: 1000).
             request_context: Request context for authentication.
 
         Returns:
-            Dict with nodes, edges, table_rows, total_units.
+            Dict with nodes, edges, table_rows, total_units, limit.
         """
         ...
 

hindsight-api/hindsight_api/engine/memory_engine.py

@@ -2264,6 +2264,7 @@ class MemoryEngine(MemoryEngineInterface):
         bank_id: str | None = None,
         fact_type: str | None = None,
         *,
+        limit: int = 1000,
         request_context: "RequestContext",
     ):
         """
@@ -2272,10 +2273,11 @@ class MemoryEngine(MemoryEngineInterface):
         Args:
             bank_id: Filter by bank ID
             fact_type: Filter by fact type (world, experience, opinion)
+            limit: Maximum number of items to return (default: 1000)
             request_context: Request context for authentication.
 
         Returns:
-            Dict with nodes, edges, and table_rows
+            Dict with nodes, edges, table_rows, total_units, and limit
         """
         await self._authenticate_tenant(request_context)
         pool = await self._get_pool()
@@ -2297,15 +2299,29 @@ class MemoryEngine(MemoryEngineInterface):
 
             where_clause = "WHERE " + " AND ".join(query_conditions) if query_conditions else ""
 
+            # Get total count first
+            total_count_result = await conn.fetchrow(
+                f"""
+                SELECT COUNT(*) as total
+                FROM {fq_table("memory_units")}
+                {where_clause}
+            """,
+                *query_params,
+            )
+            total_count = total_count_result["total"] if total_count_result else 0
+
+            # Get units with limit
+            param_count += 1
             units = await conn.fetch(
                 f"""
                 SELECT id, text, event_date, context, occurred_start, occurred_end, mentioned_at, document_id, chunk_id, fact_type
                 FROM {fq_table("memory_units")}
                 {where_clause}
                 ORDER BY mentioned_at DESC NULLS LAST, event_date DESC
-                LIMIT 1000
+                LIMIT ${param_count}
             """,
                 *query_params,
+                limit,
             )
 
             # Get links, filtering to only include links between units of the selected agent
@@ -2442,7 +2458,7 @@ class MemoryEngine(MemoryEngineInterface):
                 }
             )
 
-        return {"nodes": nodes, "edges": edges, "table_rows": table_rows, "total_units": len(units)}
+        return {"nodes": nodes, "edges": edges, "table_rows": table_rows, "total_units": total_count, "limit": limit}
 
     async def list_memory_units(
         self,

hindsight-api/tests/test_causal_relationships.py

@@ -36,7 +36,7 @@ After searching for weeks, I finally found a cheaper apartment in Brooklyn.
         context = "Personal story about housing change"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 3, 15),
             context=context,
@@ -105,7 +105,7 @@ The renovation took three months and cost $15,000.
         context = "Home repair story"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 6, 1),
             context=context,
@@ -140,7 +140,7 @@ Machine learning fascinated me so much that I changed my career to data science.
         context = "Career change story"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 1, 1),
             context=context,
@@ -172,7 +172,7 @@ The new role enabled me to lead a team of engineers.
         context = "Work promotion story"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 2, 15),
             context=context,
@@ -205,7 +205,7 @@ Reduced spending somewhat affected local businesses.
         context = "Economic impact story"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 4, 1),
             context=context,

hindsight-api/tests/test_fact_extraction_quality.py

@@ -43,7 +43,7 @@ Marcus felt anxious about the upcoming interview.
         context = "Personal journal entry"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 11, 13),
             context=context,
@@ -75,7 +75,7 @@ The music was so loud I could barely hear myself think.
         context = "Personal experience"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 11, 13),
             context=context,
@@ -108,7 +108,7 @@ Maybe we should reconsider the timeline.
         context = "Team discussion"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 11, 13),
             context=context,
@@ -141,7 +141,7 @@ I'm unable to attend the conference due to scheduling conflicts.
         context = "Personal profile discussion"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 11, 13),
             context=context,
@@ -173,7 +173,7 @@ Unlike last year, we're ahead of schedule.
         context = "Project review"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 11, 13),
             context=context,
@@ -206,7 +206,7 @@ She's enthusiastic about the opportunity.
         context = "Team meeting"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 11, 13),
             context=context,
@@ -239,7 +239,7 @@ I'm planning to switch careers because I'm not fulfilled in my current role.
         context = "Personal goals discussion"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 11, 13),
             context=context,
@@ -276,7 +276,7 @@ Family is the most important thing to her.
         context = "Personal values discussion"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 11, 13),
             context=context,
@@ -310,7 +310,7 @@ I prefer presenting in person rather than virtually because I can read the room
 
         event_date = datetime(2024, 11, 13)
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=event_date,
             context=context,
@@ -366,7 +366,7 @@ I'm planning to visit Tokyo next month.
 
         event_date = datetime(2024, 11, 13)
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=event_date,
             context=context,
@@ -419,7 +419,7 @@ with a concert surrounded by music, joy and the warm summer breeze.
 
         for attempt in range(max_retries):
             try:
-                facts, _ = await extract_facts_from_text(
+                facts, _, _ = await extract_facts_from_text(
                     text=text,
                     event_date=event_date,
                     context=context,
@@ -488,7 +488,7 @@ Yesterday I went for a morning jog for the first time in a nearby park.
 
         event_date = datetime(2024, 11, 13)
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=event_date,
             context=context,
@@ -537,7 +537,7 @@ Yesterday I went for a morning jog for the first time in a nearby park.
         This morning I had coffee with Alice.
         """
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=reference_date,
             llm_config=llm_config,
@@ -567,7 +567,7 @@ Yesterday I went for a morning jog for the first time in a nearby park.
 
         text = "Alice works at Google. She loves Python programming."
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=reference_date,
             llm_config=llm_config,
@@ -594,7 +594,7 @@ Yesterday I went for a morning jog for the first time in a nearby park.
         Bob will start his vacation on April 1st.
         """
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=reference_date,
             llm_config=llm_config,
@@ -645,7 +645,7 @@ great time! Every time I see it, I can't help but smile.
 
         event_date = datetime(2023, 2, 23)
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=event_date,
             context=context,
@@ -695,7 +695,7 @@ I've learned so much from it.
         context = "Personal update"
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 11, 13),
             context=context,
@@ -758,7 +758,7 @@ Jamie: Congratulations! I'd love to read it.
 
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=transcript,
             event_date=datetime(2024, 11, 13),
             llm_config=llm_config,
@@ -803,7 +803,7 @@ We presented our findings to the team yesterday.
 
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=text,
             event_date=datetime(2024, 11, 13),
             llm_config=llm_config,
@@ -838,7 +838,7 @@ Jamie: [teasing] We'll see who's right, my Niners pick is solid.
 
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=transcript,
             event_date=datetime(2024, 11, 14),
             context=context,
@@ -897,7 +897,7 @@ so the algorithm learns to box out. See you next week!
 
         llm_config = LLMConfig.for_memory()
 
-        facts, _ = await extract_facts_from_text(
+        facts, _, _ = await extract_facts_from_text(
             text=transcript,
             event_date=datetime(2024, 11, 13),
             llm_config=llm_config,

hindsight-clients/python/hindsight_client_api/api/memory_api.py

@@ -347,6 +347,7 @@ class MemoryApi:
         self,
         bank_id: StrictStr,
         type: Optional[StrictStr] = None,
+        limit: Optional[StrictInt] = None,
         authorization: Optional[StrictStr] = None,
         _request_timeout: Union[
             None,
@@ -363,12 +364,14 @@ class MemoryApi:
     ) -> GraphDataResponse:
         """Get memory graph data
 
-        Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). Limited to 1000 most recent items.
+        Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion).
 
         :param bank_id: (required)
         :type bank_id: str
         :param type:
         :type type: str
+        :param limit:
+        :type limit: int
         :param authorization:
         :type authorization: str
         :param _request_timeout: timeout setting for this request. If one
@@ -396,6 +399,7 @@ class MemoryApi:
         _param = self._get_graph_serialize(
             bank_id=bank_id,
             type=type,
+            limit=limit,
             authorization=authorization,
             _request_auth=_request_auth,
             _content_type=_content_type,
@@ -423,6 +427,7 @@ class MemoryApi:
         self,
         bank_id: StrictStr,
         type: Optional[StrictStr] = None,
+        limit: Optional[StrictInt] = None,
         authorization: Optional[StrictStr] = None,
         _request_timeout: Union[
             None,
@@ -439,12 +444,14 @@ class MemoryApi:
     ) -> ApiResponse[GraphDataResponse]:
         """Get memory graph data
 
-        Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). Limited to 1000 most recent items.
+        Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion).
 
         :param bank_id: (required)
         :type bank_id: str
         :param type:
         :type type: str
+        :param limit:
+        :type limit: int
         :param authorization:
         :type authorization: str
         :param _request_timeout: timeout setting for this request. If one
@@ -472,6 +479,7 @@ class MemoryApi:
         _param = self._get_graph_serialize(
             bank_id=bank_id,
             type=type,
+            limit=limit,
             authorization=authorization,
             _request_auth=_request_auth,
             _content_type=_content_type,
@@ -499,6 +507,7 @@ class MemoryApi:
         self,
         bank_id: StrictStr,
         type: Optional[StrictStr] = None,
+        limit: Optional[StrictInt] = None,
         authorization: Optional[StrictStr] = None,
         _request_timeout: Union[
             None,
@@ -515,12 +524,14 @@ class MemoryApi:
     ) -> RESTResponseType:
         """Get memory graph data
 
-        Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). Limited to 1000 most recent items.
+        Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion).
 
         :param bank_id: (required)
         :type bank_id: str
         :param type:
         :type type: str
+        :param limit:
+        :type limit: int
         :param authorization:
         :type authorization: str
         :param _request_timeout: timeout setting for this request. If one
@@ -548,6 +559,7 @@ class MemoryApi:
         _param = self._get_graph_serialize(
             bank_id=bank_id,
             type=type,
+            limit=limit,
             authorization=authorization,
             _request_auth=_request_auth,
             _content_type=_content_type,
@@ -570,6 +582,7 @@ class MemoryApi:
         self,
         bank_id,
         type,
+        limit,
         authorization,
         _request_auth,
         _content_type,
@@ -599,6 +612,10 @@ class MemoryApi:
             
             _query_params.append(('type', type))
             
+        if limit is not None:
+            
+            _query_params.append(('limit', limit))
+            
         # process the header parameters
         if authorization is not None:
             _header_params['authorization'] = authorization

hindsight-clients/python/hindsight_client_api/models/graph_data_response.py

@@ -30,7 +30,8 @@ class GraphDataResponse(BaseModel):
     edges: List[Dict[str, Any]]
     table_rows: List[Dict[str, Any]]
     total_units: StrictInt
-    __properties: ClassVar[List[str]] = ["nodes", "edges", "table_rows", "total_units"]
+    limit: StrictInt
+    __properties: ClassVar[List[str]] = ["nodes", "edges", "table_rows", "total_units", "limit"]
 
     model_config = ConfigDict(
         populate_by_name=True,
@@ -86,7 +87,8 @@ class GraphDataResponse(BaseModel):
             "nodes": obj.get("nodes"),
             "edges": obj.get("edges"),
             "table_rows": obj.get("table_rows"),
-            "total_units": obj.get("total_units")
+            "total_units": obj.get("total_units"),
+            "limit": obj.get("limit")
         })
         return _obj
 

hindsight-clients/typescript/generated/sdk.gen.ts

@@ -123,7 +123,7 @@ export const metricsEndpointMetricsGet = <ThrowOnError extends boolean = false>(
 /**
  * Get memory graph data
  *
- * Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). Limited to 1000 most recent items.
+ * Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion).
  */
 export const getGraph = <ThrowOnError extends boolean = false>(
   options: Options<GetGraphData, ThrowOnError>,

hindsight-clients/typescript/generated/types.gen.ts

@@ -570,6 +570,10 @@ export type GraphDataResponse = {
    * Total Units
    */
   total_units: number;
+  /**
+   * Limit
+   */
+  limit: number;
 };
 
 /**
@@ -1123,6 +1127,10 @@ export type GetGraphData = {
      * Type
      */
     type?: string | null;
+    /**
+     * Limit
+     */
+    limit?: number;
   };
   url: "/v1/default/banks/{bank_id}/graph";
 };

hindsight-control-plane/src/app/api/graph/route.ts

@@ -12,12 +12,15 @@ export async function GET(request: NextRequest) {
 
     // Get optional query parameters
     const type = searchParams.get("type") || searchParams.get("fact_type") || undefined;
+    const limitParam = searchParams.get("limit");
+    const limit = limitParam ? parseInt(limitParam, 10) : undefined;
 
     const response = await sdk.getGraph({
       client: lowLevelClient,
       path: { bank_id: bankId },
       query: {
         type: type,
+        limit: limit,
       },
     });
 

hindsight-control-plane/src/components/data-view.tsx

@@ -52,6 +52,9 @@ export function DataView({ factType }: DataViewProps) {
   const [selectedTableMemory, setSelectedTableMemory] = useState<any>(null);
   const itemsPerPage = 100;
 
+  // Fetch limit state - how many memories to load from the API
+  const [fetchLimit, setFetchLimit] = useState(1000);
+
   // Graph controls state
   const [showLabels, setShowLabels] = useState(true);
   const [maxNodes, setMaxNodes] = useState<number | undefined>(undefined);
@@ -93,7 +96,7 @@ export function DataView({ factType }: DataViewProps) {
     }
   };
 
-  const loadData = async () => {
+  const loadData = async (limit?: number) => {
     if (!currentBank) return;
 
     setLoading(true);
@@ -101,6 +104,7 @@ export function DataView({ factType }: DataViewProps) {
       const graphData: any = await client.getGraph({
         bank_id: currentBank,
         type: factType,
+        limit: limit ?? fetchLimit,
       });
       setData(graphData);
     } catch (error) {
@@ -265,9 +269,25 @@ export function DataView({ factType }: DataViewProps) {
 
           <div className="flex items-center justify-between mb-6">
             <div className="text-sm text-muted-foreground">
-              {searchQuery
-                ? `${filteredTableRows.length} of ${data.total_units} memories`
-                : `${data.total_units} total memories`}
+              {searchQuery ? (
+                `${filteredTableRows.length} of ${data.table_rows?.length ?? 0} loaded memories`
+              ) : data.table_rows?.length < data.total_units ? (
+                <span>
+                  Showing {data.table_rows?.length ?? 0} of {data.total_units} total memories
+                  <button
+                    onClick={() => {
+                      const newLimit = Math.min(data.total_units, fetchLimit + 1000);
+                      setFetchLimit(newLimit);
+                      loadData(newLimit);
+                    }}
+                    className="ml-2 text-primary hover:underline"
+                  >
+                    Load more
+                  </button>
+                </span>
+              ) : (
+                `${data.total_units} total memories`
+              )}
             </div>
             <div className="flex items-center gap-2 bg-muted rounded-lg p-1">
               <button

hindsight-control-plane/src/lib/api.ts

@@ -109,10 +109,11 @@ export class ControlPlaneClient {
   /**
    * Get graph data
    */
-  async getGraph(params: { bank_id: string; type?: string }) {
+  async getGraph(params: { bank_id: string; type?: string; limit?: number }) {
     const queryParams = new URLSearchParams();
     queryParams.append("bank_id", params.bank_id);
     if (params.type) queryParams.append("type", params.type);
+    if (params.limit) queryParams.append("limit", params.limit.toString());
     return this.fetchApi(`/api/graph?${queryParams}`);
   }
 

hindsight-docs/static/openapi.json

@@ -59,7 +59,7 @@
           "Memory"
         ],
         "summary": "Get memory graph data",
-        "description": "Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). Limited to 1000 most recent items.",
+        "description": "Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion).",
         "operationId": "get_graph",
         "parameters": [
           {
@@ -87,6 +87,16 @@
               "title": "Type"
             }
           },
+          {
+            "name": "limit",
+            "in": "query",
+            "required": false,
+            "schema": {
+              "type": "integer",
+              "default": 1000,
+              "title": "Limit"
+            }
+          },
           {
             "name": "authorization",
             "in": "header",
@@ -2504,6 +2514,10 @@
           "total_units": {
             "type": "integer",
             "title": "Total Units"
+          },
+          "limit": {
+            "type": "integer",
+            "title": "Limit"
           }
         },
         "type": "object",
@@ -2511,7 +2525,8 @@
           "nodes",
           "edges",
           "table_rows",
-          "total_units"
+          "total_units",
+          "limit"
         ],
         "title": "GraphDataResponse",
         "description": "Response model for graph data endpoint.",
@@ -2524,6 +2539,7 @@
               "weight": 0.8
             }
           ],
+          "limit": 1000,
           "nodes": [
             {
               "id": "1",