Duke/hindsight
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add tenant to metrics labels (#151)

Browse Source
This commit is contained in:
Nicolò Boschi
2026-01-13 15:31:58 +01:00
committed by GitHub
parent fa53917c63
commit 1ffc2a418c
4 changed files with 136 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hindsight-api/hindsight_api/metrics.py
+15
View File
@@ -28,6 +28,15 @@ from opentelemetry.sdk.resources import Resource
if TYPE_CHECKING:
import asyncpg
def _get_tenant() -> str:
"""Get current tenant (schema) from context for metrics labeling."""
# Import here to avoid circular imports
from hindsight_api.engine.memory_engine import get_current_schema
return get_current_schema()
# Custom bucket boundaries for operation duration (in seconds)
# Fine granularity in 0-30s range where most operations complete
DURATION_BUCKETS = (0.1, 0.25, 0.5, 0.75, 1.0, 2.0, 3.0, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, 120.0)
@@ -323,6 +332,7 @@ class MetricsCollector(MetricsCollectorBase):
"operation": operation,
"bank_id": bank_id,
"source": source,
"tenant": _get_tenant(),
}
if budget:
attributes["budget"] = budget
@@ -373,6 +383,7 @@ class MetricsCollector(MetricsCollectorBase):
"model": model,
"scope": scope,
"success": str(success).lower(),
"tenant": _get_tenant(),
}
# Record duration
@@ -425,10 +436,14 @@ class MetricsCollector(MetricsCollectorBase):
status_code = status_code_getter()
status_class = f"{status_code // 100}xx"
# Get tenant from context (may be set during request processing)
tenant = _get_tenant()
attributes = {
**base_attributes,
"status_code": str(status_code),
"status_class": status_class,
"tenant": tenant,
}
# Record duration and count
monitoring/grafana/dashboards/hindsight-api-service.json
+40 -11
View File
@@ -54,7 +54,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(hindsight_http_requests_total)",
"expr": "sum(hindsight_http_requests_total{tenant=~\"$tenant\"})",
"refId": "A"
}
],
@@ -99,7 +99,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(rate(hindsight_http_requests_total[1m]))",
"expr": "sum(rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[1m]))",
"refId": "A"
}
],
@@ -191,7 +191,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(rate(hindsight_http_requests_total{status_class=\"5xx\"}[5m])) / sum(rate(hindsight_http_requests_total[5m]))",
"expr": "sum(rate(hindsight_http_requests_total{status_class=\"5xx\", tenant=~\"$tenant\"}[5m])) / sum(rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[5m]))",
"refId": "A"
}
],
@@ -236,7 +236,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_http_duration_seconds_bucket[5m])))",
"expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_http_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
"refId": "A"
}
],
@@ -313,7 +313,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum by (endpoint) (rate(hindsight_http_requests_total[1m]))",
"expr": "sum by (endpoint) (rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[1m]))",
"legendFormat": "{{endpoint}}",
"refId": "A"
}
@@ -404,17 +404,17 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_http_duration_seconds_bucket[5m])))",
"expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_http_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_http_duration_seconds_bucket[5m])))",
"expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_http_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
"legendFormat": "p95",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_http_duration_seconds_bucket[5m])))",
"expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_http_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
"legendFormat": "p99",
"refId": "C"
}
@@ -505,12 +505,12 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(rate(hindsight_http_requests_total{status_class=\"5xx\"}[1m])) / sum(rate(hindsight_http_requests_total[1m]))",
"expr": "sum(rate(hindsight_http_requests_total{status_class=\"5xx\", tenant=~\"$tenant\"}[1m])) / sum(rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[1m]))",
"legendFormat": "5xx Error Rate",
"refId": "A"
},
{
"expr": "sum(rate(hindsight_http_requests_total{status_class=\"4xx\"}[1m])) / sum(rate(hindsight_http_requests_total[1m]))",
"expr": "sum(rate(hindsight_http_requests_total{status_class=\"4xx\", tenant=~\"$tenant\"}[1m])) / sum(rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[1m]))",
"legendFormat": "4xx Error Rate",
"refId": "B"
}
@@ -1276,7 +1276,36 @@
"schemaVersion": 38,
"tags": ["hindsight", "api", "service"],
"templating": {
"list": []
"list": [
{
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"definition": "label_values(hindsight_http_requests_total, tenant)",
"hide": 0,
"includeAll": true,
"label": "Tenant",
"multi": false,
"name": "tenant",
"options": [],
"query": {
"query": "label_values(hindsight_http_requests_total, tenant)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-30m",
monitoring/grafana/dashboards/hindsight-llm.json
+40 -11
View File
@@ -46,7 +46,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(hindsight_llm_calls_total)",
"expr": "sum(hindsight_llm_calls_total{tenant=~\"$tenant\"})",
"refId": "A"
}
],
@@ -91,7 +91,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(hindsight_llm_tokens_input_tokens_total) + sum(hindsight_llm_tokens_output_tokens_total)",
"expr": "sum(hindsight_llm_tokens_input_tokens_total{tenant=~\"$tenant\"}) + sum(hindsight_llm_tokens_output_tokens_total{tenant=~\"$tenant\"})",
"refId": "A"
}
],
@@ -137,7 +137,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(hindsight_llm_tokens_input_tokens_total)",
"expr": "sum(hindsight_llm_tokens_input_tokens_total{tenant=~\"$tenant\"})",
"refId": "A"
}
],
@@ -183,7 +183,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(hindsight_llm_tokens_output_tokens_total)",
"expr": "sum(hindsight_llm_tokens_output_tokens_total{tenant=~\"$tenant\"})",
"refId": "A"
}
],
@@ -260,7 +260,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum by (scope) (rate(hindsight_llm_calls_total[1m]))",
"expr": "sum by (scope) (rate(hindsight_llm_calls_total{tenant=~\"$tenant\"}[1m]))",
"legendFormat": "{{scope}}",
"refId": "A"
}
@@ -347,12 +347,12 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(rate(hindsight_llm_tokens_input_tokens_total[1m]))",
"expr": "sum(rate(hindsight_llm_tokens_input_tokens_total{tenant=~\"$tenant\"}[1m]))",
"legendFormat": "Input",
"refId": "A"
},
{
"expr": "sum(rate(hindsight_llm_tokens_output_tokens_total[1m]))",
"expr": "sum(rate(hindsight_llm_tokens_output_tokens_total{tenant=~\"$tenant\"}[1m]))",
"legendFormat": "Output",
"refId": "B"
}
@@ -430,7 +430,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (scope, le) (rate(hindsight_llm_duration_seconds_bucket[5m])))",
"expr": "histogram_quantile(0.95, sum by (scope, le) (rate(hindsight_llm_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
"legendFormat": "{{scope}}",
"refId": "A"
}
@@ -508,12 +508,12 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum by (scope) (rate(hindsight_llm_tokens_input_tokens_total[1m]))",
"expr": "sum by (scope) (rate(hindsight_llm_tokens_input_tokens_total{tenant=~\"$tenant\"}[1m]))",
"legendFormat": "{{scope}} (input)",
"refId": "A"
},
{
"expr": "sum by (scope) (rate(hindsight_llm_tokens_output_tokens_total[1m]))",
"expr": "sum by (scope) (rate(hindsight_llm_tokens_output_tokens_total{tenant=~\"$tenant\"}[1m]))",
"legendFormat": "{{scope}} (output)",
"refId": "B"
}
@@ -526,7 +526,36 @@
"schemaVersion": 38,
"tags": ["hindsight", "llm"],
"templating": {
"list": []
"list": [
{
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"definition": "label_values(hindsight_llm_calls_total, tenant)",
"hide": 0,
"includeAll": true,
"label": "Tenant",
"multi": false,
"name": "tenant",
"options": [],
"query": {
"query": "label_values(hindsight_llm_calls_total, tenant)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-30m",
monitoring/grafana/dashboards/hindsight-operations.json
+41 -12
View File
@@ -46,7 +46,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(hindsight_operation_operations_total)",
"expr": "sum(hindsight_operation_operations_total{tenant=~\"$tenant\"})",
"refId": "A"
}
],
@@ -91,7 +91,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(rate(hindsight_operation_operations_total[1m]))",
"expr": "sum(rate(hindsight_operation_operations_total{tenant=~\"$tenant\"}[1m]))",
"refId": "A"
}
],
@@ -137,7 +137,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(rate(hindsight_operation_operations_total{operation=\"retain\"}[1m]))",
"expr": "sum(rate(hindsight_operation_operations_total{operation=\"retain\", tenant=~\"$tenant\"}[1m]))",
"refId": "A"
}
],
@@ -183,7 +183,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(rate(hindsight_operation_operations_total{operation=\"recall\"}[1m]))",
"expr": "sum(rate(hindsight_operation_operations_total{operation=\"recall\", tenant=~\"$tenant\"}[1m]))",
"refId": "A"
}
],
@@ -229,7 +229,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum(rate(hindsight_operation_operations_total{operation=\"reflect\"}[1m]))",
"expr": "sum(rate(hindsight_operation_operations_total{operation=\"reflect\", tenant=~\"$tenant\"}[1m]))",
"refId": "A"
}
],
@@ -319,7 +319,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum by (operation) (rate(hindsight_operation_operations_total[1m]))",
"expr": "sum by (operation) (rate(hindsight_operation_operations_total{tenant=~\"$tenant\"}[1m]))",
"legendFormat": "{{operation}}",
"refId": "A"
}
@@ -410,17 +410,17 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))",
"expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\", tenant=~\"$tenant\"}[5m])))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))",
"expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\", tenant=~\"$tenant\"}[5m])))",
"legendFormat": "p95",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))",
"expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\", tenant=~\"$tenant\"}[5m])))",
"legendFormat": "p99",
"refId": "C"
}
@@ -498,7 +498,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (operation, le) (rate(hindsight_operation_duration_seconds_bucket[5m])))",
"expr": "histogram_quantile(0.95, sum by (operation, le) (rate(hindsight_operation_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
"legendFormat": "{{operation}}",
"refId": "A"
}
@@ -576,7 +576,7 @@
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "sum by (bank_id) (rate(hindsight_operation_operations_total[1m]))",
"expr": "sum by (bank_id) (rate(hindsight_operation_operations_total{tenant=~\"$tenant\"}[1m]))",
"legendFormat": "{{bank_id}}",
"refId": "A"
}
@@ -589,7 +589,36 @@
"schemaVersion": 38,
"tags": ["hindsight"],
"templating": {
"list": []
"list": [
{
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"definition": "label_values(hindsight_operation_operations_total, tenant)",
"hide": 0,
"includeAll": true,
"label": "Tenant",
"multi": false,
"name": "tenant",
"options": [],
"query": {
"query": "label_values(hindsight_operation_operations_total, tenant)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-30m",