{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus",
      "description": "A Prometheus-compatible store receiving CocoIndex's OpenTelemetry metrics."
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "10.0.0"
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    }
  ],
  "title": "CocoIndex",
  "uid": "cocoindex-pipeline",
  "description": "Reference dashboard for CocoIndex Plus pipelines. Reads the metrics the engine exports over OpenTelemetry, after Prometheus's name translation (dots to underscores, _total on counters, unit suffixes). Import, pick the datasource, fork as needed. Generated from the engine's metric catalog; see https://plus.cocoindex.io/docs/advanced_topics/observability/.",
  "tags": [
    "cocoindex"
  ],
  "timezone": "browser",
  "editable": true,
  "graphTooltip": 1,
  "schemaVersion": 39,
  "version": 1,
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "refresh": "30s",
  "templating": {
    "list": [
      {
        "name": "DS_PROMETHEUS",
        "label": "Datasource",
        "type": "datasource",
        "query": "prometheus",
        "refresh": 1,
        "current": {},
        "options": []
      },
      {
        "name": "env",
        "label": "Environment",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "query": {
          "query": "label_values(cocoindex_component_starts_total, env)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "sort": 1,
        "multi": true,
        "includeAll": true,
        "allValue": ".*",
        "current": {},
        "options": []
      },
      {
        "name": "app",
        "label": "App",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "query": {
          "query": "label_values(cocoindex_component_starts_total{env=~\"$env\"}, app)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "sort": 1,
        "multi": true,
        "includeAll": true,
        "allValue": ".*",
        "current": {},
        "options": []
      },
      {
        "name": "stats_group",
        "label": "Stats group",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "query": {
          "query": "label_values(cocoindex_component_starts_total{env=~\"$env\", app=~\"$app\"}, stats_group)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "sort": 1,
        "multi": true,
        "includeAll": true,
        "allValue": ".*",
        "current": {},
        "options": []
      },
      {
        "name": "processor",
        "label": "Processor",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "query": {
          "query": "label_values(cocoindex_component_starts_total{env=~\"$env\", app=~\"$app\", stats_group=~\"$stats_group\"}, processor)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "sort": 1,
        "multi": true,
        "includeAll": true,
        "allValue": ".*",
        "current": {},
        "options": []
      }
    ]
  },
  "annotations": {
    "list": []
  },
  "links": [],
  "panels": [
    {
      "id": 1,
      "title": "Freshness",
      "type": "row",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 0,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 2,
      "title": "Time since last ready",
      "type": "stat",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 1,
        "w": 6,
        "h": 5
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "time() - max by (env, app) (cocoindex_run_last_ready_time_seconds{env=~\"$env\", app=~\"$app\"})",
          "legendFormat": "{{env}}/{{app}}",
          "refId": "A",
          "instant": true,
          "range": false
        }
      ],
      "fieldConfig": {
        "defaults": {
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "orange",
                "value": 3600
              },
              {
                "color": "red",
                "value": 21600
              }
            ]
          },
          "unit": "s"
        },
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "colorMode": "value",
        "graphMode": "none"
      },
      "description": "Seconds since the last update of the app reached ready. Tracks freshness for a long-lived process that runs catch-up updates repeatedly. A live-mode update reaches ready once, so this keeps growing on a healthy live worker; for scheduled jobs that exit, the series goes stale, so check freshness in the scheduler."
    },
    {
      "id": 3,
      "title": "Updates by status",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 6,
        "y": 1,
        "w": 9,
        "h": 5
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (status) (increase(cocoindex_runs_total{env=~\"$env\", app=~\"$app\"}[$__rate_interval]))",
          "legendFormat": "{{status}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {},
      "description": "Updates finishing per interval, by terminal status. A live-mode update runs until the process stops, so a live worker shows here only when it restarts."
    },
    {
      "id": 4,
      "title": "Time to ready (p50 / p95, past hour)",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 15,
        "y": 1,
        "w": 9,
        "h": 5
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.5, sum by (le) (increase(cocoindex_run_time_to_ready_seconds_bucket{env=~\"$env\", app=~\"$app\"}[1h])))",
          "legendFormat": "p50",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.95, sum by (le) (increase(cocoindex_run_time_to_ready_seconds_bucket{env=~\"$env\", app=~\"$app\"}[1h])))",
          "legendFormat": "p95",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "spanNulls": true
          },
          "unit": "s"
        },
        "overrides": []
      },
      "options": {},
      "description": "From the start of an update until initial processing caught up, over the updates of the past hour. Updates are rare events, so a short window would leave gaps. A live-mode update reaches ready once, at its initial catch-up."
    },
    {
      "id": 5,
      "title": "Stats group: $stats_group",
      "type": "row",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 6,
        "w": 24,
        "h": 1
      },
      "panels": [],
      "repeat": "stats_group"
    },
    {
      "id": 6,
      "title": "Component runs by processor and outcome",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 7,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (processor, outcome) (increase(cocoindex_component_runs_total{env=~\"$env\", app=~\"$app\", stats_group=~\"$stats_group\", processor=~\"$processor\"}[$__rate_interval]))",
          "legendFormat": "{{processor}} {{outcome}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right"
        }
      },
      "description": "The per-processor table of the terminal display, over time: unchanged (memo hit), added, reprocessed, deleted, error."
    },
    {
      "id": 7,
      "title": "Skip ratio",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 7,
        "w": 6,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (processor) (increase(cocoindex_component_memo_decisions_total{env=~\"$env\", app=~\"$app\", stats_group=~\"$stats_group\", processor=~\"$processor\", decision=~\"hit|hit_revalidated|hit_after_wait\"}[1h])) / sum by (processor) (increase(cocoindex_component_starts_total{env=~\"$env\", app=~\"$app\", stats_group=~\"$stats_group\", processor=~\"$processor\"}[1h]))",
          "legendFormat": "{{processor}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "min": 0,
          "max": 1,
          "custom": {
            "spanNulls": true
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "options": {},
      "description": "Memo hits over runs started in the past hour, per processor. Excludes hit_same_run (a shared component's later mounts in one run) so sharing does not inflate it. A drop means unexpected reprocessing: a code change, a changed input, or a changed context value. The hour window smooths the bursts between updates."
    },
    {
      "id": 8,
      "title": "In progress",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 18,
        "y": 7,
        "w": 6,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (processor) (cocoindex_component_starts_total{env=~\"$env\", app=~\"$app\", stats_group=~\"$stats_group\", processor=~\"$processor\"}) - sum by (processor) (cocoindex_component_runs_total{env=~\"$env\", app=~\"$app\", stats_group=~\"$stats_group\", processor=~\"$processor\"})",
          "legendFormat": "{{processor}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {},
      "description": "Runs started minus runs finished, per processor. Derived at query time; nothing is gauged in the engine."
    },
    {
      "id": 9,
      "title": "Memo decisions",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 15,
        "w": 12,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (decision) (increase(cocoindex_component_memo_decisions_total{env=~\"$env\", app=~\"$app\", stats_group=~\"$stats_group\", processor=~\"$processor\"}[$__rate_interval]))",
          "legendFormat": "{{decision}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      },
      "options": {},
      "description": "Why runs were reused or rebuilt: hit, hit_same_run, hit_revalidated, hit_after_wait, miss_absent, miss_changed, miss_state_invalid, bypassed (full_reprocess), not_memoized."
    },
    {
      "id": 10,
      "title": "Errors by exception type",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 15,
        "w": 12,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (processor, exception_type, handled) (increase(cocoindex_errors_total{env=~\"$env\", app=~\"$app\", stats_group=~\"$stats_group\", processor=~\"$processor\"}[$__rate_interval]))",
          "legendFormat": "{{processor}} {{exception_type}} ({{handled}})",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "noValue": "0"
        },
        "overrides": []
      },
      "options": {},
      "description": "Component failures. handled=swallowed means an exception handler (or the default log) absorbed it; propagated means it reached the awaiting caller."
    },
    {
      "id": 11,
      "title": "Process duration p95 (user code)",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 22,
        "w": 12,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.95, sum by (processor, le) (rate(cocoindex_component_process_duration_seconds_bucket{env=~\"$env\", app=~\"$app\", stats_group=~\"$stats_group\", processor=~\"$processor\"}[$__rate_interval])))",
          "legendFormat": "{{processor}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "options": {}
    },
    {
      "id": 12,
      "title": "Submit duration p95 by phase",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 22,
        "w": 12,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.95, sum by (phase, le) (rate(cocoindex_component_submit_duration_seconds_bucket{env=~\"$env\", app=~\"$app\", stats_group=~\"$stats_group\", processor=~\"$processor\"}[$__rate_interval])))",
          "legendFormat": "{{phase}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "options": {},
      "description": "precommit (state-store reconcile), sink_apply (writes to the target), commit (state-store commit)."
    },
    {
      "id": 13,
      "title": "Targets",
      "type": "row",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 29,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 14,
      "title": "Target states reconciled by provider and result",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 30,
        "w": 12,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (provider, result) (increase(cocoindex_target_reconciles_total{env=~\"$env\", app=~\"$app\"}[$__rate_interval]))",
          "legendFormat": "{{provider}} {{result}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {},
      "description": "created / updated / deleted are what reached the target; unchanged_skipped and unchanged cost no write."
    },
    {
      "id": 15,
      "title": "Sink apply p95 by sink",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 30,
        "w": 6,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.95, sum by (sink, le) (rate(cocoindex_target_sink_duration_seconds_bucket{env=~\"$env\", app=~\"$app\"}[$__rate_interval])))",
          "legendFormat": "{{sink}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "options": {}
    },
    {
      "id": 16,
      "title": "Sink failures and batch splits",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 18,
        "y": 30,
        "w": 6,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (sink) (increase(cocoindex_target_sink_failures_total{env=~\"$env\", app=~\"$app\"}[$__rate_interval]))",
          "legendFormat": "failed {{sink}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (sink) (increase(cocoindex_target_sink_splits_total{env=~\"$env\", app=~\"$app\"}[$__rate_interval]))",
          "legendFormat": "split {{sink}}",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {},
      "description": "A split is a retry on a smaller batch after a merged sink call failed, to isolate the failing component."
    },
    {
      "id": 17,
      "title": "Concurrency",
      "type": "row",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 37,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 18,
      "title": "Available in-flight tokens",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 38,
        "w": 12,
        "h": 6
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "min by (env, app) (cocoindex_admission_available{env=~\"$env\", app=~\"$app\"})",
          "legendFormat": "{{env}}/{{app}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {},
      "description": "max_inflight_components minus tokens held by runs. Only exported for apps with a bounded pool."
    },
    {
      "id": 19,
      "title": "Admission wait p95",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 38,
        "w": 12,
        "h": 6
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.95, sum by (le) (rate(cocoindex_admission_wait_duration_seconds_bucket{env=~\"$env\", app=~\"$app\"}[$__rate_interval])))",
          "legendFormat": "p95",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "options": {},
      "description": "Time runs waited for an in-flight token. Sustained waits mean max_inflight_components is the limiter."
    }
  ]
}