# Reference alert rules for CocoIndex Plus pipelines (Grafana provisioning
# format). Every threshold below is a STARTING POINT, not a recommendation:
# set them from your own SLOs after watching the dashboard for a while.
#
# Provision with Grafana's file provisioning (`provisioning/alerting/`) after
# replacing ${DS_PROMETHEUS} with your Prometheus datasource UID, or recreate
# the rules in the UI from the expressions. Metric names are the Prometheus
# renderings of the OpenTelemetry names in the metrics reference.
apiVersion: 1
groups:
  - orgId: 1
    name: cocoindex
    folder: CocoIndex
    interval: 1m
    rules:
      - uid: cocoindex-no-recent-ready
        title: CocoIndex app has not reached ready recently
        condition: C
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.app }} in {{ $labels.env }} last reached ready {{ humanizeDuration $values.A }} ago"
          description: >-
            For a long-lived process that runs catch-up updates repeatedly. Not for live mode: a live
            update reaches ready once, at its initial catch-up, so this fires on a healthy live worker.
            Not for scheduled jobs that exit: their series go stale in Prometheus, so drive that check
            from the scheduler instead.
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: ${DS_PROMETHEUS}
            model:
              expr: time() - max by (env, app) (cocoindex_run_last_ready_time_seconds)
              instant: true
              refId: A
          - refId: C
            datasourceUid: __expr__
            model:
              type: threshold
              expression: A
              refId: C
              conditions:
                - evaluator: { type: gt, params: [21600] }   # PLACEHOLDER: 6 hours
      - uid: cocoindex-component-errors
        title: CocoIndex component error rate
        condition: C
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.processor }} in {{ $labels.app }} failing: {{ $values.A }} errors in 15 minutes"
        data:
          - refId: A
            relativeTimeRange: { from: 900, to: 0 }
            datasourceUid: ${DS_PROMETHEUS}
            model:
              expr: sum by (env, app, processor) (increase(cocoindex_errors_total[15m]))
              instant: true
              refId: A
          - refId: C
            datasourceUid: __expr__
            model:
              type: threshold
              expression: A
              refId: C
              conditions:
                - evaluator: { type: gt, params: [10] }      # PLACEHOLDER: 10 errors per 15 minutes
      - uid: cocoindex-skip-ratio-drop
        title: CocoIndex skip ratio dropped (unexpected reprocessing)
        condition: C
        for: 15m
        labels:
          severity: info
        annotations:
          summary: "{{ $labels.processor }} in {{ $labels.app }}: only {{ humanizePercentage $values.A }} of runs were memo hits"
          description: >-
            A low skip ratio on a stable source means components are rebuilding: a code change, a
            changed input, or a changed context value. Expected right after a deploy; sustained, it is
            a cost problem. hit_same_run is excluded so shared components do not inflate the ratio.
        data:
          - refId: A
            relativeTimeRange: { from: 3600, to: 0 }
            datasourceUid: ${DS_PROMETHEUS}
            model:
              expr: >-
                sum by (env, app, processor) (increase(cocoindex_component_memo_decisions_total{decision=~"hit|hit_revalidated|hit_after_wait"}[1h]))
                / sum by (env, app, processor) (increase(cocoindex_component_starts_total[1h]))
              instant: true
              refId: A
          - refId: C
            datasourceUid: __expr__
            model:
              type: threshold
              expression: A
              refId: C
              conditions:
                - evaluator: { type: lt, params: [0.5] }     # PLACEHOLDER: below 50% hits
      - uid: cocoindex-sink-failures
        title: CocoIndex target sink failures
        condition: C
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "sink {{ $labels.sink }} in {{ $labels.app }} failed {{ $values.A }} times in 15 minutes"
          description: >-
            Writes to a target are failing. The engine isolates the failing component and retries on
            the next update, so no data is lost, but the target is behind until the cause is fixed.
        data:
          - refId: A
            relativeTimeRange: { from: 900, to: 0 }
            datasourceUid: ${DS_PROMETHEUS}
            model:
              expr: sum by (env, app, sink) (increase(cocoindex_target_sink_failures_total[15m]))
              instant: true
              refId: A
          - refId: C
            datasourceUid: __expr__
            model:
              type: threshold
              expression: A
              refId: C
              conditions:
                - evaluator: { type: gt, params: [0] }       # PLACEHOLDER: any failure