{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "Prometheus stats and troubleshooting", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 5, "links": [], "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Uptime of the Prometheus application", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 0, "y": 0 }, "id": 57, "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "(time() - process_start_time_seconds{job=\"prometheus\"})", "format": "time_series", "instant": true, "intervalFactor": 2, "refId": "A", "step": 600 } ], "title": "Uptime", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Prometheus build version", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 2, "y": 0 }, "id": 51, "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "name", "wideLayout": true }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "prometheus_build_info", "format": "time_series", "instant": true, "intervalFactor": 2, "legendFormat": "{{ version }}", "refId": "A", "step": 600 } ], "title": "Version", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Service Availability over past 2 weeks (up and running because targets could be scraped)", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(245, 54, 54, 0.9)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0.99 }, { "color": "rgba(50, 172, 45, 0.97)", "value": 0.999 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 4, "y": 0 }, "id": 50, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "avg(avg_over_time(up{job=\"prometheus\"}[2w]))", "format": "time_series", "instant": true, "intervalFactor": 2, "legendFormat": "", "refId": "A", "step": 600 } ], "title": "SA %", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Data Availability : Current % of targets which are available for all Prometheus jobs, ideally it should be 100% meaning that all nodes, all containers are properly scraped, ie Prometheus DA =100%", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(245, 54, 54, 0.9)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0.84 }, { "color": "rgba(50, 172, 45, 0.97)", "value": 0.99 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 6, "y": 0 }, "id": 34, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(up)/count(up)", "format": "time_series", "instant": true, "intervalFactor": 2, "legendFormat": "", "refId": "A", "step": 600 } ], "title": "DA %", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The number of open head chunks.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(50, 172, 45, 0.97)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 1 }, { "color": "rgba(245, 54, 54, 0.9)", "value": 5 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 8, "y": 0 }, "id": 33, "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(scrape_samples_scraped)", "format": "time_series", "instant": true, "intervalFactor": 2, "legendFormat": "", "refId": "A", "step": 600 } ], "title": "Scraped", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Sample ingestion rate good to know for dimensioning the CPU and memory of Prometheus", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(50, 172, 45, 0.97)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 1 }, { "color": "rgba(245, 54, 54, 0.9)", "value": 5 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 10, "y": 0 }, "id": 36, "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "/^\\{instance=\"prometheus:8080\", job=\"prometheus\", type=\"float\"\\}$/", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(prometheus_tsdb_head_samples_appended_total[1h])", "format": "time_series", "instant": true, "intervalFactor": 1, "legendFormat": "", "range": false, "refId": "A", "step": 600 } ], "title": "Ingested/Hour", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "total WAL corruptions tsdb", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "0": { "text": "None" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(50, 172, 45, 0.97)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0.1 }, { "color": "rgba(245, 54, 54, 0.9)", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 12, "y": 0 }, "id": 38, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "max" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "prometheus_tsdb_wal_corruptions_total{job=\"prometheus\"}", "format": "time_series", "instant": true, "intervalFactor": 2, "legendFormat": "", "refId": "A", "step": 60 } ], "title": "WAL Corruptions", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "% CPU max usage (max usage/Limit), should not exceed 70%, otherwise Prometheus may be unresponsive", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(50, 172, 45, 0.97)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0.7 }, { "color": "rgba(245, 54, 54, 0.9)", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 14, "y": 0 }, "id": 45, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "max" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "max(irate(process_cpu_seconds_total{job=\"prometheus\"}[1m]))/avg(kube_pod_container_resource_limits_cpu_cores{container=\"prometheus\"})", "format": "time_series", "instant": false, "intervalFactor": 2, "legendFormat": "", "refId": "A", "step": 600 } ], "title": "CPU Usage %", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "% Mem max usage (max RSS/Limit), should not exceed 70%, otherwise Prometheus may crash", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(50, 172, 45, 0.97)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0.7 }, { "color": "rgba(245, 54, 54, 0.9)", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 16, "y": 0 }, "id": 31, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "max" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "max(process_resident_memory_bytes{job=\"prometheus\"})/avg(kube_pod_container_resource_limits_memory_bytes{container=\"prometheus\"})", "format": "time_series", "instant": false, "intervalFactor": 2, "legendFormat": "", "refId": "A", "step": 600 } ], "title": "Mem Usage %", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "storage usage available", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "#299c46", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 70 }, { "color": "#d44a3a", "value": 90 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 2, "w": 3, "x": 18, "y": 0 }, "id": 54, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "max by (persistentvolumeclaim)(kubelet_volume_stats_used_bytes{persistentvolumeclaim=~\"platform-monitoring-prometheus-volume-claim\"}*100/kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\"platform-monitoring-prometheus-volume-claim\"})", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "", "range": true, "refId": "A" } ], "title": "Storage Usage %", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "storage size available ", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 2, "w": 3, "x": 21, "y": 0 }, "id": 56, "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "max" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "max by (persistentvolumeclaim)(kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\"platform-monitoring-prometheus-volume-claim\"})", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "", "range": true, "refId": "A" } ], "title": "Storage Size", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Cores", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 2 }, "id": 18, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "irate(process_cpu_seconds_total{job=\"prometheus\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "1m rate", "metric": "prometheus_local_storage_ingested_samples_total", "refId": "A", "step": 60 }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "rate(process_cpu_seconds_total{job=\"prometheus\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "5m rate", "metric": "prometheus_local_storage_ingested_samples_total", "refId": "B", "step": 60 }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "avg(kube_pod_container_resource_requests_cpu_cores{container=\"prometheus\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Req", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "avg(kube_pod_container_resource_limits_cpu_cores{container=\"prometheus\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Limit", "refId": "D" } ], "title": "CPU", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "RAM", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Allocated bytes" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Allocated bytes - 1m max" }, "properties": [ { "id": "color", "value": { "fixedColor": "#BF1B00", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Allocated bytes - 1m min" }, "properties": [ { "id": "color", "value": { "fixedColor": "#BF1B00", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Allocated bytes - 5m max" }, "properties": [ { "id": "color", "value": { "fixedColor": "#BF1B00", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Allocated bytes - 5m min" }, "properties": [ { "id": "color", "value": { "fixedColor": "#BF1B00", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Chunks" }, "properties": [ { "id": "color", "value": { "fixedColor": "#1F78C1", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Chunks to persist" }, "properties": [ { "id": "color", "value": { "fixedColor": "#508642", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Max chunks" }, "properties": [ { "id": "color", "value": { "fixedColor": "#052B51", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Max to persist" }, "properties": [ { "id": "color", "value": { "fixedColor": "#3F6833", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "RSS" }, "properties": [ { "id": "color", "value": { "fixedColor": "#447EBC", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "/-/" }, "properties": [ { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 2 }, "id": 19, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "process_resident_memory_bytes{job=\"prometheus\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "RSS", "metric": "process_resident_memory_bytes", "refId": "B", "step": 60 }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "max_over_time(go_memstats_alloc_bytes{job=\"prometheus\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "Max Go", "metric": "go_memstats_alloc_bytes", "refId": "D", "step": 60 }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "go_memstats_alloc_bytes{job=\"prometheus\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Allocated bytes", "metric": "go_memstats_alloc_bytes", "refId": "A", "step": 60 }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "min_over_time(go_memstats_alloc_bytes{job=\"prometheus\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "Min Go", "metric": "go_memstats_alloc_bytes", "refId": "C", "step": 60 }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "avg(kube_pod_container_resource_requests_memory_bytes{container=\"prometheus\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Req", "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "avg(kube_pod_container_resource_limits_memory_bytes{container=\"prometheus\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Limit", "refId": "F" } ], "title": "Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 7 }, "id": 35, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum by (job) (scrape_samples_scraped)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A" } ], "title": "Scraped samples by job", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 7 }, "id": 52, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "topk(3,scrape_samples_scraped)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} | {{pod}} | {{instance}}", "refId": "A" } ], "title": "Top Scraped Jobs", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 12, "x": 0, "y": 12 }, "id": 40, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": " max(scrape_duration_seconds) by (job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "", "refId": "A", "step": 20 } ], "title": "Scrape Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Scrape Duration will show how long the scrapes are taking, with percentiles available as series.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 12, "x": 12, "y": 12 }, "id": 14, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "prometheus_target_interval_length_seconds{quantile=\"0.99\"}", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{quantile}} ({{interval}})", "metric": "", "refId": "A", "step": 60 } ], "title": "Scrape Duration Interval", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "% of targets which are available for all Prometheus jobs, ideally it should be 100% meaning that all nodes, all containers are properly scraped, ie Prometheus DA =100% You can look at the table on the right side to check what targets were not scraped.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "decimals": 1, "links": [], "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": 0 }, { "color": "transparent", "value": 0.84 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 16 }, "id": 42, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(up)/count(up)", "format": "time_series", "interval": "60m", "intervalFactor": 1, "legendFormat": "Coverage", "range": true, "refId": "A" } ], "timeFrom": "7d", "title": "Data Availability", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Any targets that is not scraped", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "footer": { "reducers": [] }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "displayName", "value": "Time" }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "__name__" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "beta_kubernetes_io_arch" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "beta_kubernetes_io_instance_type" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "beta_kubernetes_io_os" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "failure_domain_beta_kubernetes_io_region" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "failure_domain_beta_kubernetes_io_zone" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "instance" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "partition" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "name" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "role" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "node_role_kubernetes_io_compute" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "job" }, "properties": [ { "id": "displayName", "value": "job" }, { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" }, { "id": "custom.width", "value": 148 } ] }, { "matcher": { "id": "byName", "options": "Value #A" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "Value #B" }, "properties": [ { "id": "displayName", "value": "target coverage" }, { "id": "unit", "value": "percentunit" }, { "id": "custom.align" }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "red", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0.84 }, { "color": "green", "value": 0.99 } ] } }, { "id": "custom.cellOptions", "value": { "mode": "gradient", "type": "color-background" } }, { "id": "decimals", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "nodetype" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "endpoint" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "Value #A" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.align" } ] } ] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 }, "id": 44, "options": { "cellHeight": "sm", "showHeader": true, "sortBy": [ { "desc": false, "displayName": "target coverage" } ] }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "up == 0", "format": "table", "hide": false, "instant": true, "intervalFactor": 1, "legendFormat": "", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "(sum by (job)(up)/count by (job)(up))<.99", "format": "table", "instant": true, "intervalFactor": 1, "legendFormat": "Coverage", "refId": "B" } ], "title": "Down Targets", "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Value #A": true, "Value #B": false, "__name__": true, "app": true, "beta_kubernetes_io_arch": true, "beta_kubernetes_io_instance_type": true, "beta_kubernetes_io_os": true, "eks_amazonaws_com_capacityType": true, "eks_amazonaws_com_nodegroup": true, "eks_amazonaws_com_nodegroup_image": true, "eks_amazonaws_com_sourceLaunchTemplateId": true, "eks_amazonaws_com_sourceLaunchTemplateVersion": true, "endpoint": true, "failure_domain_beta_kubernetes_io_region": true, "failure_domain_beta_kubernetes_io_zone": true, "instance": true, "k8s_io_cloud_provider_aws": true, "karpenter_k8s_aws_ec2nodeclass": true, "karpenter_k8s_aws_instance_category": true, "karpenter_k8s_aws_instance_cpu": true, "karpenter_k8s_aws_instance_cpu_manufacturer": true, "karpenter_k8s_aws_instance_cpu_sustained_clock_speed_mhz": true, "karpenter_k8s_aws_instance_ebs_bandwidth": true, "karpenter_k8s_aws_instance_encryption_in_transit_supported": true, "karpenter_k8s_aws_instance_family": true, "karpenter_k8s_aws_instance_generation": true, "karpenter_k8s_aws_instance_hypervisor": true, "karpenter_k8s_aws_instance_memory": true, "karpenter_k8s_aws_instance_network_bandwidth": true, "karpenter_k8s_aws_instance_size": true, "karpenter_sh_capacity_type": true, "karpenter_sh_initialized": true, "karpenter_sh_nodepool": true, "karpenter_sh_registered": true, "kubernetes_io_arch": true, "kubernetes_io_hostname": true, "kubernetes_io_os": true, "name": true, "namespace": true, "nginxnode": true, "node_kubernetes_io_instance_type": true, "node_openshift_io_os_id": true, "nodetype": true, "partition": true, "pod_template_hash": true, "role": true, "topology_ebs_csi_aws_com_zone": true, "topology_k8s_aws_zone_id": true, "topology_kubernetes_io_region": true, "topology_kubernetes_io_zone": true }, "includeByName": {}, "indexByName": { "Time": 4, "Value #A": 23, "Value #B": 24, "__name__": 5, "beta_kubernetes_io_arch": 6, "beta_kubernetes_io_instance_type": 7, "beta_kubernetes_io_os": 8, "container": 3, "failure_domain_beta_kubernetes_io_region": 9, "failure_domain_beta_kubernetes_io_zone": 10, "instance": 11, "job": 0, "kubernetes_io_arch": 12, "kubernetes_io_hostname": 1, "kubernetes_io_os": 13, "name": 14, "nginxnode": 15, "node_kubernetes_io_instance_type": 16, "node_openshift_io_os_id": 17, "partition": 18, "pod": 2, "role": 19, "topology_ebs_csi_aws_com_zone": 20, "topology_kubernetes_io_region": 21, "topology_kubernetes_io_zone": 22 }, "renameByName": { "job": "", "kubernetes_io_hostname": "", "pod": "" } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "Value #B" } ] } } ], "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, "id": 49, "panels": [], "title": "Scrape Details", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 26 }, "id": 47, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum by (job,instance,pod) (scrape_samples_scraped)", "format": "time_series", "instant": false, "intervalFactor": 2, "legendFormat": "{{job}} {{instance}} {{pod}}", "refId": "A" } ], "title": "Scraped samples by job/instance/pod", "type": "timeseries" } ], "preload": false, "refresh": "", "schemaVersion": 42, "tags": [ "Prometheus" ], "templating": { "list": [ { "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "text": "2m", "value": "2m" }, "hide": 2, "name": "interval", "options": [ { "selected": true, "text": "2m", "value": "2m" } ], "query": "2m", "refresh": 2, "type": "interval" } ] }, "time": { "from": "now-5m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Prometheus Overview", "uid": "fb1438fb-d505-4a68-986c-8f8491df7f9f", "version": 1 }