{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "title": "Pods Metrics", "description": "CPU/Memory/Network/ usage for pods and containers", "editable": true, "time": { "from": "now-30m", "to": "now" }, "tags": [ "Pod" ], "panels": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "fieldConfig": { "defaults": { "unit": "short" } }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 12 }, "id": 41, "options": { "cellHeight": "sm", "footer": { "show": false }, "showHeader": true }, "pluginVersion": "12.3.1", "targets": [ { "editorMode": "code", "expr": "max by(container)(kube_pod_container_status_restarts_total{container=~\"^($container)$\"})", "format": "table", "instant": true, "legendFormat": "Restarts", "refId": "A" }, { "editorMode": "code", "expr": "max by(container)(kube_pod_container_status_ready{container=~\"^($container)$\"})", "format": "table", "instant": true, "legendFormat": "Ready(1=true)", "refId": "B" }, { "editorMode": "code", "expr": "max by(container)(kube_pod_container_status_running{container=~\"^($container)$\"})", "format": "table", "instant": true, "legendFormat": "Running(1=true)", "refId": "C" }, { "editorMode": "code", "expr": "max by(container)(kube_pod_container_status_waiting{container=~\"^($container)$\"})", "format": "table", "instant": true, "legendFormat": "Waiting(1=true)", "refId": "D" }, { "editorMode": "code", "expr": "max by(container)(kube_pod_container_status_terminated{container=~\"^($container)$\"})", "format": "table", "instant": true, "legendFormat": "Terminated(1=true)", "refId": "E" }, { "editorMode": "code", "expr": "max by(container,reason)(kube_pod_container_status_waiting_reason{container=~\"^($container)$\"})", "format": "table", "instant": true, "legendFormat": "Waiting reason={{reason}}", "refId": "F" }, { "editorMode": "code", "expr": "max by(container,reason)(kube_pod_container_status_terminated_reason{container=~\"^($container)$\"})", "format": "table", "instant": true, "legendFormat": "Terminated reason={{reason}}", "refId": "G" } ], "title": "Pod Container Status", "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "Value #A": "Restarts", "Value #B": "Ready", "Value #C": "Running", "Value #D": "Waiting", "Value #E": "Terminated", "Value #F": "Waiting reason", "Value #G": "Terminated reason" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "description": "Network activity at Pod level (eth0 network card)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "Bps" } }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 0 }, "id": 40, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "editorMode": "code", "exemplar": true, "expr": "sum by (interface) (rate (container_network_receive_bytes_total{namespace=~\"^($namespace)$\", pod=~\"^($pod)$\"}[$interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "received {{interface}}", "metric": "network", "range": true, "refId": "A", "step": 900 }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "editorMode": "code", "exemplar": true, "expr": "- sum by (interface) (rate (container_network_transmit_bytes_total{ namespace=~\"^($namespace)$\", pod=~\"^($pod)$\"}[$interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "sent {{interface}}", "metric": "network", "range": true, "refId": "B", "step": 900 } ], "title": "Network I/O ($interval avg) | $node_name | $node_arch | $node_type", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 39, "panels": [], "repeat": "container", "title": "$container", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "description": "CPU usage at container level \n\nthrottled percentage := container_cpu_cfs_throttled_periods_total / container_cpu_cfs_periods_total on 5m period \n, i.e. the percentage of CPU periods where the container ran but was throttled (stopped from running the whole CPU period).\n\nVPR New Req is the Vertical Pod Recommender recommendation for the CPU Request (based on a 7 days measurement)\n\nDrill down to \"Pods Java\" Dashboard", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "virtual cores", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "stepAfter", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "Throttling %" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FF780A", "mode": "fixed" } }, { "id": "unit", "value": "percentunit" }, { "id": "max", "value": 1 }, { "id": "min", "value": 0 } ] } ] }, "gridPos": { "h": 6, "w": 11, "x": 0, "y": 6 }, "id": 35, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "max(kube_pod_container_resource_limits_cpu_cores{pod=~\"^$pod$\",namespace=~\"^$namespace$\",container=~\"^($container)$\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Limit", "metric": "", "refId": "B", "step": 1800 }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "editorMode": "code", "expr": "sum (rate (container_cpu_usage_seconds_total{pod=~\"^$pod$\",namespace=~\"^$namespace$\",container=~\"^($container)$\"}[$interval]))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Usage", "metric": "container_cpu", "range": true, "refId": "A", "step": 1800 }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "editorMode": "code", "expr": "max(kube_pod_container_resource_requests_cpu_cores{pod=~\"^$pod$\",namespace=~\"^$namespace$\",container=~\"^($container)$\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Request", "metric": "", "range": true, "refId": "C", "step": 1800 }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "editorMode": "code", "expr": "sum(rate(container_cpu_cfs_throttled_periods_total{pod=~\"^$pod$\",namespace=~\"^$namespace$\",container=~\"^($container)$\"}[5m]) / rate(container_cpu_cfs_periods_total{pod=~\"^$pod$\",namespace=~\"^$namespace$\",container=~\"^($container)$\"}[5m]) )", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Throttling %", "metric": "", "range": true, "refId": "D", "step": 1800 }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "editorMode": "code", "expr": "max(vpr_recommendation_requests_cpu_cores{pod=~\"$podGroup.*\",container=\"$container\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "VPR New Req", "metric": "", "range": true, "refId": "E", "step": 1800 } ], "title": "CPU", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "description": "Mem usage at container level (only meaningful for non-Java based containers).if close to 100% of limit OOMKiller is triggered by OCP.The working set memory usage is without the FS cache (that can be evicted under memory pressure).\n\nVPR New Req/Limit is the Vertical Pod Recommender recommendation for the Mem Request/Limit (based on a 7 days measurement)\n\nDrill down to \"Pods Java\" Dashboard", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "stepAfter", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "bytes" } }, "gridPos": { "h": 6, "w": 13, "x": 11, "y": 6 }, "id": 36, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "max(kube_pod_container_resource_limits_memory_bytes{pod=~\"^$pod$\",namespace=~\"^$namespace$\",container=~\"^($container)$\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Limit", "metric": "", "refId": "D", "step": 1800 }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "max(container_memory_working_set_bytes{pod=~\"^$pod$\",namespace=~\"^$namespace$\",container=~\"^($container)$\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Usage", "metric": "", "refId": "A", "step": 1800 }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "max(container_memory_usage_bytes{pod=~\"^$pod$\",namespace=~\"^$namespace$\",container=~\"^($container)$\"})", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 1, "legendFormat": "Usage with Cache", "metric": "", "refId": "C", "step": 1800 }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "max(kube_pod_container_resource_requests_memory_bytes{pod=~\"^$pod$\",namespace=~\"^$namespace$\",container=~\"^($container)$\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Request", "metric": "", "refId": "B", "step": 1800 }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "editorMode": "code", "expr": "max(vpr_recommendation_requests_memory_bytes{pod=~\"$podGroup.*\",container=\"$container\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "VPR New Req", "metric": "", "range": true, "refId": "E", "step": 1800 }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "editorMode": "code", "expr": "max(vpr_recommendation_limits_memory_bytes{pod=~\"$podGroup.*\",container=\"$container\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "VPR New Limit", "metric": "", "range": true, "refId": "F", "step": 1800 } ], "title": "Memory", "type": "timeseries" } ], "templating": { "list": [ { "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "text": "2m", "value": "2m" }, "hide": 2, "name": "interval", "options": [ { "selected": true, "text": "2m", "value": "2m" } ], "query": "2m", "refresh": 2, "type": "interval" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "label_values(container_memory_swap{job=~\"kubernetes-.*\", image!=\"\"}, namespace)", "includeAll": true, "name": "namespace", "query": { "query": "label_values(container_memory_swap{job=~\"kubernetes-.*\", image!=\"\"}, namespace)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "/.*/", "sort": 1, "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "label_values(container_memory_swap{job=~\"kubernetes-.*\", image!=\"\", namespace=~\"^($namespace)$\"}, instance)", "includeAll": true, "name": "node", "query": { "query": "label_values(container_memory_swap{job=~\"kubernetes-.*\", image!=\"\", namespace=~\"^($namespace)$\"}, instance)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "sort": 1, "type": "query" }, { "current": { "text": "f5-afm-7df7566c68-899cf", "value": "f5-afm-7df7566c68-899cf" }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "label_values(container_memory_swap{job=~\"kubernetes-.*\", image!=\"\", namespace=~\"^($namespace)$\",instance=~\"$node\"}, pod)", "includeAll": false, "name": "pod", "query": { "query": "label_values(container_memory_swap{job=~\"kubernetes-.*\", image!=\"\", namespace=~\"^($namespace)$\",instance=~\"$node\"}, pod)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "sort": 1, "type": "query" }, { "current": { "text": "f5-afm", "value": "f5-afm" }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "label_values(container_memory_swap{pod=~\"$pod\"},pod)", "description": "", "hide": 2, "includeAll": false, "name": "podGroup", "query": { "qryType": 1, "query": "label_values(container_memory_swap{pod=~\"$pod\"},pod)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "(.*)-(\\w{8,10}-\\w{5}|\\d{1,3})", "sort": 1, "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "label_values(container_memory_swap{job=~\"kubernetes-.*\", image!=\"\", namespace=~\"^($namespace)$\", pod=~\"^($pod)$\",container!=\"POD\"}, container)", "includeAll": true, "name": "container", "query": { "query": "label_values(container_memory_swap{job=~\"kubernetes-.*\", image!=\"\", namespace=~\"^($namespace)$\", pod=~\"^($pod)$\",container!=\"POD\"}, container)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "sort": 1, "type": "query" }, { "current": { "text": "datkube-worker2", "value": "datkube-worker2" }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "label_values(container_memory_swap{job=~\"kubernetes-.*\", image!=\"\", namespace=~\"^($namespace)$\", pod=~\"^($pod)$\"}, instance)", "hide": 2, "includeAll": false, "name": "node_name", "query": { "query": "label_values(container_memory_swap{job=~\"kubernetes-.*\", image!=\"\", namespace=~\"^($namespace)$\", pod=~\"^($pod)$\"}, instance)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "sort": 1, "type": "query" }, { "allowCustomValue": false, "current": { "text": "", "value": "" }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "label_values(node_boot_time_seconds{instance=~\"$node_name\"},nodetype)", "description": "", "hide": 2, "name": "node_type", "query": { "qryType": 1, "query": "label_values(node_boot_time_seconds{instance=~\"$node_name\"},nodetype)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "allowCustomValue": false, "current": { "text": "", "value": "" }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "label_values(node_boot_time_seconds{instance=~\"$node_name\"},kubernetes_io_arch)", "hide": 2, "name": "node_arch", "query": { "qryType": 1, "query": "label_values(node_boot_time_seconds{instance=~\"$node_name\"},kubernetes_io_arch)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" } ] } }