diff --git a/example/performance-test/Makefile b/example/performance-test/Makefile index ebb5d25ce..511f0722c 100644 --- a/example/performance-test/Makefile +++ b/example/performance-test/Makefile @@ -9,6 +9,8 @@ JOBS ?= 10 # Number of parallel jobs per cluster LOGS ?= 1000 # Number of log lines per job LOGS_DELAY ?= 25s # Delay between log messages +SCENARIO ?= shoot # Test scenario: "shoot" or "seed" + LOGCLI_BIN := $(DIR)/bin/logcli # Default target @@ -55,7 +57,7 @@ setup: logcli check-kubeconfig # Create test clusters and start log generation run: check-kubeconfig setup @echo "Starting performance test with $(CLUSTERS) clusters, $(JOBS) jobs per cluster, $(LOGS) logs per job..." - @export CLUSTERS=$(CLUSTERS) JOBS=$(JOBS) LOGS=$(LOGS) LOGS_DELAY=$(LOGS_DELAY) && $(DIR)/up.sh + @export CLUSTERS=$(CLUSTERS) JOBS=$(JOBS) LOGS=$(LOGS) LOGS_DELAY=$(LOGS_DELAY) && $(DIR)/run.sh $(SCENARIO) # Check Current logs in Vali check: check-kubeconfig @@ -70,7 +72,7 @@ fetch: check-kubeconfig # Clean up all test resources down: check-kubeconfig @echo "Removing performance test workloads resources..." - @export CLUSTERS=$(CLUSTERS) && $(DIR)/down.sh + @export CLUSTERS=$(CLUSTERS) && $(DIR)/down.sh $(SCENARIO) # Alias for down clean: check-kubeconfig down diff --git a/example/performance-test/README.md b/example/performance-test/README.md index dc71e62b8..7391552f4 100644 --- a/example/performance-test/README.md +++ b/example/performance-test/README.md @@ -31,7 +31,7 @@ sum(count_over_time({container_name=\"shoot--logging--dev-${i}\"}[24h])) ``` The script polls aggregated logs count per namespace. (CLUSTERS) -Expected count count ≈ `JOBS * LOGS` (after all pods finish and ingestion catches up). +Expected count ≈ `JOBS * LOGS` (after all pods finish and ingestion catches up). --- @@ -75,7 +75,7 @@ Deploys test components with the helm chart in `charts/fluent-bit-plugin`. * Installs prometheus and plutono with fluent-bit dashboard for monitoring. * Installs logging backend Vali simulating a seed cluster and shared Vali for all simulated shoots. -### make up +### make run Creates the test run workloads. diff --git a/example/performance-test/charts/fluent-bit-plugin/dashboards/plutono-fluent-bit-dashboard.json b/example/performance-test/charts/fluent-bit-plugin/dashboards/plutono-fluent-bit-dashboard.json index 7b9b192a8..ed8ce5c5c 100644 --- a/example/performance-test/charts/fluent-bit-plugin/dashboards/plutono-fluent-bit-dashboard.json +++ b/example/performance-test/charts/fluent-bit-plugin/dashboards/plutono-fluent-bit-dashboard.json @@ -16,678 +16,1366 @@ "editable": true, "gnetId": 7752, "graphTooltip": 1, - "iteration": 1759842513594, + "iteration": 1763975703415, "links": [], "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 84, + "panels": [], + "title": "Fluent Bit", + "type": "row" + }, + { "datasource": "prometheus", - "description": "Input bytes processing rate of all fluent-bits.", + "description": "", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Records", + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 5, - "w": 12, + "w": 4, "x": 0, - "y": 0 + "y": 1 }, - "hiddenSeries": false, - "id": 2, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", + "id": 73, "options": { - "alertThreshold": true + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" }, - "percentage": false, "pluginVersion": "7.5.42", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(rate(fluentbit_input_bytes_total{pod=~\"$pod\"}[$__rate_interval])) by (pod, name)", - "format": "time_series", - "hide": false, + "exemplar": true, + "expr": "sum(fluentbit_input_records_total{pod=~\"$pod\"})", "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ pod }}/{{name}}", + "legendFormat": "", + "queryType": "randomWalk", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "[Fluentbit] Input Bytes Per Second", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:297", - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "title": "[Fluentbit] Input Plugin Total Records", + "type": "stat" + }, + { + "datasource": "prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Records", + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 74, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "7.5.42", + "targets": [ { - "$$hashKey": "object:298", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "exemplar": true, + "expr": "sum(fluentbit_output_proc_records_total{pod=~\"$pod\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "[Fluentbit] Output Plugin Total Records", + "type": "stat" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "prometheus", - "description": "Output bytes processing rate of all fluent-bits.", + "description": "Average Input bytes processing rate of all fluent-bits.", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "deckbytes" + }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 5, - "w": 12, - "x": 12, - "y": 0 + "w": 8, + "x": 8, + "y": 1 }, - "hiddenSeries": false, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": false, + "text": {} }, - "percentage": false, "pluginVersion": "7.5.42", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(rate(fluentbit_output_proc_bytes_total{pod=~\"$pod\"}[$__rate_interval])) by (pod, name)", + "exemplar": true, + "expr": "sum(rate(fluentbit_input_bytes_total{pod=~\"$pod\"}[$__rate_interval])) by (name) / 1024 ", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{ pod }}/{{name}}", + "legendFormat": "{{name}}", "refId": "A" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "[Fluentbit] Output Bytes Per Second", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:222", - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:223", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "[Fluentbit] Input Plugin Avg Bytes", + "type": "gauge" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "prometheus", - "description": "Input records processing rate of all fluent-bits.", + "description": "Output plugin bytes processing rate of all fluent-bits.", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "deckbytes" + }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 5, - "w": 12, - "x": 0, - "y": 5 + "w": 8, + "x": 16, + "y": 1 }, - "hiddenSeries": false, - "id": 40, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": false, + "text": {} }, - "percentage": false, "pluginVersion": "7.5.42", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(rate(fluentbit_input_records_total{pod=~\"$pod\"}[$__rate_interval])) by (pod, name)", + "exemplar": true, + "expr": "sum(rate(fluentbit_output_proc_bytes_total{pod=~\"$pod\"}[$__rate_interval])) by (name) /1024", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{ pod }}/{{name}}", + "legendFormat": "{{name}}", "refId": "A" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "[Fluentbit] Input Records Per Second", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1387", - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:1388", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "[Fluentbit] Output Plugin Avg Bytes", + "type": "gauge" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "prometheus", - "description": "Output records processing rate of all fluent-bits.", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "fixedColor": "green", + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "rps" + }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { - "h": 5, + "h": 7, "w": 12, - "x": 12, - "y": 5 + "x": 0, + "y": 6 }, - "hiddenSeries": false, - "id": 41, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", + "id": 69, "options": { - "alertThreshold": true + "graph": {}, + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } }, - "percentage": false, "pluginVersion": "7.5.42", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(rate(fluentbit_output_proc_records_total{pod=~\"$pod\"}[$__rate_interval])) by (pod, name)", - "format": "time_series", - "hide": false, + "exemplar": true, + "expr": "sum(rate(fluentbit_input_records_total{pod=~\"$pod\"}[$__rate_interval])) by (name)", "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ pod }}/{{name}}", + "legendFormat": "{{name}}", + "queryType": "randomWalk", "refId": "A" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "[Fluentbit] Output Records Per Second", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:324", - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true + "title": "[Fluentbit] Rate Input Plugin Records ", + "type": "timeseries" + }, + { + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "wps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 71, + "options": { + "graph": {}, + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right" }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.42", + "targets": [ { - "$$hashKey": "object:325", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "exemplar": true, + "expr": "sum(rate(fluentbit_output_proc_records_total{pod=~\"$pod\"}[$__rate_interval])) by (name)", + "interval": "", + "legendFormat": "{{name}}", + "queryType": "randomWalk", + "refId": "A" } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "[Fluentbit] Output Plugin Records", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "prometheus", - "description": "Output plugin total logs", + "description": "Input records processing rate of all fluent-bits.", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "rps" + }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 13 }, - "hiddenSeries": false, - "id": 56, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 40, + "links": [], "options": { - "alertThreshold": true + "graph": {}, + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } }, - "percentage": false, "pluginVersion": "7.5.42", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(rate(fluentbit_vali_gardener_incoming_logs_total{pod=~\"$pod\"}[$__rate_interval])) by (pod)", - "instant": false, + "expr": "sum(rate(fluentbit_input_records_total{pod=~\"$pod\"}[$__rate_interval])) by (pod, name)", + "format": "time_series", + "hide": false, "interval": "", - "legendFormat": "Incoming logs-{{pod}}", + "intervalFactor": 1, + "legendFormat": "{{ pod }}/{{name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "[Fluentbit] Input Records Per Second", + "type": "timeseries" + }, + { + "datasource": "prometheus", + "description": "Output records processing rate of all fluent-bits.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "wps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 41, + "links": [], + "options": { + "graph": {}, + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.42", + "targets": [ + { + "expr": "sum(rate(fluentbit_output_proc_records_total{pod=~\"$pod\"}[$__rate_interval])) by (pod, name)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ pod }}/{{name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "[Fluentbit] Output Records Per Second", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 86, + "panels": [], + "title": "Output Plugin", + "type": "row" + }, + { + "datasource": "prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Records", + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 22 + }, + "id": 75, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "7.5.42", + "targets": [ + { + "exemplar": true, + "expr": "sum(fluentbit_vali_gardener_incoming_logs_total{pod=~\"$pod\",host=~\"$host\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "[Output Plugin] Input Total Records", + "type": "stat" + }, + { + "datasource": "prometheus", + "description": "Total records send to output plugin clients", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Records", + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 22 + }, + "id": 76, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "7.5.42", + "targets": [ + { + "exemplar": true, + "expr": "sum(fluentbit_vali_gardener_forwarded_logs_total{pod=~\"$pod\",host=~\".+$host.+\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "[Output Plugin] Forwarder Total Records", + "type": "stat" + }, + { + "datasource": "prometheus", + "description": "Total errors by output plugin", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Records", + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "#EAB839", + "value": 10 + }, + { + "color": "orange", + "value": 20 + }, + { + "color": "red", + "value": 30 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 6, + "y": 22 + }, + "id": 78, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "7.5.42", + "targets": [ + { + "exemplar": true, + "expr": "sum(fluentbit_vali_gardener_errors_total{pod=~\"$pod\",host=~\"$host\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "[Output Plugin] Total Errors", + "type": "stat" + }, + { + "datasource": "prometheus", + "description": "Total records dropped by output plugin", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Records", + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 8, + "y": 22 + }, + "id": 77, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "7.5.42", + "targets": [ + { + "exemplar": true, + "expr": "sum(fluentbit_vali_gardener_dropped_logs_total{pod=~\"$pod\",host=~\"$host\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "[Output Plugin] Dropped Total Records", + "type": "stat" + }, + { + "datasource": "prometheus", + "description": "Total records without metadata", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Records", + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 10, + "y": 22 + }, + "id": 79, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "7.5.42", + "targets": [ + { + "exemplar": true, + "expr": "sum(fluentbit_vali_gardener_logs_without_metadata_total{pod=~\"$pod\",host=~\"$host\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "[Output Plugin] No Metadata Records", + "type": "stat" + }, + { + "datasource": "prometheus", + "description": "buffer sizes output plugin", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 95, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.42", + "targets": [ + { + "exemplar": true, + "expr": "fluentbit_vali_gardener_dque_size{pod=~\"$pod\",name=~\"$host\"}", + "instant": false, + "interval": "", + "legendFormat": "{{pod}}/{{name}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "[Output Plugin] DQue Current Size", + "type": "timeseries" + }, + { + "datasource": "prometheus", + "description": "Output plugin total logs", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 0, + "y": 27 + }, + "id": 56, + "options": { + "graph": {}, + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.42", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(fluentbit_vali_gardener_incoming_logs_total{pod=~\"$pod\",host=~\"$host\"}[$__rate_interval])) by (host)", + "instant": false, + "interval": "", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "[Output Plugin] Incoming Logs Rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": "prometheus", + "description": "Output plugin total logs", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 13, + "x": 11, + "y": 27 + }, + "id": 80, + "options": { + "graph": {}, + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.42", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(fluentbit_vali_gardener_forwarded_logs_total{pod=~\"$pod\",host=~\".+$host.+\"}[$__rate_interval])) by (host)", + "instant": false, + "interval": "", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "[Output Plugin] Forwarded Logs Rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": "prometheus", + "description": "Output plugin errors", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "errors" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 35 + }, + "id": 82, + "options": { + "graph": {}, + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.42", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(fluentbit_vali_gardener_errors_total{pod=~\"$pod\",host=~\"$host\"}[$__rate_interval])) by (host)", + "instant": false, + "interval": "", + "legendFormat": "{{host}}", "refId": "A" - }, - { - "expr": "sum(rate(fluentbit_vali_gardener_incoming_logs_with_endpoint_total{pod=~\"$pod\"}[$__rate_interval])) by (pod)", - "instant": false, - "interval": "", - "legendFormat": "Incoming logs with endpoint-{{pod}}", - "refId": "B" - }, - { - "expr": "sum(rate(fluentbit_vali_gardener_forwarded_logs_total{pod=~\"$pod\"}[$__rate_interval])) by (pod)", - "interval": "", - "legendFormat": "Transferred to Valitail-{{pod}}", - "refId": "C" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "[Output Plugin] Processed Logs", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, + "title": "[Output Plugin] Errors Logs Rate", "transformations": [], - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1440", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:1441", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "prometheus", - "description": "Summary for the all custom fail metrics in the output plugin", + "description": "Output plugin dropped logs", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "errors" + }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 10 - }, - "hiddenSeries": false, - "id": 59, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "h": 8, + "w": 7, + "x": 8, + "y": 35 + }, + "id": 89, "options": { - "alertThreshold": true + "graph": {}, + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } }, - "percentage": false, "pluginVersion": "7.5.42", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(rate(fluentbit_vali_gardener_dropped_logs_total{pod=~\"$pod\"}[$__rate_interval])) by (pod)", - "interval": "", - "legendFormat": "Dropped logs-{{pod}}", - "refId": "C" - }, - { - "expr": "sum(rate(prometheus_target_scrapes_sample_out_of_order_total{pod=~\"$pod\"}[$__rate_interval])) by (pod)", - "interval": "", - "legendFormat": "OutOfOrder logs-{{pod}}", - "refId": "D" - }, - { - "expr": "sum(rate(fluentbit_vali_gardener_logs_without_metadata_total{pod=~\"$pod\"}[$__rate_interval])) by (pod)", - "interval": "", - "legendFormat": "Missing metadata-{{pod}}", - "refId": "E" - }, - { - "expr": "sum(rate(fluentbit_vali_gardener_errors_total{pod=~\"$pod\"}[$__rate_interval])) by (pod)", - "interval": "", - "legendFormat": "Errors-{{pod}}", - "refId": "F" - }, - { - "expr": "sum(rate(valitail_dropped_entries_total{pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "exemplar": true, + "expr": "sum(rate(fluentbit_vali_gardener_dropped_logs_total{pod=~\"$pod\",host=~\"$host\"}[$__rate_interval])) by (host)", + "instant": false, "interval": "", - "legendFormat": "Valitail dropped-{{pod}}", - "refId": "G" + "legendFormat": "{{host}}", + "refId": "A" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "[Output Plugin] Fails Summary", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:421", - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:422", - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "[Output Plugin] Dropped Logs Rate", + "transformations": [], + "type": "timeseries" }, { "datasource": "prometheus", - "description": "Table which shows the total incoming logs from the Output plugin", + "description": "", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { - "align": "center", - "filterable": true + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true }, "mappings": [], "thresholds": { @@ -696,89 +1384,71 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "errors" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 16 + "h": 8, + "w": 9, + "x": 15, + "y": 35 }, - "hideTimeOverride": true, - "id": 62, - "interval": "", + "id": 90, "options": { - "frameIndex": 0, - "showHeader": true, - "sortBy": [] + "graph": {}, + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } }, "pluginVersion": "7.5.42", "targets": [ { "exemplar": true, - "expr": "sum(increase(fluentbit_vali_gardener_incoming_logs_total{pod=~\"$pod\"}[$__rate_interval]) != 0) by (host)", + "expr": "sum(rate(fluentbit_vali_gardener_logs_without_metadata_total{pod=~\"$pod\",host=~\".*$host.*\"}[$__rate_interval])) by (host)", + "instant": false, "interval": "", "legendFormat": "{{host}}", - "queryType": "randomWalk", "refId": "A" } ], "timeFrom": null, "timeShift": null, - "title": "[Output Plugin] Incoming Logs", - "transformations": [ - { - "id": "reduce", - "options": { - "reducers": [ - "delta" - ] - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "Delta": "Total number of logs", - "Field": "Namespace", - "Last": "Dropped logs from last 1h", - "Mean": "Average number of incoming logs per hour", - "Total": "" - } - } - }, - { - "id": "groupBy", - "options": { - "fields": { - "Namespace": { - "aggregations": [], - "operation": "aggregate" - } - } - } - } - ], - "type": "table" + "title": "[Output Plugin] Logs without metadata Rate", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 92, + "panels": [], + "title": "ValiTail", + "type": "row" }, { "datasource": "prometheus", - "description": "Table which shows the total incoming logs with attached endpoint and are ready to be forwarded to the Valitail client", "fieldConfig": { "defaults": { - "custom": { - "align": "center", - "filterable": true + "color": { + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -787,89 +1457,76 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 16 + "h": 8, + "w": 5, + "x": 0, + "y": 44 }, - "hideTimeOverride": true, - "id": 63, - "interval": "", + "id": 94, "options": { - "frameIndex": 0, - "showHeader": true, - "sortBy": [] + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" }, "pluginVersion": "7.5.42", "targets": [ { "exemplar": true, - "expr": "sum(increase(fluentbit_vali_gardener_incoming_logs_with_endpoint_total{pod=~\"$pod\"}[$__rate_interval]) != 0) by (host)", + "expr": "sum(valitail_sent_entries_total{pod=~\"$pod\",host=~\"$url\"})", "interval": "", "legendFormat": "{{host}}", "queryType": "randomWalk", "refId": "A" } ], - "timeFrom": null, - "timeShift": null, - "title": "[Output Plugin] Incoming Logs With Endpoint", - "transformations": [ - { - "id": "reduce", - "options": { - "reducers": [ - "delta" - ] - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "Delta": "Total number of logs", - "Field": "Namespace", - "Last": "Dropped logs from last 1h", - "Mean": "Average number of incoming logs per hour", - "Total": "" - } - } - }, - { - "id": "groupBy", - "options": { - "fields": { - "Namespace": { - "aggregations": [], - "operation": "aggregate" - } - } - } - } - ], - "type": "table" + "title": "ValiTail Sent Logs Total", + "type": "stat" }, { "datasource": "prometheus", - "description": "Table which shows the transferred logs from Output plugin to the Valitail per hour.", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { - "align": "center", - "filterable": true + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true }, "mappings": [], "thresholds": { @@ -884,68 +1541,73 @@ "value": 80 } ] - } + }, + "unit": "rps" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 16 + "h": 8, + "w": 19, + "x": 5, + "y": 44 }, - "id": 49, + "id": 97, "options": { - "showHeader": true, - "sortBy": [] + "graph": {}, + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } }, "pluginVersion": "7.5.42", "targets": [ { "exemplar": true, - "expr": "sum(increase(fluentbit_vali_gardener_forwarded_logs_total{pod=~\"$pod\"}[$__rate_interval]) != 0) by (host)", + "expr": "sum(rate(valitail_sent_entries_total{pod=~\"$pod\",host=~\".+$host.+\"}[$__rate_interval])) by (host)", "interval": "", "legendFormat": "{{host}}", "queryType": "randomWalk", "refId": "A" } ], - "timeFrom": null, - "timeShift": null, - "title": "[Output Plugin] Transferred Logs to Valitail", - "transformations": [ - { - "id": "reduce", - "options": { - "reducers": [ - "delta" - ] - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "Delta": "Total number of logs", - "Field": "Namespace", - "Last": "Logs from last 1h", - "Mean": "Average number of logs per hour" - } - } - } - ], - "type": "table" + "title": "ValiTail Client Send Logs", + "type": "timeseries" }, { "datasource": "prometheus", - "description": "Table which shows the total dropped entries by valitail per hour", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { - "align": "center", - "filterable": true + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true }, "mappings": [], "thresholds": { @@ -960,84 +1622,73 @@ "value": 80 } ] - } + }, + "unit": "decbytes" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "custom.width", - "value": null - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 52 }, - "hideTimeOverride": true, - "id": 58, + "id": 103, "options": { - "frameIndex": 0, - "showHeader": true, - "sortBy": [] + "graph": {}, + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } }, "pluginVersion": "7.5.42", "targets": [ { "exemplar": true, - "expr": "sum(increase(valitail_dropped_entries_total{pod=~\"$pod\"}[$__rate_interval]) != 0) by (host)", + "expr": "sum(rate(valitail_sent_bytes_total{pod=~\"$pod\",host=~\".+$host.+\"}[$__rate_interval])) by (host)", "interval": "", "legendFormat": "{{host}}", "queryType": "randomWalk", "refId": "A" } ], - "timeFrom": null, - "timeShift": null, - "title": "[Valitail] Dropped Logs", - "transformations": [ - { - "id": "reduce", - "options": { - "reducers": [ - "delta" - ] - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "Delta": "Total number of logs", - "Field": "Source", - "Last": "Dropped logs from last 1h", - "Mean": "Average number of dropped logs per hour", - "Total": "" - } - } - } - ], - "type": "table" + "title": "ValiTail Send Bytes Rate", + "type": "timeseries" }, { "datasource": "prometheus", - "description": "Table which shows dropped logs for all namespaces.", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { - "align": "center", - "filterable": true + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true }, "mappings": [], "thresholds": { @@ -1052,68 +1703,73 @@ "value": 80 } ] - } + }, + "unit": "decbytes" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 52 }, - "hideTimeOverride": true, - "id": 53, + "id": 104, "options": { - "frameIndex": 0, - "showHeader": true, - "sortBy": [] + "graph": {}, + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } }, "pluginVersion": "7.5.42", "targets": [ { "exemplar": true, - "expr": "sum(increase(fluentbit_vali_gardener_dropped_logs_total{pod=~\"$pod\"}[$__rate_interval]) != 0) by (host)", + "expr": "sum(rate(valitail_dropped_entries_total{pod=~\"$pod\",host=~\"$url\"}[$__rate_interval])) by (host)", "interval": "", "legendFormat": "{{host}}", "queryType": "randomWalk", "refId": "A" } ], - "timeFrom": null, - "timeShift": null, - "title": "[Output Plugin] Dropped Logs", - "transformations": [ - { - "id": "reduce", - "options": { - "reducers": [ - "delta" - ] - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "Delta": "Total number of logs", - "Field": "Namespace" - } - } - } - ], - "type": "table" + "title": "ValiTail Dropped Entries", + "type": "timeseries" }, { "datasource": "prometheus", - "description": "Table which shows logs without metadata for all namespaces.", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { - "align": "center", - "filterable": true + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true }, "mappings": [], "thresholds": { @@ -1128,56 +1784,90 @@ "value": 80 } ] - } + }, + "unit": "ms" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 60 }, - "hideTimeOverride": true, - "id": 52, + "id": 99, "options": { - "frameIndex": 0, - "showHeader": true + "graph": {}, + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } }, "pluginVersion": "7.5.42", "targets": [ { "exemplar": true, - "expr": "sum(increase(fluentbit_vali_gardener_logs_without_metadata_total{pod=~\"$pod\"}[$__rate_interval]) != 0) by (host)", + "expr": "histogram_quantile(0.50, sum(rate(valitail_request_duration_seconds_bucket{host=~\".+$host.+\",status_code=~\".+\"}[5m])) by (le,status_code,host)) * 1000 ", + "hide": false, "interval": "", - "legendFormat": "{{host}}", + "legendFormat": "p50 / status code {{status_code}}", + "queryType": "randomWalk", + "refId": "C" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(valitail_request_duration_seconds_bucket{host=~\".+$host.+\",status_code=~\".+\"}[5m])) by (le,status_code,host)) * 1000 ", + "interval": "", + "legendFormat": "p95 / status code {{status_code}}", "queryType": "randomWalk", "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "[Output Plugin] Missing Metadata Logs", - "transformations": [ + }, { - "id": "reduce", - "options": { - "reducers": [ - "delta" - ] - } + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(valitail_request_duration_seconds_bucket{host=~\".+$host.+\",status_code=~\".+\"}[5m])) by (le,status_code,host))* 1000", + "hide": false, + "interval": "", + "legendFormat": "p99 / status code {{status_code}}", + "queryType": "randomWalk", + "refId": "B" } ], - "type": "table" + "title": "ValiTail Request Percentiles", + "type": "timeseries" }, { "datasource": "prometheus", - "description": "Table which shows errors in the output plugin", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { - "align": "center", - "filterable": true + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true }, "mappings": [], "thresholds": { @@ -1192,47 +1882,44 @@ "value": 80 } ] - } + }, + "unit": "ms" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 60 }, - "hideTimeOverride": true, - "id": 54, + "id": 101, "options": { - "frameIndex": 0, - "showHeader": true + "graph": {}, + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltipOptions": { + "mode": "single" + } }, "pluginVersion": "7.5.42", "targets": [ { "exemplar": true, - "expr": "sum(increase(fluentbit_vali_gardener_errors_total{pod=~\"$pod\"}[$__rate_interval]) != 0) by (host)", + "expr": "sum(rate(valitail_request_duration_seconds_sum{pod=~\"$pod\",host=~\".+$host.+\"}[5m])) by (host)\n/\nsum(rate(valitail_request_duration_seconds_count{pod=~\"$pod\",host=~\".+$host.+\"}[5m])) by (host)\n* 1000", "interval": "", "legendFormat": "{{host}}", "queryType": "randomWalk", "refId": "A" } ], - "timeFrom": null, - "timeShift": null, - "title": "[Output Plugin] Errors", - "transformations": [ - { - "id": "reduce", - "options": { - "reducers": [ - "delta" - ] - } - } - ], - "type": "table" + "title": "ValiTail Avg Latency", + "type": "timeseries" } ], "refresh": "5s", @@ -1244,8 +1931,7 @@ { "allValue": "fluent-bit.+", "current": { - "selected": true, - "tags": [], + "selected": false, "text": [ "All" ], @@ -1276,11 +1962,77 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "prometheus", + "definition": "label_values(fluentbit_vali_gardener_incoming_logs_total,host)", + "description": "Output plugin host target", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "host", + "options": [], + "query": { + "query": "label_values(fluentbit_vali_gardener_incoming_logs_total,host)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "label_values(valitail_sent_entries_total, host)", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": "Url", + "multi": true, + "name": "url", + "options": [], + "query": { + "query": "label_values(valitail_sent_entries_total, host)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false } ] }, "time": { - "from": "now-5m", + "from": "now-30m", "to": "now" }, "timepicker": { diff --git a/example/performance-test/down.sh b/example/performance-test/down.sh index ee84cc86f..67fd5c6b8 100755 --- a/example/performance-test/down.sh +++ b/example/performance-test/down.sh @@ -5,42 +5,61 @@ dir=$(dirname "$0") function delete_jobs { - local shoot_namespace="shoot--logging--dev-${1}" - kubectl delete jobs --all -n "$shoot_namespace" --ignore-not-found + local namespace="${1}" + kubectl delete jobs --all -n "$namespace" --ignore-not-found } function delete_services { - local shoot_namespace="shoot--logging--dev-${1}" - kubectl delete service --all -n "$shoot_namespace" --ignore-not-found + local namespace="${1}" + kubectl delete service --all -n "$namespace" --ignore-not-found } function delete_clusters { - local cluster_name="shoot--logging--dev-${1}" + local cluster_name="${1}" kubectl delete cluster "$cluster_name" --ignore-not-found } function delete_namespaces { - local shoot_namespace="shoot--logging--dev-${1}" - kubectl delete namespace "$shoot_namespace" --ignore-not-found + local namespace="${1}" + kubectl delete namespace "$namespace" --ignore-not-found } +function shoot { + for ((i=1; i<=CLUSTERS; i++)); do + delete_jobs "shoot--logging--dev-${i}" & + done + wait -for ((i=1; i<=CLUSTERS; i++)); do - delete_jobs "$i" & -done -wait + for ((i=1; i<=CLUSTERS; i++)); do + delete_services "shoot--logging--dev-${i}" & + done + wait -for ((i=1; i<=CLUSTERS; i++)); do - delete_services "$i" & -done -wait + for ((i=1; i<=CLUSTERS; i++)); do + delete_clusters "shoot--logging--dev-${i}" & + done + wait -for ((i=1; i<=CLUSTERS; i++)); do - delete_clusters "$i" & -done -wait + for ((i=1; i<=CLUSTERS; i++)); do + delete_namespaces "shoot--logging--dev-${i}" & + done + wait -for ((i=1; i<=CLUSTERS; i++)); do - delete_namespaces "$i" & -done -wait + echo "Deleted $CLUSTERS clusters" +} + +function seed { + local namespace="seed--logging--dev" + delete_jobs "$namespace" + delete_namespaces "$namespace" +} -echo "Deleted $CLUSTERS clusters" +scenario="${1:-shoot}" +if [ "$scenario" == "shoot" ]; then + shoot +elif [ "$scenario" == "seed" ]; then + seed +else + echo "Unknown scenario: $scenario" + exit 1 + echo "Unknown scenario: $scenario" + exit 1 +fi diff --git a/example/performance-test/up.sh b/example/performance-test/run.sh similarity index 58% rename from example/performance-test/up.sh rename to example/performance-test/run.sh index 57b58d27c..93e750fff 100755 --- a/example/performance-test/up.sh +++ b/example/performance-test/run.sh @@ -8,9 +8,9 @@ nameOverride="logging" function create_namespaces { - local shoot_namespace="shoot--logging--dev-${1}" + local namespace="${1}" - kubectl create namespace "$shoot_namespace" \ + kubectl create namespace "$namespace" \ --dry-run=client -o yaml | \ kubectl apply -f - } @@ -41,14 +41,14 @@ function create_clusters { } function create_jobs { - local shoot_namespace="shoot--logging--dev-${1}" - + local namespace="${1}" + kubectl delete job logger --namespace "$namespace" --ignore-not-found >/dev/null 2>&1 cat <