diff --git a/blackbox/blackbox.yml b/blackbox/blackbox.yml new file mode 100644 index 0000000..2af3dcf --- /dev/null +++ b/blackbox/blackbox.yml @@ -0,0 +1,6 @@ +modules: + icmp_ttl5: + prober: icmp + timeout: 5s + icmp: + ttl: 10 diff --git a/docker-compose-armored.yml b/docker-compose-armored.yml index 42905f2..aa5444a 100644 --- a/docker-compose-armored.yml +++ b/docker-compose-armored.yml @@ -78,15 +78,14 @@ services: - default restart: unless-stopped - # https://github.com/SuperQ/smokeping_prober - smokeping: - image: quay.io/superq/smokeping-prober:latest - container_name: mikrotik-smokeping-prober - security_opt: - - apparmor:docker-mikrotik-monitoring - command: --config.file=/etc/smokeping/smokeping.yml + blackbox: + image: quay.io/prometheus/blackbox-exporter:latest + container_name: mikrotik-blackbox-exporter + command: --config.file=/config/blackbox.yml volumes: - - ./smokeping/smokeping.yml:/etc/smokeping/smokeping.yml + - ./blackbox/blackbox.yml:/config/blackbox.yml networks: - default restart: unless-stopped + security_opt: + - apparmor:docker-mikrotik-monitoring diff --git a/docker-compose.yml b/docker-compose.yml index cc8c025..6f391f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,7 @@ services: command: - '--config.file=/etc/prometheus/prometheus.yml' - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=1y' # <= adjust the storage retention period of Prometheus restart: unless-stopped networks: - default @@ -71,13 +72,13 @@ services: - default restart: unless-stopped - # https://github.com/SuperQ/smokeping_prober - smokeping: - image: quay.io/superq/smokeping-prober:latest - container_name: mikrotik-smokeping-prober - command: --config.file=/etc/smokeping/smokeping.yml + # https://github.com/prometheus/blackbox_exporter/blob/master/example.yml + blackbox: + image: quay.io/prometheus/blackbox-exporter:latest + container_name: mikrotik-blackbox-exporter + command: --config.file=/config/blackbox.yml volumes: - - ./smokeping/smokeping.yml:/etc/smokeping/smokeping.yml + - ./blackbox/blackbox.yml:/config/blackbox.yml networks: - default restart: unless-stopped diff --git a/grafana/provisioning/dashboards/mikrotik/monitor.json b/grafana/provisioning/dashboards/mikrotik/monitor.json index 56f1313..c81ac57 100644 --- a/grafana/provisioning/dashboards/mikrotik/monitor.json +++ b/grafana/provisioning/dashboards/mikrotik/monitor.json @@ -1353,7 +1353,7 @@ "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -1361,290 +1361,300 @@ "y": 24 }, "id": 131, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 129, - "options": { - "calculate": false, - "cellGap": 1, - "cellValues": { - "unit": "none" - }, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Oranges", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": false - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "show": false, - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(increase(smokeping_response_duration_seconds_bucket{host=\"8.8.8.8\"}[$__rate_interval])) by (le)", - "format": "heatmap", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency - ${latency_target}", - "type": "heatmap" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "fixed", - "seriesBy": "min" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 2 - }, - "id": 133, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "(smokeping_requests_total{host=\"${latency_target}\"} - smokeping_response_duration_seconds_count{host=\"${latency_target}\"})/smokeping_requests_total{host=\"${latency_target}\"}", - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Packet loss (%) - ${latency_target}", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed", - "seriesBy": "max" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 135, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(smokeping_response_duration_seconds_sum{host=\"${latency_target}\"}[$__rate_interval]) / rate(smokeping_response_duration_seconds_count{host=\"${latency_target}\"}[$__rate_interval])", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency - ${latency_target}", - "type": "timeseries" - } - ], + "panels": [], "title": "Latency", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "last" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 129, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(probe_icmp_duration_seconds{phase=\"rtt\"}[$__rate_interval])) by (instance)\n", + "format": "time_series", + "legendFormat": "{{ instance }}", + "range": true, + "refId": "A" + } + ], + "title": "Latency (RTT)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 135, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(probe_icmp_duration_seconds{phase=\"resolve\"}[$__rate_interval])) by (instance)", + "instant": false, + "legendFormat": "{{host}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency (DNS resolution)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 133, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 * avg_over_time(probe_success[$__rate_interval]) ", + "hide": false, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Availability", + "type": "timeseries" + }, { "collapsed": true, "datasource": { @@ -1654,7 +1664,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 39 }, "id": 2, "panels": [ @@ -2037,7 +2047,7 @@ "h": 1, "w": 24, "x": 0, - "y": 26 + "y": 40 }, "id": 45, "panels": [ @@ -2780,7 +2790,7 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 41 }, "id": 84, "panels": [ @@ -3729,7 +3739,7 @@ "h": 1, "w": 24, "x": 0, - "y": 28 + "y": 42 }, "id": 90, "panels": [ @@ -3866,8 +3876,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4016,7 +4025,7 @@ "h": 1, "w": 24, "x": 0, - "y": 29 + "y": 43 }, "id": 29, "panels": [ @@ -4995,7 +5004,7 @@ "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 44 }, "id": 33, "panels": [ @@ -5902,7 +5911,7 @@ "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 45 }, "id": 125, "panels": [ @@ -6057,7 +6066,7 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 46 }, "id": 80, "panels": [ @@ -6644,35 +6653,11 @@ "skipUrlSync": false, "sort": 1, "type": "query" - }, - { - "current": { - "selected": true, - "text": "9.9.9.9", - "value": "9.9.9.9" - }, - "definition": "label_values(smokeping_requests_total, host)", - "description": "Smokepings target to measure the latency", - "hide": 0, - "includeAll": false, - "label": "Latency Target", - "multi": false, - "name": "latency_target", - "options": [], - "query": { - "query": "label_values(smokeping_requests_total, host)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" } ] }, "time": { - "from": "now-30m", + "from": "now-1h", "to": "now" }, "timepicker": {}, diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 4749fcb..5e0770d 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -19,7 +19,7 @@ rule_files: # Here it's Prometheus itself. scrape_configs: # The job name is added as a label `job=` to any timeseries scraped from this config. - - job_name: 'prometheus' + - job_name: 'mikrotik' # Override the global default and scrape targets from this job every 5 seconds. scrape_interval: 15s @@ -32,3 +32,21 @@ scrape_configs: - 'mikrotik_exporter:9436' - 'mktxp:49090' - 'smokeping:9374' + + - job_name: 'blackbox' + scrape_interval: 1s + metrics_path: /probe + params: + module: [icmp_ttl5] # use ICMP as the ping protocol + static_configs: + - targets: + - 1.1.1.1 + - 8.8.8.8 + - 9.9.9.9 + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox:9115 # The blackbox exporter's real hostname:port. diff --git a/smokeping/smokeping.yml b/smokeping/smokeping.yml deleted file mode 100644 index e36c4f4..0000000 --- a/smokeping/smokeping.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -targets: -- hosts: - - 1.1.1.1 - - 8.8.8.8 - - 9.9.9.9 - interval: 10s # Duration, Default 1s. - network: ip4 # One of ip, ip4, ip6. Default: ip (automatic IPv4/IPv6) - protocol: icmp # One of icmp, udp. Default: icmp (Requires privileged operation) - size: 56 # Packet data size in bytes. Default 56 (Range: 24 - 65535)