From 304a40c0a543639da6c8cbb4660dc9ee415cd50a Mon Sep 17 00:00:00 2001 From: Jet Date: Tue, 31 Mar 2026 18:21:00 -0700 Subject: [PATCH 1/2] feat: change out uptime kuma for blackbox --- README.md | 4 +- configuration.nix | 2 +- modules/blackbox-exporter.nix | 228 ++++++++++++++++++++++++++++++++++ modules/caddy.nix | 2 +- modules/uptime-kuma.nix | 11 -- 5 files changed, 232 insertions(+), 15 deletions(-) create mode 100644 modules/blackbox-exporter.nix delete mode 100644 modules/uptime-kuma.nix diff --git a/README.md b/README.md index 83684fb..979286d 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ services: - stalwart (mail.extremist.software) - searxng (search.extremist.software) - synapse (matrix.extremist.software) -- grafana/prometheus (status.extremist.software) -- uptime-kuma (uptime.extremist.software) +- grafana/prometheus/blackbox exporter (status.extremist.software) +- uptime redirect to status.extremist.software (uptime.extremist.software) - ntfy (ntfy.extremist.software) - mymx (mymx.extremist.software) - caddy (reverse proxy + rate limiting) diff --git a/configuration.nix b/configuration.nix index 9a74c91..b957a5e 100644 --- a/configuration.nix +++ b/configuration.nix @@ -15,7 +15,7 @@ ./modules/matrix.nix ./modules/monitoring.nix ./modules/ntfy.nix - ./modules/uptime-kuma.nix + ./modules/blackbox-exporter.nix ./modules/noisebell.nix ./modules/noisepics.nix ]; diff --git a/modules/blackbox-exporter.nix b/modules/blackbox-exporter.nix new file mode 100644 index 0000000..e1b8017 --- /dev/null +++ b/modules/blackbox-exporter.nix @@ -0,0 +1,228 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + blackboxConfig = pkgs.writeText "blackbox.yml" '' + modules: + http_2xx: + prober: http + timeout: 5s + http: + preferred_ip_protocol: ip4 + tcp_connect: + prober: tcp + timeout: 5s + tcp: + preferred_ip_protocol: ip4 + icmp: + prober: icmp + timeout: 5s + icmp: + preferred_ip_protocol: ip4 + ''; + + blackboxAddress = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"; + + mkBlackboxJob = name: module: interval: monitors: { + job_name = name; + scrape_interval = interval; + metrics_path = "/probe"; + params.module = [ module ]; + static_configs = map ( + { name, target }: + { + targets = [ target ]; + labels.monitor = name; + } + ) monitors; + relabel_configs = [ + { + source_labels = [ "__address__" ]; + target_label = "__param_target"; + } + { + source_labels = [ "__param_target" ]; + target_label = "instance"; + } + { + target_label = "__address__"; + replacement = blackboxAddress; + } + ]; + }; + + http60Monitors = [ + { + name = "Website (Apex)"; + target = "https://extremist.software"; + } + { + name = "Extremist Software Forgejo"; + target = "https://git.extremist.software"; + } + { + name = "Stalwart Admin"; + target = "https://mail.extremist.software"; + } + { + name = "SearxNG"; + target = "https://search.extremist.software"; + } + { + name = "Grafana"; + target = "https://status.extremist.software"; + } + { + name = "ntfy"; + target = "https://ntfy.extremist.software"; + } + { + name = "Matrix"; + target = "https://matrix.extremist.software"; + } + { + name = "Noisebell"; + target = "https://noisebell.extremist.software"; + } + { + name = "Noisebell Discord"; + target = "https://discord.noisebell.extremist.software"; + } + { + name = "Noisebell RSS"; + target = "https://rss.noisebell.extremist.software"; + } + { + name = "Noisepics"; + target = "https://noisepics.extremist.software"; + } + { + name = "Noisebridge Mailing List"; + target = "https://lists.noisebridge.net"; + } + { + name = "Noisebridge Safespace"; + target = "https://safespace.noisebridge.net"; + } + { + name = "Noisebridge Library"; + target = "https://library.noisebridge.net"; + } + { + name = "Noisebridge Electronic Parts"; + target = "https://parts.noisebridge.net"; + } + { + name = "Noisebridge Auth"; + target = "https://auth.noisebridge.net"; + } + { + name = "Noisebridge Grafana"; + target = "https://grafana.noisebridge.net"; + } + { + name = "Noisebridge Prometheus"; + target = "https://prometheus.noisebridge.net"; + } + { + name = "jetpham.com"; + target = "https://jetpham.com"; + } + { + name = "m3.noisebridge.net"; + target = "https://m3.noisebridge.net"; + } + { + name = "m5.noisebridge.net"; + target = "https://m5.noisebridge.net"; + } + { + name = "m6.noisebridge.net"; + target = "https://m6.noisebridge.net"; + } + { + name = "m7.noisebridge.net"; + target = "https://m7.noisebridge.net"; + } + { + name = "Gitcafe"; + target = "https://app.gitcafe.dev"; + } + { + name = "Github"; + target = "https://github.com/"; + } + { + name = "Gitlab"; + target = "https://gitlab.com"; + } + { + name = "git.paperclover.net"; + target = "https://git.paperclover.net/clo"; + } + ]; + + http600Monitors = [ + { + name = "Noisebridge Wiki"; + target = "https://www.noisebridge.net/"; + } + { + name = "noisebridge.org"; + target = "https://www.noisebridge.org"; + } + { + name = "noisebridge.io"; + target = "https://noisebridge.io"; + } + ]; + + tcp60Monitors = [ + { + name = "SMTP (Standard)"; + target = "extremist.software:25"; + } + { + name = "SMTPS (Secure)"; + target = "extremist.software:465"; + } + { + name = "Submission"; + target = "mail.extremist.software:587"; + } + { + name = "IMAPS (Secure)"; + target = "extremist.software:993"; + } + { + name = "Noisebridge Mailing list SMTP"; + target = "lists.noisebridge.net:25"; + } + ]; + + icmp60Monitors = [ + { + name = "extremist.software ping"; + target = "extremist.software"; + } + ]; +in +{ + services.prometheus.exporters.blackbox = { + enable = true; + listenAddress = "127.0.0.1"; + port = 9115; + configFile = blackboxConfig; + }; + + services.prometheus.scrapeConfigs = lib.mkAfter [ + (mkBlackboxJob "blackbox-http-60s" "http_2xx" "60s" http60Monitors) + (mkBlackboxJob "blackbox-http-600s" "http_2xx" "600s" http600Monitors) + (mkBlackboxJob "blackbox-tcp-60s" "tcp_connect" "60s" tcp60Monitors) + (mkBlackboxJob "blackbox-icmp-60s" "icmp" "60s" icmp60Monitors) + ]; +} diff --git a/modules/caddy.nix b/modules/caddy.nix index 7af168e..afb852d 100644 --- a/modules/caddy.nix +++ b/modules/caddy.nix @@ -112,7 +112,7 @@ window 1m } } - reverse_proxy localhost:4001 + redir https://status.extremist.software{uri} ''; }; diff --git a/modules/uptime-kuma.nix b/modules/uptime-kuma.nix deleted file mode 100644 index 8ae0d6f..0000000 --- a/modules/uptime-kuma.nix +++ /dev/null @@ -1,11 +0,0 @@ -{ config, pkgs, ... }: - -{ - services.uptime-kuma = { - enable = true; - settings = { - PORT = "4001"; - HOST = "127.0.0.1"; - }; - }; -} From e672551481b2917127ef3644ac2bc9669374217c Mon Sep 17 00:00:00 2001 From: Jet Date: Tue, 31 Mar 2026 18:29:43 -0700 Subject: [PATCH 2/2] feat: add grafana monitor for blackbox --- grafana-dashboards/blackbox-overview.json | 759 ++++++++++++++++++++++ modules/monitoring.nix | 3 + 2 files changed, 762 insertions(+) create mode 100644 grafana-dashboards/blackbox-overview.json diff --git a/grafana-dashboards/blackbox-overview.json b/grafana-dashboards/blackbox-overview.json new file mode 100644 index 0000000..c7cbc08 --- /dev/null +++ b/grafana-dashboards/blackbox-overview.json @@ -0,0 +1,759 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(probe_success{job=~\"$job\", monitor=~\"$monitor\"})", + "instant": true, + "legendFormat": "Up", + "range": false, + "refId": "A" + } + ], + "title": "Up Now", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "count(probe_success{job=~\"$job\", monitor=~\"$monitor\"})", + "instant": true, + "legendFormat": "Total monitors", + "range": false, + "refId": "A" + } + ], + "title": "Configured Monitors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.95 + }, + { + "color": "green", + "value": 0.999 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg_over_time(probe_success{job=~\"$job\", monitor=~\"$monitor\"}[24h])", + "instant": true, + "legendFormat": "24h uptime", + "range": false, + "refId": "A" + } + ], + "title": "24h Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max(probe_duration_seconds{job=~\"$job\", monitor=~\"$monitor\"})", + "instant": true, + "legendFormat": "Slowest", + "range": false, + "refId": "A" + } + ], + "title": "Slowest Current Probe", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "probe_success{job=~\"$job\", monitor=~\"$monitor\"}", + "legendFormat": "{{monitor}}", + "range": true, + "refId": "A" + } + ], + "title": "Probe Success", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 16, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "probe_duration_seconds{job=~\"$job\", monitor=~\"$monitor\"}", + "legendFormat": "{{monitor}}", + "range": true, + "refId": "A" + } + ], + "title": "Probe Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "Down" + }, + "1": { + "color": "green", + "index": 1, + "text": "Up" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Monitor" + } + ] + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "probe_success{job=~\"$job\", monitor=~\"$monitor\"}", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "probe_duration_seconds{job=~\"$job\", monitor=~\"$monitor\"}", + "format": "table", + "instant": true, + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "probe_http_status_code{job=~\"$job\", monitor=~\"$monitor\"}", + "format": "table", + "instant": true, + "range": false, + "refId": "C" + } + ], + "title": "Current Status Table", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "monitor", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 2": true, + "Time": true, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "job": true, + "job 1": true, + "job 2": true + }, + "indexByName": { + "Value": 1, + "Value 1": 2, + "Value 2": 3, + "monitor": 0 + }, + "renameByName": { + "Value": "Status", + "Value 1": "Duration (s)", + "Value 2": "HTTP code", + "monitor": "Monitor" + } + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "blackbox", + "prometheus", + "uptime" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "All", + "value": ".*" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(probe_success{job=~\"blackbox-.*\"}, job)", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(probe_success{job=~\"blackbox-.*\"}, job)", + "refId": "Prometheus-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "All", + "value": ".*" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(probe_success{job=~\"$job\"}, monitor)", + "hide": 0, + "includeAll": true, + "label": "Monitor", + "multi": true, + "name": "monitor", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(probe_success{job=~\"$job\"}, monitor)", + "refId": "Prometheus-monitor-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Blackbox Overview", + "uid": "blackbox-overview", + "version": 1, + "weekStart": "" +} diff --git a/modules/monitoring.nix b/modules/monitoring.nix index 9f79c44..fddafcc 100644 --- a/modules/monitoring.nix +++ b/modules/monitoring.nix @@ -60,4 +60,7 @@ url = "https://grafana.com/api/dashboards/1860/revisions/37/download"; sha256 = "0qza4j8lywrj08bqbww52dgh2p2b9rkhq5p313g72i57lrlkacfl"; }; + + environment.etc."grafana-dashboards/blackbox-overview.json".source = + ../grafana-dashboards/blackbox-overview.json; }