diff --git a/README.md b/README.md index 979286d..83684fb 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ services: - stalwart (mail.extremist.software) - searxng (search.extremist.software) - synapse (matrix.extremist.software) -- grafana/prometheus/blackbox exporter (status.extremist.software) -- uptime redirect to status.extremist.software (uptime.extremist.software) +- grafana/prometheus (status.extremist.software) +- uptime-kuma (uptime.extremist.software) - ntfy (ntfy.extremist.software) - mymx (mymx.extremist.software) - caddy (reverse proxy + rate limiting) diff --git a/configuration.nix b/configuration.nix index b957a5e..9a74c91 100644 --- a/configuration.nix +++ b/configuration.nix @@ -15,7 +15,7 @@ ./modules/matrix.nix ./modules/monitoring.nix ./modules/ntfy.nix - ./modules/blackbox-exporter.nix + ./modules/uptime-kuma.nix ./modules/noisebell.nix ./modules/noisepics.nix ]; diff --git a/grafana-dashboards/blackbox-overview.json b/grafana-dashboards/blackbox-overview.json deleted file mode 100644 index c7cbc08..0000000 --- a/grafana-dashboards/blackbox-overview.json +++ /dev/null @@ -1,759 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(probe_success{job=~\"$job\", monitor=~\"$monitor\"})", - "instant": true, - "legendFormat": "Up", - "range": false, - "refId": "A" - } - ], - "title": "Up Now", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "count(probe_success{job=~\"$job\", monitor=~\"$monitor\"})", - "instant": true, - "legendFormat": "Total monitors", - "range": false, - "refId": "A" - } - ], - "title": "Configured Monitors", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 0.95 - }, - { - "color": "green", - "value": 0.999 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "avg_over_time(probe_success{job=~\"$job\", monitor=~\"$monitor\"}[24h])", - "instant": true, - "legendFormat": "24h uptime", - "range": false, - "refId": "A" - } - ], - "title": "24h Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 4, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "max(probe_duration_seconds{job=~\"$job\", monitor=~\"$monitor\"})", - "instant": true, - "legendFormat": "Slowest", - "range": false, - "refId": "A" - } - ], - "title": "Slowest Current Probe", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 4, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 4 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "probe_success{job=~\"$job\", monitor=~\"$monitor\"}", - "legendFormat": "{{monitor}}", - "range": true, - "refId": "A" - } - ], - "title": "Probe Success", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 16, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 4, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 4 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "probe_duration_seconds{job=~\"$job\", monitor=~\"$monitor\"}", - "legendFormat": "{{monitor}}", - "range": true, - "refId": "A" - } - ], - "title": "Probe Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 0, - "text": "Down" - }, - "1": { - "color": "green", - "index": 1, - "text": "Up" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 7, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "Monitor" - } - ] - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "probe_success{job=~\"$job\", monitor=~\"$monitor\"}", - "format": "table", - "instant": true, - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "probe_duration_seconds{job=~\"$job\", monitor=~\"$monitor\"}", - "format": "table", - "instant": true, - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "probe_http_status_code{job=~\"$job\", monitor=~\"$monitor\"}", - "format": "table", - "instant": true, - "range": false, - "refId": "C" - } - ], - "title": "Current Status Table", - "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "monitor", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time 1": true, - "Time 2": true, - "Time": true, - "__name__": true, - "__name__ 1": true, - "__name__ 2": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "job": true, - "job 1": true, - "job 2": true - }, - "indexByName": { - "Value": 1, - "Value 1": 2, - "Value 2": 3, - "monitor": 0 - }, - "renameByName": { - "Value": "Status", - "Value 1": "Duration (s)", - "Value 2": "HTTP code", - "monitor": "Monitor" - } - } - } - ], - "type": "table" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "style": "dark", - "tags": [ - "blackbox", - "prometheus", - "uptime" - ], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": "All", - "value": ".*" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(probe_success{job=~\"blackbox-.*\"}, job)", - "hide": 0, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(probe_success{job=~\"blackbox-.*\"}, job)", - "refId": "Prometheus-job-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": "All", - "value": ".*" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(probe_success{job=~\"$job\"}, monitor)", - "hide": 0, - "includeAll": true, - "label": "Monitor", - "multi": true, - "name": "monitor", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(probe_success{job=~\"$job\"}, monitor)", - "refId": "Prometheus-monitor-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Blackbox Overview", - "uid": "blackbox-overview", - "version": 1, - "weekStart": "" -} diff --git a/modules/blackbox-exporter.nix b/modules/blackbox-exporter.nix deleted file mode 100644 index e1b8017..0000000 --- a/modules/blackbox-exporter.nix +++ /dev/null @@ -1,228 +0,0 @@ -{ - config, - lib, - pkgs, - ... -}: - -let - blackboxConfig = pkgs.writeText "blackbox.yml" '' - modules: - http_2xx: - prober: http - timeout: 5s - http: - preferred_ip_protocol: ip4 - tcp_connect: - prober: tcp - timeout: 5s - tcp: - preferred_ip_protocol: ip4 - icmp: - prober: icmp - timeout: 5s - icmp: - preferred_ip_protocol: ip4 - ''; - - blackboxAddress = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"; - - mkBlackboxJob = name: module: interval: monitors: { - job_name = name; - scrape_interval = interval; - metrics_path = "/probe"; - params.module = [ module ]; - static_configs = map ( - { name, target }: - { - targets = [ target ]; - labels.monitor = name; - } - ) monitors; - relabel_configs = [ - { - source_labels = [ "__address__" ]; - target_label = "__param_target"; - } - { - source_labels = [ "__param_target" ]; - target_label = "instance"; - } - { - target_label = "__address__"; - replacement = blackboxAddress; - } - ]; - }; - - http60Monitors = [ - { - name = "Website (Apex)"; - target = "https://extremist.software"; - } - { - name = "Extremist Software Forgejo"; - target = "https://git.extremist.software"; - } - { - name = "Stalwart Admin"; - target = "https://mail.extremist.software"; - } - { - name = "SearxNG"; - target = "https://search.extremist.software"; - } - { - name = "Grafana"; - target = "https://status.extremist.software"; - } - { - name = "ntfy"; - target = "https://ntfy.extremist.software"; - } - { - name = "Matrix"; - target = "https://matrix.extremist.software"; - } - { - name = "Noisebell"; - target = "https://noisebell.extremist.software"; - } - { - name = "Noisebell Discord"; - target = "https://discord.noisebell.extremist.software"; - } - { - name = "Noisebell RSS"; - target = "https://rss.noisebell.extremist.software"; - } - { - name = "Noisepics"; - target = "https://noisepics.extremist.software"; - } - { - name = "Noisebridge Mailing List"; - target = "https://lists.noisebridge.net"; - } - { - name = "Noisebridge Safespace"; - target = "https://safespace.noisebridge.net"; - } - { - name = "Noisebridge Library"; - target = "https://library.noisebridge.net"; - } - { - name = "Noisebridge Electronic Parts"; - target = "https://parts.noisebridge.net"; - } - { - name = "Noisebridge Auth"; - target = "https://auth.noisebridge.net"; - } - { - name = "Noisebridge Grafana"; - target = "https://grafana.noisebridge.net"; - } - { - name = "Noisebridge Prometheus"; - target = "https://prometheus.noisebridge.net"; - } - { - name = "jetpham.com"; - target = "https://jetpham.com"; - } - { - name = "m3.noisebridge.net"; - target = "https://m3.noisebridge.net"; - } - { - name = "m5.noisebridge.net"; - target = "https://m5.noisebridge.net"; - } - { - name = "m6.noisebridge.net"; - target = "https://m6.noisebridge.net"; - } - { - name = "m7.noisebridge.net"; - target = "https://m7.noisebridge.net"; - } - { - name = "Gitcafe"; - target = "https://app.gitcafe.dev"; - } - { - name = "Github"; - target = "https://github.com/"; - } - { - name = "Gitlab"; - target = "https://gitlab.com"; - } - { - name = "git.paperclover.net"; - target = "https://git.paperclover.net/clo"; - } - ]; - - http600Monitors = [ - { - name = "Noisebridge Wiki"; - target = "https://www.noisebridge.net/"; - } - { - name = "noisebridge.org"; - target = "https://www.noisebridge.org"; - } - { - name = "noisebridge.io"; - target = "https://noisebridge.io"; - } - ]; - - tcp60Monitors = [ - { - name = "SMTP (Standard)"; - target = "extremist.software:25"; - } - { - name = "SMTPS (Secure)"; - target = "extremist.software:465"; - } - { - name = "Submission"; - target = "mail.extremist.software:587"; - } - { - name = "IMAPS (Secure)"; - target = "extremist.software:993"; - } - { - name = "Noisebridge Mailing list SMTP"; - target = "lists.noisebridge.net:25"; - } - ]; - - icmp60Monitors = [ - { - name = "extremist.software ping"; - target = "extremist.software"; - } - ]; -in -{ - services.prometheus.exporters.blackbox = { - enable = true; - listenAddress = "127.0.0.1"; - port = 9115; - configFile = blackboxConfig; - }; - - services.prometheus.scrapeConfigs = lib.mkAfter [ - (mkBlackboxJob "blackbox-http-60s" "http_2xx" "60s" http60Monitors) - (mkBlackboxJob "blackbox-http-600s" "http_2xx" "600s" http600Monitors) - (mkBlackboxJob "blackbox-tcp-60s" "tcp_connect" "60s" tcp60Monitors) - (mkBlackboxJob "blackbox-icmp-60s" "icmp" "60s" icmp60Monitors) - ]; -} diff --git a/modules/caddy.nix b/modules/caddy.nix index afb852d..7af168e 100644 --- a/modules/caddy.nix +++ b/modules/caddy.nix @@ -112,7 +112,7 @@ window 1m } } - redir https://status.extremist.software{uri} + reverse_proxy localhost:4001 ''; }; diff --git a/modules/monitoring.nix b/modules/monitoring.nix index fddafcc..9f79c44 100644 --- a/modules/monitoring.nix +++ b/modules/monitoring.nix @@ -60,7 +60,4 @@ url = "https://grafana.com/api/dashboards/1860/revisions/37/download"; sha256 = "0qza4j8lywrj08bqbww52dgh2p2b9rkhq5p313g72i57lrlkacfl"; }; - - environment.etc."grafana-dashboards/blackbox-overview.json".source = - ../grafana-dashboards/blackbox-overview.json; } diff --git a/modules/uptime-kuma.nix b/modules/uptime-kuma.nix new file mode 100644 index 0000000..8ae0d6f --- /dev/null +++ b/modules/uptime-kuma.nix @@ -0,0 +1,11 @@ +{ config, pkgs, ... }: + +{ + services.uptime-kuma = { + enable = true; + settings = { + PORT = "4001"; + HOST = "127.0.0.1"; + }; + }; +}