From f67f90dffa4e1076ceb97157c2e577cdc4aa3511 Mon Sep 17 00:00:00 2001 From: Jet Date: Wed, 18 Mar 2026 16:08:53 -0700 Subject: [PATCH] faet: add more monitoring to minecraft --- configs/minecraft-dashboard.json | 714 +++++++++++++++++++++++++++++++ modules/monitoring.nix | 37 +- 2 files changed, 748 insertions(+), 3 deletions(-) create mode 100644 configs/minecraft-dashboard.json diff --git a/configs/minecraft-dashboard.json b/configs/minecraft-dashboard.json new file mode 100644 index 0000000..3812573 --- /dev/null +++ b/configs/minecraft-dashboard.json @@ -0,0 +1,714 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Server Status", + "gridPos": { "h": 4, "w": 3, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "minecraft_status_healthy", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { "text": "Offline", "color": "red", "index": 0 }, + "1": { "text": "Online", "color": "green", "index": 1 } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + } + }, + { + "id": 2, + "type": "stat", + "title": "Players Online", + "gridPos": { "h": 4, "w": 3, "x": 3, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "minecraft_status_players_online", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Server Ping", + "gridPos": { "h": 4, "w": 3, "x": 6, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "minecraft_status_response_time", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 100 }, + { "color": "red", "value": 300 } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + } + }, + { + "id": 4, + "type": "stat", + "title": "Container Uptime", + "gridPos": { "h": 4, "w": 5, "x": 9, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "time() - container_start_time_seconds{name=\"minecraft\"}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "dtdurations", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + } + }, + { + "id": 5, + "type": "stat", + "title": "Container CPU", + "gridPos": { "h": 4, "w": 5, "x": 14, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(container_cpu_usage_seconds_total{name=\"minecraft\"}[5m])) * 100", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 90 } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + } + }, + { + "id": 6, + "type": "stat", + "title": "Container Memory", + "gridPos": { "h": 4, "w": 5, "x": 19, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "container_memory_working_set_bytes{name=\"minecraft\"}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5368709120 }, + { "color": "red", "value": 6442450944 } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Player Count", + "gridPos": { "h": 7, "w": 24, "x": 0, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "minecraft_status_players_online", + "legendFormat": "Players", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "blue" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "scaleDistribution": { "type": "linear" } + }, + "decimals": 0, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "blue", "value": null }] + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "displayMode": "hidden" } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Container CPU Usage", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(container_cpu_usage_seconds_total{name=\"minecraft\"}[5m])) * 100", + "legendFormat": "CPU %", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "orange" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "scaleDistribution": { "type": "linear" } + }, + "unit": "percent", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Container Memory Usage", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "container_memory_working_set_bytes{name=\"minecraft\"}", + "legendFormat": "Used", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "container_spec_memory_limit_bytes{name=\"minecraft\"}", + "legendFormat": "Limit", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "scaleDistribution": { "type": "linear" } + }, + "unit": "bytes", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Limit" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, + { "id": "custom.fillOpacity", "value": 0 }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Container Network I/O", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(container_network_receive_bytes_total{name=\"minecraft\"}[5m])", + "legendFormat": "Received", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "-rate(container_network_transmit_bytes_total{name=\"minecraft\"}[5m])", + "legendFormat": "Transmitted", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "scaleDistribution": { "type": "linear" } + }, + "unit": "Bps", + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Container Disk I/O", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(container_fs_reads_bytes_total{name=\"minecraft\"}[5m]))", + "legendFormat": "Reads", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "-sum(rate(container_fs_writes_bytes_total{name=\"minecraft\"}[5m]))", + "legendFormat": "Writes", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "scaleDistribution": { "type": "linear" } + }, + "unit": "Bps", + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "System CPU", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 27 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", + "legendFormat": "CPU %", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "orange" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "scaleDistribution": { "type": "linear" } + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "displayMode": "hidden" } + } + }, + { + "id": 13, + "type": "timeseries", + "title": "System Memory", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 27 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100", + "legendFormat": "Memory %", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "purple" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "scaleDistribution": { "type": "linear" } + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "displayMode": "hidden" } + } + }, + { + "id": 14, + "type": "gauge", + "title": "Disk Usage", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 27 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "100 - ((node_filesystem_avail_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"}) * 100)", + "legendFormat": "Disk %", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 90 } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true + } + }, + { + "id": 15, + "type": "timeseries", + "title": "System Network Traffic", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 34 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(node_network_receive_bytes_total{device=\"eth0\"}[5m])", + "legendFormat": "Received", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "-rate(node_network_transmit_bytes_total{device=\"eth0\"}[5m])", + "legendFormat": "Transmitted", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "scaleDistribution": { "type": "linear" } + }, + "unit": "Bps", + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 16, + "type": "timeseries", + "title": "System Disk I/O", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 34 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(node_disk_read_bytes_total{device=\"vda\"}[5m])", + "legendFormat": "Reads", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "-rate(node_disk_written_bytes_total{device=\"vda\"}[5m])", + "legendFormat": "Writes", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "scaleDistribution": { "type": "linear" } + }, + "unit": "Bps", + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + } + ], + "schemaVersion": 39, + "tags": ["minecraft"], + "templating": { "list": [] }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Minecraft Server", + "uid": "minecraft-server", + "version": 1, + "refresh": "30s" +} diff --git a/modules/monitoring.nix b/modules/monitoring.nix index 4a00884..f48b91d 100644 --- a/modules/monitoring.nix +++ b/modules/monitoring.nix @@ -1,6 +1,20 @@ { config, pkgs, ... }: { + # Minecraft Prometheus exporter (player count, server status, ping) + virtualisation.oci-containers.containers.mc-monitor = { + image = "itzg/mc-monitor:latest"; + extraOptions = [ "--network=host" ]; + dependsOn = [ "minecraft" ]; + cmd = [ "export-for-prometheus" "--servers" "localhost:25565" "--port" "9150" ]; + }; + + # Container metrics (CPU, memory, network, disk I/O per container) + services.cadvisor = { + enable = true; + port = 9180; + }; + services.prometheus = { enable = true; port = 9090; @@ -40,6 +54,22 @@ }]; scrape_interval = "15s"; } + { + job_name = "minecraft"; + static_configs = [{ + targets = [ "localhost:9150" ]; + labels = { instance = "minecraft-server"; }; + }]; + scrape_interval = "15s"; + } + { + job_name = "cadvisor"; + static_configs = [{ + targets = [ "localhost:9180" ]; + labels = { instance = "minecraft-server"; }; + }]; + scrape_interval = "15s"; + } ]; }; @@ -68,6 +98,7 @@ { name = "Prometheus"; type = "prometheus"; + uid = "prometheus"; url = "http://localhost:9090"; isDefault = true; editable = false; @@ -76,9 +107,9 @@ dashboards.settings.providers = [ { - name = "Node Exporter Full"; - options.path = "${pkgs.writeTextDir "dashboards/node-exporter-full.json" - (builtins.readFile ../configs/node-exporter-full.json)}/dashboards"; + name = "Minecraft Server"; + options.path = "${pkgs.writeTextDir "dashboards/minecraft.json" + (builtins.readFile ../configs/minecraft-dashboard.json)}/dashboards"; options.foldersFromFilesStructure = false; } ];