feat: add noisebell observability

scripts/deploy-pios-pi.sh

@@ -70,7 +70,7 @@ ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "DEPLOY_HOSTNAME='$DEPLOY_HOSTNAME' HOME_ASS
 set -euo pipefail
 
 sudo apt-get update
-sudo apt-get install -y curl rsync avahi-daemon
+sudo apt-get install -y curl jq rsync avahi-daemon prometheus-node-exporter
 
 sudo hostnamectl set-hostname "$DEPLOY_HOSTNAME"
 sudo tee /etc/hostname >/dev/null <<<"$DEPLOY_HOSTNAME"
@@ -86,7 +86,16 @@ HOSTSEOF
 if ! command -v tailscale >/dev/null 2>&1; then
   curl -fsSL https://tailscale.com/install.sh | sh
 fi
-sudo systemctl enable --now ssh avahi-daemon tailscaled
+sudo mkdir -p /etc/systemd/journald.conf.d /var/log/journal
+sudo tee /etc/systemd/journald.conf.d/noisebell-persistent.conf >/dev/null <<'JOURNALCONF'
+[Journal]
+Storage=persistent
+SystemMaxUse=200M
+MaxRetentionSec=30day
+JOURNALCONF
+sudo systemctl restart systemd-journald
+
+sudo systemctl enable --now ssh avahi-daemon tailscaled prometheus-node-exporter
 
 sudo install -m 755 "$REMOTE_TMP_DIR/noisebell" "$REMOTE_RELEASE_DIR/noisebell"
 sudo install -m 755 "$REMOTE_TMP_DIR/noisebell-relay" "$REMOTE_RELEASE_DIR/noisebell-relay"
@@ -159,12 +168,95 @@ RestartSec=5
 WantedBy=multi-user.target
 UNITEOF
 
+sudo tee /usr/local/bin/noisebell-loki-journal >/dev/null <<'SCRIPTEOF'
+#!/usr/bin/env bash
+set -uo pipefail
+
+LOKI_URL=${LOKI_URL:-http://noisebell-do:3100/loki/api/v1/push}
+HOST_LABEL=${HOST_LABEL:-$(hostname)}
+CURSOR_DIR=/var/lib/noisebell-loki-journal
+CURSOR_FILE=$CURSOR_DIR/cursor
+
+mkdir -p "$CURSOR_DIR"
+
+while true; do
+  args=(--output=json --no-pager --lines=100)
+  if [ -s "$CURSOR_FILE" ]; then
+    args+=(--after-cursor="$(cat "$CURSOR_FILE")")
+  else
+    args+=(--since=-5min)
+  fi
+
+  saw_entry=0
+  hit_error=0
+
+  while IFS= read -r entry; do
+    saw_entry=1
+    cursor=$(jq -r '.__CURSOR // empty' <<<"$entry")
+    timestamp=$(jq -r '.__REALTIME_TIMESTAMP // empty' <<<"$entry")
+    if [ -n "$timestamp" ] && [ "$timestamp" != "null" ]; then
+      timestamp="${timestamp}000"
+    else
+      timestamp=$(date +%s%N)
+    fi
+
+    unit=$(jq -r '._SYSTEMD_UNIT // .SYSLOG_IDENTIFIER // "journal"' <<<"$entry")
+    message=$(jq -r '.MESSAGE // .' <<<"$entry")
+
+    payload=$(jq -cn \
+      --arg host "$HOST_LABEL" \
+      --arg unit "$unit" \
+      --arg ts "$timestamp" \
+      --arg line "$message" \
+      '{streams:[{stream:{job:"journal",host:$host,unit:$unit},values:[[$ts,$line]]}]}')
+
+    if curl -fsS --max-time 5 \
+      -H 'Content-Type: application/json' \
+      -X POST \
+      --data "$payload" \
+      "$LOKI_URL" >/dev/null 2>&1; then
+      if [ -n "$cursor" ]; then
+        printf '%s\n' "$cursor" > "$CURSOR_FILE"
+      fi
+    else
+      hit_error=1
+      break
+    fi
+  done < <(journalctl "${args[@]}" 2>/dev/null)
+
+  if [ "$hit_error" -eq 1 ] || [ "$saw_entry" -eq 0 ]; then
+    sleep 5
+  fi
+done
+SCRIPTEOF
+sudo chmod 755 /usr/local/bin/noisebell-loki-journal
+
+sudo tee /etc/systemd/system/noisebell-loki-journal.service >/dev/null <<'UNITEOF'
+[Unit]
+Description=Noisebell journal shipper to Loki
+After=network-online.target tailscaled.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+Environment=LOKI_URL=http://noisebell-do:3100/loki/api/v1/push
+Environment=HOST_LABEL=noisebell-pi
+ExecStart=/usr/local/bin/noisebell-loki-journal
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+UNITEOF
+
 sudo ln -sfn "$REMOTE_RELEASE_DIR" "$REMOTE_CURRENT_LINK"
 sudo systemctl daemon-reload
 sudo systemctl enable noisebell.service
 sudo systemctl enable noisebell-relay.service
+sudo systemctl enable noisebell-loki-journal.service
 sudo systemctl restart noisebell.service
 sudo systemctl restart noisebell-relay.service
+sudo systemctl restart noisebell-loki-journal.service
 sudo systemctl restart avahi-daemon
 
 sudo tailscale up --auth-key="$(sudo cat /etc/noisebell/tailscale-auth-key)" --hostname=noisebell-pi || true