kernel stale warning

flake.lock

@@ -271,6 +271,22 @@
         "type": "indirect"
       }
     },
+    "nixos-infra": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1708125913,
+        "narHash": "sha256-2tFkhoIaZ7zjt9TsZIn0AxyOpGYxbqwE6IESi911g6Q=",
+        "owner": "nixos",
+        "repo": "infra",
+        "rev": "e35c6d13e9ab2ffe84015d1401df335c2690ca8e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "repo": "infra",
+        "type": "github"
+      }
+    },
     "nixos-mailserver": {
       "inputs": {
         "blobs": [],
@@ -373,6 +389,7 @@
         "nix-output-monitor": "nix-output-monitor",
         "nixfmt": "nixfmt",
         "nixos-hardware": "nixos-hardware",
+        "nixos-infra": "nixos-infra",
         "nixos-mailserver": "nixos-mailserver",
         "nixos-stable": "nixos-stable",
         "nixos-unstable": "nixos-unstable",

flake.nix

@@ -8,6 +8,10 @@
       url = "github:nix-community/disko";
       inputs.nixpkgs.follows = "nixos-stable";
     };
+    nixos-infra = {
+      url = "github:nixos/infra";
+      flake = false;
+    };
     nixfmt = {
       url = "github:piegamesde/nixfmt/rfc101-style";
       inputs = {

nixos/roles/all/default.nix

@@ -200,39 +200,6 @@ in
   services = {
     logind.killUserProcesses = false;
     journald.extraConfig = "SystemMaxUse=5G";
-    prometheus.exporters = {
-      node = {
-        enable = true;
-        enabledCollectors = [
-          "systemd"
-          "logind"
-        ];
-        disabledCollectors = [ "timex" ];
-      };
-      nginx = {
-        inherit (config.services.nginx) enable;
-      };
-    };
-    nginx = {
-      enable = lib.mkDefault (config.m-0.virtualHosts != { });
-      virtualHosts = lib.mapAttrs' (name: hostname: {
-        name = hostname;
-        value = {
-          forceSSL = true;
-          enableACME = true;
-          extraConfig = lib.mkIf (!(builtins.elem name (hosts.publicAliases.${hostName} or [ ]))) ''
-            satisfy any;
-            ${lib.concatMapStringsSep "\n" (ip_range: "allow ${ip_range};") config.m-0.headscaleIPs}
-            deny all;
-          '';
-        };
-      }) config.m-0.virtualHosts;
-      statusPage = true;
-      recommendedOptimisation = true;
-      recommendedGzipSettings = true;
-      recommendedTlsSettings = true;
-      clientMaxBodySize = "500m";
-    };
   };
   programs = {
     git.config.init.defaultBranch = "main";

nixos/roles/all/nginx.nix

@@ -0,0 +1,34 @@
+{ config, lib, ... }:
+let
+  inherit (config.m-0) hosts;
+  inherit (config.networking) hostName;
+in
+{
+  services = {
+    prometheus.exporters = {
+      nginx = {
+        inherit (config.services.nginx) enable;
+      };
+    };
+    nginx = {
+      enable = lib.mkDefault (config.m-0.virtualHosts != { });
+      virtualHosts = lib.mapAttrs' (name: hostname: {
+        name = hostname;
+        value = {
+          forceSSL = true;
+          enableACME = true;
+          extraConfig = lib.mkIf (!(builtins.elem name (hosts.publicAliases.${hostName} or [ ]))) ''
+            satisfy any;
+            ${lib.concatMapStringsSep "\n" (ip_range: "allow ${ip_range};") config.m-0.headscaleIPs}
+            deny all;
+          '';
+        };
+      }) config.m-0.virtualHosts;
+      statusPage = true;
+      recommendedOptimisation = true;
+      recommendedGzipSettings = true;
+      recommendedTlsSettings = true;
+      clientMaxBodySize = "500m";
+    };
+  };
+}

nixos/roles/all/prometheus-exporter.nix

@@ -0,0 +1,25 @@
+{ pkgs, ... }:
+{
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [
+      "systemd"
+      "logind"
+    ];
+    disabledCollectors = [ "timex" ];
+  };
+
+  systemd.services.prometheus-nixos-exporter = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+    path = [
+      pkgs.nix
+      pkgs.bash
+    ];
+    serviceConfig = {
+      Restart = "always";
+      RestartSec = "60s";
+      ExecStart = "${pkgs.prometheus-nixos-exporter}/bin/prometheus-nixos-exporter";
+    };
+  };
+}

nixos/roles/athene/prometheus.nix

@@ -1,99 +1,73 @@
 { config, lib, ... }:
 let
   inherit (config.m-0) virtualHosts;
-  targets = [
-    # Athene
+  hostJobs = name: flaky: [
     {
-      name = "athene";
-      host = "athene:9100";
+      inherit name;
+      host = "${name}:9100";
+      inherit flaky;
     }
     {
-      name = "athene user services";
-      host = "athene:9558";
+      name = "nixos on ${name}";
+      host = "${name}:9300";
+      inherit flaky;
     }
     {
-      name = "nginx on athene";
-      host = "hera:9113";
+      name = "${name} user services";
+      host = "${name}:9558";
+      inherit flaky;
     }
     {
-      name = "home assistant on athene";
-      metrics_path = "/api/prometheus";
-      host = "[::1]:8123";
-    }
-    # Hera
-    {
-      name = "hera";
-      host = "hera:9100";
-    }
-    {
-      name = "hera user services";
-      host = "hera:9558";
-    }
-    {
-      name = "nginx on hera";
-      host = "hera:9113";
-    }
-    {
-      name = "headscale on hera";
-      host = "hera:9098";
-    }
-    {
-      name = "postfix on hera";
-      host = "hera:9154";
-    }
-    {
-      name = "matrix-synapse on hera";
-      metrics_path = "/_synapse/metrics";
-      host = "hera:9148";
-    }
-    {
-      name = "forgejo on hera";
-      metrics_path = "/metrics";
-      host = "code.maralorn.de";
-    }
-    # Apollo
-    {
-      name = "apollo";
-      host = "apollo:9100";
-      flaky = true;
-    }
-    {
-      name = "apollo user services";
-      host = "apollo:9558";
-      flaky = true;
-    }
-    # Hephaistos
-    {
-      name = "hephaistos";
-      host = "hephaistos:9100";
-      flaky = true;
-    }
-    {
-      name = "hephaistos user services";
-      host = "hephaistos:9558";
-      flaky = true;
-    }
-    # Zeus
-    {
-      name = "zeus";
-      host = "zeus:9100";
-      flaky = true;
-    }
-    {
-      name = "zeus user services";
-      host = "zeus:9558";
-      flaky = true;
-    }
-    # Vocalensemble
-    {
-      name = "bach.ved";
-      host = "bach.vocalensemble-darmstadt.de:9100";
-    }
-    {
-      name = "postfix on bach.ved";
-      host = "bach.vocalensemble-darmstadt.de:9154";
+      name = "nginx on ${name}";
+      host = "${name}:9113";
+      inherit flaky;
     }
   ];
+
+  targets =
+    hostJobs "athene" false
+    ++ [
+      {
+        name = "home assistant on athene";
+        metrics_path = "/api/prometheus";
+        host = "[::1]:8123";
+      }
+    ]
+    ++ hostJobs "hera" false
+    ++ [
+      {
+        name = "headscale on hera";
+        host = "hera:9098";
+      }
+      {
+        name = "postfix on hera";
+        host = "hera:9154";
+      }
+      {
+        name = "matrix-synapse on hera";
+        metrics_path = "/_synapse/metrics";
+        host = "hera:9148";
+      }
+      {
+        name = "forgejo on hera";
+        metrics_path = "/metrics";
+        host = "code.maralorn.de";
+      }
+    ]
+    ++ hostJobs "apollo" true
+    ++ hostJobs "hephaistos" true
+    ++ hostJobs "zeus" true
+    ++ [
+      # Vocalensemble
+      {
+        name = "bach.ved";
+        host = "bach.vocalensemble-darmstadt.de:9100";
+      }
+      {
+        name = "postfix on bach.ved";
+        host = "bach.vocalensemble-darmstadt.de:9154";
+      }
+    ];
 in
 {
   environment.persistence.snapshoted.directories = [ "/var/lib/prometheus2" ];

nixos/roles/athene/rules.yml

@@ -90,9 +90,17 @@ groups:
 
         # low battery
         - alert: battery_low
-          expr: node_power_supply_energy_watthour / node_power_supply_energy_full <= 0.25
+          expr: node_power_supply_energy_watthour / node_power_supply_energy_full * on (name,power_supply) node_power_supply_info{status="Discharging"} <= 0.25
           for: 5m
           labels:
             severity: warning
           annotations:
             description: '{{ $labels.name }} has only {{ $value | humanizePercentage }} battery.'
+
+        # kernel stale
+        - alert: current_kernel_not_booted
+          expr: nixos_current_system_kernel_booted == 0
+          labels:
+            severity: warning
+          annotations:
+            description: 'The current kernel of {{ $labels.name }} is not booted.'

nixpkgs/prometheus-nixos-exporter.nix

@@ -0,0 +1,5 @@
+final: _: {
+  prometheus-nixos-exporter = final.callPackage (
+    final.flake-inputs.nixos-infra + /modules/prometheus/nixos-exporter/default.nix
+  ) { };
+}