hosts!: prometheus -> victoria metrics

7 files changed, 45 insertions, 62 deletions

diff --git a/hosts/atlas/default.nix b/hosts/atlas/default.nix
index a77d6ec..1a9691d 100644
--- a/hosts/atlas/default.nix
+++ b/hosts/atlas/default.nix
@@ -7,7 +7,6 @@
     ./hardware-configuration.nix
     ./miniflux.nix
     ./nginx.nix
-    ./prometheus.nix
   ];
 
   _module.args.nixinate = {
@@ -30,6 +29,7 @@
   networking = {
     domain = "mydadleft.me";
     hostName = "atlas";
+    firewall.allowedTCPPorts = [config.services.prometheus.exporters.node.port];
   };
 
   nix.settings.allowed-users = ["bob"];
diff --git a/hosts/atlas/prometheus.nix b/hosts/atlas/prometheus.nix
deleted file mode 100644
index 041c407..0000000
--- a/hosts/atlas/prometheus.nix
+++ /dev/null
@@ -1,14 +0,0 @@
-{config, ...}: {
-  networking.firewall.allowedTCPPorts = [config.services.prometheus.exporters.node.port];
-
-  services.prometheus.exporters.node = {
-    enable = true;
-    enabledCollectors = ["systemd"];
-  };
-
-  getchoo.server.services.promtail.clients = [
-    {
-      url = "http://p-body:3030/loki/api/v1/push";
-    }
-  ];
-}
diff --git a/hosts/p-body/default.nix b/hosts/p-body/default.nix
index 06baded..c1d4dca 100644
--- a/hosts/p-body/default.nix
+++ b/hosts/p-body/default.nix
@@ -11,7 +11,7 @@
     ./grafana.nix
     ./loki.nix
     ./nginx.nix
-    ./prometheus.nix
+    ./victoriametrics.nix
   ];
 
   _module.args.nixinate = {
diff --git a/hosts/p-body/prometheus.nix b/hosts/p-body/prometheus.nix
deleted file mode 100644
index 26e9d0e..0000000
--- a/hosts/p-body/prometheus.nix
+++ /dev/null
@@ -1,32 +0,0 @@
-{config, ...}: let
-  scrapeExporter = name: host: port: {
-    job_name = "${name}";
-    static_configs = [
-      {
-        targets = [
-          "${host}:${port}"
-        ];
-      }
-    ];
-  };
-in {
-  services.prometheus = {
-    enable = true;
-    exporters = {
-      node = {
-        enable = true;
-        enabledCollectors = ["systemd"];
-      };
-    };
-    scrapeConfigs = [
-      (scrapeExporter "p-body" "localhost" "${toString config.services.prometheus.exporters.node.port}")
-      (scrapeExporter "atlas" "atlas" "${toString config.services.prometheus.exporters.node.port}")
-    ];
-  };
-
-  getchoo.server.services.promtail.clients = [
-    {
-      url = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
-    }
-  ];
-}
diff --git a/hosts/p-body/victoriametrics.nix b/hosts/p-body/victoriametrics.nix
new file mode 100644
index 0000000..dec893e
--- /dev/null
+++ b/hosts/p-body/victoriametrics.nix
@@ -0,0 +1,26 @@
+{config, ...}: let
+  mkScrapes = let
+    mkConfig = client: {
+      job_name = "${client}";
+      static_configs = [
+        {
+          targets = [
+            "${client}:${toString config.services.prometheus.exporters.node.port}"
+          ];
+        }
+      ];
+    };
+  in
+    builtins.map mkConfig;
+in {
+  services = {
+    victoriametrics.enable = true;
+
+    vmagent = {
+      enable = true;
+      prometheusConfig = {
+        scrape_configs = mkScrapes ["p-body" "atlas"];
+      };
+    };
+  };
+}
diff --git a/hosts/profiles.nix b/hosts/profiles.nix
index 5b200c2..ae25d91 100644
--- a/hosts/profiles.nix
+++ b/hosts/profiles.nix
@@ -103,10 +103,22 @@ in {
 
           server = {
             enable = true;
-            services.promtail.enable = true;
+            services.promtail = {
+              enable = true;
+              clients = [
+                {
+                  url = "http://p-body:3030/loki/api/v1/push";
+                }
+              ];
+            };
           };
         };
 
+        services.prometheus.exporters.node = {
+          enable = true;
+          enabledCollectors = ["systemd"];
+        };
+
         nix.registry.nixpkgs.flake = nixpkgs-stable;
         system.stateVersion = "23.05";
       }
diff --git a/modules/nixos/server/services/promtail.nix b/modules/nixos/server/services/promtail.nix
index 73a8de2..95592f0 100644
--- a/modules/nixos/server/services/promtail.nix
+++ b/modules/nixos/server/services/promtail.nix
@@ -9,12 +9,6 @@ in {
   options.getchoo.server.services.promtail = {
     enable = mkEnableOption "enable promtail";
 
-    port = mkOption {
-      type = types.port;
-      default = 3031;
-      description = "port for promtail";
-    };
-
     clients = mkOption {
       type = types.listOf types.attrs;
       default = [{}];
@@ -25,17 +19,13 @@ in {
   config.services.promtail = mkIf cfg.enable {
     enable = true;
     configuration = {
-      server = {
-        http_listen_port = cfg.port;
-        grpc_listen_port = 0;
-      };
-      positions = {
-        filename = "/tmp/positions.yaml";
-      };
       inherit (cfg) clients;
+      server.disable = true;
+
       scrape_configs = [
         {
           job_name = "journal";
+
           journal = {
             max_age = "12h";
             labels = {
@@ -43,6 +33,7 @@ in {
               host = "${config.networking.hostName}";
             };
           };
+
           relabel_configs = [
             {
               source_labels = ["__journal__systemd_unit"];