From c6b38fd8f0e5908e5593d33e66cae8980f560c1b Mon Sep 17 00:00:00 2001 From: seth Date: Sun, 21 May 2023 13:46:36 -0400 Subject: move monitoring setup to modules --- hosts/p-body/default.nix | 4 +- hosts/p-body/loki.nix | 78 ------------------------- hosts/p-body/prometheus.nix | 13 ++++- hosts/p-body/promtail.nix | 37 ------------ hosts/profiles.nix | 9 ++- modules/nixos/server/services/default.nix | 2 + modules/nixos/server/services/loki.nix | 94 ++++++++++++++++++++++++++++++ modules/nixos/server/services/promtail.nix | 56 ++++++++++++++++++ 8 files changed, 171 insertions(+), 122 deletions(-) delete mode 100644 hosts/p-body/loki.nix delete mode 100644 hosts/p-body/promtail.nix create mode 100644 modules/nixos/server/services/loki.nix create mode 100644 modules/nixos/server/services/promtail.nix diff --git a/hosts/p-body/default.nix b/hosts/p-body/default.nix index ad88a7c..10abfac 100644 --- a/hosts/p-body/default.nix +++ b/hosts/p-body/default.nix @@ -9,10 +9,8 @@ (modulesPath + "/virtualisation/digital-ocean-image.nix") ./forgejo.nix ./grafana.nix - ./loki.nix ./nginx.nix ./prometheus.nix - ./promtail.nix ]; _module.args.nixinate = { @@ -23,6 +21,8 @@ hermetic = false; }; + getchoo.server.services.loki.enable = true; + networking = { domain = "mydadleft.me"; hostName = "p-body"; diff --git a/hosts/p-body/loki.nix b/hosts/p-body/loki.nix deleted file mode 100644 index 5b0541e..0000000 --- a/hosts/p-body/loki.nix +++ /dev/null @@ -1,78 +0,0 @@ -_: { - services.loki = { - enable = true; - configuration = { - server.http_listen_port = 3030; - auth_enabled = false; - - ingester = { - lifecycler = { - address = "127.0.0.1"; - ring = { - kvstore = { - store = "inmemory"; - }; - replication_factor = 1; - }; - }; - chunk_idle_period = "1h"; - max_chunk_age = "1h"; - chunk_target_size = 999999; - chunk_retain_period = "30s"; - max_transfer_retries = 0; - }; - - schema_config = { - configs = [ - { - from = "2022-06-06"; - store = "boltdb-shipper"; - object_store = "filesystem"; - schema = "v11"; - index = { - prefix = "index_"; - period = "24h"; - }; - } - ]; - }; - - storage_config = { - boltdb_shipper = { - active_index_directory = "/var/lib/loki/boltdb-shipper-active"; - cache_location = "/var/lib/loki/boltdb-shipper-cache"; - cache_ttl = "24h"; - shared_store = "filesystem"; - }; - - filesystem = { - directory = "/var/lib/loki/chunks"; - }; - }; - - limits_config = { - reject_old_samples = true; - reject_old_samples_max_age = "168h"; - }; - - chunk_store_config = { - max_look_back_period = "0s"; - }; - - table_manager = { - retention_deletes_enabled = false; - retention_period = "0s"; - }; - - compactor = { - working_directory = "/var/lib/loki"; - shared_store = "filesystem"; - compactor_ring = { - kvstore = { - store = "inmemory"; - }; - }; - }; - }; - }; -} diff --git a/hosts/p-body/prometheus.nix b/hosts/p-body/prometheus.nix index a0d272d..56e05a9 100644 --- a/hosts/p-body/prometheus.nix +++ b/hosts/p-body/prometheus.nix @@ -1,10 +1,10 @@ {config, ...}: let - scrapeExporter = name: exporter: { + scrapeExporter = name: host: port: { job_name = "${name}"; static_configs = [ { targets = [ - "127.0.0.1:${toString config.services.prometheus.exporters.${exporter}.port}" + "${host}:${port}" ]; } ]; @@ -21,7 +21,14 @@ in { }; }; scrapeConfigs = [ - (scrapeExporter "p-body" "node") + (scrapeExporter "p-body" "127.0.0.1" "${toString config.services.prometheus.exporters.node.port}") + (scrapeExporter "atlas" "atlas" "5001") ]; }; + + getchoo.server.services.promtail.clients = [ + { + url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push"; + } + ]; } diff --git a/hosts/p-body/promtail.nix b/hosts/p-body/promtail.nix deleted file mode 100644 index 1fa7f35..0000000 --- a/hosts/p-body/promtail.nix +++ /dev/null @@ -1,37 +0,0 @@ -{config, ...}: { - services.promtail = { - enable = true; - configuration = { - server = { - http_listen_port = 3031; - grpc_listen_port = 0; - }; - positions = { - filename = "/tmp/positions.yaml"; - }; - clients = [ - { - url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push"; - } - ]; - scrape_configs = [ - { - job_name = "journal"; - journal = { - max_age = "12h"; - labels = { - job = "systemd-journal"; - host = "pihole"; - }; - }; - relabel_configs = [ - { - source_labels = ["__journal__systemd_unit"]; - target_label = "unit"; - } - ]; - } - ]; - }; - }; -} diff --git a/hosts/profiles.nix b/hosts/profiles.nix index d565840..614933e 100644 --- a/hosts/profiles.nix +++ b/hosts/profiles.nix @@ -73,8 +73,13 @@ in { ../modules/nixos/features/tailscale.nix { - getchoo.features.tailscale.enable = true; - getchoo.server.enable = true; + getchoo = { + features.tailscale.enable = true; + server = { + enable = true; + services.promtail.enable = true; + }; + }; nix.registry.nixpkgs.flake = nixpkgs; } ]; diff --git a/modules/nixos/server/services/default.nix b/modules/nixos/server/services/default.nix index 68271b1..eb24d04 100644 --- a/modules/nixos/server/services/default.nix +++ b/modules/nixos/server/services/default.nix @@ -1,5 +1,7 @@ _: { imports = [ ./hercules.nix + ./loki.nix + ./promtail.nix ]; } diff --git a/modules/nixos/server/services/loki.nix b/modules/nixos/server/services/loki.nix new file mode 100644 index 0000000..1a42637 --- /dev/null +++ b/modules/nixos/server/services/loki.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + ... +}: let + cfg = config.getchoo.server.services.loki; + inherit (lib) mkEnableOption mkIf mkOption types; +in { + options.getchoo.server.services.loki = { + enable = mkEnableOption "enable loki"; + port = mkOption { + type = types.port; + default = 3030; + description = "port for loki"; + }; + }; + + config.services.loki = mkIf cfg.enable { + enable = true; + configuration = { + server.http_listen_port = cfg.port; + auth_enabled = false; + + ingester = { + lifecycler = { + address = "127.0.0.1"; + ring = { + kvstore = { + store = "inmemory"; + }; + replication_factor = 1; + }; + }; + chunk_idle_period = "1h"; + max_chunk_age = "1h"; + chunk_target_size = 999999; + chunk_retain_period = "30s"; + max_transfer_retries = 0; + }; + + schema_config = { + configs = [ + { + from = "2022-06-06"; + store = "boltdb-shipper"; + object_store = "filesystem"; + schema = "v11"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; + + storage_config = { + boltdb_shipper = { + active_index_directory = "/var/lib/loki/boltdb-shipper-active"; + cache_location = "/var/lib/loki/boltdb-shipper-cache"; + cache_ttl = "24h"; + shared_store = "filesystem"; + }; + + filesystem = { + directory = "/var/lib/loki/chunks"; + }; + }; + + limits_config = { + reject_old_samples = true; + reject_old_samples_max_age = "168h"; + }; + + chunk_store_config = { + max_look_back_period = "0s"; + }; + + table_manager = { + retention_deletes_enabled = false; + retention_period = "0s"; + }; + + compactor = { + working_directory = "/var/lib/loki"; + shared_store = "filesystem"; + compactor_ring = { + kvstore = { + store = "inmemory"; + }; + }; + }; + }; + }; +} diff --git a/modules/nixos/server/services/promtail.nix b/modules/nixos/server/services/promtail.nix new file mode 100644 index 0000000..73a8de2 --- /dev/null +++ b/modules/nixos/server/services/promtail.nix @@ -0,0 +1,56 @@ +{ + config, + lib, + ... +}: let + cfg = config.getchoo.server.services.promtail; + inherit (lib) mkEnableOption mkIf mkOption types; +in { + options.getchoo.server.services.promtail = { + enable = mkEnableOption "enable promtail"; + + port = mkOption { + type = types.port; + default = 3031; + description = "port for promtail"; + }; + + clients = mkOption { + type = types.listOf types.attrs; + default = [{}]; + description = "clients for promtail"; + }; + }; + + config.services.promtail = mkIf cfg.enable { + enable = true; + configuration = { + server = { + http_listen_port = cfg.port; + grpc_listen_port = 0; + }; + positions = { + filename = "/tmp/positions.yaml"; + }; + inherit (cfg) clients; + scrape_configs = [ + { + job_name = "journal"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = "${config.networking.hostName}"; + }; + }; + relabel_configs = [ + { + source_labels = ["__journal__systemd_unit"]; + target_label = "unit"; + } + ]; + } + ]; + }; + }; +} -- cgit v1.2.3