From 141299e3ad9f8adce0f5a623c8e3c3a1c48aa651 Mon Sep 17 00:00:00 2001 From: matt1432 Date: Fri, 26 Jan 2024 14:28:42 -0500 Subject: [PATCH] feat(servers): use my custom pacemaker module --- devices/cluster/default.nix | 4 +- devices/cluster/modules/caddy.nix | 2 +- devices/cluster/modules/corosync.nix | 23 -- devices/cluster/modules/headscale/default.nix | 2 +- devices/cluster/modules/nfs-client.nix | 30 +++ devices/cluster/modules/pacemaker.nix | 72 ++++++ devices/cluster/modules/pacemaker/default.nix | 73 ------ devices/cluster/modules/pacemaker/options.nix | 240 ------------------ flake.lock | 31 ++- flake.nix | 9 +- 10 files changed, 137 insertions(+), 349 deletions(-) delete mode 100644 devices/cluster/modules/corosync.nix create mode 100644 devices/cluster/modules/nfs-client.nix create mode 100644 devices/cluster/modules/pacemaker.nix delete mode 100644 devices/cluster/modules/pacemaker/default.nix delete mode 100644 devices/cluster/modules/pacemaker/options.nix diff --git a/devices/cluster/default.nix b/devices/cluster/default.nix index 204d352..59a844d 100644 --- a/devices/cluster/default.nix +++ b/devices/cluster/default.nix @@ -1,7 +1,7 @@ deviceName: {config, ...}: let inherit (config.vars) mainUser hostName; - clusterIP = (builtins.elemAt config.services.pacemaker.resources.caddy.virtualIps 0).ip; + clusterIP = config.services.pacemaker.virtualIps.caddy-vip.ip; in { imports = [ ./hardware-configuration.nix @@ -10,7 +10,7 @@ in { ../../modules/sshd.nix ../../modules/tailscale.nix - ./modules/pacemaker + ./modules/pacemaker.nix ]; vars = { diff --git a/devices/cluster/modules/caddy.nix b/devices/cluster/modules/caddy.nix index b46218c..72b9511 100644 --- a/devices/cluster/modules/caddy.nix +++ b/devices/cluster/modules/caddy.nix @@ -9,7 +9,7 @@ caddy = caddy-plugins.packages.${pkgs.system}.default; - clusterIP = (builtins.elemAt config.services.pacemaker.resources.caddy.virtualIps 0).ip; + clusterIP = config.services.pacemaker.virtualIps.caddy-vip.ip; in { imports = [caddy-plugins.nixosModules.default]; diff --git a/devices/cluster/modules/corosync.nix b/devices/cluster/modules/corosync.nix deleted file mode 100644 index a41fd67..0000000 --- a/devices/cluster/modules/corosync.nix +++ /dev/null @@ -1,23 +0,0 @@ -{config, ...}: { - environment.etc."corosync/authkey" = { - source = config.sops.secrets.corosync.path; - }; - - services.corosync = { - enable = true; - clusterName = "thingies"; - - nodelist = [ - { - nodeid = 1; - name = "thingone"; - ring_addrs = ["10.0.0.244"]; - } - { - nodeid = 2; - name = "thingtwo"; - ring_addrs = ["10.0.0.159"]; - } - ]; - }; -} diff --git a/devices/cluster/modules/headscale/default.nix b/devices/cluster/modules/headscale/default.nix index 0933dab..1091d61 100644 --- a/devices/cluster/modules/headscale/default.nix +++ b/devices/cluster/modules/headscale/default.nix @@ -9,7 +9,7 @@ inherit (config.vars) mainUser hostName; headscale-flake = headscale.packages.${pkgs.system}.headscale; - clusterIP = (builtins.elemAt config.services.pacemaker.resources.caddy.virtualIps 0).ip; + clusterIP = config.services.pacemaker.virtualIps.caddy-vip.ip; in { environment.systemPackages = [headscale-flake]; users.users.${mainUser}.extraGroups = ["headscale"]; diff --git a/devices/cluster/modules/nfs-client.nix b/devices/cluster/modules/nfs-client.nix new file mode 100644 index 0000000..5f7e974 --- /dev/null +++ b/devices/cluster/modules/nfs-client.nix @@ -0,0 +1,30 @@ +{pkgs, ...}: { + # NFS client setup + services.rpcbind.enable = true; + boot.supportedFilesystems = ["nfs"]; + environment.systemPackages = with pkgs; [nfs-utils]; + + systemd.mounts = let + host = "10.0.0.249"; + in [ + { + type = "nfs"; + mountConfig = { + Options = "noatime"; + }; + what = "${host}:/caddy"; + where = "/var/lib/caddy"; + requiredBy = ["caddy.service"]; + } + + { + type = "nfs"; + mountConfig = { + Options = "noatime"; + }; + what = "${host}:/headscale"; + where = "/var/lib/headscale"; + requiredBy = ["headscale.service"]; + } + ]; +} diff --git a/devices/cluster/modules/pacemaker.nix b/devices/cluster/modules/pacemaker.nix new file mode 100644 index 0000000..8036729 --- /dev/null +++ b/devices/cluster/modules/pacemaker.nix @@ -0,0 +1,72 @@ +{ + config, + pacemaker, + ... +}: let + inherit (config.sops) secrets; +in { + imports = [ + pacemaker.nixosModules.default + + ./blocky.nix + ./caddy.nix + ./headscale + ./nfs-client.nix + ./unbound.nix + ]; + + services.pacemaker = { + enable = true; + clusterName = "thingies"; + + corosyncKeyFile = secrets.corosync.path; + clusterUserPasswordFile = secrets.PASSWORD.path; + + virtualIps = { + "caddy-vip" = { + ip = "10.0.0.130"; + interface = "eno1"; + group = "caddy"; + }; + }; + + systemdResources = { + "caddy" = { + enable = true; + group = "caddy"; + startAfter = ["caddy-vip"]; + }; + + "unbound" = { + enable = true; + group = "caddy"; + startAfter = ["caddy"]; + }; + + "blocky" = { + enable = true; + group = "caddy"; + startAfter = ["unbound"]; + }; + + "headscale" = { + enable = true; + group = "caddy"; + startAfter = ["blocky"]; + }; + }; + + nodes = [ + { + nodeid = 1; + name = "thingone"; + ring_addrs = ["10.0.0.244"]; + } + { + nodeid = 2; + name = "thingtwo"; + ring_addrs = ["10.0.0.159"]; + } + ]; + }; +} diff --git a/devices/cluster/modules/pacemaker/default.nix b/devices/cluster/modules/pacemaker/default.nix deleted file mode 100644 index 850c19a..0000000 --- a/devices/cluster/modules/pacemaker/default.nix +++ /dev/null @@ -1,73 +0,0 @@ -{pkgs, ...}: { - imports = [ - ./options.nix - ../corosync.nix - - ../blocky.nix - ../caddy.nix - ../headscale - ../unbound.nix - ]; - - # TODO: update script - services.pacemaker = { - enable = true; - - resources = { - "blocky" = { - enable = true; - dependsOn = ["unbound"]; - }; - - "caddy" = { - enable = true; - virtualIps = [ - { - id = "main"; - interface = "eno1"; - ip = "10.0.0.130"; - } - ]; - }; - - "headscale" = { - enable = true; - dependsOn = ["caddy"]; - }; - - "unbound" = { - enable = true; - dependsOn = ["caddy"]; - }; - }; - }; - - # NFS client setup - services.rpcbind.enable = true; - boot.supportedFilesystems = ["nfs"]; - environment.systemPackages = with pkgs; [nfs-utils]; - - systemd.mounts = let - host = "10.0.0.249"; - in [ - { - type = "nfs"; - mountConfig = { - Options = "noatime"; - }; - what = "${host}:/caddy"; - where = "/var/lib/caddy"; - requiredBy = ["caddy.service"]; - } - - { - type = "nfs"; - mountConfig = { - Options = "noatime"; - }; - what = "${host}:/headscale"; - where = "/var/lib/headscale"; - requiredBy = ["headscale.service"]; - } - ]; -} diff --git a/devices/cluster/modules/pacemaker/options.nix b/devices/cluster/modules/pacemaker/options.nix deleted file mode 100644 index af45b1b..0000000 --- a/devices/cluster/modules/pacemaker/options.nix +++ /dev/null @@ -1,240 +0,0 @@ -{ - config, - lib, - nixpkgs-pacemaker, - pkgs, - ... -}: let - inherit - (lib) - attrNames - attrValues - concatMapStringsSep - elemAt - filterAttrs - isAttrs - mkIf - mkOption - types - ; - inherit (builtins) toFile map listToAttrs; - - pacemakerPath = "services/cluster/pacemaker/default.nix"; - cfg = config.services.pacemaker; -in { - disabledModules = [pacemakerPath]; - imports = ["${nixpkgs-pacemaker}/nixos/modules/${pacemakerPath}"]; - - options.services.pacemaker = { - resources = mkOption { - default = {}; - type = with types; - attrsOf (submodule ({name, ...}: { - options = { - enable = mkOption { - default = true; - type = types.bool; - }; - - systemdName = mkOption { - default = name; - type = types.str; - }; - - # TODO: add assertion to not have same id - virtualIps = mkOption { - default = []; - type = with types; - listOf (submodule { - options = { - id = mkOption { - type = types.str; - }; - - interface = mkOption { - default = "eno1"; - type = types.str; - }; - - ip = mkOption { - type = types.str; - }; - - cidr = mkOption { - default = 24; - type = types.int; - }; - }; - }); - }; - - # TODO: add assertion, needs to be an existing systemdName - dependsOn = mkOption { - default = []; - type = types.listOf types.str; - }; - - # TODO: Add extraResources, extraConstraints ... - }; - })); - }; - }; - - config = mkIf cfg.enable { - systemd.services = let - mkVirtIps = res: - concatMapStringsSep "\n" (vip: '' - - - - - - - - - - - - - '') - res.virtualIps; - - mkSystemdResource = res: '' - - - - - - - - ''; - - mkConstraint = res: first: let - firstName = - if isAttrs first - then first.systemdName - else first; - in '' - - - ''; - - mkDependsOn = res: let - mkConstraint' = first: - mkConstraint res first; - in - concatMapStringsSep "\n" mkConstraint' res.dependsOn; - - mkVipConstraint = res: - concatMapStringsSep "\n" ( - vip: - mkConstraint - res - "${res.systemdName}-${vip.id}-vip" - ) - res.virtualIps; - - # If we're updating resources we have to kill constraints to add new resources - constraintsEmpty = toFile "constraints.xml" '' - - - ''; - - resEnabled = filterAttrs (n: v: v.enable) cfg.resources; - - resWithIp = filterAttrs (n: v: ! isNull v.virtualIps) resEnabled; - - resources = toFile "resources.xml" '' - - ${concatMapStringsSep "\n" mkVirtIps (attrValues resWithIp)} - ${concatMapStringsSep "\n" mkSystemdResource (attrValues resEnabled)} - - ''; - - constraints = toFile "constraints.xml" '' - - ${concatMapStringsSep "\n" mkVipConstraint (attrValues resWithIp)} - ${concatMapStringsSep "\n" mkDependsOn (attrValues resEnabled)} - - ''; - - host1 = (elemAt config.services.corosync.nodelist 0).name; - in - { - "pacemaker-setup" = { - after = ["corosync.service" "pacemaker.service"]; - - path = with pkgs; [pacemaker]; - - script = '' - # The config needs to be installed from one node only - # TODO: add assertion, corosync must be enabled with at least one node - if [ "$(uname -n)" = ${host1} ]; then - # TODO: setup stonith / fencing - crm_attribute --type crm_config --name stonith-enabled --update false - crm_attribute --type crm_config --name no-quorum-policy --delete - - # Install config - cibadmin --replace --scope constraints --xml-file ${constraintsEmpty} - cibadmin --replace --scope resources --xml-file ${resources} - cibadmin --replace --scope constraints --xml-file ${constraints} - fi - ''; - }; - } - # Force all systemd units handled by pacemaker to not start automatically - // listToAttrs (map (x: { - name = x; - value = { - wantedBy = lib.mkForce []; - }; - }) (attrNames cfg.resources)); - - # FIXME: https://github.com/NixOS/nixpkgs/pull/208298 - nixpkgs.overlays = [ - (final: prev: { - inherit - (nixpkgs-pacemaker.legacyPackages.x86_64-linux) - pacemaker - ocf-resource-agents - ; - }) - ]; - }; -} diff --git a/flake.lock b/flake.lock index 9915901..3a9d3ad 100644 --- a/flake.lock +++ b/flake.lock @@ -1036,6 +1036,27 @@ "type": "github" } }, + "pacemaker": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-pacemaker": "nixpkgs-pacemaker" + }, + "locked": { + "lastModified": 1706296989, + "narHash": "sha256-qpaRjqvPVio4T3qV6Hws1kr1ohwY+Dhx82KFEPeSiLU=", + "owner": "matt1432", + "repo": "nixos-pacemaker", + "rev": "f819e743d2e2329f7c306978a5ea084955c0364d", + "type": "github" + }, + "original": { + "owner": "matt1432", + "repo": "nixos-pacemaker", + "type": "github" + } + }, "pam-fprint-grosshack-src": { "flake": false, "locked": { @@ -1156,12 +1177,12 @@ "nix-melt": "nix-melt", "nix-on-droid": "nix-on-droid", "nixpkgs": "nixpkgs_4", - "nixpkgs-pacemaker": "nixpkgs-pacemaker", "nixpkgs-wayland": "nixpkgs-wayland", "nms": "nms", "nur": "nur", "nurl": "nurl", "nvim-theme-src": "nvim-theme-src", + "pacemaker": "pacemaker", "pam-fprint-grosshack-src": "pam-fprint-grosshack-src", "persist-properties-src": "persist-properties-src", "plymouth-src": "plymouth-src", @@ -1184,11 +1205,11 @@ "sops-nix": "sops-nix" }, "locked": { - "lastModified": 1705981759, - "narHash": "sha256-4Cvwx1i+fgNOdvKheGvNiK8ZB7FENj2O7j4FLW1sLO8=", + "lastModified": 1706296642, + "narHash": "sha256-If1RZSkzjcp8IhQlObt8Ih3zW/xPof9HPlenF3ZbWq8=", "ref": "refs/heads/main", - "rev": "4655ce03e395f6438f66d521da3614379c8f9354", - "revCount": 36, + "rev": "418f51a455052c9344d2dc90731ffdf008472b56", + "revCount": 38, "type": "git", "url": "ssh://git@git.nelim.org/matt1432/nixos-secrets" }, diff --git a/flake.nix b/flake.nix index 9ab881c..998918c 100644 --- a/flake.nix +++ b/flake.nix @@ -141,11 +141,12 @@ }; # Cluster Inputs - nixpkgs-pacemaker = { + pacemaker = { type = "github"; - owner = "mitchty"; - repo = "nixpkgs"; - ref = "corosync-pacemaker-ocf"; + owner = "matt1432"; + repo = "nixos-pacemaker"; + + inputs.nixpkgs.follows = "nixpkgs"; }; # Oksys inputs