diff --git a/devices/cluster/default.nix b/devices/cluster/default.nix index 8466665..a404d1d 100644 --- a/devices/cluster/default.nix +++ b/devices/cluster/default.nix @@ -9,7 +9,7 @@ in { ../../modules/tailscale.nix ./modules/corosync.nix - ./modules/pacemaker.nix + ./modules/pacemaker ]; vars = { diff --git a/devices/cluster/modules/pacemaker.nix b/devices/cluster/modules/pacemaker.nix deleted file mode 100644 index cbc60fe..0000000 --- a/devices/cluster/modules/pacemaker.nix +++ /dev/null @@ -1,19 +0,0 @@ -{nixpkgs-pacemaker, ...}: let - pacemakerPath = "services/cluster/pacemaker/default.nix"; -in { - # FIXME: https://github.com/NixOS/nixpkgs/pull/208298 - nixpkgs.overlays = [ - (final: prev: { - inherit - (nixpkgs-pacemaker.legacyPackages.x86_64-linux) - pacemaker - ocf-resource-agents - ; - }) - ]; - - disabledModules = [pacemakerPath]; - imports = ["${nixpkgs-pacemaker}/nixos/modules/${pacemakerPath}"]; - - services.pacemaker.enable = true; -} diff --git a/devices/cluster/modules/pacemaker/default.nix b/devices/cluster/modules/pacemaker/default.nix new file mode 100644 index 0000000..c857b57 --- /dev/null +++ b/devices/cluster/modules/pacemaker/default.nix @@ -0,0 +1,9 @@ +{...}: { + imports = [./options.nix]; + + services.pacemaker = { + enable = true; + + resources = {}; + }; +} diff --git a/devices/cluster/modules/pacemaker/options.nix b/devices/cluster/modules/pacemaker/options.nix new file mode 100644 index 0000000..e6669e0 --- /dev/null +++ b/devices/cluster/modules/pacemaker/options.nix @@ -0,0 +1,203 @@ +{ + config, + lib, + nixpkgs-pacemaker, + pkgs, + ... +}: let + inherit + (lib) + attrNames + attrValues + concatMapStringsSep + elemAt + filterAttrs + mkIf + mkOption + types + ; + inherit (builtins) toFile map listToAttrs; + + pacemakerPath = "services/cluster/pacemaker/default.nix"; + cfg = config.services.pacemaker; +in { + disabledModules = [pacemakerPath]; + imports = ["${nixpkgs-pacemaker}/nixos/modules/${pacemakerPath}"]; + + options.services.pacemaker = { + networkInterface = mkOption { + default = "eno1"; + type = types.str; + }; + + resources = mkOption { + default = {}; + type = with types; + attrsOf (submodule ({name, ...}: { + options = { + enable = mkOption { + default = true; + type = types.bool; + }; + + systemdName = mkOption { + default = name; + type = types.str; + }; + + virtualIp = mkOption { + default = null; + type = types.nullOr types.str; + }; + + dependsOn = mkOption { + default = []; + # TODO: implement dependsOn + type = types.listOf types.str; + }; + + # TODO: Add extraResources, extraConstraints ... + }; + })); + }; + }; + + config = mkIf cfg.enable { + systemd.services = let + mkVirtIp = res: '' + + + + + + + + + + + + + ''; + + mkSystemdResource = res: '' + + + + + + + + ''; + + mkConstraint = first: res: '' + + + ''; + + mkVipConstraint = res: + mkConstraint + (res.systemdName + "vip") + res; + + # If we're updating resources we have to kill constraints to add new resources + constraintsEmpty = toFile "constraints.xml" '' + + + ''; + + resEnabled = filterAttrs (n: v: v.enable) cfg.resources; + + resWithIp = filterAttrs (n: v: ! isNull v.virtualIp) resEnabled; + + resources = toFile "resources.xml" '' + + ${concatMapStringsSep "\n" mkVirtIp (attrValues resWithIp)} + ${concatMapStringsSep "\n" mkSystemdResource (attrValues resEnabled)} + + ''; + + constraints = toFile "constraints.xml" '' + + ${concatMapStringsSep "\n" mkVipConstraint (attrValues resWithIp)} + + ''; + + host1 = (elemAt config.services.corosync.nodelist 0).name; + in + { + "pacemaker-setup" = { + after = ["corosync.service" "pacemaker.service"]; + + path = with pkgs; [pacemaker]; + + script = '' + # The config needs to be installed from one node only + if [ "$(uname -n)" = ${host1} ]; then + # TODO: setup stonith / fencing + crm_attribute --type crm_config --name stonith-enabled --update false + crm_attribute --type crm_config --name no-quorum-policy --delete + + # Install config + cibadmin --replace --scope constraints --xml-file ${constraintsEmpty} + cibadmin --replace --scope resources --xml-file ${resources} + cibadmin --replace --scope constraints --xml-file ${constraints} + fi + ''; + }; + } + # Force all systemd units handled by pacemaker to not start automatically + // listToAttrs (map (x: { + name = x; + value = { + wantedBy = lib.mkForce []; + }; + }) (attrNames cfg.resources)); + + # FIXME: https://github.com/NixOS/nixpkgs/pull/208298 + nixpkgs.overlays = [ + (final: prev: { + inherit + (nixpkgs-pacemaker.legacyPackages.x86_64-linux) + pacemaker + ocf-resource-agents + ; + }) + ]; + }; +}