feat(servers): setup cluster resource options
All checks were successful
Discord / discord commits (push) Successful in 57s

This commit is contained in:
matt1432 2024-01-22 21:22:48 -05:00
parent 2b14ecc1b8
commit 85b5a4b8f2
4 changed files with 213 additions and 20 deletions

View file

@ -9,7 +9,7 @@ in {
../../modules/tailscale.nix ../../modules/tailscale.nix
./modules/corosync.nix ./modules/corosync.nix
./modules/pacemaker.nix ./modules/pacemaker
]; ];
vars = { vars = {

View file

@ -1,19 +0,0 @@
{nixpkgs-pacemaker, ...}: let
pacemakerPath = "services/cluster/pacemaker/default.nix";
in {
# FIXME: https://github.com/NixOS/nixpkgs/pull/208298
nixpkgs.overlays = [
(final: prev: {
inherit
(nixpkgs-pacemaker.legacyPackages.x86_64-linux)
pacemaker
ocf-resource-agents
;
})
];
disabledModules = [pacemakerPath];
imports = ["${nixpkgs-pacemaker}/nixos/modules/${pacemakerPath}"];
services.pacemaker.enable = true;
}

View file

@ -0,0 +1,9 @@
{...}: {
imports = [./options.nix];
services.pacemaker = {
enable = true;
resources = {};
};
}

View file

@ -0,0 +1,203 @@
{
config,
lib,
nixpkgs-pacemaker,
pkgs,
...
}: let
inherit
(lib)
attrNames
attrValues
concatMapStringsSep
elemAt
filterAttrs
mkIf
mkOption
types
;
inherit (builtins) toFile map listToAttrs;
pacemakerPath = "services/cluster/pacemaker/default.nix";
cfg = config.services.pacemaker;
in {
disabledModules = [pacemakerPath];
imports = ["${nixpkgs-pacemaker}/nixos/modules/${pacemakerPath}"];
options.services.pacemaker = {
networkInterface = mkOption {
default = "eno1";
type = types.str;
};
resources = mkOption {
default = {};
type = with types;
attrsOf (submodule ({name, ...}: {
options = {
enable = mkOption {
default = true;
type = types.bool;
};
systemdName = mkOption {
default = name;
type = types.str;
};
virtualIp = mkOption {
default = null;
type = types.nullOr types.str;
};
dependsOn = mkOption {
default = [];
# TODO: implement dependsOn
type = types.listOf types.str;
};
# TODO: Add extraResources, extraConstraints ...
};
}));
};
};
config = mkIf cfg.enable {
systemd.services = let
mkVirtIp = res: ''
<primitive class="ocf" id="${res.systemdName}vip" provider="heartbeat" type="IPaddr2">
<instance_attributes id="${res.systemdName}vip-attrs">
<nvpair
id="${res.systemdName}vip-attrs-cidr_netmask"
name="cidr_netmask"
value="24"
/>
<nvpair
id="${res.systemdName}vip-attrs-ip"
name="ip"
value="${res.virtualIp}"
/>
<nvpair
id="${res.systemdName}vip-attrs-nic"
name="nic"
value="${cfg.networkInterface}"
/>
</instance_attributes>
<operations>
<op
id="${res.systemdName}vip-monitor-interval-30s"
interval="30s"
name="monitor"
/>
<op
id="${res.systemdName}vip-start-interval-0s"
interval="0s"
name="start"
timeout="20s"
/>
<op
id="${res.systemdName}vip-stop-interval-0s"
interval="0s"
name="stop"
timeout="20s"
/>
</operations>
</primitive>
'';
mkSystemdResource = res: ''
<primitive id="${res.systemdName}" class="systemd" type="${res.systemdName}">
<operations>
<op id="stop-${res.systemdName}" name="stop" interval="0" timeout="10s"/>
<op id="start-${res.systemdName}" name="start" interval="0" timeout="10s"/>
<op id="monitor-${res.systemdName}" name="monitor" interval="10s" timeout="20s"/>
</operations>
</primitive>
'';
mkConstraint = first: res: ''
<rsc_order
id="order-${res.systemdName}"
first="${first}"
then="${res.systemdName}"
kind="Mandatory"
/>
<rsc_colocation
id="colocate-${res.systemdName}"
rsc="${first}"
with-rsc="${res.systemdName}"
score="INFINITY"
/>
'';
mkVipConstraint = res:
mkConstraint
(res.systemdName + "vip")
res;
# If we're updating resources we have to kill constraints to add new resources
constraintsEmpty = toFile "constraints.xml" ''
<constraints>
</constraints>
'';
resEnabled = filterAttrs (n: v: v.enable) cfg.resources;
resWithIp = filterAttrs (n: v: ! isNull v.virtualIp) resEnabled;
resources = toFile "resources.xml" ''
<resources>
${concatMapStringsSep "\n" mkVirtIp (attrValues resWithIp)}
${concatMapStringsSep "\n" mkSystemdResource (attrValues resEnabled)}
</resources>
'';
constraints = toFile "constraints.xml" ''
<constraints>
${concatMapStringsSep "\n" mkVipConstraint (attrValues resWithIp)}
</constraints>
'';
host1 = (elemAt config.services.corosync.nodelist 0).name;
in
{
"pacemaker-setup" = {
after = ["corosync.service" "pacemaker.service"];
path = with pkgs; [pacemaker];
script = ''
# The config needs to be installed from one node only
if [ "$(uname -n)" = ${host1} ]; then
# TODO: setup stonith / fencing
crm_attribute --type crm_config --name stonith-enabled --update false
crm_attribute --type crm_config --name no-quorum-policy --delete
# Install config
cibadmin --replace --scope constraints --xml-file ${constraintsEmpty}
cibadmin --replace --scope resources --xml-file ${resources}
cibadmin --replace --scope constraints --xml-file ${constraints}
fi
'';
};
}
# Force all systemd units handled by pacemaker to not start automatically
// listToAttrs (map (x: {
name = x;
value = {
wantedBy = lib.mkForce [];
};
}) (attrNames cfg.resources));
# FIXME: https://github.com/NixOS/nixpkgs/pull/208298
nixpkgs.overlays = [
(final: prev: {
inherit
(nixpkgs-pacemaker.legacyPackages.x86_64-linux)
pacemaker
ocf-resource-agents
;
})
];
};
}