diff --git a/devices/cluster/default.nix b/devices/cluster/default.nix
index 204d352..59a844d 100644
--- a/devices/cluster/default.nix
+++ b/devices/cluster/default.nix
@@ -1,7 +1,7 @@
deviceName: {config, ...}: let
inherit (config.vars) mainUser hostName;
- clusterIP = (builtins.elemAt config.services.pacemaker.resources.caddy.virtualIps 0).ip;
+ clusterIP = config.services.pacemaker.virtualIps.caddy-vip.ip;
in {
imports = [
./hardware-configuration.nix
@@ -10,7 +10,7 @@ in {
../../modules/sshd.nix
../../modules/tailscale.nix
- ./modules/pacemaker
+ ./modules/pacemaker.nix
];
vars = {
diff --git a/devices/cluster/modules/caddy.nix b/devices/cluster/modules/caddy.nix
index b46218c..72b9511 100644
--- a/devices/cluster/modules/caddy.nix
+++ b/devices/cluster/modules/caddy.nix
@@ -9,7 +9,7 @@
caddy = caddy-plugins.packages.${pkgs.system}.default;
- clusterIP = (builtins.elemAt config.services.pacemaker.resources.caddy.virtualIps 0).ip;
+ clusterIP = config.services.pacemaker.virtualIps.caddy-vip.ip;
in {
imports = [caddy-plugins.nixosModules.default];
diff --git a/devices/cluster/modules/corosync.nix b/devices/cluster/modules/corosync.nix
deleted file mode 100644
index a41fd67..0000000
--- a/devices/cluster/modules/corosync.nix
+++ /dev/null
@@ -1,23 +0,0 @@
-{config, ...}: {
- environment.etc."corosync/authkey" = {
- source = config.sops.secrets.corosync.path;
- };
-
- services.corosync = {
- enable = true;
- clusterName = "thingies";
-
- nodelist = [
- {
- nodeid = 1;
- name = "thingone";
- ring_addrs = ["10.0.0.244"];
- }
- {
- nodeid = 2;
- name = "thingtwo";
- ring_addrs = ["10.0.0.159"];
- }
- ];
- };
-}
diff --git a/devices/cluster/modules/headscale/default.nix b/devices/cluster/modules/headscale/default.nix
index 0933dab..1091d61 100644
--- a/devices/cluster/modules/headscale/default.nix
+++ b/devices/cluster/modules/headscale/default.nix
@@ -9,7 +9,7 @@
inherit (config.vars) mainUser hostName;
headscale-flake = headscale.packages.${pkgs.system}.headscale;
- clusterIP = (builtins.elemAt config.services.pacemaker.resources.caddy.virtualIps 0).ip;
+ clusterIP = config.services.pacemaker.virtualIps.caddy-vip.ip;
in {
environment.systemPackages = [headscale-flake];
users.users.${mainUser}.extraGroups = ["headscale"];
diff --git a/devices/cluster/modules/nfs-client.nix b/devices/cluster/modules/nfs-client.nix
new file mode 100644
index 0000000..5f7e974
--- /dev/null
+++ b/devices/cluster/modules/nfs-client.nix
@@ -0,0 +1,30 @@
+{pkgs, ...}: {
+ # NFS client setup
+ services.rpcbind.enable = true;
+ boot.supportedFilesystems = ["nfs"];
+ environment.systemPackages = with pkgs; [nfs-utils];
+
+ systemd.mounts = let
+ host = "10.0.0.249";
+ in [
+ {
+ type = "nfs";
+ mountConfig = {
+ Options = "noatime";
+ };
+ what = "${host}:/caddy";
+ where = "/var/lib/caddy";
+ requiredBy = ["caddy.service"];
+ }
+
+ {
+ type = "nfs";
+ mountConfig = {
+ Options = "noatime";
+ };
+ what = "${host}:/headscale";
+ where = "/var/lib/headscale";
+ requiredBy = ["headscale.service"];
+ }
+ ];
+}
diff --git a/devices/cluster/modules/pacemaker.nix b/devices/cluster/modules/pacemaker.nix
new file mode 100644
index 0000000..8036729
--- /dev/null
+++ b/devices/cluster/modules/pacemaker.nix
@@ -0,0 +1,72 @@
+{
+ config,
+ pacemaker,
+ ...
+}: let
+ inherit (config.sops) secrets;
+in {
+ imports = [
+ pacemaker.nixosModules.default
+
+ ./blocky.nix
+ ./caddy.nix
+ ./headscale
+ ./nfs-client.nix
+ ./unbound.nix
+ ];
+
+ services.pacemaker = {
+ enable = true;
+ clusterName = "thingies";
+
+ corosyncKeyFile = secrets.corosync.path;
+ clusterUserPasswordFile = secrets.PASSWORD.path;
+
+ virtualIps = {
+ "caddy-vip" = {
+ ip = "10.0.0.130";
+ interface = "eno1";
+ group = "caddy";
+ };
+ };
+
+ systemdResources = {
+ "caddy" = {
+ enable = true;
+ group = "caddy";
+ startAfter = ["caddy-vip"];
+ };
+
+ "unbound" = {
+ enable = true;
+ group = "caddy";
+ startAfter = ["caddy"];
+ };
+
+ "blocky" = {
+ enable = true;
+ group = "caddy";
+ startAfter = ["unbound"];
+ };
+
+ "headscale" = {
+ enable = true;
+ group = "caddy";
+ startAfter = ["blocky"];
+ };
+ };
+
+ nodes = [
+ {
+ nodeid = 1;
+ name = "thingone";
+ ring_addrs = ["10.0.0.244"];
+ }
+ {
+ nodeid = 2;
+ name = "thingtwo";
+ ring_addrs = ["10.0.0.159"];
+ }
+ ];
+ };
+}
diff --git a/devices/cluster/modules/pacemaker/default.nix b/devices/cluster/modules/pacemaker/default.nix
deleted file mode 100644
index 850c19a..0000000
--- a/devices/cluster/modules/pacemaker/default.nix
+++ /dev/null
@@ -1,73 +0,0 @@
-{pkgs, ...}: {
- imports = [
- ./options.nix
- ../corosync.nix
-
- ../blocky.nix
- ../caddy.nix
- ../headscale
- ../unbound.nix
- ];
-
- # TODO: update script
- services.pacemaker = {
- enable = true;
-
- resources = {
- "blocky" = {
- enable = true;
- dependsOn = ["unbound"];
- };
-
- "caddy" = {
- enable = true;
- virtualIps = [
- {
- id = "main";
- interface = "eno1";
- ip = "10.0.0.130";
- }
- ];
- };
-
- "headscale" = {
- enable = true;
- dependsOn = ["caddy"];
- };
-
- "unbound" = {
- enable = true;
- dependsOn = ["caddy"];
- };
- };
- };
-
- # NFS client setup
- services.rpcbind.enable = true;
- boot.supportedFilesystems = ["nfs"];
- environment.systemPackages = with pkgs; [nfs-utils];
-
- systemd.mounts = let
- host = "10.0.0.249";
- in [
- {
- type = "nfs";
- mountConfig = {
- Options = "noatime";
- };
- what = "${host}:/caddy";
- where = "/var/lib/caddy";
- requiredBy = ["caddy.service"];
- }
-
- {
- type = "nfs";
- mountConfig = {
- Options = "noatime";
- };
- what = "${host}:/headscale";
- where = "/var/lib/headscale";
- requiredBy = ["headscale.service"];
- }
- ];
-}
diff --git a/devices/cluster/modules/pacemaker/options.nix b/devices/cluster/modules/pacemaker/options.nix
deleted file mode 100644
index af45b1b..0000000
--- a/devices/cluster/modules/pacemaker/options.nix
+++ /dev/null
@@ -1,240 +0,0 @@
-{
- config,
- lib,
- nixpkgs-pacemaker,
- pkgs,
- ...
-}: let
- inherit
- (lib)
- attrNames
- attrValues
- concatMapStringsSep
- elemAt
- filterAttrs
- isAttrs
- mkIf
- mkOption
- types
- ;
- inherit (builtins) toFile map listToAttrs;
-
- pacemakerPath = "services/cluster/pacemaker/default.nix";
- cfg = config.services.pacemaker;
-in {
- disabledModules = [pacemakerPath];
- imports = ["${nixpkgs-pacemaker}/nixos/modules/${pacemakerPath}"];
-
- options.services.pacemaker = {
- resources = mkOption {
- default = {};
- type = with types;
- attrsOf (submodule ({name, ...}: {
- options = {
- enable = mkOption {
- default = true;
- type = types.bool;
- };
-
- systemdName = mkOption {
- default = name;
- type = types.str;
- };
-
- # TODO: add assertion to not have same id
- virtualIps = mkOption {
- default = [];
- type = with types;
- listOf (submodule {
- options = {
- id = mkOption {
- type = types.str;
- };
-
- interface = mkOption {
- default = "eno1";
- type = types.str;
- };
-
- ip = mkOption {
- type = types.str;
- };
-
- cidr = mkOption {
- default = 24;
- type = types.int;
- };
- };
- });
- };
-
- # TODO: add assertion, needs to be an existing systemdName
- dependsOn = mkOption {
- default = [];
- type = types.listOf types.str;
- };
-
- # TODO: Add extraResources, extraConstraints ...
- };
- }));
- };
- };
-
- config = mkIf cfg.enable {
- systemd.services = let
- mkVirtIps = res:
- concatMapStringsSep "\n" (vip: ''
-
-
-
-
-
-
-
-
-
-
-
-
- '')
- res.virtualIps;
-
- mkSystemdResource = res: ''
-
-
-
-
-
-
-
- '';
-
- mkConstraint = res: first: let
- firstName =
- if isAttrs first
- then first.systemdName
- else first;
- in ''
-
-
- '';
-
- mkDependsOn = res: let
- mkConstraint' = first:
- mkConstraint res first;
- in
- concatMapStringsSep "\n" mkConstraint' res.dependsOn;
-
- mkVipConstraint = res:
- concatMapStringsSep "\n" (
- vip:
- mkConstraint
- res
- "${res.systemdName}-${vip.id}-vip"
- )
- res.virtualIps;
-
- # If we're updating resources we have to kill constraints to add new resources
- constraintsEmpty = toFile "constraints.xml" ''
-
-
- '';
-
- resEnabled = filterAttrs (n: v: v.enable) cfg.resources;
-
- resWithIp = filterAttrs (n: v: ! isNull v.virtualIps) resEnabled;
-
- resources = toFile "resources.xml" ''
-
- ${concatMapStringsSep "\n" mkVirtIps (attrValues resWithIp)}
- ${concatMapStringsSep "\n" mkSystemdResource (attrValues resEnabled)}
-
- '';
-
- constraints = toFile "constraints.xml" ''
-
- ${concatMapStringsSep "\n" mkVipConstraint (attrValues resWithIp)}
- ${concatMapStringsSep "\n" mkDependsOn (attrValues resEnabled)}
-
- '';
-
- host1 = (elemAt config.services.corosync.nodelist 0).name;
- in
- {
- "pacemaker-setup" = {
- after = ["corosync.service" "pacemaker.service"];
-
- path = with pkgs; [pacemaker];
-
- script = ''
- # The config needs to be installed from one node only
- # TODO: add assertion, corosync must be enabled with at least one node
- if [ "$(uname -n)" = ${host1} ]; then
- # TODO: setup stonith / fencing
- crm_attribute --type crm_config --name stonith-enabled --update false
- crm_attribute --type crm_config --name no-quorum-policy --delete
-
- # Install config
- cibadmin --replace --scope constraints --xml-file ${constraintsEmpty}
- cibadmin --replace --scope resources --xml-file ${resources}
- cibadmin --replace --scope constraints --xml-file ${constraints}
- fi
- '';
- };
- }
- # Force all systemd units handled by pacemaker to not start automatically
- // listToAttrs (map (x: {
- name = x;
- value = {
- wantedBy = lib.mkForce [];
- };
- }) (attrNames cfg.resources));
-
- # FIXME: https://github.com/NixOS/nixpkgs/pull/208298
- nixpkgs.overlays = [
- (final: prev: {
- inherit
- (nixpkgs-pacemaker.legacyPackages.x86_64-linux)
- pacemaker
- ocf-resource-agents
- ;
- })
- ];
- };
-}
diff --git a/flake.lock b/flake.lock
index 9915901..3a9d3ad 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1036,6 +1036,27 @@
"type": "github"
}
},
+ "pacemaker": {
+ "inputs": {
+ "nixpkgs": [
+ "nixpkgs"
+ ],
+ "nixpkgs-pacemaker": "nixpkgs-pacemaker"
+ },
+ "locked": {
+ "lastModified": 1706296989,
+ "narHash": "sha256-qpaRjqvPVio4T3qV6Hws1kr1ohwY+Dhx82KFEPeSiLU=",
+ "owner": "matt1432",
+ "repo": "nixos-pacemaker",
+ "rev": "f819e743d2e2329f7c306978a5ea084955c0364d",
+ "type": "github"
+ },
+ "original": {
+ "owner": "matt1432",
+ "repo": "nixos-pacemaker",
+ "type": "github"
+ }
+ },
"pam-fprint-grosshack-src": {
"flake": false,
"locked": {
@@ -1156,12 +1177,12 @@
"nix-melt": "nix-melt",
"nix-on-droid": "nix-on-droid",
"nixpkgs": "nixpkgs_4",
- "nixpkgs-pacemaker": "nixpkgs-pacemaker",
"nixpkgs-wayland": "nixpkgs-wayland",
"nms": "nms",
"nur": "nur",
"nurl": "nurl",
"nvim-theme-src": "nvim-theme-src",
+ "pacemaker": "pacemaker",
"pam-fprint-grosshack-src": "pam-fprint-grosshack-src",
"persist-properties-src": "persist-properties-src",
"plymouth-src": "plymouth-src",
@@ -1184,11 +1205,11 @@
"sops-nix": "sops-nix"
},
"locked": {
- "lastModified": 1705981759,
- "narHash": "sha256-4Cvwx1i+fgNOdvKheGvNiK8ZB7FENj2O7j4FLW1sLO8=",
+ "lastModified": 1706296642,
+ "narHash": "sha256-If1RZSkzjcp8IhQlObt8Ih3zW/xPof9HPlenF3ZbWq8=",
"ref": "refs/heads/main",
- "rev": "4655ce03e395f6438f66d521da3614379c8f9354",
- "revCount": 36,
+ "rev": "418f51a455052c9344d2dc90731ffdf008472b56",
+ "revCount": 38,
"type": "git",
"url": "ssh://git@git.nelim.org/matt1432/nixos-secrets"
},
diff --git a/flake.nix b/flake.nix
index 9ab881c..998918c 100644
--- a/flake.nix
+++ b/flake.nix
@@ -141,11 +141,12 @@
};
# Cluster Inputs
- nixpkgs-pacemaker = {
+ pacemaker = {
type = "github";
- owner = "mitchty";
- repo = "nixpkgs";
- ref = "corosync-pacemaker-ocf";
+ owner = "matt1432";
+ repo = "nixos-pacemaker";
+
+ inputs.nixpkgs.follows = "nixpkgs";
};
# Oksys inputs