feat(servers): move oksys services to cluster

devices/cluster/default.nix

@@ -1,5 +1,7 @@
 deviceName: {config, ...}: let
   inherit (config.vars) mainUser hostName;
+
+  clusterIP = (builtins.elemAt config.services.pacemaker.resources.caddy.virtualIps 0).ip;
 in {
   imports = [
     ./hardware-configuration.nix
@@ -34,6 +36,10 @@ in {
   networking = {
     inherit hostName;
     resolvconf.enable = true;
+    nameservers = [
+      clusterIP
+      "1.0.0.1"
+    ];
     firewall.enable = false;
   };
 

devices/cluster/modules/caddy.nix

@@ -8,6 +8,8 @@
   inherit (config.sops) secrets;
 
   caddy = caddy-plugins.packages.${pkgs.system}.default;
+
+  clusterIP = (builtins.elemAt config.services.pacemaker.resources.caddy.virtualIps 0).ip;
 in {
   imports = [caddy-plugins.nixosModules.default];
 
@@ -33,7 +35,6 @@ in {
       dockerIP = "10.0.0.122";
       jellyIP = "10.0.0.123";
       servivi = "10.0.0.249";
-      oksysIP = "10.0.0.213";
     in {
       "nelim.org" = {
         serverAliases = ["*.nelim.org"];
@@ -48,7 +49,7 @@ in {
           # Misc one-liners
           vault.reverseProxy = "${dockerIP}:8781";
           hauk.reverseProxy = "${dockerIP}:3003";
-          headscale.reverseProxy = "${oksysIP}:8085";
+          headscale.reverseProxy = "${clusterIP}:8085";
           jelly.reverseProxy = "${jellyIP}:80";
 
           # Resume builder

devices/cluster/modules/headscale/default.nix

@@ -6,8 +6,10 @@
   ...
 }: let
   inherit (builtins) readFile;
-  inherit (config.vars) mainUser;
+  inherit (config.vars) mainUser hostName;
   headscale-flake = headscale.packages.${pkgs.system}.headscale;
+
+  clusterIP = (builtins.elemAt config.services.pacemaker.resources.caddy.virtualIps 0).ip;
 in {
   environment.systemPackages = [headscale-flake];
   users.users.${mainUser}.extraGroups = ["headscale"];
@@ -19,7 +21,7 @@ in {
     enable = true;
     package = headscale-flake;
 
-    address = "10.0.0.213";
+    address = clusterIP;
     port = 8085;
 
     settings = {
@@ -36,10 +38,15 @@ in {
       private_key_path = "/var/lib/headscale/private.key";
       noise.private_key_path = "/var/lib/headscale/noise_private.key";
 
-      dns_config = {
+      dns_config = let
+        caddyIp =
+          if hostName == "thingone"
+          then "100.64.0.8"
+          else "100.64.0.9";
+      in {
         magic_dns = false;
         override_local_dns = true;
-        nameservers = ["100.64.0.1"];
+        nameservers = [caddyIp];
       };
 
       derp = {
@@ -47,7 +54,7 @@ in {
 
         server = {
           enabled = true;
-          stun_listen_addr = "0.0.0.0:3479";
+          stun_listen_addr = "${clusterIP}:3479";
           private_key_path = "/var/lib/headscale/derp_server_private.key";
 
           region_id = 995;

devices/cluster/modules/pacemaker/default.nix

@@ -3,42 +3,71 @@
     ./options.nix
     ../corosync.nix
 
+    ../blocky.nix
     ../caddy.nix
+    ../headscale
+    ../unbound.nix
   ];
 
   # TODO: update script
-  services = {
-    pacemaker = {
-      enable = true;
+  services.pacemaker = {
+    enable = true;
 
-      resources = {
-        "caddy" = {
-          enable = true;
-          virtualIps = [
-            {
-              id = "main";
-              interface = "eno1";
-              ip = "10.0.0.130";
-            }
-          ];
-        };
+    resources = {
+      "blocky" = {
+        enable = true;
+        dependsOn = ["unbound"];
+      };
+
+      "caddy" = {
+        enable = true;
+        virtualIps = [
+          {
+            id = "main";
+            interface = "eno1";
+            ip = "10.0.0.130";
+          }
+        ];
+      };
+
+      "headscale" = {
+        enable = true;
+        dependsOn = ["caddy"];
+      };
+
+      "unbound" = {
+        enable = true;
+        dependsOn = ["caddy"];
       };
     };
-
-    rpcbind.enable = true; # needed for NFS
   };
+
+  # NFS client setup
+  services.rpcbind.enable = true;
   boot.supportedFilesystems = ["nfs"];
   environment.systemPackages = with pkgs; [nfs-utils];
 
-  systemd.mounts = [
+  systemd.mounts = let
+    host = "10.0.0.249";
+  in [
     {
       type = "nfs";
       mountConfig = {
         Options = "noatime";
       };
-      what = "servivi:/caddy";
+      what = "${host}:/caddy";
       where = "/var/lib/caddy";
       requiredBy = ["caddy.service"];
     }
+
+    {
+      type = "nfs";
+      mountConfig = {
+        Options = "noatime";
+      };
+      what = "${host}:/headscale";
+      where = "/var/lib/headscale";
+      requiredBy = ["headscale.service"];
+    }
   ];
 }

devices/cluster/modules/unbound.nix

@@ -1,5 +1,5 @@
 {config, ...}: let
-  inherit (config.vars) mainUser;
+  inherit (config.vars) mainUser hostName;
 in {
   # https://github.com/MatthewVance/unbound-docker-rpi/issues/4#issuecomment-1001879602
   boot.kernel.sysctl."net.core.rmem_max" = 1048576;
@@ -27,7 +27,12 @@ in {
           "ota.nelim.org redirect"
           "nelim.org redirect"
         ];
-        local-data = [
+        local-data = let
+          caddyIp =
+            if hostName == "thingone"
+            then "100.64.0.8"
+            else "100.64.0.9";
+        in [
           "\"pve.nelim.org IN A 100.64.0.4\""
 
           "\"headscale.nelim.org. IN A 24.200.126.219\""
@@ -45,9 +50,8 @@ in {
 
           "\"ota.nelim.org. IN A 100.64.0.5\""
 
-          "\"nelim.org IN A 100.64.0.1\""
+          "\"nelim.org 0 A ${caddyIp}\""
         ];
-        #
 
         do-ip4 = true;
         do-ip6 = false;

devices/oksys/default.nix

@@ -1,12 +1,5 @@
 {config, ...}: let
   inherit (config.vars) mainUser hostName;
-  tailscaleNameservers =
-    config
-    .services
-    .headscale
-    .settings
-    .dns_config
-    .nameservers;
 in {
   imports = [
     ./hardware-configuration.nix
@@ -14,10 +7,7 @@ in {
     ../../modules/sshd.nix
     ../../modules/tailscale.nix
 
-    ./modules/blocky.nix
-    ./modules/headscale
     ./modules/remote-builder.nix
-    ./modules/unbound.nix
   ];
 
   vars = {
@@ -43,7 +33,6 @@ in {
   networking = {
     inherit hostName;
     resolvconf.enable = true;
-    nameservers = tailscaleNameservers ++ ["1.0.0.1"];
     firewall.enable = false;
   };