diff --git a/devices/cluster/default.nix b/devices/cluster/default.nix index 6161ac8a..9cad057e 100644 --- a/devices/cluster/default.nix +++ b/devices/cluster/default.nix @@ -1,5 +1,7 @@ deviceName: {config, ...}: let inherit (config.vars) mainUser hostName; + + clusterIP = (builtins.elemAt config.services.pacemaker.resources.caddy.virtualIps 0).ip; in { imports = [ ./hardware-configuration.nix @@ -34,6 +36,10 @@ in { networking = { inherit hostName; resolvconf.enable = true; + nameservers = [ + clusterIP + "1.0.0.1" + ]; firewall.enable = false; }; diff --git a/devices/oksys/modules/blocky.nix b/devices/cluster/modules/blocky.nix similarity index 100% rename from devices/oksys/modules/blocky.nix rename to devices/cluster/modules/blocky.nix diff --git a/devices/cluster/modules/caddy.nix b/devices/cluster/modules/caddy.nix index e191a661..b46218cd 100644 --- a/devices/cluster/modules/caddy.nix +++ b/devices/cluster/modules/caddy.nix @@ -8,6 +8,8 @@ inherit (config.sops) secrets; caddy = caddy-plugins.packages.${pkgs.system}.default; + + clusterIP = (builtins.elemAt config.services.pacemaker.resources.caddy.virtualIps 0).ip; in { imports = [caddy-plugins.nixosModules.default]; @@ -33,7 +35,6 @@ in { dockerIP = "10.0.0.122"; jellyIP = "10.0.0.123"; servivi = "10.0.0.249"; - oksysIP = "10.0.0.213"; in { "nelim.org" = { serverAliases = ["*.nelim.org"]; @@ -48,7 +49,7 @@ in { # Misc one-liners vault.reverseProxy = "${dockerIP}:8781"; hauk.reverseProxy = "${dockerIP}:3003"; - headscale.reverseProxy = "${oksysIP}:8085"; + headscale.reverseProxy = "${clusterIP}:8085"; jelly.reverseProxy = "${jellyIP}:80"; # Resume builder diff --git a/devices/oksys/modules/headscale/completion.bash b/devices/cluster/modules/headscale/completion.bash similarity index 100% rename from devices/oksys/modules/headscale/completion.bash rename to devices/cluster/modules/headscale/completion.bash diff --git a/devices/oksys/modules/headscale/default.nix b/devices/cluster/modules/headscale/default.nix similarity index 77% rename from devices/oksys/modules/headscale/default.nix rename to devices/cluster/modules/headscale/default.nix index 4753eb42..0933dabb 100644 --- a/devices/oksys/modules/headscale/default.nix +++ b/devices/cluster/modules/headscale/default.nix @@ -6,8 +6,10 @@ ... }: let inherit (builtins) readFile; - inherit (config.vars) mainUser; + inherit (config.vars) mainUser hostName; headscale-flake = headscale.packages.${pkgs.system}.headscale; + + clusterIP = (builtins.elemAt config.services.pacemaker.resources.caddy.virtualIps 0).ip; in { environment.systemPackages = [headscale-flake]; users.users.${mainUser}.extraGroups = ["headscale"]; @@ -19,7 +21,7 @@ in { enable = true; package = headscale-flake; - address = "10.0.0.213"; + address = clusterIP; port = 8085; settings = { @@ -36,10 +38,15 @@ in { private_key_path = "/var/lib/headscale/private.key"; noise.private_key_path = "/var/lib/headscale/noise_private.key"; - dns_config = { + dns_config = let + caddyIp = + if hostName == "thingone" + then "100.64.0.8" + else "100.64.0.9"; + in { magic_dns = false; override_local_dns = true; - nameservers = ["100.64.0.1"]; + nameservers = [caddyIp]; }; derp = { @@ -47,7 +54,7 @@ in { server = { enabled = true; - stun_listen_addr = "0.0.0.0:3479"; + stun_listen_addr = "${clusterIP}:3479"; private_key_path = "/var/lib/headscale/derp_server_private.key"; region_id = 995; diff --git a/devices/cluster/modules/pacemaker/default.nix b/devices/cluster/modules/pacemaker/default.nix index 77a2afa4..850c19ac 100644 --- a/devices/cluster/modules/pacemaker/default.nix +++ b/devices/cluster/modules/pacemaker/default.nix @@ -3,42 +3,71 @@ ./options.nix ../corosync.nix + ../blocky.nix ../caddy.nix + ../headscale + ../unbound.nix ]; # TODO: update script - services = { - pacemaker = { - enable = true; + services.pacemaker = { + enable = true; - resources = { - "caddy" = { - enable = true; - virtualIps = [ - { - id = "main"; - interface = "eno1"; - ip = "10.0.0.130"; - } - ]; - }; + resources = { + "blocky" = { + enable = true; + dependsOn = ["unbound"]; + }; + + "caddy" = { + enable = true; + virtualIps = [ + { + id = "main"; + interface = "eno1"; + ip = "10.0.0.130"; + } + ]; + }; + + "headscale" = { + enable = true; + dependsOn = ["caddy"]; + }; + + "unbound" = { + enable = true; + dependsOn = ["caddy"]; }; }; - - rpcbind.enable = true; # needed for NFS }; + + # NFS client setup + services.rpcbind.enable = true; boot.supportedFilesystems = ["nfs"]; environment.systemPackages = with pkgs; [nfs-utils]; - systemd.mounts = [ + systemd.mounts = let + host = "10.0.0.249"; + in [ { type = "nfs"; mountConfig = { Options = "noatime"; }; - what = "servivi:/caddy"; + what = "${host}:/caddy"; where = "/var/lib/caddy"; requiredBy = ["caddy.service"]; } + + { + type = "nfs"; + mountConfig = { + Options = "noatime"; + }; + what = "${host}:/headscale"; + where = "/var/lib/headscale"; + requiredBy = ["headscale.service"]; + } ]; } diff --git a/devices/oksys/modules/unbound.nix b/devices/cluster/modules/unbound.nix similarity index 88% rename from devices/oksys/modules/unbound.nix rename to devices/cluster/modules/unbound.nix index 90729e2e..663a5a41 100644 --- a/devices/oksys/modules/unbound.nix +++ b/devices/cluster/modules/unbound.nix @@ -1,5 +1,5 @@ {config, ...}: let - inherit (config.vars) mainUser; + inherit (config.vars) mainUser hostName; in { # https://github.com/MatthewVance/unbound-docker-rpi/issues/4#issuecomment-1001879602 boot.kernel.sysctl."net.core.rmem_max" = 1048576; @@ -27,7 +27,12 @@ in { "ota.nelim.org redirect" "nelim.org redirect" ]; - local-data = [ + local-data = let + caddyIp = + if hostName == "thingone" + then "100.64.0.8" + else "100.64.0.9"; + in [ "\"pve.nelim.org IN A 100.64.0.4\"" "\"headscale.nelim.org. IN A 24.200.126.219\"" @@ -45,9 +50,8 @@ in { "\"ota.nelim.org. IN A 100.64.0.5\"" - "\"nelim.org IN A 100.64.0.1\"" + "\"nelim.org 0 A ${caddyIp}\"" ]; - # do-ip4 = true; do-ip6 = false; diff --git a/devices/oksys/default.nix b/devices/oksys/default.nix index 1f834223..2cf11686 100644 --- a/devices/oksys/default.nix +++ b/devices/oksys/default.nix @@ -1,12 +1,5 @@ {config, ...}: let inherit (config.vars) mainUser hostName; - tailscaleNameservers = - config - .services - .headscale - .settings - .dns_config - .nameservers; in { imports = [ ./hardware-configuration.nix @@ -14,10 +7,7 @@ in { ../../modules/sshd.nix ../../modules/tailscale.nix - ./modules/blocky.nix - ./modules/headscale ./modules/remote-builder.nix - ./modules/unbound.nix ]; vars = { @@ -43,7 +33,6 @@ in { networking = { inherit hostName; resolvconf.enable = true; - nameservers = tailscaleNameservers ++ ["1.0.0.1"]; firewall.enable = false; };