Vesuvius dnsmasq netboot

flake.nix

@@ -55,8 +55,9 @@
 
         NIX_SSHOPTS="-o ForwardAgent=yes -J acidburn.vtluug.org" \
         ${pkgs.nixos-rebuild}/bin/nixos-rebuild switch \
-          --fast --flake ".#$TARGET_HOST_NAME" \
-          --use-remote-sudo \
+          --flake ".#$TARGET_HOST_NAME" \
+          --no-reexec \
+          --sudo \
           --target-host "papatux@$TARGET_HOST_ADDRESS" \
           --build-host "papatux@$TARGET_HOST_ADDRESS"
       '';

hosts/bastille/blade-names.nix

@@ -1,16 +1,18 @@
-# keep-sorted start
-[
-  "backbiter"
-  "damocles"
-  "durendal"
-  "eyelander"
-  "excalibur"
-  "gram"
-  "gryffindor"
-  "kusanagi"
-  "narsil"
-  "oathbringer"
-  "riptide"
-  "sting"
-]
-# keep-sorted end
+{
+  # TODO: prospit's a special case and won't remain here forever
+  "d8:9e:f3:3e:f9:41" = "prospit";
+
+  "40:f2:e9:c6:65:5f" = "backbiter";
+  "40:f2:e9:c6:69:43" = "damocles";
+  "40:f2:e9:c6:69:67" = "durendal";
+  "40:f2:e9:c6:74:59" = "eyelander";
+  "40:f2:e9:c6:75:f1" = "excalibur";
+  "40:f2:e9:c6:76:21" = "gram";
+
+  "unassigned-0" = "gryffindor";
+  "unassigned-1" = "kusanagi";
+  "unassigned-2" = "narsil";
+  "unassigned-3" = "oathbringer";
+  "unassigned-4" = "riptide";
+  "unassigned-5" = "sting";
+}

hosts/bastille/blade.nix

@@ -0,0 +1,36 @@
+{ modulesPath, pkgs, lib, ... }: {
+  imports = [
+    ./eno1-imm-disable.nix
+    (import ../common/k3s.nix { inherit lib; })
+    ../common/nix.nix
+    ../common/sshd.nix
+    ../common/users-local.nix
+    (modulesPath + "/installer/netboot/netboot-minimal.nix")
+  ];
+
+  # Get hostname from DHCP request
+  networking.hostName = "";
+
+  # Open kubernetes' ports for flannel and API server
+  networking.firewall = {
+    allowedTCPPorts = [
+      6443
+      10250
+    ];
+    allowedUDPPorts = [
+      8472
+    ];
+  };
+
+
+  # when making the ISO, the initialHashedPassword is set to "" for some reason
+  # we already set a hashed password, so null this
+  users.users.root.initialHashedPassword = lib.mkForce null;
+
+  environment.systemPackages = [
+    pkgs.fastfetch
+    pkgs.git
+  ];
+
+  system.stateVersion = "25.11";
+}

hosts/bastille/eno1-imm-disable.nix

@@ -0,0 +1,30 @@
+{ pkgs, lib, ... }:
+let
+  eno1-imm-disable = pkgs.writeShellApplication {
+    name = "eno1-imm-disable";
+
+    runtimeInputs = [
+      pkgs.iproute2
+    ];
+
+    text = ''
+      if grep "Lenovo NeXtScale nx360 M5" /sys/devices/virtual/dmi/id/product_name; then
+        ip link set down eno1
+      fi
+    '';
+  };
+in {
+  systemd.services."eno1-imm-disable" = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+
+    unitConfig = {
+      Description = "Disable eno1 on Lenovo NeXtScale nodes to avoid issues with using the imm interface";
+    };
+
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = "${lib.getExe eno1-imm-disable}";
+    };
+  };
+}

hosts/common/k3s.nix

@@ -0,0 +1,39 @@
+{
+  lib,
+  role ? "agent",
+  clusterInit ? false,
+  serverAddr ? "10.98.3.2",
+  flannelIface ? "enp1s0f1",
+}:
+{
+  networking.firewall.allowedTCPPorts = [
+    6443
+  ];
+
+  networking.firewall.allowedUDPPorts = [
+    8472
+  ];
+
+  services.k3s = {
+    inherit role clusterInit;
+
+    enable = true;
+    serverAddr = lib.mkIf (role != "server") "https://${serverAddr}:6443";
+    nodeIP = lib.mkIf (role == "server") serverAddr;
+
+    extraFlags = [
+      "--token=\"garbage secret\""
+    ]
+    ++ lib.optionals (role == "server") [
+      "--kubelet-arg=node-ip=${serverAddr}"
+      "--flannel-iface=${flannelIface}"
+      "--advertise-address=${serverAddr}"
+      "--bind-address=${serverAddr}"
+      "--write-kubeconfig-mode=0640"
+      "--write-kubeconfig-group=wheel"
+    ];
+    extraKubeletConfig = lib.mkIf (role == "server") {
+      address = serverAddr;
+    };
+  };
+}

hosts/common/nfs.nix

@@ -0,0 +1,20 @@
+{ config, pkgs, ... }:
+let
+  mkNfs = {path, options ? [ "vers=4.0" "soft" "nodev" "nosuid" ]}: {
+    device = "${path}";
+    fsType = "nfs";
+    inherit options;
+  };
+in
+{
+  environment.systemPackages = [ pkgs.nfs-utils ];
+
+  fileSystems."/nfs/cistern/share" = mkNfs {path = "10.98.0.7:/cistern/nfs/share";};
+  fileSystems."/nfs/cistern/files" = mkNfs {path = "10.98.0.7:/cistern/nfs/files";};
+  fileSystems."/nfs/cistern/home" = mkNfs {
+    path = "10.98.0.7:/cistern/nfs/home";
+    options = [ "vers=4.0" "soft" "nodev" "nosuid" ];
+  };
+  fileSystems."/nfs/cistern/libvirt" = mkNfs {path = "10.98.0.7:/cistern/nfs/libvirt";};
+  fileSystems."/nfs/cistern/docker/data" = mkNfs {path = "10.98.0.7:/cistern/nfs/docker/data";};
+}

hosts/vesuvius/README.md

@@ -10,3 +10,13 @@ Giant storage server + future LHCPISCSIPXEIDK thing maybe?
 ## Storage
 We currently have one (manually created) RAID-Z2 pool mounted at `/forge` with `8` drives of `12 Tb` each.
 We have capacity for `48`(!) drives, but still only paper (and tape) caddies.
+
+```
+# for the nix store
+zfs create -o mountpoint=legacy \
+           -o compression=zstd \
+           -o xattr=sa \
+           -o acltype=posixacl \
+           -o atime=off \
+           forge/nix
+```

hosts/vesuvius/caddy.nix

@@ -0,0 +1,81 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  gandi-key-path = "/secrets/gandi.env";
+in
+{
+  age.secrets."gandi.env".file = ../../secrets/vesuvius/gandi.env.age;
+
+  containers.caddy-proxy = {
+    autoStart = true;
+    ephemeral = true;
+    macvlans = [ "eno0" ];
+    privateNetwork = false;
+    bindMounts = {
+      "${gandi-key-path}" = {
+        hostPath = config.age.secrets."gandi.env".path;
+      };
+    };
+    config =
+      {
+        config,
+        lib,
+        pkgs,
+        ...
+      }:
+      {
+        networking.interfaces.mv-eno0 = {
+          useDHCP = true;
+          ipv4.addresses = [
+            {
+              address = "128.173.89.163";
+              prefixLength = 24;
+            }
+          ];
+          ipv6.addresses = [
+            {
+              address = "2607:b400:6:cc80:0:aff:fe62:f";
+              prefixLength = 64;
+            }
+          ];
+        };
+
+        # Force container to get DNS settings from network
+        networking.useHostResolvConf = false;
+
+        services.caddy = {
+          enable = true;
+          virtualHosts."*.vtluug.org".extraConfig = ''
+            reverse_proxy svc.bastille.vtluug.org:80 {
+              header_up Host {labels.2}.svc.bastille.vtluug.org
+            }
+          '';
+          package = pkgs.caddy.withPlugins {
+            plugins = [ "github.com/caddy-dns/gandi@v1.1.0" ];
+            hash = "sha256-5mjD0CY7f5+sRtV1rXysj8PvId2gQaWiXlIaTg2Lv8A=";
+          };
+          globalConfig = ''
+            acme_dns gandi {env.GANDI_AUTH_TOKEN}
+          '';
+        };
+        systemd.services.caddy.serviceConfig.EnvironmentFile = [ "${gandi-key-path}" ];
+
+        networking.firewall = {
+          allowedTCPPorts = [
+            80
+            443
+          ];
+          allowedUDPPorts = [
+            80
+            443
+          ];
+        };
+
+        system.stateVersion = "26.05";
+      };
+  };
+}