test: separate app and sandbox

Signed-off-by: Ophestra <cat@gensokyo.uk>

10 files changed, 239 insertions, 49 deletions

diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml
index 0ca93ab1..dcf71fde 100644
--- a/.gitea/workflows/test.yml
+++ b/.gitea/workflows/test.yml
@@ -22,37 +22,71 @@ jobs:
           path: result/*
           retention-days: 1
 
-  fpkg:
-    name: Fpkg
+  race:
+    name: Fortify (race detector)
     runs-on: nix
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Run NixOS test
-        run: nix build --out-link "result" --print-out-paths --print-build-logs .#checks.x86_64-linux.fpkg
+        run: nix build --out-link "result" --print-out-paths --print-build-logs .#checks.x86_64-linux.race
 
       - name: Upload test output
         uses: actions/upload-artifact@v3
         with:
-          name: "fpkg-vm-output"
+          name: "fortify-race-vm-output"
           path: result/*
           retention-days: 1
 
-  race:
-    name: Data race detector
+  sandbox:
+    name: Sandbox
     runs-on: nix
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Run NixOS test
-        run: nix build --out-link "result" --print-out-paths --print-build-logs .#checks.x86_64-linux.race
+        run: nix build --out-link "result" --print-out-paths --print-build-logs .#checks.x86_64-linux.sandbox
 
       - name: Upload test output
         uses: actions/upload-artifact@v3
         with:
-          name: "fortify-race-vm-output"
+          name: "sandbox-vm-output"
+          path: result/*
+          retention-days: 1
+
+  sandbox-race:
+    name: Sandbox (race detector)
+    runs-on: nix
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run NixOS test
+        run: nix build --out-link "result" --print-out-paths --print-build-logs .#checks.x86_64-linux.sandbox-race
+
+      - name: Upload test output
+        uses: actions/upload-artifact@v3
+        with:
+          name: "sandbox-race-vm-output"
+          path: result/*
+          retention-days: 1
+
+  fpkg:
+    name: Fpkg
+    runs-on: nix
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run NixOS test
+        run: nix build --out-link "result" --print-out-paths --print-build-logs .#checks.x86_64-linux.fpkg
+
+      - name: Upload test output
+        uses: actions/upload-artifact@v3
+        with:
+          name: "fpkg-vm-output"
           path: result/*
           retention-days: 1
 
@@ -60,8 +94,10 @@ jobs:
     name: Flake checks
     needs:
       - fortify
-      - fpkg
       - race
+      - sandbox
+      - sandbox-race
+      - fpkg
     runs-on: nix
     steps:
       - name: Checkout
diff --git a/flake.nix b/flake.nix
index 1d251ab2..93e21bec 100644
--- a/flake.nix
+++ b/flake.nix
@@ -58,12 +58,19 @@
         in
         {
           fortify = callPackage ./test { inherit system self; };
-          fpkg = callPackage ./cmd/fpkg/test { inherit system self; };
           race = callPackage ./test {
             inherit system self;
             withRace = true;
           };
 
+          sandbox = callPackage ./test/sandbox { inherit self; };
+          sandbox-race = callPackage ./test/sandbox {
+            inherit self;
+            withRace = true;
+          };
+
+          fpkg = callPackage ./cmd/fpkg/test { inherit system self; };
+
           formatting = runCommandLocal "check-formatting" { nativeBuildInputs = [ nixfmt-rfc-style ]; } ''
             cd ${./.}
 
diff --git a/test/configuration.nix b/test/configuration.nix
index 96b87a9d..8afe0cac 100644
--- a/test/configuration.nix
+++ b/test/configuration.nix
@@ -4,10 +4,6 @@
   config,
   ...
 }:
-let
-  testProgram = pkgs.callPackage ./sandbox/tool/package.nix { inherit (config.environment.fortify.package) version; };
-  testCases = import ./sandbox/case lib testProgram;
-in
 {
   users.users = {
     alice = {
@@ -41,9 +37,6 @@ in
       # For D-Bus tests:
       libnotify
       mako
-
-      # For checking seccomp outcome:
-      testProgram
     ];
 
     variables = {
@@ -109,10 +102,6 @@ in
     home-manager = _: _: { home.stateVersion = "23.05"; };
 
     apps = [
-      testCases.preset
-      testCases.tty
-      testCases.mapuid
-
       {
         name = "ne-foot";
         verbose = true;
diff --git a/test/sandbox/case/mapuid.nix b/test/sandbox/case/mapuid.nix
index bf9837d4..ea3d5b20 100644
--- a/test/sandbox/case/mapuid.nix
+++ b/test/sandbox/case/mapuid.nix
@@ -97,7 +97,6 @@
         "pki" = fs "80001ff" null null;
         "polkit-1" = fs "80001ff" null null;
         "profile" = fs "80001ff" null null;
-        "profiles" = fs "80001ff" null null;
         "protocols" = fs "80001ff" null null;
         "resolv.conf" = fs "80001ff" null null;
         "resolvconf.conf" = fs "80001ff" null null;
diff --git a/test/sandbox/case/preset.nix b/test/sandbox/case/preset.nix
index cedd9d79..6e83b726 100644
--- a/test/sandbox/case/preset.nix
+++ b/test/sandbox/case/preset.nix
@@ -97,7 +97,6 @@
         "pki" = fs "80001ff" null null;
         "polkit-1" = fs "80001ff" null null;
         "profile" = fs "80001ff" null null;
-        "profiles" = fs "80001ff" null null;
         "protocols" = fs "80001ff" null null;
         "resolv.conf" = fs "80001ff" null null;
         "resolvconf.conf" = fs "80001ff" null null;
diff --git a/test/sandbox/case/tty.nix b/test/sandbox/case/tty.nix
index f295b926..19c28968 100644
--- a/test/sandbox/case/tty.nix
+++ b/test/sandbox/case/tty.nix
@@ -98,7 +98,6 @@
         "pki" = fs "80001ff" null null;
         "polkit-1" = fs "80001ff" null null;
         "profile" = fs "80001ff" null null;
-        "profiles" = fs "80001ff" null null;
         "protocols" = fs "80001ff" null null;
         "resolv.conf" = fs "80001ff" null null;
         "resolvconf.conf" = fs "80001ff" null null;
diff --git a/test/sandbox/configuration.nix b/test/sandbox/configuration.nix
new file mode 100644
index 00000000..5bca7fed
--- /dev/null
+++ b/test/sandbox/configuration.nix
@@ -0,0 +1,76 @@
+{
+  lib,
+  pkgs,
+  config,
+  ...
+}:
+let
+  testProgram = pkgs.callPackage ./tool/package.nix { inherit (config.environment.fortify.package) version; };
+  testCases = import ./case lib testProgram;
+in
+{
+  users.users = {
+    alice = {
+      isNormalUser = true;
+      description = "Alice Foobar";
+      password = "foobar";
+      uid = 1000;
+    };
+  };
+
+  home-manager.users.alice.home.stateVersion = "24.11";
+
+  # Automatically login on tty1 as a normal user:
+  services.getty.autologinUser = "alice";
+
+  environment = {
+    systemPackages = with pkgs; [
+      # For checking seccomp outcome:
+      testProgram
+    ];
+
+    variables = {
+      SWAYSOCK = "/tmp/sway-ipc.sock";
+      WLR_RENDERER = "pixman";
+    };
+  };
+
+  # Automatically configure and start Sway when logging in on tty1:
+  programs.bash.loginShellInit = ''
+    if [ "$(tty)" = "/dev/tty1" ]; then
+      set -e
+
+      mkdir -p ~/.config/sway
+      (sed s/Mod4/Mod1/ /etc/sway/config &&
+      echo 'output * bg ${pkgs.nixos-artwork.wallpapers.simple-light-gray.gnomeFilePath} fill' &&
+      echo 'output Virtual-1 res 1680x1050') > ~/.config/sway/config
+
+      sway --validate
+      systemd-cat --identifier=session sway && touch /tmp/sway-exit-ok
+    fi
+  '';
+
+  programs.sway.enable = true;
+
+  virtualisation.qemu.options = [
+    # Need to switch to a different GPU driver than the default one (-vga std) so that Sway can launch:
+    "-vga none -device virtio-gpu-pci"
+
+    # Increase performance:
+    "-smp 8"
+  ];
+
+  environment.fortify = {
+    enable = true;
+    stateDir = "/var/lib/fortify";
+    users.alice = 0;
+
+    home-manager = _: _: { home.stateVersion = "23.05"; };
+
+    apps = [
+      testCases.preset
+      testCases.tty
+      testCases.mapuid
+    ];
+  };
+}
diff --git a/test/sandbox/default.nix b/test/sandbox/default.nix
new file mode 100644
index 00000000..6ffc4fd5
--- /dev/null
+++ b/test/sandbox/default.nix
@@ -0,0 +1,39 @@
+{
+  lib,
+  nixosTest,
+
+  self,
+  withRace ? false,
+}:
+
+nixosTest {
+  name = "fortify-sandbox" + (if withRace then "-race" else "");
+  nodes.machine =
+    { options, pkgs, ... }:
+    {
+      # Run with Go race detector:
+      environment.fortify = lib.mkIf withRace rec {
+        # race detector does not support static linking
+        package = (pkgs.callPackage ../../package.nix { }).overrideAttrs (previousAttrs: {
+          GOFLAGS = previousAttrs.GOFLAGS ++ [ "-race" ];
+        });
+        fsuPackage = options.environment.fortify.fsuPackage.default.override { fortify = package; };
+      };
+
+      imports = [
+        ./configuration.nix
+
+        self.nixosModules.fortify
+        self.inputs.home-manager.nixosModules.home-manager
+      ];
+    };
+
+  # adapted from nixos sway integration tests
+
+  # testScriptWithTypes:49: error: Cannot call function of unknown type
+  #           (machine.succeed if succeed else machine.execute)(
+  #           ^
+  # Found 1 error in 1 file (checked 1 source file)
+  skipTypeCheck = true;
+  testScript = builtins.readFile ./test.py;
+}
diff --git a/test/sandbox/test.py b/test/sandbox/test.py
new file mode 100644
index 00000000..4bd7d20a
--- /dev/null
+++ b/test/sandbox/test.py
@@ -0,0 +1,71 @@
+import json
+import shlex
+
+q = shlex.quote
+
+
+def swaymsg(command: str = "", succeed=True, type="command"):
+    assert command != "" or type != "command", "Must specify command or type"
+    shell = q(f"swaymsg -t {q(type)} -- {q(command)}")
+    with machine.nested(
+            f"sending swaymsg {shell!r}" + " (allowed to fail)" * (not succeed)
+    ):
+        ret = (machine.succeed if succeed else machine.execute)(
+            f"su - alice -c {shell}"
+        )
+
+    # execute also returns a status code, but disregard.
+    if not succeed:
+        _, ret = ret
+
+    if not succeed and not ret:
+        return None
+
+    parsed = json.loads(ret)
+    return parsed
+
+
+start_all()
+machine.wait_for_unit("multi-user.target")
+
+# To check fortify's version:
+print(machine.succeed("sudo -u alice -i fortify version"))
+
+# Wait for Sway to complete startup:
+machine.wait_for_file("/run/user/1000/wayland-1")
+machine.wait_for_file("/tmp/sway-ipc.sock")
+
+# Check seccomp outcome:
+swaymsg("exec fortify run cat")
+pid = int(machine.wait_until_succeeds("pgrep -U 1000000 -x cat", timeout=5))
+print(machine.succeed(f"fortify-test filter {pid} c698b081ff957afe17a6d94374537d37f2a63f6f9dd75da7546542407a9e32476ebda3312ba7785d7f618542bcfaf27ca27dcc2dddba852069d28bcfe8cad39a &>/dev/stdout", timeout=5))
+machine.succeed(f"kill -TERM {pid}")
+
+# Verify capabilities/securebits in user namespace:
+print(machine.succeed("sudo -u alice -i fortify run capsh --print"))
+print(machine.succeed("sudo -u alice -i fortify run capsh --has-no-new-privs"))
+print(machine.fail("sudo -u alice -i fortify run capsh --has-a=CAP_SYS_ADMIN"))
+print(machine.fail("sudo -u alice -i fortify run capsh --has-b=CAP_SYS_ADMIN"))
+print(machine.fail("sudo -u alice -i fortify run capsh --has-i=CAP_SYS_ADMIN"))
+print(machine.fail("sudo -u alice -i fortify run capsh --has-p=CAP_SYS_ADMIN"))
+print(machine.fail("sudo -u alice -i fortify run umount -R /dev"))
+
+# Check sandbox outcome:
+check_offset = 0
+def check_sandbox(name):
+    global check_offset
+    check_offset += 1
+    swaymsg(f"exec script /dev/null -E always -qec check-sandbox-{name}")
+    machine.wait_for_file(f"/tmp/fortify.1000/tmpdir/{check_offset}/sandbox-ok", timeout=15)
+
+
+check_sandbox("preset")
+check_sandbox("tty")
+check_sandbox("mapuid")
+
+# Exit Sway and verify process exit status 0:
+swaymsg("exit", succeed=False)
+machine.wait_for_file("/tmp/sway-exit-ok")
+
+# Print fortify runDir contents:
+print(machine.succeed("find /run/user/1000/fortify"))
diff --git a/test/test.py b/test/test.py
index baf074e2..0ab8970c 100644
--- a/test/test.py
+++ b/test/test.py
@@ -99,39 +99,14 @@ print(denyOutputVerbose)
 # Fail direct fsu call:
 print(machine.fail("sudo -u alice -i fsu"))
 
-# Check seccomp outcome:
-swaymsg("exec fortify run cat")
-pid = int(machine.wait_until_succeeds("pgrep -U 1000000 -x cat", timeout=5))
-print(machine.succeed(f"fortify-test filter {pid} c698b081ff957afe17a6d94374537d37f2a63f6f9dd75da7546542407a9e32476ebda3312ba7785d7f618542bcfaf27ca27dcc2dddba852069d28bcfe8cad39a &>/dev/stdout", timeout=5))
-machine.succeed(f"kill -TERM {pid}")
-
-# Verify capabilities/securebits in user namespace:
-print(machine.succeed("sudo -u alice -i fortify run capsh --print"))
-print(machine.succeed("sudo -u alice -i fortify run capsh --has-no-new-privs"))
-print(machine.fail("sudo -u alice -i fortify run capsh --has-a=CAP_SYS_ADMIN"))
-print(machine.fail("sudo -u alice -i fortify run capsh --has-b=CAP_SYS_ADMIN"))
-print(machine.fail("sudo -u alice -i fortify run capsh --has-i=CAP_SYS_ADMIN"))
-print(machine.fail("sudo -u alice -i fortify run capsh --has-p=CAP_SYS_ADMIN"))
-print(machine.fail("sudo -u alice -i fortify run umount -R /dev"))
-
 # Verify PrintBaseError behaviour:
 if denyOutput != "fsu: uid 1001 is not in the fsurc file\n":
     raise Exception(f"unexpected deny output:\n{denyOutput}")
 if denyOutputVerbose != "fsu: uid 1001 is not in the fsurc file\nfortify: *cannot obtain uid from fsu: permission denied\n":
     raise Exception(f"unexpected deny verbose output:\n{denyOutputVerbose}")
 
-# Check sandbox outcome:
 check_offset = 0
-def check_sandbox(name):
-    global check_offset
-    check_offset += 1
-    swaymsg(f"exec script /dev/null -E always -qec check-sandbox-{name}")
-    machine.wait_for_file(f"/tmp/fortify.1000/tmpdir/{check_offset}/sandbox-ok", timeout=15)
-
 
-check_sandbox("preset")
-check_sandbox("tty")
-check_sandbox("mapuid")
 
 def aid(offset):
     return 1+check_offset+offset