From 8ddc826bd02ff0d58123e1af5de2f48ae0dbc7c9 Mon Sep 17 00:00:00 2001 From: Ophestra Date: Thu, 1 Oct 2026 22:34:51 +0900 Subject: test/sandbox: migrate tests This benefits even more than the cmd/sharefs test suite, the slow python-based test script was a major bottleneck. Replacing the nix-represented test cases with compound literals also significantly increases readability. Signed-off-by: Ophestra --- .gitea/workflows/test.yml | 2 +- internal/workflows/doc.go | 15 +- internal/workflows/step.go | 18 +- internal/workflows/test.go | 7 +- test/flake.nix | 55 ------ test/interactive/configuration.nix | 68 ------- test/interactive/hakurei.nix | 47 ----- test/interactive/raceattr.nix | 30 --- test/interactive/trace.nix | 35 ---- test/interactive/vm.nix | 56 ------ test/internal/testsuite/proc.go | 348 +++++++++++++++++++++++++++++++++++ test/internal/testsuite/proc_test.go | 33 ++++ test/internal/testsuite/testsuite.go | 45 +++++ test/sandbox/main.go | 83 +++++++++ test/sandbox/ptrace.go | 153 +++++++++++++++ test/sandbox/seccomp.patch | 18 ++ test/sandbox/sum_amd64.go | 7 + test/sharefs/main.go | 6 +- 18 files changed, 718 insertions(+), 308 deletions(-) delete mode 100644 test/interactive/configuration.nix delete mode 100644 test/interactive/hakurei.nix delete mode 100644 test/interactive/raceattr.nix delete mode 100644 test/interactive/trace.nix delete mode 100644 test/interactive/vm.nix create mode 100644 test/internal/testsuite/proc.go create mode 100644 test/internal/testsuite/proc_test.go create mode 100644 test/sandbox/main.go create mode 100644 test/sandbox/ptrace.go create mode 100644 test/sandbox/seccomp.patch create mode 100644 test/sandbox/sum_amd64.go diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index b42ba205..52ae291a 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -1 +1 @@ -{"name":"Test","on":["push"],"jobs":{"hakurei":{"name":"Hakurei","runs-on":"nix","steps":[{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Run NixOS test","run":"nix build --out-link result --print-out-paths --print-build-logs ./test#checks.x86_64-linux.hakurei"},{"name":"Upload test output","uses":"actions/upload-artifact@v3","with":{"name":"hakurei-vm-output","path":"result/*","retention-days":1}}]},"race":{"name":"Hakurei (race detector)","runs-on":"nix","steps":[{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Run NixOS test","run":"nix build --out-link result --print-out-paths --print-build-logs ./test#checks.x86_64-linux.race"},{"name":"Upload test output","uses":"actions/upload-artifact@v3","with":{"name":"hakurei-race-vm-output","path":"result/*","retention-days":1}}]},"sandbox":{"name":"Sandbox","runs-on":"nix","steps":[{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Run NixOS test","run":"nix build --out-link result --print-out-paths --print-build-logs ./test#checks.x86_64-linux.sandbox"},{"name":"Upload test output","uses":"actions/upload-artifact@v3","with":{"name":"sandbox-vm-output","path":"result/*","retention-days":1}}]},"sandbox-race":{"name":"Sandbox (race detector)","runs-on":"nix","steps":[{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Run NixOS test","run":"nix build --out-link result --print-out-paths --print-build-logs ./test#checks.x86_64-linux.sandbox-race"},{"name":"Upload test output","uses":"actions/upload-artifact@v3","with":{"name":"sandbox-race-vm-output","path":"result/*","retention-days":1}}]},"sharefs":{"name":"ShareFS","runs-on":"rosa","steps":[{"name":"Fix container filesystem","run":"rm /var/run \u0026\u0026 ln -sf ../run /var"},{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Set up Go toolchain","uses":"actions/setup-go@v6","with":{"go-version-file":"go.mod"}},{"name":"install fuse and fs_mark","run":"apt-get update \u0026\u0026 apt-get install -y fuse3 fsmark"},{"name":"Request distribution","id":"dist","run":"HAKUREI_REV=\"$(git rev-parse --short HEAD)\" \u0026\u0026 /rosa/bin/mbf ci dist -o result . \"$(cat cmd/dist/VERSION)-$HAKUREI_REV\" \u0026\u0026 echo \"rev=$HAKUREI_REV\" \u003e\u003e \"$GITHUB_OUTPUT\""},{"name":"Install hakurei","run":"HAKUREI_VERSION=\"$(cat cmd/dist/VERSION)-${{ steps.dist.outputs.rev }}\" \u0026\u0026 tar xf \"result/hakurei-$HAKUREI_VERSION-amd64.tar.gz\" \u0026\u0026 \"./hakurei-$HAKUREI_VERSION-amd64/install.sh\" \u0026\u0026 sudo -u ubuntu hakurei version"},{"name":"Mount sharefs","run":"useradd -ru 1023 -md /var/lib/sdcard -k /var/empty -s /sbin/nologin media_rw \u0026\u0026 install -dm0 /sdcard \u0026\u0026 sharefs -o rw,noexec,nosuid,nodev,noatime,allow_other,mkdir,source=/var/lib/sdcard,setuid=1023,setgid=1023 /sdcard"},{"name":"Compile and run test suite","run":"sharefs -V \u0026\u0026 rm -rf result \u0026\u0026 go run -tags=testsuite ./test/sharefs ubuntu"},{"name":"Upload test output","uses":"actions/upload-artifact@v3","with":{"name":"fs_mark","path":"result/*","retention-days":1}}]},"check":{"name":"Flake checks","needs":["hakurei","race","sandbox","sandbox-race"],"runs-on":"nix","steps":[{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Run checks","run":"nix --print-build-logs --experimental-features 'nix-command flakes' flake check ./test"}]},"dist":{"name":"Create distribution","runs-on":"rosa","steps":[{"name":"Fix container filesystem","run":"rm /var/run \u0026\u0026 ln -sf ../run /var"},{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Request distribution","id":"dist-test","run":"HAKUREI_REV=\"$(git rev-parse --short HEAD)\" \u0026\u0026 /rosa/bin/mbf ci dist -o result . \"$(cat cmd/dist/VERSION)-$HAKUREI_REV\" \u0026\u0026 echo \"rev=$HAKUREI_REV\" \u003e\u003e \"$GITHUB_OUTPUT\""},{"name":"Upload distribution","uses":"actions/upload-artifact@v3","with":{"name":"hakurei-${{ steps.dist-test.outputs.rev }}","path":"result/*","retention-days":1}}]}}} +{"name":"Test","on":["push"],"jobs":{"hakurei":{"name":"Hakurei","runs-on":"nix","steps":[{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Run NixOS test","run":"nix build --out-link result --print-out-paths --print-build-logs ./test#checks.x86_64-linux.hakurei"},{"name":"Upload test output","uses":"actions/upload-artifact@v3","with":{"name":"hakurei-vm-output","path":"result/*","retention-days":1}}]},"race":{"name":"Hakurei (race detector)","runs-on":"nix","steps":[{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Run NixOS test","run":"nix build --out-link result --print-out-paths --print-build-logs ./test#checks.x86_64-linux.race"},{"name":"Upload test output","uses":"actions/upload-artifact@v3","with":{"name":"hakurei-race-vm-output","path":"result/*","retention-days":1}}]},"sandbox":{"name":"Sandbox","runs-on":"rosa","steps":[{"name":"Fix container filesystem","run":"rm /var/run \u0026\u0026 ln -sf ../run /var"},{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Set up Go toolchain","uses":"actions/setup-go@v6","with":{"go-version-file":"go.mod"}},{"name":"Request distribution","id":"dist","run":"HAKUREI_REV=\"$(git rev-parse --short HEAD)\" \u0026\u0026 /rosa/bin/mbf ci dist -o result . \"$(cat cmd/dist/VERSION)-$HAKUREI_REV\" \u0026\u0026 echo \"rev=$HAKUREI_REV\" \u003e\u003e \"$GITHUB_OUTPUT\""},{"name":"Install hakurei","run":"HAKUREI_VERSION=\"$(cat cmd/dist/VERSION)-${{ steps.dist.outputs.rev }}\" \u0026\u0026 tar xf result/hakurei-$HAKUREI_VERSION*-amd64.tar.gz \u0026\u0026 ./hakurei-$HAKUREI_VERSION*-amd64/install.sh \u0026\u0026 sudo -u ubuntu hakurei version \u0026\u0026 echo 'Defaults closefrom_override' \u003e /etc/sudoers.d/closefrom_override \u0026\u0026 mkdir /var/empty"},{"name":"Compile and run test suite","run":"rm -rf result \u0026\u0026 go run -tags=testsuite ./test/sandbox ubuntu"}]},"sandbox-race":{"name":"Sandbox (with race instrument)","runs-on":"rosa","steps":[{"name":"Fix container filesystem","run":"rm /var/run \u0026\u0026 ln -sf ../run /var"},{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Set up Go toolchain","uses":"actions/setup-go@v6","with":{"go-version-file":"go.mod"}},{"name":"Request distribution","id":"dist","run":"HAKUREI_REV=\"$(git rev-parse --short HEAD)\" \u0026\u0026 /rosa/bin/mbf ci race -o result . \"$(cat cmd/dist/VERSION)-$HAKUREI_REV\" \u0026\u0026 echo \"rev=$HAKUREI_REV\" \u003e\u003e \"$GITHUB_OUTPUT\""},{"name":"Install hakurei","run":"HAKUREI_VERSION=\"$(cat cmd/dist/VERSION)-${{ steps.dist.outputs.rev }}\" \u0026\u0026 tar xf result/hakurei-$HAKUREI_VERSION*-amd64.tar.gz \u0026\u0026 ./hakurei-$HAKUREI_VERSION*-amd64/install.sh \u0026\u0026 sudo -u ubuntu hakurei version \u0026\u0026 echo 'Defaults closefrom_override' \u003e /etc/sudoers.d/closefrom_override \u0026\u0026 mkdir /var/empty"},{"name":"Compile and run test suite","run":"rm -rf result \u0026\u0026 go run -tags=testsuite ./test/sandbox ubuntu"}]},"sharefs":{"name":"ShareFS","runs-on":"rosa","steps":[{"name":"Fix container filesystem","run":"rm /var/run \u0026\u0026 ln -sf ../run /var"},{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Set up Go toolchain","uses":"actions/setup-go@v6","with":{"go-version-file":"go.mod"}},{"name":"install fuse and fs_mark","run":"apt-get update \u0026\u0026 apt-get install -y fuse3 fsmark"},{"name":"Request distribution","id":"dist","run":"HAKUREI_REV=\"$(git rev-parse --short HEAD)\" \u0026\u0026 /rosa/bin/mbf ci dist -o result . \"$(cat cmd/dist/VERSION)-$HAKUREI_REV\" \u0026\u0026 echo \"rev=$HAKUREI_REV\" \u003e\u003e \"$GITHUB_OUTPUT\""},{"name":"Install hakurei","run":"HAKUREI_VERSION=\"$(cat cmd/dist/VERSION)-${{ steps.dist.outputs.rev }}\" \u0026\u0026 tar xf result/hakurei-$HAKUREI_VERSION*-amd64.tar.gz \u0026\u0026 ./hakurei-$HAKUREI_VERSION*-amd64/install.sh \u0026\u0026 sudo -u ubuntu hakurei version \u0026\u0026 echo 'Defaults closefrom_override' \u003e /etc/sudoers.d/closefrom_override \u0026\u0026 mkdir /var/empty"},{"name":"Mount sharefs","run":"useradd -ru 1023 -md /var/lib/sdcard -k /var/empty -s /sbin/nologin media_rw \u0026\u0026 install -dm0 /sdcard \u0026\u0026 sharefs -o rw,noexec,nosuid,nodev,noatime,allow_other,mkdir,source=/var/lib/sdcard,setuid=1023,setgid=1023 /sdcard"},{"name":"Compile and run test suite","run":"sharefs -V \u0026\u0026 rm -rf result \u0026\u0026 go run -tags=testsuite ./test/sharefs ubuntu"},{"name":"Upload test output","uses":"actions/upload-artifact@v3","with":{"name":"fs_mark","path":"result/*","retention-days":1}}]},"check":{"name":"Flake checks","needs":["hakurei","race"],"runs-on":"nix","steps":[{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Run checks","run":"nix --print-build-logs --experimental-features 'nix-command flakes' flake check ./test"}]},"dist":{"name":"Create distribution","runs-on":"rosa","steps":[{"name":"Fix container filesystem","run":"rm /var/run \u0026\u0026 ln -sf ../run /var"},{"name":"Checkout","uses":"actions/checkout@v4"},{"name":"Request distribution","id":"dist-test","run":"HAKUREI_REV=\"$(git rev-parse --short HEAD)\" \u0026\u0026 /rosa/bin/mbf ci dist -o result . \"$(cat cmd/dist/VERSION)-$HAKUREI_REV\" \u0026\u0026 echo \"rev=$HAKUREI_REV\" \u003e\u003e \"$GITHUB_OUTPUT\""},{"name":"Upload distribution","uses":"actions/upload-artifact@v3","with":{"name":"hakurei-${{ steps.dist-test.outputs.rev }}","path":"result/*","retention-days":1}}]}}} diff --git a/internal/workflows/doc.go b/internal/workflows/doc.go index e34c59c8..b0dab000 100644 --- a/internal/workflows/doc.go +++ b/internal/workflows/doc.go @@ -67,6 +67,7 @@ Before starting the container, configure act_runner via config.yaml: -v /var/lib/rosa:/rosa --security-opt='unmask=/proc/*' --cap-add=SYS_ADMIN + --cap-add=SYS_PTRACE --device=/dev/kvm --device=/dev/fuse valid_volumes: @@ -76,8 +77,9 @@ where /var/lib/rosa is the absolute pathname of the cache directory in the init namespace. Setting MBF_POISON_OPEN enables cmd/mbf to run as root. It is also a good idea here to set runner.capacity to reflect the capacity of the guest, so jobs can be consumed quicker. Removing mount points covering /proc enables -testing of cmd/hakurei. Exposing the fuse device and adding capability SYS_ADMIN -enables testing of cmd/sharefs. +testing of cmd/hakurei. Exposing the fuse device and adding capability +CAP_SYS_ADMIN enables testing of cmd/sharefs. Adding capability CAP_SYS_PTRACE +enables dumping seccomp filters via ptrace on the patched kernel. Build a statically-linked cmd/mbf: @@ -120,6 +122,15 @@ this can be achieved by the init script: It is often a good idea to populate the cache from a mirror service before the first workflow job is started and re-populate it after every cmd/mbf update. +# Configuring the kernel + +In order to attach to the container process, the sysctl kernel.yama.ptrace_scope +must be set to 0. After which, apply the patch test/sandbox/seccomp.patch to +your kernel sources, compile and install the new kernel. Refer to +https://wiki.alpinelinux.org/wiki/Custom_Kernel if the guest runs Alpine Linux. +If running podman or docker as root, the patch is not required. Do not apply +this patch on a system meant to be secure. + # Security The design of Microsoft Github workflows is inherently insecure: it requires diff --git a/internal/workflows/step.go b/internal/workflows/step.go index 219ed78b..6d715196 100644 --- a/internal/workflows/step.go +++ b/internal/workflows/step.go @@ -55,9 +55,21 @@ func newCIRequest(display, name, id string) Step { var install = Step{ Name: "Install hakurei", Run: "HAKUREI_VERSION=\"$(cat cmd/dist/VERSION)-${{ steps.dist.outputs.rev }}\" && " + - "tar xf \"result/hakurei-$HAKUREI_VERSION-amd64.tar.gz\" && " + - "\"./hakurei-$HAKUREI_VERSION-amd64/install.sh\" && " + - "sudo -u ubuntu hakurei version", + "tar xf result/hakurei-$HAKUREI_VERSION*-amd64.tar.gz && " + + "./hakurei-$HAKUREI_VERSION*-amd64/install.sh && " + + "sudo -u ubuntu hakurei version && " + + "echo 'Defaults closefrom_override' > " + + "/etc/sudoers.d/closefrom_override && " + + "mkdir /var/empty", +} + +// newTestsuite returns a step for running an integration test suite. +func newTestsuite(name, prefix string) Step { + return Step{ + Name: "Compile and run test suite", + Run: prefix + "rm -rf result && " + + "go run -tags=testsuite ./test/" + name + " ubuntu", + } } // newNixOSTest returns a step for running the named NixOS test. diff --git a/internal/workflows/test.go b/internal/workflows/test.go index d3290f16..37f41e51 100644 --- a/internal/workflows/test.go +++ b/internal/workflows/test.go @@ -82,12 +82,7 @@ var _ = (&Workflow{ "setuid=1023,setgid=1023 /sdcard", }, - { - Name: "Compile and run test suite", - Run: "sharefs -V && rm -rf result && " + - "go run -tags=testsuite ./test/sharefs ubuntu", - }, - + newTestsuite("sharefs", "sharefs -V && "), newUploadArtifact("test output", "fs_mark"), }, }}, diff --git a/test/flake.nix b/test/flake.nix index b106ec7a..ff1f6237 100644 --- a/test/flake.nix +++ b/test/flake.nix @@ -49,12 +49,6 @@ withRace = true; }; - sandbox = callPackage ./sandbox { inherit self; }; - sandbox-race = callPackage ./sandbox { - inherit self; - withRace = true; - }; - formatting = runCommandLocal "check-formatting" { nativeBuildInputs = [ nixfmt ]; } '' cd ${./.} @@ -155,55 +149,6 @@ }; withPackage = pkgs.mkShell { buildInputs = [ hakurei ] ++ hakurei.targetPkgs; }; - vm = - let - nixos = nixpkgs.lib.nixosSystem { - inherit system; - modules = [ - { - environment = { - systemPackages = [ - (pkgs.buildFHSEnv { - pname = "hakurei-fhs"; - inherit (hakurei) version; - targetPkgs = _: hakurei.targetPkgs; - extraOutputsToInstall = [ "dev" ]; - profile = '' - export PKG_CONFIG_PATH="/usr/share/pkgconfig:$PKG_CONFIG_PATH" - ''; - }) - ]; - - hakurei = - let - # this is used for interactive vm testing during development, where tests might be broken - package = self.packages.${pkgs.stdenv.hostPlatform.system}.hakurei.override { - buildGo127Module = previousArgs: pkgs.pkgsStatic.buildGo127Module (previousArgs // { doCheck = false; }); - }; - in - { - inherit package; - hsuPackage = self.packages.${pkgs.stdenv.hostPlatform.system}.hsu.override { hakurei = package; }; - }; - }; - } - - ./interactive/configuration.nix - ./interactive/vm.nix - ./interactive/hakurei.nix - ./interactive/trace.nix - ./interactive/raceattr.nix - - self.nixosModules.hakurei - home-manager.nixosModules.home-manager - ]; - }; - in - pkgs.mkShell { - buildInputs = [ nixos.config.system.build.vm ]; - shellHook = "exec run-nixos-vm $@"; - }; - generateDoc = let inherit (pkgs) lib; diff --git a/test/interactive/configuration.nix b/test/interactive/configuration.nix deleted file mode 100644 index 9d1bddd8..00000000 --- a/test/interactive/configuration.nix +++ /dev/null @@ -1,68 +0,0 @@ -{ pkgs, ... }: -{ - system.stateVersion = "23.05"; - - users.users = { - alice = { - isNormalUser = true; - description = "Alice Foobar"; - password = "foobar"; - uid = 1000; - extraGroups = [ "wheel" ]; - }; - untrusted = { - isNormalUser = true; - description = "Untrusted user"; - password = "foobar"; - uid = 1001; - }; - }; - - home-manager.users.alice.home.stateVersion = "24.11"; - - security = { - sudo.wheelNeedsPassword = false; - rtkit.enable = true; - pam.loginLimits = [ - { - domain = "@users"; - item = "rtprio"; - type = "-"; - value = 1; - } - ]; - }; - - services = { - getty.autologinUser = "alice"; - pipewire = { - enable = true; - alsa.enable = true; - alsa.support32Bit = true; - pulse.enable = true; - jack.enable = true; - }; - }; - - environment.variables = { - SWAYSOCK = "/tmp/sway-ipc.sock"; - WLR_RENDERER = "pixman"; - }; - - programs = { - sway.enable = true; - - bash.loginShellInit = '' - if [ "$(tty)" = "/dev/tty1" ]; then - set -e - - mkdir -p ~/.config/sway - (sed s/Mod4/Mod1/ /etc/sway/config && - echo 'output * bg ${pkgs.nixos-artwork.wallpapers.simple-light-gray.gnomeFilePath} fill') > ~/.config/sway/config - - sway --validate - systemd-cat --identifier=session sway && touch /tmp/sway-exit-ok - fi - ''; - }; -} diff --git a/test/interactive/hakurei.nix b/test/interactive/hakurei.nix deleted file mode 100644 index ef5d0ada..00000000 --- a/test/interactive/hakurei.nix +++ /dev/null @@ -1,47 +0,0 @@ -{ pkgs, ... }: -{ - environment.hakurei = rec { - enable = true; - stateDir = "/var/lib/hakurei"; - sharefs.source = "${stateDir}/sdcard"; - users.alice = 0; - apps = { - "cat.gensokyo.extern.foot.noEnablements" = { - name = "ne-foot"; - identity = 1; - shareUid = true; - verbose = true; - share = pkgs.foot; - packages = [ pkgs.foot ]; - command = "foot"; - enablements = { - dbus = false; - pipewire = false; - }; - }; - - "cat.gensokyo.extern.foot.badDaemon" = { - name = "bd-foot"; - identity = 1; - shareUid = true; - verbose = true; - share = pkgs.foot; - packages = [ pkgs.foot ]; - command = "foot"; - enablements = { - dbus = false; - }; - extraPaths = [ - { - type = "daemon"; - dst = "/proc/nonexistent"; - path = "/usr/bin/env"; - args = [ "false" ]; - } - ]; - }; - }; - - extraHomeConfig.home.stateVersion = "23.05"; - }; -} diff --git a/test/interactive/raceattr.nix b/test/interactive/raceattr.nix deleted file mode 100644 index de54f4cf..00000000 --- a/test/interactive/raceattr.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ lib, pkgs, ... }: -{ - security.wrappers.raceattr = - let - inherit (pkgs) buildGoModule; - in - { - setuid = true; - owner = "root"; - group = "root"; - source = "${ - (buildGoModule rec { - name = "raceattr"; - pname = name; - tags = [ "raceattr" ]; - - src = builtins.path { - name = "${pname}-src"; - path = lib.cleanSource ../sharefs; - filter = path: type: (type == "directory") || (type == "regular" && lib.hasSuffix ".go" path); - }; - vendorHash = null; - - preBuild = '' - go mod init hakurei.app/raceattr >& /dev/null - ''; - }) - }/bin/raceattr"; - }; -} diff --git a/test/interactive/trace.nix b/test/interactive/trace.nix deleted file mode 100644 index b9ef872c..00000000 --- a/test/interactive/trace.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ lib, pkgs, ... }: -let - tracing = name: "\"/sys/kernel/debug/tracing/${name}\""; -in -{ - environment.systemPackages = [ - (pkgs.writeShellScriptBin "hakurei-set-up-tracing" '' - set -e - echo "$1" > ${tracing "set_graph_function"} - echo function_graph > ${tracing "current_tracer"} - echo funcgraph-tail > ${tracing "trace_options"} - echo funcgraph-retval > ${tracing "trace_options"} - echo nofuncgraph-cpu > ${tracing "trace_options"} - echo nofuncgraph-overhead > ${tracing "trace_options"} - echo nofuncgraph-duration > ${tracing "trace_options"} - '') - (pkgs.writeShellScriptBin "hakurei-print-trace" "exec cat ${tracing "trace"}") - (pkgs.writeShellScriptBin "hakurei-consume-trace" "exec cat ${tracing "trace_pipe"}") - ]; - - boot.kernelPatches = [ - { - name = "funcgraph-retval"; - patch = null; - structuredExtraConfig = with lib.kernel; { - FUNCTION_GRAPH_RETVAL = yes; - - RUST = lib.mkForce unset; - DRM_NOVA = lib.mkForce unset; - DRM_PANIC_SCREEN_QR_CODE = lib.mkForce unset; - NOVA_CORE = lib.mkForce unset; - }; - } - ]; -} diff --git a/test/interactive/vm.nix b/test/interactive/vm.nix deleted file mode 100644 index 522878e9..00000000 --- a/test/interactive/vm.nix +++ /dev/null @@ -1,56 +0,0 @@ -{ - virtualisation.vmVariant.virtualisation = { - memorySize = 4096; - qemu.options = [ - "-vga none -device virtio-gpu-pci" - "-smp 8" - ]; - - mountHostNixStore = true; - writableStore = true; - writableStoreUseTmpfs = false; - - sharedDirectories = { - cwd = { - target = "/mnt/.ro-cwd"; - source = ''"$OLDPWD"''; - securityModel = "none"; - }; - }; - - fileSystems = { - "/mnt/.ro-cwd".options = [ - "ro" - "noatime" - ]; - "/mnt/cwd".overlay = { - lowerdir = [ "/mnt/.ro-cwd" ]; - upperdir = "/tmp/.cwd/upper"; - workdir = "/tmp/.cwd/work"; - }; - - "/mnt/src".overlay = { - lowerdir = [ ../.. ]; - upperdir = "/tmp/.src/upper"; - workdir = "/tmp/.src/work"; - }; - }; - }; - - systemd.services = { - logrotate-checkconf.enable = false; - hakurei-src-fix-ownership = { - wantedBy = [ "multi-user.target" ]; - wants = [ "mnt-src.mount" ]; - after = [ "mnt-src.mount" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - script = '' - chown -R alice:users /mnt/src/ - chmod -R +w /mnt/src/ - ''; - }; - }; -} diff --git a/test/internal/testsuite/proc.go b/test/internal/testsuite/proc.go new file mode 100644 index 00000000..868e1dda --- /dev/null +++ b/test/internal/testsuite/proc.go @@ -0,0 +1,348 @@ +package testsuite + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "unsafe" + + "hakurei.app/fhs" +) + +// Stat represents status information read from /proc/pid/stat. +type Stat struct { + // The process ID. + PID int + // The filename of the executable, with parenthesis stripped. + Comm string + // One of the following characters, indicating process state: + // + // R Running + // + // S Sleeping in an interruptible wait + // + // D Waiting in uninterruptible disk sleep + // + // Z Zombie + // + // T Stopped (on a signal) or (before Linux + // 2.6.33) trace stopped + // + // t Tracing stop (Linux 2.6.33 onward) + // + // W Paging (only before Linux 2.6.0) + // + // X Dead (from Linux 2.6.0 onward) + // + // x Dead (Linux 2.6.33 to 3.13 only) + // + // K Wakekill (Linux 2.6.33 to 3.13 only) + // + // W Waking (Linux 2.6.33 to 3.13 only) + // + // P Parked (Linux 3.9 to 3.13 only) + // + // I Idle (Linux 4.14 onward) + State byte + // The process ID of the parent of this process. + PPID int + // The process group ID of the process. + PGRP int + // The session ID of the process. + Session int + // The controlling terminal of the process. + TTYNR int + // The ID of the foreground process group of the controlling terminal of the + // process. + TPGID int + // The kernel flags word of the process. For bit meanings, see the PF_* + // defines in the Linux kernel source file include/linux/sched.h. + Flags uint + // The number of minor faults the process has made which have not required + // loading a memory page from disk. + MinFlt uint + // The number of minor faults that the process's waited-for children have + // made. + CMinFlt uint + // The number of major faults the process has made which have required + // loading a memory page from disk. + MajFlt uint + // The number of major faults that the process's waited-for children have + // made. + CMajFlt uint + // Amount of time that this process has been scheduled in user mode, + // measured in clock ticks. + UTime uint + // Amount of time that this process has been scheduled in kernel mode, + // measured in clock ticks. + STime uint + // Amount of time that this process's waited-for children have been + // scheduled in user mode, measured in clock ticks. + CUTime int + // Amount of time that this process's waited-for children have been + // scheduled in kernel mode, measured in clock ticks. + CSTime int + // For processes running a real-time scheduling policy, this is the negated + // scheduling priority, minus one. + Priority int + // The nice value, a value in the range 19 (low priority) to -20 (high + // priority). + Nice int + // Number of threads in this process. + NumThreads int + + // unmaintained field: itrealvalue + + // The time the process started after system boot. Since Linux 2.6, the + // value is expressed in clock ticks. + StartTime uint64 + // Virtual memory size in bytes. + VSize uint + // Resident set size in pages. + RSS int + // Soft limit in bytes on the rss of the process. + RSSLim uint64 + // The address above which program text can run. + StartCode uint64 + // The address below which program text can run. + EndCode uint64 + // The address of the start (i.e., bottom) of the stack. + StartStack uint64 + // The current value of ESP (stack pointer), as found in the kernel stack + // page for the process. + KSTKESP uint64 + // The current EIP (instruction pointer). + KSTKEIP uint64 + + // obsolete fields: signal, blocked, sigignore, sigcatch + + // This is the "channel" in which the process is waiting. It is the address + // of a location in the kernel where the process is sleeping. + WChan uint64 + + // unmaintained fields: nswap, cnswap + + // Signal to be sent to parent when we die. + ExitSignal int + // CPU number last executed on. + Processor int + // Real-time scheduling priority, a number in the range 1 to 99 for processes + // scheduled under a real-time policy, or 0, for non-real-time processes. + RTPriority uint + // Scheduling policy (see sched_setscheduler(2)). Decode using the SCHED_* + // constants in linux/sched.h. + Policy uint + // Aggregated block I/O delays, measured in clock ticks (centiseconds). + DelayAcctBlkIOTicks uint64 + // Guest time of the process (time spent running a virtual CPU for a guest + // operating system), measured in clock ticks. + GuestTime int + // Guest time of the process's children, measured in clock ticks. + CGuestTime int +} + +// Executable is like [os.Executable], but for the process referred to by s. +func (s *Stat) Executable() (string, error) { + path, err := os.Readlink(filepath.Join(fhs.Proc, strconv.Itoa(s.PID), "exe")) + + // When the executable has been deleted then Readlink returns a + // path appended with " (deleted)". + return strings.TrimSuffix(path, " (deleted)"), err +} + +// Stat populates stat with the proc filesystem entry referred to by s. +func (s *Stat) Stat(stat *syscall.Stat_t) (err error) { + err = syscall.Stat(filepath.Join(fhs.Proc, strconv.Itoa(s.PID)), stat) + if err != nil { + err = os.NewSyscallError("stat", err) + } + return +} + +// Args reads arguments of the process referred to by s. +func (s *Stat) Args() ([]string, error) { + p, err := os.ReadFile(filepath.Join(fhs.Proc, strconv.Itoa(s.PID), "cmdline")) + if err != nil { + return nil, err + } + a := bytes.Split(p, []byte{0}) + if len(a) > 0 && len(a[len(a)-1]) == 0 { + a = a[:len(a)-1] + } + + args := make([]string, len(a)) + for i, arg := range a { + args[i] = unsafe.String(unsafe.SliceData(arg), len(arg)) + } + return args, nil +} + +// ErrBadDelimiters is returned by [Stat.UnmarshalText] if one or both bytes of +// the comm delimiter pair were missing or misplaced. +var ErrBadDelimiters = errors.New("missing comm delimiters") + +// UnmarshalText populates the structure pointed to by s from text. +func (s *Stat) UnmarshalText(text []byte) (err error) { + var ( + discard uint64 + _uint64 = &discard + _int64 = (*int64)(unsafe.Pointer(&discard)) + + ld = bytes.Index(text, []byte("(")) + rd = bytes.LastIndex(text, []byte(")")) + ) + + if ld <= 0 || rd < 0 { + return ErrBadDelimiters + } + + if s.PID, err = strconv.Atoi( + unsafe.String(unsafe.SliceData(text), ld-1), + ); err != nil { + return + } + + s.Comm = string(text[ld+1 : rd]) + + var ( + n int + + state string + ) + n, err = fmt.Fscan( + bytes.NewBuffer(text[rd+2:]), + &state, + &s.PPID, + &s.PGRP, + &s.Session, + &s.TTYNR, + &s.TPGID, + &s.Flags, + &s.MinFlt, + &s.CMinFlt, + &s.MajFlt, + &s.CMajFlt, + &s.UTime, + &s.STime, + &s.CUTime, + &s.CSTime, + &s.Priority, + &s.Nice, + &s.NumThreads, + _int64, + &s.StartTime, + &s.VSize, + &s.RSS, + &s.RSSLim, + &s.StartCode, + &s.EndCode, + &s.StartStack, + &s.KSTKESP, + &s.KSTKEIP, + _uint64, + _uint64, + _uint64, + _uint64, + &s.WChan, + _uint64, + _uint64, + &s.ExitSignal, + &s.Processor, + &s.RTPriority, + &s.Policy, + &s.DelayAcctBlkIOTicks, + &s.GuestTime, + &s.CGuestTime, + ) + if err != nil { + err = fmt.Errorf("field %d: %w", n, err) + } else if len(state) != 1 { + err = fmt.Errorf("invalid state %q", state) + } else { + s.State = state[0] + } + return +} + +// A StatScanner continuously scans the proc filesystem for process status +// information in /proc/pid/stat. +type StatScanner struct { + // Current entry. + stat Stat + // Cached top-level /proc entries. + dents []os.DirEntry + // Current progress through dents. + i int + // Whether the previous call to Scan had repopulated dents. + wrapped bool + // First stored error: a non-nil err disables the scanner. + err error +} + +// Scan reads a process status information entry. It returns false if an +// unrecoverable error is encountered, after which Scan no longer scans new +// entries. +func (s *StatScanner) Scan() bool { + if s.err != nil { + return false + } + + if s.wrapped = s.i == len(s.dents); s.wrapped { + if s.dents, s.err = os.ReadDir(fhs.Proc); s.err != nil { + return false + } + s.i = 0 + if len(s.dents) == 0 { + s.err = syscall.ENOTRECOVERABLE + return false + } + } + + for s.i < len(s.dents) { + dent := s.dents[s.i] + s.i++ + if !dent.IsDir() { + continue + } + + pid, err := strconv.Atoi(dent.Name()) + if err != nil { + continue + } + + var p []byte + p, err = os.ReadFile(filepath.Join(fhs.Proc, dent.Name(), "stat")) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + s.err = err + return false + } + + s.err = s.stat.UnmarshalText(p) + if s.err == nil && pid != s.stat.PID { + s.err = fmt.Errorf( + "bad status information: dent=%d, stat=%d", + pid, s.stat.PID, + ) + } + return s.err == nil + } + return s.Scan() +} + +// Stat returns the address of the [Stat] structure populated by the last call +// to Scan. +func (s *StatScanner) Stat() *Stat { return &s.stat } + +// Err returns the stored error value. +func (s *StatScanner) Err() error { return s.err } + +// Repopulated returns whether the last Scan call had re-read the proc filesystem. +func (s *StatScanner) Repopulated() bool { return s.wrapped } diff --git a/test/internal/testsuite/proc_test.go b/test/internal/testsuite/proc_test.go new file mode 100644 index 00000000..a8698e73 --- /dev/null +++ b/test/internal/testsuite/proc_test.go @@ -0,0 +1,33 @@ +package testsuite_test + +import ( + "testing" + + "hakurei.app/test/internal/testsuite" +) + +func BenchmarkStatScanner(b *testing.B) { + var s testsuite.StatScanner + + for b.Loop() { + if !s.Scan() { + b.Fatal(s.Err()) + } + } +} + +func BenchmarkStatScannerFull(b *testing.B) { + var s testsuite.StatScanner + + for b.Loop() { + for s.Scan() { + if s.Repopulated() { + break + } + } + + if err := s.Err(); err != nil { + b.Fatal(err) + } + } +} diff --git a/test/internal/testsuite/testsuite.go b/test/internal/testsuite/testsuite.go index 4b70da34..f2df326c 100644 --- a/test/internal/testsuite/testsuite.go +++ b/test/internal/testsuite/testsuite.go @@ -2,14 +2,21 @@ package testsuite import ( + "context" "log" "os" "os/exec" "os/signal" + "strconv" "syscall" + "testing" ) func init() { + if testing.Testing() { + return + } + log.SetFlags(0) log.SetPrefix("testsuite: ") @@ -25,6 +32,14 @@ func ReceiveSignals() { log.Fatalf("terminating on signal %s", <-s) } +// GetUsername returns the first command-line argument. +func GetUsername() string { + if len(os.Args) != 2 { + log.Fatal("expecting 1 argument") + } + return os.Args[1] +} + // MustRun runs command and terminates the testsuite on error. func MustRun(command ...string) { cmd := exec.Command(command[0], command[1:]...) @@ -38,3 +53,33 @@ func MustRun(command ...string) { func MustRunAs(username string, command ...string) { MustRun(append([]string{"sudo", "-u", username}, command...)...) } + +// MustStart starts cmd and returns a channel delivering its wait error. +func MustStart(cmd *exec.Cmd) (done <-chan error) { + if err := cmd.Start(); err != nil { + log.Fatal(err) + } + d := make(chan error) + go func() { d <- cmd.Wait() }() + return d +} + +// MustStartAs wraps [MustStart] for sudo. +func MustStartAs( + ctx context.Context, + username string, + files []*os.File, + command ...string, +) (proc *os.Process, done <-chan error) { + sudoArgs := []string{ + "-u", username, + } + if len(files) != 0 { + sudoArgs = append(sudoArgs, "-C", strconv.Itoa(len(files)+4)) + } + sudoArgs = append(sudoArgs, "--") + cmd := exec.CommandContext(ctx, "sudo", append(sudoArgs, command...)...) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + cmd.ExtraFiles = files + return cmd.Process, MustStart(cmd) +} diff --git a/test/sandbox/main.go b/test/sandbox/main.go new file mode 100644 index 00000000..fd129577 --- /dev/null +++ b/test/sandbox/main.go @@ -0,0 +1,83 @@ +//go:build testsuite + +// The sandbox test program runs cmd/hakurei with configurations simulating +// several common workloads and inspects the resulting container states. +package main + +import ( + "context" + "log" + "slices" + "sync" + "syscall" + + "hakurei.app/hst" + "hakurei.app/test/internal/testsuite" +) + +func main() { + go testsuite.ReceiveSignals() + username := testsuite.GetUsername() + + // the signal handler does not wait for termination + ctx := context.Background() + + var wg sync.WaitGroup + defer wg.Wait() + + wg.Go(func() { + log.Println("checking pd seccomp outcome") + c, cancel := context.WithCancel(ctx) + defer cancel() + + _, done := testsuite.MustStartAs( + c, username, nil, + "hakurei", "exec", "sleep", "infinity", + ) + + var ( + s testsuite.StatScanner + + stat syscall.Stat_t + ) + for s.Scan() { + select { + case err := <-done: + if err == nil { + log.Fatal("test process terminated unexpectedly") + } + log.Fatal(err) + default: + break + } + + if s.Stat().Comm != "sleep" { + continue + } + + if args, err := s.Stat().Args(); err != nil { + log.Fatal(err) + } else if !slices.Equal(args, []string{"sleep", "infinity"}) { + continue + } + + if err := s.Stat().Stat(&stat); err != nil { + log.Fatal(err) + } + + id := hst.ToUser[uint32](0, 0) + if stat.Uid != id || stat.Gid != id { + continue + } + + break + } + if err := s.Err(); err != nil { + log.Fatal(err) + } + mustCheckFilter(s.Stat().PID, pdSum) + if err := <-done; err != nil { + log.Fatal(err) + } + }) +} diff --git a/test/sandbox/ptrace.go b/test/sandbox/ptrace.go new file mode 100644 index 00000000..2647b19a --- /dev/null +++ b/test/sandbox/ptrace.go @@ -0,0 +1,153 @@ +//go:build testsuite + +package main + +import ( + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "log" + "os" + "syscall" + "unsafe" +) + +const ( + // PTRACE_ATTACH attaches to the process specified in pid. + PTRACE_ATTACH = 16 + // PTRACE_DETACH restarts the stopped tracee as for PTRACE_CONT, but first + // detaches from it. + PTRACE_DETACH = 17 + + // PTRACE_SECCOMP_GET_FILTER allows the tracer to dump the tracee's classic + // BPF filters. + PTRACE_SECCOMP_GET_FILTER = 0x420c +) + +// ptrace wraps the ptrace syscall. +func ptrace( + op uintptr, + pid, addr int, + data unsafe.Pointer, +) (r uintptr, errno syscall.Errno) { + r, _, errno = syscall.Syscall6( + syscall.SYS_PTRACE, + op, + uintptr(pid), + uintptr(addr), + uintptr(data), + 0, 0, + ) + return +} + +// ptraceAttach attaches to the process referred to by pid. +func ptraceAttach(pid int) error { + if _, errno := ptrace(PTRACE_ATTACH, pid, 0, nil); errno != 0 { + return os.NewSyscallError("PTRACE_ATTACH", errno) + } + + var status syscall.WaitStatus + for { + if _, err := syscall.Wait4( + pid, + &status, + syscall.WALL, + nil, + ); err != nil { + if errors.Is(err, syscall.EINTR) { + continue + } + return os.NewSyscallError("wait4", err) + } + break + } + + return nil +} + +// ptraceDetach detaches from the attached process referred to by pid. +func ptraceDetach(pid int) error { + if _, errno := ptrace(PTRACE_DETACH, pid, 0, nil); errno != 0 { + return os.NewSyscallError("PTRACE_DETACH", errno) + } + return nil +} + +// getFilter dumps the specified tracee's cBPF filter at the specified index +// and returns the resulting payload. T must be eight bytes long and must not +// contain pointers. +func getFilter[T comparable](pid, index int) ([]T, error) { + if s := unsafe.Sizeof(*new(T)); s != 8 { + return nil, fmt.Errorf("invalid filter block size %d", s) + } + + var buf []T + if n, errno := ptrace( + PTRACE_SECCOMP_GET_FILTER, + pid, index, nil, + ); errno != 0 { + return nil, os.NewSyscallError("PTRACE_SECCOMP_GET_FILTER", errno) + } else { + buf = make([]T, n) + } + if _, errno := ptrace( + PTRACE_SECCOMP_GET_FILTER, + pid, index, unsafe.Pointer(&buf[0]), + ); errno != 0 { + return nil, os.NewSyscallError("PTRACE_SECCOMP_GET_FILTER", errno) + } + return buf, nil +} + +// checkFilter checks the process at pid to have its first filter's contents +// match the specified sha512 checksum in hexadecimal string representation. +func checkFilter(pid int, sum string) (err error) { + if err = ptraceAttach(pid); err != nil { + return + } + defer func() { + if detachErr := ptraceDetach(pid); err == nil { + err = detachErr + } + }() + + var ( + buf [][8]byte + want []byte + ) + + if want, err = hex.DecodeString(sum); err != nil { + return + } + + h := sha512.New() + if buf, err = getFilter[[8]byte](pid, 0); err != nil { + return err + } else { + for _, w := range buf { + h.Write(w[:]) + } + } + + if got := h.Sum(nil); string(got) != string(want) { + return fmt.Errorf( + "bad filter\n\t got: %s\n\twant: %s", + hex.EncodeToString(got), + hex.EncodeToString(want), + ) + } + return +} + +// mustCheckFilter is like checkFilter, but terminates the test suite if a +// non-nil error is returned. Otherwise, the tracee is interrupted after it +// resumes. +func mustCheckFilter(pid int, sum string) { + if err := checkFilter(pid, sum); err != nil { + log.Fatal(err) + } else if err = syscall.Kill(pid, syscall.SIGINT); err != nil { + log.Fatalf("cannot terminate tracee: %v", err) + } +} diff --git a/test/sandbox/seccomp.patch b/test/sandbox/seccomp.patch new file mode 100644 index 00000000..ddabc71e --- /dev/null +++ b/test/sandbox/seccomp.patch @@ -0,0 +1,18 @@ +diff --git a/kernel/seccomp.c b/kernel/seccomp.c +index 25f62867a16d..7b63ccc8daf4 100644 +--- a/kernel/seccomp.c ++++ b/kernel/seccomp.c +@@ -2216,8 +2216,12 @@ long seccomp_get_filter(struct task_struct *task, unsigned long filter_off, + struct seccomp_filter *filter; + struct sock_fprog_kern *fprog; + long ret; ++ struct user_namespace *user_ns = current_user_ns(); + +- if (!capable(CAP_SYS_ADMIN) || ++ if (in_userns(user_ns, task_cred_xxx(task, user_ns))) { ++ if (!ns_capable(user_ns, CAP_SYS_ADMIN)) ++ return -EACCES; ++ } else if (!capable(CAP_SYS_ADMIN) || + current->seccomp.mode != SECCOMP_MODE_DISABLED) { + return -EACCES; + } diff --git a/test/sandbox/sum_amd64.go b/test/sandbox/sum_amd64.go new file mode 100644 index 00000000..da03f12b --- /dev/null +++ b/test/sandbox/sum_amd64.go @@ -0,0 +1,7 @@ +//go:build testsuite + +package main + +const ( + pdSum = "c698b081ff957afe17a6d94374537d37f2a63f6f9dd75da7546542407a9e32476ebda3312ba7785d7f618542bcfaf27ca27dcc2dddba852069d28bcfe8cad39a" +) diff --git a/test/sharefs/main.go b/test/sharefs/main.go index 536a61b3..2ebdc6c7 100644 --- a/test/sharefs/main.go +++ b/test/sharefs/main.go @@ -51,16 +51,12 @@ func checkBadOpts(username, opts, want string) { func main() { go testsuite.ReceiveSignals() + username := testsuite.GetUsername() if err := os.Mkdir("result", 0755); err != nil { log.Fatal(err) } - if len(os.Args) != 2 { - log.Fatal("expecting 1 argument") - } - username := os.Args[1] - done := make(chan struct{}) go func() { defer close(done) -- cgit v1.3.1