diff options
Diffstat (limited to 'internal/workflows')
| -rw-r--r-- | internal/workflows/doc.go | 150 | ||||
| -rw-r--r-- | internal/workflows/main.go | 151 | ||||
| -rw-r--r-- | internal/workflows/release.go | 52 | ||||
| -rw-r--r-- | internal/workflows/seccomp.patch | 18 | ||||
| -rw-r--r-- | internal/workflows/step.go | 124 | ||||
| -rw-r--r-- | internal/workflows/test.go | 150 |
6 files changed, 645 insertions, 0 deletions
diff --git a/internal/workflows/doc.go b/internal/workflows/doc.go new file mode 100644 index 00000000..24e86b8c --- /dev/null +++ b/internal/workflows/doc.go @@ -0,0 +1,150 @@ +//go:build !workflows + +/* +The workflows program manages the .gitea directory to avoid having to write +yaml. It is never invoked directly or built for the distribution tarball. + +This program is an internal detail of the hakurei project and is not usable +on its own. It is not covered by the compatibility promise. + +# Configuring act_runner + +It is unreasonable and impractical to bootstrap Rosa OS each time a job runs. +This section documents an ideal configuration for exposing the cmd/mbf CI +service to all job containers, while avoiding some issues with the container +setup documented upstream. + +Microsoft Github workflows expect a docker socket in the job container. This is +problematic because docker has a hard time running inside a docker container. +The Gitea act_runner simply bind mounts whatever socket it sees into the +container. With a regular docker daemon, this allows not only a simple container +escape, but also privilege escalation as unconstrained root in the init +namespace. To mitigate this, set up an unprivileged podman daemon and expose its +socket to the container instead. Since mountinfo always use credentials from the +init user namespace, subordinate user and group ID must always be 100000. + +On Alpine Linux, this is achieved by: + + # Configuration for /etc/init.d/podman + + # See podman-system-service(1) for service description + # and available options. + #podman_opts="--time 0" + + # API endpoint in URI form. Leave empty to use defaults. + podman_uri="unix:///srv/act_runner/podman" + + # Setting root user will start rootful service. + # Use any other user for rootless mode. + podman_user="act_runner" + +A symlink must then be created at /var/run/docker.sock pointing to the socket +owned by the unprivileged daemon. This should be done in a /etc/local.d script. + +The container can be specified via docker-compose: + + version: "3.8" + services: + runner: + restart: always + image: docker.io/gitea/act_runner:nightly + network_mode: host + environment: + CONFIG_FILE: /config.yaml + GITEA_INSTANCE_URL: "https://git.gensokyo.uk" + GITEA_RUNNER_LABELS: "rosa:docker://docker.gitea.com/runner-images:ubuntu-latest" + + GITEA_RUNNER_REGISTRATION_TOKEN: "${REGISTRATION_TOKEN}" + GITEA_RUNNER_NAME: "${RUNNER_NAME}" + volumes: + - ./config.yaml:/config.yaml + - ./data:/data + - ./podman:/var/run/docker.sock + +Before starting the container, configure act_runner via config.yaml: + + container: + options: -e PAGER=cat -e MBF_CACHE_DIR=/rosa -e MBF_POISON_OPEN=1 + -v /var/lib/rosa:/rosa + --security-opt='unmask=/proc/*' + --cap-add=SYS_ADMIN + --cap-add=SYS_PTRACE + --device=/dev/kvm + --device=/dev/fuse + valid_volumes: + - /var/lib/rosa + +where /var/lib/rosa is the absolute pathname of the cache directory in the init +namespace. Setting MBF_POISON_OPEN enables cmd/mbf to run as root. It is also +a good idea here to set runner.capacity to reflect the capacity of the guest, so +jobs can be consumed quicker. Removing mount points covering /proc enables +testing of cmd/hakurei. Exposing the fuse device and adding capability +CAP_SYS_ADMIN enables testing of cmd/sharefs. Adding capability CAP_SYS_PTRACE +enables dumping seccomp filters via ptrace on the patched kernel. + +Build a statically-linked cmd/mbf: + + go build -trimpath \ + -ldflags='-s -w + -buildid= + -linkmode external + -extldflags=-static' \ + ./cmd/mbf + +Create a directory named "bin" in the cache directory and install the resulting +binary. + +The cmd/mbf CI service should generally run as a separate user. To achieve this, +change the socket permission to 0777 after the service starts. On Alpine Linux, +this can be achieved by the init script: + + #!/sbin/openrc-run + supervisor=supervise-daemon + + name="Rosa OS cmd/mbf CI backend" + description="Listening service that services CI workloads from act_runner" + + command=/var/lib/rosa/bin/mbf + command_args="-d /var/lib/rosa ci daemon" + command_user="${mbf_user:=mbf}" + + output_logger="logger" + error_logger="logger" + + depend() { + need sysfs + } + + start_post() { + while [ ! -e /var/lib/rosa/ci ]; do sleep 0.1; done + chmod 0777 /var/lib/rosa/ci + } + +It is often a good idea to populate the cache from a mirror service before the +first workflow job is started and re-populate it after every cmd/mbf update. + +# Configuring the kernel + +In order to attach to the container process, the sysctl kernel.yama.ptrace_scope +must be set to 0. After which, apply the patch internal/workflows/seccomp.patch +to your kernel sources, compile and install the new kernel. Refer to +https://wiki.alpinelinux.org/wiki/Custom_Kernel if the guest runs Alpine Linux. +If running podman or docker as root, the patch is not required. Do not apply +this patch on a system meant to be secure. + +# Security + +The design of Microsoft Github workflows is inherently insecure: it requires +internet access just to fetch actions, and these actions all expect internet +access as well. Gitea, being a drop-in replacement, retains the same problems. + +The setup documented above already avoids the obvious container escape via the +bind-mounted docker socket, but it is still easily possible to escape as the +context of the unconstrained runner user. For this reason, the runner must +reside in a hardened guest. Since internet access cannot be disabled, extra care +must be taken when setting up the host firewall to prevent access to internal +systems from the untrusted guest. +*/ +package workflows + +//go:generate go run -tags=workflows . diff --git a/internal/workflows/main.go b/internal/workflows/main.go new file mode 100644 index 00000000..81b14a12 --- /dev/null +++ b/internal/workflows/main.go @@ -0,0 +1,151 @@ +//go:build workflows + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" +) + +// KV represents a string to value pair. +type KV[T any] struct { + K string + V T +} + +// A Map represents an ordered set of key-value pairs. +type Map[T any] []KV[T] + +// MarshalJSON encodes m as a key-value pair in JSON. +func (m Map[T]) MarshalJSON() ([]byte, error) { + if m == nil || len(m) == 0 { + return []byte("{}"), nil + } + + buf := bytes.NewBuffer(make([]byte, 0, len(m)<<5)) + buf.WriteByte('{') + + last := len(m) - 1 + for i, pair := range m { + p, err := json.Marshal(pair.K) + if err != nil { + return nil, fmt.Errorf("key %q: %w", pair.K, err) + } + buf.Write(p) + buf.WriteByte(':') + + p, err = json.Marshal(pair.V) + if err != nil { + return nil, fmt.Errorf("value of key %q: %w", pair.K, err) + } + buf.Write(p) + + if i < last { + buf.WriteByte(',') + } + } + + buf.WriteByte('}') + return buf.Bytes(), nil +} + +// A Step is a task ran as part of a [Job]. +type Step struct { + // A name for the step. + Name string `json:"name,omitempty"` + // Selects an action to run as part of a step in the job. + Uses string `json:"uses,omitempty"` + // A unique identifier for the step. + ID string `json:"id,omitempty"` + // Prevent a step from running unless a condition is met. + If string `json:"if,omitempty"` + // Runs command-line programs that do not exceed 21,000 characters using the + // operating system's shell. + Run string `json:"run,omitempty"` + // Key-value pairs of input parameters defined by the action. + With Map[any] `json:"with,omitempty"` +} + +// A Job is an entry in a [Workflow] document. +type Job struct { + // Set a name for the job. + Name string `json:"name,omitempty"` + // Identifies any jobs that must complete successfully before this job will + // run. + Needs []string `json:"needs,omitempty"` + // Define the type of machine to run the job on. + On string `json:"runs-on,omitempty"` + // Create a map of outputs for a job. + Outputs Map[string] `json:"outputs,omitempty"` + // A sequence of tasks. + Steps []Step `json:"steps,omitempty"` +} + +// Workflow describes a workflow document. +type Workflow struct { + // The name of the workflow. + Name string `json:"name,omitempty"` + // The name for workflow runs generated from the workflow. + RunName string `json:"run-name,omitempty"` + // Define which events can cause the workflow to run. + On any `json:"on"` + + // Named jobs part of the workflow run. + Jobs Map[Job] `json:"jobs,omitempty"` +} + +// workflowsDir is the relative pathname of the workflows directory. +const workflowsDir = "../../.gitea/workflows" + +// writeFile writes the action to the .gitea directory. +func (w *Workflow) writeFile(name string) error { + f, err := os.OpenFile( + filepath.Join(workflowsDir, name+".yml"), + os.O_CREATE|os.O_EXCL|os.O_WRONLY, + 0444, + ) + if err != nil { + return err + } + + err = json.NewEncoder(f).Encode(w) + if closeErr := f.Close(); err == nil { + err = closeErr + } + return err +} + +// removeOnce synchronises removal of stale workflow files. +var removeOnce sync.Once + +// removeWorkflows removes existing workflow files once. +func removeWorkflows() { + removeOnce.Do(func() { + if err := os.RemoveAll(workflowsDir); err != nil { + fmt.Println(err) + os.Exit(1) + } + if err := os.Mkdir(workflowsDir, 0755); err != nil { + fmt.Println(err) + os.Exit(1) + } + }) +} + +// mustWriteFile is like writeFile, but terminates the program if a non-nil +// error is returned. +func (w *Workflow) mustWriteFile(name string) (_ struct{}) { + removeWorkflows() + + if err := w.writeFile(name); err != nil { + fmt.Println(err) + os.Exit(1) + } + return +} + +func main() {} diff --git a/internal/workflows/release.go b/internal/workflows/release.go new file mode 100644 index 00000000..f220c0f7 --- /dev/null +++ b/internal/workflows/release.go @@ -0,0 +1,52 @@ +//go:build workflows + +package main + +var _ = (&Workflow{ + Name: "Release", + On: Map[any]{{"push", Map[[]string]{{"tags", []string{"v*"}}}}}, + + Jobs: []KV[Job]{ + {"release", Job{ + Name: "Create release", + On: "rosa", + + Steps: []Step{ + fixup, + + { + Name: "Checkout", + Uses: "actions/checkout@v4", + + With: []KV[any]{ + {"fetch-depth", 0}, + {"fetch-tags", true}, + }, + }, + + { + Name: "Request distribution", + ID: "dist", + Run: "/rosa/bin/mbf ci dist -o result " + + ". \"$(cat cmd/dist/VERSION)-act\" && " + + "git tag " + + "--list " + + "--format='%(contents:body)' " + + "\"$(cat cmd/dist/VERSION)\" >> " + + "\"/tmp/release-body\"", + }, + + { + Name: "Create release", + Uses: "https://gitea.com/actions/gitea-release-action@v1.3.7", + + With: []KV[any]{ + {"files", "result/hakurei-**"}, + {"body_path", "/tmp/release-body"}, + {"draft", true}, + }, + }, + }, + }}, + }, +}).mustWriteFile("release") diff --git a/internal/workflows/seccomp.patch b/internal/workflows/seccomp.patch new file mode 100644 index 00000000..ddabc71e --- /dev/null +++ b/internal/workflows/seccomp.patch @@ -0,0 +1,18 @@ +diff --git a/kernel/seccomp.c b/kernel/seccomp.c +index 25f62867a16d..7b63ccc8daf4 100644 +--- a/kernel/seccomp.c ++++ b/kernel/seccomp.c +@@ -2216,8 +2216,12 @@ long seccomp_get_filter(struct task_struct *task, unsigned long filter_off, + struct seccomp_filter *filter; + struct sock_fprog_kern *fprog; + long ret; ++ struct user_namespace *user_ns = current_user_ns(); + +- if (!capable(CAP_SYS_ADMIN) || ++ if (in_userns(user_ns, task_cred_xxx(task, user_ns))) { ++ if (!ns_capable(user_ns, CAP_SYS_ADMIN)) ++ return -EACCES; ++ } else if (!capable(CAP_SYS_ADMIN) || + current->seccomp.mode != SECCOMP_MODE_DISABLED) { + return -EACCES; + } diff --git a/internal/workflows/step.go b/internal/workflows/step.go new file mode 100644 index 00000000..fc95fd69 --- /dev/null +++ b/internal/workflows/step.go @@ -0,0 +1,124 @@ +//go:build workflows + +package main + +import "strings" + +// fixup is a step to work around a Podman regression introduced by a bad +// container escape mitigation. +var fixup = Step{ + Name: "Fix container filesystem", + Run: "rm /var/run && ln -sf ../run /var", +} + +// checkout is the standard actions/checkout step. +var checkout = Step{Name: "Checkout", Uses: "actions/checkout@v4"} + +// toolchain is a step for setting up the Go toolchain. +var toolchain = Step{ + Name: "Set up Go toolchain", + Uses: "actions/setup-go@v6", + + With: []KV[any]{ + {"go-version-file", "go.mod"}, + }, +} + +// newUploadArtifact returns an actions/upload-artifact step uploading +// everything in the result directory as the specified name. +func newUploadArtifact(display, name string) Step { + return Step{ + Name: "Upload " + display, + Uses: "actions/upload-artifact@v3", + + With: Map[any]{ + {"name", name}, + {"path", "result/*"}, + + {"retention-days", 1}, + }, + } +} + +// newCIRequest returns a step for requesting a mbf CI task. +func newCIRequest(display, name, id string) Step { + return Step{ + Name: "Request " + display, + ID: id, + Run: "HAKUREI_REV=\"$(git rev-parse --short HEAD)\" && " + + "/rosa/bin/mbf ci " + name + " " + + ". \"$(cat cmd/dist/VERSION)-$HAKUREI_REV\" && " + + "echo \"rev=$HAKUREI_REV\" >> \"$GITHUB_OUTPUT\"", + } +} + +// download is a step to obtain a hakurei distribution. +var download = Step{ + Name: "Download hakurei distribution", + ID: "download", + Uses: "actions/download-artifact@v3", + + With: []KV[any]{ + {"name", "dist-${{ needs.dist.outputs.rev }}"}, + {"path", "result"}, + }, +} + +// _download is like download, but with race instrument. +var _download = Step{ + Name: "Download hakurei distribution", + ID: "download", + Uses: "actions/download-artifact@v3", + + With: []KV[any]{ + {"name", "dist-${{ needs.dist-race.outputs.rev }}-race"}, + {"path", "result"}, + }, +} + +// install is a step to unpack and deploy a hakurei distribution created by a +// step with identifier dist. +var install = Step{ + Name: "Install hakurei", + Run: "HAKUREI_VERSION=\"$(cat cmd/dist/VERSION)\" && " + + "tar xf result/hakurei-$HAKUREI_VERSION*-amd64.tar.gz && " + + "./hakurei-$HAKUREI_VERSION*-amd64/install.sh && " + + "sudo -u ubuntu hakurei version && " + + "mkdir /var/empty", +} + +// newTestsuite returns a step for running an integration test suite. +func newTestsuite(name, prefix string) Step { + return Step{ + Name: "Compile and run test suite", + Run: prefix + "rm -rf result && " + + "go run -tags=testsuite ./cmd/" + name, + } +} + +// newPackages returns a job for installing the specified packages with +// best-effort caching. Package names must not contain spaces. +func newPackages(rev int, repos []string, packages ...string) Step { + return Step{ + Name: "Install packages", + Uses: "awalsh128/cache-apt-pkgs-action@v1", + With: []KV[any]{ + {"add-repository", strings.Join(repos, " ")}, + {"packages", strings.Join(packages, " ")}, + {"version", rev}, + {"execute_install_scripts", true}, + }, + } +} + +// newNixOSTest returns a step for running the named NixOS test. +func newNixOSTest(name string) Step { + return Step{ + Name: "Run NixOS test", + Run: "nix build " + + "--out-link result " + + "--print-out-paths " + + "--print-build-logs " + + "./cmd/hakurei/testsuite#checks.x86_64-linux." + name, + } +} diff --git a/internal/workflows/test.go b/internal/workflows/test.go new file mode 100644 index 00000000..1f9bba58 --- /dev/null +++ b/internal/workflows/test.go @@ -0,0 +1,150 @@ +//go:build workflows + +package main + +var _ = (&Workflow{ + Name: "Test", + On: []any{"push"}, + + Jobs: Map[Job]{ + {"hakurei-legacy", Job{ + Name: "Hakurei (legacy)", + On: "nix", + + Steps: []Step{ + checkout, + newNixOSTest("hakurei"), + newUploadArtifact("test output", "hakurei-vm-output"), + }, + }}, + + {"hakurei-race-legacy", Job{ + Name: "Hakurei (legacy with race instrument)", + On: "nix", + + Steps: []Step{ + checkout, + newNixOSTest("race"), + newUploadArtifact("test output", "hakurei-race-vm-output"), + }, + }}, + + {"sandbox", Job{ + Name: "Sandbox", + On: "rosa", + + Needs: []string{"dist"}, + Steps: []Step{ + fixup, + checkout, + toolchain, + newPackages(0, []string{"ppa:savoury1/pipewire"}, + "libmount-dev", + "sway", + "xwayland", + "xdg-dbus-proxy", + "pipewire", + ), + + download, + install, + newTestsuite("hakurei/testsuite/sandbox", ""), + }, + }}, + + {"sandbox-race", Job{ + Name: "Sandbox (with race instrument)", + On: "rosa", + + Needs: []string{"dist-race"}, + Steps: []Step{ + fixup, + checkout, + toolchain, + + newPackages(0, []string{"ppa:savoury1/pipewire"}, + "libmount-dev", + "sway", + "xwayland", + "xdg-dbus-proxy", + "pipewire", + ), + + _download, + install, + newTestsuite("hakurei/testsuite/sandbox", ""), + }, + }}, + + {"sharefs", Job{ + Name: "ShareFS", + On: "rosa", + + Needs: []string{"dist"}, + Steps: []Step{ + fixup, + checkout, + toolchain, + newPackages(0, nil, "fuse3", "fsmark"), + + download, + install, + + { + Name: "Mount sharefs", + Run: "useradd " + + "-ru 1023 " + + "-md /var/lib/sdcard " + + "-k /var/empty " + + "-s /sbin/nologin " + + "media_rw && install -dm0 /sdcard && " + + "sharefs " + + "-o rw,noexec,nosuid,nodev,noatime," + + "allow_other,mkdir,source=/var/lib/sdcard," + + "setuid=1023,setgid=1023 /sdcard", + }, + + newTestsuite("sharefs/testsuite", "sharefs -V && "), + newUploadArtifact("test output", "fs_mark"), + }, + }}, + + {"dist", Job{ + Name: "Create distribution", + On: "rosa", + + Outputs: []KV[string]{ + {"rev", "${{ steps.dist.outputs.rev }}"}, + }, + + Steps: []Step{ + fixup, + checkout, + newCIRequest("distribution", "dist -o result", "dist"), + newUploadArtifact( + "distribution", + "dist-${{ steps.dist.outputs.rev }}", + ), + }, + }}, + + {"dist-race", Job{ + Name: "Create distribution (with race instrument)", + On: "rosa", + + Outputs: []KV[string]{ + {"rev", "${{ steps.dist.outputs.rev }}"}, + }, + + Steps: []Step{ + fixup, + checkout, + newCIRequest("distribution", "race -o result", "dist"), + newUploadArtifact( + "distribution", + "dist-${{ steps.dist.outputs.rev }}-race", + ), + }, + }}, + }, +}).mustWriteFile("test") |