aboutsummaryrefslogtreecommitdiffhomepage
path: root/pkg
diff options
context:
space:
mode:
Diffstat (limited to 'pkg')
-rw-r--r--pkg/archive.go411
-rw-r--r--pkg/archive_test.go240
-rw-r--r--pkg/clean.go161
-rw-r--r--pkg/clean_test.go293
-rw-r--r--pkg/compress.go151
-rw-r--r--pkg/compress_test.go72
-rw-r--r--pkg/exec.go808
-rw-r--r--pkg/exec_test.go670
-rw-r--r--pkg/file.go100
-rw-r--r--pkg/file_test.go56
-rw-r--r--pkg/internal/testtool/expected/expected.go9
-rw-r--r--pkg/internal/testtool/expected/sum_amd64.go11
-rw-r--r--pkg/internal/testtool/expected/sum_arm64.go11
-rw-r--r--pkg/internal/testtool/expected/sum_riscv64.go11
-rw-r--r--pkg/internal/testtool/main.go277
-rw-r--r--pkg/ir.go865
-rw-r--r--pkg/ir_test.go170
-rw-r--r--pkg/net.go109
-rw-r--r--pkg/net_test.go168
-rw-r--r--pkg/pkg.go2952
-rw-r--r--pkg/pkg_test.go2375
-rw-r--r--pkg/tar.go226
-rw-r--r--pkg/tar_test.go225
23 files changed, 10371 insertions, 0 deletions
diff --git a/pkg/archive.go b/pkg/archive.go
new file mode 100644
index 00000000..ea4017d9
--- /dev/null
+++ b/pkg/archive.go
@@ -0,0 +1,411 @@
+package pkg
+
+import (
+ "crypto/sha512"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "unsafe"
+
+ "hakurei.app/check"
+)
+
+/*
+| mode uint32 | path_sz uint32 |
+| data_sz uint64 |
+| path string |
+| data []byte |
+*/
+
+// An ArchiveHeader represents a single header in an archive.
+type ArchiveHeader struct {
+ Mode fs.FileMode // file mode bits
+ Path string // pathname of the file
+ Size uint64 // size of data segment
+}
+
+// Writer implements sequential writing of an archive. [Writer.WriteHeader]
+// begins a new file with the provided [ArchiveHeader], and then Writer can be
+// treated as an [io.Writer] to supply that file's data.
+//
+// It is the caller's responsibility to write entries in lexical order.
+type Writer struct {
+ // Underlying writer.
+ w io.Writer
+ // Current header.
+ h ArchiveHeader
+ // Fixed-size header segment.
+ buf [wordSize * 2]byte
+ // Current position in data segment.
+ n uint64
+}
+
+// NewWriter returns the address of a new [Writer] writing to w.
+func NewWriter(w io.Writer) *Writer { return &Writer{w: w} }
+
+var zero [wordSize]byte
+
+// padSize returns the padding size for aligning sz.
+func padSize[T int | uint64](sz T) T {
+ return (wordSize - (sz)%wordSize) % wordSize
+}
+
+// flush concludes writing to the current file and writes padding.
+func (aw *Writer) flush() error {
+ if aw.h.Size > aw.n {
+ return fmt.Errorf("missed writing %d bytes", aw.h.Size-aw.n)
+ } else if aw.h.Size < aw.n {
+ return fmt.Errorf("wrote %d bytes beyond end of file", aw.n-aw.h.Size)
+ }
+
+ if psz := padSize(aw.h.Size); psz != 0 {
+ if _, err := aw.w.Write(zero[:psz]); err != nil {
+ return err
+ }
+ }
+
+ aw.n = 0
+ return nil
+}
+
+// WriteHeader writes h and begins accepting its corresponding file.
+func (aw *Writer) WriteHeader(h *ArchiveHeader) error {
+ if err := aw.flush(); err != nil {
+ return err
+ }
+
+ aw.h = *h
+ binary.LittleEndian.PutUint32(aw.buf[:], uint32(aw.h.Mode))
+ binary.LittleEndian.PutUint32(aw.buf[wordSize/2:], uint32(len(aw.h.Path)))
+ binary.LittleEndian.PutUint64(aw.buf[wordSize:], aw.h.Size)
+ if _, err := aw.w.Write(aw.buf[:]); err != nil {
+ return err
+ } else if _, err = aw.w.Write(
+ unsafe.Slice(unsafe.StringData(aw.h.Path), len(aw.h.Path)),
+ ); err != nil {
+ return err
+ } else if psz := padSize(len(aw.h.Path)); psz != 0 {
+ if _, err = aw.w.Write(zero[:psz]); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// Write writes p to the underlying writer and records the new position. Invalid
+// positions are reported by WriteHeader and Close.
+func (aw *Writer) Write(p []byte) (n int, err error) {
+ n, err = aw.w.Write(p)
+ aw.n += uint64(n)
+ return
+}
+
+// Close concludes writing to the archive stream.
+func (aw *Writer) Close() (err error) {
+ err = aw.flush()
+ aw.w = nil
+ return
+}
+
+// ErrInsecurePath is returned by [FlatEntry.Decode] if validation is requested
+// and a nonlocal path is encountered in the stream.
+var ErrInsecurePath = errors.New("insecure file path")
+
+// Reader implements sequential reading of an archive. [Reader.Next] advances to
+// the next file in the archive (including the first), and then Reader can be
+// treated as an [io.Reader] to access the file's data.
+type Reader struct {
+ // Underlying reader.
+ r io.Reader
+ // Fixed-size header segment.
+ buf [wordSize * 2]byte
+ // Remaining bytes in current data segment.
+ n, pad uint64
+}
+
+// NewReader returns the address of a new [Reader] reading from r.
+func NewReader(r io.Reader) *Reader { return &Reader{r: r} }
+
+// Next advances ar to the next entry. Remaining bytes of the current data
+// segment are discarded. Advancing beyond the final entry returns [io.EOF].
+func (ar *Reader) Next() (*ArchiveHeader, error) {
+ if dsz := int64(ar.n + ar.pad); dsz > 0 {
+ if n, err := io.CopyN(io.Discard, ar.r, dsz); err != nil {
+ if errors.Is(err, io.EOF) && n != dsz {
+ err = io.ErrUnexpectedEOF
+ }
+ return nil, err
+ }
+ }
+
+ if _, err := io.ReadFull(ar.r, ar.buf[:]); err != nil {
+ return nil, err
+ }
+
+ h := ArchiveHeader{
+ Mode: fs.FileMode(binary.LittleEndian.Uint32(ar.buf[:])),
+ Size: binary.LittleEndian.Uint64(ar.buf[wordSize:]),
+ }
+ pathSize := int(binary.LittleEndian.Uint32(ar.buf[wordSize/2:]))
+ pPathSize := alignSize(pathSize)
+
+ buf := make([]byte, pPathSize)
+ if _, err := io.ReadFull(ar.r, buf); err != nil {
+ if errors.Is(err, io.EOF) {
+ err = io.ErrUnexpectedEOF
+ }
+ return nil, err
+ }
+
+ h.Path = unsafe.String(unsafe.SliceData(buf), pathSize)
+ if !filepath.IsLocal(h.Path) {
+ return &h, ErrInsecurePath
+ }
+
+ ar.n = h.Size
+ ar.pad = padSize(h.Size)
+ return &h, nil
+}
+
+// Read implements [io.Reader] for the data segment of the current entry.
+func (ar *Reader) Read(p []byte) (n int, err error) {
+ if uint64(len(p)) > ar.n {
+ p = p[:ar.n]
+ }
+
+ if len(p) > 0 {
+ n, err = ar.r.Read(p)
+ ar.n -= uint64(n)
+ }
+
+ switch err {
+ case io.EOF:
+ if ar.n > 0 {
+ return n, io.ErrUnexpectedEOF
+ }
+
+ case nil:
+ if ar.n == 0 {
+ return n, io.EOF
+ }
+ }
+ return
+}
+
+// Write writes a deterministic representation of the contents of fsys to w.
+// The resulting data can be hashed to produce a deterministic checksum for the
+// directory.
+func Write(fsys fs.FS, root string, w io.Writer) error {
+ aw := NewWriter(w)
+ if err := fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+
+ var fi fs.FileInfo
+ fi, err = d.Info()
+ if err != nil {
+ return err
+ }
+
+ h := ArchiveHeader{
+ Path: path,
+ Mode: fi.Mode(),
+ }
+ if h.Mode.IsRegular() {
+ h.Size = uint64(fi.Size())
+ if err = aw.WriteHeader(&h); err != nil {
+ return err
+ }
+
+ var r fs.File
+ r, err = fsys.Open(path)
+ if err != nil {
+ return err
+ }
+ _, err = io.Copy(aw, r)
+ if _err := r.Close(); err == nil {
+ err = _err
+ }
+ return err
+ } else if h.Mode&fs.ModeSymlink != 0 {
+ var newpath string
+ if newpath, err = fs.ReadLink(fsys, path); err != nil {
+ return err
+ }
+
+ h.Size = uint64(len(newpath))
+ if err = aw.WriteHeader(&h); err != nil {
+ return err
+ }
+
+ _, err = aw.Write(unsafe.Slice(unsafe.StringData(newpath), len(newpath)))
+ return err
+ } else if !h.Mode.IsDir() {
+ return InvalidFileModeError(h.Mode)
+ }
+ return aw.WriteHeader(&h)
+ }); err != nil {
+ return err
+ }
+ return aw.Close()
+}
+
+// SumFS saves checksum of the archive of fsys to the value pointed to by buf.
+func SumFS(buf *Checksum, fsys fs.FS, root string) error {
+ h := sha512.New384()
+ if err := Write(fsys, root, h); err != nil {
+ return err
+ }
+ h.Sum(buf[:0])
+ return nil
+}
+
+// SumDir saves checksum of the archive of directory at pathname to the value
+// pointed to by buf.
+func SumDir(buf *Checksum, pathname *check.Absolute) error {
+ return SumFS(buf, os.DirFS(pathname.String()), ".")
+}
+
+// archiveArtifact is an [Artifact] unpacking an archive supported by [Reader]
+// backed by a [FileArtifact].
+type archiveArtifact struct {
+ // Caller-supplied backing archive.
+ f Artifact
+}
+
+var _ CuresExempt = archiveArtifact{}
+
+// NewArchive returns a new [Artifact] backed by the supplied [Artifact]. The
+// source [Artifact] must be a [FileArtifact] and produce a stream compatible
+// with [Reader].
+func NewArchive(a Artifact) Artifact {
+ return archiveArtifact{a}
+}
+
+// Kind returns the hardcoded [Kind] constant.
+func (archiveArtifact) Kind() Kind { return KindArchive }
+
+// Params is a noop.
+func (archiveArtifact) Params(*IContext) {}
+
+func init() {
+ register(KindArchive, func(r *IRReader) Artifact {
+ a := NewArchive(r.Next())
+ if _, ok := r.Finalise(); ok {
+ panic(ErrUnexpectedChecksum)
+ }
+ return a
+ })
+}
+
+// Inputs returns a slice containing the backing file.
+func (a archiveArtifact) Inputs() []Artifact {
+ return []Artifact{a.f}
+}
+
+// IsExclusive returns false: [Reader] is fully sequential.
+func (archiveArtifact) IsExclusive() bool { return false }
+
+// Cure cures the [Artifact], producing a directory located at work.
+func (a archiveArtifact) Cure(t *TContext) (err error) {
+ var r io.ReadCloser
+ if r, err = t.Open(a.f); err != nil {
+ return
+ }
+
+ defer func() {
+ closeErr := r.Close()
+ if err == nil {
+ err = closeErr
+ }
+ }()
+
+ type dirTargetPerm struct {
+ path string
+ mode fs.FileMode
+ }
+ var madeDirectories []dirTargetPerm
+
+ if err = os.MkdirAll(t.GetWorkDir().String(), 0700); err != nil {
+ return
+ }
+ var root *os.Root
+ if root, err = os.OpenRoot(t.GetWorkDir().String()); err != nil {
+ return
+ }
+ defer func() {
+ closeErr := root.Close()
+ if err == nil {
+ err = closeErr
+ }
+ }()
+
+ var header *ArchiveHeader
+ ar := NewReader(r)
+ for header, err = ar.Next(); err == nil; header, err = ar.Next() {
+ if header.Mode.IsRegular() {
+ var f *os.File
+ if f, err = root.OpenFile(
+ header.Path,
+ os.O_CREATE|os.O_EXCL|os.O_WRONLY,
+ header.Mode.Perm(),
+ ); err != nil {
+ return
+ }
+ if _, err = io.Copy(f, ar); err != nil {
+ _ = f.Close()
+ return
+ } else if err = f.Close(); err != nil {
+ return
+ }
+ } else if header.Mode&fs.ModeSymlink != 0 {
+ var p []byte
+ if p, err = io.ReadAll(ar); err != nil {
+ return
+ }
+
+ if err = root.Symlink(
+ unsafe.String(unsafe.SliceData(p), len(p)),
+ header.Path,
+ ); err != nil {
+ return
+ }
+ } else if header.Mode.IsDir() {
+ if header.Path == "." {
+ continue
+ }
+
+ madeDirectories = append(madeDirectories, dirTargetPerm{
+ path: header.Path,
+ mode: header.Mode,
+ })
+ if err = root.Mkdir(header.Path, 0700); err != nil {
+ return
+ }
+ } else {
+ return InvalidFileModeError(header.Mode)
+ }
+ }
+ if errors.Is(err, io.EOF) {
+ err = nil
+ }
+ if err == nil {
+ for _, e := range madeDirectories {
+ if err = root.Chmod(e.path, e.mode.Perm()); err != nil {
+ return
+ }
+ }
+ } else {
+ return
+ }
+ return
+}
+
+// CuresExempt exempts the cheap [KindArchive] implementation often found at
+// the end of a [FileArtifact] pipeline.
+func (archiveArtifact) CuresExempt() {}
diff --git a/pkg/archive_test.go b/pkg/archive_test.go
new file mode 100644
index 00000000..44ea56a7
--- /dev/null
+++ b/pkg/archive_test.go
@@ -0,0 +1,240 @@
+package pkg_test
+
+import (
+ "bytes"
+ "io"
+ "io/fs"
+ "maps"
+ "reflect"
+ "testing"
+ "testing/fstest"
+ "unsafe"
+
+ "hakurei.app/check"
+ "hakurei.app/pkg"
+)
+
+func TestArchive(t *testing.T) {
+ t.Parallel()
+
+ type entry struct {
+ path string
+ mode fs.FileMode
+ data string
+ }
+ testCases := []struct {
+ name string
+ fsys fs.FS
+ entries []entry
+ sum pkg.Checksum
+ err error
+ }{
+ {"bad type", fstest.MapFS{
+ ".": {Mode: fs.ModeDir | 0700},
+ "invalid": {Mode: fs.ModeCharDevice | 0400},
+ }, nil, pkg.Checksum{}, pkg.InvalidFileModeError(
+ fs.ModeCharDevice | 0400,
+ )},
+
+ {"coldboot", fstest.MapFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "devices": {Mode: fs.ModeDir | 0700},
+ "devices/uevent": {Mode: 0600, Data: []byte("add")},
+ "devices/empty": {Mode: fs.ModeDir | 0700},
+
+ "devices/sub": {Mode: fs.ModeDir | 0700},
+ "devices/sub/uevent": {Mode: 0600, Data: []byte("add")},
+
+ "block": {Mode: fs.ModeDir | 0700},
+ "block/uevent": {Mode: 0600},
+ }, []entry{
+ {".", fs.ModeDir | 0700, ""},
+
+ {"block", fs.ModeDir | 0700, ""},
+ {"block/uevent", 0600, ""},
+
+ {"devices", fs.ModeDir | 0700, ""},
+ {"devices/empty", fs.ModeDir | 0700, ""},
+ {"devices/sub", fs.ModeDir | 0700, ""},
+ {"devices/sub/uevent", 0600, "add"},
+ {"devices/uevent", 0600, "add"},
+ }, pkg.MustDecode("mEy_Lf5KotThm7OwMx7yTKZh5HCCyaB41pVAvI9uDMgVQFM91iosBLYsRm8bDsX8"), nil},
+
+ {"empty", fstest.MapFS{
+ ".": {Mode: fs.ModeDir | 0700},
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }, []entry{
+ {".", fs.ModeDir | 0700, ""},
+ {"checksum", fs.ModeDir | 0700, ""},
+ {"identifier", fs.ModeDir | 0700, ""},
+ {"work", fs.ModeDir | 0700, ""},
+ }, pkg.MustDecode("E4vEZKhCcL2gPZ2Tt59FS3lDng-d_2SKa2i5G_RbDfwGn6EemptFaGLPUDiOa94C"), nil},
+
+ {"sample directory step garbage", fstest.MapFS{
+ ".": {Mode: fs.ModeDir | 0500},
+
+ "lib": {Mode: fs.ModeDir | 0500},
+ "lib/check": {Mode: 0400},
+
+ "lib/pkgconfig": {Mode: fs.ModeDir | 0500},
+ }, []entry{
+ {".", fs.ModeDir | 0500, ""},
+
+ {"lib", fs.ModeDir | 0500, ""},
+ {"lib/check", 0400, ""},
+
+ {"lib/pkgconfig", fs.ModeDir | 0500, ""},
+ }, pkg.MustDecode("CUx-3hSbTWPsbMfDhgalG4Ni_GmR9TnVX8F99tY_P5GtkYvczg9RrF5zO0jX9XYT"), nil},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ t.Run("roundtrip", func(t *testing.T) {
+ t.Parallel()
+
+ var buf bytes.Buffer
+ if err := pkg.Write(
+ tc.fsys,
+ ".",
+ &buf,
+ ); !reflect.DeepEqual(err, tc.err) {
+ t.Fatalf("Flatten: error = %v, want %v", err, tc.err)
+ } else if tc.err != nil {
+ return
+ }
+
+ r := pkg.NewReader(bytes.NewReader(buf.Bytes()))
+ var got []entry
+ for {
+ h, err := r.Next()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ t.Fatalf("Next: error = %v", err)
+ }
+
+ var data []byte
+ if data, err = io.ReadAll(r); err != nil {
+ t.Fatalf("Read: error = %v", err)
+ }
+
+ got = append(got, entry{
+ path: h.Path,
+ mode: h.Mode,
+ data: unsafe.String(unsafe.SliceData(data), len(data)),
+ })
+ }
+
+ if !reflect.DeepEqual(got, tc.entries) {
+ t.Fatalf("Reader: %#v, want %#v", got, tc.entries)
+ }
+ })
+
+ if tc.err != nil {
+ return
+ }
+
+ t.Run("hash", func(t *testing.T) {
+ t.Parallel()
+
+ var got pkg.Checksum
+ if err := pkg.SumFS(&got, tc.fsys, "."); err != nil {
+ t.Fatalf("SumFS: error = %v", err)
+ } else if got != tc.sum {
+ t.Fatalf("SumFS: %v", &pkg.ChecksumMismatchError{
+ Got: got,
+ Want: tc.sum,
+ })
+ }
+ })
+ })
+ }
+}
+
+var archiveTestdata = fstest.MapFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "devices": {Mode: fs.ModeDir | 0700},
+ "devices/uevent": {Mode: 0600, Data: []byte("add")},
+ "devices/empty": {Mode: fs.ModeDir | 0700},
+
+ "devices/sub": {Mode: fs.ModeDir | 0700},
+ "devices/sub/uevent": {Mode: 0600, Data: []byte("add")},
+
+ "block": {Mode: fs.ModeDir | 0700},
+ "block/uevent": {Mode: 0600},
+}
+
+func TestArchiveArtifact(t *testing.T) {
+ t.Parallel()
+
+ want := maps.Clone(archiveTestdata)
+ want["."].Mode = fs.ModeDir | 0500
+
+ checkWithCache(t, []cacheTestCase{
+ {"unpack", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ var buf bytes.Buffer
+ if err := pkg.Write(archiveTestdata, ".", &buf); err != nil {
+ t.Fatal(err)
+ }
+
+ cureMany(t, c, []cureStep{
+ {"sample", pkg.NewArchive(
+ pkg.NewFile("", buf.Bytes()),
+ ), ignorePathname, expectsFS(want), pkg.WNew, nil},
+ })
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F": {Mode: fs.ModeDir | 0500},
+ "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/block": {Mode: fs.ModeDir | 0700},
+ "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/block/uevent": {Mode: 0600},
+ "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices": {Mode: fs.ModeDir | 0700},
+ "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices/empty": {Mode: fs.ModeDir | 0700},
+ "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices/sub": {Mode: fs.ModeDir | 0700},
+ "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices/sub/uevent": {Mode: 0600, Data: []byte("add")},
+ "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices/uevent": {Mode: 0600, Data: []byte("add")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/-zXKfphyWM2Ko7VMVURaEMSuixNokFSe0xFbnwR2RtRtvMrcPGWalV0LIn45PXTY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+ })
+}
+
+func BenchmarkArchiveRead(b *testing.B) {
+ var buf bytes.Buffer
+ if err := pkg.Write(archiveTestdata, ".", &buf); err != nil {
+ b.Fatal(err)
+ }
+ testdata := buf.Bytes()
+
+ for b.Loop() {
+ r := pkg.NewReader(bytes.NewReader(testdata))
+ for {
+ _, err := r.Next()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ b.Fatal(err)
+ }
+ }
+ }
+}
+
+func BenchmarkArchiveWrite(b *testing.B) {
+ for b.Loop() {
+ if err := pkg.Write(archiveTestdata, ".", io.Discard); err != nil {
+ b.Fatal(err)
+ }
+ }
+}
diff --git a/pkg/clean.go b/pkg/clean.go
new file mode 100644
index 00000000..f07706cf
--- /dev/null
+++ b/pkg/clean.go
@@ -0,0 +1,161 @@
+package pkg
+
+import (
+ "errors"
+ "os"
+ "unique"
+)
+
+// Clean destroys checksum backing entries without any identifier or substitute
+// entry referring to it. If at least one keep [Artifact] is specified,
+// identifier and substitute entries not kept alive by them are destroyed first.
+func (c *Cache) Clean(dry, inputs bool, keep ...Artifact) (
+ []unique.Handle[ID],
+ []unique.Handle[Checksum],
+ error,
+) {
+ c.identMu.Lock()
+ defer c.identMu.Unlock()
+
+ c.checksumMu.Lock()
+ defer c.checksumMu.Unlock()
+
+ dents, err := os.ReadDir(c.base.Append(dirChecksum).String())
+ if err != nil {
+ return nil, nil, err
+ }
+ checksums := make(map[unique.Handle[Checksum]]string, len(dents))
+ var buf Checksum
+ for _, dent := range dents {
+ name := dent.Name()
+ if err = Decode(&buf, name); err != nil {
+ return nil, nil, err
+ }
+ checksums[unique.Make(buf)] = name
+ }
+
+ type identPair struct {
+ id unique.Handle[ID]
+ name string
+ }
+ dents, err = os.ReadDir(c.base.Append(dirIdentifier).String())
+ if err != nil {
+ return nil, nil, err
+ }
+
+ keepIdents := make(map[unique.Handle[ID]]struct{})
+ if inputs {
+ for _, id := range Inputs((*Collect)(&keep)) {
+ keepIdents[id] = struct{}{}
+ }
+ } else {
+ for _, a := range keep {
+ keepIdents[c.Ident(a)] = struct{}{}
+ }
+ }
+
+ idents := make([]identPair, 0, len(dents))
+ for _, dent := range dents {
+ name := dent.Name()
+ if err = Decode(&buf, name); err != nil {
+ return nil, nil, err
+ }
+ id := unique.Make(ID(buf))
+
+ if _, ok := keepIdents[id]; len(keep) == 0 || ok {
+ if err = readlinkChecksum(c.base.Append(
+ dirIdentifier,
+ name,
+ ), &buf); err != nil {
+ return nil, nil, err
+ }
+ delete(checksums, unique.Make(buf))
+ continue
+ }
+
+ c.msg.Verbosef(
+ "arranging for destruction of %s%s%s...",
+ c.sgrIdent, name, c.sgrRes,
+ )
+ idents = append(idents, identPair{id, name})
+ }
+
+ destroyedIdents := make([]unique.Handle[ID], 0, len(idents))
+ for _, pair := range idents {
+ if !dry {
+ if err = os.Remove(c.base.Append(
+ dirStatus,
+ pair.name,
+ ).String()); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return destroyedIdents, nil, err
+ }
+
+ if err = os.Remove(c.base.Append(
+ dirIdentifier,
+ pair.name,
+ ).String()); err != nil {
+ return destroyedIdents, nil, err
+ }
+ }
+ destroyedIdents = append(destroyedIdents, pair.id)
+ }
+
+ destroyedChecksums := make([]unique.Handle[Checksum], 0, len(checksums))
+ for checksum, name := range checksums {
+ if err = c.parent.Err(); err != nil {
+ return destroyedIdents, destroyedChecksums, err
+ }
+ c.msg.Verbosef(
+ "destroying checksum %s%s%s...",
+ c.sgrIdent, name, c.sgrRes,
+ )
+ if !dry {
+ if err = errors.Join(removeAll(c.base.Append(
+ dirChecksum,
+ name,
+ ))); err != nil {
+ return destroyedIdents, destroyedChecksums, err
+ }
+ }
+ destroyedChecksums = append(destroyedChecksums, checksum)
+ }
+
+ dents, err = os.ReadDir(c.base.Append(dirSubstitute).String())
+ if err != nil {
+ return destroyedIdents, destroyedChecksums, err
+ }
+ for _, dent := range dents {
+ name := dent.Name()
+ if err = readlinkChecksum(c.base.Append(
+ dirSubstitute,
+ name,
+ ), &buf); err != nil {
+ return destroyedIdents, destroyedChecksums, err
+ }
+ if _, ok := checksums[unique.Make(buf)]; !ok {
+ continue
+ }
+
+ c.msg.Verbosef(
+ "destroying substitute %s%s%s...",
+ c.sgrIdent, name, c.sgrRes,
+ )
+ if !dry {
+ if err = os.Remove(c.base.Append(
+ dirStatus,
+ name,
+ ).String()); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return destroyedIdents, nil, err
+ }
+
+ if err = os.Remove(c.base.Append(
+ dirSubstitute,
+ name,
+ ).String()); err != nil {
+ return destroyedIdents, destroyedChecksums, err
+ }
+ }
+ }
+
+ return destroyedIdents, destroyedChecksums, nil
+}
diff --git a/pkg/clean_test.go b/pkg/clean_test.go
new file mode 100644
index 00000000..e31d52aa
--- /dev/null
+++ b/pkg/clean_test.go
@@ -0,0 +1,293 @@
+package pkg_test
+
+import (
+ "bytes"
+ "crypto/sha512"
+ "io/fs"
+ "log"
+ "os"
+ "slices"
+ "strings"
+ "testing"
+ "unique"
+
+ "hakurei.app/message"
+ "hakurei.app/pkg"
+)
+
+// formatHandles returns a user-facing string representing h.
+func formatHandles[T pkg.ID | pkg.Checksum](handles ...unique.Handle[T]) string {
+ var buf strings.Builder
+ for _, h := range handles {
+ buf.WriteString(pkg.Encode(pkg.Checksum(h.Value())))
+ buf.WriteString(", ")
+ }
+ return strings.TrimSuffix(buf.String(), ", ")
+}
+
+func TestClean(t *testing.T) {
+ t.Parallel()
+ ic := pkg.NewIR()
+
+ testCases := []struct {
+ name string
+ a []pkg.Artifact
+ keep []pkg.Artifact
+ inputs bool
+ want expectsFS
+
+ wantIdents []unique.Handle[pkg.ID]
+ wantChecksums []unique.Handle[pkg.Checksum]
+ }{
+ {"simple", []pkg.Artifact{
+ pkg.NewFile("file", nil),
+ }, nil, false, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/h6qx7m0vsyhO6oLEJgY2rKa1c6RZuIZ4QXxZSh__NZJ9VU0MIj5bFB-KuaPcsfpE": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")},
+
+ "lock": {Mode: 0644},
+ "variant": {Mode: 0400},
+ "status": {Mode: fs.ModeDir | 0700},
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "fault": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }, nil, nil},
+
+ {"keep", []pkg.Artifact{
+ pkg.NewFile("removed-file", []byte("removed file")),
+ }, []pkg.Artifact{
+ pkg.NewFile("file", []byte("\xfd")),
+ }, false, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/KgZ-FjbGuU-XP2QEHInpgv-2Zn0cTH5NqFMgTU0XrSdKmSwyC-3baVs1BMCP5spk": {Mode: 0400, Data: []byte("\xfd")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/tkrOYhQfTRaArk4nNYNjTmHQUmuJMVIoDeAKIQBm3I4uuMimBKXKMIY2gGZqwQg4": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/KgZ-FjbGuU-XP2QEHInpgv-2Zn0cTH5NqFMgTU0XrSdKmSwyC-3baVs1BMCP5spk")},
+
+ "lock": {Mode: 0644},
+ "variant": {Mode: 0400},
+ "status": {Mode: fs.ModeDir | 0700},
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "fault": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }, []unique.Handle[pkg.ID]{
+ ic.Ident(pkg.NewFile("removed-file", []byte("removed file"))),
+ }, []unique.Handle[pkg.Checksum]{
+ unique.Make(sha512.Sum384([]byte("removed file"))),
+ }},
+
+ {"inputs anchored substitute", []pkg.Artifact{
+ &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("destroyed"),
+ deps: []pkg.Artifact{
+ pkg.NewFile("destroyed-input", []byte("destroyed")),
+ },
+ cure: func(f *pkg.FContext) error {
+ p := f.GetWorkDir()
+ if err := os.MkdirAll(p.String(), 0755); err != nil {
+ return err
+ }
+ return os.WriteFile(p.Append("result").String(), nil, 0444)
+ },
+ },
+ }, []pkg.Artifact{
+ &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("kept"),
+ deps: []pkg.Artifact{
+ pkg.NewFile("kept-input", []byte("kept")),
+ },
+ cure: func(f *pkg.FContext) error {
+ p := f.GetWorkDir()
+ if err := os.MkdirAll(p.String(), 0755); err != nil {
+ return err
+ }
+ return os.WriteFile(p.Append("result").String(), nil, 0444)
+ },
+ },
+ }, true, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/H-eSiCo227-xdqyNl2R-5G3eqXPtbb8XegAB70I5OQb2majeZXJoCxTq9wJy5qqv": {Mode: 0400, Data: []byte("kept")},
+ "checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE": {Mode: fs.ModeDir | 0500},
+ "checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE/result": {Mode: 0444},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/KpxTUsUfGbsX6C6HU2j9cd9He3_tt2o2m2MgH4m0zial8lmoqSWY-Gc3KrS0k-UK": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/H-eSiCo227-xdqyNl2R-5G3eqXPtbb8XegAB70I5OQb2majeZXJoCxTq9wJy5qqv")},
+ "identifier/xxQ43VMIS1WHMSEFnUdBNd6pulbtgiTorP_mtJpEsZENYhGBiDEL6Y2D8yqn0Sra": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")},
+
+ "lock": {Mode: 0644},
+ "variant": {Mode: 0400},
+ "status": {Mode: fs.ModeDir | 0700},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "substitute/dP-6_wIDRRouOaOF-nkBy-IaUbLdYHbOUTrptu7j_qfW01mnHSjYJ0oykUmvUd2x": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")},
+ "substitute/td1tf1kb3z8iUFg2k1xlHeTTEz3xR4e77WSWdv2JhjmZyY6iuvodhdT8BblDOi0E": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")},
+
+ "fault": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }, []unique.Handle[pkg.ID]{
+ ic.Ident(pkg.NewFile("destroyed-input", []byte("destroyed"))),
+ ic.Ident(&stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("destroyed"),
+ deps: []pkg.Artifact{
+ pkg.NewFile("destroyed-input", []byte("destroyed")),
+ },
+ }),
+ }, []unique.Handle[pkg.Checksum]{
+ unique.Make(sha512.Sum384([]byte("destroyed"))),
+ }},
+
+ {"inputs", []pkg.Artifact{
+ &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("destroyed"),
+ deps: []pkg.Artifact{
+ pkg.NewFile("destroyed-input", []byte("destroyed")),
+ },
+ cure: func(f *pkg.FContext) error {
+ if w, err := f.GetStatusWriter(); err != nil {
+ return err
+ } else if _, err = w.Write([]byte("destroyed")); err != nil {
+ return err
+ }
+
+ p := f.GetWorkDir()
+ if err := os.MkdirAll(p.String(), 0755); err != nil {
+ return err
+ }
+ return os.WriteFile(p.Append("result").String(), nil, 0444)
+ },
+ },
+ }, []pkg.Artifact{
+ &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("kept"),
+ deps: []pkg.Artifact{
+ pkg.NewFile("kept-input", []byte("kept")),
+ },
+ cure: func(f *pkg.FContext) error {
+ if w, err := f.GetStatusWriter(); err != nil {
+ return err
+ } else if _, err = w.Write([]byte("kept")); err != nil {
+ return err
+ }
+
+ p := f.GetWorkDir()
+ if err := os.MkdirAll(p.String(), 0755); err != nil {
+ return err
+ }
+ return os.WriteFile(p.Append("result").String(), []byte{0}, 0444)
+ },
+ },
+ }, true, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/CyDnDvF-LaeGPcSW70tPosNCoclByWkTjznUUF1DcgzlIwkN9yzz1ZFME1TlPj6W": {Mode: fs.ModeDir | 0500},
+ "checksum/CyDnDvF-LaeGPcSW70tPosNCoclByWkTjznUUF1DcgzlIwkN9yzz1ZFME1TlPj6W/result": {Mode: 0444, Data: []byte("\x00")},
+ "checksum/H-eSiCo227-xdqyNl2R-5G3eqXPtbb8XegAB70I5OQb2majeZXJoCxTq9wJy5qqv": {Mode: 0400, Data: []byte("kept")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/KpxTUsUfGbsX6C6HU2j9cd9He3_tt2o2m2MgH4m0zial8lmoqSWY-Gc3KrS0k-UK": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/H-eSiCo227-xdqyNl2R-5G3eqXPtbb8XegAB70I5OQb2majeZXJoCxTq9wJy5qqv")},
+ "identifier/xxQ43VMIS1WHMSEFnUdBNd6pulbtgiTorP_mtJpEsZENYhGBiDEL6Y2D8yqn0Sra": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/CyDnDvF-LaeGPcSW70tPosNCoclByWkTjznUUF1DcgzlIwkN9yzz1ZFME1TlPj6W")},
+
+ "lock": {Mode: 0644},
+ "variant": {Mode: 0400},
+
+ "status": {Mode: fs.ModeDir | 0700},
+ "status/xxQ43VMIS1WHMSEFnUdBNd6pulbtgiTorP_mtJpEsZENYhGBiDEL6Y2D8yqn0Sra": {Mode: 0400, Data: []byte(statusHeader + "kept")},
+ "status/td1tf1kb3z8iUFg2k1xlHeTTEz3xR4e77WSWdv2JhjmZyY6iuvodhdT8BblDOi0E": {Mode: 0400, Data: []byte(statusHeader + "kept")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "substitute/td1tf1kb3z8iUFg2k1xlHeTTEz3xR4e77WSWdv2JhjmZyY6iuvodhdT8BblDOi0E": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/CyDnDvF-LaeGPcSW70tPosNCoclByWkTjznUUF1DcgzlIwkN9yzz1ZFME1TlPj6W")},
+
+ "fault": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }, []unique.Handle[pkg.ID]{
+ ic.Ident(pkg.NewFile("destroyed-input", []byte("destroyed"))),
+ ic.Ident(&stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("destroyed"),
+ deps: []pkg.Artifact{
+ pkg.NewFile("destroyed-input", []byte("destroyed")),
+ },
+ }),
+ }, []unique.Handle[pkg.Checksum]{
+ unique.Make(expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+ "result": {Mode: 0444},
+ }.hash()),
+ unique.Make(sha512.Sum384([]byte("destroyed"))),
+ }},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ base := makeBase(t)
+ msg := message.New(log.New(os.Stderr, "clean: ", 0))
+ msg.SwapVerbose(testing.Verbose())
+ c, err := pkg.Open(t.Context(), msg, base, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(c.Close)
+
+ all := pkg.Collect(slices.Concat(tc.a, tc.keep))
+ if _, _, err = c.Cure(&all); !pkg.IsCollected(err) {
+ t.Fatal(err)
+ }
+
+ var (
+ idents []unique.Handle[pkg.ID]
+ checksums []unique.Handle[pkg.Checksum]
+ )
+ idents, checksums, err = c.Clean(false, tc.inputs, tc.keep...)
+ if err != nil {
+ t.Fatalf("Clean: error = %v", err)
+ }
+ var buf [2]pkg.Checksum
+
+ slices.SortFunc(idents, func(a, b unique.Handle[pkg.ID]) int {
+ buf[0], buf[1] = a.Value(), b.Value()
+ return bytes.Compare(buf[0][:], buf[1][:])
+ })
+ slices.SortFunc(checksums, func(a, b unique.Handle[pkg.Checksum]) int {
+ buf[0], buf[1] = a.Value(), b.Value()
+ return bytes.Compare(buf[0][:], buf[1][:])
+ })
+
+ if !slices.Equal(idents, tc.wantIdents) {
+ t.Errorf(
+ "Clean: idents = %s, want %s",
+ formatHandles(idents...), formatHandles(tc.wantIdents...),
+ )
+ }
+ if !slices.Equal(checksums, tc.wantChecksums) {
+ t.Errorf(
+ "Clean: checksums = %s, want %s",
+ formatHandles(checksums...), formatHandles(tc.wantChecksums...),
+ )
+ }
+
+ want := tc.want.hash()
+ var checksum pkg.Checksum
+ if err = pkg.SumDir(&checksum, base); err != nil {
+ t.Fatalf("SumDir: error = %v", err)
+ } else if checksum != want {
+ t.Error(expectsFrom(base.String()))
+ }
+ })
+ }
+}
diff --git a/pkg/compress.go b/pkg/compress.go
new file mode 100644
index 00000000..22f990f1
--- /dev/null
+++ b/pkg/compress.go
@@ -0,0 +1,151 @@
+package pkg
+
+import (
+ "compress/bzip2"
+ "compress/gzip"
+ "fmt"
+ "io"
+ "os"
+
+ "hakurei.app/internal/xz"
+ "hakurei.app/internal/zstd"
+)
+
+const (
+ // Gzip denotes a stream compressed via [gzip].
+ Gzip = iota
+ // Bzip2 denotes a stream compressed via [bzip2].
+ Bzip2
+ // Zstd denotes a stream compressed via [zstd].
+ Zstd
+ // XZ denotes a stream compressed via [xz].
+ XZ
+)
+
+// A decompressArtifact is a [FileArtifact] decompressing a backing
+// [FileArtifact] stream.
+type decompressArtifact struct {
+ // Caller-supplied backing stream.
+ f Artifact
+ // Compression on top of the stream.
+ compress uint32
+}
+
+var _ FileArtifact = new(decompressArtifact)
+var _ CuresExempt = new(decompressArtifact)
+
+// decompressArtifactNamed embeds decompressArtifact for a [fmt.Stringer] stream.
+type decompressArtifactNamed struct {
+ decompressArtifact
+ // Copied from decompressArtifact.f.
+ name string
+}
+
+var _ fmt.Stringer = new(decompressArtifactNamed)
+
+// NewDecompress returns a [FileArtifact] decompressing the supplied [Artifact].
+func NewDecompress(a Artifact, compress uint32) FileArtifact {
+ da := decompressArtifact{a, compress}
+ if s, ok := a.(fmt.Stringer); ok {
+ if name := s.String(); name != "" {
+ return &decompressArtifactNamed{da, name}
+ }
+ }
+ return &da
+}
+
+// String returns the name of the underlying [Artifact] prefixed with decompress.
+func (a *decompressArtifactNamed) String() string { return "decompress-" + a.name }
+
+// Kind returns the hardcoded [Kind] constant.
+func (a *decompressArtifact) Kind() Kind { return KindDecompress }
+
+// Params writes value of compression enum.
+func (a *decompressArtifact) Params(ctx *IContext) { ctx.WriteUint32(a.compress) }
+
+func init() {
+ register(KindDecompress, func(r *IRReader) Artifact {
+ a := NewDecompress(r.Next(), r.ReadUint32())
+ if _, ok := r.Finalise(); ok {
+ panic(ErrUnexpectedChecksum)
+ }
+ return a
+ })
+}
+
+// Inputs returns a slice containing the backing file.
+func (a *decompressArtifact) Inputs() []Artifact {
+ return []Artifact{a.f}
+}
+
+// IsExclusive returns false: decompressor is fully sequential.
+func (a *decompressArtifact) IsExclusive() bool { return false }
+
+// compoundCloser is an [io.ReadCloser] with an additional [io.Closer] attached.
+type compoundCloser struct {
+ io.ReadCloser
+ c io.Closer
+}
+
+// Close closes [io.ReadCloser] and the additional [io.Closer]. It returns the
+// non-nil error returned by the underlying [io.ReadCloser], otherwise it
+// returns the error returned by the additional [io.Closer].
+func (c compoundCloser) Close() error {
+ err := c.ReadCloser.Close()
+ if _err := c.c.Close(); err == nil {
+ err = _err
+ }
+ return err
+}
+
+// IsExecutable returns false.
+func (*decompressArtifact) IsExecutable() bool { return false }
+
+// Cure returns a decompressor [io.ReadCloser].
+func (a *decompressArtifact) Cure(r *RContext) (io.ReadCloser, error) {
+ sr, err := r.Open(a.f)
+ if err != nil {
+ return nil, err
+ }
+ br := r.cache.getReaderRC(sr)
+
+ var dr io.ReadCloser
+ switch a.compress {
+ case Gzip:
+ if dr, err = gzip.NewReader(br); err != nil {
+ _ = br.Close()
+ return nil, err
+ }
+ return compoundCloser{dr, br}, nil
+
+ case Bzip2:
+ return struct {
+ io.Reader
+ io.Closer
+ }{bzip2.NewReader(br), br}, nil
+
+ case Zstd:
+ return struct {
+ io.Reader
+ io.Closer
+ }{zstd.NewReader(br), br}, nil
+
+ case XZ:
+ var _dr io.Reader
+ if _dr, err = xz.NewReader(br, 0); err != nil {
+ _ = br.Close()
+ return nil, err
+ }
+ return struct {
+ io.Reader
+ io.Closer
+ }{_dr, br}, nil
+
+ default:
+ return nil, os.ErrInvalid
+ }
+}
+
+// CuresExempt exempts the cheap [KindDecompress] implementation often part of
+// a [FileArtifact] pipeline.
+func (*decompressArtifact) CuresExempt() {}
diff --git a/pkg/compress_test.go b/pkg/compress_test.go
new file mode 100644
index 00000000..d737cdd7
--- /dev/null
+++ b/pkg/compress_test.go
@@ -0,0 +1,72 @@
+package pkg_test
+
+import (
+ "bytes"
+ "compress/gzip"
+ "crypto/sha512"
+ "io/fs"
+ "net/http"
+ "testing"
+ "testing/fstest"
+
+ "hakurei.app/check"
+ "hakurei.app/pkg"
+)
+
+func TestDecompress(t *testing.T) {
+ t.Parallel()
+
+ var buf bytes.Buffer
+ gw := gzip.NewWriter(&buf)
+ if _, err := gw.Write([]byte{0}); err != nil {
+ t.Fatal(err)
+ } else if err = gw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ testdata := buf.String()
+
+ var transport http.Transport
+ client := http.Client{Transport: &transport}
+ transport.RegisterProtocol("file", http.NewFileTransportFS(fstest.MapFS{
+ "testdata": {Data: []byte(testdata), Mode: 0400},
+ }))
+ testdataChecksum := func() pkg.Checksum {
+ h := sha512.New384()
+ h.Write([]byte(testdata))
+ return (pkg.Checksum)(h.Sum(nil))
+ }()
+
+ gh := pkg.NewDecompress(pkg.NewHTTPGet(
+ &client,
+ "file:///testdata",
+ testdataChecksum,
+ ), pkg.Gzip)
+
+ checkWithCache(t, []cacheTestCase{
+ {"decompress", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ cureMany(t, c, []cureStep{
+ {"close", pkg.NewDecompress(pkg.NewHTTPGet(
+ &client,
+ "file:///testdata",
+ pkg.Checksum{0xfd},
+ ), pkg.Gzip), nil, nil, pkg.WNew, &pkg.ChecksumMismatchError{
+ Got: testdataChecksum,
+ Want: pkg.Checksum{0xfd},
+ }},
+
+ {"gzip", gh, ignorePathname, expectsChecksum(sha512.Sum384([]byte{0})), pkg.WNew, nil},
+ })
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/" + pkg.Encode(sha512.Sum384([]byte{0})): {Mode: 0400, Data: []byte{0}},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/" + pkg.Encode(pkg.NewIR().Ident(gh).Value()): {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+ })
+}
diff --git a/pkg/exec.go b/pkg/exec.go
new file mode 100644
index 00000000..b40ae84b
--- /dev/null
+++ b/pkg/exec.go
@@ -0,0 +1,808 @@
+package pkg
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "slices"
+ "strconv"
+ "sync"
+ "syscall"
+ "time"
+ "unique"
+
+ "hakurei.app/check"
+ "hakurei.app/container"
+ "hakurei.app/container/seccomp"
+ "hakurei.app/container/std"
+ "hakurei.app/ext"
+ "hakurei.app/fhs"
+ "hakurei.app/message"
+)
+
+// AbsWork is the container pathname [TContext.GetWorkDir] is mounted on.
+var AbsWork = fhs.AbsRoot.Append("work/")
+
+const (
+ // EnvJobs is the name of the environment variable holding a decimal
+ // representation of the preferred job count. Its value must not affect cure
+ // outcome.
+ EnvJobs = "CURE_JOBS"
+ // EnvLoad is the name of the environment variable holding a decimal
+ // representation of the preferred loadavg target. Its value must not affect
+ // cure outcome.
+ EnvLoad = "CURE_LOAD"
+)
+
+// ExecPath is a slice of [Artifact] and the [check.Absolute] pathname to make
+// it available at under in the container.
+type ExecPath struct {
+ // Pathname in the container mount namespace.
+ P *check.Absolute
+ // Artifacts to mount on the pathname, must contain at least one [Artifact].
+ // If there are multiple entries or W is true, P is set up as an overlay
+ // mount, and entries of A must not implement [FileArtifact].
+ A []Artifact
+ // Whether to make the mount point writable via the temp directory.
+ W bool
+}
+
+// GetArtifactFunc is the function signature of [FContext.GetArtifact].
+type GetArtifactFunc func(Artifact) (*check.Absolute, unique.Handle[Checksum])
+
+// PromoteLayers returns artifacts with identical-by-content layers promoted to
+// the highest priority instance, as if mounted via [ExecPath].
+func PromoteLayers(
+ artifacts []Artifact,
+ getArtifact GetArtifactFunc,
+ report func(i int, d Artifact),
+) []*check.Absolute {
+ layers := make([]*check.Absolute, 0, len(artifacts))
+ checksums := make(map[unique.Handle[Checksum]]struct{}, len(artifacts))
+ for i := range artifacts {
+ d := artifacts[len(artifacts)-1-i]
+ pathname, checksum := getArtifact(d)
+ if _, ok := checksums[checksum]; ok {
+ report(len(artifacts)-1-i, d)
+ continue
+ }
+ checksums[checksum] = struct{}{}
+ layers = append(layers, pathname)
+ }
+ slices.Reverse(layers)
+ return layers
+}
+
+// layers returns pathnames collected from A deduplicated via [PromoteLayers].
+func (p *ExecPath) layers(
+ msg message.Msg,
+ getArtifact GetArtifactFunc,
+ ident func(a Artifact) unique.Handle[ID],
+) []*check.Absolute {
+ return PromoteLayers(p.A, getArtifact, func(i int, d Artifact) {
+ if msg.IsVerbose() {
+ msg.Verbosef("promoted layer %d as %s", i, reportName(d, ident(d)))
+ }
+ })
+}
+
+// Path returns a populated [ExecPath].
+func Path(pathname *check.Absolute, writable bool, a ...Artifact) ExecPath {
+ return ExecPath{pathname, a, writable}
+}
+
+// MustPath is like [Path], but takes a string pathname via [check.MustAbs].
+func MustPath(pathname string, writable bool, a ...Artifact) ExecPath {
+ return ExecPath{check.MustAbs(pathname), a, writable}
+}
+
+var (
+ binfmt map[string]container.BinfmtEntry
+ binfmtMu sync.RWMutex
+)
+
+// RegisterArch arranges for [KindExec] and [KindExecNet] to support a new
+// architecture via a binfmt_misc entry. Each architecture must be registered
+// at most once.
+func RegisterArch(arch string, e container.BinfmtEntry) {
+ if arch == "" {
+ panic(UnsupportedArchError(arch))
+ }
+
+ binfmtMu.Lock()
+ defer binfmtMu.Unlock()
+
+ if binfmt == nil {
+ binfmt = make(map[string]container.BinfmtEntry)
+ }
+
+ if _, ok := binfmt[arch]; ok {
+ panic("attempting to register " + strconv.Quote(arch) + " twice")
+ }
+ binfmt[arch] = e
+}
+
+const (
+ // ExecTimeoutDefault replaces out of range [NewExec] timeout values.
+ ExecTimeoutDefault = 15 * time.Minute
+ // ExecTimeoutMax is the arbitrary upper bound of [NewExec] timeout.
+ ExecTimeoutMax = 48 * time.Hour
+)
+
+// An execArtifact is an [Artifact] that produces output by running a program
+// part of another [Artifact] in a [container] to produce its output.
+//
+// Methods of execArtifact does not modify any struct field or underlying arrays
+// referred to by slices.
+type execArtifact struct {
+ // Caller-supplied user-facing reporting name, guaranteed to be nonzero
+ // during initialisation.
+ name string
+ // Target architecture.
+ arch string
+ // Caller-supplied inner mount points.
+ paths []ExecPath
+
+ // Passed through to [container.Params].
+ dir *check.Absolute
+ // Passed through to [container.Params].
+ env []string
+ // Passed through to [container.Params].
+ path *check.Absolute
+ // Passed through to [container.Params].
+ args []string
+
+ // Duration the initial process is allowed to run. The zero value is
+ // equivalent to [ExecTimeoutDefault].
+ timeout time.Duration
+
+ // Caller-supplied exclusivity value, returned as is by IsExclusive.
+ exclusive bool
+}
+
+var _ fmt.Stringer = new(execArtifact)
+
+// execMeasuredArtifact is like execArtifact but implements [KnownChecksum] and
+// has its resulting container optionally keep the host net namespace.
+type execMeasuredArtifact struct {
+ checksum Checksum
+
+ // Whether to keep host net namespace.
+ hostNet bool
+
+ execArtifact
+}
+
+var _ KnownChecksum = new(execMeasuredArtifact)
+
+// Checksum returns the caller-supplied checksum.
+func (a *execMeasuredArtifact) Checksum() Checksum { return a.checksum }
+
+// Kind returns [KindExecNet], or [KindExec] if hostNet is false.
+func (a *execMeasuredArtifact) Kind() Kind {
+ if a == nil || a.hostNet {
+ return KindExecNet
+ }
+ return KindExec
+}
+
+// Cure cures the [Artifact] in the container described by the caller. The
+// container optionally retains host networking.
+func (a *execMeasuredArtifact) Cure(f *FContext) error {
+ return a.cure(f, a.hostNet)
+}
+
+// ErrNetChecksum is panicked by [NewExec] if host net namespace is requested
+// with a nil checksum.
+var ErrNetChecksum = errors.New("attempting to keep net namespace without checksum")
+
+// NewExec returns a new [Artifact] that executes the program path in a
+// container with specified paths bind mounted read-only in order. A private
+// instance of /proc and /dev is made available to the container.
+//
+// The working and temporary directories are both created and mounted writable
+// on [AbsWork] and [fhs.AbsTmp] respectively. If one or more paths target
+// [AbsWork], the final entry is set up as a writable overlay mount on /work for
+// which the upperdir is the host side work directory. In this configuration,
+// the W field is ignored, and the program must avoid causing whiteout files to
+// be created. Cure fails if upperdir ends up with entries other than directory,
+// regular or symlink.
+//
+// If checksum is non-nil, the resulting [Artifact] implements [KnownChecksum]
+// and its container optionally runs in the host net namespace.
+//
+// The container is allowed to run for the specified duration before the initial
+// process and all processes originating from it is terminated. A zero or
+// negative timeout value is equivalent tp [ExecTimeoutDefault], a timeout value
+// greater than [ExecTimeoutMax] is equivalent to [ExecTimeoutMax].
+//
+// The user-facing name and exclusivity value are not accessible from the
+// container and does not affect curing outcome. Because of this, it is omitted
+// from parameter data for computing identifier.
+func NewExec(
+ name, arch string,
+ checksum *Checksum,
+ timeout time.Duration,
+ hostNet, exclusive bool,
+
+ dir *check.Absolute,
+ env []string,
+ pathname *check.Absolute,
+ args []string,
+
+ paths ...ExecPath,
+) Artifact {
+ if name == "" {
+ name = "exec-" + filepath.Base(pathname.String())
+ }
+ if arch == "" {
+ arch = runtime.GOARCH
+ }
+ if timeout <= 0 {
+ timeout = ExecTimeoutDefault
+ }
+ if timeout > ExecTimeoutMax {
+ timeout = ExecTimeoutMax
+ }
+ a := execArtifact{name, arch, paths, dir, env, pathname, args, timeout, exclusive}
+ if checksum == nil {
+ if hostNet {
+ panic(ErrNetChecksum)
+ }
+ return &a
+ }
+ return &execMeasuredArtifact{*checksum, hostNet, a}
+}
+
+// Kind returns the hardcoded [Kind] constant.
+func (*execArtifact) Kind() Kind { return KindExec }
+
+// Params writes paths, executable pathname and args.
+func (a *execArtifact) Params(ctx *IContext) {
+ ctx.WriteString(a.arch)
+ ctx.WriteString(a.name)
+
+ ctx.WriteUint32(uint32(len(a.paths)))
+ for _, p := range a.paths {
+ if p.P != nil {
+ ctx.WriteString(p.P.String())
+ } else {
+ ctx.WriteString("invalid P\x00")
+ }
+
+ ctx.WriteUint32(uint32(len(p.A)))
+ for _, d := range p.A {
+ ctx.WriteIdent(d)
+ }
+
+ if p.W {
+ ctx.WriteUint32(1)
+ } else {
+ ctx.WriteUint32(0)
+ }
+ }
+
+ ctx.WriteString(a.dir.String())
+
+ ctx.WriteUint32(uint32(len(a.env)))
+ for _, e := range a.env {
+ ctx.WriteString(e)
+ }
+
+ ctx.WriteString(a.path.String())
+
+ ctx.WriteUint32(uint32(len(a.args)))
+ for _, arg := range a.args {
+ ctx.WriteString(arg)
+ }
+
+ ctx.WriteUint32(uint32(a.timeout & 0xffffffff))
+ ctx.WriteUint32(uint32(a.timeout >> 32))
+
+ if a.exclusive {
+ ctx.WriteUint32(1)
+ } else {
+ ctx.WriteUint32(0)
+ }
+}
+
+// UnsupportedArchError describes an unsupported or invalid architecture.
+type UnsupportedArchError string
+
+func (e UnsupportedArchError) Error() string {
+ if e == "" {
+ return "invalid architecture name"
+ }
+ return "unsupported architecture " + string(e)
+}
+
+// readExecArtifact interprets IR values and returns the address of execArtifact
+// or execNetArtifact.
+func readExecArtifact(r *IRReader, net bool) Artifact {
+ r.DiscardAll()
+
+ arch := r.ReadString()
+ if arch == "" {
+ panic(UnsupportedArchError(arch))
+ }
+
+ name := r.ReadString()
+
+ sz := r.ReadUint32()
+ if sz > irMaxDeps {
+ panic(ErrIRDepend)
+ }
+ paths := make([]ExecPath, sz)
+ for i := range paths {
+ paths[i].P = check.MustAbs(r.ReadString())
+
+ sz = r.ReadUint32()
+ if sz > irMaxDeps {
+ panic(ErrIRDepend)
+ }
+ paths[i].A = make([]Artifact, sz)
+ for j := range paths[i].A {
+ paths[i].A[j] = r.ReadIdent()
+ }
+
+ paths[i].W = r.ReadUint32() != 0
+ }
+
+ dir := check.MustAbs(r.ReadString())
+
+ sz = r.ReadUint32()
+ if sz > irMaxValues {
+ panic(ErrIRValues)
+ }
+ env := make([]string, sz)
+ for i := range env {
+ env[i] = r.ReadString()
+ }
+
+ pathname := check.MustAbs(r.ReadString())
+
+ sz = r.ReadUint32()
+ if sz > irMaxValues {
+ panic(ErrIRValues)
+ }
+ args := make([]string, sz)
+ for i := range args {
+ args[i] = r.ReadString()
+ }
+
+ timeout := time.Duration(r.ReadUint32())
+ timeout |= time.Duration(r.ReadUint32()) << 32
+
+ exclusive := r.ReadUint32() != 0
+
+ checksum, ok := r.Finalise()
+ var checksumP *Checksum
+ if ok {
+ checksumP = new(checksum.Value())
+ }
+
+ if net && !ok {
+ panic(ErrExpectedChecksum)
+ }
+
+ return NewExec(
+ name, arch, checksumP, timeout, net, exclusive, dir, env, pathname, args, paths...,
+ )
+}
+
+func init() {
+ register(KindExec,
+ func(r *IRReader) Artifact { return readExecArtifact(r, false) })
+ register(KindExecNet,
+ func(r *IRReader) Artifact { return readExecArtifact(r, true) })
+}
+
+// Inputs returns a slice of all artifacts collected from caller-supplied
+// [ExecPath].
+func (a *execArtifact) Inputs() []Artifact {
+ artifacts := make([][]Artifact, 0, len(a.paths))
+ for _, p := range a.paths {
+ artifacts = append(artifacts, p.A)
+ }
+ return slices.Concat(artifacts...)
+}
+
+// IsExclusive returns the caller-supplied exclusivity value.
+func (a *execArtifact) IsExclusive() bool { return a.exclusive }
+
+// String returns the caller-supplied reporting name.
+func (a *execArtifact) String() string { return a.name }
+
+// Cure cures the [Artifact] in the container described by the caller.
+func (a *execArtifact) Cure(f *FContext) (err error) {
+ return a.cure(f, false)
+}
+
+const (
+ // execWaitDelay is passed through to [container.Params].
+ execWaitDelay = time.Nanosecond
+)
+
+// scanLinesCR is like [bufio.ScanLines], but also treats a bare \r as an
+// end-of-line marker.
+func scanLinesCR(data []byte, atEOF bool) (advance int, token []byte, err error) {
+ if atEOF && len(data) == 0 {
+ return 0, nil, nil
+ }
+ ri, ni := bytes.IndexByte(data, '\r'), bytes.IndexByte(data, '\n')
+
+ if ri >= 0 && (ni < 0 || ri < ni) {
+ if ri+1 == ni {
+ // We have a full \r\n-terminated line.
+ return ri + 2, data[:ri], nil
+ }
+ // We have a bare \r, probably some kind of progress indicator.
+ return ri + 1, data[:ri], nil
+ }
+ if ni >= 0 && (ri < 0 || ni < ri) {
+ // We have a full newline-terminated line.
+ return ni + 1, data[:ni], nil
+ }
+ // If we're at EOF, we have a final, non-terminated line. Return it.
+ if atEOF {
+ return len(data), data, nil
+ }
+ // Request more data.
+ return 0, nil, nil
+}
+
+// scanVerbose prefixes program output for a verbose [message.Msg].
+func scanVerbose(
+ msg message.Msg,
+ cancel context.CancelFunc,
+ done chan<- struct{},
+ prefix, suffix string,
+ r io.Reader,
+) {
+ defer close(done)
+ s := bufio.NewScanner(r)
+ s.Split(scanLinesCR)
+ s.Buffer(
+ make([]byte, bufio.MaxScanTokenSize),
+ bufio.MaxScanTokenSize<<12,
+ )
+ for s.Scan() {
+ msg.Verbose(prefix, s.Text()+suffix)
+ }
+ if err := s.Err(); err != nil && !errors.Is(err, os.ErrClosed) {
+ cancel()
+ msg.Verbose("*"+prefix, err.Error()+suffix)
+ }
+}
+
+var (
+ // ErrInvalidPaths is returned for an [Artifact] of [KindExec] or
+ // [KindExecNet] specified with invalid paths.
+ ErrInvalidPaths = errors.New("invalid mount point")
+)
+
+// SeccompPresets is the [seccomp] presets used by exec artifacts.
+const SeccompPresets = std.PresetStrict &
+ ^(std.PresetDenyNS | std.PresetDenyDevel)
+
+// makeContainer sets up the specified temp and work directories and returns the
+// corresponding [container.Container] that would have run for cure.
+func (a *execArtifact) makeContainer(
+ ctx context.Context,
+ msg message.Msg,
+ flags, jobs, load int,
+ hostNet bool,
+ temp, work *check.Absolute,
+ getArtifact GetArtifactFunc,
+ ident func(a Artifact) unique.Handle[ID],
+) (z *container.Container, err error) {
+ overlayWorkIndex := -1
+ for i, p := range a.paths {
+ if p.P == nil || len(p.A) == 0 {
+ return nil, ErrInvalidPaths
+ }
+ if p.P.Is(AbsWork) {
+ overlayWorkIndex = i
+ }
+ }
+
+ var artifactCount int
+ for _, p := range a.paths {
+ artifactCount += len(p.A)
+ }
+
+ z = container.New(ctx, msg)
+ z.WaitDelay = execWaitDelay
+ z.SeccompPresets = SeccompPresets
+ z.SeccompFlags |= seccomp.AllowMultiarch
+ z.ParentPerm = 0700
+ z.HostNet = hostNet
+ z.HostAbstract = flags&CHostAbstract != 0
+ z.Hostname = "cure"
+ z.SetScheduler = flags&CSchedIdle != 0
+ z.SchedPolicy = ext.SCHED_IDLE
+ if z.HostNet {
+ z.Hostname = "cure-net"
+ }
+ z.Quiet = flags&CSuppressInit != 0
+ z.Uid, z.Gid = (1<<10)-1, (1<<10)-1
+ z.Dir, z.Path, z.Args = a.dir, a.path, a.args
+ z.Env = slices.Concat(a.env, []string{
+ EnvJobs + "=" + strconv.Itoa(jobs),
+ EnvLoad + "=" + strconv.Itoa(load),
+ })
+ z.Grow(len(a.paths) + 4)
+
+ if a.arch != runtime.GOARCH {
+ binfmtMu.RLock()
+ e, ok := binfmt[a.arch]
+ binfmtMu.RUnlock()
+ if !ok {
+ return nil, UnsupportedArchError(a.arch)
+ }
+ z.Binfmt = []container.BinfmtEntry{e}
+ z.InitAsRoot = true
+ }
+
+ for i, b := range a.paths {
+ if i == overlayWorkIndex {
+ if err = os.MkdirAll(work.String(), 0700); err != nil {
+ return
+ }
+ tempWork := temp.Append(".work")
+ if err = os.MkdirAll(tempWork.String(), 0700); err != nil {
+ return
+ }
+ z.Overlay(
+ AbsWork,
+ work,
+ tempWork,
+ b.layers(msg, getArtifact, ident)...,
+ )
+ continue
+ }
+
+ if a.paths[i].W {
+ tempUpper, tempWork := temp.Append(
+ ".upper", strconv.Itoa(i),
+ ), temp.Append(
+ ".work", strconv.Itoa(i),
+ )
+ if err = os.MkdirAll(tempUpper.String(), 0700); err != nil {
+ return
+ }
+ if err = os.MkdirAll(tempWork.String(), 0700); err != nil {
+ return
+ }
+ z.Overlay(b.P, tempUpper, tempWork, b.layers(msg, getArtifact, ident)...)
+ } else if len(b.A) == 1 {
+ pathname, _ := getArtifact(b.A[0])
+ z.Bind(pathname, b.P, 0)
+ } else {
+ z.OverlayReadonly(b.P, b.layers(msg, getArtifact, ident)...)
+ }
+ }
+ if overlayWorkIndex < 0 {
+ z.Bind(
+ work,
+ AbsWork,
+ std.BindWritable|std.BindEnsure,
+ )
+ }
+ z.Bind(
+ temp,
+ fhs.AbsTmp,
+ std.BindWritable|std.BindEnsure,
+ )
+ z.Proc(fhs.AbsProc).Dev(fhs.AbsDev, true)
+ return
+}
+
+var (
+ // ErrExecBusy is returned entering [Cache.EnterExec] while another
+ // goroutine has not yet returned from it.
+ ErrExecBusy = errors.New("scratch directories in use")
+ // ErrNotExec is returned for unsupported implementations of [Artifact]
+ // passed to [Cache.EnterExec].
+ ErrNotExec = errors.New("attempting to run a non-exec artifact")
+)
+
+// EnterExec runs the container of an [Artifact] of [KindExec] or [KindExecNet]
+// with its entry point, argument, and standard streams replaced with values
+// supplied by the caller.
+func (c *Cache) EnterExec(
+ ctx context.Context,
+ a Artifact,
+ hostname string,
+ retainSession bool,
+ stdin io.Reader,
+ stdout, stderr io.Writer,
+ path *check.Absolute,
+ args ...string,
+) (err error) {
+ if !c.inExec.CompareAndSwap(false, true) {
+ return ErrExecBusy
+ }
+ defer c.inExec.Store(false)
+
+ var hostNet bool
+ var e *execArtifact
+ switch f := a.(type) {
+ case *execArtifact:
+ e = f
+
+ case *execMeasuredArtifact:
+ e = &f.execArtifact
+ hostNet = f.hostNet
+
+ default:
+ return ErrNotExec
+ }
+
+ deps := Collect(a.Inputs())
+ if _, _, err = c.Cure(&deps); err == nil {
+ return errors.New("unreachable")
+ } else if !IsCollected(err) {
+ return
+ }
+
+ dm := make(map[Artifact]cureRes)
+ for i, p := range deps {
+ var res cureRes
+ res.pathname, res.checksum, err = c.Cure(p)
+ if err != nil {
+ return
+ }
+ dm[deps[i]] = res
+ }
+
+ scratch := c.base.Append(dirExecScratch)
+ temp, work := scratch.Append("temp"), scratch.Append("work")
+ // work created during makeContainer
+ if err = os.MkdirAll(temp.String(), 0700); err != nil {
+ return
+ }
+ defer func() {
+ if chmodErr, removeErr := removeAll(scratch); chmodErr != nil || removeErr != nil {
+ err = errors.Join(err, chmodErr, removeErr)
+ }
+ }()
+
+ var z *container.Container
+ z, err = e.makeContainer(
+ ctx, c.msg,
+ c.attr.Flags,
+ c.attr.Jobs,
+ c.attr.Load,
+ hostNet,
+ temp, work,
+ func(a Artifact) (*check.Absolute, unique.Handle[Checksum]) {
+ if res, ok := dm[a]; ok {
+ return res.pathname, res.checksum
+ }
+ panic(InvalidLookupError(c.Ident(a).Value()))
+ },
+ c.Ident,
+ )
+ if err != nil {
+ return
+ }
+ z.Stdin, z.Stdout, z.Stderr = stdin, stdout, stderr
+ z.Path, z.Args = path, args
+ z.RetainSession = retainSession
+ if stdin == os.Stdin {
+ if s, ok := os.LookupEnv("TERM"); ok {
+ z.Env = append(z.Env, "TERM="+s)
+ }
+ }
+ if hostname != "" {
+ z.Hostname = hostname
+ }
+
+ if err = z.Start(); err != nil {
+ return
+ }
+ if err = z.Serve(); err != nil {
+ return
+ }
+ return z.Wait()
+}
+
+// cure is like Cure but allows optional host net namespace.
+func (a *execArtifact) cure(f *FContext, hostNet bool) (err error) {
+ ctx, cancel := context.WithTimeout(f.Unwrap(), a.timeout)
+ defer cancel()
+
+ msg := f.GetMessage()
+ var z *container.Container
+ if z, err = a.makeContainer(
+ ctx, msg, f.cache.attr.Flags, f.GetJobs(), f.GetLoad(), hostNet,
+ f.GetTempDir(), f.GetWorkDir(),
+ f.GetArtifact,
+ f.cache.Ident,
+ ); err != nil {
+ return
+ }
+
+ var status io.Writer
+ if status, err = f.GetStatusWriter(); err != nil {
+ return
+ }
+
+ if msg.IsVerbose() {
+ var stdout, stderr io.ReadCloser
+ if stdout, err = z.StdoutPipe(); err != nil {
+ return
+ }
+ if stderr, err = z.StderrPipe(); err != nil {
+ _ = stdout.Close()
+ return
+ }
+
+ brStdout, brStderr := f.cache.getReader(stdout), f.cache.getReader(stderr)
+ stdoutDone, stderrDone := make(chan struct{}), make(chan struct{})
+
+ var suffix, prefixO, prefixE string
+ if f.cache.attr.Flags&CColourOutput != 0 {
+ suffix = "\x1b[0m"
+ prefixO = "\x1b[1;37m(" + a.name + ")\x1b[0m"
+ prefixE = "\x1b[1;97m(" + a.name + ")\x1b[0m"
+ } else {
+ prefixO = "(" + a.name + ":1)"
+ prefixE = "(" + a.name + ":2)"
+ }
+
+ go scanVerbose(
+ msg, cancel, stdoutDone,
+ prefixO, suffix,
+ io.TeeReader(brStdout, status),
+ )
+ go scanVerbose(
+ msg, cancel, stderrDone,
+ prefixE, suffix,
+ io.TeeReader(brStderr, status),
+ )
+ defer func() {
+ if err != nil && !errors.As(err, new(*exec.ExitError)) {
+ _ = stdout.Close()
+ _ = stderr.Close()
+ }
+
+ <-stdoutDone
+ <-stderrDone
+ f.cache.putReader(brStdout)
+ f.cache.putReader(brStderr)
+ }()
+ } else {
+ z.Stdout, z.Stderr = status, status
+ }
+
+ if err = z.Start(); err != nil {
+ return
+ }
+ if err = z.Serve(); err != nil {
+ return
+ }
+ if err = z.Wait(); err != nil {
+ return
+ }
+
+ // do not allow empty directories to succeed
+ for {
+ err = syscall.Rmdir(f.GetWorkDir().String())
+ if err != syscall.EINTR {
+ break
+ }
+ }
+ if err != nil && errors.Is(err, syscall.ENOTEMPTY) {
+ err = nil
+ }
+ return
+}
diff --git a/pkg/exec_test.go b/pkg/exec_test.go
new file mode 100644
index 00000000..051af551
--- /dev/null
+++ b/pkg/exec_test.go
@@ -0,0 +1,670 @@
+package pkg_test
+
+import (
+ "bufio"
+ "bytes"
+ _ "embed"
+ "encoding/gob"
+ "errors"
+ "io/fs"
+ "net"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "slices"
+ "strings"
+ "testing"
+ _ "unsafe" // for go:linkname
+
+ "hakurei.app/check"
+ "hakurei.app/container"
+ "hakurei.app/hst"
+ "hakurei.app/internal/info"
+ "hakurei.app/internal/stub"
+ "hakurei.app/pkg"
+
+ "hakurei.app/pkg/internal/testtool/expected"
+)
+
+// testtoolBin is the container test tool binary made available to the
+// execArtifact for testing its curing environment.
+//
+//go:generate env CGO_ENABLED=0 go build -tags testtool -o internal/testtool ./internal/testtool
+//go:embed internal/testtool/testtool
+var testtoolBin []byte
+
+func init() {
+ pathname, err := filepath.Abs("internal/testtool/testtool")
+ if err != nil {
+ panic(err)
+ }
+ pkg.RegisterArch("cafe", container.BinfmtEntry{
+ Magic: expected.Magic,
+ Interpreter: check.MustAbs(pathname),
+ })
+}
+
+// scanLinesCR is like [bufio.ScanLines], but also treats a bare \r as an
+// end-of-line marker.
+//
+//go:linkname scanLinesCR hakurei.app/pkg.scanLinesCR
+func scanLinesCR(data []byte, atEOF bool) (advance int, token []byte, err error)
+
+func TestScan(t *testing.T) {
+ t.Parallel()
+
+ testCases := []struct {
+ name string
+ data string
+ want []string
+ }{
+ {"progress indicator", "Updating files: 76% (8655/11256)\r" +
+ "Updating files: 77% (8668/11256)\r" +
+ "Updating files: 78% (8780/11256)\r" +
+ "Updating files: 79% (8893/11256)\r" +
+ "Updating files: 80% (9005/11256)\r" +
+ "Updating files: 81% (9118/11256)\r" +
+ "Updating files: 82% (9230/11256)\r" +
+ "Updating files: 83% (9343/11256)\r" +
+ "Updating files: 84% (9456/11256)\r" +
+ "Updating files: 85% (9568/11256)\r" +
+ "Updating files: 86% (9681/11256)\r" +
+ "Updating files: 87% (9793/11256)\r" +
+ "Updating files: 88% (9906/11256)\r" +
+ "Updating files: 89% (10018/11256)\r" +
+ "Updating files: 90% (10131/11256)\r" +
+ "Updating files: 91% (10243/11256)\r" +
+ "Updating files: 92% (10356/11256)\r" +
+ "Updating files: 93% (10469/11256)\r" +
+ "Updating files: 94% (10581/11256)\r" +
+ "Updating files: 95% (10694/11256)\r" +
+ "Updating files: 96% (10806/11256)\r" +
+ "Updating files: 97% (10919/11256)\r" +
+ "Updating files: 98% (11031/11256)\r" +
+ "Updating files: 99% (11144/11256)\r" +
+ "Updating files: 100% (11256/11256)\r" +
+ "Updating files: 100% (11256/11256), done.\n", []string{
+ "Updating files: 76% (8655/11256)",
+ "Updating files: 77% (8668/11256)",
+ "Updating files: 78% (8780/11256)",
+ "Updating files: 79% (8893/11256)",
+ "Updating files: 80% (9005/11256)",
+ "Updating files: 81% (9118/11256)",
+ "Updating files: 82% (9230/11256)",
+ "Updating files: 83% (9343/11256)",
+ "Updating files: 84% (9456/11256)",
+ "Updating files: 85% (9568/11256)",
+ "Updating files: 86% (9681/11256)",
+ "Updating files: 87% (9793/11256)",
+ "Updating files: 88% (9906/11256)",
+ "Updating files: 89% (10018/11256)",
+ "Updating files: 90% (10131/11256)",
+ "Updating files: 91% (10243/11256)",
+ "Updating files: 92% (10356/11256)",
+ "Updating files: 93% (10469/11256)",
+ "Updating files: 94% (10581/11256)",
+ "Updating files: 95% (10694/11256)",
+ "Updating files: 96% (10806/11256)",
+ "Updating files: 97% (10919/11256)",
+ "Updating files: 98% (11031/11256)",
+ "Updating files: 99% (11144/11256)",
+ "Updating files: 100% (11256/11256)",
+ "Updating files: 100% (11256/11256), done.",
+ }},
+
+ {"crlf", "0\r1\n2\n3\r\n4\n5\r6\r\n7\n", []string{
+ "0", "1", "2", "3", "4", "5", "6", "7",
+ }},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ s := bufio.NewScanner(strings.NewReader(tc.data))
+ s.Split(scanLinesCR)
+ got := make([]string, 0, len(tc.want))
+ for s.Scan() {
+ got = append(got, s.Text())
+ }
+ if err := s.Err(); err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(got, tc.want) {
+ t.Fatalf("Scan: %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
+
+func TestExec(t *testing.T) {
+ t.Parallel()
+
+ wantOffline := expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+
+ "check": {Mode: 0400, Data: []byte{0}},
+ }
+ wantOfflineEncode := pkg.Encode(wantOffline.hash())
+ failingArtifact := &stubArtifact{
+ kind: pkg.KindTar,
+ params: []byte("doomed artifact"),
+ cure: func(t *pkg.TContext) error {
+ return stub.UniqueError(0xcafe)
+ },
+ }
+
+ checkWithCache(t, []cacheTestCase{
+ {"offline", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ testtool, testtoolDestroy := newTesttool()
+
+ cureMany(t, c, []cureStep{
+ {"container", pkg.NewExec(
+ "exec-offline", "", new(wantOffline.hash()), 0, false, false,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1"},
+ check.MustAbs("/opt/bin/testtool"),
+ []string{"testtool"},
+
+ pkg.MustPath("/file", false, newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xfe, 0},
+ nil,
+ nil, nil,
+ )),
+ pkg.MustPath("/.hakurei", false, &stubArtifact{
+ kind: pkg.KindTar,
+ params: []byte("empty directory"),
+ cure: func(t *pkg.TContext) error {
+ return os.MkdirAll(t.GetWorkDir().String(), 0700)
+ },
+ }),
+ pkg.MustPath("/opt", false, testtool),
+ ), ignorePathname, wantOffline, pkg.WNew, nil},
+
+ {"substitution", pkg.NewExec(
+ "exec-offline", "", new(wantOffline.hash()), 0, false, false,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1"},
+ check.MustAbs("/opt/bin/testtool"),
+ []string{"testtool"},
+
+ pkg.MustPath("/file", false, newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xfe, 0},
+ nil,
+ nil, nil,
+ )),
+ // substitution miss fails in testtool due to differing idents
+ pkg.MustPath("/.hakurei", false, &stubArtifact{
+ kind: pkg.KindTar,
+ params: []byte("empty directory (substituted)"),
+ cure: func(t *pkg.TContext) error {
+ return os.MkdirAll(t.GetWorkDir().String(), 0700)
+ },
+ }),
+ pkg.MustPath("/opt", false, testtool),
+ ), ignorePathname, wantOffline, pkg.WSubstitute, nil},
+
+ {"error passthrough", pkg.NewExec(
+ "", "", nil, 0, false, true,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1"},
+ check.MustAbs("/opt/bin/testtool"),
+ []string{"testtool"},
+
+ pkg.MustPath("/proc/nonexistent", false, failingArtifact),
+ ), nil, nil, pkg.WNew, pkg.InputError{
+ failingArtifact: stub.UniqueError(0xcafe),
+ }},
+
+ {"invalid paths", pkg.NewExec(
+ "", "", nil, 0, false, false,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1"},
+ check.MustAbs("/opt/bin/testtool"),
+ []string{"testtool"},
+
+ pkg.ExecPath{},
+ ), nil, nil, pkg.WNew, pkg.ErrInvalidPaths},
+ })
+
+ // check init failure passthrough
+ initFailureArtifact := pkg.NewExec(
+ "", "", nil, 0, false, false,
+ pkg.AbsWork,
+ nil,
+ check.MustAbs("/opt/bin/testtool"),
+ []string{"testtool"},
+ )
+ var exitError *exec.ExitError
+ if _, _, err := c.Cure(initFailureArtifact); !errors.As(err, &exitError) ||
+ exitError.ExitCode() != hst.ExitFailure {
+ t.Fatalf("Cure: error = %v, want init exit status 1", err)
+ }
+
+ var faultStatus []byte
+ if faults, err := c.ReadFaults(initFailureArtifact); err != nil {
+ t.Fatal(err)
+ } else if len(faults) != 1 {
+ t.Fatalf("ReadFaults: %v", faults)
+ } else if faultStatus, err = os.ReadFile(faults[0].String()); err != nil {
+ t.Fatal(err)
+ } else if err = faults[0].Destroy(); err != nil {
+ t.Fatal(err)
+ } else {
+ t.Logf("destroyed expected fault at %s", faults[0].Time().UTC())
+ }
+
+ if !bytes.HasPrefix(faultStatus, []byte(
+ "hakurei.app/pkg ",
+ )) || !bytes.Contains(faultStatus, []byte(
+ "\ninit: fork/exec /opt/bin/testtool: no such file or directory\n",
+ )) {
+ t.Errorf("unexpected status:\n%s", string(faultStatus))
+ }
+
+ destroyStatus(t, base, 2, 1)
+ testtoolDestroy(t, base, c)
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500},
+ "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}},
+ "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500},
+ "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400, Data: []byte{}},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+ "identifier/_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")},
+ "identifier/" + expected.Offline: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)},
+ "identifier/" + expected.OfflineS: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)},
+ "identifier/_OHzj_xdmfpaWfFN-_D9JpGpPlIjh84873E3Ui9xwv9PPPIt0BlzvH-cnPNeRsTl": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "temp": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"net", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ testtool, testtoolDestroy := newTesttool()
+
+ wantNet := expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+
+ "check": {Mode: 0400, Data: []byte("net")},
+ }
+ cureMany(t, c, []cureStep{
+ {"container", pkg.NewExec(
+ "exec-net", "", new(wantNet.hash()), 0, true, false,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1"},
+ check.MustAbs("/opt/bin/testtool"),
+ []string{"testtool", "net"},
+
+ pkg.MustPath("/file", false, newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xfe, 0},
+ nil,
+ nil, nil,
+ )),
+ pkg.MustPath("/.hakurei", false, &stubArtifact{
+ kind: pkg.KindTar,
+ params: []byte("empty directory"),
+ cure: func(t *pkg.TContext) error {
+ return os.MkdirAll(t.GetWorkDir().String(), 0700)
+ },
+ }),
+ pkg.MustPath("/opt", false, testtool),
+ ), ignorePathname, wantNet, pkg.WNew, nil},
+ })
+
+ destroyStatus(t, base, 2, 0)
+ testtoolDestroy(t, base, c)
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500},
+ "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400, Data: []byte{}},
+ "checksum/a1F_i9PVQI4qMcoHgTQkORuyWLkC1GLIxOhDt2JpU1NGAxWc5VJzdlfRK-PYBh3W": {Mode: fs.ModeDir | 0500},
+ "checksum/a1F_i9PVQI4qMcoHgTQkORuyWLkC1GLIxOhDt2JpU1NGAxWc5VJzdlfRK-PYBh3W/check": {Mode: 0400, Data: []byte("net")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/" + expected.Net: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/a1F_i9PVQI4qMcoHgTQkORuyWLkC1GLIxOhDt2JpU1NGAxWc5VJzdlfRK-PYBh3W")},
+ "identifier/_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")},
+ "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "temp": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"overlay root", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ testtool, testtoolDestroy := newTesttool()
+
+ cureMany(t, c, []cureStep{
+ {"container", pkg.NewExec(
+ "exec-overlay-root", "", nil, 0, false, false,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"},
+ check.MustAbs("/opt/bin/testtool"),
+ []string{"testtool"},
+
+ pkg.MustPath("/", true, &stubArtifact{
+ kind: pkg.KindTar,
+ params: []byte("empty directory"),
+ cure: func(t *pkg.TContext) error {
+ return os.MkdirAll(t.GetWorkDir().String(), 0700)
+ },
+ }),
+ pkg.MustPath("/opt", false, testtool),
+ ), ignorePathname, wantOffline, pkg.WNew, nil},
+ })
+
+ destroyStatus(t, base, 2, 0)
+ testtoolDestroy(t, base, c)
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500},
+ "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}},
+ "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/" + expected.OvlRoot: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)},
+ "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "temp": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"overlay work", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ testtool, testtoolDestroy := newTesttool()
+
+ cureMany(t, c, []cureStep{
+ {"container", pkg.NewExec(
+ "exec-overlay-work", "", nil, 0, false, false,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"},
+ check.MustAbs("/work/bin/testtool"),
+ []string{"testtool"},
+
+ pkg.MustPath("/", true, &stubArtifact{
+ kind: pkg.KindTar,
+ params: []byte("empty directory"),
+ cure: func(t *pkg.TContext) error {
+ return os.MkdirAll(t.GetWorkDir().String(), 0700)
+ },
+ }), pkg.MustPath("/work/", false, &stubArtifact{
+ kind: pkg.KindTar,
+ params: []byte("empty directory"),
+ cure: func(t *pkg.TContext) error {
+ return os.MkdirAll(t.GetWorkDir().String(), 0700)
+ },
+ }), pkg.Path(pkg.AbsWork, false /* ignored */, testtool),
+ ), ignorePathname, wantOffline, pkg.WNew, nil},
+ })
+
+ destroyStatus(t, base, 2, 0)
+ testtoolDestroy(t, base, c)
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500},
+ "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}},
+ "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/" + expected.Work: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)},
+ "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "temp": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"multiple layers", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ testtool, testtoolDestroy := newTesttool()
+
+ cureMany(t, c, []cureStep{
+ {"container", pkg.NewExec(
+ "exec-multiple-layers", "", nil, 0, false, false,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"},
+ check.MustAbs("/opt/bin/testtool"),
+ []string{"testtool", "layers"},
+
+ pkg.MustPath("/", true, &stubArtifact{
+ kind: pkg.KindTar,
+ params: []byte("empty directory"),
+ cure: func(t *pkg.TContext) error {
+ return os.MkdirAll(t.GetWorkDir().String(), 0700)
+ },
+ }, &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("test sample with dependencies"),
+
+ deps: slices.Repeat([]pkg.Artifact{newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xfe, 0},
+ nil,
+ nil, nil,
+ ), &stubArtifact{
+ kind: pkg.KindTar,
+ params: []byte("empty directory"),
+
+ // this is queued and might run instead of the other
+ // one so do not leave it as nil
+ cure: func(t *pkg.TContext) error {
+ return os.MkdirAll(t.GetWorkDir().String(), 0700)
+ },
+ }}, 1<<5 /* concurrent cache hits */),
+
+ cure: func(f *pkg.FContext) error {
+ work := f.GetWorkDir()
+ if err := os.MkdirAll(work.String(), 0700); err != nil {
+ return err
+ }
+ return os.WriteFile(work.Append("check").String(), []byte("layers"), 0400)
+ },
+ }),
+ pkg.MustPath("/opt", false, testtool),
+ ), ignorePathname, wantOffline, pkg.WNew, nil},
+ })
+
+ destroyStatus(t, base, 2, 0)
+ testtoolDestroy(t, base, c)
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500},
+ "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}},
+ "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500},
+ "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400, Data: []byte{}},
+ "checksum/nY_CUdiaUM1OL4cPr5TS92FCJ3rCRV7Hm5oVTzAvMXwC03_QnTRfQ5PPs7mOU9fK": {Mode: fs.ModeDir | 0500},
+ "checksum/nY_CUdiaUM1OL4cPr5TS92FCJ3rCRV7Hm5oVTzAvMXwC03_QnTRfQ5PPs7mOU9fK/check": {Mode: 0400, Data: []byte("layers")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")},
+ "identifier/2uIvmnqtzAx4Ed4PWV8xjZCWeJCwfE2IcCc0evM-Nd8mDUlgSU6nLtPLcpPUcMlG": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/nY_CUdiaUM1OL4cPr5TS92FCJ3rCRV7Hm5oVTzAvMXwC03_QnTRfQ5PPs7mOU9fK")},
+ "identifier/" + expected.Layers: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)},
+ "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "temp": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"overlay layer promotion", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ testtool, testtoolDestroy := newTesttool()
+
+ cureMany(t, c, []cureStep{
+ {"container", pkg.NewExec(
+ "exec-layer-promotion", "", nil, 0, false, true,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"},
+ check.MustAbs("/opt/bin/testtool"),
+ []string{"testtool", "promote"},
+
+ pkg.MustPath("/", true, &stubArtifact{
+ kind: pkg.KindTar,
+ params: []byte("another empty directory"),
+ cure: func(t *pkg.TContext) error {
+ return os.MkdirAll(t.GetWorkDir().String(), 0700)
+ },
+ }, &stubArtifact{
+ kind: pkg.KindTar,
+ params: []byte("empty directory"),
+ cure: func(t *pkg.TContext) error {
+ return os.MkdirAll(t.GetWorkDir().String(), 0700)
+ },
+ }),
+ pkg.MustPath("/opt", false, testtool),
+ ), ignorePathname, wantOffline, pkg.WNew, nil},
+ })
+
+ destroyStatus(t, base, 2, 0)
+ testtoolDestroy(t, base, c)
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500},
+ "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}},
+ "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/Vzj9qaSVBtNyBQv_PW6uojFOWTwraHoaC0C6xdaGG1q_HTvSh4KLVxRcEQU1jd2Y": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+ "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+ "identifier/" + expected.Promote: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "temp": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"binfmt", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ if info.CanDegrade && os.Getenv("ROSA_SKIP_BINFMT") != "" {
+ t.Skip("binfmt_misc test explicitly skipped")
+ }
+
+ cureMany(t, c, []cureStep{
+ {"container", pkg.NewExec(
+ "exec-binfmt", "cafe", nil, 0, false, true,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1", "HAKUREI_BINFMT=1"},
+ check.MustAbs("/opt/bin/sample"),
+ []string{"sample"},
+
+ pkg.MustPath("/", true, &stubArtifact{
+ kind: pkg.KindTar,
+ params: []byte("empty directory"),
+ cure: func(t *pkg.TContext) error {
+ return os.MkdirAll(t.GetWorkDir().String(), 0700)
+ },
+ }),
+ pkg.MustPath("/opt", false, overrideIdent{pkg.ID{0xfe, 0xff}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ work := t.GetWorkDir()
+ if err := os.MkdirAll(
+ work.Append("bin").String(),
+ 0700,
+ ); err != nil {
+ return err
+ }
+
+ return os.WriteFile(t.GetWorkDir().Append(
+ "bin",
+ "sample",
+ ).String(), []byte(expected.Full), 0500)
+ },
+ }}),
+ ), ignorePathname, expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+
+ "check": {Mode: 0400, Data: []byte("binfmt")},
+ }, pkg.WNew, nil},
+ })
+
+ destroyStatus(t, base, 2, 0)
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/5aevg3YpDxjqQZ-pdvXK7YqgkL5JKqcoStYQxeD96kuYar6K2mRQWMHib6NQRnpV": {Mode: fs.ModeDir | 0500},
+ "checksum/5aevg3YpDxjqQZ-pdvXK7YqgkL5JKqcoStYQxeD96kuYar6K2mRQWMHib6NQRnpV/bin": {Mode: fs.ModeDir | 0700},
+ "checksum/5aevg3YpDxjqQZ-pdvXK7YqgkL5JKqcoStYQxeD96kuYar6K2mRQWMHib6NQRnpV/bin/sample": {Mode: 0500, Data: []byte("\xca\xfe\xba\xbe\xfd\xfd:3")},
+ "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500},
+ "checksum/UnDo4B5KneEUY5b4vRUk_y9MWgkWuw2N8f8a2XayO686xXur-aZmX2-7n_8tKMe3": {Mode: fs.ModeDir | 0500},
+ "checksum/UnDo4B5KneEUY5b4vRUk_y9MWgkWuw2N8f8a2XayO686xXur-aZmX2-7n_8tKMe3/check": {Mode: 0400, Data: []byte("binfmt")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/c-HodmoEG0f_WGdst9aN-duLMH7QDDXOUi7UcyArXMeM1fc9lYKKQlTCW8ktaK8X": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UnDo4B5KneEUY5b4vRUk_y9MWgkWuw2N8f8a2XayO686xXur-aZmX2-7n_8tKMe3")},
+ "identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/5aevg3YpDxjqQZ-pdvXK7YqgkL5JKqcoStYQxeD96kuYar6K2mRQWMHib6NQRnpV")},
+ "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "temp": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+ })
+}
+
+// newTesttool returns an [Artifact] that cures into testtoolBin. The returned
+// function must be called at the end of the test but not deferred.
+func newTesttool() (
+ testtool pkg.Artifact,
+ testtoolDestroy func(t *testing.T, base *check.Absolute, c *pkg.Cache),
+) {
+ // testtoolBin is built during go:generate and is not deterministic
+ testtool = overrideIdent{pkg.ID{0xfe, 0xff}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ work := t.GetWorkDir()
+ if err := os.MkdirAll(
+ work.Append("bin").String(),
+ 0700,
+ ); err != nil {
+ return err
+ }
+
+ if ift, err := net.Interfaces(); err != nil {
+ return err
+ } else {
+ var f *os.File
+ if f, err = os.Create(t.GetWorkDir().Append(
+ "ift",
+ ).String()); err != nil {
+ return err
+ } else {
+ err = gob.NewEncoder(f).Encode(ift)
+ closeErr := f.Close()
+ if err != nil {
+ return err
+ }
+ if closeErr != nil {
+ return closeErr
+ }
+ }
+ }
+
+ return os.WriteFile(t.GetWorkDir().Append(
+ "bin",
+ "testtool",
+ ).String(), testtoolBin, 0500)
+ },
+ }}
+ testtoolDestroy = newDestroyArtifactFunc(testtool)
+ return
+}
diff --git a/pkg/file.go b/pkg/file.go
new file mode 100644
index 00000000..14467153
--- /dev/null
+++ b/pkg/file.go
@@ -0,0 +1,100 @@
+package pkg
+
+import (
+ "bytes"
+ "crypto/sha512"
+ "fmt"
+ "io"
+)
+
+// A fileArtifact is an [Artifact] that cures into data known ahead of time.
+type fileArtifact []byte
+
+var _ KnownChecksum = new(fileArtifact)
+var _ CuresExempt = new(fileArtifact)
+var _ RevisionArtifact = new(fileArtifact)
+
+// fileArtifactNamed embeds fileArtifact alongside a caller-supplied name.
+type fileArtifactNamed struct {
+ fileArtifact
+ // Caller-supplied user-facing reporting name.
+ name string
+}
+
+var _ fmt.Stringer = new(fileArtifactNamed)
+var _ KnownChecksum = new(fileArtifactNamed)
+
+// String returns the caller-supplied reporting name.
+func (a *fileArtifactNamed) String() string { return a.name }
+
+// Params writes the caller-supplied reporting name and the file body.
+func (a *fileArtifactNamed) Params(ctx *IContext) {
+ ctx.WriteString(a.name)
+ ctx.Write(a.fileArtifact)
+}
+
+// NewFile returns a [FileArtifact] that cures into a caller-supplied byte slice.
+//
+// Caller must not modify data after NewFile returns.
+func NewFile(name string, data []byte) FileArtifact {
+ f := fileArtifact(data)
+ if name != "" {
+ return &fileArtifactNamed{f, name}
+ }
+ return &f
+}
+
+// Kind returns the hardcoded [Kind] constant.
+func (*fileArtifact) Kind() Kind { return KindFile }
+
+// Params writes an empty string and the file body.
+func (a *fileArtifact) Params(ctx *IContext) {
+ ctx.WriteString("")
+ ctx.Write(*a)
+}
+
+func init() {
+ register(KindFile, func(r *IRReader) Artifact {
+ name := r.ReadString()
+ data := r.ReadStringBytes()
+ if _, ok := r.Finalise(); !ok {
+ panic(ErrExpectedChecksum)
+ }
+ return NewFile(name, data)
+ })
+}
+
+// Inputs returns a nil slice.
+func (*fileArtifact) Inputs() []Artifact { return nil }
+
+// IsExclusive returns false: Cure returns a prepopulated buffer.
+func (*fileArtifact) IsExclusive() bool { return false }
+
+// Checksum computes and returns the checksum of caller-supplied data.
+func (a *fileArtifact) Checksum() Checksum {
+ h := sha512.New384()
+ h.Write(*a)
+ return Checksum(h.Sum(nil))
+}
+
+// Revision satisfies [RevisionArtifact] for incompatible IsExecutable behaviour.
+func (*fileArtifact) Revision() uint64 { return 0 }
+
+// IsExecutable returns whether the contents begin with shebang or the ELF
+// magic number.
+func (a *fileArtifact) IsExecutable() bool {
+ return a != nil &&
+ (bytes.HasPrefix(
+ *a, []byte{'#', '!'},
+ ) || bytes.HasPrefix(
+ *a, []byte{0x7f, 'E', 'L', 'F'},
+ ))
+}
+
+// Cure returns the caller-supplied data.
+func (a *fileArtifact) Cure(*RContext) (io.ReadCloser, error) {
+ return io.NopCloser(bytes.NewReader(*a)), nil
+}
+
+// CuresExempt exempts the cheap [KindFile] implementation.
+func (*fileArtifact) CuresExempt() {}
diff --git a/pkg/file_test.go b/pkg/file_test.go
new file mode 100644
index 00000000..5742ca04
--- /dev/null
+++ b/pkg/file_test.go
@@ -0,0 +1,56 @@
+package pkg_test
+
+import (
+ "io/fs"
+ "testing"
+
+ "hakurei.app/check"
+ "hakurei.app/pkg"
+)
+
+func TestFile(t *testing.T) {
+ t.Parallel()
+
+ want := expectsFile{0}
+ const ident = "N7M6W8-e55Wky_1q1a6qRKFVDw1S5XtPdB4HE2YZObG8OFum8HittyJgZUG5Ru2u"
+ wantShebang := expectsFile("#!/bin/sh\n")
+ const identShebang = "oKTGZb_TRtJf5G1iGGj4SOx1WUbI6YhmsUeEsCNvmK9tZYRwdwQf-DLFUxRPx5qm"
+ wantELF := expectsFile("\x7fELF\xde\xad\xbe\xef")
+ const identELF = "LjeHmIJgD1SiXIH8ewpGRed2_LU84NLapstnOAa_0hxLd8YY7x4KM0aApsj-RwCh"
+ checkWithCache(t, []cacheTestCase{
+ {"file", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ cureMany(t, c, []cureStep{
+ {"short", pkg.NewFile("null", want), base.Append(
+ "identifier",
+ ident,
+ ), want, pkg.WNew, nil},
+
+ {"shebang", pkg.NewFile("shebang", wantShebang), base.Append(
+ "identifier",
+ identShebang,
+ ), wantShebang, pkg.WNew, nil},
+
+ {"elf", pkg.NewFile("ELF", wantELF), base.Append(
+ "identifier",
+ identELF,
+ ), wantELF, pkg.WNew, nil},
+ })
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/" + pkg.Encode(want.hash()): {Mode: 0400, Data: want},
+ "checksum/" + pkg.Encode(wantShebang.hash()): {Mode: 0500, Data: wantShebang},
+ "checksum/" + pkg.Encode(wantELF.hash()): {Mode: 0500, Data: wantELF},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/" + ident: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + pkg.Encode(want.hash()))},
+ "identifier/" + identShebang: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + pkg.Encode(wantShebang.hash()))},
+ "identifier/" + identELF: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + pkg.Encode(wantELF.hash()))},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+ })
+}
diff --git a/pkg/internal/testtool/expected/expected.go b/pkg/internal/testtool/expected/expected.go
new file mode 100644
index 00000000..c0e8e4a2
--- /dev/null
+++ b/pkg/internal/testtool/expected/expected.go
@@ -0,0 +1,9 @@
+// Package expected contains data shared between test helper and test harness.
+package expected
+
+const (
+ // Magic are magic bytes in the binfmt test case.
+ Magic = "\xca\xfe\xba\xbe\xfd\xfd"
+ // Full is the full content of the binfmt test case executable.
+ Full = Magic + ":3"
+)
diff --git a/pkg/internal/testtool/expected/sum_amd64.go b/pkg/internal/testtool/expected/sum_amd64.go
new file mode 100644
index 00000000..354931be
--- /dev/null
+++ b/pkg/internal/testtool/expected/sum_amd64.go
@@ -0,0 +1,11 @@
+package expected
+
+const (
+ Offline = "IKmVhALejtYA3JhN1PCfjKF-Dio1XVcOeMWDHhfSHCbb4iBNhGjFy2UspZveZvb8"
+ OfflineS = "6ZubjSyCnz4-TKgQZi_BbY-JUwsStYCtD_f-BTDOOotfEbqkm47W20AazHRaLviz"
+ OvlRoot = "Yox7WpG8B_OqIXWF4HWFZkUfsEl7DGzpFyZM0qtd9T8nQ-7SAkEUurlAe6_DJ6Iu"
+ Layers = "aAwfxDB-efVo379FNl178jK7HEKU3LEjHbZdaVVjk7GnXGZGEANNsXnB3SCUxM9A"
+ Net = "IU3GwU2jTBd7HjotvFo2wsDWBQGVGpamwiSKdc2G1EnhM-ZaacxAuK6q0wbTVxra"
+ Promote = "rOjetGHzT_jPErrBR24IAZKDj8Q9dLLqxFQqloMgjsULss0VXDI--n6LxuMBHvmZ"
+ Work = "7f9XDeREfvJIjCe0qesND0s0tasExAIcc-OH33-eq9tsrQPUXhJp4KEkpsYH4hCD"
+)
diff --git a/pkg/internal/testtool/expected/sum_arm64.go b/pkg/internal/testtool/expected/sum_arm64.go
new file mode 100644
index 00000000..2e6f7cbf
--- /dev/null
+++ b/pkg/internal/testtool/expected/sum_arm64.go
@@ -0,0 +1,11 @@
+package expected
+
+const (
+ Offline = "MTgCSzwPrzGigp9phvDPRwUTNGLm1KM2EsEtbz-l2cNieqjaGtnzm65CDXlSxXrl"
+ OfflineS = "p89bQg7_jR4nwaSbpEd08ne1jHOEw7Wlnlyv5FHq7ba1Kml3HSBZwEKRzu0gCqEb"
+ OvlRoot = "9ifwO6MAaV9kTqRR2O5PJzm8XhfXbr50mFLWyEHQm8OHHcJtAEUgXZzJYoZt9YVV"
+ Layers = "INuGActFPwJtS7mAR9Bkt7FD4yoHlXHSJ9CaSAQJrpkUBL_k815Cs_jvRfxy5-a7"
+ Net = "kJvy24uk4ePvHoxSKf4vMOhR4NxLbmyWlA9bTS7d3z3fE9WF1Tvpao9g-QuYU4vm"
+ Promote = "lYqyYrfjPawIbld5Qz0NbKvd_bmI3nJC1eEnlIQWzsjWE9uA1HuKPAaClnDeyG6L"
+ Work = "V6k4FqOBMRCmHiqchs5-joVGZ_MhNWZyXm3gdbU0S7S5eMze7G4SwYMTf66ojKQL"
+)
diff --git a/pkg/internal/testtool/expected/sum_riscv64.go b/pkg/internal/testtool/expected/sum_riscv64.go
new file mode 100644
index 00000000..53872c08
--- /dev/null
+++ b/pkg/internal/testtool/expected/sum_riscv64.go
@@ -0,0 +1,11 @@
+package expected
+
+const (
+ Offline = "LlieSHVDe1lnKvfI8t4oGRg_2hPxwyy1drmaqc-EKCOENuS9C2YHrp2VY5pSyhbg"
+ OfflineS = "qccjM8oYjUuEYPKFFzAxDFVzMerSUKUZc18NdVQoiPWf-70XOsNK8NaUQqjTgTAD"
+ OvlRoot = "lpV2bb_KGYFHPiOn-jtGop8K_ew0y1p3BezRp0ebM93_a-T9J-vkqGD5uokda1aN"
+ Layers = "sM3DJm210CHPGHkAquIbrXhspB81FpVC60ApQrvJGD5El2i7cSONj7LWf7EWqKpT"
+ Net = "JihQUwsVETRYjb2-yIXLYZ6WVsaFebwl8btSYAsNQ2QapS5MZVaI-dYwrVoySycR"
+ Promote = "lGqwR4W2OjylSVitMQP1lc6wApOBkbMjsmIqkxQbfEZ3DHEQBbJr84ErW6Xb54zo"
+ Work = "aZgtyv6aMtoHFkHAaJr6nLaA7N_SHjShgWPyQ5UdzaIqYSiVqvCrZqSokGTCcTii"
+)
diff --git a/pkg/internal/testtool/main.go b/pkg/internal/testtool/main.go
new file mode 100644
index 00000000..edffab7e
--- /dev/null
+++ b/pkg/internal/testtool/main.go
@@ -0,0 +1,277 @@
+//go:build testtool
+
+package main
+
+import (
+ "encoding/gob"
+ "log"
+ "net"
+ "os"
+ "path/filepath"
+ "reflect"
+ "runtime"
+ "slices"
+ "strconv"
+ "strings"
+
+ "hakurei.app/fhs"
+ "hakurei.app/vfs"
+
+ "hakurei.app/pkg/internal/testtool/expected"
+)
+
+func main() {
+ log.SetFlags(0)
+ log.SetPrefix("testtool: ")
+
+ if os.Getenv("HAKUREI_BINFMT") == "1" {
+ wantArgs := []string{"/interpreter", "/opt/bin/sample"}
+ if !slices.Equal(os.Args, wantArgs) {
+ log.Fatalf("Args: %q, want %q", os.Args, wantArgs)
+ }
+
+ if err := os.WriteFile("check", []byte("binfmt"), 0400); err != nil {
+ log.Fatal(err)
+ }
+
+ return
+ }
+
+ environ := slices.DeleteFunc(slices.Clone(os.Environ()), func(s string) bool {
+ for _, t := range []string{
+ "CURE_JOBS=" + strconv.Itoa(runtime.NumCPU()),
+ "CURE_LOAD=" + strconv.Itoa(runtime.NumCPU()+2),
+ } {
+ if s == t {
+ return true
+ }
+ }
+ return false
+ })
+
+ var hostNet, layers, promote bool
+ if len(os.Args) == 2 && os.Args[0] == "testtool" {
+ switch os.Args[1] {
+ case "net":
+ hostNet = true
+ log.SetPrefix("testtool(net): ")
+ break
+
+ case "layers":
+ layers = true
+ log.SetPrefix("testtool(layers): ")
+ break
+
+ case "promote":
+ promote = true
+ log.SetPrefix("testtool(promote): ")
+
+ default:
+ log.Fatalf("Args: %q", os.Args)
+ return
+ }
+ } else if wantArgs := []string{"testtool"}; !slices.Equal(os.Args, wantArgs) {
+ log.Fatalf("Args: %q, want %q", os.Args, wantArgs)
+ }
+
+ var overlayRoot bool
+ wantEnv := []string{"HAKUREI_TEST=1"}
+ if len(environ) == 2 {
+ overlayRoot = true
+ if !layers && !promote {
+ log.SetPrefix("testtool(overlay root): ")
+ }
+ wantEnv = []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"}
+ }
+ if !slices.Equal(wantEnv, environ) {
+ log.Fatalf("Environ: %q, want %q", environ, wantEnv)
+ }
+
+ var overlayWork bool
+ const (
+ wantExec = "/opt/bin/testtool"
+ wantExecWork = "/work/bin/testtool"
+ )
+ var iftPath string
+ if got, err := os.Executable(); err != nil {
+ log.Fatalf("Executable: error = %v", err)
+ } else {
+ iftPath = filepath.Join(filepath.Dir(filepath.Dir(got)), "ift")
+
+ if got != wantExec {
+ switch got {
+ case wantExecWork:
+ overlayWork = true
+ log.SetPrefix("testtool(overlay work): ")
+
+ default:
+ log.Fatalf("Executable: %q, want %q", got, wantExec)
+ }
+ }
+ }
+
+ wantHostname := "cure"
+ if hostNet {
+ wantHostname += "-net"
+ }
+
+ if hostname, err := os.Hostname(); err != nil {
+ log.Fatalf("Hostname: error = %v", err)
+ } else if hostname != wantHostname {
+ log.Fatalf("Hostname: %q, want %q", hostname, wantHostname)
+ }
+
+ var m *vfs.MountInfo
+ if f, err := os.Open(fhs.Proc + "self/mountinfo"); err != nil {
+ log.Fatalf("Open: error = %v", err)
+ } else {
+ err = vfs.NewMountInfoDecoder(f).Decode(&m)
+ closeErr := f.Close()
+ if err != nil {
+ log.Fatalf("Decode: error = %v", err)
+ }
+ if closeErr != nil {
+ log.Fatalf("Close: error = %v", err)
+ }
+ }
+
+ if ift, err := net.Interfaces(); err != nil {
+ log.Fatal(err)
+ } else if !hostNet {
+ if len(ift) != 1 || ift[0].Name != "lo" {
+ log.Fatalln("got interfaces", strings.Join(slices.Collect(func(yield func(ifn string) bool) {
+ for _, ifi := range ift {
+ if !yield(ifi.Name) {
+ break
+ }
+ }
+ }), ", "))
+ }
+ } else {
+ var iftParent []net.Interface
+
+ var r *os.File
+ if r, err = os.Open(iftPath); err != nil {
+ log.Fatal(err)
+ } else {
+ err = gob.NewDecoder(r).Decode(&iftParent)
+ closeErr := r.Close()
+ if err != nil {
+ log.Fatal(err)
+ }
+ if closeErr != nil {
+ log.Fatal(closeErr)
+ }
+ }
+
+ if !reflect.DeepEqual(ift, iftParent) {
+ log.Fatalf("Interfaces: %#v, want %#v", ift, iftParent)
+ }
+ }
+
+ const checksumEmptyDir = "MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU"
+ ident := expected.Offline
+ log.Println(m)
+ next := func() { m = m.Next; log.Println(m) }
+
+ if overlayRoot {
+ ident = expected.OvlRoot
+
+ if m.Root != "/" || m.Target != "/" ||
+ m.Source != "overlay" || m.FsType != "overlay" {
+ log.Fatal("unexpected root mount entry")
+ }
+ var lowerdir []string
+ for _, o := range strings.Split(m.FsOptstr, ",") {
+ const lowerdirKey = "lowerdir+="
+ if strings.HasPrefix(o, lowerdirKey) {
+ lowerdir = append(lowerdir, o[len(lowerdirKey):])
+ }
+ }
+ if !layers {
+ if len(lowerdir) != 1 || filepath.Base(lowerdir[0]) != checksumEmptyDir {
+ log.Fatal("unexpected artifact checksum")
+ }
+ } else {
+ ident = expected.Layers
+
+ if len(lowerdir) != 2 ||
+ filepath.Base(lowerdir[0]) != "MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU" ||
+ filepath.Base(lowerdir[1]) != "nY_CUdiaUM1OL4cPr5TS92FCJ3rCRV7Hm5oVTzAvMXwC03_QnTRfQ5PPs7mOU9fK" {
+ log.Fatalf("unexpected lowerdirs %s", strings.Join(lowerdir, ", "))
+ }
+ }
+ } else {
+ if hostNet {
+ ident = expected.Net
+ }
+
+ if m.Root != "/sysroot" || m.Target != "/" {
+ log.Fatal("unexpected root mount entry")
+ }
+
+ next()
+ if filepath.Base(m.Root) != "OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb" {
+ log.Fatal("unexpected file artifact checksum")
+ }
+
+ next()
+ if filepath.Base(m.Root) != checksumEmptyDir {
+ log.Fatal("unexpected artifact checksum")
+ }
+ }
+
+ if promote {
+ ident = expected.Promote
+ }
+
+ next() // testtool artifact
+
+ next()
+ if overlayWork {
+ ident = expected.Work
+ if m.Root != "/" || m.Target != "/work" ||
+ m.Source != "overlay" || m.FsType != "overlay" {
+ log.Fatal("unexpected work mount entry")
+ }
+ } else {
+ if filepath.Base(m.Root) != ident || m.Target != "/work" {
+ log.Fatal("unexpected work mount entry")
+ }
+ }
+
+ next()
+ if filepath.Base(m.Root) != ident || m.Target != "/tmp" {
+ log.Fatal("unexpected temp mount entry")
+ }
+
+ next()
+ if m.Root != "/" || m.Target != "/proc" || m.Source != "proc" || m.FsType != "proc" {
+ log.Fatal("unexpected proc mount entry")
+ }
+
+ next()
+ if m.Root != "/" || m.Target != "/dev" || m.Source != "devtmpfs" || m.FsType != "tmpfs" {
+ log.Fatal("unexpected dev mount entry")
+ }
+
+ for i := 0; i < 9; i++ { // private /dev entries
+ next()
+ }
+
+ if m.Next != nil {
+ log.Println("unexpected extra mount entries")
+ for m.Next != nil {
+ next()
+ }
+ os.Exit(1)
+ }
+
+ checkData := []byte{0}
+ if hostNet {
+ checkData = []byte("net")
+ }
+ if err := os.WriteFile("check", checkData, 0400); err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/pkg/ir.go b/pkg/ir.go
new file mode 100644
index 00000000..d9f984a7
--- /dev/null
+++ b/pkg/ir.go
@@ -0,0 +1,865 @@
+package pkg
+
+import (
+ "bufio"
+ "bytes"
+ "crypto/sha512"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+ "iter"
+ "slices"
+ "strconv"
+ "sync"
+ "syscall"
+ "unique"
+ "unsafe"
+)
+
+// wordSize is the boundary which binary segments are always aligned to.
+const wordSize = 8
+
+// alignSize returns the padded size for aligning sz.
+func alignSize[T int | uint64](sz T) T {
+ return sz + (wordSize-(sz)%wordSize)%wordSize
+}
+
+// panicToError recovers from a panic and replaces a nil error with the panicked
+// error value. If the value does not implement error, it is re-panicked.
+func panicToError(errP *error) {
+ r := recover()
+ if r == nil {
+ return
+ }
+
+ if err, ok := r.(error); !ok {
+ panic(r)
+ } else if *errP == nil {
+ *errP = err
+ }
+}
+
+// irCache implements [IRCache].
+type irCache struct {
+ // Artifact to [unique.Handle] of identifier cache.
+ artifact sync.Map
+ // Identifier free list, must not be accessed directly.
+ identPool sync.Pool
+}
+
+// zeroIRCache returns the initialised value of irCache.
+func zeroIRCache() irCache {
+ return irCache{
+ identPool: sync.Pool{New: func() any { return new(extIdent) }},
+ }
+}
+
+// IRCache provides memory management and caching primitives for IR and
+// identifier operations against [Artifact] implementations.
+//
+// The zero value is not safe for use.
+type IRCache struct{ irCache }
+
+// NewIR returns the address of a new [IRCache].
+func NewIR() *IRCache {
+ return &IRCache{zeroIRCache()}
+}
+
+// Inputs returns an iterator over direct and transitive inputs of an [Artifact]
+// in randomised order.
+func Inputs(a Artifact) iter.Seq2[Artifact, unique.Handle[ID]] {
+ ic := NewIR()
+ ic.Ident(a)
+ return func(yield func(Artifact, unique.Handle[ID]) bool) {
+ ic.artifact.Range(func(key, value any) bool {
+ return yield(key.(Artifact), value.(unique.Handle[ID]))
+ })
+ }
+}
+
+// IContext is passed to [Artifact.Params] and provides methods for writing
+// values to the IR writer. It does not expose the underlying [io.Writer].
+//
+// IContext is valid until [Artifact.Params] returns.
+type IContext struct {
+ // Address of underlying irCache, should be zeroed or made unusable after
+ // [Artifact.Params] returns and must not be exposed directly.
+ ic *irCache
+ // Written to by various methods, should be zeroed after [Artifact.Params]
+ // returns and must not be exposed directly.
+ w io.Writer
+ // Optional [Artifact] to cureRes cache, replaces [IRKindIdent] with
+ // checksum values if non-nil. The pathname field may not be populated.
+ inputs map[Artifact]cureRes
+}
+
+// irZero is a zero IR word.
+var irZero [wordSize]byte
+
+// IRValueKind denotes the kind of encoded value.
+type IRValueKind uint32
+
+const (
+ // IRKindEnd denotes the end of the current parameters stream. The ancillary
+ // value is interpreted as [IREndFlag].
+ IRKindEnd IRValueKind = iota
+ // IRKindIdent denotes the identifier of a dependency [Artifact]. The
+ // ancillary value is reserved for future use.
+ IRKindIdent
+ // IRKindUint32 denotes an inlined uint32 value.
+ IRKindUint32
+ // IRKindString denotes a string with its true length encoded in header
+ // ancillary data. Its wire length is always aligned to 8 byte boundary.
+ IRKindString
+
+ irHeaderShift = 32
+ irHeaderMask = 0xffffffff
+)
+
+// String returns a user-facing name of k.
+func (k IRValueKind) String() string {
+ switch k {
+ case IRKindEnd:
+ return "terminator"
+ case IRKindIdent:
+ return "ident"
+ case IRKindUint32:
+ return "uint32"
+ case IRKindString:
+ return "string"
+ default:
+ return "invalid kind " + strconv.Itoa(int(k))
+ }
+}
+
+// irValueHeader encodes [IRValueKind] and a 32-bit ancillary value.
+type irValueHeader uint64
+
+// encodeHeader returns irValueHeader encoding [IRValueKind] and ancillary data.
+func (k IRValueKind) encodeHeader(v uint32) irValueHeader {
+ return irValueHeader(v)<<irHeaderShift | irValueHeader(k)
+}
+
+// put stores h in b[0:8].
+func (h irValueHeader) put(b []byte) {
+ binary.LittleEndian.PutUint64(b[:], uint64(h))
+}
+
+// append appends the bytes of h to b and returns the appended slice.
+func (h irValueHeader) append(b []byte) []byte {
+ return binary.LittleEndian.AppendUint64(b, uint64(h))
+}
+
+// IREndFlag is ancillary data encoded in the header of an [IRKindEnd] value and
+// specifies the presence of optional fields in the remaining [IRKindEnd] data.
+// Order of present fields is the order of their corresponding constants defined
+// below.
+type IREndFlag uint32
+
+const (
+ // IREndKnownChecksum denotes a [KnownChecksum] artifact. For an [IRKindEnd]
+ // value with this flag set, the remaining data contains the [Checksum].
+ IREndKnownChecksum IREndFlag = 1 << iota
+)
+
+// mustWrite writes to IContext.w and panics on error. The panic is recovered
+// from by the caller and used as the return value.
+func (i *IContext) mustWrite(p []byte) {
+ if _, err := i.w.Write(p); err != nil {
+ panic(err)
+ }
+}
+
+// WriteIdent writes the identifier of [Artifact] to the IR. The behaviour of
+// WriteIdent is not defined for an [Artifact] not part of the slice returned by
+// [Artifact.Inputs].
+func (i *IContext) WriteIdent(a Artifact) {
+ buf := i.ic.getIdentBuf()
+ defer i.ic.putIdentBuf(buf)
+
+ IRKindIdent.encodeHeader(0).put(buf[:])
+ if i.inputs != nil {
+ res, ok := i.inputs[a]
+ if !ok {
+ panic(InvalidLookupError(i.ic.Ident(a).Value()))
+ }
+ *(*ID)(buf[wordSize:]) = res.checksum.Value()
+ } else {
+ *(*ID)(buf[wordSize:]) = i.ic.Ident(a).Value()
+ }
+ i.mustWrite(buf[:])
+}
+
+// WriteUint32 writes a uint32 value to the IR.
+func (i *IContext) WriteUint32(v uint32) {
+ i.mustWrite(IRKindUint32.encodeHeader(v).append(nil))
+}
+
+// irMaxStringLength is the maximum acceptable wire size of [IRKindString].
+const irMaxStringLength = 1 << 24
+
+// IRStringError is a string value too big to encode in IR.
+type IRStringError string
+
+func (IRStringError) Error() string {
+ return "params value too big to encode in IR"
+}
+
+// Write writes p as a string value to the IR.
+func (i *IContext) Write(p []byte) {
+ sz := alignSize(len(p))
+ if len(p) > irMaxStringLength || sz > irMaxStringLength {
+ panic(IRStringError(p))
+ }
+
+ i.mustWrite(IRKindString.encodeHeader(uint32(len(p))).append(nil))
+ i.mustWrite(p)
+
+ psz := sz - len(p)
+ if psz > 0 {
+ i.mustWrite(irZero[:psz])
+ }
+}
+
+// WriteString writes s as a string value to the IR.
+func (i *IContext) WriteString(s string) {
+ p := unsafe.Slice(unsafe.StringData(s), len(s))
+ i.Write(p)
+}
+
+// Encode writes a deterministic, efficient representation of a to w and returns
+// the first non-nil error encountered while writing to w.
+func (ic *irCache) Encode(w io.Writer, a Artifact) (err error) {
+ return ic.encode(w, a, nil)
+}
+
+// encode implements Encode but replaces identifiers with their cured checksums
+// for a non-nil ident. The pathname field is unused. Caller must acquire
+// Cache.identMu.
+func (ic *irCache) encode(
+ w io.Writer,
+ a Artifact,
+ inputs map[Artifact]cureRes,
+) (err error) {
+ deps := a.Inputs()
+ idents := make([]*extIdent, len(deps))
+ if inputs == nil {
+ for i, d := range deps {
+ dbuf, did := ic.unsafeIdent(d, true)
+ if dbuf == nil {
+ dbuf = ic.getIdentBuf()
+ binary.LittleEndian.PutUint64(dbuf[:], uint64(d.Kind()))
+ *(*ID)(dbuf[wordSize:]) = did.Value()
+ } else {
+ ic.storeIdent(d, dbuf)
+ }
+ defer ic.putIdentBuf(dbuf)
+ idents[i] = dbuf
+ }
+ } else {
+ for i, d := range deps {
+ res, ok := inputs[d]
+ if !ok {
+ return InvalidLookupError(ic.Ident(d).Value())
+ }
+
+ dbuf := ic.getIdentBuf()
+ binary.LittleEndian.PutUint64(dbuf[:], uint64(d.Kind()))
+ *(*ID)(dbuf[wordSize:]) = res.checksum.Value()
+ defer ic.putIdentBuf(dbuf)
+ idents[i] = dbuf
+ }
+ }
+ slices.SortFunc(idents, func(a, b *extIdent) int {
+ return bytes.Compare(a[:], b[:])
+ })
+ idents = slices.CompactFunc(idents, func(a, b *extIdent) bool {
+ return *a == *b
+ })
+
+ // kind uint64 | rev uint64 | deps_sz uint64
+ var buf [wordSize * 3]byte
+ binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind()))
+ binary.LittleEndian.PutUint64(buf[wordSize:], GetRevision(a))
+ binary.LittleEndian.PutUint64(buf[wordSize*2:], uint64(len(idents)))
+ if _, err = w.Write(buf[:]); err != nil {
+ return
+ }
+
+ for _, dn := range idents {
+ // kind uint64 | ident ID
+ if _, err = w.Write(dn[:]); err != nil {
+ return
+ }
+ }
+
+ func() {
+ i := IContext{ic, w, inputs}
+
+ defer panicToError(&err)
+ defer func() { i.ic, i.w = nil, nil }()
+
+ a.Params(&i)
+ }()
+ if err != nil {
+ return
+ }
+
+ var f IREndFlag
+ kcBuf := ic.getIdentBuf()
+ sz := wordSize
+ if kc, ok := a.(KnownChecksum); ok {
+ f |= IREndKnownChecksum
+ *(*Checksum)(kcBuf[wordSize:]) = kc.Checksum()
+ sz += len(Checksum{})
+ }
+ IRKindEnd.encodeHeader(uint32(f)).put(kcBuf[:])
+
+ _, err = w.Write(kcBuf[:sz])
+ ic.putIdentBuf(kcBuf)
+ return
+}
+
+// encodeAll implements EncodeAll by recursively encoding dependencies and
+// performs deduplication by value via the encoded map.
+func (ic *irCache) encodeAll(
+ w io.Writer,
+ a Artifact,
+ encoded map[Artifact]struct{},
+) (err error) {
+ if _, ok := encoded[a]; ok {
+ return
+ }
+
+ for _, d := range a.Inputs() {
+ if err = ic.encodeAll(w, d, encoded); err != nil {
+ return
+ }
+ }
+
+ encoded[a] = struct{}{}
+ return ic.Encode(w, a)
+}
+
+// EncodeAll writes a self-describing IR stream of a to w and returns the first
+// non-nil error encountered while writing to w.
+//
+// EncodeAll tries to avoid encoding the same [Artifact] more than once, however
+// it will fail to do so if they do not compare equal by value, as that will
+// require buffering and greatly reduce performance. It is therefore up to the
+// caller to avoid causing dependencies to be represented in a way such that
+// two equivalent artifacts do not compare equal. While an IR stream with
+// repeated artifacts is valid, it is somewhat inefficient, and the reference
+// [IRDecoder] implementation produces a warning for it.
+//
+// Note that while EncodeAll makes use of the ident free list, it does not use
+// the ident cache, nor does it contribute identifiers it computes back to the
+// ident cache. Because of this, multiple invocations of EncodeAll will have
+// similar cost and does not amortise when combined with a call to Cure.
+func (ic *irCache) EncodeAll(w io.Writer, a Artifact) error {
+ return ic.encodeAll(w, a, make(map[Artifact]struct{}))
+}
+
+// ErrRemainingIR is returned for a [IRReadFunc] that failed to call
+// [IRReader.Finalise] before returning.
+var ErrRemainingIR = errors.New("implementation did not consume final value")
+
+// DanglingIdentError is an identifier in a [IRKindIdent] value that was never
+// described in the IR stream before it was encountered.
+type DanglingIdentError unique.Handle[ID]
+
+func (e DanglingIdentError) Error() string {
+ return "artifact " + Encode(unique.Handle[ID](e).Value()) +
+ " was never described"
+}
+
+type (
+ // IRDecoder decodes [Artifact] from an IR stream. The stream is read to
+ // EOF and the final [Artifact] is returned. Previous artifacts may be
+ // looked up by their identifier.
+ //
+ // An [Artifact] may appear more than once in the same IR stream. A
+ // repeating [Artifact] generates a warning via [Cache] and will appear if
+ // verbose logging is enabled. Artifacts may only depend on artifacts
+ // previously described in the IR stream.
+ //
+ // IRDecoder rejects an IR stream on the first decoding error, it does not
+ // check against nonzero reserved ancillary data or incorrectly ordered or
+ // redundant unstructured dependencies. An invalid IR stream as such will
+ // yield [Artifact] values with identifiers disagreeing with those computed
+ // by IRDecoder. For this reason, IRDecoder does not access the ident cache
+ // to avoid putting [Cache] into an inconsistent state.
+ //
+ // Methods of IRDecoder are not safe for concurrent use.
+ IRDecoder struct {
+ // Address of underlying [Cache], must not be exposed directly.
+ c *Cache
+
+ // Underlying IR reader. Methods of [IRReader] must not use this as it
+ // bypasses ident measurement.
+ r io.Reader
+ // Artifacts already seen in the IR stream.
+ ident map[unique.Handle[ID]]Artifact
+
+ // Whether Decode returned, and the entire IR stream was decoded.
+ done, ok bool
+ }
+
+ // IRReader provides methods to decode the IR wire format and read values
+ // from the reader embedded in the underlying [IRDecoder]. It is
+ // deliberately impossible to obtain the [IRValueKind] of the next value,
+ // and callers must never recover from panics in any read method.
+ //
+ // It is the responsibility of the caller to call Finalise after all IR
+ // values have been read. Failure to call Finalise causes the resulting
+ // [Artifact] to be rejected with [ErrRemainingIR].
+ //
+ // For an [Artifact] expected to have dependencies, the caller must consume
+ // all dependencies by calling Next until all dependencies are depleted, or
+ // call DiscardAll to explicitly discard them and rely on values encoded as
+ // [IRKindIdent] instead. Failure to consume all unstructured dependencies
+ // causes the resulting [Artifact] to be rejected with [MissedDependencyError].
+ //
+ // Requesting the value of an unstructured dependency not yet described in
+ // the IR stream via Next, or reading an [IRKindIdent] value not part of
+ // unstructured dependencies via ReadIdent may cause the resulting
+ // [Artifact] to be rejected with [DanglingIdentError], however either
+ // method may return a non-nil [Artifact] implementation of unspecified
+ // value.
+ IRReader struct {
+ // Address of underlying [IRDecoder], should be zeroed or made unusable
+ // after finalisation and must not be exposed directly.
+ d *IRDecoder
+ // Common buffer for word-sized reads.
+ buf [wordSize]byte
+
+ // Inputs sent before params, sorted by identifier. Resliced on
+ // each call to Next and checked to be depleted during Finalise.
+ inputs []*extIdent
+
+ // Number of values already read, -1 denotes a finalised IRReader.
+ count int
+ // Header of value currently being read.
+ h irValueHeader
+
+ // Measured IR reader. All reads for the current [Artifact] must go
+ // through this to produce a correct ident.
+ r io.Reader
+ // Buffers measure writes. Flushed and returned to d during Finalise.
+ ibw *bufio.Writer
+ }
+
+ // IRReadFunc reads IR values written by [Artifact.Params] to produce an
+ // instance of [Artifact] identical to the one to produce these values.
+ IRReadFunc func(r *IRReader) Artifact
+)
+
+// kind returns the [IRValueKind] encoded in h.
+func (h irValueHeader) kind() IRValueKind {
+ return IRValueKind(h & irHeaderMask)
+}
+
+// value returns ancillary data encoded in h.
+func (h irValueHeader) value() uint32 {
+ return uint32(h >> irHeaderShift)
+}
+
+// irArtifact refers to artifact IR interpretation functions and must not be
+// written to directly.
+var irArtifact = make(map[Kind]IRReadFunc)
+
+// InvalidKindError is an unregistered [Kind] value.
+type InvalidKindError Kind
+
+func (e InvalidKindError) Error() string {
+ return "invalid artifact kind " + strconv.Itoa(int(e))
+}
+
+// register records the [IRReadFunc] of an implementation of [Artifact] under
+// the specified [Kind]. Expecting to be used only during initialization, it
+// panics if the mapping between [Kind] and [IRReadFunc] is not a bijection.
+//
+// register is not safe for concurrent use. register must not be called after
+// the first instance of [Cache] has been opened.
+func register(k Kind, f IRReadFunc) {
+ openMu.Lock()
+ defer openMu.Unlock()
+
+ if opened {
+ panic("attempting to register after open")
+ }
+ if _, ok := irArtifact[k]; ok {
+ panic("attempting to register " + strconv.Itoa(int(k)) + " twice")
+ }
+ irArtifact[k] = f
+}
+
+// Register records the [IRReadFunc] of a custom implementation of [Artifact]
+// under the specified [Kind]. Expecting to be used only during initialization,
+// it panics if the mapping between [Kind] and [IRReadFunc] is not a bijection,
+// or the specified [Kind] is below [KindCustomOffset].
+//
+// Register is not safe for concurrent use. Register must not be called after
+// the first instance of [Cache] has been opened.
+func Register(k Kind, f IRReadFunc) {
+ if k < KindCustomOffset {
+ panic("attempting to register within internal kind range")
+ }
+ register(k, f)
+}
+
+// NewDecoder returns a new [IRDecoder] that reads from the [io.Reader].
+func (c *Cache) NewDecoder(r io.Reader) *IRDecoder {
+ return &IRDecoder{c, r, make(map[unique.Handle[ID]]Artifact), false, false}
+}
+
+const (
+ // irMaxValues is the arbitrary maximum number of values allowed to be
+ // written by [Artifact.Params] and subsequently read via [IRReader].
+ irMaxValues = 1 << 12
+
+ // irMaxDeps is the arbitrary maximum number of direct dependencies allowed
+ // to be returned by [Artifact.Inputs] and subsequently decoded by
+ // [IRDecoder].
+ irMaxDeps = 1 << 10
+)
+
+var (
+ // ErrIRValues is returned for an [Artifact] with too many parameter values.
+ ErrIRValues = errors.New("artifact has too many IR parameter values")
+
+ // ErrIRDepend is returned for an [Artifact] with too many dependencies.
+ ErrIRDepend = errors.New("artifact has too many dependencies")
+
+ // ErrAlreadyFinalised is returned when attempting to use an [IRReader] that
+ // has already been finalised.
+ ErrAlreadyFinalised = errors.New("reader has already finalised")
+)
+
+// enterReader panics with an appropriate error for an out-of-bounds count and
+// must be called at some point in any exported method.
+func (ir *IRReader) enterReader(read bool) {
+ if ir.count < 0 {
+ panic(ErrAlreadyFinalised)
+ }
+ if ir.count >= irMaxValues {
+ panic(ErrIRValues)
+ }
+
+ if read {
+ ir.count++
+ }
+}
+
+// IRKindError describes an attempt to read an IR value of unexpected kind.
+type IRKindError struct {
+ Got, Want IRValueKind
+ Ancillary uint32
+}
+
+func (e *IRKindError) Error() string {
+ return fmt.Sprintf(
+ "got %s IR value (%#x) instead of %s",
+ e.Got, e.Ancillary, e.Want,
+ )
+}
+
+// readFull reads until either p is filled or an error is encountered.
+func (ir *IRReader) readFull(p []byte) (n int, err error) {
+ for n < len(p) && err == nil {
+ var nn int
+ nn, err = ir.r.Read(p[n:])
+ n += nn
+ }
+ return
+}
+
+// mustRead reads from the underlying measured reader and panics on error. If
+// an [io.EOF] is encountered and n != len(p), the error is promoted to a
+// [io.ErrUnexpectedEOF], if n == 0, [io.EOF] is kept as is, otherwise it is
+// zeroed.
+func (ir *IRReader) mustRead(p []byte) {
+ n, err := ir.readFull(p)
+ if err == nil {
+ return
+ }
+
+ if errors.Is(err, io.EOF) {
+ if n == len(p) {
+ return
+ }
+ err = io.ErrUnexpectedEOF
+ }
+ panic(err)
+}
+
+// mustReadHeader reads the next header via d and checks its kind.
+func (ir *IRReader) mustReadHeader(k IRValueKind) {
+ ir.mustRead(ir.buf[:])
+ ir.h = irValueHeader(binary.LittleEndian.Uint64(ir.buf[:]))
+ if wk := ir.h.kind(); wk != k {
+ panic(&IRKindError{wk, k, ir.h.value()})
+ }
+}
+
+// putAll returns all dependency buffers to the underlying [Cache].
+func (ir *IRReader) putAll() {
+ for _, buf := range ir.inputs {
+ ir.d.c.putIdentBuf(buf)
+ }
+ ir.inputs = nil
+}
+
+// DiscardAll discards all unstructured dependencies. This is useful to
+// implementations that encode dependencies as [IRKindIdent] which are read back
+// via ReadIdent.
+func (ir *IRReader) DiscardAll() {
+ if ir.inputs == nil {
+ panic("attempting to discard dependencies twice")
+ }
+ ir.putAll()
+}
+
+// ErrDependencyDepleted is returned when attempting to advance to the next
+// unstructured dependency when there are none left.
+var ErrDependencyDepleted = errors.New("reading past end of dependencies")
+
+// Next returns the next unstructured dependency.
+func (ir *IRReader) Next() Artifact {
+ if len(ir.inputs) == 0 {
+ panic(ErrDependencyDepleted)
+ }
+
+ id := unique.Make(ID(ir.inputs[0][wordSize:]))
+ ir.d.c.putIdentBuf(ir.inputs[0])
+ ir.inputs = ir.inputs[1:]
+
+ if a, ok := ir.d.ident[id]; !ok {
+ ir.putAll()
+ panic(DanglingIdentError(id))
+ } else {
+ return a
+ }
+}
+
+// MissedDependencyError is the number of unstructured dependencies remaining
+// in [IRReader] that was never requested or explicitly discarded before
+// finalisation.
+type MissedDependencyError int
+
+func (e MissedDependencyError) Error() string {
+ return "missed " + strconv.Itoa(int(e)) + " unstructured dependencies"
+}
+
+var (
+ // ErrUnexpectedChecksum is returned by a [IRReadFunc] that does not expect
+ // a checksum but received one in [IRKindEnd] anyway.
+ ErrUnexpectedChecksum = errors.New("checksum specified on unsupported artifact")
+ // ErrExpectedChecksum is returned by a [IRReadFunc] that expects a checksum
+ // but did not receive one in [IRKindEnd].
+ ErrExpectedChecksum = errors.New("checksum required but not specified")
+)
+
+// Finalise reads the final [IRKindEnd] value and marks r as finalised. Methods
+// of r are invalid upon entry into Finalise. If a [Checksum] is available via
+// [IREndKnownChecksum], its handle is returned and the caller must store its
+// value in the resulting [Artifact].
+func (ir *IRReader) Finalise() (checksum unique.Handle[Checksum], ok bool) {
+ ir.enterReader(true)
+ ir.count = -1
+
+ ir.mustReadHeader(IRKindEnd)
+ f := IREndFlag(ir.h.value())
+
+ if f&IREndKnownChecksum != 0 {
+ buf := ir.d.c.getIdentBuf()
+ defer ir.d.c.putIdentBuf(buf)
+
+ ir.mustRead(buf[wordSize:])
+ checksum = unique.Make(Checksum(buf[wordSize:]))
+ ok = true
+ }
+
+ if err := ir.ibw.Flush(); err != nil {
+ panic(err)
+ }
+ ir.r, ir.ibw = nil, nil
+
+ if len(ir.inputs) != 0 {
+ panic(MissedDependencyError(len(ir.inputs)))
+ }
+
+ return
+}
+
+// ReadIdent reads the next value as [IRKindIdent].
+func (ir *IRReader) ReadIdent() Artifact {
+ ir.enterReader(true)
+ ir.mustReadHeader(IRKindIdent)
+
+ buf := ir.d.c.getIdentBuf()
+ defer ir.d.c.putIdentBuf(buf)
+
+ ir.mustRead(buf[wordSize:])
+ id := unique.Make(ID(buf[wordSize:]))
+
+ if a, ok := ir.d.ident[id]; !ok {
+ panic(DanglingIdentError(id))
+ } else {
+ return a
+ }
+}
+
+// ReadUint32 reads the next value as [IRKindUint32].
+func (ir *IRReader) ReadUint32() uint32 {
+ ir.enterReader(true)
+ ir.mustReadHeader(IRKindUint32)
+ return ir.h.value()
+}
+
+// ReadStringBytes reads the next value as [IRKindString] but returns it as a
+// byte slice instead.
+func (ir *IRReader) ReadStringBytes() []byte {
+ ir.enterReader(true)
+ ir.mustReadHeader(IRKindString)
+
+ sz := int(ir.h.value())
+ szWire := alignSize(sz)
+ if szWire > irMaxStringLength {
+ panic(IRStringError("\x00"))
+ }
+
+ p := make([]byte, szWire)
+ ir.mustRead(p)
+ return p[:sz]
+}
+
+// ReadString reads the next value as [IRKindString].
+func (ir *IRReader) ReadString() string {
+ p := ir.ReadStringBytes()
+ return unsafe.String(unsafe.SliceData(p), len(p))
+}
+
+// A RevisionError describes the first [Artifact] in an IR stream claiming a
+// revision not supported by the registered implementation.
+type RevisionError [2]uint64
+
+func (e RevisionError) Error() string {
+ return "claimed revision " + strconv.FormatUint(e[0], 10) +
+ " differs from implementation value " + strconv.FormatUint(e[1], 10)
+}
+
+// decode decodes the next [Artifact] in the IR stream and returns any buffer
+// originating from [Cache] before returning. decode returns [io.EOF] if and
+// only if the underlying [io.Reader] is already read to EOF.
+func (d *IRDecoder) decode() (a Artifact, err error) {
+ defer panicToError(&err)
+ var ir IRReader
+
+ defer func() { ir.d = nil }()
+ ir.d = d
+
+ h := sha512.New384()
+ ir.ibw = d.c.getWriter(h)
+ defer d.c.putWriter(ir.ibw)
+ ir.r = io.TeeReader(d.r, ir.ibw)
+
+ if n, _err := ir.readFull(ir.buf[:]); _err != nil {
+ if errors.Is(_err, io.EOF) {
+ if n != 0 {
+ _err = io.ErrUnexpectedEOF
+ }
+ }
+
+ err = _err
+ return
+ }
+ ak := Kind(binary.LittleEndian.Uint64(ir.buf[:]))
+ f, ok := irArtifact[ak]
+ if !ok {
+ err = InvalidKindError(ak)
+ return
+ }
+
+ ir.mustRead(ir.buf[:])
+ rev := binary.LittleEndian.Uint64(ir.buf[:])
+
+ defer ir.putAll()
+ ir.mustRead(ir.buf[:])
+ sz := binary.LittleEndian.Uint64(ir.buf[:])
+ if sz > irMaxDeps {
+ err = ErrIRDepend
+ return
+ }
+ ir.inputs = make([]*extIdent, sz)
+ for i := range ir.inputs {
+ ir.inputs[i] = d.c.getIdentBuf()
+ }
+ for _, buf := range ir.inputs {
+ ir.mustRead(buf[:])
+ }
+
+ a = f(&ir)
+ if a == nil {
+ err = syscall.ENOTRECOVERABLE
+ return
+ }
+ if _rev := GetRevision(a); _rev != rev {
+ err = RevisionError{rev, _rev}
+ return
+ }
+
+ if ir.count != -1 {
+ err = ErrRemainingIR
+ return
+ }
+
+ buf := d.c.getIdentBuf()
+ h.Sum(buf[wordSize:wordSize])
+ id := unique.Make(ID(buf[wordSize:]))
+ d.c.putIdentBuf(buf)
+ if _, ok = d.ident[id]; !ok {
+ d.ident[id] = a
+ } else {
+ d.c.msg.Verbosef(
+ "artifact %s%s%s appeared more than once in IR stream",
+ d.c.sgrIdent, Encode(id.Value()), d.c.sgrRes,
+ )
+ }
+
+ return
+}
+
+// Decode consumes the IR stream to EOF and returns the final [Artifact]. After
+// Decode returns, Lookup is available and Decode must not be called again.
+func (d *IRDecoder) Decode() (a Artifact, err error) {
+ if d.done {
+ panic("attempting to decode an IR stream twice")
+ }
+ defer func() { d.done = true }()
+
+ var cur Artifact
+next:
+ a, err = d.decode()
+
+ if err == nil {
+ cur = a
+ goto next
+ }
+
+ if errors.Is(err, io.EOF) {
+ a, err = cur, nil
+ d.ok = true
+ }
+ return
+}
+
+// Lookup looks up an [Artifact] described by the IR stream by its identifier.
+func (d *IRDecoder) Lookup(id unique.Handle[ID]) (a Artifact, ok bool) {
+ if !d.ok {
+ panic("attempting to look up artifact without full IR stream")
+ }
+ a, ok = d.ident[id]
+ return
+}
diff --git a/pkg/ir_test.go b/pkg/ir_test.go
new file mode 100644
index 00000000..9cc024ae
--- /dev/null
+++ b/pkg/ir_test.go
@@ -0,0 +1,170 @@
+package pkg_test
+
+import (
+ "bytes"
+ "io"
+ "io/fs"
+ "reflect"
+ "testing"
+
+ "hakurei.app/check"
+ "hakurei.app/pkg"
+)
+
+func TestIRRoundtrip(t *testing.T) {
+ t.Parallel()
+
+ testCases := []struct {
+ name string
+ a pkg.Artifact
+ }{
+ {"http get aligned", pkg.NewHTTPGet(
+ nil, "file:///testdata",
+ pkg.Checksum(bytes.Repeat([]byte{0xfd}, len(pkg.Checksum{}))),
+ )},
+ {"http get unaligned", pkg.NewHTTPGet(
+ nil, "https://hakurei.app",
+ pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))),
+ )},
+
+ {"http get tar", pkg.NewTar(pkg.NewDecompress(pkg.NewHTTPGet(
+ nil, "file:///testdata",
+ pkg.Checksum(bytes.Repeat([]byte{0xff}, len(pkg.Checksum{}))),
+ ), pkg.Bzip2))},
+ {"http get tar unaligned", pkg.NewTar(pkg.NewHTTPGet(
+ nil, "https://hakurei.app",
+ pkg.Checksum(bytes.Repeat([]byte{0xfe}, len(pkg.Checksum{}))),
+ ))},
+
+ {"exec offline", pkg.NewExec(
+ "exec-offline", "", nil, 0, false, false,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1"},
+ check.MustAbs("/opt/bin/testtool"),
+ []string{"testtool"},
+
+ pkg.MustPath("/file", false, pkg.NewFile("file", []byte(
+ "stub file",
+ ))), pkg.MustPath("/.hakurei", false, pkg.NewTar(pkg.NewHTTPGet(
+ nil, "file:///hakurei.tar",
+ pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))),
+ ))), pkg.MustPath("/opt", false, pkg.NewTar(pkg.NewDecompress(pkg.NewHTTPGet(
+ nil, "file:///testtool.tar.gz",
+ pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))),
+ ), pkg.Gzip))),
+ )},
+
+ {"exec net", pkg.NewExec(
+ "exec-net", "",
+ (*pkg.Checksum)(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))),
+ 0, false, false,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1"},
+ check.MustAbs("/opt/bin/testtool"),
+ []string{"testtool", "net"},
+
+ pkg.MustPath("/file", false, pkg.NewFile("file", []byte(
+ "stub file",
+ ))), pkg.MustPath("/.hakurei", false, pkg.NewTar(pkg.NewHTTPGet(
+ nil, "file:///hakurei.tar",
+ pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))),
+ ))), pkg.MustPath("/opt", false, pkg.NewTar(pkg.NewDecompress(pkg.NewHTTPGet(
+ nil, "file:///testtool.tar.gz",
+ pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))),
+ ), pkg.Gzip))),
+ )},
+
+ {"exec measured", pkg.NewExec(
+ "exec-measured", "",
+ (*pkg.Checksum)(bytes.Repeat([]byte{0xfd}, len(pkg.Checksum{}))),
+ 0, false, false,
+ pkg.AbsWork,
+ []string{"HAKUREI_TEST=1"},
+ check.MustAbs("/opt/bin/testtool"),
+ []string{"testtool", "measured"},
+
+ pkg.MustPath("/file", false, pkg.NewFile("file", []byte(
+ "stub file",
+ ))), pkg.MustPath("/.hakurei", false, pkg.NewTar(pkg.NewHTTPGet(
+ nil, "file:///hakurei.tar",
+ pkg.Checksum(bytes.Repeat([]byte{0xfd}, len(pkg.Checksum{}))),
+ ))), pkg.MustPath("/opt", false, pkg.NewTar(pkg.NewDecompress(pkg.NewHTTPGet(
+ nil, "file:///testtool.tar.gz",
+ pkg.Checksum(bytes.Repeat([]byte{0xfd}, len(pkg.Checksum{}))),
+ ), pkg.Gzip))),
+ )},
+
+ {"file anonymous", pkg.NewFile("", []byte{0})},
+ {"file", pkg.NewFile("stub", []byte("stub"))},
+
+ {"decompress", pkg.NewDecompress(pkg.NewFile("", []byte{0}), pkg.Bzip2)},
+
+ {"archive", pkg.NewArchive(pkg.NewFile("", []byte{0}))},
+ }
+ testCasesCache := make([]cacheTestCase, len(testCases))
+ for i, tc := range testCases {
+ want := tc.a
+ testCasesCache[i] = cacheTestCase{tc.name, 0, nil,
+ func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ r, w := io.Pipe()
+
+ done := make(chan error, 1)
+ go func() {
+ t.Helper()
+ done <- c.EncodeAll(w, want)
+ _ = w.Close()
+ }()
+
+ if got, err := c.NewDecoder(r).Decode(); err != nil {
+ t.Fatalf("Decode: error = %v", err)
+ } else if !reflect.DeepEqual(got, want) {
+ t.Fatalf("Decode: %#v, want %#v", got, want)
+ }
+
+ if err := <-done; err != nil {
+ t.Fatalf("EncodeAll: error = %v", err)
+ }
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ },
+ }
+ }
+ checkWithCache(t, testCasesCache)
+}
+
+func TestRevision(t *testing.T) {
+ checkWithCache(t, []cacheTestCase{{"revision", 0, nil,
+ func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ a := pkg.NewFile("", nil)
+ var buf bytes.Buffer
+ if err := c.EncodeAll(&buf, a); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := c.NewDecoder(
+ bytes.NewReader(buf.Bytes()),
+ ).Decode(); err != nil {
+ t.Fatalf("Decode: error = %v", err)
+ }
+
+ p := buf.Bytes()
+ p[8] = 0xfd
+ wantErr := pkg.RevisionError{0xfd, 0}
+ if _, err := c.NewDecoder(
+ bytes.NewReader(p),
+ ).Decode(); !reflect.DeepEqual(err, wantErr) {
+ t.Fatalf("Decode: error = %v, want %v", err, wantErr)
+ }
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ },
+ }})
+}
diff --git a/pkg/net.go b/pkg/net.go
new file mode 100644
index 00000000..cd30e3bb
--- /dev/null
+++ b/pkg/net.go
@@ -0,0 +1,109 @@
+package pkg
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+ "path"
+ "unique"
+)
+
+// An httpArtifact is an [Artifact] backed by a [http] url string. The method is
+// hardcoded as [http.MethodGet]. Request body is not allowed because it cannot
+// be deterministically represented by Params.
+type httpArtifact struct {
+ // Caller-supplied url string.
+ url string
+
+ // Caller-supplied checksum of the response body. This is validated when
+ // closing the [io.ReadCloser] returned by Cure.
+ checksum unique.Handle[Checksum]
+
+ // client is the address of the caller-supplied [http.Client].
+ client *http.Client
+}
+
+var _ KnownChecksum = new(httpArtifact)
+var _ fmt.Stringer = new(httpArtifact)
+
+// NewHTTPGet returns a new [FileArtifact] backed by the supplied client. A GET
+// request is set up for url. If c is nil, [http.DefaultClient] is used instead.
+func NewHTTPGet(
+ c *http.Client,
+ url string,
+ checksum Checksum,
+) FileArtifact {
+ return &httpArtifact{url: url, checksum: unique.Make(checksum), client: c}
+}
+
+// Kind returns the hardcoded [Kind] constant.
+func (*httpArtifact) Kind() Kind { return KindHTTPGet }
+
+// Params writes the backing url string. Client is not represented as it does
+// not affect [Cache.Cure] outcome.
+func (a *httpArtifact) Params(ctx *IContext) { ctx.WriteString(a.url) }
+
+func init() {
+ register(KindHTTPGet, func(r *IRReader) Artifact {
+ url := r.ReadString()
+ checksum, ok := r.Finalise()
+ if !ok {
+ panic(ErrExpectedChecksum)
+ }
+ return NewHTTPGet(nil, url, checksum.Value())
+ })
+}
+
+// Inputs returns a nil slice.
+func (*httpArtifact) Inputs() []Artifact { return nil }
+
+// IsExclusive returns false: Cure returns as soon as a response is received.
+func (*httpArtifact) IsExclusive() bool { return false }
+
+// Checksum returns the caller-supplied checksum.
+func (a *httpArtifact) Checksum() Checksum { return a.checksum.Value() }
+
+// String returns [path.Base] over the backing url.
+func (a *httpArtifact) String() string { return path.Base(a.url) }
+
+// ResponseStatusError is returned for a response returned by an [http.Client]
+// with a status code other than [http.StatusOK].
+type ResponseStatusError int
+
+func (e ResponseStatusError) Error() string {
+ return "the requested URL returned non-OK status: " + http.StatusText(int(e))
+}
+
+// IsExecutable returns false.
+func (*httpArtifact) IsExecutable() bool { return false }
+
+// Cure sends the http request and returns the resulting response body reader
+// wrapped to perform checksum validation. It is valid but not encouraged to
+// close the resulting [io.ReadCloser] before it is read to EOF, as that causes
+// Close to block until all remaining data is consumed and validated.
+func (a *httpArtifact) Cure(r *RContext) (rc io.ReadCloser, err error) {
+ var req *http.Request
+ req, err = http.NewRequestWithContext(r.Unwrap(), http.MethodGet, a.url, nil)
+ if err != nil {
+ return
+ }
+ req.Header.Set("User-Agent", "Hakurei/1.1")
+
+ c := a.client
+ if c == nil {
+ c = http.DefaultClient
+ }
+
+ var resp *http.Response
+ if resp, err = c.Do(req); err != nil {
+ return
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ _ = resp.Body.Close()
+ return nil, ResponseStatusError(resp.StatusCode)
+ }
+
+ rc = r.NewMeasuredReader(resp.Body, a.checksum)
+ return
+}
diff --git a/pkg/net_test.go b/pkg/net_test.go
new file mode 100644
index 00000000..b16a2019
--- /dev/null
+++ b/pkg/net_test.go
@@ -0,0 +1,168 @@
+package pkg_test
+
+import (
+ "crypto/sha512"
+ "io"
+ "io/fs"
+ "net/http"
+ "reflect"
+ "testing"
+ "testing/fstest"
+ "unique"
+
+ "hakurei.app/check"
+ "hakurei.app/pkg"
+)
+
+func TestHTTPGet(t *testing.T) {
+ t.Parallel()
+
+ const testdata = "\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69"
+
+ testdataChecksum := func() unique.Handle[pkg.Checksum] {
+ h := sha512.New384()
+ h.Write([]byte(testdata))
+ return unique.Make(pkg.Checksum(h.Sum(nil)))
+ }()
+
+ var transport http.Transport
+ client := http.Client{Transport: &transport}
+ transport.RegisterProtocol("file", http.NewFileTransportFS(fstest.MapFS{
+ "testdata": {Data: []byte(testdata), Mode: 0400},
+ }))
+
+ checkWithCache(t, []cacheTestCase{
+ {"direct", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ r := newRContext(t, c)
+ f := pkg.NewHTTPGet(
+ &client,
+ "file:///testdata",
+ testdataChecksum.Value(),
+ )
+ var got []byte
+ if rc, err := f.Cure(r); err != nil {
+ t.Fatalf("Cure: error = %v", err)
+ } else if got, err = io.ReadAll(rc); err != nil {
+ t.Fatalf("ReadAll: error = %v", err)
+ } else if string(got) != testdata {
+ t.Fatalf("Cure: %x, want %x", got, testdata)
+ } else if err = rc.Close(); err != nil {
+ t.Fatalf("Close: error = %v", err)
+ }
+
+ // check direct validation
+ f = pkg.NewHTTPGet(
+ &client,
+ "file:///testdata",
+ pkg.Checksum{},
+ )
+ wantErrMismatch := &pkg.ChecksumMismatchError{
+ Got: testdataChecksum.Value(),
+ }
+ if rc, err := f.Cure(r); err != nil {
+ t.Fatalf("Cure: error = %v", err)
+ } else if got, err = io.ReadAll(rc); err != nil {
+ t.Fatalf("ReadAll: error = %v", err)
+ } else if string(got) != testdata {
+ t.Fatalf("Cure: %x, want %x", got, testdata)
+ } else if err = rc.Close(); !reflect.DeepEqual(err, wantErrMismatch) {
+ t.Fatalf("Close: error = %#v, want %#v", err, wantErrMismatch)
+ }
+
+ // check fallback validation
+ if rc, err := f.Cure(r); err != nil {
+ t.Fatalf("Cure: error = %v", err)
+ } else if err = rc.Close(); !reflect.DeepEqual(err, wantErrMismatch) {
+ t.Fatalf("Close: error = %#v, want %#v", err, wantErrMismatch)
+ }
+
+ // check direct response error
+ f = pkg.NewHTTPGet(
+ &client,
+ "file:///nonexistent",
+ pkg.Checksum{},
+ )
+ wantErrNotFound := pkg.ResponseStatusError(http.StatusNotFound)
+ if _, err := f.Cure(r); !reflect.DeepEqual(err, wantErrNotFound) {
+ t.Fatalf("Cure: error = %#v, want %#v", err, wantErrNotFound)
+ }
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"cure", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ r := newRContext(t, c)
+
+ f := pkg.NewHTTPGet(
+ &client,
+ "file:///testdata",
+ testdataChecksum.Value(),
+ )
+ wantPathname := base.Append(
+ "identifier",
+ "COBRUwkxpIvZWJggpRxn7HVXiw6epXZOex8jGRJKpowpwukV7GRBY_LBvOJNQHue",
+ )
+ if pathname, checksum, err := c.Cure(f); err != nil {
+ t.Fatalf("Cure: error = %v", err)
+ } else if !pathname.Is(wantPathname) {
+ t.Fatalf("Cure: %q, want %q", pathname, wantPathname)
+ } else if checksum != testdataChecksum {
+ t.Fatalf("Cure: %x, want %x", checksum.Value(), testdataChecksum.Value())
+ }
+
+ var got []byte
+ if rc, err := f.Cure(r); err != nil {
+ t.Fatalf("Cure: error = %v", err)
+ } else if got, err = io.ReadAll(rc); err != nil {
+ t.Fatalf("ReadAll: error = %v", err)
+ } else if string(got) != testdata {
+ t.Fatalf("Cure: %x, want %x", got, testdata)
+ } else if err = rc.Close(); err != nil {
+ t.Fatalf("Close: error = %v", err)
+ }
+
+ // check load from cache
+ f = pkg.NewHTTPGet(
+ &client,
+ "file:///testdata",
+ testdataChecksum.Value(),
+ )
+ if rc, err := f.Cure(r); err != nil {
+ t.Fatalf("Cure: error = %v", err)
+ } else if got, err = io.ReadAll(rc); err != nil {
+ t.Fatalf("ReadAll: error = %v", err)
+ } else if string(got) != testdata {
+ t.Fatalf("Cure: %x, want %x", got, testdata)
+ } else if err = rc.Close(); err != nil {
+ t.Fatalf("Close: error = %v", err)
+ }
+
+ // check error passthrough
+ f = pkg.NewHTTPGet(
+ &client,
+ "file:///nonexistent",
+ pkg.Checksum{},
+ )
+ wantErrNotFound := pkg.ResponseStatusError(http.StatusNotFound)
+ if _, _, err := c.Cure(f); !reflect.DeepEqual(err, wantErrNotFound) {
+ t.Fatalf("Pathname: error = %#v, want %#v", err, wantErrNotFound)
+ }
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU": {Mode: 0400, Data: []byte("\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/COBRUwkxpIvZWJggpRxn7HVXiw6epXZOex8jGRJKpowpwukV7GRBY_LBvOJNQHue": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+ })
+}
diff --git a/pkg/pkg.go b/pkg/pkg.go
new file mode 100644
index 00000000..5d9d8d48
--- /dev/null
+++ b/pkg/pkg.go
@@ -0,0 +1,2952 @@
+// Package pkg provides low-level primitives for packaging software.
+//
+// The public interface, IR format, and IR representation of [Artifact]
+// implementations not satisfying [RevisionArtifact] are covered by the
+// compatibility promise. [RevisionArtifact] revisions may change between any
+// two releases.
+package pkg
+
+import (
+ "bufio"
+ "bytes"
+ "cmp"
+ "context"
+ "crypto/rand"
+ "crypto/sha512"
+ "encoding/base64"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "hash"
+ "io"
+ "io/fs"
+ "iter"
+ "maps"
+ "math"
+ "os"
+ "path/filepath"
+ "runtime"
+ "slices"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "syscall"
+ "testing"
+ "time"
+ "unique"
+ "unsafe"
+
+ "hakurei.app/check"
+ "hakurei.app/internal/info"
+ "hakurei.app/internal/lockedfile"
+ "hakurei.app/message"
+)
+
+const (
+ // programName is the string identifying this build system.
+ programName = "hakurei.app/pkg"
+)
+
+type (
+ // A Checksum is a SHA-384 checksum computed for a cured [Artifact].
+ Checksum = [sha512.Size384]byte
+
+ // An ID is a unique identifier returned by [KnownIdent.ID]. This value must
+ // be deterministically determined ahead of time.
+ ID Checksum
+)
+
+// Encode is abbreviation for base64.URLEncoding.EncodeToString(checksum[:]).
+func Encode(checksum Checksum) string {
+ return base64.URLEncoding.EncodeToString(checksum[:])
+}
+
+// Decode is abbreviation for base64.URLEncoding.Decode(checksum[:], []byte(s)).
+func Decode(buf *Checksum, s string) (err error) {
+ var n int
+ n, err = base64.URLEncoding.Decode(buf[:], []byte(s))
+ if err == nil && n != len(buf) {
+ err = io.ErrUnexpectedEOF
+ }
+ return
+}
+
+// MustDecode decodes a string representation of [Checksum] and panics if there
+// is a decoding error or the resulting data is too short.
+func MustDecode(s string) (checksum Checksum) {
+ if err := Decode(&checksum, s); err != nil {
+ panic(err)
+ }
+ return
+}
+
+var (
+ // extension is a string uniquely identifying a set of custom [Artifact]
+ // implementations registered by calling [Register].
+ extension string
+
+ // openMu synchronises access to global state for initialisation.
+ openMu sync.Mutex
+ // opened is false if [Open] was never called.
+ opened bool
+)
+
+// Extension returns a string uniquely identifying the currently registered set
+// of custom [Artifact], or the zero value if none was registered.
+func Extension() string { return extension }
+
+// ValidExtension returns whether s is valid for use in a call to SetExtension.
+func ValidExtension(s string) bool {
+ if l := len(s); l == 0 || l > 128 {
+ return false
+ }
+ for _, v := range s {
+ if v < 'a' || v > 'z' {
+ return false
+ }
+ }
+ return true
+}
+
+// ErrInvalidExtension is returned for a variant identification string for which
+// [ValidExtension] returns false.
+var ErrInvalidExtension = errors.New("invalid extension variant identification string")
+
+// SetExtension sets the extension variant identification string. SetExtension
+// must be called before [Open] if custom [Artifact] implementations had been
+// recorded by calling [Register].
+//
+// The variant identification string must be between 1 and 128 bytes long and
+// consists of only bytes between 'a' and 'z'.
+//
+// SetExtension is not safe for concurrent use. SetExtension is called at most
+// once and must not be called after the first instance of Cache has been opened.
+func SetExtension(s string) {
+ openMu.Lock()
+ defer openMu.Unlock()
+
+ if opened {
+ panic("attempting to set extension after open")
+ }
+ if extension != "" {
+ panic("attempting to set extension twice")
+ }
+ if !ValidExtension(s) {
+ panic(ErrInvalidExtension)
+ }
+ extension = s
+ statusHeader = makeStatusHeader(s)
+}
+
+// common holds elements and receives methods shared between different contexts.
+type common struct {
+ // Context specific to this [Artifact]. The toplevel context in [Cache] must
+ // not be exposed directly.
+ ctx context.Context
+
+ // Address of underlying [Cache], should be zeroed or made unusable after
+ // Cure returns and must not be exposed directly.
+ cache *Cache
+}
+
+// TContext is passed to [TrivialArtifact.Cure] and provides information and
+// methods required for curing the [TrivialArtifact].
+//
+// Methods of TContext are safe for concurrent use. TContext is valid
+// until [TrivialArtifact.Cure] returns.
+type TContext struct {
+ // Populated during [Cache.Cure].
+ work, temp *check.Absolute
+
+ // Target [Artifact] encoded identifier.
+ ids string
+ // Pathname status was created at.
+ statusPath, statusSPath *check.Absolute
+ // File statusHeader and logs are written to.
+ status *os.File
+ // Error value during prepareStatus.
+ statusErr error
+
+ common
+}
+
+// makeStatusHeader creates the header written to every status file. This should
+// not be called directly, its result is stored in statusHeader and will not
+// change after the first [Cache] is opened.
+func makeStatusHeader(extension string) string {
+ s := programName
+ if v := info.Version(); v != info.FallbackVersion {
+ s += " " + v
+ }
+ if extension != "" {
+ s += " with " + extension + " extensions"
+ }
+ s += " (" + runtime.GOARCH + ")"
+ if name, err := os.Hostname(); err == nil {
+ s += " on " + name
+ }
+ s += "\n\n"
+ return s
+}
+
+// statusHeader is the header written to all status files in dirStatus.
+var statusHeader = makeStatusHeader("")
+
+// prepareStatus initialises the status file once.
+func (t *TContext) prepareStatus(writeHeader bool) error {
+ if t.statusPath != nil || t.status != nil {
+ return t.statusErr
+ }
+
+ t.statusPath = t.cache.base.Append(
+ dirStatus,
+ t.ids,
+ )
+ if t.status, t.statusErr = os.OpenFile(
+ t.statusPath.String(),
+ syscall.O_CREAT|syscall.O_EXCL|syscall.O_WRONLY,
+ 0400,
+ ); t.statusErr != nil {
+ return t.statusErr
+ }
+
+ if writeHeader {
+ _, t.statusErr = t.status.WriteString(statusHeader)
+ }
+ return t.statusErr
+}
+
+// GetStatusWriter returns a [io.Writer] for build logs. The caller must not
+// seek this writer before the position it was first returned in.
+func (t *TContext) GetStatusWriter() (io.Writer, error) {
+ err := t.prepareStatus(true)
+ return t.status, err
+}
+
+// destroy destroys the temporary directory and joins its errors with the error
+// referred to by errP. If the error referred to by errP is non-nil, the work
+// directory is removed similarly. [Cache] is responsible for making sure work
+// is never left behind for a successful [Cache.Cure].
+//
+// If implementation had requested status, it is closed with error joined with
+// the error referred to by errP. If the error referred to by errP is non-nil,
+// the status file is removed from the filesystem.
+//
+// destroy must be deferred by [Cache.Cure] if [TContext] is passed to any Cure
+// implementation. It should not be called prior to that point.
+func (t *TContext) destroy(errP *error) {
+ if chmodErr, removeErr := removeAll(t.temp); chmodErr != nil || removeErr != nil {
+ *errP = errors.Join(*errP, chmodErr, removeErr)
+ }
+
+ if *errP != nil {
+ chmodErr, removeErr := removeAll(t.work)
+ if chmodErr != nil || removeErr != nil {
+ *errP = errors.Join(*errP, chmodErr, removeErr)
+ } else if errors.Is(*errP, os.ErrExist) {
+ if linkError, ok := errors.AsType[*os.LinkError](*errP); ok &&
+ linkError != nil &&
+ linkError.Op == "rename" {
+ // two artifacts may be backed by the same file
+ *errP = nil
+ }
+ }
+ }
+
+ if t.status != nil {
+ if err := t.status.Close(); err != nil {
+ *errP = errors.Join(*errP, err)
+ }
+ if *errP != nil {
+ *errP = errors.Join(*errP, os.Rename(
+ t.statusPath.String(), t.cache.base.Append(
+ dirFault,
+ t.ids+"."+strconv.FormatUint(uint64(
+ time.Now().UnixNano(),
+ ), 10),
+ ).String(),
+ ))
+ if t.statusSPath != nil {
+ t.cache.checksumMu.Lock()
+ *errP = errors.Join(*errP, os.Remove(t.statusSPath.String()))
+ t.cache.checksumMu.Unlock()
+ }
+ }
+ t.status = nil
+ }
+}
+
+// Unwrap returns the underlying [context.Context].
+func (c *common) Unwrap() context.Context { return c.ctx }
+
+// GetMessage returns [message.Msg] held by the underlying [Cache].
+func (c *common) GetMessage() message.Msg { return c.cache.msg }
+
+// GetJobs returns the preferred number of jobs to run, when applicable. Its
+// value must not affect cure outcome.
+func (c *common) GetJobs() int { return c.cache.attr.Jobs }
+
+// GetLoad returns the preferred load average target, when applicable. Its
+// value must not affect cure outcome.
+func (c *common) GetLoad() int { return c.cache.attr.Load }
+
+// GetWorkDir returns a pathname to a directory which [Artifact] is expected to
+// write its output to. This is not the final resting place of the [Artifact]
+// and this pathname should not be directly referred to in the final contents.
+func (t *TContext) GetWorkDir() *check.Absolute { return t.work }
+
+// GetTempDir returns a pathname which implementations may use as scratch space.
+// A directory is not created automatically, implementations are expected to
+// create it if they wish to use it, using [os.MkdirAll].
+func (t *TContext) GetTempDir() *check.Absolute { return t.temp }
+
+// Open tries to open [Artifact] for reading. If a implements [FileArtifact],
+// its reader might be used directly, eliminating the roundtrip to vfs.
+// Otherwise, it must cure into a directory containing a single regular file.
+//
+// If err is nil, the caller must close the resulting [io.ReadCloser] and return
+// its error, if any. Failure to read r to EOF may result in a spurious
+// [ChecksumMismatchError], or the underlying implementation may block on Close.
+func (c *common) Open(a Artifact) (r io.ReadCloser, err error) {
+ if f, ok := a.(FileArtifact); ok {
+ return c.cache.openFile(c.ctx, f)
+ }
+
+ var pathname *check.Absolute
+ if pathname, _, _, err = c.cache.cure(a, true, false); err != nil {
+ return
+ }
+
+ var entries []os.DirEntry
+ if entries, err = os.ReadDir(pathname.String()); err != nil {
+ return
+ }
+
+ if len(entries) != 1 || !entries[0].Type().IsRegular() {
+ err = errors.New(
+ "input directory does not contain a single regular file",
+ )
+ return
+ } else {
+ return os.Open(pathname.Append(entries[0].Name()).String())
+ }
+}
+
+// FContext is passed to [FloodArtifact.Cure] and provides information and
+// methods required for curing the [FloodArtifact].
+//
+// Methods of FContext are safe for concurrent use. FContext is valid
+// until [FloodArtifact.Cure] returns.
+type FContext struct {
+ TContext
+
+ // Cured top-level inputs looked up by Pathname.
+ inputs map[Artifact]cureRes
+}
+
+// linkSubstitute links status for substitute if populated.
+func (f *FContext) linkSubstitute(ids, substitutes string) (err error) {
+ if f.status == nil || ids == substitutes {
+ return
+ }
+
+ statusS := f.cache.base.Append(
+ dirStatus,
+ substitutes,
+ )
+ f.cache.checksumMu.Lock()
+ err = os.Link(f.cache.base.Append(
+ dirStatus,
+ ids,
+ ).String(), statusS.String())
+ f.cache.checksumMu.Unlock()
+ if err == nil {
+ f.statusSPath = statusS
+ }
+ return
+}
+
+// InvalidLookupError is the identifier of non-input [Artifact] looked up
+// via [FContext.GetArtifact] by a misbehaving [Artifact] implementation.
+type InvalidLookupError ID
+
+func (e InvalidLookupError) Error() string {
+ return "attempting to look up non-input artifact " + Encode(e)
+}
+
+var _ error = InvalidLookupError{}
+
+// GetArtifact returns the identifier pathname and checksum of an [Artifact].
+// Calling Pathname with an [Artifact] not part of the slice returned by
+// [Artifact.Inputs] panics.
+func (f *FContext) GetArtifact(a Artifact) (
+ pathname *check.Absolute,
+ checksum unique.Handle[Checksum],
+) {
+ if res, ok := f.inputs[a]; ok {
+ return res.pathname, res.checksum
+ }
+ panic(InvalidLookupError(f.cache.Ident(a).Value()))
+}
+
+// RContext is passed to [FileArtifact.Cure] and provides helper methods useful
+// for curing the [FileArtifact].
+//
+// Methods of RContext are safe for concurrent use. RContext is valid
+// until [FileArtifact.Cure] returns.
+type RContext struct{ common }
+
+// An Artifact is a read-only reference to a piece of data that may be created
+// deterministically but might not currently be available in memory or on the
+// filesystem.
+type Artifact interface {
+ // Kind returns the [Kind] of artifact. This is usually unique to the
+ // concrete type but two functionally identical implementations of
+ // [Artifact] is allowed to return the same [Kind] value.
+ Kind() Kind
+
+ // Params writes deterministic values describing [Artifact]. Implementations
+ // must guarantee that these values are unique among differing instances
+ // of the same implementation with identical dependencies and conveys enough
+ // information to create another instance of [Artifact] identical to the
+ // instance emitting these values. The new instance created via [IRReadFunc]
+ // from these values must then produce identical IR values.
+ //
+ // Result must remain identical across multiple invocations.
+ Params(ctx *IContext)
+
+ // Inputs returns a slice of [Artifact] the current instance has access to
+ // while producing its output.
+ //
+ // Callers must not modify the retuned slice.
+ //
+ // Result must remain identical across multiple invocations.
+ Inputs() []Artifact
+
+ // IsExclusive returns whether the [Artifact] is exclusive. Exclusive
+ // artifacts might not run in parallel with each other, and are still
+ // subject to the cures limit.
+ //
+ // Some implementations may saturate the CPU for a nontrivial amount of
+ // time. Curing multiple such implementations simultaneously causes
+ // significant CPU scheduler overhead. An exclusive artifact will generally
+ // not be cured alongside another exclusive artifact, thus alleviating this
+ // overhead.
+ //
+ // Note that [Cache] reserves the right to still cure exclusive
+ // artifacts concurrently as this is not a synchronisation primitive but
+ // an optimisation one. Implementations are forbidden from accessing global
+ // state regardless of exclusivity.
+ //
+ // Result must remain identical across multiple invocations.
+ IsExclusive() bool
+}
+
+// FloodArtifact refers to an [Artifact] requiring its entire dependency graph
+// to be cured prior to curing itself.
+type FloodArtifact interface {
+ // Cure cures the current [Artifact] to the working directory obtained via
+ // [TContext.GetWorkDir] embedded in [FContext].
+ //
+ // Implementations must not retain c.
+ Cure(f *FContext) (err error)
+
+ Artifact
+}
+
+// TrivialArtifact refers to an [Artifact] that cures without requiring that
+// any other [Artifact] is cured before it. Its dependency tree is ignored after
+// computing its identifier.
+//
+// TrivialArtifact is unable to cure any other [Artifact] and it cannot access
+// pathnames. This type of [Artifact] is primarily intended for dependency-less
+// artifacts or direct dependencies that only consists of [FileArtifact].
+type TrivialArtifact interface {
+ // Cure cures the current [Artifact] to the working directory obtained via
+ // [TContext.GetWorkDir].
+ //
+ // Implementations must not retain c.
+ Cure(t *TContext) (err error)
+
+ Artifact
+}
+
+// KnownIdent is optionally implemented by [Artifact] and is used instead of
+// [Cache.Ident] when it is available.
+//
+// This is very subtle to use correctly. The implementation must ensure that
+// this value is globally unique, otherwise [Cache] can enter an inconsistent
+// state. This should not be implemented outside of testing.
+type KnownIdent interface {
+ // ID returns a globally unique identifier referring to the current
+ // [Artifact]. This value must be known ahead of time and guaranteed to be
+ // unique without having obtained the full contents of the [Artifact].
+ ID() ID
+
+ Artifact
+}
+
+// KnownChecksum is optionally implemented by [Artifact] for an artifact with
+// output known ahead of time.
+type KnownChecksum interface {
+ // Checksum returns the address of a known checksum.
+ //
+ // Callers must not modify the [Checksum].
+ //
+ // Result must remain identical across multiple invocations.
+ Checksum() Checksum
+
+ Artifact
+}
+
+// CuresExempt is optionally implemented for an artifact exempt to the
+// cache-wide cures counter and limit.
+type CuresExempt interface {
+ Artifact
+
+ // CuresExempt is a no-op function but serves to distinguish implementations
+ // that are cures-exempt.
+ CuresExempt()
+}
+
+// FileArtifact refers to an [Artifact] backed by a single file.
+//
+// FileArtifact does not support fine-grained cancellation. Its context is
+// inherited from the first [TrivialArtifact] or [FloodArtifact] that opens it.
+type FileArtifact interface {
+ // IsExecutable returns whether the resulting filesystem entry should be made
+ // executable, if the [FileArtifact] is cured to the on-disk cache.
+ //
+ // Result must remain identical across multiple invocations.
+ IsExecutable() bool
+
+ // Cure returns [io.ReadCloser] of the full contents of [FileArtifact]. If
+ // [FileArtifact] implements [KnownChecksum], Cure is responsible for
+ // validating any data it produces and must return [ChecksumMismatchError]
+ // if validation fails. This error is conventionally returned during the
+ // first call to Close, but may be returned during any call to Read before
+ // EOF, or by Cure itself.
+ //
+ // Callers are responsible for closing the resulting [io.ReadCloser].
+ //
+ // The resulting [io.ReadCloser] across multiple invocations must have
+ // identical behaviour.
+ Cure(r *RContext) (io.ReadCloser, error)
+
+ Artifact
+}
+
+// RevisionArtifact is optionally implemented by an artifact that had undergone
+// internal changes affecting its behaviour while retaining its IR structure.
+type RevisionArtifact interface {
+ // Revision returns the revision number of [Artifact]. This value is always
+ // represented in the IR. An IR stream produced from the same [Kind] with
+ // differing revision is rejected.
+ //
+ // Result must remain identical across multiple invocations.
+ Revision() uint64
+
+ Artifact
+}
+
+// GetRevision returns the revision number of an [Artifact].
+func GetRevision(a Artifact) (revision uint64) {
+ revision = math.MaxUint64
+ if r, ok := a.(RevisionArtifact); ok {
+ revision = r.Revision()
+ }
+ return
+}
+
+// reportName returns a string describing [Artifact] presented to the user.
+func reportName(a Artifact, id unique.Handle[ID]) string {
+ r := Encode(id.Value())
+ if s, ok := a.(fmt.Stringer); ok {
+ if name := s.String(); name != "" {
+ r += "-" + name
+ }
+ }
+ return r
+}
+
+// Kind corresponds to the concrete type of [Artifact] and is used to create
+// identifier for an [Artifact] with dependencies.
+type Kind uint64
+
+const (
+ // KindHTTPGet is the kind of [Artifact] returned by [NewHTTPGet].
+ KindHTTPGet Kind = iota
+ // KindTar is the kind of [Artifact] returned by [NewTar].
+ KindTar
+ // KindExec is the kind of [Artifact] returned by [NewExec].
+ KindExec
+ // KindExecNet is the kind of [Artifact] returned by [NewExec] but with a
+ // non-nil checksum.
+ KindExecNet
+ // KindFile is the kind of [Artifact] returned by [NewFile].
+ KindFile
+ // KindDecompress is the kind of [Artifact] returned by [NewDecompress].
+ KindDecompress
+ // KindArchive is the kind of [Artifact] returned by [NewArchive].
+ KindArchive
+
+ // _kindEnd is the total number of kinds and does not denote a kind.
+ _kindEnd
+
+ // KindCustomOffset is the first [Kind] value reserved for implementations
+ // not from this package.
+ KindCustomOffset = 1 << 31
+)
+
+const (
+ // kindCollection is the kind of [Collect]. It never cures successfully.
+ kindCollection Kind = KindCustomOffset - 1 - iota
+)
+
+const (
+ // fileLock is the lock file for exclusive access to the cache directory.
+ fileLock = "lock"
+ // fileVariant is a file holding the variant identification string set by a
+ // prior call to [SetExtension].
+ fileVariant = "variant"
+
+ // dirSubstitute holds symlinks to artifacts by checksum, named after their
+ // substitute identifier.
+ dirSubstitute = "substitute"
+ // dirIdentifier holds symlinks to artifacts by checksum, named after their
+ // IR-based identifier.
+ dirIdentifier = "identifier"
+ // dirChecksum holds artifacts named after their [Checksum].
+ dirChecksum = "checksum"
+ // dirStatus holds artifact metadata and logs named after their IR-based
+ // identifier. For [FloodArtifact], the same file is also available under
+ // its substitute identifier.
+ dirStatus = "status"
+ // dirFault holds status files of faulted cures.
+ dirFault = "fault"
+
+ // dirWork holds working pathnames set up during [Cache.Cure].
+ dirWork = "work"
+ // dirTemp holds scratch space allocated during [Cache.Cure].
+ dirTemp = "temp"
+
+ // dirExecScratch is scratch space set up for the container started by
+ // [Cache.EnterExec]. Exclusivity via Cache.inExec.
+ dirExecScratch = "scratch"
+
+ // checksumLinknamePrefix is prepended to the encoded [Checksum] value
+ // of an [Artifact] when creating a symbolic link to dirChecksum.
+ checksumLinknamePrefix = "../" + dirChecksum + "/"
+)
+
+// cureRes are the non-error results returned by [Cache.Cure].
+type cureRes struct {
+ pathname *check.Absolute
+ checksum unique.Handle[Checksum]
+}
+
+// A pendingArtifactDep is an input [Artifact] pending concurrent curing,
+// subject to the cures limit. Values pointed to by result addresses are safe
+// to access after the [sync.WaitGroup] associated with this pendingArtifactDep
+// is done. pendingArtifactDep must not be reused or modified after it is sent
+// to cure.
+type pendingArtifactDep struct {
+ // Dependency artifact populated during [Cache.Cure].
+ a Artifact
+
+ // Address of result pathname populated during [Cache.Cure] and dereferenced
+ // if curing succeeds.
+ resP *cureRes
+
+ // Address of result error map populated during [Cache.Cure], dereferenced
+ // after acquiring errsMu if curing fails. No additional action is taken,
+ // [Cache] and its caller are responsible for further error handling.
+ errs InputError
+ // Address of mutex synchronising access to errs.
+ errsMu *sync.Mutex
+
+ // For synchronising access to result buffer.
+ *sync.WaitGroup
+}
+
+const (
+ // CValidateKnown arranges for [KnownChecksum] outcomes to be validated to
+ // match its intended checksum.
+ //
+ // A correct implementation of [KnownChecksum] does not successfully cure
+ // with output not matching its intended checksum. When an implementation
+ // fails to perform this validation correctly, the on-disk format enters
+ // an inconsistent state (correctable by [Cache.Scrub]).
+ //
+ // This flag causes [Cache.Cure] to always compute the checksum, and reject
+ // a cure if it does not match the intended checksum.
+ //
+ // This behaviour significantly reduces performance and is not recommended
+ // outside of testing a custom [Artifact] implementation.
+ CValidateKnown = 1 << iota
+
+ // CSchedIdle arranges for the [ext.SCHED_IDLE] scheduling priority to be
+ // set for [KindExec] and [KindExecNet] containers.
+ CSchedIdle
+
+ // CAssumeChecksum enables the use of [KnownChecksum] for duplicate function
+ // call suppression via the on-disk cache.
+ //
+ // This may cause incorrect cure outcome if an impossible checksum is
+ // specified that matches an output already present in the on-disk cache.
+ // This may be avoided by purposefully specifying a statistically
+ // unattainable checksum, like the zero value.
+ //
+ // While this optimisation might seem appealing, it is almost never
+ // applicable in real world use. Almost every time this path was taken, it
+ // was caused by an incorrect checksum accidentally left behind while
+ // bumping a package. Only enable this if you are really sure you need it.
+ CAssumeChecksum
+
+ // CHostAbstract disables restriction of sandboxed processes from connecting
+ // to an abstract UNIX socket created by a host process.
+ //
+ // This is considered less secure in some systems, but does not introduce
+ // impurity due to [KindExecNet] being [KnownChecksum]. This flag exists
+ // to support kernels without Landlock LSM enabled.
+ CHostAbstract
+
+ // CPromoteVariant allows [pkg.Open] to promote an unextended on-disk cache
+ // to the current extension variant. This is a one-way operation.
+ CPromoteVariant
+
+ // CSuppressInit arranges for verbose output of the container init to be
+ // suppressed regardless of [message.Msg] state.
+ CSuppressInit
+
+ // CIgnoreSubstitutes disables content-based input substitution.
+ CIgnoreSubstitutes
+
+ // CExternShallow arranges for only non-flood inputs to be fetched when
+ // curing an [Artifact] available via the external cache.
+ CExternShallow
+
+ // CColourOutput enables output colouring via ANSI control sequences.
+ CColourOutput
+)
+
+// toplevel holds [context.WithCancel] over caller-supplied context, where all
+// [Artifact] context are derived from.
+type toplevel struct {
+ ctx context.Context
+ cancel context.CancelFunc
+}
+
+// newToplevel returns the address of a new toplevel via ctx.
+func newToplevel(ctx context.Context) *toplevel {
+ var t toplevel
+ t.ctx, t.cancel = context.WithCancel(ctx)
+ return &t
+}
+
+// pendingCure provides synchronisation and cancellation for pending cures.
+type pendingCure struct {
+ // Closed on cure completion.
+ done <-chan struct{}
+ // Error outcome, safe to access after done is closed.
+ err error
+ // Cancels the corresponding cure.
+ cancel context.CancelFunc
+}
+
+// An External cache provides prepared [Artifact] cure outcomes.
+type External interface {
+ // Artifact returns the address of the [Checksum] of the cure outcome of
+ // an [Artifact] corresponding to id, or nil if this [Artifact] is not
+ // available in the external cache.
+ Artifact(ctx context.Context, id unique.Handle[ID]) (*Checksum, error)
+ // Checksum returns an [Artifact] producing the specified checksum.
+ Checksum(checksum unique.Handle[Checksum]) Artifact
+ // Status returns [io.ReadCloser] of the status file of an [Artifact]
+ // corresponding to id, or nil if this [Artifact] is not available or a
+ // status file is not present.
+ Status(r *RContext, id unique.Handle[ID]) (io.ReadCloser, error)
+}
+
+// Cache is a support layer that implementations of [Artifact] can use to store
+// cured [Artifact] data in a content addressed fashion.
+type Cache struct {
+ // Cures of any variant of [Artifact] sends to cures before entering the
+ // implementation and receives an equal amount of elements after.
+ cures chan struct{}
+
+ // Parent context which toplevel was derived from.
+ parent context.Context
+ // For deriving curing context, must not be accessed directly.
+ toplevel atomic.Pointer[toplevel]
+ // For waiting on input curing goroutines.
+ wg sync.WaitGroup
+ // Reports new cures and passed to [Artifact].
+ msg message.Msg
+ // Select graphics rendition sequences, populated by Open.
+ sgrRes, sgrIdent, sgrWarn, sgrErr string
+
+ // Directory where all [Cache] related files are placed.
+ base *check.Absolute
+ // Immutable [CacheAttr] populated by [Open].
+ attr CacheAttr
+
+ // Must not be exposed directly.
+ irCache
+
+ // Synchronises access to dirChecksum.
+ checksumMu sync.RWMutex
+
+ // Presence of an alternative in the cache. Keys are not valid identifiers
+ // and must not be used as such.
+ substitute map[unique.Handle[ID]]unique.Handle[Checksum]
+ // Synchronises access to substitute and corresponding filesystem entries.
+ substituteMu sync.RWMutex
+ // Identifier to content pair cache.
+ ident map[unique.Handle[ID]]unique.Handle[Checksum]
+ // Identifier to error pair for unrecoverably faulted [Artifact].
+ identErr map[unique.Handle[ID]]error
+ // Pending identifiers, accessed through Cure for entries not in ident.
+ identPending map[unique.Handle[ID]]*pendingCure
+ // Synchronises access to ident and corresponding filesystem entries.
+ identMu sync.RWMutex
+ // Synchronises entry into Abort and Cure.
+ abortMu sync.RWMutex
+
+ // Synchronises entry into exclusive artifacts for the cure method.
+ exclMu sync.Mutex
+ // Buffered I/O free list, must not be accessed directly.
+ brPool, bwPool sync.Pool
+
+ // Optional external cache implementation.
+ extern External
+ // Caches responses from extern.
+ externCache map[unique.Handle[ID]]unique.Handle[Checksum]
+ // Synchronises access to extern.
+ externMu sync.RWMutex
+
+ // Unlocks the on-filesystem cache. Must only be called from Close.
+ unlock func()
+ // Whether [Cache] is considered closed.
+ closed bool
+ // Synchronises calls to Abort and Close.
+ closeMu sync.Mutex
+
+ // Whether EnterExec has not yet returned.
+ inExec atomic.Bool
+}
+
+// extIdent is a [Kind] concatenated with [ID].
+type extIdent [wordSize + len(ID{})]byte
+
+// getIdentBuf returns the address of an extIdent for Ident.
+func (ic *irCache) getIdentBuf() *extIdent { return ic.identPool.Get().(*extIdent) }
+
+// putIdentBuf adds buf to identPool.
+func (ic *irCache) putIdentBuf(buf *extIdent) { ic.identPool.Put(buf) }
+
+// storeIdent adds an [Artifact] to the artifact cache.
+func (ic *irCache) storeIdent(a Artifact, buf *extIdent) unique.Handle[ID] {
+ idu := unique.Make(ID(buf[wordSize:]))
+ ic.artifact.Store(a, idu)
+ return idu
+}
+
+// Ident returns the identifier of an [Artifact].
+func (ic *irCache) Ident(a Artifact) unique.Handle[ID] {
+ buf, idu := ic.unsafeIdent(a, false)
+ if buf != nil {
+ idu = ic.storeIdent(a, buf)
+ ic.putIdentBuf(buf)
+ }
+ return idu
+}
+
+// unsafeIdent implements Ident but returns the underlying buffer for a newly
+// computed identifier. Callers must return this buffer to identPool. encodeKind
+// is only a hint, kind may still be encoded in the buffer.
+func (ic *irCache) unsafeIdent(a Artifact, encodeKind bool) (
+ buf *extIdent,
+ idu unique.Handle[ID],
+) {
+ if id, ok := ic.artifact.Load(a); ok {
+ idu = id.(unique.Handle[ID])
+ return
+ }
+
+ if ki, ok := a.(KnownIdent); ok {
+ buf = ic.getIdentBuf()
+ if encodeKind {
+ binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind()))
+ }
+ *(*ID)(buf[wordSize:]) = ki.ID()
+ return
+ }
+
+ buf = ic.getIdentBuf()
+ h := sha512.New384()
+ if err := ic.Encode(h, a); err != nil {
+ // unreachable
+ panic(err)
+ }
+ binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind()))
+ h.Sum(buf[wordSize:wordSize])
+ return
+}
+
+// getReader is like [bufio.NewReader] but for brPool.
+func (c *Cache) getReader(r io.Reader) *bufio.Reader {
+ br := c.brPool.Get().(*bufio.Reader)
+ br.Reset(r)
+ return br
+}
+
+// putReader adds br to brPool.
+func (c *Cache) putReader(br *bufio.Reader) { c.brPool.Put(br) }
+
+// bufioReadCloser is the concrete type of value returned by Cache.getReaderRC.
+type bufioReadCloser struct {
+ // Saved close error.
+ closeErr error
+ // Synchronises calls to Close.
+ closeOnce sync.Once
+
+ // For backing freelist.
+ c *Cache
+ // Underlying reader.
+ r io.ReadCloser
+ // Allocated from c.
+ *bufio.Reader
+}
+
+// Close closes the underlying reader, saves its return value, and returns the
+// [bufio.Reader] instance to the backing [Cache].
+func (brc *bufioReadCloser) Close() error {
+ brc.closeOnce.Do(func() {
+ br := brc.Reader
+ brc.Reader = nil
+ brc.c.putReader(br)
+ brc.closeErr = brc.r.Close()
+ })
+ return brc.closeErr
+}
+
+// getReaderRC is like getReader, but returns an [io.ReadCloser].
+func (c *Cache) getReaderRC(r io.ReadCloser) io.ReadCloser {
+ return &bufioReadCloser{c: c, r: r, Reader: c.getReader(r)}
+}
+
+// getWriter is like [bufio.NewWriter] but for bwPool.
+func (c *Cache) getWriter(w io.Writer) *bufio.Writer {
+ bw := c.bwPool.Get().(*bufio.Writer)
+ bw.Reset(w)
+ return bw
+}
+
+// putWriter adds bw to bwPool.
+func (c *Cache) putWriter(bw *bufio.Writer) { c.bwPool.Put(bw) }
+
+// A ChecksumMismatchError describes an [Artifact] with unexpected content.
+type ChecksumMismatchError struct {
+ // Actual and expected checksums.
+ Got, Want Checksum
+}
+
+func (e *ChecksumMismatchError) Error() string {
+ return "got " + Encode(e.Got) +
+ " instead of " + Encode(e.Want)
+}
+
+// LinknamePrefixError describes a malformed linkname to a [Checksum].
+type LinknamePrefixError string
+
+func (e LinknamePrefixError) Error() string {
+ return "linkname " + strconv.Quote(string(e)) + " missing prefix"
+}
+
+// readlinkChecksum reads a symbolic link to a dirChecksum entry and saves the
+// decoded [Checksum] to the value pointed to by buf. The checksumLinknamePrefix
+// is required.
+func readlinkChecksum(a *check.Absolute, buf *Checksum) error {
+ linkname, err := os.Readlink(a.String())
+ if err != nil {
+ return nil
+ }
+
+ if !strings.HasPrefix(linkname, checksumLinknamePrefix) {
+ return LinknamePrefixError(linkname)
+ }
+ return Decode(buf, linkname[len(checksumLinknamePrefix):])
+}
+
+// SetExternal sets e as the [External] implementation of c.
+func (c *Cache) SetExternal(e External) {
+ c.externMu.Lock()
+ c.externCache = make(map[unique.Handle[ID]]unique.Handle[Checksum])
+ c.extern = e
+ c.externMu.Unlock()
+}
+
+// ScrubError describes the outcome of a [Cache.Scrub] call where errors were
+// found and removed from the underlying storage of [Cache].
+type ScrubError struct {
+ // Content-addressed entries not matching their checksum. This can happen
+ // if an incorrect [FileArtifact] implementation was cured against
+ // a non-strict [Cache].
+ ChecksumMismatches []ChecksumMismatchError
+ // Dangling identifier symlinks. This can happen if the content-addressed
+ // entry was removed while scrubbing due to a checksum mismatch.
+ DanglingIdentifiers []ID
+ // Dangling status files. This can happen if a dangling status symlink was
+ // removed while scrubbing.
+ DanglingStatus []ID
+ // Miscellaneous errors, including [os.ReadDir] on checksum and identifier
+ // directories, [Decode] on entry names and [os.RemoveAll] on inconsistent
+ // entries.
+ Errs map[unique.Handle[string]][]error
+}
+
+// errs is a deterministic iterator over Errs.
+func (e *ScrubError) errs(yield func(unique.Handle[string], []error) bool) {
+ keys := slices.AppendSeq(
+ make([]unique.Handle[string], 0, len(e.Errs)),
+ maps.Keys(e.Errs),
+ )
+ slices.SortFunc(keys, func(a, b unique.Handle[string]) int {
+ return strings.Compare(a.Value(), b.Value())
+ })
+ for _, key := range keys {
+ if !yield(key, e.Errs[key]) {
+ break
+ }
+ }
+}
+
+// Unwrap returns a concatenation of ChecksumMismatches and Errs.
+func (e *ScrubError) Unwrap() []error {
+ s := make([]error, 0, len(e.ChecksumMismatches)+len(e.Errs))
+ for _, err := range e.ChecksumMismatches {
+ s = append(s, &err)
+ }
+ for _, errs := range e.errs {
+ s = append(s, errs...)
+ }
+ return s
+}
+
+// Error returns a multi-line representation of [ScrubError].
+func (e *ScrubError) Error() string {
+ var segments []string
+ var buf strings.Builder
+
+ if len(e.ChecksumMismatches) > 0 {
+ buf.Reset()
+ buf.WriteString("checksum mismatches:\n")
+ for _, m := range e.ChecksumMismatches {
+ buf.WriteString(m.Error() + "\n")
+ }
+ segments = append(segments, buf.String())
+ }
+ if len(e.DanglingIdentifiers) > 0 {
+ buf.Reset()
+ buf.WriteString("dangling identifiers:\n")
+ for _, id := range e.DanglingIdentifiers {
+ buf.WriteString(Encode(id) + "\n")
+ }
+ segments = append(segments, buf.String())
+ }
+ if len(e.DanglingStatus) > 0 {
+ buf.Reset()
+ buf.WriteString("dangling status:\n")
+ for _, id := range e.DanglingStatus {
+ buf.WriteString(Encode(id) + "\n")
+ }
+ segments = append(segments, buf.String())
+ }
+ if len(e.Errs) > 0 {
+ buf.Reset()
+ buf.WriteString("errors during scrub:\n")
+ for pathname, errs := range e.errs {
+ buf.WriteString(" " + pathname.Value() + ":\n")
+ for _, err := range errs {
+ buf.WriteString(" " + err.Error() + "\n")
+ }
+ }
+ segments = append(segments, buf.String())
+ }
+ return strings.Join(segments, "\n")
+}
+
+// Scrub frees internal in-memory identifier to content pair cache, verifies all
+// cached artifacts against their checksums, checks for dangling identifier
+// symlinks and removes them if found.
+//
+// This method is not safe for concurrent use with any other method.
+func (c *Cache) Scrub(checks int) error {
+ if checks <= 0 {
+ checks = runtime.NumCPU()
+ }
+
+ c.substituteMu.Lock()
+ defer c.substituteMu.Unlock()
+ c.identMu.Lock()
+ defer c.identMu.Unlock()
+ c.checksumMu.Lock()
+ defer c.checksumMu.Unlock()
+
+ c.substitute = make(map[unique.Handle[ID]]unique.Handle[Checksum])
+ c.ident = make(map[unique.Handle[ID]]unique.Handle[Checksum])
+ c.identErr = make(map[unique.Handle[ID]]error)
+ c.artifact.Clear()
+
+ var (
+ se = ScrubError{Errs: make(map[unique.Handle[string]][]error)}
+ seMu sync.Mutex
+
+ addErr = func(pathname *check.Absolute, err error) {
+ seMu.Lock()
+ se.Errs[pathname.Handle()] = append(se.Errs[pathname.Handle()], err)
+ seMu.Unlock()
+ }
+ )
+
+ type checkEntry struct {
+ ent os.DirEntry
+ check func(ent os.DirEntry, want *Checksum) bool
+ }
+ var (
+ dir *check.Absolute
+ wg sync.WaitGroup
+ w = make(chan checkEntry, checks)
+ p = sync.Pool{New: func() any { return new(Checksum) }}
+ )
+ condemn := func(ent os.DirEntry) {
+ pathname := dir.Append(ent.Name())
+ chmodErr, removeErr := removeAll(pathname)
+ if chmodErr != nil {
+ addErr(pathname, chmodErr)
+ }
+ if removeErr != nil {
+ addErr(pathname, removeErr)
+ }
+ }
+ for i := 0; i < checks; i++ {
+ go func() {
+ for ce := range w {
+ want := p.Get().(*Checksum)
+ ent := ce.ent
+ if err := Decode(want, ent.Name()); err != nil {
+ addErr(dir.Append(ent.Name()), err)
+ wg.Go(func() { condemn(ent) })
+ } else if !ce.check(ent, want) {
+ wg.Go(func() { condemn(ent) })
+ } else {
+ c.msg.Verbosef(
+ "%s%s%s is consistent",
+ c.sgrIdent, ent.Name(), c.sgrRes,
+ )
+ }
+ p.Put(want)
+ wg.Done()
+ }
+ }()
+ }
+ defer close(w)
+
+ dir = c.base.Append(dirChecksum)
+ if entries, readdirErr := os.ReadDir(dir.String()); readdirErr != nil {
+ addErr(dir, readdirErr)
+ } else {
+ wg.Add(len(entries))
+ for _, ent := range entries {
+ w <- checkEntry{ent, func(ent os.DirEntry, want *Checksum) bool {
+ got := p.Get().(*Checksum)
+ defer p.Put(got)
+
+ pathname := dir.Append(ent.Name())
+ if ent.IsDir() {
+ if err := SumDir(got, pathname); err != nil {
+ addErr(pathname, err)
+ return true
+ }
+ } else if ent.Type().IsRegular() {
+ h := sha512.New384()
+
+ if r, err := os.Open(pathname.String()); err != nil {
+ addErr(pathname, err)
+ return true
+ } else {
+ _, err = io.Copy(h, r)
+ closeErr := r.Close()
+ if closeErr != nil {
+ addErr(pathname, closeErr)
+ }
+ if err != nil {
+ addErr(pathname, err)
+ }
+ }
+ h.Sum(got[:0])
+ } else {
+ addErr(pathname, InvalidFileModeError(ent.Type()))
+ return false
+ }
+
+ if *got != *want {
+ seMu.Lock()
+ se.ChecksumMismatches = append(se.ChecksumMismatches,
+ ChecksumMismatchError{Got: *got, Want: *want},
+ )
+ seMu.Unlock()
+ return false
+ }
+ return true
+ }}
+ }
+ wg.Wait()
+ }
+
+ for _, suffix := range []string{
+ dirSubstitute,
+ dirIdentifier,
+ } {
+ dir = c.base.Append(suffix)
+ if entries, readdirErr := os.ReadDir(dir.String()); readdirErr != nil {
+ addErr(dir, readdirErr)
+ } else {
+ wg.Add(len(entries))
+ for _, ent := range entries {
+ w <- checkEntry{ent, func(ent os.DirEntry, want *Checksum) bool {
+ got := p.Get().(*Checksum)
+ defer p.Put(got)
+
+ pathname := dir.Append(ent.Name())
+ if linkname, err := os.Readlink(
+ pathname.String(),
+ ); err != nil {
+ seMu.Lock()
+ se.Errs[pathname.Handle()] = append(se.Errs[pathname.Handle()], err)
+ se.DanglingIdentifiers = append(se.DanglingIdentifiers, *want)
+ seMu.Unlock()
+ return false
+ } else if err = Decode(got, filepath.Base(linkname)); err != nil {
+ seMu.Lock()
+ lnp := dir.Append(linkname)
+ se.Errs[lnp.Handle()] = append(se.Errs[lnp.Handle()], err)
+ se.DanglingIdentifiers = append(se.DanglingIdentifiers, *want)
+ seMu.Unlock()
+ return false
+ }
+
+ if _, err := os.Stat(pathname.String()); err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ addErr(pathname, err)
+ }
+ seMu.Lock()
+ se.DanglingIdentifiers = append(se.DanglingIdentifiers, *want)
+ seMu.Unlock()
+ return false
+ }
+ return true
+ }}
+ }
+ wg.Wait()
+ }
+ }
+
+ dir = c.base.Append(dirStatus)
+ if entries, readdirErr := os.ReadDir(dir.String()); readdirErr != nil {
+ if !errors.Is(readdirErr, os.ErrNotExist) {
+ addErr(dir, readdirErr)
+ }
+ } else {
+ wg.Add(len(entries))
+ for _, ent := range entries {
+ w <- checkEntry{ent, func(ent os.DirEntry, want *Checksum) bool {
+ got := p.Get().(*Checksum)
+ defer p.Put(got)
+
+ var ok bool
+ for _, name := range [...]string{
+ dirIdentifier,
+ dirSubstitute,
+ } {
+ if _, err := os.Stat(c.base.Append(
+ name,
+ ent.Name(),
+ ).String()); err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ addErr(dir.Append(ent.Name()), err)
+ }
+ continue
+ }
+ ok = true
+ }
+ if !ok {
+ seMu.Lock()
+ se.DanglingStatus = append(se.DanglingStatus, *want)
+ seMu.Unlock()
+ }
+ return ok
+ }}
+ }
+ wg.Wait()
+ }
+
+ if len(c.identPending) > 0 {
+ addErr(c.base, errors.New(
+ "scrub began with pending artifacts",
+ ))
+ } else {
+ pathname := c.base.Append(dirWork)
+ chmodErr, removeErr := removeAll(pathname)
+ if chmodErr != nil {
+ addErr(pathname, chmodErr)
+ }
+ if removeErr != nil {
+ addErr(pathname, removeErr)
+ }
+
+ if err := os.Mkdir(pathname.String(), 0700); err != nil {
+ addErr(pathname, err)
+ }
+
+ pathname = c.base.Append(dirTemp)
+ chmodErr, removeErr = removeAll(pathname)
+ if chmodErr != nil {
+ addErr(pathname, chmodErr)
+ }
+ if removeErr != nil {
+ addErr(pathname, removeErr)
+ }
+ }
+
+ if len(se.ChecksumMismatches) > 0 ||
+ len(se.DanglingIdentifiers) > 0 ||
+ len(se.DanglingStatus) > 0 ||
+ len(se.Errs) > 0 {
+ slices.SortFunc(se.ChecksumMismatches, func(a, b ChecksumMismatchError) int {
+ return bytes.Compare(a.Want[:], b.Want[:])
+ })
+ slices.SortFunc(se.DanglingIdentifiers, func(a, b ID) int {
+ return bytes.Compare(a[:], b[:])
+ })
+ slices.SortFunc(se.DanglingStatus, func(a, b ID) int {
+ return bytes.Compare(a[:], b[:])
+ })
+ return &se
+ } else {
+ return nil
+ }
+}
+
+// loadOrStoreIdent attempts to load a cached [Artifact] by its identifier or
+// wait for a pending [Artifact] to cure. If neither is possible, the current
+// identifier is stored in identPending and a non-nil channel is returned.
+//
+// Since identErr is treated as grow-only, loadOrStoreIdent must not be entered
+// without holding a read lock on abortMu.
+func (c *Cache) loadOrStoreIdent(id unique.Handle[ID]) (
+ ctx context.Context,
+ done chan<- struct{},
+ checksum unique.Handle[Checksum],
+ err error,
+) {
+ var ok bool
+
+ c.identMu.Lock()
+ if checksum, ok = c.ident[id]; ok {
+ c.identMu.Unlock()
+ return
+ }
+ if err, ok = c.identErr[id]; ok {
+ c.identMu.Unlock()
+ return
+ }
+
+ var pending *pendingCure
+ if pending, ok = c.identPending[id]; ok {
+ c.identMu.Unlock()
+ <-pending.done
+ c.identMu.RLock()
+ if checksum, ok = c.ident[id]; !ok {
+ err = pending.err
+ }
+ c.identMu.RUnlock()
+ return
+ }
+
+ d := make(chan struct{})
+ pending = &pendingCure{done: d}
+ ctx, pending.cancel = context.WithCancel(c.toplevel.Load().ctx)
+ c.wg.Add(1)
+ c.identPending[id] = pending
+ c.identMu.Unlock()
+ done = d
+ return
+}
+
+// finaliseIdent commits a checksum or error to ident for an identifier
+// previously submitted to identPending.
+func (c *Cache) finaliseIdent(
+ done chan<- struct{},
+ id unique.Handle[ID],
+ checksum unique.Handle[Checksum],
+ err error,
+) {
+ c.identMu.Lock()
+ if err != nil {
+ c.identPending[id].err = err
+ c.identErr[id] = err
+ } else {
+ c.ident[id] = checksum
+ }
+ delete(c.identPending, id)
+ c.identMu.Unlock()
+ c.wg.Done()
+
+ close(done)
+}
+
+// zeroChecksum is a zero [Checksum] handle, used for comparison only.
+var zeroChecksum unique.Handle[Checksum]
+
+// loadSubstitute returns a checksum corresponding to a substitute identifier,
+// or zeroChecksum if an alternative is not available.
+func (c *Cache) loadSubstitute(
+ substitute unique.Handle[ID],
+) (unique.Handle[Checksum], error) {
+ c.substituteMu.RLock()
+ if checksum, ok := c.substitute[substitute]; ok {
+ c.substituteMu.RUnlock()
+ return checksum, nil
+ }
+
+ linkname, err := os.Readlink(c.base.Append(
+ dirSubstitute,
+ Encode(substitute.Value()),
+ ).String())
+ c.substituteMu.RUnlock()
+
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ return zeroChecksum, err
+ }
+
+ c.substituteMu.Lock()
+ c.substitute[substitute] = zeroChecksum
+ c.substituteMu.Unlock()
+ return zeroChecksum, nil
+ }
+
+ var checksum unique.Handle[Checksum]
+ buf := c.getIdentBuf()
+ err = Decode((*Checksum)(buf[:]), filepath.Base(linkname))
+ if err == nil {
+ checksum = unique.Make(Checksum(buf[:]))
+
+ c.substituteMu.Lock()
+ c.substitute[substitute] = checksum
+ c.substituteMu.Unlock()
+ }
+ c.putIdentBuf(buf)
+
+ return checksum, err
+}
+
+// Done returns a channel that is closed when the ongoing cure of an [Artifact]
+// referred to by the specified identifier completes. Done may return nil if
+// no ongoing cure of the specified identifier exists.
+func (c *Cache) Done(id unique.Handle[ID]) <-chan struct{} {
+ c.identMu.RLock()
+ pending, ok := c.identPending[id]
+ c.identMu.RUnlock()
+ if !ok || pending == nil {
+ return nil
+ }
+ return pending.done
+}
+
+// Cancel cancels the ongoing cure of an [Artifact] referred to by the specified
+// identifier. Cancel returns whether the [context.CancelFunc] has been killed.
+// Cancel returns after the cure is complete.
+func (c *Cache) Cancel(id unique.Handle[ID]) bool {
+ c.identMu.RLock()
+ pending, ok := c.identPending[id]
+ c.identMu.RUnlock()
+ if !ok || pending == nil || pending.cancel == nil {
+ return false
+ }
+ pending.cancel()
+ <-pending.done
+
+ c.abortMu.Lock()
+ c.identMu.Lock()
+ delete(c.identErr, id)
+ c.identMu.Unlock()
+ c.abortMu.Unlock()
+ return true
+}
+
+// openFile tries to load [FileArtifact] from [Cache], and if that fails,
+// obtains it via [FileArtifact.Cure] instead. Notably, it does not cure
+// [FileArtifact] to the filesystem. If err is nil, the caller is responsible
+// for closing the resulting [io.ReadCloser].
+//
+// The context must originate from loadOrStoreIdent to enable cancellation.
+func (c *Cache) openFile(
+ ctx context.Context,
+ f FileArtifact,
+) (r io.ReadCloser, err error) {
+ if kc, ok := f.(KnownChecksum); c.attr.Flags&CAssumeChecksum != 0 && ok {
+ c.checksumMu.RLock()
+ r, err = os.Open(c.base.Append(
+ dirChecksum,
+ Encode(kc.Checksum()),
+ ).String())
+ c.checksumMu.RUnlock()
+ } else {
+ c.identMu.RLock()
+ r, err = os.Open(c.base.Append(
+ dirIdentifier,
+ Encode(c.Ident(f).Value()),
+ ).String())
+ c.identMu.RUnlock()
+ }
+
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ return
+ }
+ id := c.Ident(f)
+ if c.msg.IsVerbose() {
+ rn := reportName(f, id)
+ c.msg.Verbosef("curing %s%s%s in memory...", c.sgrIdent, rn, c.sgrRes)
+ defer func() {
+ if err == nil {
+ c.msg.Verbosef("opened %s%s%s for reading", c.sgrIdent, rn, c.sgrRes)
+ }
+ }()
+ }
+ return f.Cure(&RContext{common{ctx, c}})
+ }
+ return
+}
+
+// InvalidFileModeError describes a [FloodArtifact.Cure] or
+// [TrivialArtifact.Cure] that did not result in a regular file or directory
+// located at the work pathname.
+type InvalidFileModeError fs.FileMode
+
+// Error returns a constant string.
+func (e InvalidFileModeError) Error() string {
+ return "artifact did not produce a regular file or directory"
+}
+
+// NoOutputError describes a [FloodArtifact.Cure] or [TrivialArtifact.Cure]
+// that did not populate its work pathname despite completing successfully.
+type NoOutputError struct{}
+
+// Unwrap returns [os.ErrNotExist].
+func (NoOutputError) Unwrap() error { return os.ErrNotExist }
+
+// Error returns a constant string.
+func (NoOutputError) Error() string {
+ return "artifact cured successfully but did not produce any output"
+}
+
+// removeAll is similar to [os.RemoveAll] but is robust against any permissions.
+func removeAll(pathname *check.Absolute) (chmodErr, removeErr error) {
+ chmodErr = filepath.WalkDir(pathname.String(), func(
+ path string,
+ d fs.DirEntry,
+ err error,
+ ) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ return os.Chmod(path, 0700)
+ }
+ return nil
+ })
+ if errors.Is(chmodErr, os.ErrNotExist) {
+ chmodErr = nil
+ }
+ removeErr = os.RemoveAll(pathname.String())
+ return
+}
+
+// zeroTimes zeroes atime and mtime for the named file.
+func zeroTimes(path string) (err error) {
+ // include/uapi/linux/fcntl.h
+ const (
+ AT_FDCWD = -100
+ AT_SYMLINK_NOFOLLOW = 0x100
+ )
+ _AT_FDCWD := AT_FDCWD
+
+ var _p0 *byte
+ _p0, err = syscall.BytePtrFromString(path)
+ if err != nil {
+ return
+ }
+ if _, _, errno := syscall.Syscall6(
+ syscall.SYS_UTIMENSAT,
+ uintptr(_AT_FDCWD),
+ uintptr(unsafe.Pointer(_p0)),
+ uintptr(unsafe.Pointer(new([2]syscall.Timespec))),
+ AT_SYMLINK_NOFOLLOW,
+ 0, 0,
+ ); errno != 0 {
+ return os.NewSyscallError("utimensat", errno)
+ }
+ return
+}
+
+// overrideFileInfo overrides the permission bits of [fs.FileInfo] to 0500 and
+// is the concrete type returned by overrideFile.Stat.
+type overrideFileInfo struct{ fs.FileInfo }
+
+// Mode returns [fs.FileMode] with its permission bits set to 0500.
+func (fi overrideFileInfo) Mode() fs.FileMode {
+ return fi.FileInfo.Mode()&(^fs.FileMode(0777)) | 0500
+}
+
+// Sys returns nil to avoid passing the original permission bits.
+func (fi overrideFileInfo) Sys() any { return nil }
+
+// overrideFile overrides the permission bits of [fs.File] to 0500 and is the
+// concrete type returned by dotOverrideFS for calls with "." passed as name.
+type overrideFile struct{ fs.File }
+
+func (f overrideFile) Stat() (fi fs.FileInfo, err error) {
+ fi, err = f.File.Stat()
+ if err != nil {
+ return
+ }
+ fi = overrideFileInfo{fi}
+ return
+}
+
+// dirFS is implemented by the concrete type of the return value of [os.DirFS].
+type dirFS interface {
+ fs.StatFS
+ fs.ReadFileFS
+ fs.ReadDirFS
+ fs.ReadLinkFS
+}
+
+// dotOverrideFS overrides the permission bits of "." to 0500 to avoid the extra
+// system calls to add and remove write bit from the target directory.
+type dotOverrideFS struct{ dirFS }
+
+// Open wraps the underlying [fs.FS] with "." special case.
+func (fsys dotOverrideFS) Open(name string) (f fs.File, err error) {
+ f, err = fsys.dirFS.Open(name)
+ if err != nil || name != "." {
+ return
+ }
+ f = overrideFile{f}
+ return
+}
+
+// Stat wraps the underlying [fs.FS] with "." special case.
+func (fsys dotOverrideFS) Stat(name string) (fi fs.FileInfo, err error) {
+ fi, err = fsys.dirFS.Stat(name)
+ if err != nil || name != "." {
+ return
+ }
+ fi = overrideFileInfo{fi}
+ return
+}
+
+// InvalidArtifactError describes an artifact that does not implement a
+// supported Cure method.
+type InvalidArtifactError ID
+
+func (e InvalidArtifactError) Error() string {
+ return "artifact " + Encode(e) + " cannot be cured"
+}
+
+// Cure cures the [Artifact] and returns its pathname and [Checksum]. Direct
+// calls to Cure are not subject to the cures limit.
+func (c *Cache) Cure(a Artifact) (
+ pathname *check.Absolute,
+ checksum unique.Handle[Checksum],
+ err error,
+) {
+ c.abortMu.RLock()
+ defer c.abortMu.RUnlock()
+
+ if err = c.toplevel.Load().ctx.Err(); err != nil {
+ return
+ }
+
+ pathname, checksum, _, err = c.cure(a, true, false)
+ return
+}
+
+// CureWhence is like Cure, but returns the whence value.
+func (c *Cache) CureWhence(a Artifact) (
+ pathname *check.Absolute,
+ checksum unique.Handle[Checksum],
+ whence int,
+ err error,
+) {
+ c.abortMu.RLock()
+ defer c.abortMu.RUnlock()
+
+ if err = c.toplevel.Load().ctx.Err(); err != nil {
+ return
+ }
+
+ return c.cure(a, true, false)
+}
+
+// CureNew is like Cure, but always enters the implementation.
+func (c *Cache) CureNew(a Artifact) (
+ pathname *check.Absolute,
+ checksum unique.Handle[Checksum],
+ err error,
+) {
+ c.abortMu.RLock()
+ defer c.abortMu.RUnlock()
+
+ if err = c.toplevel.Load().ctx.Err(); err != nil {
+ return
+ }
+
+ var whence int
+retry:
+ pathname, checksum, whence, err = c.cure(a, true, true)
+ if err != nil || whence == WNew {
+ return
+ }
+ goto retry
+}
+
+// An InputError describes inputs of a [FloodArtifact] which had failed to cure.
+type InputError map[Artifact]error
+
+// unwrap returns an iterator over sorted, deduplicated [Artifact] and their
+// corresponding identifier.
+func (e InputError) unwrap() iter.Seq2[Artifact, unique.Handle[ID]] {
+ ir := NewIR()
+
+ type input struct {
+ a Artifact
+ id unique.Handle[ID]
+ }
+ p := make([]input, 0, len(e))
+ for a := range e {
+ p = append(p, input{a, ir.Ident(a)})
+ }
+
+ var identBuf [2]ID
+ slices.SortFunc(p, func(a, b input) int {
+ identBuf[0], identBuf[1] = a.id.Value(), b.id.Value()
+ return slices.Compare(identBuf[0][:], identBuf[1][:])
+ })
+ p = slices.CompactFunc(p, func(a, b input) bool { return a.id == b.id })
+
+ return func(yield func(Artifact, unique.Handle[ID]) bool) {
+ for _, i := range p {
+ if !yield(i.a, i.id) {
+ return
+ }
+ }
+ }
+}
+
+// Error returns a user-facing, deterministic text representation of e.
+func (e InputError) Error() string {
+ var buf strings.Builder
+ buf.WriteString("errors curing inputs:")
+ for a, id := range e.unwrap() {
+ buf.WriteString("\n\t")
+ buf.WriteString(reportName(a, id))
+ buf.WriteString(": ")
+ buf.WriteString(e[a].Error())
+ }
+ return buf.String()
+}
+
+// Unwrap returns a slice of underlying errors sorted by identifier.
+func (e InputError) Unwrap() []error {
+ errs := make([]error, 0, len(e))
+ for a := range e.unwrap() {
+ errs = append(errs, e[a])
+ }
+ return errs
+}
+
+// enterCure must be called before entering an [Artifact] implementation.
+func (c *Cache) enterCure(a Artifact, curesExempt bool) error {
+ if c.attr.Notify != nil {
+ c.attr.Notify <- true
+ }
+
+ if a.IsExclusive() {
+ c.exclMu.Lock()
+ }
+ if curesExempt {
+ return nil
+ }
+
+ ctx := c.toplevel.Load().ctx
+ select {
+ case c.cures <- struct{}{}:
+ return nil
+
+ case <-ctx.Done():
+ if a.IsExclusive() {
+ c.exclMu.Unlock()
+ }
+ return ctx.Err()
+ }
+}
+
+// exitCure must be called after exiting an [Artifact] implementation.
+func (c *Cache) exitCure(a Artifact, curesExempt bool) {
+ if c.attr.Notify != nil {
+ c.attr.Notify <- false
+ }
+
+ if a.IsExclusive() {
+ c.exclMu.Unlock()
+ }
+ if curesExempt {
+ return
+ }
+
+ <-c.cures
+}
+
+// measuredReader implements [io.ReadCloser] and measures the checksum during
+// Close. If the underlying reader is not read to EOF, Close blocks until all
+// remaining data is consumed and validated.
+type measuredReader struct {
+ // Underlying reader. Never exposed directly.
+ r io.ReadCloser
+ // For validating checksum. Never exposed directly.
+ h hash.Hash
+ // Buffers writes to h, initialised by [Cache]. Never exposed directly.
+ hbw *bufio.Writer
+ // Expected checksum, compared during Close.
+ want unique.Handle[Checksum]
+
+ // For accessing free lists.
+ c *Cache
+
+ // Set up via [io.TeeReader] by [Cache].
+ io.Reader
+}
+
+// Close reads the underlying [io.ReadCloser] to EOF, closes it and measures its
+// outcome. It returns a [ChecksumMismatchError] for an unexpected checksum.
+func (mr *measuredReader) Close() (err error) {
+ if mr.hbw == nil || mr.Reader == nil {
+ return os.ErrInvalid
+ }
+ err = mr.hbw.Flush()
+ mr.c.putWriter(mr.hbw)
+ mr.hbw, mr.Reader = nil, nil
+ if err != nil {
+ _ = mr.r.Close()
+ return
+ }
+ var n int64
+ if n, err = io.Copy(mr.h, mr.r); err != nil {
+ _ = mr.r.Close()
+ return
+ }
+
+ if n > 0 {
+ mr.c.msg.Verbosef(
+ "%smissed %d bytes on measured reader%s",
+ mr.c.sgrWarn, n, mr.c.sgrRes,
+ )
+ }
+
+ if err = mr.r.Close(); err != nil {
+ return
+ }
+
+ buf := mr.c.getIdentBuf()
+ mr.h.Sum(buf[:0])
+
+ if got := Checksum(buf[:]); got != mr.want.Value() {
+ err = &ChecksumMismatchError{
+ Got: got,
+ Want: mr.want.Value(),
+ }
+ }
+
+ mr.c.putIdentBuf(buf)
+ return
+}
+
+// newMeasuredReader implements [RContext.NewMeasuredReader].
+func (c *Cache) newMeasuredReader(
+ r io.ReadCloser,
+ checksum unique.Handle[Checksum],
+) io.ReadCloser {
+ mr := measuredReader{r: r, h: sha512.New384(), want: checksum, c: c}
+ mr.hbw = c.getWriter(mr.h)
+ mr.Reader = io.TeeReader(r, mr.hbw)
+ return &mr
+}
+
+// NewMeasuredReader returns an [io.ReadCloser] implementing behaviour required
+// by [FileArtifact]. The resulting [io.ReadCloser] holds a buffer originating
+// from [Cache] and must be closed to return this buffer.
+func (r *RContext) NewMeasuredReader(
+ rc io.ReadCloser,
+ checksum unique.Handle[Checksum],
+) io.ReadCloser {
+ return r.cache.newMeasuredReader(rc, checksum)
+}
+
+// tryChecksum dereferences a symlink to a cure outcome.
+func (c *Cache) tryChecksum(pathname *check.Absolute) (
+ checksum unique.Handle[Checksum],
+ err error,
+) {
+ _, err = os.Lstat(pathname.String())
+ if err == nil {
+ var name string
+ if name, err = os.Readlink(pathname.String()); err != nil {
+ return
+ }
+ buf := c.getIdentBuf()
+ err = Decode((*Checksum)(buf[:]), filepath.Base(name))
+ if err == nil {
+ checksum = unique.Make(Checksum(buf[:]))
+ }
+ c.putIdentBuf(buf)
+ }
+ return
+}
+
+// tryLocal attempts to obtain an [Artifact] outcome from the filesystem.
+func (c *Cache) tryLocal(id unique.Handle[ID]) (unique.Handle[Checksum], error) {
+ return c.tryChecksum(c.base.Append(
+ dirIdentifier,
+ Encode(id.Value()),
+ ))
+}
+
+// tryExtern attempts to obtain an [Artifact] outcome from extern.
+func (c *Cache) tryExtern(ctx context.Context, id unique.Handle[ID]) (
+ unique.Handle[Checksum],
+ error,
+) {
+ c.externMu.RLock()
+ defer c.externMu.RUnlock()
+
+ checksum, ok := c.externCache[id]
+ if !ok {
+ if c.extern == nil {
+ return zeroChecksum, nil
+ }
+
+ v, err := c.extern.Artifact(ctx, id)
+ if err != nil {
+ return zeroChecksum, err
+ }
+ if v == nil {
+ return zeroChecksum, nil
+ }
+ checksum = unique.Make(*v)
+
+ var got unique.Handle[Checksum]
+ if _, got, err = c.Cure(c.extern.Checksum(checksum)); err != nil {
+ return checksum, err
+ } else if got != checksum {
+ return zeroChecksum, &ChecksumMismatchError{got.Value(), checksum.Value()}
+ }
+ }
+ return checksum, nil
+}
+
+// cureMany concurrently collects outcome of multiple [Artifact].
+func (c *Cache) cureMany(
+ inputs []Artifact,
+ r map[Artifact]cureRes,
+ shallow bool,
+) ([]bool, error) {
+ var wg sync.WaitGroup
+ wg.Add(len(inputs))
+ var mask []bool
+ res := make([]cureRes, len(inputs))
+ errs := make(InputError)
+ var errsMu sync.Mutex
+ if shallow {
+ mask = make([]bool, len(inputs))
+ }
+ for i, d := range inputs {
+ if shallow {
+ if _, ok := d.(FloodArtifact); ok {
+ mask[i] = true
+ wg.Done()
+ continue
+ }
+
+ if kc, ok := d.(KnownChecksum); ok {
+ res[i].checksum = unique.Make(kc.Checksum())
+ wg.Done()
+ continue
+ }
+ }
+ pending := pendingArtifactDep{d, &res[i], errs, &errsMu, &wg}
+ go pending.cure(c)
+ }
+ wg.Wait()
+
+ if len(errs) > 0 {
+ return mask, errs
+ }
+ for i, p := range res {
+ if shallow && mask[i] {
+ continue
+ }
+ r[inputs[i]] = p
+ }
+ return mask, nil
+}
+
+// HangingInputError describes an input of an [Artifact] on a sparse cache
+// without an outcome available locally or via [External].
+type HangingInputError unique.Handle[ID]
+
+func (e HangingInputError) Error() string {
+ return Encode(unique.Handle[ID](e).Value()) + " is unavailable"
+}
+
+const (
+ // WNew indicates a cure entering the implementation.
+ WNew = iota
+ // WCache indicates an [Artifact] present in the cache.
+ WCache
+ // WSubstitute indicates an [Artifact] hitting a content-based input
+ // substitution.
+ WSubstitute
+ // WExternal indicates a cure entering the external cache.
+ WExternal
+)
+
+// WhenceString returns a printable string for a whence value.
+func WhenceString(whence int) string {
+ switch whence {
+ case WNew:
+ return "new"
+ case WCache:
+ return "cache"
+ case WSubstitute:
+ return "substitute"
+ case WExternal:
+ return "external"
+
+ default:
+ return "invalid whence " + strconv.Itoa(whence)
+ }
+}
+
+// cure implements Cure without acquiring a read lock on abortMu. cure must not
+// be entered during Abort.
+func (c *Cache) cure(a Artifact, curesExempt, rebuild bool) (
+ pathname *check.Absolute,
+ checksum unique.Handle[Checksum],
+ whence int,
+ err error,
+) {
+ id := c.Ident(a)
+ if rebuild {
+ var v ID
+ _, _ = rand.Read(v[:])
+ id = unique.Make(v)
+ }
+
+ ids := Encode(id.Value())
+ pathname = c.base.Append(
+ dirIdentifier,
+ ids,
+ )
+ defer func() {
+ if err != nil {
+ pathname = nil
+ checksum = unique.Handle[Checksum]{}
+ }
+ }()
+
+ if _, ok := a.(CuresExempt); ok {
+ curesExempt = true
+ }
+
+ var (
+ ctx context.Context
+ done chan<- struct{}
+ )
+ ctx, done, checksum, err = c.loadOrStoreIdent(id)
+ if done == nil {
+ whence = WCache
+ return
+ } else {
+ defer func() { c.finaliseIdent(done, id, checksum, err) }()
+ }
+
+ checksum, err = c.tryChecksum(pathname)
+ if err == nil || !errors.Is(err, os.ErrNotExist) {
+ whence = WCache
+ return
+ }
+
+ var (
+ checksums string
+ substitute unique.Handle[ID]
+ alternative *check.Absolute
+ )
+ defer func() {
+ if err == nil && checksums != "" {
+ linkname := checksumLinknamePrefix + checksums
+
+ err = os.Symlink(
+ linkname,
+ pathname.String(),
+ )
+ if err == nil {
+ err = zeroTimes(pathname.String())
+ }
+
+ if err == nil && alternative != nil && substitute != id {
+ c.substituteMu.Lock()
+ err = os.Symlink(
+ linkname,
+ alternative.String(),
+ )
+ if errors.Is(err, os.ErrExist) {
+ c.msg.Verbosef(
+ "creating alternative over %s%s%s for artifact %s%s%s",
+ c.sgrIdent, Encode(substitute.Value()), c.sgrRes,
+ c.sgrIdent, ids, c.sgrRes,
+ )
+ err = nil
+ }
+ if err == nil {
+ err = zeroTimes(alternative.String())
+ }
+ if err == nil && checksum != zeroChecksum {
+ c.substitute[substitute] = checksum
+ }
+ c.substituteMu.Unlock()
+ }
+ }
+ }()
+
+ var checksumPathname *check.Absolute
+ var checksumFi os.FileInfo
+ if kc, ok := a.(KnownChecksum); ok {
+ checksum = unique.Make(kc.Checksum())
+ checksums = Encode(checksum.Value())
+ checksumPathname = c.base.Append(
+ dirChecksum,
+ checksums,
+ )
+
+ if c.attr.Flags&CAssumeChecksum != 0 {
+ c.checksumMu.RLock()
+ checksumFi, err = os.Stat(checksumPathname.String())
+ c.checksumMu.RUnlock()
+
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ return
+ }
+
+ checksumFi, err = nil, nil
+ }
+ }
+ }
+
+ whence = WNew
+ if c.msg.IsVerbose() {
+ rn := reportName(a, id)
+ c.msg.Verbosef("curing %s%s%s...", c.sgrIdent, rn, c.sgrRes)
+ defer func() {
+ if err != nil {
+ return
+ }
+ if checksums != "" {
+ c.msg.Verbosef(
+ "cured %s%s%s checksum %s%s%s",
+ c.sgrIdent, rn, c.sgrRes,
+ c.sgrIdent, checksums, c.sgrRes)
+ } else {
+ c.msg.Verbosef("cured %s%s%s", c.sgrIdent, rn, c.sgrRes)
+ }
+ }()
+ }
+
+ // cure FileArtifact outside type switch to skip TContext initialisation
+ if f, ok := a.(FileArtifact); ok {
+ if checksumFi != nil {
+ whence = WCache
+ if !checksumFi.Mode().IsRegular() {
+ // unreachable
+ err = InvalidFileModeError(checksumFi.Mode())
+ }
+ return
+ }
+
+ perm := os.FileMode(0400)
+ if f.IsExecutable() {
+ perm = 0500
+ }
+
+ work := c.base.Append(dirWork, ids)
+ var w *os.File
+ if w, err = os.OpenFile(
+ work.String(),
+ os.O_CREATE|os.O_EXCL|os.O_WRONLY,
+ perm,
+ ); err != nil {
+ return
+ }
+ defer func() {
+ closeErr := w.Close()
+ if err == nil {
+ err = closeErr
+ }
+
+ removeErr := os.Remove(work.String())
+ if err == nil && !errors.Is(removeErr, os.ErrNotExist) {
+ err = removeErr
+ }
+ }()
+
+ var r io.ReadCloser
+ if err = c.enterCure(a, curesExempt); err != nil {
+ return
+ }
+ r, err = f.Cure(&RContext{common{ctx, c}})
+ if err == nil {
+ if checksumPathname == nil || c.attr.Flags&CValidateKnown != 0 {
+ h := sha512.New384()
+ hbw := c.getWriter(h)
+ _, err = io.Copy(w, io.TeeReader(r, hbw))
+ flushErr := hbw.Flush()
+ c.putWriter(hbw)
+ if err == nil {
+ err = flushErr
+ }
+
+ if err == nil {
+ buf := c.getIdentBuf()
+ h.Sum(buf[:0])
+
+ if checksumPathname == nil {
+ checksum = unique.Make(Checksum(buf[:]))
+ checksums = Encode(Checksum(buf[:]))
+ } else if c.attr.Flags&CValidateKnown != 0 {
+ if got := Checksum(buf[:]); got != checksum.Value() {
+ err = &ChecksumMismatchError{
+ Got: got,
+ Want: checksum.Value(),
+ }
+ }
+ }
+
+ c.putIdentBuf(buf)
+
+ if checksumPathname == nil {
+ checksumPathname = c.base.Append(
+ dirChecksum,
+ checksums,
+ )
+ }
+ }
+ } else {
+ _, err = io.Copy(w, r)
+ }
+
+ closeErr := r.Close()
+ if err == nil {
+ err = closeErr
+ }
+ }
+ c.exitCure(a, curesExempt)
+ if err != nil {
+ if c.msg.IsVerbose() {
+ c.msg.Verbosef(
+ "cure file %s%s%s: %s%v%s",
+ c.sgrIdent, reportName(f, id), c.sgrRes,
+ c.sgrErr, err, c.sgrRes,
+ )
+ }
+ return
+ }
+
+ c.checksumMu.Lock()
+ if err = os.Rename(
+ work.String(),
+ checksumPathname.String(),
+ ); err != nil {
+ c.checksumMu.Unlock()
+ return
+ }
+ timeErr := zeroTimes(checksumPathname.String())
+ c.checksumMu.Unlock()
+
+ if err == nil {
+ err = timeErr
+ }
+ return
+ }
+
+ if checksumFi != nil {
+ whence = WCache
+ if !checksumFi.Mode().IsDir() {
+ // unreachable
+ err = InvalidFileModeError(checksumFi.Mode())
+ }
+ return
+ }
+
+ t := TContext{
+ c.base.Append(dirWork, ids),
+ c.base.Append(dirTemp, ids),
+ ids, nil, nil, nil, nil,
+ common{ctx, c},
+ }
+ switch ca := a.(type) {
+ case TrivialArtifact:
+ defer t.destroy(&err)
+ if err = c.enterCure(a, curesExempt); err != nil {
+ return
+ }
+ err = ca.Cure(&t)
+ c.exitCure(a, curesExempt)
+ if err != nil {
+ if c.msg.IsVerbose() {
+ c.msg.Verbosef(
+ "cure trivial %s%s%s: %s%v%s",
+ c.sgrIdent, reportName(ca, id), c.sgrRes,
+ c.sgrErr, err, c.sgrRes,
+ )
+ }
+ return
+ }
+ break
+
+ case FloodArtifact:
+ var externChecksum unique.Handle[Checksum]
+ if externChecksum, err = c.tryExtern(ctx, id); err != nil {
+ if c.msg.IsVerbose() {
+ c.msg.Verbosef(
+ "extern %s%s%s: %s%v%s",
+ c.sgrIdent, reportName(ca, id), c.sgrRes,
+ c.sgrErr, err, c.sgrRes,
+ )
+ }
+ return
+ }
+ extern := externChecksum != zeroChecksum
+ shallow := extern && c.attr.Flags&CExternShallow != 0
+
+ inputs := a.Inputs()
+ f := FContext{t, make(map[Artifact]cureRes, len(inputs))}
+ var mask []bool
+ if mask, err = c.cureMany(inputs, f.inputs, shallow); err != nil {
+ return
+ }
+
+ if shallow {
+ for i, d := range inputs {
+ if !mask[i] {
+ continue
+ }
+
+ if kc, ok := d.(KnownChecksum); ok {
+ f.inputs[d] = cureRes{checksum: unique.Make(kc.Checksum())}
+ continue
+ }
+
+ var sum unique.Handle[Checksum]
+ did := c.Ident(d)
+ sum, err = c.tryLocal(did)
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ return
+ }
+
+ sum, err = c.tryExtern(ctx, did)
+ if err != nil {
+ return
+ }
+ if sum == zeroChecksum {
+ if c.msg.IsVerbose() {
+ c.msg.Verbosef(
+ "input %s%s%s not available",
+ c.sgrIdent, reportName(d, did), c.sgrRes,
+ )
+ }
+ err = HangingInputError(did)
+ return
+ }
+ }
+
+ f.inputs[d] = cureRes{checksum: sum}
+ }
+ }
+
+ sh := sha512.New384()
+ err = c.encode(sh, a, f.inputs)
+ if err != nil {
+ return
+ }
+
+ buf := c.getIdentBuf()
+ sh.Sum(buf[wordSize:wordSize])
+ substitute = unique.Make(ID(buf[wordSize:]))
+ substitutes := Encode(substitute.Value())
+ c.putIdentBuf(buf)
+ alternative = c.base.Append(
+ dirSubstitute,
+ substitutes,
+ )
+
+ if !rebuild && c.attr.Flags&CIgnoreSubstitutes == 0 {
+ var substituteChecksum unique.Handle[Checksum]
+ substituteChecksum, err = c.loadSubstitute(substitute)
+ if err != nil {
+ return
+ }
+ if substituteChecksum != zeroChecksum {
+ whence = WSubstitute
+ checksum = substituteChecksum
+ checksums = Encode(checksum.Value())
+ checksumPathname = c.base.Append(
+ dirChecksum,
+ checksums,
+ )
+ if _, err = os.Lstat(c.base.Append(
+ dirStatus,
+ substitutes,
+ ).String()); err == nil {
+ err = os.Symlink(substitutes, c.base.Append(
+ dirStatus,
+ ids,
+ ).String())
+ } else if errors.Is(err, os.ErrNotExist) {
+ err = nil
+ }
+ return
+ }
+ }
+
+ defer f.destroy(&err)
+
+ if extern {
+ whence = WExternal
+ if checksum != zeroChecksum && externChecksum != checksum {
+ err = &ChecksumMismatchError{externChecksum.Value(), checksum.Value()}
+ if c.msg.IsVerbose() {
+ c.msg.Verbosef(
+ "extern %s%s%s: %s%v%s",
+ c.sgrIdent, reportName(ca, id), c.sgrRes,
+ c.sgrErr, err, c.sgrRes,
+ )
+ }
+ return
+ }
+
+ var externStatus io.ReadCloser
+ c.externMu.RLock()
+ externStatus, err = c.extern.Status(&RContext{common{ctx, c}}, id)
+ c.externMu.RUnlock()
+ if err != nil {
+ return
+ }
+
+ checksum = externChecksum
+ checksums = Encode(checksum.Value())
+ checksumPathname = c.base.Append(
+ dirChecksum,
+ checksums,
+ )
+
+ if externStatus != nil {
+ if err = f.prepareStatus(false); err != nil {
+ _ = externStatus.Close()
+ return
+ } else if _, err = io.Copy(f.status, externStatus); err != nil {
+ _ = externStatus.Close()
+ return
+ } else if err = externStatus.Close(); err != nil {
+ return
+ } else if !rebuild {
+ if err = f.linkSubstitute(ids, substitutes); err != nil {
+ return
+ }
+ }
+ }
+ return
+ }
+
+ if err = c.enterCure(a, curesExempt); err != nil {
+ return
+ }
+ err = ca.Cure(&f)
+ c.exitCure(a, curesExempt)
+
+ if !rebuild && err == nil {
+ err = f.linkSubstitute(ids, substitutes)
+ }
+ if err != nil {
+ if c.msg.IsVerbose() {
+ c.msg.Verbosef(
+ "cure %s%s%s: %s%v%s",
+ c.sgrIdent, reportName(ca, id), c.sgrRes,
+ c.sgrErr, err, c.sgrRes,
+ )
+ }
+ return
+ }
+ break
+
+ default:
+ err = InvalidArtifactError(id.Value())
+ return
+ }
+ t.cache = nil
+
+ var fi os.FileInfo
+ if fi, err = os.Lstat(t.work.String()); err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ err = NoOutputError{}
+ }
+ return
+ }
+
+ if !fi.IsDir() {
+ if !fi.Mode().IsRegular() {
+ err = InvalidFileModeError(fi.Mode())
+ } else {
+ err = errors.New("non-file artifact produced regular file")
+ }
+ return
+ }
+
+ var gotChecksum Checksum
+ if err = SumFS(
+ &gotChecksum,
+ dotOverrideFS{os.DirFS(t.work.String()).(dirFS)},
+ ".",
+ ); err != nil {
+ return
+ }
+
+ if checksumPathname == nil {
+ checksum = unique.Make(gotChecksum)
+ checksums = Encode(gotChecksum)
+ checksumPathname = c.base.Append(
+ dirChecksum,
+ checksums,
+ )
+ } else if gotChecksum != checksum.Value() {
+ err = &ChecksumMismatchError{
+ Got: gotChecksum,
+ Want: checksum.Value(),
+ }
+ if c.msg.IsVerbose() {
+ c.msg.Verbosef(
+ "validate %s%s%s: %s%v%s",
+ c.sgrIdent, reportName(a, id), c.sgrRes,
+ c.sgrErr, err, c.sgrRes,
+ )
+ }
+ return
+ }
+
+ if err = os.Chmod(t.work.String(), 0700); err != nil {
+ return
+ }
+ if err = filepath.WalkDir(t.work.String(), func(path string, _ fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ return zeroTimes(path)
+ }); err != nil {
+ return
+ }
+ c.checksumMu.Lock()
+ if err = os.Rename(
+ t.work.String(),
+ checksumPathname.String(),
+ ); err != nil {
+ if !errors.Is(err, os.ErrExist) {
+ c.checksumMu.Unlock()
+ return
+ }
+ // err is zeroed during deferred cleanup
+ } else {
+ err = os.Chmod(checksumPathname.String(), 0500)
+ }
+ c.checksumMu.Unlock()
+ return
+}
+
+// cure cures the pending [Artifact], stores its result and notifies the caller.
+func (pending *pendingArtifactDep) cure(c *Cache) {
+ defer pending.Done()
+
+ var err error
+ pending.resP.pathname, pending.resP.checksum, _, err = c.cure(pending.a, false, false)
+ if err == nil {
+ return
+ }
+
+ pending.errsMu.Lock()
+ if errs, ok := err.(InputError); ok {
+ maps.Copy(pending.errs, errs)
+ } else {
+ pending.errs[pending.a] = err
+ }
+ pending.errsMu.Unlock()
+}
+
+// OpenStatus attempts to open the status file associated to an [Artifact]. If
+// err is nil, the caller must close the resulting reader.
+func (c *Cache) OpenStatus(a Artifact) (r io.ReadSeekCloser, err error) {
+ c.identMu.RLock()
+ r, err = os.Open(c.base.Append(
+ dirStatus,
+ Encode(c.Ident(a).Value())).String(),
+ )
+ c.identMu.RUnlock()
+ return
+}
+
+// Fault holds the pathname and termination time of an [Artifact] fault entry.
+type Fault struct {
+ *check.Absolute
+ t uint64
+}
+
+// Time returns the instant in time where the fault occurred.
+func (f Fault) Time() time.Time { return time.Unix(0, int64(f.t)) }
+
+// Open opens the underlying entry for reading.
+func (f Fault) Open() (io.ReadCloser, error) { return os.Open(f.Absolute.String()) }
+
+// Destroy removes the underlying fault entry.
+func (f Fault) Destroy() error { return os.Remove(f.Absolute.String()) }
+
+// ReadFaults returns fault entries for an [Artifact].
+func (c *Cache) ReadFaults(a Artifact) (faults []Fault, err error) {
+ prefix := Encode(c.Ident(a).Value()) + "."
+ var dents []os.DirEntry
+ if dents, err = os.ReadDir(c.base.Append(dirFault).String()); err != nil {
+ return
+ }
+
+ for _, dent := range dents {
+ name := dent.Name()
+ if !strings.HasPrefix(name, prefix) {
+ continue
+ }
+ var t uint64
+ t, err = strconv.ParseUint(name[len(prefix):], 10, 64)
+ if err != nil {
+ return
+ }
+
+ faults = append(faults, Fault{c.base.Append(
+ dirFault,
+ name,
+ ), t})
+ }
+
+ slices.SortFunc(faults, func(a, b Fault) int {
+ return cmp.Compare(a.t, b.t)
+ })
+ return
+}
+
+// Abort cancels all pending cures and waits for them to clean up, but does not
+// close the cache.
+func (c *Cache) Abort() {
+ c.closeMu.Lock()
+ defer c.closeMu.Unlock()
+
+ if c.closed {
+ return
+ }
+
+ c.toplevel.Load().cancel()
+ c.abortMu.Lock()
+ defer c.abortMu.Unlock()
+
+ // holding abortMu, identPending stays empty
+ c.wg.Wait()
+ c.identMu.Lock()
+ c.toplevel.Store(newToplevel(c.parent))
+ clear(c.identErr)
+ c.identMu.Unlock()
+}
+
+// Close cancels all pending cures and waits for them to clean up.
+func (c *Cache) Close() {
+ c.closeMu.Lock()
+ defer c.closeMu.Unlock()
+
+ if c.closed {
+ return
+ }
+
+ c.closed = true
+ c.toplevel.Load().cancel()
+ c.wg.Wait()
+ close(c.cures)
+ c.unlock()
+
+ if c.attr.Notify != nil {
+ close(c.attr.Notify)
+ }
+}
+
+// UnsupportedVariantError describes an on-disk cache with an extension variant
+// identification string that differs from the value returned by [Extension].
+type UnsupportedVariantError string
+
+func (e UnsupportedVariantError) Error() string {
+ return "unsupported variant " + strconv.Quote(string(e))
+}
+
+var (
+ // ErrWouldPromote is returned by [Open] if the [CPromoteVariant] bit is not
+ // set and the on-disk cache requires variant promotion.
+ ErrWouldPromote = errors.New("operation would promote unextended cache")
+)
+
+// CacheAttr holds the attributes that will be applied to a new [Cache] opened
+// by [Open].
+type CacheAttr struct {
+ // Concurrent cures of a [FloodArtifact] dependency graph.
+ Cures int
+ // Options affecting [Cache] behaviour.
+ Flags int
+ // Preferred job count, when applicable.
+ Jobs int
+ // Preferred loadavg target, when applicable.
+ Load int
+ // Optional cure entry and exit notification.
+ Notify chan<- bool
+
+ // Omit the [lockedfile] lock.
+ skipLock bool
+}
+
+// Open returns the address of a newly opened instance of [Cache].
+//
+// Concurrent cures of a [FloodArtifact] dependency graph is limited to the
+// caller-supplied value, however direct calls to [Cache.Cure] is not subject
+// to this limitation.
+//
+// A cures or jobs value of 0 or lower is equivalent to the value returned by
+// [runtime.NumCPU].
+//
+// A successful call to Open guarantees exclusive access to the on-filesystem
+// cache for the resulting instance of [Cache]. The [Cache.Close] method cancels
+// and waits for pending cures on [Cache] before releasing this lock and must be
+// called once the [Cache] is no longer needed.
+func Open(
+ ctx context.Context,
+ msg message.Msg,
+ base *check.Absolute,
+ attr *CacheAttr,
+) (*Cache, error) {
+ openMu.Lock()
+ defer openMu.Unlock()
+ opened = true
+
+ if extension == "" && len(irArtifact) != int(_kindEnd) {
+ panic("attempting to open cache with incomplete variant setup")
+ }
+
+ var a CacheAttr
+ if attr != nil {
+ a = *attr
+ }
+
+ if a.Cures < 1 {
+ a.Cures = runtime.NumCPU()
+ }
+ if a.Jobs < 1 {
+ a.Jobs = runtime.NumCPU()
+ }
+ if a.Load < 1 {
+ a.Load = runtime.NumCPU() + 2
+ }
+
+ for _, name := range []string{
+ dirSubstitute,
+ dirIdentifier,
+ dirChecksum,
+ dirStatus,
+ dirFault,
+ dirWork,
+ } {
+ if err := os.MkdirAll(
+ base.Append(name).String(),
+ 0700,
+ ); err != nil && !errors.Is(err, os.ErrExist) {
+ return nil, err
+ }
+ }
+
+ c := Cache{
+ parent: ctx,
+
+ cures: make(chan struct{}, a.Cures),
+ attr: a,
+
+ msg: msg,
+ base: base,
+
+ irCache: zeroIRCache(),
+
+ substitute: make(map[unique.Handle[ID]]unique.Handle[Checksum]),
+ ident: make(map[unique.Handle[ID]]unique.Handle[Checksum]),
+ identErr: make(map[unique.Handle[ID]]error),
+ identPending: make(map[unique.Handle[ID]]*pendingCure),
+
+ brPool: sync.Pool{New: func() any { return new(bufio.Reader) }},
+ bwPool: sync.Pool{New: func() any { return new(bufio.Writer) }},
+ }
+ c.toplevel.Store(newToplevel(ctx))
+
+ if !a.skipLock || !testing.Testing() {
+ if unlock, err := lockedfile.MutexAt(
+ base.Append(fileLock).String(),
+ ).Lock(); err != nil {
+ return nil, err
+ } else {
+ c.unlock = unlock
+ }
+ } else {
+ c.unlock = func() {}
+ }
+
+ for _, name := range []string{
+ dirWork,
+ dirTemp,
+ } {
+ dents, err := os.ReadDir(base.Append(name).String())
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ continue
+ }
+ c.unlock()
+ return nil, err
+ }
+ if len(dents) != 0 {
+ c.unlock()
+ return nil, fmt.Errorf(
+ "%s is not empty, scrub likely required",
+ name,
+ )
+ }
+ }
+
+ if _, err := os.ReadDir(base.Append(
+ dirExecScratch,
+ ).String()); !errors.Is(err, os.ErrNotExist) {
+ c.unlock()
+ if err != nil {
+ return nil, err
+ }
+ return nil, errors.New(dirExecScratch + " is present, scrub likely required")
+ }
+
+ variantPath := base.Append(fileVariant).String()
+ if p, err := os.ReadFile(variantPath); err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ c.unlock()
+ return nil, err
+ }
+ // nonexistence implies newly created cache, or a cache predating
+ // variant identification strings, in which case it is silently promoted
+ if err = os.WriteFile(
+ variantPath,
+ []byte(extension),
+ 0400,
+ ); err != nil {
+ c.unlock()
+ return nil, err
+ }
+ } else if s := string(p); s == "" {
+ if extension != "" {
+ if a.Flags&CPromoteVariant == 0 {
+ c.unlock()
+ return nil, ErrWouldPromote
+ }
+ if err = os.WriteFile(
+ variantPath,
+ []byte(extension),
+ 0400,
+ ); err != nil {
+ c.unlock()
+ return nil, err
+ }
+ }
+ } else if !ValidExtension(s) {
+ c.unlock()
+ return nil, ErrInvalidExtension
+ } else if s != extension {
+ c.unlock()
+ return nil, UnsupportedVariantError(s)
+ }
+
+ if a.Flags&CColourOutput != 0 {
+ c.sgrRes = "\x1b[0m"
+ c.sgrIdent = "\x1b[1m"
+ c.sgrWarn = "\x1b[35m"
+ c.sgrErr = "\x1b[1;31m"
+ }
+
+ return &c, nil
+}
+
+// Collected is returned by [Collect.Cure] to indicate a successful collection.
+type Collected struct{}
+
+// Error returns a constant string to satisfy error, but should never be seen
+// by the user.
+func (Collected) Error() string { return "artifacts successfully collected" }
+
+// IsCollected returns whether the underlying error contains that of the result
+// of curing a [Collect] helper.
+func IsCollected(err error) bool { return errors.As(err, new(Collected)) }
+
+// Collect implements [pkg.FloodArtifact] to concurrently cure multiple
+// [pkg.Artifact]. It returns [Collected].
+type Collect []Artifact
+
+var _ Artifact = new(Collect)
+
+// Cure returns [Collected].
+func (*Collect) Cure(*FContext) error { return Collected{} }
+
+// Kind returns the hardcoded [pkg.Kind] value.
+func (*Collect) Kind() Kind { return kindCollection }
+
+// Params is a noop: dependencies are already represented in the header.
+func (*Collect) Params(*IContext) {}
+
+// Inputs returns [Collect] as is.
+func (c *Collect) Inputs() []Artifact { return *c }
+
+// IsExclusive returns false: Cure is a noop.
+func (*Collect) IsExclusive() bool { return false }
diff --git a/pkg/pkg_test.go b/pkg/pkg_test.go
new file mode 100644
index 00000000..c11b6ac4
--- /dev/null
+++ b/pkg/pkg_test.go
@@ -0,0 +1,2375 @@
+package pkg_test
+
+import (
+ "archive/tar"
+ "bytes"
+ "context"
+ "crypto/sha512"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "log"
+ "net/http"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strconv"
+ "strings"
+ "sync"
+ "syscall"
+ "testing"
+ "testing/fstest"
+ "time"
+ "unique"
+ "unsafe"
+
+ "hakurei.app/check"
+ "hakurei.app/container"
+ "hakurei.app/fhs"
+ "hakurei.app/internal/info"
+ "hakurei.app/internal/landlock"
+ "hakurei.app/internal/stub"
+ "hakurei.app/message"
+ "hakurei.app/pkg"
+)
+
+var skipLock = func() pkg.CacheAttr {
+ var attr pkg.CacheAttr
+ *(*bool)(unsafe.Pointer(reflect.ValueOf(&attr).
+ Elem().
+ FieldByName("skipLock").
+ UnsafeAddr())) = true
+ return attr
+}()
+
+var (
+ // extension is a string uniquely identifying a set of custom [Artifact]
+ // implementations registered by calling [Register].
+ //
+ //go:linkname extension hakurei.app/pkg.extension
+ extension string
+
+ // opened is false if [Open] was never called.
+ //
+ //go:linkname opened hakurei.app/pkg.opened
+ opened bool
+
+ // irArtifact refers to artifact IR interpretation functions and must not be
+ // written to directly.
+ //
+ //go:linkname irArtifact hakurei.app/pkg.irArtifact
+ irArtifact map[pkg.Kind]pkg.IRReadFunc
+
+ // statusHeader is the header written to all status files in dirStatus.
+ //
+ //go:linkname statusHeader hakurei.app/pkg.statusHeader
+ statusHeader string
+)
+
+// newRContext returns the address of a new [pkg.RContext] unsafely created for
+// the specified [testing.TB].
+func newRContext(tb testing.TB, c *pkg.Cache) *pkg.RContext {
+ var r pkg.RContext
+ rContextVal := reflect.ValueOf(&r).Elem().FieldByName("ctx")
+ reflect.NewAt(
+ rContextVal.Type(),
+ unsafe.Pointer(rContextVal.UnsafeAddr()),
+ ).Elem().Set(reflect.ValueOf(tb.Context()))
+ rCacheVal := reflect.ValueOf(&r).Elem().FieldByName("cache")
+ reflect.NewAt(
+ rCacheVal.Type(),
+ unsafe.Pointer(rCacheVal.UnsafeAddr()),
+ ).Elem().Set(reflect.ValueOf(c))
+ return &r
+}
+
+// overrideIdent overrides the ID method of [Artifact].
+type overrideIdent struct {
+ id pkg.ID
+ pkg.TrivialArtifact
+}
+
+func (a overrideIdent) ID() pkg.ID { return a.id }
+
+// overrideIdentFile overrides the ID method of [FileArtifact].
+type overrideIdentFile struct {
+ id pkg.ID
+ pkg.FileArtifact
+}
+
+func (a overrideIdentFile) ID() pkg.ID { return a.id }
+
+// A knownIdentArtifact implements [pkg.KnownIdent] and [Artifact]
+type knownIdentArtifact interface {
+ pkg.KnownIdent
+ pkg.TrivialArtifact
+}
+
+// A knownIdentFile implements [pkg.KnownIdent] and [FileArtifact]
+type knownIdentFile interface {
+ pkg.KnownIdent
+ pkg.FileArtifact
+}
+
+// overrideChecksum overrides the Checksum method of [Artifact].
+type overrideChecksum struct {
+ checksum pkg.Checksum
+ knownIdentArtifact
+}
+
+func (a overrideChecksum) Checksum() pkg.Checksum { return a.checksum }
+
+// overrideChecksumFile overrides the Checksum method of [FileArtifact].
+type overrideChecksumFile struct {
+ checksum pkg.Checksum
+ knownIdentFile
+}
+
+func (a overrideChecksumFile) Checksum() pkg.Checksum { return a.checksum }
+
+// A stubArtifact implements [TrivialArtifact] with hardcoded behaviour.
+type stubArtifact struct {
+ kind pkg.Kind
+ params []byte
+ deps []pkg.Artifact
+
+ cure func(t *pkg.TContext) error
+}
+
+func (a *stubArtifact) Kind() pkg.Kind { return a.kind }
+func (a *stubArtifact) Params(ctx *pkg.IContext) { ctx.Write(a.params) }
+func (a *stubArtifact) Inputs() []pkg.Artifact { return a.deps }
+func (a *stubArtifact) Cure(t *pkg.TContext) error { return a.cure(t) }
+func (*stubArtifact) IsExclusive() bool { return false }
+
+// A stubArtifactF implements [FloodArtifact] with hardcoded behaviour.
+type stubArtifactF struct {
+ kind pkg.Kind
+ params []byte
+ deps []pkg.Artifact
+ excl bool
+
+ cure func(f *pkg.FContext) error
+}
+
+func (a *stubArtifactF) Kind() pkg.Kind { return a.kind }
+func (a *stubArtifactF) Params(ctx *pkg.IContext) { ctx.Write(a.params) }
+func (a *stubArtifactF) Inputs() []pkg.Artifact { return a.deps }
+func (a *stubArtifactF) Cure(f *pkg.FContext) error { return a.cure(f) }
+func (a *stubArtifactF) IsExclusive() bool { return a.excl }
+
+// A stubFile implements [FileArtifact] with hardcoded behaviour.
+type stubFile struct {
+ data []byte
+ err error
+
+ stubArtifact
+}
+
+func (*stubFile) IsExecutable() bool { return false }
+
+func (a *stubFile) Cure(*pkg.RContext) (io.ReadCloser, error) {
+ return io.NopCloser(bytes.NewReader(a.data)), a.err
+}
+
+// newStubFile returns an implementation of [pkg.File] with hardcoded behaviour.
+func newStubFile(
+ kind pkg.Kind,
+ id pkg.ID,
+ sum *pkg.Checksum,
+ data []byte,
+ err error,
+) pkg.FileArtifact {
+ f := overrideIdentFile{id, &stubFile{data, err, stubArtifact{
+ kind,
+ nil,
+ nil,
+ func(*pkg.TContext) error {
+ panic("unreachable")
+ },
+ }}}
+ if sum == nil {
+ return f
+ } else {
+ return overrideChecksumFile{*sum, f}
+ }
+}
+
+// stubExtern implements [External] with hardcoded prepared outcomes.
+type stubExtern struct {
+ artifact map[unique.Handle[pkg.ID]]pkg.Checksum
+ checksum map[unique.Handle[pkg.Checksum]]fstest.MapFS
+ status map[unique.Handle[pkg.ID]]string
+}
+
+func (e stubExtern) Artifact(_ context.Context, id unique.Handle[pkg.ID]) (*pkg.Checksum, error) {
+ if checksum, ok := e.artifact[id]; ok {
+ return &checksum, nil
+ }
+ return nil, nil
+}
+
+func (e stubExtern) Checksum(checksum unique.Handle[pkg.Checksum]) pkg.Artifact {
+ var buf bytes.Buffer
+ if err := pkg.Write(e.checksum[checksum], ".", &buf); err != nil {
+ panic(err)
+ }
+ return pkg.NewArchive(pkg.NewFile("", buf.Bytes()))
+}
+
+func (e stubExtern) Status(_ *pkg.RContext, id unique.Handle[pkg.ID]) (io.ReadCloser, error) {
+ if status, ok := e.status[id]; ok {
+ return io.NopCloser(strings.NewReader(status)), nil
+ }
+ return nil, nil
+}
+
+// destroyArtifact removes all traces of an [Artifact] from the on-disk cache.
+// Do not use this in a test case without a very good reason to do so.
+func destroyArtifact(
+ t *testing.T,
+ base *check.Absolute,
+ c *pkg.Cache,
+ a pkg.Artifact,
+) {
+ if pathname, checksum, err := c.Cure(a); err != nil {
+ t.Fatalf("Cure: error = %v", err)
+ } else if err = os.Remove(pathname.String()); err != nil {
+ t.Fatal(err)
+ } else {
+ p := base.Append(
+ "checksum",
+ pkg.Encode(checksum.Value()),
+ )
+ if err = filepath.WalkDir(p.String(), func(
+ path string,
+ d fs.DirEntry,
+ err error,
+ ) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ return os.Chmod(path, 0700)
+ }
+ return nil
+ }); err != nil && !errors.Is(err, os.ErrNotExist) {
+ t.Fatal(err)
+ }
+ if err = os.RemoveAll(p.String()); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+// newDestroyArtifactFunc returns a function that calls destroyArtifact.
+func newDestroyArtifactFunc(a pkg.Artifact) func(
+ t *testing.T,
+ base *check.Absolute,
+ c *pkg.Cache,
+) {
+ return func(
+ t *testing.T,
+ base *check.Absolute,
+ c *pkg.Cache,
+ ) {
+ destroyArtifact(t, base, c, a)
+ }
+}
+
+// destroyStatus counts non-substitution status entries and destroys them.
+func destroyStatus(t *testing.T, base *check.Absolute, c, s int) {
+ dents, err := os.ReadDir(base.Append("status").String())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var gotC, gotS int
+ for _, dent := range dents {
+ if err = os.Remove(base.Append(
+ "status",
+ dent.Name(),
+ ).String()); err != nil {
+ t.Fatal(err)
+ }
+
+ if dent.Type().IsRegular() {
+ gotC++
+ continue
+ }
+ if dent.Type()&fs.ModeSymlink == fs.ModeSymlink {
+ gotS++
+ continue
+ }
+ t.Errorf("%s: %s", dent.Name(), dent.Type())
+ }
+
+ if gotC != c {
+ t.Errorf("status: c = %d, want %d", gotC, c)
+ }
+ if gotS != s {
+ t.Errorf("status: s = %d, want %d", gotS, s)
+ }
+}
+
+func TestIdent(t *testing.T) {
+ t.Parallel()
+
+ testCases := []struct {
+ name string
+ a pkg.Artifact
+ want unique.Handle[pkg.ID]
+ }{
+ {"decompress", &stubArtifact{
+ pkg.KindDecompress,
+ []byte{pkg.Gzip, 0, 0, 0, 0, 0, 0, 0},
+ []pkg.Artifact{
+ overrideIdent{pkg.ID{}, new(stubArtifact)},
+ },
+ nil,
+ }, unique.Make[pkg.ID](pkg.MustDecode(
+ "fkmge-H56Ph2LzNNbEUjTypes8LfOxrF_CEXg07kq6ho_nbTNvzQde9-bjB7gyB_",
+ ))},
+ }
+
+ msg := message.New(log.New(os.Stderr, "ident: ", 0))
+ msg.SwapVerbose(true)
+ var cache *pkg.Cache
+ if a, err := check.NewAbs(t.TempDir()); err != nil {
+ t.Fatal(err)
+ } else if cache, err = pkg.Open(t.Context(), msg, a, nil); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(cache.Close)
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ if got := cache.Ident(tc.a); got != tc.want {
+ t.Errorf("Ident: %s, want %s",
+ pkg.Encode(got.Value()),
+ pkg.Encode(tc.want.Value()),
+ )
+ }
+ })
+ }
+}
+
+// An expectsKnown describes an expected file or directory.
+type expectsKnown interface {
+ // hash returns the checksum of the represented data.
+ hash() (checksum pkg.Checksum)
+}
+
+// An expectsChecksum is a prepared checksum value.
+type expectsChecksum pkg.Checksum
+
+// hash returns e.
+func (e expectsChecksum) hash() pkg.Checksum { return e }
+
+// An expectsFile is the contents of a file expected by the test suite.
+type expectsFile []byte
+
+// hash computes the checksum of e.
+func (e expectsFile) hash() (checksum pkg.Checksum) {
+ h := sha512.New384()
+ h.Write(e)
+ h.Sum(checksum[:0])
+ return
+}
+
+// An expectsFS describes the state of a filesystem expected by the test suite.
+type expectsFS fstest.MapFS
+
+// hash computes the checksum of e.
+func (e expectsFS) hash() (checksum pkg.Checksum) {
+ if err := pkg.SumFS(&checksum, fstest.MapFS(e), "."); err != nil {
+ panic(err)
+ }
+ return
+}
+
+// expectsFrom generates expectsFS for a filesystem directory.
+func expectsFrom(pathname string) string {
+ var buf strings.Builder
+ buf.WriteString("expectsFS{\n")
+ if err := filepath.WalkDir(pathname, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+
+ var rel string
+ if rel, err = filepath.Rel(pathname, path); err != nil {
+ return err
+ }
+ buf.WriteString("\t" + strconv.Quote(rel) + ": {Mode: ")
+
+ var fi fs.FileInfo
+ if fi, err = d.Info(); err != nil {
+ return err
+ }
+ mode := fi.Mode()
+
+ switch {
+ case mode.IsDir():
+ buf.WriteString("fs.ModeDir | 0" +
+ strconv.FormatInt(int64(mode&^fs.ModeDir), 8))
+
+ case mode&fs.ModeSymlink != 0:
+ buf.WriteString("fs.ModeSymlink | 0" +
+ strconv.FormatInt(int64(mode&^fs.ModeSymlink), 8) +
+ ", Data: []byte(")
+ var linkname string
+ if linkname, err = os.Readlink(path); err != nil {
+ return err
+ }
+ buf.WriteString(strconv.Quote(linkname))
+ buf.WriteByte(')')
+
+ case mode.IsRegular():
+ buf.WriteString("0" + strconv.FormatInt(int64(mode), 8))
+ var p []byte
+ if p, err = os.ReadFile(path); err != nil {
+ return err
+ }
+
+ if len(p) > 0 {
+ buf.WriteString(", Data: []byte(")
+ buf.WriteString(strconv.Quote(unsafe.String(unsafe.SliceData(p), len(p))))
+ buf.WriteByte(')')
+ }
+ }
+ buf.WriteString("},\n")
+ return nil
+ }); err != nil {
+ panic(err)
+ }
+ buf.WriteString("}")
+ return buf.String()
+}
+
+// cacheTestCase is a test case passed to checkWithCache where a new instance
+// of [pkg.Cache] is prepared for the test case, and is validated and removed
+// on test completion.
+type cacheTestCase struct {
+ name string
+ flags int
+ early func(t *testing.T, base *check.Absolute)
+ f func(t *testing.T, base *check.Absolute, c *pkg.Cache)
+ want expectsFS
+}
+
+const (
+ // checkDestroySubstitutes arranges for substitutes to be destroyed before
+ // measurement during checkWithCache.
+ checkDestroySubstitutes = 1 << (iota + 32)
+)
+
+// makeBase returns a [pkg.Cache] base directory created for tb.
+func makeBase(tb testing.TB) (base *check.Absolute) {
+ tb.Helper()
+
+ base = check.MustAbs(tb.TempDir())
+ if err := os.Chmod(base.String(), 0700); err != nil {
+ tb.Fatal(err)
+ }
+ tb.Cleanup(func() {
+ if err := filepath.WalkDir(base.String(), func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ tb.Error(err)
+ return nil
+ }
+ if !d.IsDir() {
+ return nil
+ }
+ return os.Chmod(path, 0700)
+ }); err != nil {
+ tb.Fatal(err)
+ }
+ })
+ return
+}
+
+// checkWithCache runs a slice of cacheTestCase.
+func checkWithCache(t *testing.T, testCases []cacheTestCase) {
+ t.Helper()
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Helper()
+ t.Parallel()
+
+ base := makeBase(t)
+ msg := message.New(log.New(os.Stderr, "cache: ", 0))
+ msg.SwapVerbose(testing.Verbose())
+
+ flags := tc.flags | pkg.CSuppressInit
+
+ if info.CanDegrade {
+ if _, err := landlock.GetABI(); err != nil {
+ if !errors.Is(err, syscall.ENOSYS) {
+ t.Fatalf("LandlockGetABI: error = %v", err)
+ }
+ flags |= pkg.CHostAbstract
+ t.Log("Landlock LSM is unavailable, setting CHostAbstract")
+ }
+ }
+
+ var scrubFunc func() error // scrub after hashing
+ if c, err := pkg.Open(t.Context(), msg, base, &pkg.CacheAttr{
+ Cures: 1 << 4,
+ Flags: flags,
+ }); err != nil {
+ t.Fatalf("Open: error = %v", err)
+ } else {
+ t.Cleanup(c.Close)
+ if tc.early != nil {
+ tc.early(t, base)
+ }
+ tc.f(t, base, c)
+ scrubFunc = func() error {
+ err = c.Scrub(1 << 7)
+ idents, checksums, cleanErr := c.Clean(false, false)
+ if len(idents) > 0 {
+ t.Errorf("destroyed %d idents", len(idents))
+ }
+ if len(checksums) > 0 {
+ t.Errorf("destroyed %d checksums", len(checksums))
+ }
+ return errors.Join(err, cleanErr)
+ }
+ }
+
+ var restoreTemp bool
+ if _, err := os.Lstat(base.Append("temp").String()); err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ t.Fatal(err)
+ }
+ } else {
+ restoreTemp = true
+ }
+
+ // destroy lock and variant file to avoid changing cache checksums
+ for _, s := range []string{
+ "lock",
+ "variant",
+ } {
+ pathname := base.Append(s)
+ if p, err := os.ReadFile(pathname.String()); err != nil {
+ t.Fatal(err)
+ } else if len(p) != 0 {
+ t.Fatalf("file %q: %q", s, string(p))
+ }
+ if err := os.Remove(pathname.String()); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ // destroy non-deterministic substitutes
+ if tc.flags&checkDestroySubstitutes != 0 {
+ substitute := base.Append("substitute")
+ if err := os.RemoveAll(substitute.String()); err != nil {
+ t.Fatal(err)
+ } else if err = os.Mkdir(substitute.String(), 0700); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ // destroy empty status directory
+ if err := syscall.Rmdir(base.Append("status").String()); err != nil {
+ if !errors.Is(err, syscall.ENOTEMPTY) {
+ t.Fatal(err)
+ }
+ }
+
+ // destroy empty fault directory
+ if err := os.Remove(base.Append("fault").String()); err != nil {
+ t.Fatal(err)
+ }
+
+ want := tc.want.hash()
+
+ var checksum pkg.Checksum
+ if err := pkg.SumDir(&checksum, base); err != nil {
+ t.Fatalf("SumDir: error = %v", err)
+ } else if checksum != want {
+ t.Fatal(expectsFrom(base.String()))
+ }
+
+ if err := scrubFunc(); err != nil {
+ t.Fatal("cache contains inconsistencies\n\n" + err.Error())
+ }
+
+ if restoreTemp {
+ if err := os.Mkdir(
+ base.Append("temp").String(),
+ 0700,
+ ); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ // validate again to make sure scrub did not condemn anything
+ if err := pkg.SumDir(&checksum, base); err != nil {
+ t.Fatalf("SumDir: error = %v", err)
+ } else if checksum != want {
+ t.Fatalf("(scrubbed) %s", expectsFrom(base.String()))
+ }
+ })
+ }
+}
+
+// A cureStep contains an [Artifact] to be cured, and the expected outcome.
+type cureStep struct {
+ name string
+
+ a pkg.Artifact
+
+ pathname *check.Absolute
+ output expectsKnown
+ whence int
+ err error
+}
+
+// ignorePathname is passed to cureMany to skip the pathname check.
+var ignorePathname = check.MustAbs("/\x00")
+
+// cureMany cures many artifacts against a [Cache] and checks their outcomes.
+func cureMany(t *testing.T, c *pkg.Cache, steps []cureStep) {
+ t.Helper()
+
+ makeChecksumH := func(checksum pkg.Checksum) unique.Handle[pkg.Checksum] {
+ if checksum == (pkg.Checksum{}) {
+ return unique.Handle[pkg.Checksum]{}
+ }
+ return unique.Make(checksum)
+ }
+
+ for _, step := range steps {
+ t.Log("cure step:", step.name)
+ if pathname, checksum, whence, err := c.CureWhence(step.a); !reflect.DeepEqual(err, step.err) {
+ faults, _err := c.ReadFaults(step.a)
+ if _err != nil {
+ t.Errorf("ReadFaults: error = %v", _err)
+ }
+
+ var p []byte
+ for _, fault := range faults {
+ p, _err = os.ReadFile(fault.String())
+ if _err != nil {
+ t.Error(_err)
+ continue
+ }
+ t.Log(string(p))
+ t.Logf("faulting cure terminated %s ago", time.Since(faults[0].Time()))
+ }
+ t.Fatalf("Cure: error = %v, want %v", err, step.err)
+ } else if step.whence != whence {
+ t.Fatalf("Cure: whence = %s, want %s", pkg.WhenceString(whence), pkg.WhenceString(step.whence))
+ } else if step.pathname != ignorePathname && !pathname.Is(step.pathname) {
+ t.Fatalf("Cure: pathname = %q, want %q", pathname, step.pathname)
+ } else if step.output == nil || checksum != makeChecksumH(step.output.hash()) {
+ if pathname != nil {
+ if name, _err := filepath.EvalSymlinks(pathname.String()); _err != nil {
+ t.Fatal(_err)
+ } else {
+ t.Fatal(expectsFrom(name))
+ }
+ } else if checksum != (unique.Handle[pkg.Checksum]{}) {
+ t.Fatalf("Cure: unexpected checksum %s", pkg.Encode(checksum.Value()))
+ }
+ } else {
+ v := any(err)
+ if err == nil {
+ v = pathname
+ }
+ var checksumVal pkg.Checksum
+ if checksum != (unique.Handle[pkg.Checksum]{}) {
+ checksumVal = checksum.Value()
+ }
+ t.Log(pkg.Encode(checksumVal)+":", v)
+ }
+ }
+}
+
+// newWantScrubError returns the address to a new [ScrubError] for base.
+func newWantScrubError(base *check.Absolute) *pkg.ScrubError {
+ return &pkg.ScrubError{
+ ChecksumMismatches: []pkg.ChecksumMismatchError{
+ {Got: pkg.MustDecode(
+ "vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX",
+ ), Want: pkg.Checksum{0xff, 0}},
+ },
+ DanglingIdentifiers: []pkg.ID{
+ {0xfe, 0},
+ {0xfe, 0xfe},
+ {0xfe, 0xff},
+ },
+ Errs: map[unique.Handle[string]][]error{
+ base.Append("checksum", "__8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA").Handle(): {
+ pkg.InvalidFileModeError(fs.ModeSymlink),
+ },
+
+ base.Append("checksum", "invalid").Handle(): {
+ base64.CorruptInputError(4),
+ },
+
+ base.Append("nonexistent").Handle(): {
+ base64.CorruptInputError(8),
+ },
+
+ base.Append("identifier", pkg.Encode(pkg.ID{0xfe, 0xff})).Handle(): {
+ &os.PathError{
+ Op: "readlink",
+ Path: base.Append(
+ "identifier",
+ pkg.Encode(pkg.ID{0xfe, 0xff}),
+ ).String(),
+ Err: syscall.EINVAL,
+ },
+ },
+ base.Append("identifier", "invalid").Handle(): {
+ base64.CorruptInputError(4),
+ },
+ },
+ }
+}
+
+func TestCache(t *testing.T) {
+ t.Parallel()
+
+ testdata := expectsFile("" +
+ "\x00\x00\x00\x00" +
+ "\xad\x0b\x00" +
+ "\x04" +
+ "\xfe\xfe\x00\x00" +
+ "\xfe\xca\x00\x00")
+
+ testCases := []cacheTestCase{
+ {"file", pkg.CValidateKnown | pkg.CAssumeChecksum, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ identifier := (pkg.ID)(bytes.Repeat([]byte{
+ 0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f,
+ }, 8))
+ wantPathname := base.Append(
+ "identifier",
+ "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
+ )
+ identifier0 := (pkg.ID)(bytes.Repeat([]byte{
+ 0x71, 0xa7, 0xde, 0x6d, 0xa6, 0xde,
+ }, 8))
+ wantPathname0 := base.Append(
+ "identifier",
+ "cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe",
+ )
+ failingFile := newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xff, 3},
+ nil,
+ nil, struct {
+ _ []byte
+ stub.UniqueError
+ }{UniqueError: 0xbad},
+ )
+
+ cureMany(t, c, []cureStep{
+ {"initial file", newStubFile(
+ pkg.KindHTTPGet,
+ identifier,
+ new(testdata.hash()),
+ testdata, nil,
+ ), wantPathname, testdata, pkg.WNew, nil},
+
+ {"identical content", newStubFile(
+ pkg.KindHTTPGet,
+ identifier0,
+ new(testdata.hash()),
+ testdata, nil,
+ ), wantPathname0, testdata, pkg.WCache, nil},
+
+ {"existing entry", newStubFile(
+ pkg.KindHTTPGet,
+ identifier,
+ new(testdata.hash()),
+ testdata, nil,
+ ), wantPathname, testdata, pkg.WCache, nil},
+
+ {"checksum mismatch", newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xff, 0},
+ new(pkg.Checksum),
+ testdata, nil,
+ ), nil, nil, pkg.WNew, &pkg.ChecksumMismatchError{
+ Got: testdata.hash(),
+ }},
+
+ {"store without validation", newStubFile(
+ pkg.KindHTTPGet,
+ pkg.MustDecode("vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX"),
+ nil,
+ []byte{0}, nil,
+ ), base.Append(
+ "identifier",
+ "vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX",
+ ), expectsChecksum{
+ 0xbe, 0xc0, 0x21, 0xb4, 0xf3, 0x68,
+ 0xe3, 0x06, 0x91, 0x34, 0xe0, 0x12,
+ 0xc2, 0xb4, 0x30, 0x70, 0x83, 0xd3,
+ 0xa9, 0xbd, 0xd2, 0x06, 0xe2, 0x4e,
+ 0x5f, 0x0d, 0x86, 0xe1, 0x3d, 0x66,
+ 0x36, 0x65, 0x59, 0x33, 0xec, 0x2b,
+ 0x41, 0x34, 0x65, 0x96, 0x68, 0x17,
+ 0xa9, 0xc2, 0x08, 0xa1, 0x17, 0x17,
+ }, pkg.WNew, nil},
+
+ {"incomplete implementation", struct{ pkg.Artifact }{&stubArtifact{
+ kind: pkg.KindExec,
+ params: []byte("artifact overridden to be incomplete"),
+ }}, nil, nil, pkg.WNew, pkg.InvalidArtifactError(pkg.MustDecode(
+ "poVrv3zNINbHd-WUJihwgWQwJpbHURFHfpY7Zux-ztto_bQhhS5PdpbNMjdkXv-V",
+ ))},
+
+ {"error passthrough", newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xff, 1},
+ nil,
+ nil, stub.UniqueError(0xcafe),
+ ), nil, nil, pkg.WNew, stub.UniqueError(0xcafe)},
+
+ {"error caching", newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xff, 1},
+ nil,
+ nil, nil,
+ ), nil, nil, pkg.WCache, stub.UniqueError(0xcafe)},
+
+ {"cache hit bad type", overrideChecksum{testdata.hash(), overrideIdent{pkg.ID{0xff, 2}, &stubArtifact{
+ kind: pkg.KindTar,
+ }}}, nil, nil, pkg.WCache, pkg.InvalidFileModeError(
+ 0400,
+ )},
+
+ {"noncomparable error", &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("artifact with dependency returning noncomparable error"),
+ deps: []pkg.Artifact{failingFile},
+
+ cure: func(f *pkg.FContext) error {
+ panic("attempting to cure impossible artifact")
+ },
+ }, nil, nil, pkg.WNew, pkg.InputError{
+ failingFile: struct {
+ _ []byte
+ stub.UniqueError
+ }{UniqueError: 0xbad},
+ }},
+ })
+
+ if c0, err := pkg.Open(
+ t.Context(),
+ message.New(nil),
+ base, &skipLock,
+ ); err != nil {
+ t.Fatalf("open: error = %v", err)
+ } else {
+ t.Cleanup(c.Close) // check doubled cancel
+ cureMany(t, c0, []cureStep{
+ {"cache hit ident", overrideIdent{
+ id: identifier,
+ }, wantPathname, testdata, pkg.WCache, nil},
+
+ {"cache miss checksum match", newStubFile(
+ pkg.KindHTTPGet,
+ testdata.hash(),
+ nil,
+ testdata,
+ nil,
+ ), base.Append(
+ "identifier",
+ pkg.Encode(testdata.hash()),
+ ), testdata, pkg.WNew, nil},
+ })
+
+ // cure after close
+ c.Close()
+ if _, _, err = c.Cure(&stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("unreachable artifact cured after cancel"),
+ deps: []pkg.Artifact{pkg.NewFile("", []byte("unreachable dependency"))},
+ }); !reflect.DeepEqual(err, context.Canceled) {
+ t.Fatalf("(closed) Cure: error = %v", err)
+ }
+ }
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX": {Mode: 0400, Data: []byte{0}},
+ "checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq": {Mode: 0400, Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX")},
+ "identifier/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq")},
+ "identifier/cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq")},
+ "identifier/deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"directory", pkg.CAssumeChecksum, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ id := pkg.MustDecode(
+ "HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY",
+ )
+ makeSample := func(t *pkg.TContext) error {
+ work := t.GetWorkDir()
+ if err := os.Mkdir(work.String(), 0700); err != nil {
+ return err
+ }
+
+ if err := os.WriteFile(
+ work.Append("check").String(),
+ []byte{0, 0},
+ 0400,
+ ); err != nil {
+ return err
+ }
+
+ if err := os.MkdirAll(work.Append(
+ "lib",
+ "pkgconfig",
+ ).String(), 0700); err != nil {
+ return err
+ }
+
+ return os.Symlink(
+ "/proc/nonexistent/libedac.so",
+ work.Append(
+ "lib",
+ "libedac.so",
+ ).String(),
+ )
+ }
+ want := expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+
+ "check": {Mode: 0400, Data: []byte{0, 0}},
+
+ "lib": {Mode: fs.ModeDir | 0700},
+ "lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")},
+
+ "lib/pkgconfig": {Mode: fs.ModeDir | 0700},
+ }
+ wantPathname := base.Append(
+ "identifier",
+ pkg.Encode(id),
+ )
+
+ id0 := pkg.MustDecode(
+ "Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa",
+ )
+ wantPathname0 := base.Append(
+ "identifier",
+ pkg.Encode(id0),
+ )
+
+ makeGarbage := func(work *check.Absolute, wantErr error) error {
+ if err := os.Mkdir(work.String(), 0700); err != nil {
+ return err
+ }
+
+ mode := fs.FileMode(0)
+ if wantErr == nil {
+ mode = 0500
+ }
+
+ if err := os.MkdirAll(work.Append(
+ "lib",
+ "pkgconfig",
+ ).String(), 0700); err != nil {
+ return err
+ }
+
+ if err := os.WriteFile(work.Append(
+ "lib",
+ "check",
+ ).String(), nil, 0400&mode); err != nil {
+ return err
+ }
+
+ if err := os.Chmod(work.Append(
+ "lib",
+ "pkgconfig",
+ ).String(), 0500&mode); err != nil {
+ return err
+ }
+ if err := os.Chmod(work.Append(
+ "lib",
+ ).String(), 0500&mode); err != nil {
+ return err
+ }
+
+ return wantErr
+ }
+
+ cureMany(t, c, []cureStep{
+ {"initial directory", overrideChecksum{want.hash(), overrideIdent{id, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: makeSample,
+ }}}, wantPathname, want, pkg.WNew, nil},
+
+ {"identical identifier", overrideChecksum{want.hash(), overrideIdent{id, &stubArtifact{
+ kind: pkg.KindTar,
+ }}}, wantPathname, want, pkg.WCache, nil},
+
+ {"identical checksum", overrideIdent{id0, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: makeSample,
+ }}, wantPathname0, want, pkg.WNew, nil},
+
+ {"cure fault", overrideIdent{pkg.ID{0xff, 0}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ return makeGarbage(t.GetWorkDir(), stub.UniqueError(0xcafe))
+ },
+ }}, nil, nil, pkg.WNew, stub.UniqueError(0xcafe)},
+
+ {"checksum mismatch", overrideChecksum{pkg.Checksum{}, overrideIdent{pkg.ID{0xff, 1}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ return makeGarbage(t.GetWorkDir(), nil)
+ },
+ }}}, nil, nil, pkg.WNew, &pkg.ChecksumMismatchError{
+ Got: pkg.MustDecode(
+ "CUx-3hSbTWPsbMfDhgalG4Ni_GmR9TnVX8F99tY_P5GtkYvczg9RrF5zO0jX9XYT",
+ ),
+ }},
+
+ {"cache hit bad type", newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xff, 2},
+ new(want.hash()),
+ testdata, nil,
+ ), nil, nil, pkg.WCache, pkg.InvalidFileModeError(
+ fs.ModeDir | 0500,
+ )},
+
+ {"openFile directory", overrideIdent{pkg.ID{0xff, 3}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ r, err := t.Open(overrideChecksumFile{checksum: want.hash()})
+ if err != nil {
+ panic(err)
+ }
+ _, err = io.ReadAll(r)
+ return err
+ },
+ }}, nil, nil, pkg.WNew, &os.PathError{
+ Op: "read",
+ Path: base.Append(
+ "checksum",
+ pkg.Encode(want.hash()),
+ ).String(),
+ Err: syscall.EISDIR,
+ }},
+
+ {"no output", overrideIdent{pkg.ID{0xff, 4}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ return nil
+ },
+ }}, nil, nil, pkg.WNew, pkg.NoOutputError{}},
+
+ {"file output", overrideIdent{pkg.ID{0xff, 5}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ return os.WriteFile(t.GetWorkDir().String(), []byte{0}, 0400)
+ },
+ }}, nil, nil, pkg.WNew, errors.New("non-file artifact produced regular file")},
+
+ {"symlink output", overrideIdent{pkg.ID{0xff, 6}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ return os.Symlink(
+ t.GetWorkDir().String(),
+ t.GetWorkDir().String(),
+ )
+ },
+ }}, nil, nil, pkg.WNew, pkg.InvalidFileModeError(
+ fs.ModeSymlink | 0777,
+ )},
+
+ {"alternative", &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("substitutable artifact"),
+ deps: []pkg.Artifact{newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xff, 8},
+ nil,
+ []byte("substitutable dependency"),
+ nil,
+ )},
+
+ cure: func(f *pkg.FContext) error {
+ return makeSample(&f.TContext)
+ },
+ }, base.Append(
+ "identifier",
+ "gIH5eti07nBAhVti3ZTn8EIGOBCZWWvNpxg5CmCZlDzIC4mBdomPmQLeEpf1IQg9",
+ ), want, pkg.WNew, nil},
+
+ {"substitutable", &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("substitutable artifact"),
+ deps: []pkg.Artifact{newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xff, 10},
+ nil,
+ []byte("substitutable dependency"),
+ nil,
+ )},
+
+ cure: func(f *pkg.FContext) error {
+ panic("substitution missed")
+ },
+ }, base.Append(
+ "identifier",
+ "VGjpp7NVw3OYLpgOQumfM0KcJgY3suz6torykUczcg4bGT24ySkPn5hy-4dNqqkR",
+ ), want, pkg.WSubstitute, nil},
+ })
+
+ if c0, err := pkg.Open(
+ t.Context(),
+ message.New(nil),
+ base, &skipLock,
+ ); err != nil {
+ t.Fatalf("open: error = %v", err)
+ } else {
+ t.Cleanup(c.Close)
+ cureMany(t, c0, []cureStep{
+ {"substitutable", &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("substitutable artifact"),
+ deps: []pkg.Artifact{newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xff, 0xff, 0xfd, 0xfd},
+ nil,
+ []byte("substitutable dependency"),
+ nil,
+ )},
+
+ cure: func(f *pkg.FContext) error {
+ panic("substitution missed")
+ },
+ }, base.Append(
+ "identifier",
+ "zeZ4JEZ0atAKl6Am67cBID6-KR18nZGgeYfU4qTuYaWhQC4eAn0dyTXdIBH9Bos8",
+ ), want, pkg.WSubstitute, nil},
+ })
+ }
+
+ if dents, err := os.ReadDir(base.Append("status").String()); err != nil {
+ t.Fatal(err)
+ } else if len(dents) > 0 {
+ t.Errorf("ReadDir: %v", dents)
+ }
+
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO": {Mode: 0400, Data: []byte("substitutable dependency")},
+ "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b": {Mode: fs.ModeDir | 0500},
+ "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/check": {Mode: 0400, Data: []byte{0, 0}},
+ "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/lib": {Mode: fs.ModeDir | 0700},
+ "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/lib/pkgconfig": {Mode: fs.ModeDir | 0700},
+ "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")},
+ "identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")},
+ "identifier/zeZ4JEZ0atAKl6Am67cBID6-KR18nZGgeYfU4qTuYaWhQC4eAn0dyTXdIBH9Bos8": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")},
+ "identifier/___9_QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO")},
+ "identifier/_wgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO")},
+ "identifier/_woAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO")},
+ "identifier/VGjpp7NVw3OYLpgOQumfM0KcJgY3suz6torykUczcg4bGT24ySkPn5hy-4dNqqkR": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")},
+ "identifier/gIH5eti07nBAhVti3ZTn8EIGOBCZWWvNpxg5CmCZlDzIC4mBdomPmQLeEpf1IQg9": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "substitute/nWPg893oqn_YNN8kgfZrs_LVLwP90e0mKxqU8UwoazQh-yBPwhoviD7of6YALx-M": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")},
+
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"pending", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ wantErr := stub.UniqueError(0xcafe)
+ n, ready := make(chan struct{}), make(chan struct{})
+ go func() {
+ if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ close(ready)
+ <-n
+ return wantErr
+ },
+ }}); !reflect.DeepEqual(err, wantErr) {
+ panic(fmt.Sprintf("Cure: error = %v, want %v", err, wantErr))
+ }
+ }()
+
+ <-ready
+ wCureDone := make(chan struct{})
+ go func() {
+ if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, &stubArtifact{
+ kind: pkg.KindTar,
+ }}); !reflect.DeepEqual(err, wantErr) {
+ panic(fmt.Sprintf("Cure: error = %v, want %v", err, wantErr))
+ }
+ close(wCureDone)
+ }()
+
+ // check cache activity while a cure is blocking
+ cureMany(t, c, []cureStep{
+ {"error passthrough", newStubFile(
+ pkg.KindHTTPGet,
+ pkg.ID{0xff, 1},
+ nil,
+ nil, stub.UniqueError(0xbad),
+ ), nil, nil, pkg.WNew, stub.UniqueError(0xbad)},
+
+ {"file output", overrideIdent{pkg.ID{0xff, 2}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ return os.WriteFile(
+ t.GetWorkDir().String(),
+ []byte{0},
+ 0400,
+ )
+ },
+ }}, nil, nil, pkg.WNew, errors.New(
+ "non-file artifact produced regular file",
+ )},
+ })
+
+ wantErrScrub := &pkg.ScrubError{
+ Errs: map[unique.Handle[string]][]error{
+ base.Handle(): {errors.New("scrub began with pending artifacts")},
+ },
+ }
+ if err := c.Scrub(1 << 6); !reflect.DeepEqual(err, wantErrScrub) {
+ t.Fatalf("Scrub: error = %#v, want %#v", err, wantErrScrub)
+ }
+
+ notify := c.Done(unique.Make(pkg.ID{0xff}))
+ go close(n)
+ if notify != nil {
+ <-notify
+ }
+ for c.Done(unique.Make(pkg.ID{0xff})) != nil {
+ }
+ <-wCureDone
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"cancel abort block", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ var wg sync.WaitGroup
+ defer wg.Wait()
+
+ var started sync.WaitGroup
+ defer started.Wait()
+
+ blockCures := func(d byte, e stub.UniqueError, n int) {
+ started.Add(n)
+ for i := range n {
+ wg.Go(func() {
+ if _, _, err := c.Cure(overrideIdent{pkg.ID{d, byte(i)}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ started.Done()
+ <-t.Unwrap().Done()
+ return e + stub.UniqueError(i)
+ },
+ }}); !reflect.DeepEqual(err, e+stub.UniqueError(i)) {
+ panic(err)
+ }
+ })
+ }
+ started.Wait()
+ }
+
+ blockCures(0xfd, 0xbad, 16)
+ c.Abort()
+ wg.Wait()
+
+ blockCures(0xfd, 0xcafe, 16)
+ c.Abort()
+ wg.Wait()
+
+ blockCures(0xff, 0xbad, 1)
+ if !c.Cancel(unique.Make(pkg.ID{0xff})) {
+ t.Fatal("missed cancellation")
+ }
+ wg.Wait()
+
+ blockCures(0xff, 0xcafe, 1)
+ if !c.Cancel(unique.Make(pkg.ID{0xff})) {
+ t.Fatal("missed cancellation")
+ }
+ wg.Wait()
+
+ for c.Cancel(unique.Make(pkg.ID{0xff})) {
+ }
+
+ c.Close()
+ c.Abort()
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"no assume checksum", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ makeGarbage := func(work *check.Absolute, wantErr error) error {
+ if err := os.Mkdir(work.String(), 0700); err != nil {
+ return err
+ }
+
+ if err := os.WriteFile(work.Append(
+ "check",
+ ).String(), nil, 0400); err != nil {
+ return err
+ }
+
+ return wantErr
+ }
+
+ want := expectsChecksum(pkg.MustDecode(
+ "Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M",
+ ))
+
+ cureMany(t, c, []cureStep{
+ {"create", overrideChecksum{want.hash(), overrideIdent{pkg.ID{0xff, 0}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ return makeGarbage(t.GetWorkDir(), nil)
+ },
+ }}}, base.Append(
+ "identifier",
+ pkg.Encode(pkg.ID{0xff, 0}),
+ ), want, pkg.WNew, nil},
+
+ {"reject", overrideChecksum{want.hash(), overrideIdent{pkg.ID{0xfe, 1}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ return makeGarbage(t.GetWorkDir(), stub.UniqueError(0xbad))
+ },
+ }}}, nil, nil, pkg.WNew, stub.UniqueError(0xbad)},
+
+ {"match", overrideChecksum{want.hash(), overrideIdent{pkg.ID{0xff, 1}, &stubArtifact{
+ kind: pkg.KindTar,
+ cure: func(t *pkg.TContext) error {
+ return makeGarbage(t.GetWorkDir(), nil)
+ },
+ }}}, base.Append(
+ "identifier",
+ pkg.Encode(pkg.ID{0xff, 1}),
+ ), want, pkg.WNew, nil},
+ })
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M": {Mode: fs.ModeDir | 0500},
+ "checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M/check": {Mode: 0400, Data: []byte{}},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/_wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M")},
+ "identifier/_wEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"scrub", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ cureMany(t, c, []cureStep{
+ {"bad measured file", newStubFile(
+ pkg.KindHTTPGet,
+ pkg.Checksum{0xfe, 0},
+ &pkg.Checksum{0xff, 0},
+ []byte{0}, nil,
+ ), base.Append(
+ "identifier",
+ pkg.Encode(pkg.Checksum{0xfe, 0}),
+ ), expectsChecksum{0xff, 0}, pkg.WNew, nil},
+ })
+
+ for _, p := range [][]string{
+ {"identifier", "invalid"},
+ {"identifier", pkg.Encode(pkg.ID{0xfe, 0xff})},
+ {"checksum", "invalid"},
+ } {
+ if err := os.WriteFile(
+ base.Append(p...).String(),
+ nil,
+ 0400,
+ ); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ for _, p := range [][]string{
+ {"../nonexistent", "checksum", pkg.Encode(pkg.Checksum{0xff, 0xff})},
+ {"../nonexistent", "identifier", pkg.Encode(pkg.Checksum{0xfe, 0xfe})},
+ } {
+ if err := os.Symlink(
+ p[0],
+ base.Append(p[1:]...).String(),
+ ); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ wantErr := newWantScrubError(base)
+ if err := c.Scrub(1 << 6); !reflect.DeepEqual(err, wantErr) {
+ t.Fatalf("Scrub: error =\n%s\nwant\n%s", err, wantErr)
+ }
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"status substitute clean", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ destroyed := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("destroyed"),
+ deps: []pkg.Artifact{
+ pkg.NewFile("destroyed-input", []byte("destroyed")),
+ },
+ cure: func(f *pkg.FContext) error {
+ if w, err := f.GetStatusWriter(); err != nil {
+ return err
+ } else if _, err = w.Write([]byte("destroyed")); err != nil {
+ return err
+ }
+
+ p := f.GetWorkDir()
+ if err := os.MkdirAll(p.String(), 0755); err != nil {
+ return err
+ }
+ return os.WriteFile(p.Append("result").String(), nil, 0444)
+ },
+ }
+ substituted := new(*destroyed)
+ substituted.deps = []pkg.Artifact{
+ pkg.NewFile("destroyed-input-0", []byte("destroyed")),
+ }
+ substituted.cure = func(*pkg.FContext) error {
+ panic("substitutable cure reached")
+ }
+
+ cureMany(t, c, []cureStep{
+ {"destroyed", destroyed, base.Append(
+ "identifier",
+ pkg.Encode(c.Ident(destroyed).Value()),
+ ), expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+ "result": {Mode: 0444},
+ }, pkg.WNew, nil},
+
+ {"substituted", substituted, base.Append(
+ "identifier",
+ pkg.Encode(c.Ident(substituted).Value()),
+ ), expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+ "result": {Mode: 0444},
+ }, pkg.WSubstitute, nil},
+ })
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE": {Mode: fs.ModeDir | 0500},
+ "checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE/result": {Mode: 0444},
+ "checksum/wILUy2izpj2sgKJVhGUGIAde1XVuqvp5BpFMIQHanT5Q8R6jK4QPVSrJsjZh-njV": {Mode: 0400, Data: []byte("destroyed")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/CLyyxJWn1yUGFs_0KnJ4ZSU2hH4-elvGFRB36x14LrNrKBJPxvORhL9O4Rc7oeYU": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/wILUy2izpj2sgKJVhGUGIAde1XVuqvp5BpFMIQHanT5Q8R6jK4QPVSrJsjZh-njV")},
+ "identifier/arv8a6z4i0x6pUb5Z2T-3iE4ETvyZj-uReHGKsS7uiBgMgKNEOzTGG3LIZb8vp0L": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/wILUy2izpj2sgKJVhGUGIAde1XVuqvp5BpFMIQHanT5Q8R6jK4QPVSrJsjZh-njV")},
+ "identifier/heRgWhNJmhm898V68kw0ta76t4rP1o29tYW_TrdyBWXGfo37K3QWA7Z-JkLXAwcn": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")},
+ "identifier/q-VoVBI3IjcWTqwIokpi3y05CP113CHgHslc-XWnhOvDULwzVc3q9Y5xOCm_YyLZ": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")},
+
+ "status": {Mode: fs.ModeDir | 0700},
+ "status/q-VoVBI3IjcWTqwIokpi3y05CP113CHgHslc-XWnhOvDULwzVc3q9Y5xOCm_YyLZ": {Mode: fs.ModeSymlink | 0777, Data: []byte("dP-6_wIDRRouOaOF-nkBy-IaUbLdYHbOUTrptu7j_qfW01mnHSjYJ0oykUmvUd2x")},
+ "status/heRgWhNJmhm898V68kw0ta76t4rP1o29tYW_TrdyBWXGfo37K3QWA7Z-JkLXAwcn": {Mode: 0400, Data: []byte(statusHeader + "destroyed")},
+ "status/dP-6_wIDRRouOaOF-nkBy-IaUbLdYHbOUTrptu7j_qfW01mnHSjYJ0oykUmvUd2x": {Mode: 0400, Data: []byte(statusHeader + "destroyed")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "substitute/dP-6_wIDRRouOaOF-nkBy-IaUbLdYHbOUTrptu7j_qfW01mnHSjYJ0oykUmvUd2x": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")},
+
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"extern", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ a := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("extern"),
+ }
+ wantIdent := c.Ident(a)
+ wantOutput := expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+ "result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")},
+ }
+ var wantChecksum pkg.Checksum
+ if err := pkg.SumFS(
+ &wantChecksum,
+ fstest.MapFS(wantOutput),
+ ".",
+ ); err != nil {
+ t.Fatal(err)
+ }
+ wantChecksumH := unique.Make(wantChecksum)
+
+ _a := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("extern substitute"),
+ deps: []pkg.Artifact{pkg.NewFile("", nil)},
+ }
+ _wantIdent := c.Ident(_a)
+ _wantOutput := expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+ }
+ var _wantChecksum pkg.Checksum
+ if err := pkg.SumFS(
+ &_wantChecksum,
+ fstest.MapFS(_wantOutput),
+ ".",
+ ); err != nil {
+ t.Fatal(err)
+ }
+ _wantChecksumH := unique.Make(_wantChecksum)
+
+ kca := pkg.NewExec(
+ "", "",
+ new(pkg.Checksum), 0, false, false,
+ fhs.AbsRoot, nil, fhs.AbsRoot, nil,
+ )
+ kcIdent := c.Ident(kca)
+
+ c.SetExternal(stubExtern{
+ artifact: map[unique.Handle[pkg.ID]]pkg.Checksum{
+ wantIdent: wantChecksum,
+ _wantIdent: _wantChecksum,
+ kcIdent: wantChecksum,
+ },
+ checksum: map[unique.Handle[pkg.Checksum]]fstest.MapFS{
+ wantChecksumH: fstest.MapFS(wantOutput),
+ _wantChecksumH: fstest.MapFS(_wantOutput),
+ },
+ status: map[unique.Handle[pkg.ID]]string{
+ wantIdent: "\x00",
+ kcIdent: "unreachable",
+ },
+ })
+
+ cureMany(t, c, []cureStep{
+ {"extern", a, base.Append(
+ "identifier",
+ pkg.Encode(wantIdent.Value()),
+ ), wantOutput, pkg.WExternal, nil},
+
+ {"substitute", _a, base.Append(
+ "identifier",
+ pkg.Encode(_wantIdent.Value()),
+ ), _wantOutput, pkg.WExternal, nil},
+
+ {"mismatch", kca, nil, nil, pkg.WExternal, &pkg.ChecksumMismatchError{
+ Got: wantChecksum,
+ }},
+ })
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500},
+ "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400},
+ "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl": {Mode: fs.ModeDir | 0500},
+ "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl/result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/k8yyhlBV3JpYzzywAnrRWkfp2TmgIDNC6FqMFwrrR2x0C3unBLveDNKgigOjG9MM": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+ "identifier/Xys8hPpJsKirZmTTuT7SwQ6661TCp_ZRu8iUjPPD7pC5E3ll7xw4zhNlZ2PVcz_-": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+ "identifier/8sInO_dhPW9kweit7jVuH_vtMkRxlZT7XE4_yEKPZmpGkgVxkmxz0OpXgbMsH6ET": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+ "identifier/ofaXbB5gIza_7W4Rzo5XDe_8FwlCPKWn3TBcmGgFAkkTWMl5GXGmOzX89Jd2Mk2A": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")},
+ "identifier/8VL-egZySw-RbnHZtxGWdf5Z2v9S9CihrdFwa54Bc5EYMQcEgWEem7QrdCTWlCb8": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+
+ "status": {Mode: fs.ModeDir | 0700},
+ "status/8VL-egZySw-RbnHZtxGWdf5Z2v9S9CihrdFwa54Bc5EYMQcEgWEem7QrdCTWlCb8": {Mode: 0400, Data: []byte("\x00")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "substitute/socGRnhdsJlJ4Z01ZwHFZdLzFH0rWvoaJrylcUKtvhBdGsE7VoYl-vdUuvfK8B4I": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"extern shallow", pkg.CExternShallow, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ a := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("extern"),
+ }
+ wantIdent := c.Ident(a)
+ wantOutput := expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+ "result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")},
+ }
+ var wantChecksum pkg.Checksum
+ if err := pkg.SumFS(
+ &wantChecksum,
+ fstest.MapFS(wantOutput),
+ ".",
+ ); err != nil {
+ t.Fatal(err)
+ }
+ wantChecksumH := unique.Make(wantChecksum)
+
+ _a := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("extern substitute"),
+ deps: []pkg.Artifact{pkg.NewFile("", nil)},
+ }
+ _wantIdent := c.Ident(_a)
+ _wantOutput := expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+ }
+ var _wantChecksum pkg.Checksum
+ if err := pkg.SumFS(
+ &_wantChecksum,
+ fstest.MapFS(_wantOutput),
+ ".",
+ ); err != nil {
+ t.Fatal(err)
+ }
+ _wantChecksumH := unique.Make(_wantChecksum)
+
+ kca := pkg.NewExec(
+ "", "",
+ new(pkg.Checksum), 0, false, false,
+ fhs.AbsRoot, nil, fhs.AbsRoot, nil,
+ )
+ kcIdent := c.Ident(kca)
+
+ fia := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("flood input"),
+ deps: []pkg.Artifact{_a},
+ }
+ fiaIdent := c.Ident(fia)
+
+ flia := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("flood input local"),
+ deps: []pkg.Artifact{_a},
+ }
+ fliaIdent := c.Ident(flia)
+
+ h := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("hanging input"),
+ }
+
+ fhia := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("flood input local"),
+ deps: []pkg.Artifact{h},
+ }
+ fhiaIdent := c.Ident(fhia)
+
+ c.SetExternal(stubExtern{
+ artifact: map[unique.Handle[pkg.ID]]pkg.Checksum{
+ wantIdent: wantChecksum,
+ _wantIdent: _wantChecksum,
+ kcIdent: wantChecksum,
+ fiaIdent: wantChecksum,
+ fliaIdent: wantChecksum,
+ fhiaIdent: wantChecksum,
+ },
+ checksum: map[unique.Handle[pkg.Checksum]]fstest.MapFS{
+ wantChecksumH: fstest.MapFS(wantOutput),
+ _wantChecksumH: fstest.MapFS(_wantOutput),
+ },
+ status: map[unique.Handle[pkg.ID]]string{
+ wantIdent: "\x00",
+ kcIdent: "unreachable",
+ },
+ })
+
+ cureMany(t, c, []cureStep{
+ {"extern", a, base.Append(
+ "identifier",
+ pkg.Encode(wantIdent.Value()),
+ ), wantOutput, pkg.WExternal, nil},
+
+ {"flood shallow extern", fia, base.Append(
+ "identifier",
+ pkg.Encode(fiaIdent.Value()),
+ ), wantOutput, pkg.WExternal, nil},
+
+ {"substitute", _a, base.Append(
+ "identifier",
+ pkg.Encode(_wantIdent.Value()),
+ ), _wantOutput, pkg.WExternal, nil},
+
+ {"flood shallow local", flia, base.Append(
+ "identifier",
+ pkg.Encode(fliaIdent.Value()),
+ ), wantOutput, pkg.WExternal, nil},
+
+ {"flood shallow hanging", fhia, nil, nil,
+ pkg.WNew, pkg.HangingInputError(c.Ident(h))},
+
+ {"mismatch", kca, nil, nil, pkg.WExternal, &pkg.ChecksumMismatchError{
+ Got: wantChecksum,
+ }},
+ })
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500},
+ "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl": {Mode: fs.ModeDir | 0500},
+ "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl/result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/3fYJjSiWhklt7TK9MPVLGk5_8AIU8ostTcFPXULJf_qeAVG6pmC-Cqqu4ZhEXRxn": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+ "identifier/8VL-egZySw-RbnHZtxGWdf5Z2v9S9CihrdFwa54Bc5EYMQcEgWEem7QrdCTWlCb8": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+ "identifier/8sInO_dhPW9kweit7jVuH_vtMkRxlZT7XE4_yEKPZmpGkgVxkmxz0OpXgbMsH6ET": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+ "identifier/Xys8hPpJsKirZmTTuT7SwQ6661TCp_ZRu8iUjPPD7pC5E3ll7xw4zhNlZ2PVcz_-": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+ "identifier/k8yyhlBV3JpYzzywAnrRWkfp2TmgIDNC6FqMFwrrR2x0C3unBLveDNKgigOjG9MM": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+ "identifier/mXlNz_v77nW7jstn_rJn4RL7dCQlkakM0y-HjO6tkDubtzaj-wCmW1hIr2Gk64_O": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+
+ "status": {Mode: fs.ModeDir | 0700},
+ "status/8VL-egZySw-RbnHZtxGWdf5Z2v9S9CihrdFwa54Bc5EYMQcEgWEem7QrdCTWlCb8": {Mode: 0400, Data: []byte("\x00")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "substitute/socGRnhdsJlJ4Z01ZwHFZdLzFH0rWvoaJrylcUKtvhBdGsE7VoYl-vdUuvfK8B4I": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+ "substitute/MccYezQ4pshkeBV4w_aAEIwvZdKHL3OAgAkR3ouZ43EF94Ur1HBHaN7njPfJAj50": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+ "substitute/VV5W0_DiA1iylaqbKFThp1ZpV_y1ccVS1XyqDnDaXoHQ-EEyPI4XFrQHcHOF2hWG": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+ }
+ checkWithCache(t, testCases)
+}
+
+func TestErrors(t *testing.T) {
+ t.Parallel()
+
+ testCases := []struct {
+ name string
+ err error
+ want string
+ }{
+ {"InvalidLookupError", pkg.InvalidLookupError{
+ 0xff, 0xf0,
+ }, "attempting to look up non-input artifact __AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"},
+
+ {"InvalidArtifactError", pkg.InvalidArtifactError{
+ 0xff, 0xfd,
+ }, "artifact __0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA cannot be cured"},
+
+ {"ChecksumMismatchError", &pkg.ChecksumMismatchError{
+ Want: (pkg.Checksum)(bytes.Repeat([]byte{
+ 0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f,
+ }, 8)),
+ }, "got AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +
+ " instead of deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"},
+
+ {"ResponseStatusError", pkg.ResponseStatusError(
+ http.StatusNotAcceptable,
+ ), "the requested URL returned non-OK status: Not Acceptable"},
+
+ {"DisallowedTypeflagError", pkg.DisallowedTypeflagError(
+ tar.TypeChar,
+ ), "disallowed typeflag '3'"},
+
+ {"InvalidFileModeError", pkg.InvalidFileModeError(
+ fs.ModeSymlink | 0777,
+ ), "artifact did not produce a regular file or directory"},
+
+ {"NoOutputError", pkg.NoOutputError{
+ // empty struct
+ }, "artifact cured successfully but did not produce any output"},
+
+ {"IRKindError", &pkg.IRKindError{
+ Got: pkg.IRKindEnd,
+ Want: pkg.IRKindIdent,
+ Ancillary: 0xcafebabe,
+ }, "got terminator IR value (0xcafebabe) instead of ident"},
+ {"IRKindError invalid", &pkg.IRKindError{
+ Got: 0xbeef,
+ Want: pkg.IRKindIdent,
+ Ancillary: 0xcafe,
+ }, "got invalid kind 48879 IR value (0xcafe) instead of ident"},
+
+ {"UnsupportedVariantError", pkg.UnsupportedVariantError(
+ "rosa",
+ ), `unsupported variant "rosa"`},
+
+ {"UnsupportedArchError zero", pkg.UnsupportedArchError(""),
+ "invalid architecture name"},
+ {"UnsupportedArchError", pkg.UnsupportedArchError("riscv64"),
+ "unsupported architecture riscv64"},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ if got := tc.err.Error(); got != tc.want {
+ t.Errorf("Error: %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
+
+func TestScrubError(t *testing.T) {
+ t.Parallel()
+
+ testCases := []struct {
+ name string
+ err pkg.ScrubError
+ want string
+ unwrap []error
+ }{
+ {"sample", *newWantScrubError(
+ fhs.AbsVarLib.Append("cure"),
+ ), `checksum mismatches:
+got vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX instead of _wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+
+dangling identifiers:
+_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+_v4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+
+errors during scrub:
+ /var/lib/cure/checksum/__8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA:
+ artifact did not produce a regular file or directory
+ /var/lib/cure/checksum/invalid:
+ illegal base64 data at input byte 4
+ /var/lib/cure/identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA:
+ readlink /var/lib/cure/identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: invalid argument
+ /var/lib/cure/identifier/invalid:
+ illegal base64 data at input byte 4
+ /var/lib/cure/nonexistent:
+ illegal base64 data at input byte 8
+`, []error{
+ &pkg.ChecksumMismatchError{Got: pkg.MustDecode(
+ "vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX",
+ ), Want: pkg.Checksum{0xff, 0}},
+
+ pkg.InvalidFileModeError(fs.ModeSymlink),
+ base64.CorruptInputError(4),
+
+ &os.PathError{
+ Op: "readlink",
+ Path: fhs.AbsVarLib.Append("cure").Append(
+ "identifier",
+ pkg.Encode(pkg.ID{0xfe, 0xff}),
+ ).String(),
+ Err: syscall.EINVAL,
+ },
+
+ base64.CorruptInputError(4),
+ base64.CorruptInputError(8),
+ }},
+
+ {"full", pkg.ScrubError{
+ ChecksumMismatches: []pkg.ChecksumMismatchError{
+ {Want: pkg.MustDecode("CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN")},
+ },
+ DanglingIdentifiers: []pkg.ID{
+ (pkg.ID)(bytes.Repeat([]byte{0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f}, 8)),
+ (pkg.ID)(bytes.Repeat([]byte{0x71, 0xa7, 0xde, 0x6d, 0xa6, 0xde}, 8)),
+ },
+ Errs: map[unique.Handle[string]][]error{
+ unique.Make("/proc/nonexistent"): {
+ stub.UniqueError(0xcafe),
+ stub.UniqueError(0xbad),
+ stub.UniqueError(0xff),
+ },
+ },
+ }, `checksum mismatches:
+got AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA instead of CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN
+
+dangling identifiers:
+deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef
+cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe
+
+errors during scrub:
+ /proc/nonexistent:
+ unique error 51966 injected by the test suite
+ unique error 2989 injected by the test suite
+ unique error 255 injected by the test suite
+`, []error{
+ &pkg.ChecksumMismatchError{Want: pkg.MustDecode("CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN")},
+ stub.UniqueError(0xcafe),
+ stub.UniqueError(0xbad),
+ stub.UniqueError(0xff),
+ }},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ if got := tc.err.Error(); got != tc.want {
+ t.Errorf("Error:\n\n%s\n\nwant\n\n%s", got, tc.want)
+ }
+
+ if unwrap := tc.err.Unwrap(); !reflect.DeepEqual(unwrap, tc.unwrap) {
+ t.Errorf("Unwrap: %#v, want %#v", unwrap, tc.unwrap)
+ }
+ })
+ }
+}
+
+func TestInputError(t *testing.T) {
+ t.Parallel()
+
+ makeIdent := func(ident ...byte) pkg.Artifact {
+ var a overrideIdent
+ copy(a.id[:], ident)
+ // does not compare equal
+ a.TrivialArtifact = new(stubArtifact)
+ return a
+ }
+
+ testCases := []struct {
+ name string
+ err pkg.InputError
+ want string
+ unwrap []error
+ }{
+ {"simple", pkg.InputError{
+ makeIdent(0xff, 9): stub.UniqueError(0xbad09),
+ makeIdent(0xff, 0): stub.UniqueError(0xbad00),
+ makeIdent(0xff, 0xf): stub.UniqueError(0xbad0f),
+ makeIdent(0xff, 1): stub.UniqueError(0xbad01),
+ }, `errors curing inputs:
+ _wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765184 injected by the test suite
+ _wEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765185 injected by the test suite
+ _wkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765193 injected by the test suite
+ _w8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765199 injected by the test suite`, []error{
+ stub.UniqueError(0xbad00),
+ stub.UniqueError(0xbad01),
+ stub.UniqueError(0xbad09),
+ stub.UniqueError(0xbad0f),
+ }},
+
+ {"dedup", pkg.InputError{
+ makeIdent(0xff, 9): stub.UniqueError(0xbad09),
+ makeIdent(0xff, 9): stub.UniqueError(0xbad09),
+ makeIdent(0xff, 9): stub.UniqueError(0xbad09),
+ makeIdent(0xff, 0): stub.UniqueError(0xbad00),
+ makeIdent(0xff, 0): stub.UniqueError(0xbad00),
+ makeIdent(0xff, 1): stub.UniqueError(0xbad01),
+ }, `errors curing inputs:
+ _wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765184 injected by the test suite
+ _wEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765185 injected by the test suite
+ _wkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765193 injected by the test suite`, []error{
+ stub.UniqueError(0xbad00),
+ stub.UniqueError(0xbad01),
+ stub.UniqueError(0xbad09),
+ }},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ if got := tc.err.Error(); got != tc.want {
+ t.Errorf("Error:\n%s\nwant\n%s", got, tc.want)
+ }
+
+ if unwrap := tc.err.Unwrap(); !reflect.DeepEqual(unwrap, tc.unwrap) {
+ t.Errorf("Unwrap: %#v, want %#v", unwrap, tc.unwrap)
+ }
+ })
+ }
+}
+
+// earlyFailureF is a [FloodArtifact] with a large dependency graph resulting in
+// a large [DependencyCureError].
+type earlyFailureF int
+
+func (earlyFailureF) Kind() pkg.Kind { return pkg.KindExec }
+func (earlyFailureF) Params(*pkg.IContext) {}
+func (earlyFailureF) IsExclusive() bool { return false }
+
+func (a earlyFailureF) Inputs() []pkg.Artifact {
+ deps := make([]pkg.Artifact, a)
+ for i := range deps {
+ deps[i] = a - 1
+ }
+ return deps
+}
+
+func (a earlyFailureF) Cure(*pkg.FContext) error {
+ if a != 0 {
+ panic("unexpected cure on " + strconv.Itoa(int(a)))
+ }
+ return stub.UniqueError(0xcafe)
+}
+
+func BenchmarkEarlyDCE(b *testing.B) {
+ msg := message.New(log.New(os.Stderr, "dce: ", 0))
+ msg.SwapVerbose(testing.Verbose())
+
+ for b.Loop() {
+ b.StopTimer()
+ c, err := pkg.Open(b.Context(), msg, check.MustAbs(b.TempDir()), nil)
+ if err != nil {
+ b.Fatal(err)
+ }
+ b.StartTimer()
+ _, _, err = c.Cure(earlyFailureF(8))
+ b.StopTimer()
+ if !errors.Is(err, stub.UniqueError(0xcafe)) {
+ b.Fatalf("Cure: error = %v", err)
+ }
+ c.Close()
+ b.StartTimer()
+ }
+}
+
+func TestDependencyCureErrorEarly(t *testing.T) {
+ t.Parallel()
+
+ checkWithCache(t, []cacheTestCase{
+ {"early", 0, nil, func(t *testing.T, _ *check.Absolute, c *pkg.Cache) {
+ _, _, err := c.Cure(earlyFailureF(8))
+ if !errors.Is(err, stub.UniqueError(0xcafe)) {
+ t.Fatalf("Cure: error = %v", err)
+ }
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+ })
+}
+
+func TestOpen(t *testing.T) {
+ t.Parallel()
+
+ t.Run("nonexistent", func(t *testing.T) {
+ t.Parallel()
+
+ wantErr := &os.PathError{
+ Op: "mkdir",
+ Path: container.Nonexistent,
+ Err: syscall.ENOENT,
+ }
+ if _, err := pkg.Open(
+ t.Context(),
+ message.New(nil),
+ check.MustAbs(container.Nonexistent),
+ nil,
+ ); !reflect.DeepEqual(err, wantErr) {
+ t.Errorf("Open: error = %#v, want %#v", err, wantErr)
+ }
+ })
+
+ t.Run("permission", func(t *testing.T) {
+ t.Parallel()
+
+ tempDir := check.MustAbs(t.TempDir())
+ if err := os.Chmod(tempDir.String(), 0); err != nil {
+ t.Fatal(err)
+ } else {
+ t.Cleanup(func() {
+ if err = os.Chmod(tempDir.String(), 0700); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+
+ wantErr := &os.PathError{
+ Op: "mkdir",
+ Path: tempDir.Append("cache").String(),
+ Err: syscall.EACCES,
+ }
+ if _, err := pkg.Open(
+ t.Context(),
+ message.New(nil),
+ tempDir.Append("cache"),
+ nil,
+ ); !reflect.DeepEqual(err, wantErr) {
+ t.Errorf("Open: error = %#v, want %#v", err, wantErr)
+ }
+ })
+
+ t.Run("dirty", func(t *testing.T) {
+ t.Parallel()
+
+ tempDir := check.MustAbs(t.TempDir())
+ if err := os.MkdirAll(tempDir.Append(
+ "cache",
+ "work",
+ "dirty",
+ ).String(), 0755); err != nil {
+ t.Fatal(err)
+ }
+
+ wantErr := errors.New("work is not empty, scrub likely required")
+ if _, err := pkg.Open(
+ t.Context(),
+ message.New(nil),
+ tempDir.Append("cache"),
+ nil,
+ ); !reflect.DeepEqual(err, wantErr) {
+ t.Errorf("Open: error = %#v, want %#v", err, wantErr)
+ }
+ })
+
+ t.Run("scratch", func(t *testing.T) {
+ t.Parallel()
+
+ tempDir := check.MustAbs(t.TempDir())
+ if err := os.MkdirAll(tempDir.Append(
+ "cache",
+ "scratch",
+ ).String(), 0755); err != nil {
+ t.Fatal(err)
+ }
+
+ wantErr := errors.New("scratch is present, scrub likely required")
+ if _, err := pkg.Open(
+ t.Context(),
+ message.New(nil),
+ tempDir.Append("cache"), nil,
+ ); !reflect.DeepEqual(err, wantErr) {
+ t.Errorf("Open: error = %#v, want %#v", err, wantErr)
+ }
+ })
+}
+
+func TestExtensionRegister(t *testing.T) {
+ extensionOld := extension
+ openedOld := opened
+ t.Cleanup(func() { extension = extensionOld; opened = openedOld })
+ extension = ""
+ opened = false
+
+ t.Run("set", func(t *testing.T) {
+ t.Cleanup(func() { extension = "" })
+
+ const want = "rosa"
+ pkg.SetExtension(want)
+ if got := pkg.Extension(); got != want {
+ t.Fatalf("Extension: %q, want %q", got, want)
+ }
+ })
+
+ t.Run("twice", func(t *testing.T) {
+ t.Cleanup(func() { extension = "" })
+
+ defer func() {
+ const wantPanic = "attempting to set extension twice"
+ if r := recover(); r != wantPanic {
+ t.Errorf("panic: %#v, want %q", r, wantPanic)
+ }
+ }()
+ pkg.SetExtension("rosa")
+ pkg.SetExtension("rosa")
+ })
+
+ t.Run("invalid", func(t *testing.T) {
+ defer func() {
+ var wantPanic = pkg.ErrInvalidExtension
+ if r := recover(); r != wantPanic {
+ t.Errorf("panic: %#v, want %#v", r, wantPanic)
+ }
+ }()
+ pkg.SetExtension(" ")
+ })
+
+ t.Run("opened", func(t *testing.T) {
+ t.Cleanup(func() { opened = false })
+
+ if _, err := pkg.Open(
+ t.Context(),
+ message.New(log.Default()),
+ check.MustAbs(container.Nonexistent),
+ nil,
+ ); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("Open: error = %v", err)
+ }
+
+ t.Run("variant", func(t *testing.T) {
+ defer func() {
+ const wantPanic = "attempting to set extension after open"
+ if r := recover(); r != wantPanic {
+ t.Errorf("panic: %#v, want %q", r, wantPanic)
+ }
+ }()
+ pkg.SetExtension("rosa")
+ })
+
+ t.Run("register", func(t *testing.T) {
+ defer func() {
+ const wantPanic = "attempting to register after open"
+ if r := recover(); r != wantPanic {
+ t.Errorf("panic: %#v, want %q", r, wantPanic)
+ }
+ }()
+ pkg.Register(pkg.KindCustomOffset, nil)
+ })
+ })
+
+ t.Run("incomplete", func(t *testing.T) {
+ t.Cleanup(func() { delete(irArtifact, pkg.KindCustomOffset) })
+
+ defer func() {
+ const wantPanic = "attempting to open cache with incomplete variant setup"
+ if r := recover(); r != wantPanic {
+ t.Errorf("panic: %#v, want %q", r, wantPanic)
+ }
+ }()
+ pkg.Register(pkg.KindCustomOffset, nil)
+
+ t.Cleanup(func() { opened = false })
+ _, _ = pkg.Open(nil, nil, nil, nil)
+ panic("unreachable")
+ })
+
+ t.Run("create", func(t *testing.T) {
+ t.Cleanup(func() { extension = "" })
+ const want = "rosa"
+ pkg.SetExtension(want)
+
+ base := check.MustAbs(t.TempDir())
+ t.Cleanup(func() { opened = false })
+ if c, err := pkg.Open(t.Context(), nil, base, nil); err != nil {
+ t.Fatal(err)
+ } else {
+ c.Close()
+ }
+
+ if got, err := os.ReadFile(base.Append("variant").String()); err != nil {
+ t.Fatal(err)
+ } else if string(got) != want {
+ t.Fatalf("variant: %q", string(got))
+ }
+ })
+
+ t.Run("access", func(t *testing.T) {
+ base := check.MustAbs(t.TempDir())
+ t.Cleanup(func() { opened = false })
+
+ if err := os.WriteFile(base.Append("variant").String(), nil, 0); err != nil {
+ t.Fatal(err)
+ }
+
+ wantErr := &os.PathError{
+ Op: "open",
+ Path: base.Append("variant").String(),
+ Err: syscall.EACCES,
+ }
+ if _, err := pkg.Open(
+ t.Context(), nil,
+ base, nil,
+ ); !reflect.DeepEqual(err, wantErr) {
+ t.Fatalf("Open: error = %v, want %v", err, wantErr)
+ }
+ })
+
+ t.Run("promote", func(t *testing.T) {
+ t.Cleanup(func() { extension = "" })
+ const want = "rosa"
+ pkg.SetExtension(want)
+
+ base := check.MustAbs(t.TempDir())
+ t.Cleanup(func() { opened = false })
+
+ variantPath := base.Append("variant")
+ if err := os.WriteFile(variantPath.String(), nil, 0600); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := pkg.Open(
+ t.Context(), nil,
+ base,
+ nil,
+ ); !reflect.DeepEqual(err, pkg.ErrWouldPromote) {
+ t.Fatalf("Open: error = %v", err)
+ }
+
+ if p, err := os.ReadFile(variantPath.String()); err != nil {
+ t.Fatal(err)
+ } else if len(p) != 0 {
+ t.Fatalf("variant: %q", string(p))
+ }
+
+ if c, err := pkg.Open(
+ t.Context(), nil,
+ base, &pkg.CacheAttr{Flags: pkg.CPromoteVariant},
+ ); err != nil {
+ t.Fatalf("Open: error = %v", err)
+ } else {
+ c.Close()
+ }
+
+ if p, err := os.ReadFile(variantPath.String()); err != nil {
+ t.Fatal(err)
+ } else if string(p) != want {
+ t.Fatalf("variant: %q, want %q", string(p), want)
+ }
+ })
+
+ t.Run("open invalid", func(t *testing.T) {
+ base := check.MustAbs(t.TempDir())
+ t.Cleanup(func() { opened = false })
+
+ variantPath := base.Append("variant")
+ if err := os.WriteFile(variantPath.String(), make([]byte, 129), 0400); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := pkg.Open(
+ t.Context(), nil,
+ base, nil,
+ ); !reflect.DeepEqual(err, pkg.ErrInvalidExtension) {
+ t.Fatalf("Open: error = %v", err)
+ }
+ })
+
+ t.Run("unsupported", func(t *testing.T) {
+ base := check.MustAbs(t.TempDir())
+ t.Cleanup(func() { opened = false })
+
+ variantPath := base.Append("variant")
+ if err := os.WriteFile(variantPath.String(), []byte("rosa"), 0400); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := pkg.Open(
+ t.Context(), nil,
+ base, nil,
+ ); !reflect.DeepEqual(err, pkg.UnsupportedVariantError("rosa")) {
+ t.Fatalf("Open: error = %v", err)
+ }
+ })
+}
diff --git a/pkg/tar.go b/pkg/tar.go
new file mode 100644
index 00000000..9a8c4b76
--- /dev/null
+++ b/pkg/tar.go
@@ -0,0 +1,226 @@
+package pkg
+
+import (
+ "archive/tar"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+)
+
+// A tarArtifact is an [Artifact] unpacking a tarball backed by a [FileArtifact].
+type tarArtifact struct {
+ // Caller-supplied backing tarball.
+ f Artifact
+}
+
+var _ CuresExempt = new(tarArtifact)
+
+// tarArtifactNamed embeds tarArtifact for a [fmt.Stringer] tarball.
+type tarArtifactNamed struct {
+ tarArtifact
+ // Copied from tarArtifact.f.
+ name string
+}
+
+var _ fmt.Stringer = new(tarArtifactNamed)
+
+// String returns the name of the underlying [Artifact] prefixed with unpack.
+func (a *tarArtifactNamed) String() string { return "unpack-" + a.name }
+
+// NewTar returns a new [Artifact] unpacking the tar stream produced by the
+// backing [Artifact]. The source [Artifact] must be a [FileArtifact].
+func NewTar(a Artifact) Artifact {
+ ta := tarArtifact{a}
+ if s, ok := a.(fmt.Stringer); ok {
+ if name := s.String(); name != "" {
+ return &tarArtifactNamed{ta, name}
+ }
+ }
+ return &ta
+}
+
+// Kind returns the hardcoded [Kind] constant.
+func (a *tarArtifact) Kind() Kind { return KindTar }
+
+// Params is a noop.
+func (a *tarArtifact) Params(*IContext) {}
+
+func init() {
+ register(KindTar, func(r *IRReader) Artifact {
+ a := NewTar(r.Next())
+ if _, ok := r.Finalise(); ok {
+ panic(ErrUnexpectedChecksum)
+ }
+ return a
+ })
+}
+
+// Inputs returns a slice containing the backing file.
+func (a *tarArtifact) Inputs() []Artifact {
+ return []Artifact{a.f}
+}
+
+// IsExclusive returns false: decompressor and tar reader are fully sequential.
+func (a *tarArtifact) IsExclusive() bool { return false }
+
+// A DisallowedTypeflagError describes a disallowed typeflag encountered while
+// unpacking a tarball.
+type DisallowedTypeflagError byte
+
+func (e DisallowedTypeflagError) Error() string {
+ return "disallowed typeflag '" + string(e) + "'"
+}
+
+// Cure cures the [Artifact], producing a directory located at work.
+func (a *tarArtifact) Cure(t *TContext) (err error) {
+ var r io.ReadCloser
+ if r, err = t.Open(a.f); err != nil {
+ return
+ }
+
+ defer func() {
+ closeErr := r.Close()
+ if err == nil {
+ err = closeErr
+ }
+ }()
+
+ type dirTargetPerm struct {
+ path string
+ mode fs.FileMode
+ }
+ var madeDirectories []dirTargetPerm
+
+ if err = os.MkdirAll(t.GetTempDir().String(), 0700); err != nil {
+ return
+ }
+ var root *os.Root
+ if root, err = os.OpenRoot(t.GetTempDir().String()); err != nil {
+ return
+ }
+ defer func() {
+ closeErr := root.Close()
+ if err == nil {
+ err = closeErr
+ }
+ }()
+
+ var header *tar.Header
+ tr := tar.NewReader(r)
+ for header, err = tr.Next(); err == nil; header, err = tr.Next() {
+ typeflag := header.Typeflag
+ if typeflag == 0 {
+ if len(header.Name) > 0 && header.Name[len(header.Name)-1] == '/' {
+ typeflag = tar.TypeDir
+ } else {
+ typeflag = tar.TypeReg
+ }
+ }
+
+ if typeflag >= '0' && typeflag <= '9' && typeflag != tar.TypeDir {
+ if err = root.MkdirAll(filepath.Dir(header.Name), 0700); err != nil {
+ return
+ }
+ }
+
+ switch typeflag {
+ case tar.TypeReg:
+ var f *os.File
+ if f, err = root.OpenFile(
+ header.Name,
+ os.O_CREATE|os.O_EXCL|os.O_WRONLY,
+ header.FileInfo().Mode()&0500,
+ ); err != nil {
+ return
+ }
+ if _, err = io.Copy(f, tr); err != nil {
+ _ = f.Close()
+ return
+ } else if err = f.Close(); err != nil {
+ return
+ }
+ break
+
+ case tar.TypeLink:
+ if err = root.Link(
+ header.Linkname,
+ header.Name,
+ ); err != nil {
+ return
+ }
+ break
+
+ case tar.TypeSymlink:
+ if err = root.Symlink(
+ header.Linkname,
+ header.Name,
+ ); err != nil {
+ return
+ }
+ break
+
+ case tar.TypeDir:
+ madeDirectories = append(madeDirectories, dirTargetPerm{
+ path: header.Name,
+ mode: header.FileInfo().Mode(),
+ })
+ if err = root.MkdirAll(header.Name, 0700); err != nil {
+ return
+ }
+ break
+
+ case tar.TypeChar:
+ t.GetMessage().Verbosef(
+ "%sskipping character device %d, %d %q%s",
+ t.cache.sgrWarn, header.Devmajor, header.Devminor, header.Name, t.cache.sgrRes,
+ )
+ continue
+
+ case tar.TypeXGlobalHeader:
+ continue // ignore
+
+ default:
+ return DisallowedTypeflagError(typeflag)
+ }
+ }
+ if errors.Is(err, io.EOF) {
+ err = nil
+ }
+ if err == nil {
+ for _, e := range madeDirectories {
+ if err = root.Chmod(e.path, e.mode&0500); err != nil {
+ return
+ }
+ }
+ } else {
+ return
+ }
+
+ temp := t.GetTempDir()
+ if err = os.Chmod(temp.String(), 0700); err != nil {
+ return
+ }
+
+ var entries []os.DirEntry
+ if entries, err = os.ReadDir(temp.String()); err != nil {
+ return
+ }
+
+ if len(entries) == 1 && entries[0].IsDir() {
+ p := temp.Append(entries[0].Name())
+ if err = os.Chmod(p.String(), 0700); err != nil {
+ return
+ }
+ err = os.Rename(p.String(), t.GetWorkDir().String())
+ } else {
+ err = os.Rename(temp.String(), t.GetWorkDir().String())
+ }
+ return
+}
+
+// CuresExempt exempts the cheap [KindTar] implementation often at the end of a
+// [FileArtifact] pipeline.
+func (*tarArtifact) CuresExempt() {}
diff --git a/pkg/tar_test.go b/pkg/tar_test.go
new file mode 100644
index 00000000..5462714f
--- /dev/null
+++ b/pkg/tar_test.go
@@ -0,0 +1,225 @@
+package pkg_test
+
+import (
+ "archive/tar"
+ "bytes"
+ "crypto/sha512"
+ "errors"
+ "io/fs"
+ "net/http"
+ "os"
+ "testing"
+ "testing/fstest"
+
+ "hakurei.app/check"
+ "hakurei.app/internal/stub"
+ "hakurei.app/pkg"
+)
+
+func TestTar(t *testing.T) {
+ t.Parallel()
+
+ want := expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+
+ "checksum": {Mode: fs.ModeDir | 0500},
+ "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP": {Mode: fs.ModeDir | 0500},
+ "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check": {Mode: 0400, Data: []byte{0, 0}},
+ "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib": {Mode: fs.ModeDir | 0500},
+ "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig": {Mode: fs.ModeDir | 0500},
+ "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")},
+
+ "identifier": {Mode: fs.ModeDir | 0500},
+ "identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")},
+ "identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")},
+
+ "work": {Mode: fs.ModeDir | 0500},
+ }
+ wantEncode := pkg.Encode(want.hash())
+
+ wantExpand := expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+
+ "libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")},
+ }
+ wantExpandEncode := pkg.Encode(wantExpand.hash())
+
+ checkWithCache(t, []cacheTestCase{
+ {"http", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ checkTarHTTP(t, base, c, fstest.MapFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP": {Mode: fs.ModeDir | 0700},
+ "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check": {Mode: 0400, Data: []byte{0, 0}},
+ "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib": {Mode: fs.ModeDir | 0700},
+ "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig": {Mode: fs.ModeDir | 0700},
+ "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")},
+ "identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")},
+
+ "work": {Mode: fs.ModeDir | 0700},
+ }, want)
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/" + wantEncode: {Mode: fs.ModeDir | 0500},
+ "checksum/" + wantEncode + "/checksum": {Mode: fs.ModeDir | 0500},
+ "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP": {Mode: fs.ModeDir | 0500},
+ "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check": {Mode: 0400, Data: []byte{0, 0}},
+ "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib": {Mode: fs.ModeDir | 0500},
+ "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")},
+ "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig": {Mode: fs.ModeDir | 0500},
+ "checksum/" + wantEncode + "/identifier": {Mode: fs.ModeDir | 0500},
+ "checksum/" + wantEncode + "/identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")},
+ "checksum/" + wantEncode + "/identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")},
+ "checksum/" + wantEncode + "/work": {Mode: fs.ModeDir | 0500},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/snWp53xxNdx6cV5KSJqzyVWzLlTsFd8udOisIu8Rgxly09mGcxufYCoQ5CQrHDyZ": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantEncode)},
+ "identifier/v9DUj2R4YK_3ae0m9VuUYI-HJysDDCENu1k10thxioycRzFH1Qejw5bBGukm4IdL": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantEncode)},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "temp": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+
+ {"http expand", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ checkTarHTTP(t, base, c, fstest.MapFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "lib": {Mode: fs.ModeDir | 0700},
+ "lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")},
+ }, wantExpand)
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/" + wantExpandEncode: {Mode: fs.ModeDir | 0500},
+ "checksum/" + wantExpandEncode + "/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/1SuvgGltcsgqps9tWqjULG8RCdpx_6KVWXdsMd5xPKvdtysOxqpE-3bLu4j0WpGk": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantExpandEncode)},
+ "identifier/snWp53xxNdx6cV5KSJqzyVWzLlTsFd8udOisIu8Rgxly09mGcxufYCoQ5CQrHDyZ": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantExpandEncode)},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+
+ "temp": {Mode: fs.ModeDir | 0700},
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
+ })
+}
+
+func checkTarHTTP(
+ t *testing.T,
+ base *check.Absolute,
+ c *pkg.Cache,
+ testdataFsys fs.FS,
+ want expectsKnown,
+) {
+ var testdata string
+ {
+ var buf bytes.Buffer
+ w := tar.NewWriter(&buf)
+ if err := w.AddFS(testdataFsys); err != nil {
+ t.Fatalf("AddFS: error = %v", err)
+ }
+ if err := w.Close(); err != nil {
+ t.Fatalf("Close: error = %v", err)
+ }
+ testdata = buf.String()
+ }
+
+ testdataChecksum := func() pkg.Checksum {
+ h := sha512.New384()
+ h.Write([]byte(testdata))
+ return (pkg.Checksum)(h.Sum(nil))
+ }()
+
+ var transport http.Transport
+ client := http.Client{Transport: &transport}
+ transport.RegisterProtocol("file", http.NewFileTransportFS(fstest.MapFS{
+ "testdata": {Data: []byte(testdata), Mode: 0400},
+ }))
+
+ tarDir := stubArtifact{
+ kind: pkg.KindExec,
+ params: []byte("directory containing a single regular file"),
+ cure: func(t *pkg.TContext) error {
+ work := t.GetWorkDir()
+ if err := os.MkdirAll(work.String(), 0700); err != nil {
+ return err
+ }
+ return os.WriteFile(
+ work.Append("sample.tar.gz").String(),
+ []byte(testdata),
+ 0400,
+ )
+ },
+ }
+ tarDirMulti := stubArtifact{
+ kind: pkg.KindExec,
+ params: []byte("directory containing a multiple entries"),
+ cure: func(t *pkg.TContext) error {
+ work := t.GetWorkDir()
+ if err := os.MkdirAll(work.Append(
+ "garbage",
+ ).String(), 0700); err != nil {
+ return err
+ }
+ return os.WriteFile(
+ work.Append("sample.tar.gz").String(),
+ []byte(testdata),
+ 0400,
+ )
+ },
+ }
+ tarDirType := stubArtifact{
+ kind: pkg.KindExec,
+ params: []byte("directory containing a symbolic link"),
+ cure: func(t *pkg.TContext) error {
+ work := t.GetWorkDir()
+ if err := os.MkdirAll(work.String(), 0700); err != nil {
+ return err
+ }
+ return os.Symlink(
+ work.String(),
+ work.Append("sample.tar.gz").String(),
+ )
+ },
+ }
+ // destroy these to avoid including it in flatten test case
+ defer newDestroyArtifactFunc(&tarDir)(t, base, c)
+ defer newDestroyArtifactFunc(&tarDirMulti)(t, base, c)
+ defer newDestroyArtifactFunc(&tarDirType)(t, base, c)
+
+ cureMany(t, c, []cureStep{
+ {"file", pkg.NewTar(pkg.NewHTTPGet(
+ &client,
+ "file:///testdata",
+ testdataChecksum,
+ )), ignorePathname, want, pkg.WNew, nil},
+
+ {"directory", pkg.NewTar(&tarDir), ignorePathname, want, pkg.WNew, nil},
+
+ {"multiple entries", pkg.NewTar(&tarDirMulti), nil, nil, pkg.WNew, errors.New(
+ "input directory does not contain a single regular file",
+ )},
+
+ {"bad type", pkg.NewTar(&tarDirType), nil, nil, pkg.WNew, errors.New(
+ "input directory does not contain a single regular file",
+ )},
+
+ {"error passthrough", pkg.NewTar(&stubArtifact{
+ kind: pkg.KindExec,
+ params: []byte("doomed artifact"),
+ cure: func(t *pkg.TContext) error {
+ return stub.UniqueError(0xcafe)
+ },
+ }), nil, nil, pkg.WNew, stub.UniqueError(0xcafe)},
+ })
+}
generated by cgit v1.3.1 (git 2.54.0) at 2026-08-29 23:05:28 +0000