diff options
Diffstat (limited to 'internal')
38 files changed, 15 insertions, 10381 deletions
diff --git a/internal/pkg/archive.go b/internal/pkg/archive.go deleted file mode 100644 index ea4017d9..00000000 --- a/internal/pkg/archive.go +++ /dev/null @@ -1,411 +0,0 @@ -package pkg - -import ( - "crypto/sha512" - "encoding/binary" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "unsafe" - - "hakurei.app/check" -) - -/* -| mode uint32 | path_sz uint32 | -| data_sz uint64 | -| path string | -| data []byte | -*/ - -// An ArchiveHeader represents a single header in an archive. -type ArchiveHeader struct { - Mode fs.FileMode // file mode bits - Path string // pathname of the file - Size uint64 // size of data segment -} - -// Writer implements sequential writing of an archive. [Writer.WriteHeader] -// begins a new file with the provided [ArchiveHeader], and then Writer can be -// treated as an [io.Writer] to supply that file's data. -// -// It is the caller's responsibility to write entries in lexical order. -type Writer struct { - // Underlying writer. - w io.Writer - // Current header. - h ArchiveHeader - // Fixed-size header segment. - buf [wordSize * 2]byte - // Current position in data segment. - n uint64 -} - -// NewWriter returns the address of a new [Writer] writing to w. -func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } - -var zero [wordSize]byte - -// padSize returns the padding size for aligning sz. -func padSize[T int | uint64](sz T) T { - return (wordSize - (sz)%wordSize) % wordSize -} - -// flush concludes writing to the current file and writes padding. -func (aw *Writer) flush() error { - if aw.h.Size > aw.n { - return fmt.Errorf("missed writing %d bytes", aw.h.Size-aw.n) - } else if aw.h.Size < aw.n { - return fmt.Errorf("wrote %d bytes beyond end of file", aw.n-aw.h.Size) - } - - if psz := padSize(aw.h.Size); psz != 0 { - if _, err := aw.w.Write(zero[:psz]); err != nil { - return err - } - } - - aw.n = 0 - return nil -} - -// WriteHeader writes h and begins accepting its corresponding file. -func (aw *Writer) WriteHeader(h *ArchiveHeader) error { - if err := aw.flush(); err != nil { - return err - } - - aw.h = *h - binary.LittleEndian.PutUint32(aw.buf[:], uint32(aw.h.Mode)) - binary.LittleEndian.PutUint32(aw.buf[wordSize/2:], uint32(len(aw.h.Path))) - binary.LittleEndian.PutUint64(aw.buf[wordSize:], aw.h.Size) - if _, err := aw.w.Write(aw.buf[:]); err != nil { - return err - } else if _, err = aw.w.Write( - unsafe.Slice(unsafe.StringData(aw.h.Path), len(aw.h.Path)), - ); err != nil { - return err - } else if psz := padSize(len(aw.h.Path)); psz != 0 { - if _, err = aw.w.Write(zero[:psz]); err != nil { - return err - } - } - return nil -} - -// Write writes p to the underlying writer and records the new position. Invalid -// positions are reported by WriteHeader and Close. -func (aw *Writer) Write(p []byte) (n int, err error) { - n, err = aw.w.Write(p) - aw.n += uint64(n) - return -} - -// Close concludes writing to the archive stream. -func (aw *Writer) Close() (err error) { - err = aw.flush() - aw.w = nil - return -} - -// ErrInsecurePath is returned by [FlatEntry.Decode] if validation is requested -// and a nonlocal path is encountered in the stream. -var ErrInsecurePath = errors.New("insecure file path") - -// Reader implements sequential reading of an archive. [Reader.Next] advances to -// the next file in the archive (including the first), and then Reader can be -// treated as an [io.Reader] to access the file's data. -type Reader struct { - // Underlying reader. - r io.Reader - // Fixed-size header segment. - buf [wordSize * 2]byte - // Remaining bytes in current data segment. - n, pad uint64 -} - -// NewReader returns the address of a new [Reader] reading from r. -func NewReader(r io.Reader) *Reader { return &Reader{r: r} } - -// Next advances ar to the next entry. Remaining bytes of the current data -// segment are discarded. Advancing beyond the final entry returns [io.EOF]. -func (ar *Reader) Next() (*ArchiveHeader, error) { - if dsz := int64(ar.n + ar.pad); dsz > 0 { - if n, err := io.CopyN(io.Discard, ar.r, dsz); err != nil { - if errors.Is(err, io.EOF) && n != dsz { - err = io.ErrUnexpectedEOF - } - return nil, err - } - } - - if _, err := io.ReadFull(ar.r, ar.buf[:]); err != nil { - return nil, err - } - - h := ArchiveHeader{ - Mode: fs.FileMode(binary.LittleEndian.Uint32(ar.buf[:])), - Size: binary.LittleEndian.Uint64(ar.buf[wordSize:]), - } - pathSize := int(binary.LittleEndian.Uint32(ar.buf[wordSize/2:])) - pPathSize := alignSize(pathSize) - - buf := make([]byte, pPathSize) - if _, err := io.ReadFull(ar.r, buf); err != nil { - if errors.Is(err, io.EOF) { - err = io.ErrUnexpectedEOF - } - return nil, err - } - - h.Path = unsafe.String(unsafe.SliceData(buf), pathSize) - if !filepath.IsLocal(h.Path) { - return &h, ErrInsecurePath - } - - ar.n = h.Size - ar.pad = padSize(h.Size) - return &h, nil -} - -// Read implements [io.Reader] for the data segment of the current entry. -func (ar *Reader) Read(p []byte) (n int, err error) { - if uint64(len(p)) > ar.n { - p = p[:ar.n] - } - - if len(p) > 0 { - n, err = ar.r.Read(p) - ar.n -= uint64(n) - } - - switch err { - case io.EOF: - if ar.n > 0 { - return n, io.ErrUnexpectedEOF - } - - case nil: - if ar.n == 0 { - return n, io.EOF - } - } - return -} - -// Write writes a deterministic representation of the contents of fsys to w. -// The resulting data can be hashed to produce a deterministic checksum for the -// directory. -func Write(fsys fs.FS, root string, w io.Writer) error { - aw := NewWriter(w) - if err := fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - var fi fs.FileInfo - fi, err = d.Info() - if err != nil { - return err - } - - h := ArchiveHeader{ - Path: path, - Mode: fi.Mode(), - } - if h.Mode.IsRegular() { - h.Size = uint64(fi.Size()) - if err = aw.WriteHeader(&h); err != nil { - return err - } - - var r fs.File - r, err = fsys.Open(path) - if err != nil { - return err - } - _, err = io.Copy(aw, r) - if _err := r.Close(); err == nil { - err = _err - } - return err - } else if h.Mode&fs.ModeSymlink != 0 { - var newpath string - if newpath, err = fs.ReadLink(fsys, path); err != nil { - return err - } - - h.Size = uint64(len(newpath)) - if err = aw.WriteHeader(&h); err != nil { - return err - } - - _, err = aw.Write(unsafe.Slice(unsafe.StringData(newpath), len(newpath))) - return err - } else if !h.Mode.IsDir() { - return InvalidFileModeError(h.Mode) - } - return aw.WriteHeader(&h) - }); err != nil { - return err - } - return aw.Close() -} - -// SumFS saves checksum of the archive of fsys to the value pointed to by buf. -func SumFS(buf *Checksum, fsys fs.FS, root string) error { - h := sha512.New384() - if err := Write(fsys, root, h); err != nil { - return err - } - h.Sum(buf[:0]) - return nil -} - -// SumDir saves checksum of the archive of directory at pathname to the value -// pointed to by buf. -func SumDir(buf *Checksum, pathname *check.Absolute) error { - return SumFS(buf, os.DirFS(pathname.String()), ".") -} - -// archiveArtifact is an [Artifact] unpacking an archive supported by [Reader] -// backed by a [FileArtifact]. -type archiveArtifact struct { - // Caller-supplied backing archive. - f Artifact -} - -var _ CuresExempt = archiveArtifact{} - -// NewArchive returns a new [Artifact] backed by the supplied [Artifact]. The -// source [Artifact] must be a [FileArtifact] and produce a stream compatible -// with [Reader]. -func NewArchive(a Artifact) Artifact { - return archiveArtifact{a} -} - -// Kind returns the hardcoded [Kind] constant. -func (archiveArtifact) Kind() Kind { return KindArchive } - -// Params is a noop. -func (archiveArtifact) Params(*IContext) {} - -func init() { - register(KindArchive, func(r *IRReader) Artifact { - a := NewArchive(r.Next()) - if _, ok := r.Finalise(); ok { - panic(ErrUnexpectedChecksum) - } - return a - }) -} - -// Inputs returns a slice containing the backing file. -func (a archiveArtifact) Inputs() []Artifact { - return []Artifact{a.f} -} - -// IsExclusive returns false: [Reader] is fully sequential. -func (archiveArtifact) IsExclusive() bool { return false } - -// Cure cures the [Artifact], producing a directory located at work. -func (a archiveArtifact) Cure(t *TContext) (err error) { - var r io.ReadCloser - if r, err = t.Open(a.f); err != nil { - return - } - - defer func() { - closeErr := r.Close() - if err == nil { - err = closeErr - } - }() - - type dirTargetPerm struct { - path string - mode fs.FileMode - } - var madeDirectories []dirTargetPerm - - if err = os.MkdirAll(t.GetWorkDir().String(), 0700); err != nil { - return - } - var root *os.Root - if root, err = os.OpenRoot(t.GetWorkDir().String()); err != nil { - return - } - defer func() { - closeErr := root.Close() - if err == nil { - err = closeErr - } - }() - - var header *ArchiveHeader - ar := NewReader(r) - for header, err = ar.Next(); err == nil; header, err = ar.Next() { - if header.Mode.IsRegular() { - var f *os.File - if f, err = root.OpenFile( - header.Path, - os.O_CREATE|os.O_EXCL|os.O_WRONLY, - header.Mode.Perm(), - ); err != nil { - return - } - if _, err = io.Copy(f, ar); err != nil { - _ = f.Close() - return - } else if err = f.Close(); err != nil { - return - } - } else if header.Mode&fs.ModeSymlink != 0 { - var p []byte - if p, err = io.ReadAll(ar); err != nil { - return - } - - if err = root.Symlink( - unsafe.String(unsafe.SliceData(p), len(p)), - header.Path, - ); err != nil { - return - } - } else if header.Mode.IsDir() { - if header.Path == "." { - continue - } - - madeDirectories = append(madeDirectories, dirTargetPerm{ - path: header.Path, - mode: header.Mode, - }) - if err = root.Mkdir(header.Path, 0700); err != nil { - return - } - } else { - return InvalidFileModeError(header.Mode) - } - } - if errors.Is(err, io.EOF) { - err = nil - } - if err == nil { - for _, e := range madeDirectories { - if err = root.Chmod(e.path, e.mode.Perm()); err != nil { - return - } - } - } else { - return - } - return -} - -// CuresExempt exempts the cheap [KindArchive] implementation often found at -// the end of a [FileArtifact] pipeline. -func (archiveArtifact) CuresExempt() {} diff --git a/internal/pkg/archive_test.go b/internal/pkg/archive_test.go deleted file mode 100644 index 1cb4bea9..00000000 --- a/internal/pkg/archive_test.go +++ /dev/null @@ -1,240 +0,0 @@ -package pkg_test - -import ( - "bytes" - "io" - "io/fs" - "maps" - "reflect" - "testing" - "testing/fstest" - "unsafe" - - "hakurei.app/check" - "hakurei.app/internal/pkg" -) - -func TestArchive(t *testing.T) { - t.Parallel() - - type entry struct { - path string - mode fs.FileMode - data string - } - testCases := []struct { - name string - fsys fs.FS - entries []entry - sum pkg.Checksum - err error - }{ - {"bad type", fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0700}, - "invalid": {Mode: fs.ModeCharDevice | 0400}, - }, nil, pkg.Checksum{}, pkg.InvalidFileModeError( - fs.ModeCharDevice | 0400, - )}, - - {"coldboot", fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "devices": {Mode: fs.ModeDir | 0700}, - "devices/uevent": {Mode: 0600, Data: []byte("add")}, - "devices/empty": {Mode: fs.ModeDir | 0700}, - - "devices/sub": {Mode: fs.ModeDir | 0700}, - "devices/sub/uevent": {Mode: 0600, Data: []byte("add")}, - - "block": {Mode: fs.ModeDir | 0700}, - "block/uevent": {Mode: 0600}, - }, []entry{ - {".", fs.ModeDir | 0700, ""}, - - {"block", fs.ModeDir | 0700, ""}, - {"block/uevent", 0600, ""}, - - {"devices", fs.ModeDir | 0700, ""}, - {"devices/empty", fs.ModeDir | 0700, ""}, - {"devices/sub", fs.ModeDir | 0700, ""}, - {"devices/sub/uevent", 0600, "add"}, - {"devices/uevent", 0600, "add"}, - }, pkg.MustDecode("mEy_Lf5KotThm7OwMx7yTKZh5HCCyaB41pVAvI9uDMgVQFM91iosBLYsRm8bDsX8"), nil}, - - {"empty", fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, []entry{ - {".", fs.ModeDir | 0700, ""}, - {"checksum", fs.ModeDir | 0700, ""}, - {"identifier", fs.ModeDir | 0700, ""}, - {"work", fs.ModeDir | 0700, ""}, - }, pkg.MustDecode("E4vEZKhCcL2gPZ2Tt59FS3lDng-d_2SKa2i5G_RbDfwGn6EemptFaGLPUDiOa94C"), nil}, - - {"sample directory step garbage", fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "lib": {Mode: fs.ModeDir | 0500}, - "lib/check": {Mode: 0400}, - - "lib/pkgconfig": {Mode: fs.ModeDir | 0500}, - }, []entry{ - {".", fs.ModeDir | 0500, ""}, - - {"lib", fs.ModeDir | 0500, ""}, - {"lib/check", 0400, ""}, - - {"lib/pkgconfig", fs.ModeDir | 0500, ""}, - }, pkg.MustDecode("CUx-3hSbTWPsbMfDhgalG4Ni_GmR9TnVX8F99tY_P5GtkYvczg9RrF5zO0jX9XYT"), nil}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - t.Run("roundtrip", func(t *testing.T) { - t.Parallel() - - var buf bytes.Buffer - if err := pkg.Write( - tc.fsys, - ".", - &buf, - ); !reflect.DeepEqual(err, tc.err) { - t.Fatalf("Flatten: error = %v, want %v", err, tc.err) - } else if tc.err != nil { - return - } - - r := pkg.NewReader(bytes.NewReader(buf.Bytes())) - var got []entry - for { - h, err := r.Next() - if err != nil { - if err == io.EOF { - break - } - t.Fatalf("Next: error = %v", err) - } - - var data []byte - if data, err = io.ReadAll(r); err != nil { - t.Fatalf("Read: error = %v", err) - } - - got = append(got, entry{ - path: h.Path, - mode: h.Mode, - data: unsafe.String(unsafe.SliceData(data), len(data)), - }) - } - - if !reflect.DeepEqual(got, tc.entries) { - t.Fatalf("Reader: %#v, want %#v", got, tc.entries) - } - }) - - if tc.err != nil { - return - } - - t.Run("hash", func(t *testing.T) { - t.Parallel() - - var got pkg.Checksum - if err := pkg.SumFS(&got, tc.fsys, "."); err != nil { - t.Fatalf("SumFS: error = %v", err) - } else if got != tc.sum { - t.Fatalf("SumFS: %v", &pkg.ChecksumMismatchError{ - Got: got, - Want: tc.sum, - }) - } - }) - }) - } -} - -var archiveTestdata = fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "devices": {Mode: fs.ModeDir | 0700}, - "devices/uevent": {Mode: 0600, Data: []byte("add")}, - "devices/empty": {Mode: fs.ModeDir | 0700}, - - "devices/sub": {Mode: fs.ModeDir | 0700}, - "devices/sub/uevent": {Mode: 0600, Data: []byte("add")}, - - "block": {Mode: fs.ModeDir | 0700}, - "block/uevent": {Mode: 0600}, -} - -func TestArchiveArtifact(t *testing.T) { - t.Parallel() - - want := maps.Clone(archiveTestdata) - want["."].Mode = fs.ModeDir | 0500 - - checkWithCache(t, []cacheTestCase{ - {"unpack", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - var buf bytes.Buffer - if err := pkg.Write(archiveTestdata, ".", &buf); err != nil { - t.Fatal(err) - } - - cureMany(t, c, []cureStep{ - {"sample", pkg.NewArchive( - pkg.NewFile("", buf.Bytes()), - ), ignorePathname, expectsFS(want), pkg.WNew, nil}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F": {Mode: fs.ModeDir | 0500}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/block": {Mode: fs.ModeDir | 0700}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/block/uevent": {Mode: 0600}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices": {Mode: fs.ModeDir | 0700}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices/empty": {Mode: fs.ModeDir | 0700}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices/sub": {Mode: fs.ModeDir | 0700}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices/sub/uevent": {Mode: 0600, Data: []byte("add")}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices/uevent": {Mode: 0600, Data: []byte("add")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/-zXKfphyWM2Ko7VMVURaEMSuixNokFSe0xFbnwR2RtRtvMrcPGWalV0LIn45PXTY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} - -func BenchmarkArchiveRead(b *testing.B) { - var buf bytes.Buffer - if err := pkg.Write(archiveTestdata, ".", &buf); err != nil { - b.Fatal(err) - } - testdata := buf.Bytes() - - for b.Loop() { - r := pkg.NewReader(bytes.NewReader(testdata)) - for { - _, err := r.Next() - if err != nil { - if err == io.EOF { - break - } - b.Fatal(err) - } - } - } -} - -func BenchmarkArchiveWrite(b *testing.B) { - for b.Loop() { - if err := pkg.Write(archiveTestdata, ".", io.Discard); err != nil { - b.Fatal(err) - } - } -} diff --git a/internal/pkg/clean.go b/internal/pkg/clean.go deleted file mode 100644 index f07706cf..00000000 --- a/internal/pkg/clean.go +++ /dev/null @@ -1,161 +0,0 @@ -package pkg - -import ( - "errors" - "os" - "unique" -) - -// Clean destroys checksum backing entries without any identifier or substitute -// entry referring to it. If at least one keep [Artifact] is specified, -// identifier and substitute entries not kept alive by them are destroyed first. -func (c *Cache) Clean(dry, inputs bool, keep ...Artifact) ( - []unique.Handle[ID], - []unique.Handle[Checksum], - error, -) { - c.identMu.Lock() - defer c.identMu.Unlock() - - c.checksumMu.Lock() - defer c.checksumMu.Unlock() - - dents, err := os.ReadDir(c.base.Append(dirChecksum).String()) - if err != nil { - return nil, nil, err - } - checksums := make(map[unique.Handle[Checksum]]string, len(dents)) - var buf Checksum - for _, dent := range dents { - name := dent.Name() - if err = Decode(&buf, name); err != nil { - return nil, nil, err - } - checksums[unique.Make(buf)] = name - } - - type identPair struct { - id unique.Handle[ID] - name string - } - dents, err = os.ReadDir(c.base.Append(dirIdentifier).String()) - if err != nil { - return nil, nil, err - } - - keepIdents := make(map[unique.Handle[ID]]struct{}) - if inputs { - for _, id := range Inputs((*Collect)(&keep)) { - keepIdents[id] = struct{}{} - } - } else { - for _, a := range keep { - keepIdents[c.Ident(a)] = struct{}{} - } - } - - idents := make([]identPair, 0, len(dents)) - for _, dent := range dents { - name := dent.Name() - if err = Decode(&buf, name); err != nil { - return nil, nil, err - } - id := unique.Make(ID(buf)) - - if _, ok := keepIdents[id]; len(keep) == 0 || ok { - if err = readlinkChecksum(c.base.Append( - dirIdentifier, - name, - ), &buf); err != nil { - return nil, nil, err - } - delete(checksums, unique.Make(buf)) - continue - } - - c.msg.Verbosef( - "arranging for destruction of %s%s%s...", - c.sgrIdent, name, c.sgrRes, - ) - idents = append(idents, identPair{id, name}) - } - - destroyedIdents := make([]unique.Handle[ID], 0, len(idents)) - for _, pair := range idents { - if !dry { - if err = os.Remove(c.base.Append( - dirStatus, - pair.name, - ).String()); err != nil && !errors.Is(err, os.ErrNotExist) { - return destroyedIdents, nil, err - } - - if err = os.Remove(c.base.Append( - dirIdentifier, - pair.name, - ).String()); err != nil { - return destroyedIdents, nil, err - } - } - destroyedIdents = append(destroyedIdents, pair.id) - } - - destroyedChecksums := make([]unique.Handle[Checksum], 0, len(checksums)) - for checksum, name := range checksums { - if err = c.parent.Err(); err != nil { - return destroyedIdents, destroyedChecksums, err - } - c.msg.Verbosef( - "destroying checksum %s%s%s...", - c.sgrIdent, name, c.sgrRes, - ) - if !dry { - if err = errors.Join(removeAll(c.base.Append( - dirChecksum, - name, - ))); err != nil { - return destroyedIdents, destroyedChecksums, err - } - } - destroyedChecksums = append(destroyedChecksums, checksum) - } - - dents, err = os.ReadDir(c.base.Append(dirSubstitute).String()) - if err != nil { - return destroyedIdents, destroyedChecksums, err - } - for _, dent := range dents { - name := dent.Name() - if err = readlinkChecksum(c.base.Append( - dirSubstitute, - name, - ), &buf); err != nil { - return destroyedIdents, destroyedChecksums, err - } - if _, ok := checksums[unique.Make(buf)]; !ok { - continue - } - - c.msg.Verbosef( - "destroying substitute %s%s%s...", - c.sgrIdent, name, c.sgrRes, - ) - if !dry { - if err = os.Remove(c.base.Append( - dirStatus, - name, - ).String()); err != nil && !errors.Is(err, os.ErrNotExist) { - return destroyedIdents, nil, err - } - - if err = os.Remove(c.base.Append( - dirSubstitute, - name, - ).String()); err != nil { - return destroyedIdents, destroyedChecksums, err - } - } - } - - return destroyedIdents, destroyedChecksums, nil -} diff --git a/internal/pkg/clean_test.go b/internal/pkg/clean_test.go deleted file mode 100644 index 67dceb09..00000000 --- a/internal/pkg/clean_test.go +++ /dev/null @@ -1,293 +0,0 @@ -package pkg_test - -import ( - "bytes" - "crypto/sha512" - "io/fs" - "log" - "os" - "slices" - "strings" - "testing" - "unique" - - "hakurei.app/internal/pkg" - "hakurei.app/message" -) - -// formatHandles returns a user-facing string representing h. -func formatHandles[T pkg.ID | pkg.Checksum](handles ...unique.Handle[T]) string { - var buf strings.Builder - for _, h := range handles { - buf.WriteString(pkg.Encode(pkg.Checksum(h.Value()))) - buf.WriteString(", ") - } - return strings.TrimSuffix(buf.String(), ", ") -} - -func TestClean(t *testing.T) { - t.Parallel() - ic := pkg.NewIR() - - testCases := []struct { - name string - a []pkg.Artifact - keep []pkg.Artifact - inputs bool - want expectsFS - - wantIdents []unique.Handle[pkg.ID] - wantChecksums []unique.Handle[pkg.Checksum] - }{ - {"simple", []pkg.Artifact{ - pkg.NewFile("file", nil), - }, nil, false, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/h6qx7m0vsyhO6oLEJgY2rKa1c6RZuIZ4QXxZSh__NZJ9VU0MIj5bFB-KuaPcsfpE": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")}, - - "lock": {Mode: 0644}, - "variant": {Mode: 0400}, - "status": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "fault": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, nil, nil}, - - {"keep", []pkg.Artifact{ - pkg.NewFile("removed-file", []byte("removed file")), - }, []pkg.Artifact{ - pkg.NewFile("file", []byte("\xfd")), - }, false, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/KgZ-FjbGuU-XP2QEHInpgv-2Zn0cTH5NqFMgTU0XrSdKmSwyC-3baVs1BMCP5spk": {Mode: 0400, Data: []byte("\xfd")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/tkrOYhQfTRaArk4nNYNjTmHQUmuJMVIoDeAKIQBm3I4uuMimBKXKMIY2gGZqwQg4": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/KgZ-FjbGuU-XP2QEHInpgv-2Zn0cTH5NqFMgTU0XrSdKmSwyC-3baVs1BMCP5spk")}, - - "lock": {Mode: 0644}, - "variant": {Mode: 0400}, - "status": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "fault": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, []unique.Handle[pkg.ID]{ - ic.Ident(pkg.NewFile("removed-file", []byte("removed file"))), - }, []unique.Handle[pkg.Checksum]{ - unique.Make(sha512.Sum384([]byte("removed file"))), - }}, - - {"inputs anchored substitute", []pkg.Artifact{ - &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("destroyed"), - deps: []pkg.Artifact{ - pkg.NewFile("destroyed-input", []byte("destroyed")), - }, - cure: func(f *pkg.FContext) error { - p := f.GetWorkDir() - if err := os.MkdirAll(p.String(), 0755); err != nil { - return err - } - return os.WriteFile(p.Append("result").String(), nil, 0444) - }, - }, - }, []pkg.Artifact{ - &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("kept"), - deps: []pkg.Artifact{ - pkg.NewFile("kept-input", []byte("kept")), - }, - cure: func(f *pkg.FContext) error { - p := f.GetWorkDir() - if err := os.MkdirAll(p.String(), 0755); err != nil { - return err - } - return os.WriteFile(p.Append("result").String(), nil, 0444) - }, - }, - }, true, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/H-eSiCo227-xdqyNl2R-5G3eqXPtbb8XegAB70I5OQb2majeZXJoCxTq9wJy5qqv": {Mode: 0400, Data: []byte("kept")}, - "checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE": {Mode: fs.ModeDir | 0500}, - "checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE/result": {Mode: 0444}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/KpxTUsUfGbsX6C6HU2j9cd9He3_tt2o2m2MgH4m0zial8lmoqSWY-Gc3KrS0k-UK": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/H-eSiCo227-xdqyNl2R-5G3eqXPtbb8XegAB70I5OQb2majeZXJoCxTq9wJy5qqv")}, - "identifier/xxQ43VMIS1WHMSEFnUdBNd6pulbtgiTorP_mtJpEsZENYhGBiDEL6Y2D8yqn0Sra": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")}, - - "lock": {Mode: 0644}, - "variant": {Mode: 0400}, - "status": {Mode: fs.ModeDir | 0700}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "substitute/dP-6_wIDRRouOaOF-nkBy-IaUbLdYHbOUTrptu7j_qfW01mnHSjYJ0oykUmvUd2x": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")}, - "substitute/td1tf1kb3z8iUFg2k1xlHeTTEz3xR4e77WSWdv2JhjmZyY6iuvodhdT8BblDOi0E": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")}, - - "fault": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, []unique.Handle[pkg.ID]{ - ic.Ident(pkg.NewFile("destroyed-input", []byte("destroyed"))), - ic.Ident(&stubArtifactF{ - kind: pkg.KindExec, - params: []byte("destroyed"), - deps: []pkg.Artifact{ - pkg.NewFile("destroyed-input", []byte("destroyed")), - }, - }), - }, []unique.Handle[pkg.Checksum]{ - unique.Make(sha512.Sum384([]byte("destroyed"))), - }}, - - {"inputs", []pkg.Artifact{ - &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("destroyed"), - deps: []pkg.Artifact{ - pkg.NewFile("destroyed-input", []byte("destroyed")), - }, - cure: func(f *pkg.FContext) error { - if w, err := f.GetStatusWriter(); err != nil { - return err - } else if _, err = w.Write([]byte("destroyed")); err != nil { - return err - } - - p := f.GetWorkDir() - if err := os.MkdirAll(p.String(), 0755); err != nil { - return err - } - return os.WriteFile(p.Append("result").String(), nil, 0444) - }, - }, - }, []pkg.Artifact{ - &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("kept"), - deps: []pkg.Artifact{ - pkg.NewFile("kept-input", []byte("kept")), - }, - cure: func(f *pkg.FContext) error { - if w, err := f.GetStatusWriter(); err != nil { - return err - } else if _, err = w.Write([]byte("kept")); err != nil { - return err - } - - p := f.GetWorkDir() - if err := os.MkdirAll(p.String(), 0755); err != nil { - return err - } - return os.WriteFile(p.Append("result").String(), []byte{0}, 0444) - }, - }, - }, true, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/CyDnDvF-LaeGPcSW70tPosNCoclByWkTjznUUF1DcgzlIwkN9yzz1ZFME1TlPj6W": {Mode: fs.ModeDir | 0500}, - "checksum/CyDnDvF-LaeGPcSW70tPosNCoclByWkTjznUUF1DcgzlIwkN9yzz1ZFME1TlPj6W/result": {Mode: 0444, Data: []byte("\x00")}, - "checksum/H-eSiCo227-xdqyNl2R-5G3eqXPtbb8XegAB70I5OQb2majeZXJoCxTq9wJy5qqv": {Mode: 0400, Data: []byte("kept")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/KpxTUsUfGbsX6C6HU2j9cd9He3_tt2o2m2MgH4m0zial8lmoqSWY-Gc3KrS0k-UK": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/H-eSiCo227-xdqyNl2R-5G3eqXPtbb8XegAB70I5OQb2majeZXJoCxTq9wJy5qqv")}, - "identifier/xxQ43VMIS1WHMSEFnUdBNd6pulbtgiTorP_mtJpEsZENYhGBiDEL6Y2D8yqn0Sra": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/CyDnDvF-LaeGPcSW70tPosNCoclByWkTjznUUF1DcgzlIwkN9yzz1ZFME1TlPj6W")}, - - "lock": {Mode: 0644}, - "variant": {Mode: 0400}, - - "status": {Mode: fs.ModeDir | 0700}, - "status/xxQ43VMIS1WHMSEFnUdBNd6pulbtgiTorP_mtJpEsZENYhGBiDEL6Y2D8yqn0Sra": {Mode: 0400, Data: []byte(statusHeader + "kept")}, - "status/td1tf1kb3z8iUFg2k1xlHeTTEz3xR4e77WSWdv2JhjmZyY6iuvodhdT8BblDOi0E": {Mode: 0400, Data: []byte(statusHeader + "kept")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "substitute/td1tf1kb3z8iUFg2k1xlHeTTEz3xR4e77WSWdv2JhjmZyY6iuvodhdT8BblDOi0E": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/CyDnDvF-LaeGPcSW70tPosNCoclByWkTjznUUF1DcgzlIwkN9yzz1ZFME1TlPj6W")}, - - "fault": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, []unique.Handle[pkg.ID]{ - ic.Ident(pkg.NewFile("destroyed-input", []byte("destroyed"))), - ic.Ident(&stubArtifactF{ - kind: pkg.KindExec, - params: []byte("destroyed"), - deps: []pkg.Artifact{ - pkg.NewFile("destroyed-input", []byte("destroyed")), - }, - }), - }, []unique.Handle[pkg.Checksum]{ - unique.Make(expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - "result": {Mode: 0444}, - }.hash()), - unique.Make(sha512.Sum384([]byte("destroyed"))), - }}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - base := makeBase(t) - msg := message.New(log.New(os.Stderr, "clean: ", 0)) - msg.SwapVerbose(testing.Verbose()) - c, err := pkg.Open(t.Context(), msg, base, nil) - if err != nil { - t.Fatal(err) - } - t.Cleanup(c.Close) - - all := pkg.Collect(slices.Concat(tc.a, tc.keep)) - if _, _, err = c.Cure(&all); !pkg.IsCollected(err) { - t.Fatal(err) - } - - var ( - idents []unique.Handle[pkg.ID] - checksums []unique.Handle[pkg.Checksum] - ) - idents, checksums, err = c.Clean(false, tc.inputs, tc.keep...) - if err != nil { - t.Fatalf("Clean: error = %v", err) - } - var buf [2]pkg.Checksum - - slices.SortFunc(idents, func(a, b unique.Handle[pkg.ID]) int { - buf[0], buf[1] = a.Value(), b.Value() - return bytes.Compare(buf[0][:], buf[1][:]) - }) - slices.SortFunc(checksums, func(a, b unique.Handle[pkg.Checksum]) int { - buf[0], buf[1] = a.Value(), b.Value() - return bytes.Compare(buf[0][:], buf[1][:]) - }) - - if !slices.Equal(idents, tc.wantIdents) { - t.Errorf( - "Clean: idents = %s, want %s", - formatHandles(idents...), formatHandles(tc.wantIdents...), - ) - } - if !slices.Equal(checksums, tc.wantChecksums) { - t.Errorf( - "Clean: checksums = %s, want %s", - formatHandles(checksums...), formatHandles(tc.wantChecksums...), - ) - } - - want := tc.want.hash() - var checksum pkg.Checksum - if err = pkg.SumDir(&checksum, base); err != nil { - t.Fatalf("SumDir: error = %v", err) - } else if checksum != want { - t.Error(expectsFrom(base.String())) - } - }) - } -} diff --git a/internal/pkg/compress.go b/internal/pkg/compress.go deleted file mode 100644 index 22f990f1..00000000 --- a/internal/pkg/compress.go +++ /dev/null @@ -1,151 +0,0 @@ -package pkg - -import ( - "compress/bzip2" - "compress/gzip" - "fmt" - "io" - "os" - - "hakurei.app/internal/xz" - "hakurei.app/internal/zstd" -) - -const ( - // Gzip denotes a stream compressed via [gzip]. - Gzip = iota - // Bzip2 denotes a stream compressed via [bzip2]. - Bzip2 - // Zstd denotes a stream compressed via [zstd]. - Zstd - // XZ denotes a stream compressed via [xz]. - XZ -) - -// A decompressArtifact is a [FileArtifact] decompressing a backing -// [FileArtifact] stream. -type decompressArtifact struct { - // Caller-supplied backing stream. - f Artifact - // Compression on top of the stream. - compress uint32 -} - -var _ FileArtifact = new(decompressArtifact) -var _ CuresExempt = new(decompressArtifact) - -// decompressArtifactNamed embeds decompressArtifact for a [fmt.Stringer] stream. -type decompressArtifactNamed struct { - decompressArtifact - // Copied from decompressArtifact.f. - name string -} - -var _ fmt.Stringer = new(decompressArtifactNamed) - -// NewDecompress returns a [FileArtifact] decompressing the supplied [Artifact]. -func NewDecompress(a Artifact, compress uint32) FileArtifact { - da := decompressArtifact{a, compress} - if s, ok := a.(fmt.Stringer); ok { - if name := s.String(); name != "" { - return &decompressArtifactNamed{da, name} - } - } - return &da -} - -// String returns the name of the underlying [Artifact] prefixed with decompress. -func (a *decompressArtifactNamed) String() string { return "decompress-" + a.name } - -// Kind returns the hardcoded [Kind] constant. -func (a *decompressArtifact) Kind() Kind { return KindDecompress } - -// Params writes value of compression enum. -func (a *decompressArtifact) Params(ctx *IContext) { ctx.WriteUint32(a.compress) } - -func init() { - register(KindDecompress, func(r *IRReader) Artifact { - a := NewDecompress(r.Next(), r.ReadUint32()) - if _, ok := r.Finalise(); ok { - panic(ErrUnexpectedChecksum) - } - return a - }) -} - -// Inputs returns a slice containing the backing file. -func (a *decompressArtifact) Inputs() []Artifact { - return []Artifact{a.f} -} - -// IsExclusive returns false: decompressor is fully sequential. -func (a *decompressArtifact) IsExclusive() bool { return false } - -// compoundCloser is an [io.ReadCloser] with an additional [io.Closer] attached. -type compoundCloser struct { - io.ReadCloser - c io.Closer -} - -// Close closes [io.ReadCloser] and the additional [io.Closer]. It returns the -// non-nil error returned by the underlying [io.ReadCloser], otherwise it -// returns the error returned by the additional [io.Closer]. -func (c compoundCloser) Close() error { - err := c.ReadCloser.Close() - if _err := c.c.Close(); err == nil { - err = _err - } - return err -} - -// IsExecutable returns false. -func (*decompressArtifact) IsExecutable() bool { return false } - -// Cure returns a decompressor [io.ReadCloser]. -func (a *decompressArtifact) Cure(r *RContext) (io.ReadCloser, error) { - sr, err := r.Open(a.f) - if err != nil { - return nil, err - } - br := r.cache.getReaderRC(sr) - - var dr io.ReadCloser - switch a.compress { - case Gzip: - if dr, err = gzip.NewReader(br); err != nil { - _ = br.Close() - return nil, err - } - return compoundCloser{dr, br}, nil - - case Bzip2: - return struct { - io.Reader - io.Closer - }{bzip2.NewReader(br), br}, nil - - case Zstd: - return struct { - io.Reader - io.Closer - }{zstd.NewReader(br), br}, nil - - case XZ: - var _dr io.Reader - if _dr, err = xz.NewReader(br, 0); err != nil { - _ = br.Close() - return nil, err - } - return struct { - io.Reader - io.Closer - }{_dr, br}, nil - - default: - return nil, os.ErrInvalid - } -} - -// CuresExempt exempts the cheap [KindDecompress] implementation often part of -// a [FileArtifact] pipeline. -func (*decompressArtifact) CuresExempt() {} diff --git a/internal/pkg/compress_test.go b/internal/pkg/compress_test.go deleted file mode 100644 index 7ce00edf..00000000 --- a/internal/pkg/compress_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package pkg_test - -import ( - "bytes" - "compress/gzip" - "crypto/sha512" - "io/fs" - "net/http" - "testing" - "testing/fstest" - - "hakurei.app/check" - "hakurei.app/internal/pkg" -) - -func TestDecompress(t *testing.T) { - t.Parallel() - - var buf bytes.Buffer - gw := gzip.NewWriter(&buf) - if _, err := gw.Write([]byte{0}); err != nil { - t.Fatal(err) - } else if err = gw.Close(); err != nil { - t.Fatal(err) - } - testdata := buf.String() - - var transport http.Transport - client := http.Client{Transport: &transport} - transport.RegisterProtocol("file", http.NewFileTransportFS(fstest.MapFS{ - "testdata": {Data: []byte(testdata), Mode: 0400}, - })) - testdataChecksum := func() pkg.Checksum { - h := sha512.New384() - h.Write([]byte(testdata)) - return (pkg.Checksum)(h.Sum(nil)) - }() - - gh := pkg.NewDecompress(pkg.NewHTTPGet( - &client, - "file:///testdata", - testdataChecksum, - ), pkg.Gzip) - - checkWithCache(t, []cacheTestCase{ - {"decompress", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - cureMany(t, c, []cureStep{ - {"close", pkg.NewDecompress(pkg.NewHTTPGet( - &client, - "file:///testdata", - pkg.Checksum{0xfd}, - ), pkg.Gzip), nil, nil, pkg.WNew, &pkg.ChecksumMismatchError{ - Got: testdataChecksum, - Want: pkg.Checksum{0xfd}, - }}, - - {"gzip", gh, ignorePathname, expectsChecksum(sha512.Sum384([]byte{0})), pkg.WNew, nil}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + pkg.Encode(sha512.Sum384([]byte{0})): {Mode: 0400, Data: []byte{0}}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/" + pkg.Encode(pkg.NewIR().Ident(gh).Value()): {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} diff --git a/internal/pkg/exec.go b/internal/pkg/exec.go deleted file mode 100644 index b40ae84b..00000000 --- a/internal/pkg/exec.go +++ /dev/null @@ -1,808 +0,0 @@ -package pkg - -import ( - "bufio" - "bytes" - "context" - "errors" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "runtime" - "slices" - "strconv" - "sync" - "syscall" - "time" - "unique" - - "hakurei.app/check" - "hakurei.app/container" - "hakurei.app/container/seccomp" - "hakurei.app/container/std" - "hakurei.app/ext" - "hakurei.app/fhs" - "hakurei.app/message" -) - -// AbsWork is the container pathname [TContext.GetWorkDir] is mounted on. -var AbsWork = fhs.AbsRoot.Append("work/") - -const ( - // EnvJobs is the name of the environment variable holding a decimal - // representation of the preferred job count. Its value must not affect cure - // outcome. - EnvJobs = "CURE_JOBS" - // EnvLoad is the name of the environment variable holding a decimal - // representation of the preferred loadavg target. Its value must not affect - // cure outcome. - EnvLoad = "CURE_LOAD" -) - -// ExecPath is a slice of [Artifact] and the [check.Absolute] pathname to make -// it available at under in the container. -type ExecPath struct { - // Pathname in the container mount namespace. - P *check.Absolute - // Artifacts to mount on the pathname, must contain at least one [Artifact]. - // If there are multiple entries or W is true, P is set up as an overlay - // mount, and entries of A must not implement [FileArtifact]. - A []Artifact - // Whether to make the mount point writable via the temp directory. - W bool -} - -// GetArtifactFunc is the function signature of [FContext.GetArtifact]. -type GetArtifactFunc func(Artifact) (*check.Absolute, unique.Handle[Checksum]) - -// PromoteLayers returns artifacts with identical-by-content layers promoted to -// the highest priority instance, as if mounted via [ExecPath]. -func PromoteLayers( - artifacts []Artifact, - getArtifact GetArtifactFunc, - report func(i int, d Artifact), -) []*check.Absolute { - layers := make([]*check.Absolute, 0, len(artifacts)) - checksums := make(map[unique.Handle[Checksum]]struct{}, len(artifacts)) - for i := range artifacts { - d := artifacts[len(artifacts)-1-i] - pathname, checksum := getArtifact(d) - if _, ok := checksums[checksum]; ok { - report(len(artifacts)-1-i, d) - continue - } - checksums[checksum] = struct{}{} - layers = append(layers, pathname) - } - slices.Reverse(layers) - return layers -} - -// layers returns pathnames collected from A deduplicated via [PromoteLayers]. -func (p *ExecPath) layers( - msg message.Msg, - getArtifact GetArtifactFunc, - ident func(a Artifact) unique.Handle[ID], -) []*check.Absolute { - return PromoteLayers(p.A, getArtifact, func(i int, d Artifact) { - if msg.IsVerbose() { - msg.Verbosef("promoted layer %d as %s", i, reportName(d, ident(d))) - } - }) -} - -// Path returns a populated [ExecPath]. -func Path(pathname *check.Absolute, writable bool, a ...Artifact) ExecPath { - return ExecPath{pathname, a, writable} -} - -// MustPath is like [Path], but takes a string pathname via [check.MustAbs]. -func MustPath(pathname string, writable bool, a ...Artifact) ExecPath { - return ExecPath{check.MustAbs(pathname), a, writable} -} - -var ( - binfmt map[string]container.BinfmtEntry - binfmtMu sync.RWMutex -) - -// RegisterArch arranges for [KindExec] and [KindExecNet] to support a new -// architecture via a binfmt_misc entry. Each architecture must be registered -// at most once. -func RegisterArch(arch string, e container.BinfmtEntry) { - if arch == "" { - panic(UnsupportedArchError(arch)) - } - - binfmtMu.Lock() - defer binfmtMu.Unlock() - - if binfmt == nil { - binfmt = make(map[string]container.BinfmtEntry) - } - - if _, ok := binfmt[arch]; ok { - panic("attempting to register " + strconv.Quote(arch) + " twice") - } - binfmt[arch] = e -} - -const ( - // ExecTimeoutDefault replaces out of range [NewExec] timeout values. - ExecTimeoutDefault = 15 * time.Minute - // ExecTimeoutMax is the arbitrary upper bound of [NewExec] timeout. - ExecTimeoutMax = 48 * time.Hour -) - -// An execArtifact is an [Artifact] that produces output by running a program -// part of another [Artifact] in a [container] to produce its output. -// -// Methods of execArtifact does not modify any struct field or underlying arrays -// referred to by slices. -type execArtifact struct { - // Caller-supplied user-facing reporting name, guaranteed to be nonzero - // during initialisation. - name string - // Target architecture. - arch string - // Caller-supplied inner mount points. - paths []ExecPath - - // Passed through to [container.Params]. - dir *check.Absolute - // Passed through to [container.Params]. - env []string - // Passed through to [container.Params]. - path *check.Absolute - // Passed through to [container.Params]. - args []string - - // Duration the initial process is allowed to run. The zero value is - // equivalent to [ExecTimeoutDefault]. - timeout time.Duration - - // Caller-supplied exclusivity value, returned as is by IsExclusive. - exclusive bool -} - -var _ fmt.Stringer = new(execArtifact) - -// execMeasuredArtifact is like execArtifact but implements [KnownChecksum] and -// has its resulting container optionally keep the host net namespace. -type execMeasuredArtifact struct { - checksum Checksum - - // Whether to keep host net namespace. - hostNet bool - - execArtifact -} - -var _ KnownChecksum = new(execMeasuredArtifact) - -// Checksum returns the caller-supplied checksum. -func (a *execMeasuredArtifact) Checksum() Checksum { return a.checksum } - -// Kind returns [KindExecNet], or [KindExec] if hostNet is false. -func (a *execMeasuredArtifact) Kind() Kind { - if a == nil || a.hostNet { - return KindExecNet - } - return KindExec -} - -// Cure cures the [Artifact] in the container described by the caller. The -// container optionally retains host networking. -func (a *execMeasuredArtifact) Cure(f *FContext) error { - return a.cure(f, a.hostNet) -} - -// ErrNetChecksum is panicked by [NewExec] if host net namespace is requested -// with a nil checksum. -var ErrNetChecksum = errors.New("attempting to keep net namespace without checksum") - -// NewExec returns a new [Artifact] that executes the program path in a -// container with specified paths bind mounted read-only in order. A private -// instance of /proc and /dev is made available to the container. -// -// The working and temporary directories are both created and mounted writable -// on [AbsWork] and [fhs.AbsTmp] respectively. If one or more paths target -// [AbsWork], the final entry is set up as a writable overlay mount on /work for -// which the upperdir is the host side work directory. In this configuration, -// the W field is ignored, and the program must avoid causing whiteout files to -// be created. Cure fails if upperdir ends up with entries other than directory, -// regular or symlink. -// -// If checksum is non-nil, the resulting [Artifact] implements [KnownChecksum] -// and its container optionally runs in the host net namespace. -// -// The container is allowed to run for the specified duration before the initial -// process and all processes originating from it is terminated. A zero or -// negative timeout value is equivalent tp [ExecTimeoutDefault], a timeout value -// greater than [ExecTimeoutMax] is equivalent to [ExecTimeoutMax]. -// -// The user-facing name and exclusivity value are not accessible from the -// container and does not affect curing outcome. Because of this, it is omitted -// from parameter data for computing identifier. -func NewExec( - name, arch string, - checksum *Checksum, - timeout time.Duration, - hostNet, exclusive bool, - - dir *check.Absolute, - env []string, - pathname *check.Absolute, - args []string, - - paths ...ExecPath, -) Artifact { - if name == "" { - name = "exec-" + filepath.Base(pathname.String()) - } - if arch == "" { - arch = runtime.GOARCH - } - if timeout <= 0 { - timeout = ExecTimeoutDefault - } - if timeout > ExecTimeoutMax { - timeout = ExecTimeoutMax - } - a := execArtifact{name, arch, paths, dir, env, pathname, args, timeout, exclusive} - if checksum == nil { - if hostNet { - panic(ErrNetChecksum) - } - return &a - } - return &execMeasuredArtifact{*checksum, hostNet, a} -} - -// Kind returns the hardcoded [Kind] constant. -func (*execArtifact) Kind() Kind { return KindExec } - -// Params writes paths, executable pathname and args. -func (a *execArtifact) Params(ctx *IContext) { - ctx.WriteString(a.arch) - ctx.WriteString(a.name) - - ctx.WriteUint32(uint32(len(a.paths))) - for _, p := range a.paths { - if p.P != nil { - ctx.WriteString(p.P.String()) - } else { - ctx.WriteString("invalid P\x00") - } - - ctx.WriteUint32(uint32(len(p.A))) - for _, d := range p.A { - ctx.WriteIdent(d) - } - - if p.W { - ctx.WriteUint32(1) - } else { - ctx.WriteUint32(0) - } - } - - ctx.WriteString(a.dir.String()) - - ctx.WriteUint32(uint32(len(a.env))) - for _, e := range a.env { - ctx.WriteString(e) - } - - ctx.WriteString(a.path.String()) - - ctx.WriteUint32(uint32(len(a.args))) - for _, arg := range a.args { - ctx.WriteString(arg) - } - - ctx.WriteUint32(uint32(a.timeout & 0xffffffff)) - ctx.WriteUint32(uint32(a.timeout >> 32)) - - if a.exclusive { - ctx.WriteUint32(1) - } else { - ctx.WriteUint32(0) - } -} - -// UnsupportedArchError describes an unsupported or invalid architecture. -type UnsupportedArchError string - -func (e UnsupportedArchError) Error() string { - if e == "" { - return "invalid architecture name" - } - return "unsupported architecture " + string(e) -} - -// readExecArtifact interprets IR values and returns the address of execArtifact -// or execNetArtifact. -func readExecArtifact(r *IRReader, net bool) Artifact { - r.DiscardAll() - - arch := r.ReadString() - if arch == "" { - panic(UnsupportedArchError(arch)) - } - - name := r.ReadString() - - sz := r.ReadUint32() - if sz > irMaxDeps { - panic(ErrIRDepend) - } - paths := make([]ExecPath, sz) - for i := range paths { - paths[i].P = check.MustAbs(r.ReadString()) - - sz = r.ReadUint32() - if sz > irMaxDeps { - panic(ErrIRDepend) - } - paths[i].A = make([]Artifact, sz) - for j := range paths[i].A { - paths[i].A[j] = r.ReadIdent() - } - - paths[i].W = r.ReadUint32() != 0 - } - - dir := check.MustAbs(r.ReadString()) - - sz = r.ReadUint32() - if sz > irMaxValues { - panic(ErrIRValues) - } - env := make([]string, sz) - for i := range env { - env[i] = r.ReadString() - } - - pathname := check.MustAbs(r.ReadString()) - - sz = r.ReadUint32() - if sz > irMaxValues { - panic(ErrIRValues) - } - args := make([]string, sz) - for i := range args { - args[i] = r.ReadString() - } - - timeout := time.Duration(r.ReadUint32()) - timeout |= time.Duration(r.ReadUint32()) << 32 - - exclusive := r.ReadUint32() != 0 - - checksum, ok := r.Finalise() - var checksumP *Checksum - if ok { - checksumP = new(checksum.Value()) - } - - if net && !ok { - panic(ErrExpectedChecksum) - } - - return NewExec( - name, arch, checksumP, timeout, net, exclusive, dir, env, pathname, args, paths..., - ) -} - -func init() { - register(KindExec, - func(r *IRReader) Artifact { return readExecArtifact(r, false) }) - register(KindExecNet, - func(r *IRReader) Artifact { return readExecArtifact(r, true) }) -} - -// Inputs returns a slice of all artifacts collected from caller-supplied -// [ExecPath]. -func (a *execArtifact) Inputs() []Artifact { - artifacts := make([][]Artifact, 0, len(a.paths)) - for _, p := range a.paths { - artifacts = append(artifacts, p.A) - } - return slices.Concat(artifacts...) -} - -// IsExclusive returns the caller-supplied exclusivity value. -func (a *execArtifact) IsExclusive() bool { return a.exclusive } - -// String returns the caller-supplied reporting name. -func (a *execArtifact) String() string { return a.name } - -// Cure cures the [Artifact] in the container described by the caller. -func (a *execArtifact) Cure(f *FContext) (err error) { - return a.cure(f, false) -} - -const ( - // execWaitDelay is passed through to [container.Params]. - execWaitDelay = time.Nanosecond -) - -// scanLinesCR is like [bufio.ScanLines], but also treats a bare \r as an -// end-of-line marker. -func scanLinesCR(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF && len(data) == 0 { - return 0, nil, nil - } - ri, ni := bytes.IndexByte(data, '\r'), bytes.IndexByte(data, '\n') - - if ri >= 0 && (ni < 0 || ri < ni) { - if ri+1 == ni { - // We have a full \r\n-terminated line. - return ri + 2, data[:ri], nil - } - // We have a bare \r, probably some kind of progress indicator. - return ri + 1, data[:ri], nil - } - if ni >= 0 && (ri < 0 || ni < ri) { - // We have a full newline-terminated line. - return ni + 1, data[:ni], nil - } - // If we're at EOF, we have a final, non-terminated line. Return it. - if atEOF { - return len(data), data, nil - } - // Request more data. - return 0, nil, nil -} - -// scanVerbose prefixes program output for a verbose [message.Msg]. -func scanVerbose( - msg message.Msg, - cancel context.CancelFunc, - done chan<- struct{}, - prefix, suffix string, - r io.Reader, -) { - defer close(done) - s := bufio.NewScanner(r) - s.Split(scanLinesCR) - s.Buffer( - make([]byte, bufio.MaxScanTokenSize), - bufio.MaxScanTokenSize<<12, - ) - for s.Scan() { - msg.Verbose(prefix, s.Text()+suffix) - } - if err := s.Err(); err != nil && !errors.Is(err, os.ErrClosed) { - cancel() - msg.Verbose("*"+prefix, err.Error()+suffix) - } -} - -var ( - // ErrInvalidPaths is returned for an [Artifact] of [KindExec] or - // [KindExecNet] specified with invalid paths. - ErrInvalidPaths = errors.New("invalid mount point") -) - -// SeccompPresets is the [seccomp] presets used by exec artifacts. -const SeccompPresets = std.PresetStrict & - ^(std.PresetDenyNS | std.PresetDenyDevel) - -// makeContainer sets up the specified temp and work directories and returns the -// corresponding [container.Container] that would have run for cure. -func (a *execArtifact) makeContainer( - ctx context.Context, - msg message.Msg, - flags, jobs, load int, - hostNet bool, - temp, work *check.Absolute, - getArtifact GetArtifactFunc, - ident func(a Artifact) unique.Handle[ID], -) (z *container.Container, err error) { - overlayWorkIndex := -1 - for i, p := range a.paths { - if p.P == nil || len(p.A) == 0 { - return nil, ErrInvalidPaths - } - if p.P.Is(AbsWork) { - overlayWorkIndex = i - } - } - - var artifactCount int - for _, p := range a.paths { - artifactCount += len(p.A) - } - - z = container.New(ctx, msg) - z.WaitDelay = execWaitDelay - z.SeccompPresets = SeccompPresets - z.SeccompFlags |= seccomp.AllowMultiarch - z.ParentPerm = 0700 - z.HostNet = hostNet - z.HostAbstract = flags&CHostAbstract != 0 - z.Hostname = "cure" - z.SetScheduler = flags&CSchedIdle != 0 - z.SchedPolicy = ext.SCHED_IDLE - if z.HostNet { - z.Hostname = "cure-net" - } - z.Quiet = flags&CSuppressInit != 0 - z.Uid, z.Gid = (1<<10)-1, (1<<10)-1 - z.Dir, z.Path, z.Args = a.dir, a.path, a.args - z.Env = slices.Concat(a.env, []string{ - EnvJobs + "=" + strconv.Itoa(jobs), - EnvLoad + "=" + strconv.Itoa(load), - }) - z.Grow(len(a.paths) + 4) - - if a.arch != runtime.GOARCH { - binfmtMu.RLock() - e, ok := binfmt[a.arch] - binfmtMu.RUnlock() - if !ok { - return nil, UnsupportedArchError(a.arch) - } - z.Binfmt = []container.BinfmtEntry{e} - z.InitAsRoot = true - } - - for i, b := range a.paths { - if i == overlayWorkIndex { - if err = os.MkdirAll(work.String(), 0700); err != nil { - return - } - tempWork := temp.Append(".work") - if err = os.MkdirAll(tempWork.String(), 0700); err != nil { - return - } - z.Overlay( - AbsWork, - work, - tempWork, - b.layers(msg, getArtifact, ident)..., - ) - continue - } - - if a.paths[i].W { - tempUpper, tempWork := temp.Append( - ".upper", strconv.Itoa(i), - ), temp.Append( - ".work", strconv.Itoa(i), - ) - if err = os.MkdirAll(tempUpper.String(), 0700); err != nil { - return - } - if err = os.MkdirAll(tempWork.String(), 0700); err != nil { - return - } - z.Overlay(b.P, tempUpper, tempWork, b.layers(msg, getArtifact, ident)...) - } else if len(b.A) == 1 { - pathname, _ := getArtifact(b.A[0]) - z.Bind(pathname, b.P, 0) - } else { - z.OverlayReadonly(b.P, b.layers(msg, getArtifact, ident)...) - } - } - if overlayWorkIndex < 0 { - z.Bind( - work, - AbsWork, - std.BindWritable|std.BindEnsure, - ) - } - z.Bind( - temp, - fhs.AbsTmp, - std.BindWritable|std.BindEnsure, - ) - z.Proc(fhs.AbsProc).Dev(fhs.AbsDev, true) - return -} - -var ( - // ErrExecBusy is returned entering [Cache.EnterExec] while another - // goroutine has not yet returned from it. - ErrExecBusy = errors.New("scratch directories in use") - // ErrNotExec is returned for unsupported implementations of [Artifact] - // passed to [Cache.EnterExec]. - ErrNotExec = errors.New("attempting to run a non-exec artifact") -) - -// EnterExec runs the container of an [Artifact] of [KindExec] or [KindExecNet] -// with its entry point, argument, and standard streams replaced with values -// supplied by the caller. -func (c *Cache) EnterExec( - ctx context.Context, - a Artifact, - hostname string, - retainSession bool, - stdin io.Reader, - stdout, stderr io.Writer, - path *check.Absolute, - args ...string, -) (err error) { - if !c.inExec.CompareAndSwap(false, true) { - return ErrExecBusy - } - defer c.inExec.Store(false) - - var hostNet bool - var e *execArtifact - switch f := a.(type) { - case *execArtifact: - e = f - - case *execMeasuredArtifact: - e = &f.execArtifact - hostNet = f.hostNet - - default: - return ErrNotExec - } - - deps := Collect(a.Inputs()) - if _, _, err = c.Cure(&deps); err == nil { - return errors.New("unreachable") - } else if !IsCollected(err) { - return - } - - dm := make(map[Artifact]cureRes) - for i, p := range deps { - var res cureRes - res.pathname, res.checksum, err = c.Cure(p) - if err != nil { - return - } - dm[deps[i]] = res - } - - scratch := c.base.Append(dirExecScratch) - temp, work := scratch.Append("temp"), scratch.Append("work") - // work created during makeContainer - if err = os.MkdirAll(temp.String(), 0700); err != nil { - return - } - defer func() { - if chmodErr, removeErr := removeAll(scratch); chmodErr != nil || removeErr != nil { - err = errors.Join(err, chmodErr, removeErr) - } - }() - - var z *container.Container - z, err = e.makeContainer( - ctx, c.msg, - c.attr.Flags, - c.attr.Jobs, - c.attr.Load, - hostNet, - temp, work, - func(a Artifact) (*check.Absolute, unique.Handle[Checksum]) { - if res, ok := dm[a]; ok { - return res.pathname, res.checksum - } - panic(InvalidLookupError(c.Ident(a).Value())) - }, - c.Ident, - ) - if err != nil { - return - } - z.Stdin, z.Stdout, z.Stderr = stdin, stdout, stderr - z.Path, z.Args = path, args - z.RetainSession = retainSession - if stdin == os.Stdin { - if s, ok := os.LookupEnv("TERM"); ok { - z.Env = append(z.Env, "TERM="+s) - } - } - if hostname != "" { - z.Hostname = hostname - } - - if err = z.Start(); err != nil { - return - } - if err = z.Serve(); err != nil { - return - } - return z.Wait() -} - -// cure is like Cure but allows optional host net namespace. -func (a *execArtifact) cure(f *FContext, hostNet bool) (err error) { - ctx, cancel := context.WithTimeout(f.Unwrap(), a.timeout) - defer cancel() - - msg := f.GetMessage() - var z *container.Container - if z, err = a.makeContainer( - ctx, msg, f.cache.attr.Flags, f.GetJobs(), f.GetLoad(), hostNet, - f.GetTempDir(), f.GetWorkDir(), - f.GetArtifact, - f.cache.Ident, - ); err != nil { - return - } - - var status io.Writer - if status, err = f.GetStatusWriter(); err != nil { - return - } - - if msg.IsVerbose() { - var stdout, stderr io.ReadCloser - if stdout, err = z.StdoutPipe(); err != nil { - return - } - if stderr, err = z.StderrPipe(); err != nil { - _ = stdout.Close() - return - } - - brStdout, brStderr := f.cache.getReader(stdout), f.cache.getReader(stderr) - stdoutDone, stderrDone := make(chan struct{}), make(chan struct{}) - - var suffix, prefixO, prefixE string - if f.cache.attr.Flags&CColourOutput != 0 { - suffix = "\x1b[0m" - prefixO = "\x1b[1;37m(" + a.name + ")\x1b[0m" - prefixE = "\x1b[1;97m(" + a.name + ")\x1b[0m" - } else { - prefixO = "(" + a.name + ":1)" - prefixE = "(" + a.name + ":2)" - } - - go scanVerbose( - msg, cancel, stdoutDone, - prefixO, suffix, - io.TeeReader(brStdout, status), - ) - go scanVerbose( - msg, cancel, stderrDone, - prefixE, suffix, - io.TeeReader(brStderr, status), - ) - defer func() { - if err != nil && !errors.As(err, new(*exec.ExitError)) { - _ = stdout.Close() - _ = stderr.Close() - } - - <-stdoutDone - <-stderrDone - f.cache.putReader(brStdout) - f.cache.putReader(brStderr) - }() - } else { - z.Stdout, z.Stderr = status, status - } - - if err = z.Start(); err != nil { - return - } - if err = z.Serve(); err != nil { - return - } - if err = z.Wait(); err != nil { - return - } - - // do not allow empty directories to succeed - for { - err = syscall.Rmdir(f.GetWorkDir().String()) - if err != syscall.EINTR { - break - } - } - if err != nil && errors.Is(err, syscall.ENOTEMPTY) { - err = nil - } - return -} diff --git a/internal/pkg/exec_test.go b/internal/pkg/exec_test.go deleted file mode 100644 index 408d6257..00000000 --- a/internal/pkg/exec_test.go +++ /dev/null @@ -1,670 +0,0 @@ -package pkg_test - -import ( - "bufio" - "bytes" - _ "embed" - "encoding/gob" - "errors" - "io/fs" - "net" - "os" - "os/exec" - "path/filepath" - "slices" - "strings" - "testing" - _ "unsafe" // for go:linkname - - "hakurei.app/check" - "hakurei.app/container" - "hakurei.app/hst" - "hakurei.app/internal/info" - "hakurei.app/internal/pkg" - "hakurei.app/internal/stub" - - "hakurei.app/internal/pkg/internal/testtool/expected" -) - -// testtoolBin is the container test tool binary made available to the -// execArtifact for testing its curing environment. -// -//go:generate env CGO_ENABLED=0 go build -tags testtool -o internal/testtool ./internal/testtool -//go:embed internal/testtool/testtool -var testtoolBin []byte - -func init() { - pathname, err := filepath.Abs("internal/testtool/testtool") - if err != nil { - panic(err) - } - pkg.RegisterArch("cafe", container.BinfmtEntry{ - Magic: expected.Magic, - Interpreter: check.MustAbs(pathname), - }) -} - -// scanLinesCR is like [bufio.ScanLines], but also treats a bare \r as an -// end-of-line marker. -// -//go:linkname scanLinesCR hakurei.app/internal/pkg.scanLinesCR -func scanLinesCR(data []byte, atEOF bool) (advance int, token []byte, err error) - -func TestScan(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - data string - want []string - }{ - {"progress indicator", "Updating files: 76% (8655/11256)\r" + - "Updating files: 77% (8668/11256)\r" + - "Updating files: 78% (8780/11256)\r" + - "Updating files: 79% (8893/11256)\r" + - "Updating files: 80% (9005/11256)\r" + - "Updating files: 81% (9118/11256)\r" + - "Updating files: 82% (9230/11256)\r" + - "Updating files: 83% (9343/11256)\r" + - "Updating files: 84% (9456/11256)\r" + - "Updating files: 85% (9568/11256)\r" + - "Updating files: 86% (9681/11256)\r" + - "Updating files: 87% (9793/11256)\r" + - "Updating files: 88% (9906/11256)\r" + - "Updating files: 89% (10018/11256)\r" + - "Updating files: 90% (10131/11256)\r" + - "Updating files: 91% (10243/11256)\r" + - "Updating files: 92% (10356/11256)\r" + - "Updating files: 93% (10469/11256)\r" + - "Updating files: 94% (10581/11256)\r" + - "Updating files: 95% (10694/11256)\r" + - "Updating files: 96% (10806/11256)\r" + - "Updating files: 97% (10919/11256)\r" + - "Updating files: 98% (11031/11256)\r" + - "Updating files: 99% (11144/11256)\r" + - "Updating files: 100% (11256/11256)\r" + - "Updating files: 100% (11256/11256), done.\n", []string{ - "Updating files: 76% (8655/11256)", - "Updating files: 77% (8668/11256)", - "Updating files: 78% (8780/11256)", - "Updating files: 79% (8893/11256)", - "Updating files: 80% (9005/11256)", - "Updating files: 81% (9118/11256)", - "Updating files: 82% (9230/11256)", - "Updating files: 83% (9343/11256)", - "Updating files: 84% (9456/11256)", - "Updating files: 85% (9568/11256)", - "Updating files: 86% (9681/11256)", - "Updating files: 87% (9793/11256)", - "Updating files: 88% (9906/11256)", - "Updating files: 89% (10018/11256)", - "Updating files: 90% (10131/11256)", - "Updating files: 91% (10243/11256)", - "Updating files: 92% (10356/11256)", - "Updating files: 93% (10469/11256)", - "Updating files: 94% (10581/11256)", - "Updating files: 95% (10694/11256)", - "Updating files: 96% (10806/11256)", - "Updating files: 97% (10919/11256)", - "Updating files: 98% (11031/11256)", - "Updating files: 99% (11144/11256)", - "Updating files: 100% (11256/11256)", - "Updating files: 100% (11256/11256), done.", - }}, - - {"crlf", "0\r1\n2\n3\r\n4\n5\r6\r\n7\n", []string{ - "0", "1", "2", "3", "4", "5", "6", "7", - }}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - s := bufio.NewScanner(strings.NewReader(tc.data)) - s.Split(scanLinesCR) - got := make([]string, 0, len(tc.want)) - for s.Scan() { - got = append(got, s.Text()) - } - if err := s.Err(); err != nil { - t.Fatal(err) - } - if !slices.Equal(got, tc.want) { - t.Fatalf("Scan: %q, want %q", got, tc.want) - } - }) - } -} - -func TestExec(t *testing.T) { - t.Parallel() - - wantOffline := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "check": {Mode: 0400, Data: []byte{0}}, - } - wantOfflineEncode := pkg.Encode(wantOffline.hash()) - failingArtifact := &stubArtifact{ - kind: pkg.KindTar, - params: []byte("doomed artifact"), - cure: func(t *pkg.TContext) error { - return stub.UniqueError(0xcafe) - }, - } - - checkWithCache(t, []cacheTestCase{ - {"offline", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - testtool, testtoolDestroy := newTesttool() - - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-offline", "", new(wantOffline.hash()), 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - - pkg.MustPath("/file", false, newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xfe, 0}, - nil, - nil, nil, - )), - pkg.MustPath("/.hakurei", false, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), - pkg.MustPath("/opt", false, testtool), - ), ignorePathname, wantOffline, pkg.WNew, nil}, - - {"substitution", pkg.NewExec( - "exec-offline", "", new(wantOffline.hash()), 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - - pkg.MustPath("/file", false, newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xfe, 0}, - nil, - nil, nil, - )), - // substitution miss fails in testtool due to differing idents - pkg.MustPath("/.hakurei", false, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory (substituted)"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), - pkg.MustPath("/opt", false, testtool), - ), ignorePathname, wantOffline, pkg.WSubstitute, nil}, - - {"error passthrough", pkg.NewExec( - "", "", nil, 0, false, true, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - - pkg.MustPath("/proc/nonexistent", false, failingArtifact), - ), nil, nil, pkg.WNew, pkg.InputError{ - failingArtifact: stub.UniqueError(0xcafe), - }}, - - {"invalid paths", pkg.NewExec( - "", "", nil, 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - - pkg.ExecPath{}, - ), nil, nil, pkg.WNew, pkg.ErrInvalidPaths}, - }) - - // check init failure passthrough - initFailureArtifact := pkg.NewExec( - "", "", nil, 0, false, false, - pkg.AbsWork, - nil, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - ) - var exitError *exec.ExitError - if _, _, err := c.Cure(initFailureArtifact); !errors.As(err, &exitError) || - exitError.ExitCode() != hst.ExitFailure { - t.Fatalf("Cure: error = %v, want init exit status 1", err) - } - - var faultStatus []byte - if faults, err := c.ReadFaults(initFailureArtifact); err != nil { - t.Fatal(err) - } else if len(faults) != 1 { - t.Fatalf("ReadFaults: %v", faults) - } else if faultStatus, err = os.ReadFile(faults[0].String()); err != nil { - t.Fatal(err) - } else if err = faults[0].Destroy(); err != nil { - t.Fatal(err) - } else { - t.Logf("destroyed expected fault at %s", faults[0].Time().UTC()) - } - - if !bytes.HasPrefix(faultStatus, []byte( - "internal/pkg ", - )) || !bytes.Contains(faultStatus, []byte( - "\ninit: fork/exec /opt/bin/testtool: no such file or directory\n", - )) { - t.Errorf("unexpected status:\n%s", string(faultStatus)) - } - - destroyStatus(t, base, 2, 1) - testtoolDestroy(t, base, c) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400, Data: []byte{}}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")}, - "identifier/" + expected.Offline: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)}, - "identifier/" + expected.OfflineS: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)}, - "identifier/_OHzj_xdmfpaWfFN-_D9JpGpPlIjh84873E3Ui9xwv9PPPIt0BlzvH-cnPNeRsTl": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"net", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - testtool, testtoolDestroy := newTesttool() - - wantNet := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "check": {Mode: 0400, Data: []byte("net")}, - } - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-net", "", new(wantNet.hash()), 0, true, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool", "net"}, - - pkg.MustPath("/file", false, newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xfe, 0}, - nil, - nil, nil, - )), - pkg.MustPath("/.hakurei", false, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), - pkg.MustPath("/opt", false, testtool), - ), ignorePathname, wantNet, pkg.WNew, nil}, - }) - - destroyStatus(t, base, 2, 0) - testtoolDestroy(t, base, c) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400, Data: []byte{}}, - "checksum/a1F_i9PVQI4qMcoHgTQkORuyWLkC1GLIxOhDt2JpU1NGAxWc5VJzdlfRK-PYBh3W": {Mode: fs.ModeDir | 0500}, - "checksum/a1F_i9PVQI4qMcoHgTQkORuyWLkC1GLIxOhDt2JpU1NGAxWc5VJzdlfRK-PYBh3W/check": {Mode: 0400, Data: []byte("net")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/" + expected.Net: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/a1F_i9PVQI4qMcoHgTQkORuyWLkC1GLIxOhDt2JpU1NGAxWc5VJzdlfRK-PYBh3W")}, - "identifier/_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"overlay root", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - testtool, testtoolDestroy := newTesttool() - - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-overlay-root", "", nil, 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - - pkg.MustPath("/", true, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), - pkg.MustPath("/opt", false, testtool), - ), ignorePathname, wantOffline, pkg.WNew, nil}, - }) - - destroyStatus(t, base, 2, 0) - testtoolDestroy(t, base, c) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/" + expected.OvlRoot: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"overlay work", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - testtool, testtoolDestroy := newTesttool() - - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-overlay-work", "", nil, 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"}, - check.MustAbs("/work/bin/testtool"), - []string{"testtool"}, - - pkg.MustPath("/", true, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), pkg.MustPath("/work/", false, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), pkg.Path(pkg.AbsWork, false /* ignored */, testtool), - ), ignorePathname, wantOffline, pkg.WNew, nil}, - }) - - destroyStatus(t, base, 2, 0) - testtoolDestroy(t, base, c) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/" + expected.Work: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"multiple layers", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - testtool, testtoolDestroy := newTesttool() - - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-multiple-layers", "", nil, 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool", "layers"}, - - pkg.MustPath("/", true, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }, &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("test sample with dependencies"), - - deps: slices.Repeat([]pkg.Artifact{newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xfe, 0}, - nil, - nil, nil, - ), &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - - // this is queued and might run instead of the other - // one so do not leave it as nil - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }}, 1<<5 /* concurrent cache hits */), - - cure: func(f *pkg.FContext) error { - work := f.GetWorkDir() - if err := os.MkdirAll(work.String(), 0700); err != nil { - return err - } - return os.WriteFile(work.Append("check").String(), []byte("layers"), 0400) - }, - }), - pkg.MustPath("/opt", false, testtool), - ), ignorePathname, wantOffline, pkg.WNew, nil}, - }) - - destroyStatus(t, base, 2, 0) - testtoolDestroy(t, base, c) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400, Data: []byte{}}, - "checksum/nY_CUdiaUM1OL4cPr5TS92FCJ3rCRV7Hm5oVTzAvMXwC03_QnTRfQ5PPs7mOU9fK": {Mode: fs.ModeDir | 0500}, - "checksum/nY_CUdiaUM1OL4cPr5TS92FCJ3rCRV7Hm5oVTzAvMXwC03_QnTRfQ5PPs7mOU9fK/check": {Mode: 0400, Data: []byte("layers")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")}, - "identifier/2uIvmnqtzAx4Ed4PWV8xjZCWeJCwfE2IcCc0evM-Nd8mDUlgSU6nLtPLcpPUcMlG": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/nY_CUdiaUM1OL4cPr5TS92FCJ3rCRV7Hm5oVTzAvMXwC03_QnTRfQ5PPs7mOU9fK")}, - "identifier/" + expected.Layers: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"overlay layer promotion", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - testtool, testtoolDestroy := newTesttool() - - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-layer-promotion", "", nil, 0, false, true, - pkg.AbsWork, - []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool", "promote"}, - - pkg.MustPath("/", true, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("another empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), - pkg.MustPath("/opt", false, testtool), - ), ignorePathname, wantOffline, pkg.WNew, nil}, - }) - - destroyStatus(t, base, 2, 0) - testtoolDestroy(t, base, c) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/Vzj9qaSVBtNyBQv_PW6uojFOWTwraHoaC0C6xdaGG1q_HTvSh4KLVxRcEQU1jd2Y": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/" + expected.Promote: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"binfmt", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - if info.CanDegrade && os.Getenv("ROSA_SKIP_BINFMT") != "" { - t.Skip("binfmt_misc test explicitly skipped") - } - - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-binfmt", "cafe", nil, 0, false, true, - pkg.AbsWork, - []string{"HAKUREI_TEST=1", "HAKUREI_BINFMT=1"}, - check.MustAbs("/opt/bin/sample"), - []string{"sample"}, - - pkg.MustPath("/", true, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), - pkg.MustPath("/opt", false, overrideIdent{pkg.ID{0xfe, 0xff}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - work := t.GetWorkDir() - if err := os.MkdirAll( - work.Append("bin").String(), - 0700, - ); err != nil { - return err - } - - return os.WriteFile(t.GetWorkDir().Append( - "bin", - "sample", - ).String(), []byte(expected.Full), 0500) - }, - }}), - ), ignorePathname, expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "check": {Mode: 0400, Data: []byte("binfmt")}, - }, pkg.WNew, nil}, - }) - - destroyStatus(t, base, 2, 0) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/5aevg3YpDxjqQZ-pdvXK7YqgkL5JKqcoStYQxeD96kuYar6K2mRQWMHib6NQRnpV": {Mode: fs.ModeDir | 0500}, - "checksum/5aevg3YpDxjqQZ-pdvXK7YqgkL5JKqcoStYQxeD96kuYar6K2mRQWMHib6NQRnpV/bin": {Mode: fs.ModeDir | 0700}, - "checksum/5aevg3YpDxjqQZ-pdvXK7YqgkL5JKqcoStYQxeD96kuYar6K2mRQWMHib6NQRnpV/bin/sample": {Mode: 0500, Data: []byte("\xca\xfe\xba\xbe\xfd\xfd:3")}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - "checksum/UnDo4B5KneEUY5b4vRUk_y9MWgkWuw2N8f8a2XayO686xXur-aZmX2-7n_8tKMe3": {Mode: fs.ModeDir | 0500}, - "checksum/UnDo4B5KneEUY5b4vRUk_y9MWgkWuw2N8f8a2XayO686xXur-aZmX2-7n_8tKMe3/check": {Mode: 0400, Data: []byte("binfmt")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/c-HodmoEG0f_WGdst9aN-duLMH7QDDXOUi7UcyArXMeM1fc9lYKKQlTCW8ktaK8X": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UnDo4B5KneEUY5b4vRUk_y9MWgkWuw2N8f8a2XayO686xXur-aZmX2-7n_8tKMe3")}, - "identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/5aevg3YpDxjqQZ-pdvXK7YqgkL5JKqcoStYQxeD96kuYar6K2mRQWMHib6NQRnpV")}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} - -// newTesttool returns an [Artifact] that cures into testtoolBin. The returned -// function must be called at the end of the test but not deferred. -func newTesttool() ( - testtool pkg.Artifact, - testtoolDestroy func(t *testing.T, base *check.Absolute, c *pkg.Cache), -) { - // testtoolBin is built during go:generate and is not deterministic - testtool = overrideIdent{pkg.ID{0xfe, 0xff}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - work := t.GetWorkDir() - if err := os.MkdirAll( - work.Append("bin").String(), - 0700, - ); err != nil { - return err - } - - if ift, err := net.Interfaces(); err != nil { - return err - } else { - var f *os.File - if f, err = os.Create(t.GetWorkDir().Append( - "ift", - ).String()); err != nil { - return err - } else { - err = gob.NewEncoder(f).Encode(ift) - closeErr := f.Close() - if err != nil { - return err - } - if closeErr != nil { - return closeErr - } - } - } - - return os.WriteFile(t.GetWorkDir().Append( - "bin", - "testtool", - ).String(), testtoolBin, 0500) - }, - }} - testtoolDestroy = newDestroyArtifactFunc(testtool) - return -} diff --git a/internal/pkg/file.go b/internal/pkg/file.go deleted file mode 100644 index 14467153..00000000 --- a/internal/pkg/file.go +++ /dev/null @@ -1,100 +0,0 @@ -package pkg - -import ( - "bytes" - "crypto/sha512" - "fmt" - "io" -) - -// A fileArtifact is an [Artifact] that cures into data known ahead of time. -type fileArtifact []byte - -var _ KnownChecksum = new(fileArtifact) -var _ CuresExempt = new(fileArtifact) -var _ RevisionArtifact = new(fileArtifact) - -// fileArtifactNamed embeds fileArtifact alongside a caller-supplied name. -type fileArtifactNamed struct { - fileArtifact - // Caller-supplied user-facing reporting name. - name string -} - -var _ fmt.Stringer = new(fileArtifactNamed) -var _ KnownChecksum = new(fileArtifactNamed) - -// String returns the caller-supplied reporting name. -func (a *fileArtifactNamed) String() string { return a.name } - -// Params writes the caller-supplied reporting name and the file body. -func (a *fileArtifactNamed) Params(ctx *IContext) { - ctx.WriteString(a.name) - ctx.Write(a.fileArtifact) -} - -// NewFile returns a [FileArtifact] that cures into a caller-supplied byte slice. -// -// Caller must not modify data after NewFile returns. -func NewFile(name string, data []byte) FileArtifact { - f := fileArtifact(data) - if name != "" { - return &fileArtifactNamed{f, name} - } - return &f -} - -// Kind returns the hardcoded [Kind] constant. -func (*fileArtifact) Kind() Kind { return KindFile } - -// Params writes an empty string and the file body. -func (a *fileArtifact) Params(ctx *IContext) { - ctx.WriteString("") - ctx.Write(*a) -} - -func init() { - register(KindFile, func(r *IRReader) Artifact { - name := r.ReadString() - data := r.ReadStringBytes() - if _, ok := r.Finalise(); !ok { - panic(ErrExpectedChecksum) - } - return NewFile(name, data) - }) -} - -// Inputs returns a nil slice. -func (*fileArtifact) Inputs() []Artifact { return nil } - -// IsExclusive returns false: Cure returns a prepopulated buffer. -func (*fileArtifact) IsExclusive() bool { return false } - -// Checksum computes and returns the checksum of caller-supplied data. -func (a *fileArtifact) Checksum() Checksum { - h := sha512.New384() - h.Write(*a) - return Checksum(h.Sum(nil)) -} - -// Revision satisfies [RevisionArtifact] for incompatible IsExecutable behaviour. -func (*fileArtifact) Revision() uint64 { return 0 } - -// IsExecutable returns whether the contents begin with shebang or the ELF -// magic number. -func (a *fileArtifact) IsExecutable() bool { - return a != nil && - (bytes.HasPrefix( - *a, []byte{'#', '!'}, - ) || bytes.HasPrefix( - *a, []byte{0x7f, 'E', 'L', 'F'}, - )) -} - -// Cure returns the caller-supplied data. -func (a *fileArtifact) Cure(*RContext) (io.ReadCloser, error) { - return io.NopCloser(bytes.NewReader(*a)), nil -} - -// CuresExempt exempts the cheap [KindFile] implementation. -func (*fileArtifact) CuresExempt() {} diff --git a/internal/pkg/file_test.go b/internal/pkg/file_test.go deleted file mode 100644 index f6fcc96f..00000000 --- a/internal/pkg/file_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package pkg_test - -import ( - "io/fs" - "testing" - - "hakurei.app/check" - "hakurei.app/internal/pkg" -) - -func TestFile(t *testing.T) { - t.Parallel() - - want := expectsFile{0} - const ident = "N7M6W8-e55Wky_1q1a6qRKFVDw1S5XtPdB4HE2YZObG8OFum8HittyJgZUG5Ru2u" - wantShebang := expectsFile("#!/bin/sh\n") - const identShebang = "oKTGZb_TRtJf5G1iGGj4SOx1WUbI6YhmsUeEsCNvmK9tZYRwdwQf-DLFUxRPx5qm" - wantELF := expectsFile("\x7fELF\xde\xad\xbe\xef") - const identELF = "LjeHmIJgD1SiXIH8ewpGRed2_LU84NLapstnOAa_0hxLd8YY7x4KM0aApsj-RwCh" - checkWithCache(t, []cacheTestCase{ - {"file", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - cureMany(t, c, []cureStep{ - {"short", pkg.NewFile("null", want), base.Append( - "identifier", - ident, - ), want, pkg.WNew, nil}, - - {"shebang", pkg.NewFile("shebang", wantShebang), base.Append( - "identifier", - identShebang, - ), wantShebang, pkg.WNew, nil}, - - {"elf", pkg.NewFile("ELF", wantELF), base.Append( - "identifier", - identELF, - ), wantELF, pkg.WNew, nil}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + pkg.Encode(want.hash()): {Mode: 0400, Data: want}, - "checksum/" + pkg.Encode(wantShebang.hash()): {Mode: 0500, Data: wantShebang}, - "checksum/" + pkg.Encode(wantELF.hash()): {Mode: 0500, Data: wantELF}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/" + ident: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + pkg.Encode(want.hash()))}, - "identifier/" + identShebang: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + pkg.Encode(wantShebang.hash()))}, - "identifier/" + identELF: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + pkg.Encode(wantELF.hash()))}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} diff --git a/internal/pkg/internal/testtool/expected/expected.go b/internal/pkg/internal/testtool/expected/expected.go deleted file mode 100644 index c0e8e4a2..00000000 --- a/internal/pkg/internal/testtool/expected/expected.go +++ /dev/null @@ -1,9 +0,0 @@ -// Package expected contains data shared between test helper and test harness. -package expected - -const ( - // Magic are magic bytes in the binfmt test case. - Magic = "\xca\xfe\xba\xbe\xfd\xfd" - // Full is the full content of the binfmt test case executable. - Full = Magic + ":3" -) diff --git a/internal/pkg/internal/testtool/expected/sum_amd64.go b/internal/pkg/internal/testtool/expected/sum_amd64.go deleted file mode 100644 index 354931be..00000000 --- a/internal/pkg/internal/testtool/expected/sum_amd64.go +++ /dev/null @@ -1,11 +0,0 @@ -package expected - -const ( - Offline = "IKmVhALejtYA3JhN1PCfjKF-Dio1XVcOeMWDHhfSHCbb4iBNhGjFy2UspZveZvb8" - OfflineS = "6ZubjSyCnz4-TKgQZi_BbY-JUwsStYCtD_f-BTDOOotfEbqkm47W20AazHRaLviz" - OvlRoot = "Yox7WpG8B_OqIXWF4HWFZkUfsEl7DGzpFyZM0qtd9T8nQ-7SAkEUurlAe6_DJ6Iu" - Layers = "aAwfxDB-efVo379FNl178jK7HEKU3LEjHbZdaVVjk7GnXGZGEANNsXnB3SCUxM9A" - Net = "IU3GwU2jTBd7HjotvFo2wsDWBQGVGpamwiSKdc2G1EnhM-ZaacxAuK6q0wbTVxra" - Promote = "rOjetGHzT_jPErrBR24IAZKDj8Q9dLLqxFQqloMgjsULss0VXDI--n6LxuMBHvmZ" - Work = "7f9XDeREfvJIjCe0qesND0s0tasExAIcc-OH33-eq9tsrQPUXhJp4KEkpsYH4hCD" -) diff --git a/internal/pkg/internal/testtool/expected/sum_arm64.go b/internal/pkg/internal/testtool/expected/sum_arm64.go deleted file mode 100644 index 2e6f7cbf..00000000 --- a/internal/pkg/internal/testtool/expected/sum_arm64.go +++ /dev/null @@ -1,11 +0,0 @@ -package expected - -const ( - Offline = "MTgCSzwPrzGigp9phvDPRwUTNGLm1KM2EsEtbz-l2cNieqjaGtnzm65CDXlSxXrl" - OfflineS = "p89bQg7_jR4nwaSbpEd08ne1jHOEw7Wlnlyv5FHq7ba1Kml3HSBZwEKRzu0gCqEb" - OvlRoot = "9ifwO6MAaV9kTqRR2O5PJzm8XhfXbr50mFLWyEHQm8OHHcJtAEUgXZzJYoZt9YVV" - Layers = "INuGActFPwJtS7mAR9Bkt7FD4yoHlXHSJ9CaSAQJrpkUBL_k815Cs_jvRfxy5-a7" - Net = "kJvy24uk4ePvHoxSKf4vMOhR4NxLbmyWlA9bTS7d3z3fE9WF1Tvpao9g-QuYU4vm" - Promote = "lYqyYrfjPawIbld5Qz0NbKvd_bmI3nJC1eEnlIQWzsjWE9uA1HuKPAaClnDeyG6L" - Work = "V6k4FqOBMRCmHiqchs5-joVGZ_MhNWZyXm3gdbU0S7S5eMze7G4SwYMTf66ojKQL" -) diff --git a/internal/pkg/internal/testtool/expected/sum_riscv64.go b/internal/pkg/internal/testtool/expected/sum_riscv64.go deleted file mode 100644 index 53872c08..00000000 --- a/internal/pkg/internal/testtool/expected/sum_riscv64.go +++ /dev/null @@ -1,11 +0,0 @@ -package expected - -const ( - Offline = "LlieSHVDe1lnKvfI8t4oGRg_2hPxwyy1drmaqc-EKCOENuS9C2YHrp2VY5pSyhbg" - OfflineS = "qccjM8oYjUuEYPKFFzAxDFVzMerSUKUZc18NdVQoiPWf-70XOsNK8NaUQqjTgTAD" - OvlRoot = "lpV2bb_KGYFHPiOn-jtGop8K_ew0y1p3BezRp0ebM93_a-T9J-vkqGD5uokda1aN" - Layers = "sM3DJm210CHPGHkAquIbrXhspB81FpVC60ApQrvJGD5El2i7cSONj7LWf7EWqKpT" - Net = "JihQUwsVETRYjb2-yIXLYZ6WVsaFebwl8btSYAsNQ2QapS5MZVaI-dYwrVoySycR" - Promote = "lGqwR4W2OjylSVitMQP1lc6wApOBkbMjsmIqkxQbfEZ3DHEQBbJr84ErW6Xb54zo" - Work = "aZgtyv6aMtoHFkHAaJr6nLaA7N_SHjShgWPyQ5UdzaIqYSiVqvCrZqSokGTCcTii" -) diff --git a/internal/pkg/internal/testtool/main.go b/internal/pkg/internal/testtool/main.go deleted file mode 100644 index 96a85058..00000000 --- a/internal/pkg/internal/testtool/main.go +++ /dev/null @@ -1,277 +0,0 @@ -//go:build testtool - -package main - -import ( - "encoding/gob" - "log" - "net" - "os" - "path/filepath" - "reflect" - "runtime" - "slices" - "strconv" - "strings" - - "hakurei.app/fhs" - "hakurei.app/vfs" - - "hakurei.app/internal/pkg/internal/testtool/expected" -) - -func main() { - log.SetFlags(0) - log.SetPrefix("testtool: ") - - if os.Getenv("HAKUREI_BINFMT") == "1" { - wantArgs := []string{"/interpreter", "/opt/bin/sample"} - if !slices.Equal(os.Args, wantArgs) { - log.Fatalf("Args: %q, want %q", os.Args, wantArgs) - } - - if err := os.WriteFile("check", []byte("binfmt"), 0400); err != nil { - log.Fatal(err) - } - - return - } - - environ := slices.DeleteFunc(slices.Clone(os.Environ()), func(s string) bool { - for _, t := range []string{ - "CURE_JOBS=" + strconv.Itoa(runtime.NumCPU()), - "CURE_LOAD=" + strconv.Itoa(runtime.NumCPU()+2), - } { - if s == t { - return true - } - } - return false - }) - - var hostNet, layers, promote bool - if len(os.Args) == 2 && os.Args[0] == "testtool" { - switch os.Args[1] { - case "net": - hostNet = true - log.SetPrefix("testtool(net): ") - break - - case "layers": - layers = true - log.SetPrefix("testtool(layers): ") - break - - case "promote": - promote = true - log.SetPrefix("testtool(promote): ") - - default: - log.Fatalf("Args: %q", os.Args) - return - } - } else if wantArgs := []string{"testtool"}; !slices.Equal(os.Args, wantArgs) { - log.Fatalf("Args: %q, want %q", os.Args, wantArgs) - } - - var overlayRoot bool - wantEnv := []string{"HAKUREI_TEST=1"} - if len(environ) == 2 { - overlayRoot = true - if !layers && !promote { - log.SetPrefix("testtool(overlay root): ") - } - wantEnv = []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"} - } - if !slices.Equal(wantEnv, environ) { - log.Fatalf("Environ: %q, want %q", environ, wantEnv) - } - - var overlayWork bool - const ( - wantExec = "/opt/bin/testtool" - wantExecWork = "/work/bin/testtool" - ) - var iftPath string - if got, err := os.Executable(); err != nil { - log.Fatalf("Executable: error = %v", err) - } else { - iftPath = filepath.Join(filepath.Dir(filepath.Dir(got)), "ift") - - if got != wantExec { - switch got { - case wantExecWork: - overlayWork = true - log.SetPrefix("testtool(overlay work): ") - - default: - log.Fatalf("Executable: %q, want %q", got, wantExec) - } - } - } - - wantHostname := "cure" - if hostNet { - wantHostname += "-net" - } - - if hostname, err := os.Hostname(); err != nil { - log.Fatalf("Hostname: error = %v", err) - } else if hostname != wantHostname { - log.Fatalf("Hostname: %q, want %q", hostname, wantHostname) - } - - var m *vfs.MountInfo - if f, err := os.Open(fhs.Proc + "self/mountinfo"); err != nil { - log.Fatalf("Open: error = %v", err) - } else { - err = vfs.NewMountInfoDecoder(f).Decode(&m) - closeErr := f.Close() - if err != nil { - log.Fatalf("Decode: error = %v", err) - } - if closeErr != nil { - log.Fatalf("Close: error = %v", err) - } - } - - if ift, err := net.Interfaces(); err != nil { - log.Fatal(err) - } else if !hostNet { - if len(ift) != 1 || ift[0].Name != "lo" { - log.Fatalln("got interfaces", strings.Join(slices.Collect(func(yield func(ifn string) bool) { - for _, ifi := range ift { - if !yield(ifi.Name) { - break - } - } - }), ", ")) - } - } else { - var iftParent []net.Interface - - var r *os.File - if r, err = os.Open(iftPath); err != nil { - log.Fatal(err) - } else { - err = gob.NewDecoder(r).Decode(&iftParent) - closeErr := r.Close() - if err != nil { - log.Fatal(err) - } - if closeErr != nil { - log.Fatal(closeErr) - } - } - - if !reflect.DeepEqual(ift, iftParent) { - log.Fatalf("Interfaces: %#v, want %#v", ift, iftParent) - } - } - - const checksumEmptyDir = "MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU" - ident := expected.Offline - log.Println(m) - next := func() { m = m.Next; log.Println(m) } - - if overlayRoot { - ident = expected.OvlRoot - - if m.Root != "/" || m.Target != "/" || - m.Source != "overlay" || m.FsType != "overlay" { - log.Fatal("unexpected root mount entry") - } - var lowerdir []string - for _, o := range strings.Split(m.FsOptstr, ",") { - const lowerdirKey = "lowerdir+=" - if strings.HasPrefix(o, lowerdirKey) { - lowerdir = append(lowerdir, o[len(lowerdirKey):]) - } - } - if !layers { - if len(lowerdir) != 1 || filepath.Base(lowerdir[0]) != checksumEmptyDir { - log.Fatal("unexpected artifact checksum") - } - } else { - ident = expected.Layers - - if len(lowerdir) != 2 || - filepath.Base(lowerdir[0]) != "MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU" || - filepath.Base(lowerdir[1]) != "nY_CUdiaUM1OL4cPr5TS92FCJ3rCRV7Hm5oVTzAvMXwC03_QnTRfQ5PPs7mOU9fK" { - log.Fatalf("unexpected lowerdirs %s", strings.Join(lowerdir, ", ")) - } - } - } else { - if hostNet { - ident = expected.Net - } - - if m.Root != "/sysroot" || m.Target != "/" { - log.Fatal("unexpected root mount entry") - } - - next() - if filepath.Base(m.Root) != "OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb" { - log.Fatal("unexpected file artifact checksum") - } - - next() - if filepath.Base(m.Root) != checksumEmptyDir { - log.Fatal("unexpected artifact checksum") - } - } - - if promote { - ident = expected.Promote - } - - next() // testtool artifact - - next() - if overlayWork { - ident = expected.Work - if m.Root != "/" || m.Target != "/work" || - m.Source != "overlay" || m.FsType != "overlay" { - log.Fatal("unexpected work mount entry") - } - } else { - if filepath.Base(m.Root) != ident || m.Target != "/work" { - log.Fatal("unexpected work mount entry") - } - } - - next() - if filepath.Base(m.Root) != ident || m.Target != "/tmp" { - log.Fatal("unexpected temp mount entry") - } - - next() - if m.Root != "/" || m.Target != "/proc" || m.Source != "proc" || m.FsType != "proc" { - log.Fatal("unexpected proc mount entry") - } - - next() - if m.Root != "/" || m.Target != "/dev" || m.Source != "devtmpfs" || m.FsType != "tmpfs" { - log.Fatal("unexpected dev mount entry") - } - - for i := 0; i < 9; i++ { // private /dev entries - next() - } - - if m.Next != nil { - log.Println("unexpected extra mount entries") - for m.Next != nil { - next() - } - os.Exit(1) - } - - checkData := []byte{0} - if hostNet { - checkData = []byte("net") - } - if err := os.WriteFile("check", checkData, 0400); err != nil { - log.Fatal(err) - } -} diff --git a/internal/pkg/ir.go b/internal/pkg/ir.go deleted file mode 100644 index d9f984a7..00000000 --- a/internal/pkg/ir.go +++ /dev/null @@ -1,865 +0,0 @@ -package pkg - -import ( - "bufio" - "bytes" - "crypto/sha512" - "encoding/binary" - "errors" - "fmt" - "io" - "iter" - "slices" - "strconv" - "sync" - "syscall" - "unique" - "unsafe" -) - -// wordSize is the boundary which binary segments are always aligned to. -const wordSize = 8 - -// alignSize returns the padded size for aligning sz. -func alignSize[T int | uint64](sz T) T { - return sz + (wordSize-(sz)%wordSize)%wordSize -} - -// panicToError recovers from a panic and replaces a nil error with the panicked -// error value. If the value does not implement error, it is re-panicked. -func panicToError(errP *error) { - r := recover() - if r == nil { - return - } - - if err, ok := r.(error); !ok { - panic(r) - } else if *errP == nil { - *errP = err - } -} - -// irCache implements [IRCache]. -type irCache struct { - // Artifact to [unique.Handle] of identifier cache. - artifact sync.Map - // Identifier free list, must not be accessed directly. - identPool sync.Pool -} - -// zeroIRCache returns the initialised value of irCache. -func zeroIRCache() irCache { - return irCache{ - identPool: sync.Pool{New: func() any { return new(extIdent) }}, - } -} - -// IRCache provides memory management and caching primitives for IR and -// identifier operations against [Artifact] implementations. -// -// The zero value is not safe for use. -type IRCache struct{ irCache } - -// NewIR returns the address of a new [IRCache]. -func NewIR() *IRCache { - return &IRCache{zeroIRCache()} -} - -// Inputs returns an iterator over direct and transitive inputs of an [Artifact] -// in randomised order. -func Inputs(a Artifact) iter.Seq2[Artifact, unique.Handle[ID]] { - ic := NewIR() - ic.Ident(a) - return func(yield func(Artifact, unique.Handle[ID]) bool) { - ic.artifact.Range(func(key, value any) bool { - return yield(key.(Artifact), value.(unique.Handle[ID])) - }) - } -} - -// IContext is passed to [Artifact.Params] and provides methods for writing -// values to the IR writer. It does not expose the underlying [io.Writer]. -// -// IContext is valid until [Artifact.Params] returns. -type IContext struct { - // Address of underlying irCache, should be zeroed or made unusable after - // [Artifact.Params] returns and must not be exposed directly. - ic *irCache - // Written to by various methods, should be zeroed after [Artifact.Params] - // returns and must not be exposed directly. - w io.Writer - // Optional [Artifact] to cureRes cache, replaces [IRKindIdent] with - // checksum values if non-nil. The pathname field may not be populated. - inputs map[Artifact]cureRes -} - -// irZero is a zero IR word. -var irZero [wordSize]byte - -// IRValueKind denotes the kind of encoded value. -type IRValueKind uint32 - -const ( - // IRKindEnd denotes the end of the current parameters stream. The ancillary - // value is interpreted as [IREndFlag]. - IRKindEnd IRValueKind = iota - // IRKindIdent denotes the identifier of a dependency [Artifact]. The - // ancillary value is reserved for future use. - IRKindIdent - // IRKindUint32 denotes an inlined uint32 value. - IRKindUint32 - // IRKindString denotes a string with its true length encoded in header - // ancillary data. Its wire length is always aligned to 8 byte boundary. - IRKindString - - irHeaderShift = 32 - irHeaderMask = 0xffffffff -) - -// String returns a user-facing name of k. -func (k IRValueKind) String() string { - switch k { - case IRKindEnd: - return "terminator" - case IRKindIdent: - return "ident" - case IRKindUint32: - return "uint32" - case IRKindString: - return "string" - default: - return "invalid kind " + strconv.Itoa(int(k)) - } -} - -// irValueHeader encodes [IRValueKind] and a 32-bit ancillary value. -type irValueHeader uint64 - -// encodeHeader returns irValueHeader encoding [IRValueKind] and ancillary data. -func (k IRValueKind) encodeHeader(v uint32) irValueHeader { - return irValueHeader(v)<<irHeaderShift | irValueHeader(k) -} - -// put stores h in b[0:8]. -func (h irValueHeader) put(b []byte) { - binary.LittleEndian.PutUint64(b[:], uint64(h)) -} - -// append appends the bytes of h to b and returns the appended slice. -func (h irValueHeader) append(b []byte) []byte { - return binary.LittleEndian.AppendUint64(b, uint64(h)) -} - -// IREndFlag is ancillary data encoded in the header of an [IRKindEnd] value and -// specifies the presence of optional fields in the remaining [IRKindEnd] data. -// Order of present fields is the order of their corresponding constants defined -// below. -type IREndFlag uint32 - -const ( - // IREndKnownChecksum denotes a [KnownChecksum] artifact. For an [IRKindEnd] - // value with this flag set, the remaining data contains the [Checksum]. - IREndKnownChecksum IREndFlag = 1 << iota -) - -// mustWrite writes to IContext.w and panics on error. The panic is recovered -// from by the caller and used as the return value. -func (i *IContext) mustWrite(p []byte) { - if _, err := i.w.Write(p); err != nil { - panic(err) - } -} - -// WriteIdent writes the identifier of [Artifact] to the IR. The behaviour of -// WriteIdent is not defined for an [Artifact] not part of the slice returned by -// [Artifact.Inputs]. -func (i *IContext) WriteIdent(a Artifact) { - buf := i.ic.getIdentBuf() - defer i.ic.putIdentBuf(buf) - - IRKindIdent.encodeHeader(0).put(buf[:]) - if i.inputs != nil { - res, ok := i.inputs[a] - if !ok { - panic(InvalidLookupError(i.ic.Ident(a).Value())) - } - *(*ID)(buf[wordSize:]) = res.checksum.Value() - } else { - *(*ID)(buf[wordSize:]) = i.ic.Ident(a).Value() - } - i.mustWrite(buf[:]) -} - -// WriteUint32 writes a uint32 value to the IR. -func (i *IContext) WriteUint32(v uint32) { - i.mustWrite(IRKindUint32.encodeHeader(v).append(nil)) -} - -// irMaxStringLength is the maximum acceptable wire size of [IRKindString]. -const irMaxStringLength = 1 << 24 - -// IRStringError is a string value too big to encode in IR. -type IRStringError string - -func (IRStringError) Error() string { - return "params value too big to encode in IR" -} - -// Write writes p as a string value to the IR. -func (i *IContext) Write(p []byte) { - sz := alignSize(len(p)) - if len(p) > irMaxStringLength || sz > irMaxStringLength { - panic(IRStringError(p)) - } - - i.mustWrite(IRKindString.encodeHeader(uint32(len(p))).append(nil)) - i.mustWrite(p) - - psz := sz - len(p) - if psz > 0 { - i.mustWrite(irZero[:psz]) - } -} - -// WriteString writes s as a string value to the IR. -func (i *IContext) WriteString(s string) { - p := unsafe.Slice(unsafe.StringData(s), len(s)) - i.Write(p) -} - -// Encode writes a deterministic, efficient representation of a to w and returns -// the first non-nil error encountered while writing to w. -func (ic *irCache) Encode(w io.Writer, a Artifact) (err error) { - return ic.encode(w, a, nil) -} - -// encode implements Encode but replaces identifiers with their cured checksums -// for a non-nil ident. The pathname field is unused. Caller must acquire -// Cache.identMu. -func (ic *irCache) encode( - w io.Writer, - a Artifact, - inputs map[Artifact]cureRes, -) (err error) { - deps := a.Inputs() - idents := make([]*extIdent, len(deps)) - if inputs == nil { - for i, d := range deps { - dbuf, did := ic.unsafeIdent(d, true) - if dbuf == nil { - dbuf = ic.getIdentBuf() - binary.LittleEndian.PutUint64(dbuf[:], uint64(d.Kind())) - *(*ID)(dbuf[wordSize:]) = did.Value() - } else { - ic.storeIdent(d, dbuf) - } - defer ic.putIdentBuf(dbuf) - idents[i] = dbuf - } - } else { - for i, d := range deps { - res, ok := inputs[d] - if !ok { - return InvalidLookupError(ic.Ident(d).Value()) - } - - dbuf := ic.getIdentBuf() - binary.LittleEndian.PutUint64(dbuf[:], uint64(d.Kind())) - *(*ID)(dbuf[wordSize:]) = res.checksum.Value() - defer ic.putIdentBuf(dbuf) - idents[i] = dbuf - } - } - slices.SortFunc(idents, func(a, b *extIdent) int { - return bytes.Compare(a[:], b[:]) - }) - idents = slices.CompactFunc(idents, func(a, b *extIdent) bool { - return *a == *b - }) - - // kind uint64 | rev uint64 | deps_sz uint64 - var buf [wordSize * 3]byte - binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind())) - binary.LittleEndian.PutUint64(buf[wordSize:], GetRevision(a)) - binary.LittleEndian.PutUint64(buf[wordSize*2:], uint64(len(idents))) - if _, err = w.Write(buf[:]); err != nil { - return - } - - for _, dn := range idents { - // kind uint64 | ident ID - if _, err = w.Write(dn[:]); err != nil { - return - } - } - - func() { - i := IContext{ic, w, inputs} - - defer panicToError(&err) - defer func() { i.ic, i.w = nil, nil }() - - a.Params(&i) - }() - if err != nil { - return - } - - var f IREndFlag - kcBuf := ic.getIdentBuf() - sz := wordSize - if kc, ok := a.(KnownChecksum); ok { - f |= IREndKnownChecksum - *(*Checksum)(kcBuf[wordSize:]) = kc.Checksum() - sz += len(Checksum{}) - } - IRKindEnd.encodeHeader(uint32(f)).put(kcBuf[:]) - - _, err = w.Write(kcBuf[:sz]) - ic.putIdentBuf(kcBuf) - return -} - -// encodeAll implements EncodeAll by recursively encoding dependencies and -// performs deduplication by value via the encoded map. -func (ic *irCache) encodeAll( - w io.Writer, - a Artifact, - encoded map[Artifact]struct{}, -) (err error) { - if _, ok := encoded[a]; ok { - return - } - - for _, d := range a.Inputs() { - if err = ic.encodeAll(w, d, encoded); err != nil { - return - } - } - - encoded[a] = struct{}{} - return ic.Encode(w, a) -} - -// EncodeAll writes a self-describing IR stream of a to w and returns the first -// non-nil error encountered while writing to w. -// -// EncodeAll tries to avoid encoding the same [Artifact] more than once, however -// it will fail to do so if they do not compare equal by value, as that will -// require buffering and greatly reduce performance. It is therefore up to the -// caller to avoid causing dependencies to be represented in a way such that -// two equivalent artifacts do not compare equal. While an IR stream with -// repeated artifacts is valid, it is somewhat inefficient, and the reference -// [IRDecoder] implementation produces a warning for it. -// -// Note that while EncodeAll makes use of the ident free list, it does not use -// the ident cache, nor does it contribute identifiers it computes back to the -// ident cache. Because of this, multiple invocations of EncodeAll will have -// similar cost and does not amortise when combined with a call to Cure. -func (ic *irCache) EncodeAll(w io.Writer, a Artifact) error { - return ic.encodeAll(w, a, make(map[Artifact]struct{})) -} - -// ErrRemainingIR is returned for a [IRReadFunc] that failed to call -// [IRReader.Finalise] before returning. -var ErrRemainingIR = errors.New("implementation did not consume final value") - -// DanglingIdentError is an identifier in a [IRKindIdent] value that was never -// described in the IR stream before it was encountered. -type DanglingIdentError unique.Handle[ID] - -func (e DanglingIdentError) Error() string { - return "artifact " + Encode(unique.Handle[ID](e).Value()) + - " was never described" -} - -type ( - // IRDecoder decodes [Artifact] from an IR stream. The stream is read to - // EOF and the final [Artifact] is returned. Previous artifacts may be - // looked up by their identifier. - // - // An [Artifact] may appear more than once in the same IR stream. A - // repeating [Artifact] generates a warning via [Cache] and will appear if - // verbose logging is enabled. Artifacts may only depend on artifacts - // previously described in the IR stream. - // - // IRDecoder rejects an IR stream on the first decoding error, it does not - // check against nonzero reserved ancillary data or incorrectly ordered or - // redundant unstructured dependencies. An invalid IR stream as such will - // yield [Artifact] values with identifiers disagreeing with those computed - // by IRDecoder. For this reason, IRDecoder does not access the ident cache - // to avoid putting [Cache] into an inconsistent state. - // - // Methods of IRDecoder are not safe for concurrent use. - IRDecoder struct { - // Address of underlying [Cache], must not be exposed directly. - c *Cache - - // Underlying IR reader. Methods of [IRReader] must not use this as it - // bypasses ident measurement. - r io.Reader - // Artifacts already seen in the IR stream. - ident map[unique.Handle[ID]]Artifact - - // Whether Decode returned, and the entire IR stream was decoded. - done, ok bool - } - - // IRReader provides methods to decode the IR wire format and read values - // from the reader embedded in the underlying [IRDecoder]. It is - // deliberately impossible to obtain the [IRValueKind] of the next value, - // and callers must never recover from panics in any read method. - // - // It is the responsibility of the caller to call Finalise after all IR - // values have been read. Failure to call Finalise causes the resulting - // [Artifact] to be rejected with [ErrRemainingIR]. - // - // For an [Artifact] expected to have dependencies, the caller must consume - // all dependencies by calling Next until all dependencies are depleted, or - // call DiscardAll to explicitly discard them and rely on values encoded as - // [IRKindIdent] instead. Failure to consume all unstructured dependencies - // causes the resulting [Artifact] to be rejected with [MissedDependencyError]. - // - // Requesting the value of an unstructured dependency not yet described in - // the IR stream via Next, or reading an [IRKindIdent] value not part of - // unstructured dependencies via ReadIdent may cause the resulting - // [Artifact] to be rejected with [DanglingIdentError], however either - // method may return a non-nil [Artifact] implementation of unspecified - // value. - IRReader struct { - // Address of underlying [IRDecoder], should be zeroed or made unusable - // after finalisation and must not be exposed directly. - d *IRDecoder - // Common buffer for word-sized reads. - buf [wordSize]byte - - // Inputs sent before params, sorted by identifier. Resliced on - // each call to Next and checked to be depleted during Finalise. - inputs []*extIdent - - // Number of values already read, -1 denotes a finalised IRReader. - count int - // Header of value currently being read. - h irValueHeader - - // Measured IR reader. All reads for the current [Artifact] must go - // through this to produce a correct ident. - r io.Reader - // Buffers measure writes. Flushed and returned to d during Finalise. - ibw *bufio.Writer - } - - // IRReadFunc reads IR values written by [Artifact.Params] to produce an - // instance of [Artifact] identical to the one to produce these values. - IRReadFunc func(r *IRReader) Artifact -) - -// kind returns the [IRValueKind] encoded in h. -func (h irValueHeader) kind() IRValueKind { - return IRValueKind(h & irHeaderMask) -} - -// value returns ancillary data encoded in h. -func (h irValueHeader) value() uint32 { - return uint32(h >> irHeaderShift) -} - -// irArtifact refers to artifact IR interpretation functions and must not be -// written to directly. -var irArtifact = make(map[Kind]IRReadFunc) - -// InvalidKindError is an unregistered [Kind] value. -type InvalidKindError Kind - -func (e InvalidKindError) Error() string { - return "invalid artifact kind " + strconv.Itoa(int(e)) -} - -// register records the [IRReadFunc] of an implementation of [Artifact] under -// the specified [Kind]. Expecting to be used only during initialization, it -// panics if the mapping between [Kind] and [IRReadFunc] is not a bijection. -// -// register is not safe for concurrent use. register must not be called after -// the first instance of [Cache] has been opened. -func register(k Kind, f IRReadFunc) { - openMu.Lock() - defer openMu.Unlock() - - if opened { - panic("attempting to register after open") - } - if _, ok := irArtifact[k]; ok { - panic("attempting to register " + strconv.Itoa(int(k)) + " twice") - } - irArtifact[k] = f -} - -// Register records the [IRReadFunc] of a custom implementation of [Artifact] -// under the specified [Kind]. Expecting to be used only during initialization, -// it panics if the mapping between [Kind] and [IRReadFunc] is not a bijection, -// or the specified [Kind] is below [KindCustomOffset]. -// -// Register is not safe for concurrent use. Register must not be called after -// the first instance of [Cache] has been opened. -func Register(k Kind, f IRReadFunc) { - if k < KindCustomOffset { - panic("attempting to register within internal kind range") - } - register(k, f) -} - -// NewDecoder returns a new [IRDecoder] that reads from the [io.Reader]. -func (c *Cache) NewDecoder(r io.Reader) *IRDecoder { - return &IRDecoder{c, r, make(map[unique.Handle[ID]]Artifact), false, false} -} - -const ( - // irMaxValues is the arbitrary maximum number of values allowed to be - // written by [Artifact.Params] and subsequently read via [IRReader]. - irMaxValues = 1 << 12 - - // irMaxDeps is the arbitrary maximum number of direct dependencies allowed - // to be returned by [Artifact.Inputs] and subsequently decoded by - // [IRDecoder]. - irMaxDeps = 1 << 10 -) - -var ( - // ErrIRValues is returned for an [Artifact] with too many parameter values. - ErrIRValues = errors.New("artifact has too many IR parameter values") - - // ErrIRDepend is returned for an [Artifact] with too many dependencies. - ErrIRDepend = errors.New("artifact has too many dependencies") - - // ErrAlreadyFinalised is returned when attempting to use an [IRReader] that - // has already been finalised. - ErrAlreadyFinalised = errors.New("reader has already finalised") -) - -// enterReader panics with an appropriate error for an out-of-bounds count and -// must be called at some point in any exported method. -func (ir *IRReader) enterReader(read bool) { - if ir.count < 0 { - panic(ErrAlreadyFinalised) - } - if ir.count >= irMaxValues { - panic(ErrIRValues) - } - - if read { - ir.count++ - } -} - -// IRKindError describes an attempt to read an IR value of unexpected kind. -type IRKindError struct { - Got, Want IRValueKind - Ancillary uint32 -} - -func (e *IRKindError) Error() string { - return fmt.Sprintf( - "got %s IR value (%#x) instead of %s", - e.Got, e.Ancillary, e.Want, - ) -} - -// readFull reads until either p is filled or an error is encountered. -func (ir *IRReader) readFull(p []byte) (n int, err error) { - for n < len(p) && err == nil { - var nn int - nn, err = ir.r.Read(p[n:]) - n += nn - } - return -} - -// mustRead reads from the underlying measured reader and panics on error. If -// an [io.EOF] is encountered and n != len(p), the error is promoted to a -// [io.ErrUnexpectedEOF], if n == 0, [io.EOF] is kept as is, otherwise it is -// zeroed. -func (ir *IRReader) mustRead(p []byte) { - n, err := ir.readFull(p) - if err == nil { - return - } - - if errors.Is(err, io.EOF) { - if n == len(p) { - return - } - err = io.ErrUnexpectedEOF - } - panic(err) -} - -// mustReadHeader reads the next header via d and checks its kind. -func (ir *IRReader) mustReadHeader(k IRValueKind) { - ir.mustRead(ir.buf[:]) - ir.h = irValueHeader(binary.LittleEndian.Uint64(ir.buf[:])) - if wk := ir.h.kind(); wk != k { - panic(&IRKindError{wk, k, ir.h.value()}) - } -} - -// putAll returns all dependency buffers to the underlying [Cache]. -func (ir *IRReader) putAll() { - for _, buf := range ir.inputs { - ir.d.c.putIdentBuf(buf) - } - ir.inputs = nil -} - -// DiscardAll discards all unstructured dependencies. This is useful to -// implementations that encode dependencies as [IRKindIdent] which are read back -// via ReadIdent. -func (ir *IRReader) DiscardAll() { - if ir.inputs == nil { - panic("attempting to discard dependencies twice") - } - ir.putAll() -} - -// ErrDependencyDepleted is returned when attempting to advance to the next -// unstructured dependency when there are none left. -var ErrDependencyDepleted = errors.New("reading past end of dependencies") - -// Next returns the next unstructured dependency. -func (ir *IRReader) Next() Artifact { - if len(ir.inputs) == 0 { - panic(ErrDependencyDepleted) - } - - id := unique.Make(ID(ir.inputs[0][wordSize:])) - ir.d.c.putIdentBuf(ir.inputs[0]) - ir.inputs = ir.inputs[1:] - - if a, ok := ir.d.ident[id]; !ok { - ir.putAll() - panic(DanglingIdentError(id)) - } else { - return a - } -} - -// MissedDependencyError is the number of unstructured dependencies remaining -// in [IRReader] that was never requested or explicitly discarded before -// finalisation. -type MissedDependencyError int - -func (e MissedDependencyError) Error() string { - return "missed " + strconv.Itoa(int(e)) + " unstructured dependencies" -} - -var ( - // ErrUnexpectedChecksum is returned by a [IRReadFunc] that does not expect - // a checksum but received one in [IRKindEnd] anyway. - ErrUnexpectedChecksum = errors.New("checksum specified on unsupported artifact") - // ErrExpectedChecksum is returned by a [IRReadFunc] that expects a checksum - // but did not receive one in [IRKindEnd]. - ErrExpectedChecksum = errors.New("checksum required but not specified") -) - -// Finalise reads the final [IRKindEnd] value and marks r as finalised. Methods -// of r are invalid upon entry into Finalise. If a [Checksum] is available via -// [IREndKnownChecksum], its handle is returned and the caller must store its -// value in the resulting [Artifact]. -func (ir *IRReader) Finalise() (checksum unique.Handle[Checksum], ok bool) { - ir.enterReader(true) - ir.count = -1 - - ir.mustReadHeader(IRKindEnd) - f := IREndFlag(ir.h.value()) - - if f&IREndKnownChecksum != 0 { - buf := ir.d.c.getIdentBuf() - defer ir.d.c.putIdentBuf(buf) - - ir.mustRead(buf[wordSize:]) - checksum = unique.Make(Checksum(buf[wordSize:])) - ok = true - } - - if err := ir.ibw.Flush(); err != nil { - panic(err) - } - ir.r, ir.ibw = nil, nil - - if len(ir.inputs) != 0 { - panic(MissedDependencyError(len(ir.inputs))) - } - - return -} - -// ReadIdent reads the next value as [IRKindIdent]. -func (ir *IRReader) ReadIdent() Artifact { - ir.enterReader(true) - ir.mustReadHeader(IRKindIdent) - - buf := ir.d.c.getIdentBuf() - defer ir.d.c.putIdentBuf(buf) - - ir.mustRead(buf[wordSize:]) - id := unique.Make(ID(buf[wordSize:])) - - if a, ok := ir.d.ident[id]; !ok { - panic(DanglingIdentError(id)) - } else { - return a - } -} - -// ReadUint32 reads the next value as [IRKindUint32]. -func (ir *IRReader) ReadUint32() uint32 { - ir.enterReader(true) - ir.mustReadHeader(IRKindUint32) - return ir.h.value() -} - -// ReadStringBytes reads the next value as [IRKindString] but returns it as a -// byte slice instead. -func (ir *IRReader) ReadStringBytes() []byte { - ir.enterReader(true) - ir.mustReadHeader(IRKindString) - - sz := int(ir.h.value()) - szWire := alignSize(sz) - if szWire > irMaxStringLength { - panic(IRStringError("\x00")) - } - - p := make([]byte, szWire) - ir.mustRead(p) - return p[:sz] -} - -// ReadString reads the next value as [IRKindString]. -func (ir *IRReader) ReadString() string { - p := ir.ReadStringBytes() - return unsafe.String(unsafe.SliceData(p), len(p)) -} - -// A RevisionError describes the first [Artifact] in an IR stream claiming a -// revision not supported by the registered implementation. -type RevisionError [2]uint64 - -func (e RevisionError) Error() string { - return "claimed revision " + strconv.FormatUint(e[0], 10) + - " differs from implementation value " + strconv.FormatUint(e[1], 10) -} - -// decode decodes the next [Artifact] in the IR stream and returns any buffer -// originating from [Cache] before returning. decode returns [io.EOF] if and -// only if the underlying [io.Reader] is already read to EOF. -func (d *IRDecoder) decode() (a Artifact, err error) { - defer panicToError(&err) - var ir IRReader - - defer func() { ir.d = nil }() - ir.d = d - - h := sha512.New384() - ir.ibw = d.c.getWriter(h) - defer d.c.putWriter(ir.ibw) - ir.r = io.TeeReader(d.r, ir.ibw) - - if n, _err := ir.readFull(ir.buf[:]); _err != nil { - if errors.Is(_err, io.EOF) { - if n != 0 { - _err = io.ErrUnexpectedEOF - } - } - - err = _err - return - } - ak := Kind(binary.LittleEndian.Uint64(ir.buf[:])) - f, ok := irArtifact[ak] - if !ok { - err = InvalidKindError(ak) - return - } - - ir.mustRead(ir.buf[:]) - rev := binary.LittleEndian.Uint64(ir.buf[:]) - - defer ir.putAll() - ir.mustRead(ir.buf[:]) - sz := binary.LittleEndian.Uint64(ir.buf[:]) - if sz > irMaxDeps { - err = ErrIRDepend - return - } - ir.inputs = make([]*extIdent, sz) - for i := range ir.inputs { - ir.inputs[i] = d.c.getIdentBuf() - } - for _, buf := range ir.inputs { - ir.mustRead(buf[:]) - } - - a = f(&ir) - if a == nil { - err = syscall.ENOTRECOVERABLE - return - } - if _rev := GetRevision(a); _rev != rev { - err = RevisionError{rev, _rev} - return - } - - if ir.count != -1 { - err = ErrRemainingIR - return - } - - buf := d.c.getIdentBuf() - h.Sum(buf[wordSize:wordSize]) - id := unique.Make(ID(buf[wordSize:])) - d.c.putIdentBuf(buf) - if _, ok = d.ident[id]; !ok { - d.ident[id] = a - } else { - d.c.msg.Verbosef( - "artifact %s%s%s appeared more than once in IR stream", - d.c.sgrIdent, Encode(id.Value()), d.c.sgrRes, - ) - } - - return -} - -// Decode consumes the IR stream to EOF and returns the final [Artifact]. After -// Decode returns, Lookup is available and Decode must not be called again. -func (d *IRDecoder) Decode() (a Artifact, err error) { - if d.done { - panic("attempting to decode an IR stream twice") - } - defer func() { d.done = true }() - - var cur Artifact -next: - a, err = d.decode() - - if err == nil { - cur = a - goto next - } - - if errors.Is(err, io.EOF) { - a, err = cur, nil - d.ok = true - } - return -} - -// Lookup looks up an [Artifact] described by the IR stream by its identifier. -func (d *IRDecoder) Lookup(id unique.Handle[ID]) (a Artifact, ok bool) { - if !d.ok { - panic("attempting to look up artifact without full IR stream") - } - a, ok = d.ident[id] - return -} diff --git a/internal/pkg/ir_test.go b/internal/pkg/ir_test.go deleted file mode 100644 index 312474bc..00000000 --- a/internal/pkg/ir_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package pkg_test - -import ( - "bytes" - "io" - "io/fs" - "reflect" - "testing" - - "hakurei.app/check" - "hakurei.app/internal/pkg" -) - -func TestIRRoundtrip(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - a pkg.Artifact - }{ - {"http get aligned", pkg.NewHTTPGet( - nil, "file:///testdata", - pkg.Checksum(bytes.Repeat([]byte{0xfd}, len(pkg.Checksum{}))), - )}, - {"http get unaligned", pkg.NewHTTPGet( - nil, "https://hakurei.app", - pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))), - )}, - - {"http get tar", pkg.NewTar(pkg.NewDecompress(pkg.NewHTTPGet( - nil, "file:///testdata", - pkg.Checksum(bytes.Repeat([]byte{0xff}, len(pkg.Checksum{}))), - ), pkg.Bzip2))}, - {"http get tar unaligned", pkg.NewTar(pkg.NewHTTPGet( - nil, "https://hakurei.app", - pkg.Checksum(bytes.Repeat([]byte{0xfe}, len(pkg.Checksum{}))), - ))}, - - {"exec offline", pkg.NewExec( - "exec-offline", "", nil, 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - - pkg.MustPath("/file", false, pkg.NewFile("file", []byte( - "stub file", - ))), pkg.MustPath("/.hakurei", false, pkg.NewTar(pkg.NewHTTPGet( - nil, "file:///hakurei.tar", - pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))), - ))), pkg.MustPath("/opt", false, pkg.NewTar(pkg.NewDecompress(pkg.NewHTTPGet( - nil, "file:///testtool.tar.gz", - pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))), - ), pkg.Gzip))), - )}, - - {"exec net", pkg.NewExec( - "exec-net", "", - (*pkg.Checksum)(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))), - 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool", "net"}, - - pkg.MustPath("/file", false, pkg.NewFile("file", []byte( - "stub file", - ))), pkg.MustPath("/.hakurei", false, pkg.NewTar(pkg.NewHTTPGet( - nil, "file:///hakurei.tar", - pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))), - ))), pkg.MustPath("/opt", false, pkg.NewTar(pkg.NewDecompress(pkg.NewHTTPGet( - nil, "file:///testtool.tar.gz", - pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))), - ), pkg.Gzip))), - )}, - - {"exec measured", pkg.NewExec( - "exec-measured", "", - (*pkg.Checksum)(bytes.Repeat([]byte{0xfd}, len(pkg.Checksum{}))), - 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool", "measured"}, - - pkg.MustPath("/file", false, pkg.NewFile("file", []byte( - "stub file", - ))), pkg.MustPath("/.hakurei", false, pkg.NewTar(pkg.NewHTTPGet( - nil, "file:///hakurei.tar", - pkg.Checksum(bytes.Repeat([]byte{0xfd}, len(pkg.Checksum{}))), - ))), pkg.MustPath("/opt", false, pkg.NewTar(pkg.NewDecompress(pkg.NewHTTPGet( - nil, "file:///testtool.tar.gz", - pkg.Checksum(bytes.Repeat([]byte{0xfd}, len(pkg.Checksum{}))), - ), pkg.Gzip))), - )}, - - {"file anonymous", pkg.NewFile("", []byte{0})}, - {"file", pkg.NewFile("stub", []byte("stub"))}, - - {"decompress", pkg.NewDecompress(pkg.NewFile("", []byte{0}), pkg.Bzip2)}, - - {"archive", pkg.NewArchive(pkg.NewFile("", []byte{0}))}, - } - testCasesCache := make([]cacheTestCase, len(testCases)) - for i, tc := range testCases { - want := tc.a - testCasesCache[i] = cacheTestCase{tc.name, 0, nil, - func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - r, w := io.Pipe() - - done := make(chan error, 1) - go func() { - t.Helper() - done <- c.EncodeAll(w, want) - _ = w.Close() - }() - - if got, err := c.NewDecoder(r).Decode(); err != nil { - t.Fatalf("Decode: error = %v", err) - } else if !reflect.DeepEqual(got, want) { - t.Fatalf("Decode: %#v, want %#v", got, want) - } - - if err := <-done; err != nil { - t.Fatalf("EncodeAll: error = %v", err) - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, - } - } - checkWithCache(t, testCasesCache) -} - -func TestRevision(t *testing.T) { - checkWithCache(t, []cacheTestCase{{"revision", 0, nil, - func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - a := pkg.NewFile("", nil) - var buf bytes.Buffer - if err := c.EncodeAll(&buf, a); err != nil { - t.Fatal(err) - } - - if _, err := c.NewDecoder( - bytes.NewReader(buf.Bytes()), - ).Decode(); err != nil { - t.Fatalf("Decode: error = %v", err) - } - - p := buf.Bytes() - p[8] = 0xfd - wantErr := pkg.RevisionError{0xfd, 0} - if _, err := c.NewDecoder( - bytes.NewReader(p), - ).Decode(); !reflect.DeepEqual(err, wantErr) { - t.Fatalf("Decode: error = %v, want %v", err, wantErr) - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, - }}) -} diff --git a/internal/pkg/net.go b/internal/pkg/net.go deleted file mode 100644 index cd30e3bb..00000000 --- a/internal/pkg/net.go +++ /dev/null @@ -1,109 +0,0 @@ -package pkg - -import ( - "fmt" - "io" - "net/http" - "path" - "unique" -) - -// An httpArtifact is an [Artifact] backed by a [http] url string. The method is -// hardcoded as [http.MethodGet]. Request body is not allowed because it cannot -// be deterministically represented by Params. -type httpArtifact struct { - // Caller-supplied url string. - url string - - // Caller-supplied checksum of the response body. This is validated when - // closing the [io.ReadCloser] returned by Cure. - checksum unique.Handle[Checksum] - - // client is the address of the caller-supplied [http.Client]. - client *http.Client -} - -var _ KnownChecksum = new(httpArtifact) -var _ fmt.Stringer = new(httpArtifact) - -// NewHTTPGet returns a new [FileArtifact] backed by the supplied client. A GET -// request is set up for url. If c is nil, [http.DefaultClient] is used instead. -func NewHTTPGet( - c *http.Client, - url string, - checksum Checksum, -) FileArtifact { - return &httpArtifact{url: url, checksum: unique.Make(checksum), client: c} -} - -// Kind returns the hardcoded [Kind] constant. -func (*httpArtifact) Kind() Kind { return KindHTTPGet } - -// Params writes the backing url string. Client is not represented as it does -// not affect [Cache.Cure] outcome. -func (a *httpArtifact) Params(ctx *IContext) { ctx.WriteString(a.url) } - -func init() { - register(KindHTTPGet, func(r *IRReader) Artifact { - url := r.ReadString() - checksum, ok := r.Finalise() - if !ok { - panic(ErrExpectedChecksum) - } - return NewHTTPGet(nil, url, checksum.Value()) - }) -} - -// Inputs returns a nil slice. -func (*httpArtifact) Inputs() []Artifact { return nil } - -// IsExclusive returns false: Cure returns as soon as a response is received. -func (*httpArtifact) IsExclusive() bool { return false } - -// Checksum returns the caller-supplied checksum. -func (a *httpArtifact) Checksum() Checksum { return a.checksum.Value() } - -// String returns [path.Base] over the backing url. -func (a *httpArtifact) String() string { return path.Base(a.url) } - -// ResponseStatusError is returned for a response returned by an [http.Client] -// with a status code other than [http.StatusOK]. -type ResponseStatusError int - -func (e ResponseStatusError) Error() string { - return "the requested URL returned non-OK status: " + http.StatusText(int(e)) -} - -// IsExecutable returns false. -func (*httpArtifact) IsExecutable() bool { return false } - -// Cure sends the http request and returns the resulting response body reader -// wrapped to perform checksum validation. It is valid but not encouraged to -// close the resulting [io.ReadCloser] before it is read to EOF, as that causes -// Close to block until all remaining data is consumed and validated. -func (a *httpArtifact) Cure(r *RContext) (rc io.ReadCloser, err error) { - var req *http.Request - req, err = http.NewRequestWithContext(r.Unwrap(), http.MethodGet, a.url, nil) - if err != nil { - return - } - req.Header.Set("User-Agent", "Hakurei/1.1") - - c := a.client - if c == nil { - c = http.DefaultClient - } - - var resp *http.Response - if resp, err = c.Do(req); err != nil { - return - } - - if resp.StatusCode != http.StatusOK { - _ = resp.Body.Close() - return nil, ResponseStatusError(resp.StatusCode) - } - - rc = r.NewMeasuredReader(resp.Body, a.checksum) - return -} diff --git a/internal/pkg/net_test.go b/internal/pkg/net_test.go deleted file mode 100644 index d809d35f..00000000 --- a/internal/pkg/net_test.go +++ /dev/null @@ -1,168 +0,0 @@ -package pkg_test - -import ( - "crypto/sha512" - "io" - "io/fs" - "net/http" - "reflect" - "testing" - "testing/fstest" - "unique" - - "hakurei.app/check" - "hakurei.app/internal/pkg" -) - -func TestHTTPGet(t *testing.T) { - t.Parallel() - - const testdata = "\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69" - - testdataChecksum := func() unique.Handle[pkg.Checksum] { - h := sha512.New384() - h.Write([]byte(testdata)) - return unique.Make(pkg.Checksum(h.Sum(nil))) - }() - - var transport http.Transport - client := http.Client{Transport: &transport} - transport.RegisterProtocol("file", http.NewFileTransportFS(fstest.MapFS{ - "testdata": {Data: []byte(testdata), Mode: 0400}, - })) - - checkWithCache(t, []cacheTestCase{ - {"direct", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - r := newRContext(t, c) - f := pkg.NewHTTPGet( - &client, - "file:///testdata", - testdataChecksum.Value(), - ) - var got []byte - if rc, err := f.Cure(r); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if got, err = io.ReadAll(rc); err != nil { - t.Fatalf("ReadAll: error = %v", err) - } else if string(got) != testdata { - t.Fatalf("Cure: %x, want %x", got, testdata) - } else if err = rc.Close(); err != nil { - t.Fatalf("Close: error = %v", err) - } - - // check direct validation - f = pkg.NewHTTPGet( - &client, - "file:///testdata", - pkg.Checksum{}, - ) - wantErrMismatch := &pkg.ChecksumMismatchError{ - Got: testdataChecksum.Value(), - } - if rc, err := f.Cure(r); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if got, err = io.ReadAll(rc); err != nil { - t.Fatalf("ReadAll: error = %v", err) - } else if string(got) != testdata { - t.Fatalf("Cure: %x, want %x", got, testdata) - } else if err = rc.Close(); !reflect.DeepEqual(err, wantErrMismatch) { - t.Fatalf("Close: error = %#v, want %#v", err, wantErrMismatch) - } - - // check fallback validation - if rc, err := f.Cure(r); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if err = rc.Close(); !reflect.DeepEqual(err, wantErrMismatch) { - t.Fatalf("Close: error = %#v, want %#v", err, wantErrMismatch) - } - - // check direct response error - f = pkg.NewHTTPGet( - &client, - "file:///nonexistent", - pkg.Checksum{}, - ) - wantErrNotFound := pkg.ResponseStatusError(http.StatusNotFound) - if _, err := f.Cure(r); !reflect.DeepEqual(err, wantErrNotFound) { - t.Fatalf("Cure: error = %#v, want %#v", err, wantErrNotFound) - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"cure", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - r := newRContext(t, c) - - f := pkg.NewHTTPGet( - &client, - "file:///testdata", - testdataChecksum.Value(), - ) - wantPathname := base.Append( - "identifier", - "COBRUwkxpIvZWJggpRxn7HVXiw6epXZOex8jGRJKpowpwukV7GRBY_LBvOJNQHue", - ) - if pathname, checksum, err := c.Cure(f); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if !pathname.Is(wantPathname) { - t.Fatalf("Cure: %q, want %q", pathname, wantPathname) - } else if checksum != testdataChecksum { - t.Fatalf("Cure: %x, want %x", checksum.Value(), testdataChecksum.Value()) - } - - var got []byte - if rc, err := f.Cure(r); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if got, err = io.ReadAll(rc); err != nil { - t.Fatalf("ReadAll: error = %v", err) - } else if string(got) != testdata { - t.Fatalf("Cure: %x, want %x", got, testdata) - } else if err = rc.Close(); err != nil { - t.Fatalf("Close: error = %v", err) - } - - // check load from cache - f = pkg.NewHTTPGet( - &client, - "file:///testdata", - testdataChecksum.Value(), - ) - if rc, err := f.Cure(r); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if got, err = io.ReadAll(rc); err != nil { - t.Fatalf("ReadAll: error = %v", err) - } else if string(got) != testdata { - t.Fatalf("Cure: %x, want %x", got, testdata) - } else if err = rc.Close(); err != nil { - t.Fatalf("Close: error = %v", err) - } - - // check error passthrough - f = pkg.NewHTTPGet( - &client, - "file:///nonexistent", - pkg.Checksum{}, - ) - wantErrNotFound := pkg.ResponseStatusError(http.StatusNotFound) - if _, _, err := c.Cure(f); !reflect.DeepEqual(err, wantErrNotFound) { - t.Fatalf("Pathname: error = %#v, want %#v", err, wantErrNotFound) - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU": {Mode: 0400, Data: []byte("\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/COBRUwkxpIvZWJggpRxn7HVXiw6epXZOex8jGRJKpowpwukV7GRBY_LBvOJNQHue": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} diff --git a/internal/pkg/pkg.go b/internal/pkg/pkg.go deleted file mode 100644 index 268d0248..00000000 --- a/internal/pkg/pkg.go +++ /dev/null @@ -1,2947 +0,0 @@ -// Package pkg provides utilities for packaging software. -package pkg - -import ( - "bufio" - "bytes" - "cmp" - "context" - "crypto/rand" - "crypto/sha512" - "encoding/base64" - "encoding/binary" - "errors" - "fmt" - "hash" - "io" - "io/fs" - "iter" - "maps" - "math" - "os" - "path/filepath" - "runtime" - "slices" - "strconv" - "strings" - "sync" - "sync/atomic" - "syscall" - "testing" - "time" - "unique" - "unsafe" - - "hakurei.app/check" - "hakurei.app/internal/info" - "hakurei.app/internal/lockedfile" - "hakurei.app/message" -) - -const ( - // programName is the string identifying this build system. - programName = "internal/pkg" -) - -type ( - // A Checksum is a SHA-384 checksum computed for a cured [Artifact]. - Checksum = [sha512.Size384]byte - - // An ID is a unique identifier returned by [KnownIdent.ID]. This value must - // be deterministically determined ahead of time. - ID Checksum -) - -// Encode is abbreviation for base64.URLEncoding.EncodeToString(checksum[:]). -func Encode(checksum Checksum) string { - return base64.URLEncoding.EncodeToString(checksum[:]) -} - -// Decode is abbreviation for base64.URLEncoding.Decode(checksum[:], []byte(s)). -func Decode(buf *Checksum, s string) (err error) { - var n int - n, err = base64.URLEncoding.Decode(buf[:], []byte(s)) - if err == nil && n != len(buf) { - err = io.ErrUnexpectedEOF - } - return -} - -// MustDecode decodes a string representation of [Checksum] and panics if there -// is a decoding error or the resulting data is too short. -func MustDecode(s string) (checksum Checksum) { - if err := Decode(&checksum, s); err != nil { - panic(err) - } - return -} - -var ( - // extension is a string uniquely identifying a set of custom [Artifact] - // implementations registered by calling [Register]. - extension string - - // openMu synchronises access to global state for initialisation. - openMu sync.Mutex - // opened is false if [Open] was never called. - opened bool -) - -// Extension returns a string uniquely identifying the currently registered set -// of custom [Artifact], or the zero value if none was registered. -func Extension() string { return extension } - -// ValidExtension returns whether s is valid for use in a call to SetExtension. -func ValidExtension(s string) bool { - if l := len(s); l == 0 || l > 128 { - return false - } - for _, v := range s { - if v < 'a' || v > 'z' { - return false - } - } - return true -} - -// ErrInvalidExtension is returned for a variant identification string for which -// [ValidExtension] returns false. -var ErrInvalidExtension = errors.New("invalid extension variant identification string") - -// SetExtension sets the extension variant identification string. SetExtension -// must be called before [Open] if custom [Artifact] implementations had been -// recorded by calling [Register]. -// -// The variant identification string must be between 1 and 128 bytes long and -// consists of only bytes between 'a' and 'z'. -// -// SetExtension is not safe for concurrent use. SetExtension is called at most -// once and must not be called after the first instance of Cache has been opened. -func SetExtension(s string) { - openMu.Lock() - defer openMu.Unlock() - - if opened { - panic("attempting to set extension after open") - } - if extension != "" { - panic("attempting to set extension twice") - } - if !ValidExtension(s) { - panic(ErrInvalidExtension) - } - extension = s - statusHeader = makeStatusHeader(s) -} - -// common holds elements and receives methods shared between different contexts. -type common struct { - // Context specific to this [Artifact]. The toplevel context in [Cache] must - // not be exposed directly. - ctx context.Context - - // Address of underlying [Cache], should be zeroed or made unusable after - // Cure returns and must not be exposed directly. - cache *Cache -} - -// TContext is passed to [TrivialArtifact.Cure] and provides information and -// methods required for curing the [TrivialArtifact]. -// -// Methods of TContext are safe for concurrent use. TContext is valid -// until [TrivialArtifact.Cure] returns. -type TContext struct { - // Populated during [Cache.Cure]. - work, temp *check.Absolute - - // Target [Artifact] encoded identifier. - ids string - // Pathname status was created at. - statusPath, statusSPath *check.Absolute - // File statusHeader and logs are written to. - status *os.File - // Error value during prepareStatus. - statusErr error - - common -} - -// makeStatusHeader creates the header written to every status file. This should -// not be called directly, its result is stored in statusHeader and will not -// change after the first [Cache] is opened. -func makeStatusHeader(extension string) string { - s := programName - if v := info.Version(); v != info.FallbackVersion { - s += " " + v - } - if extension != "" { - s += " with " + extension + " extensions" - } - s += " (" + runtime.GOARCH + ")" - if name, err := os.Hostname(); err == nil { - s += " on " + name - } - s += "\n\n" - return s -} - -// statusHeader is the header written to all status files in dirStatus. -var statusHeader = makeStatusHeader("") - -// prepareStatus initialises the status file once. -func (t *TContext) prepareStatus(writeHeader bool) error { - if t.statusPath != nil || t.status != nil { - return t.statusErr - } - - t.statusPath = t.cache.base.Append( - dirStatus, - t.ids, - ) - if t.status, t.statusErr = os.OpenFile( - t.statusPath.String(), - syscall.O_CREAT|syscall.O_EXCL|syscall.O_WRONLY, - 0400, - ); t.statusErr != nil { - return t.statusErr - } - - if writeHeader { - _, t.statusErr = t.status.WriteString(statusHeader) - } - return t.statusErr -} - -// GetStatusWriter returns a [io.Writer] for build logs. The caller must not -// seek this writer before the position it was first returned in. -func (t *TContext) GetStatusWriter() (io.Writer, error) { - err := t.prepareStatus(true) - return t.status, err -} - -// destroy destroys the temporary directory and joins its errors with the error -// referred to by errP. If the error referred to by errP is non-nil, the work -// directory is removed similarly. [Cache] is responsible for making sure work -// is never left behind for a successful [Cache.Cure]. -// -// If implementation had requested status, it is closed with error joined with -// the error referred to by errP. If the error referred to by errP is non-nil, -// the status file is removed from the filesystem. -// -// destroy must be deferred by [Cache.Cure] if [TContext] is passed to any Cure -// implementation. It should not be called prior to that point. -func (t *TContext) destroy(errP *error) { - if chmodErr, removeErr := removeAll(t.temp); chmodErr != nil || removeErr != nil { - *errP = errors.Join(*errP, chmodErr, removeErr) - } - - if *errP != nil { - chmodErr, removeErr := removeAll(t.work) - if chmodErr != nil || removeErr != nil { - *errP = errors.Join(*errP, chmodErr, removeErr) - } else if errors.Is(*errP, os.ErrExist) { - if linkError, ok := errors.AsType[*os.LinkError](*errP); ok && - linkError != nil && - linkError.Op == "rename" { - // two artifacts may be backed by the same file - *errP = nil - } - } - } - - if t.status != nil { - if err := t.status.Close(); err != nil { - *errP = errors.Join(*errP, err) - } - if *errP != nil { - *errP = errors.Join(*errP, os.Rename( - t.statusPath.String(), t.cache.base.Append( - dirFault, - t.ids+"."+strconv.FormatUint(uint64( - time.Now().UnixNano(), - ), 10), - ).String(), - )) - if t.statusSPath != nil { - t.cache.checksumMu.Lock() - *errP = errors.Join(*errP, os.Remove(t.statusSPath.String())) - t.cache.checksumMu.Unlock() - } - } - t.status = nil - } -} - -// Unwrap returns the underlying [context.Context]. -func (c *common) Unwrap() context.Context { return c.ctx } - -// GetMessage returns [message.Msg] held by the underlying [Cache]. -func (c *common) GetMessage() message.Msg { return c.cache.msg } - -// GetJobs returns the preferred number of jobs to run, when applicable. Its -// value must not affect cure outcome. -func (c *common) GetJobs() int { return c.cache.attr.Jobs } - -// GetLoad returns the preferred load average target, when applicable. Its -// value must not affect cure outcome. -func (c *common) GetLoad() int { return c.cache.attr.Load } - -// GetWorkDir returns a pathname to a directory which [Artifact] is expected to -// write its output to. This is not the final resting place of the [Artifact] -// and this pathname should not be directly referred to in the final contents. -func (t *TContext) GetWorkDir() *check.Absolute { return t.work } - -// GetTempDir returns a pathname which implementations may use as scratch space. -// A directory is not created automatically, implementations are expected to -// create it if they wish to use it, using [os.MkdirAll]. -func (t *TContext) GetTempDir() *check.Absolute { return t.temp } - -// Open tries to open [Artifact] for reading. If a implements [FileArtifact], -// its reader might be used directly, eliminating the roundtrip to vfs. -// Otherwise, it must cure into a directory containing a single regular file. -// -// If err is nil, the caller must close the resulting [io.ReadCloser] and return -// its error, if any. Failure to read r to EOF may result in a spurious -// [ChecksumMismatchError], or the underlying implementation may block on Close. -func (c *common) Open(a Artifact) (r io.ReadCloser, err error) { - if f, ok := a.(FileArtifact); ok { - return c.cache.openFile(c.ctx, f) - } - - var pathname *check.Absolute - if pathname, _, _, err = c.cache.cure(a, true, false); err != nil { - return - } - - var entries []os.DirEntry - if entries, err = os.ReadDir(pathname.String()); err != nil { - return - } - - if len(entries) != 1 || !entries[0].Type().IsRegular() { - err = errors.New( - "input directory does not contain a single regular file", - ) - return - } else { - return os.Open(pathname.Append(entries[0].Name()).String()) - } -} - -// FContext is passed to [FloodArtifact.Cure] and provides information and -// methods required for curing the [FloodArtifact]. -// -// Methods of FContext are safe for concurrent use. FContext is valid -// until [FloodArtifact.Cure] returns. -type FContext struct { - TContext - - // Cured top-level inputs looked up by Pathname. - inputs map[Artifact]cureRes -} - -// linkSubstitute links status for substitute if populated. -func (f *FContext) linkSubstitute(ids, substitutes string) (err error) { - if f.status == nil || ids == substitutes { - return - } - - statusS := f.cache.base.Append( - dirStatus, - substitutes, - ) - f.cache.checksumMu.Lock() - err = os.Link(f.cache.base.Append( - dirStatus, - ids, - ).String(), statusS.String()) - f.cache.checksumMu.Unlock() - if err == nil { - f.statusSPath = statusS - } - return -} - -// InvalidLookupError is the identifier of non-input [Artifact] looked up -// via [FContext.GetArtifact] by a misbehaving [Artifact] implementation. -type InvalidLookupError ID - -func (e InvalidLookupError) Error() string { - return "attempting to look up non-input artifact " + Encode(e) -} - -var _ error = InvalidLookupError{} - -// GetArtifact returns the identifier pathname and checksum of an [Artifact]. -// Calling Pathname with an [Artifact] not part of the slice returned by -// [Artifact.Inputs] panics. -func (f *FContext) GetArtifact(a Artifact) ( - pathname *check.Absolute, - checksum unique.Handle[Checksum], -) { - if res, ok := f.inputs[a]; ok { - return res.pathname, res.checksum - } - panic(InvalidLookupError(f.cache.Ident(a).Value())) -} - -// RContext is passed to [FileArtifact.Cure] and provides helper methods useful -// for curing the [FileArtifact]. -// -// Methods of RContext are safe for concurrent use. RContext is valid -// until [FileArtifact.Cure] returns. -type RContext struct{ common } - -// An Artifact is a read-only reference to a piece of data that may be created -// deterministically but might not currently be available in memory or on the -// filesystem. -type Artifact interface { - // Kind returns the [Kind] of artifact. This is usually unique to the - // concrete type but two functionally identical implementations of - // [Artifact] is allowed to return the same [Kind] value. - Kind() Kind - - // Params writes deterministic values describing [Artifact]. Implementations - // must guarantee that these values are unique among differing instances - // of the same implementation with identical dependencies and conveys enough - // information to create another instance of [Artifact] identical to the - // instance emitting these values. The new instance created via [IRReadFunc] - // from these values must then produce identical IR values. - // - // Result must remain identical across multiple invocations. - Params(ctx *IContext) - - // Inputs returns a slice of [Artifact] the current instance has access to - // while producing its output. - // - // Callers must not modify the retuned slice. - // - // Result must remain identical across multiple invocations. - Inputs() []Artifact - - // IsExclusive returns whether the [Artifact] is exclusive. Exclusive - // artifacts might not run in parallel with each other, and are still - // subject to the cures limit. - // - // Some implementations may saturate the CPU for a nontrivial amount of - // time. Curing multiple such implementations simultaneously causes - // significant CPU scheduler overhead. An exclusive artifact will generally - // not be cured alongside another exclusive artifact, thus alleviating this - // overhead. - // - // Note that [Cache] reserves the right to still cure exclusive - // artifacts concurrently as this is not a synchronisation primitive but - // an optimisation one. Implementations are forbidden from accessing global - // state regardless of exclusivity. - // - // Result must remain identical across multiple invocations. - IsExclusive() bool -} - -// FloodArtifact refers to an [Artifact] requiring its entire dependency graph -// to be cured prior to curing itself. -type FloodArtifact interface { - // Cure cures the current [Artifact] to the working directory obtained via - // [TContext.GetWorkDir] embedded in [FContext]. - // - // Implementations must not retain c. - Cure(f *FContext) (err error) - - Artifact -} - -// TrivialArtifact refers to an [Artifact] that cures without requiring that -// any other [Artifact] is cured before it. Its dependency tree is ignored after -// computing its identifier. -// -// TrivialArtifact is unable to cure any other [Artifact] and it cannot access -// pathnames. This type of [Artifact] is primarily intended for dependency-less -// artifacts or direct dependencies that only consists of [FileArtifact]. -type TrivialArtifact interface { - // Cure cures the current [Artifact] to the working directory obtained via - // [TContext.GetWorkDir]. - // - // Implementations must not retain c. - Cure(t *TContext) (err error) - - Artifact -} - -// KnownIdent is optionally implemented by [Artifact] and is used instead of -// [Cache.Ident] when it is available. -// -// This is very subtle to use correctly. The implementation must ensure that -// this value is globally unique, otherwise [Cache] can enter an inconsistent -// state. This should not be implemented outside of testing. -type KnownIdent interface { - // ID returns a globally unique identifier referring to the current - // [Artifact]. This value must be known ahead of time and guaranteed to be - // unique without having obtained the full contents of the [Artifact]. - ID() ID - - Artifact -} - -// KnownChecksum is optionally implemented by [Artifact] for an artifact with -// output known ahead of time. -type KnownChecksum interface { - // Checksum returns the address of a known checksum. - // - // Callers must not modify the [Checksum]. - // - // Result must remain identical across multiple invocations. - Checksum() Checksum - - Artifact -} - -// CuresExempt is optionally implemented for an artifact exempt to the -// cache-wide cures counter and limit. -type CuresExempt interface { - Artifact - - // CuresExempt is a no-op function but serves to distinguish implementations - // that are cures-exempt. - CuresExempt() -} - -// FileArtifact refers to an [Artifact] backed by a single file. -// -// FileArtifact does not support fine-grained cancellation. Its context is -// inherited from the first [TrivialArtifact] or [FloodArtifact] that opens it. -type FileArtifact interface { - // IsExecutable returns whether the resulting filesystem entry should be made - // executable, if the [FileArtifact] is cured to the on-disk cache. - // - // Result must remain identical across multiple invocations. - IsExecutable() bool - - // Cure returns [io.ReadCloser] of the full contents of [FileArtifact]. If - // [FileArtifact] implements [KnownChecksum], Cure is responsible for - // validating any data it produces and must return [ChecksumMismatchError] - // if validation fails. This error is conventionally returned during the - // first call to Close, but may be returned during any call to Read before - // EOF, or by Cure itself. - // - // Callers are responsible for closing the resulting [io.ReadCloser]. - // - // The resulting [io.ReadCloser] across multiple invocations must have - // identical behaviour. - Cure(r *RContext) (io.ReadCloser, error) - - Artifact -} - -// RevisionArtifact is optionally implemented by an artifact that had undergone -// internal changes affecting its behaviour while retaining its IR structure. -type RevisionArtifact interface { - // Revision returns the revision number of [Artifact]. This value is always - // represented in the IR. An IR stream produced from the same [Kind] with - // differing revision is rejected. - // - // Result must remain identical across multiple invocations. - Revision() uint64 - - Artifact -} - -// GetRevision returns the revision number of an [Artifact]. -func GetRevision(a Artifact) (revision uint64) { - revision = math.MaxUint64 - if r, ok := a.(RevisionArtifact); ok { - revision = r.Revision() - } - return -} - -// reportName returns a string describing [Artifact] presented to the user. -func reportName(a Artifact, id unique.Handle[ID]) string { - r := Encode(id.Value()) - if s, ok := a.(fmt.Stringer); ok { - if name := s.String(); name != "" { - r += "-" + name - } - } - return r -} - -// Kind corresponds to the concrete type of [Artifact] and is used to create -// identifier for an [Artifact] with dependencies. -type Kind uint64 - -const ( - // KindHTTPGet is the kind of [Artifact] returned by [NewHTTPGet]. - KindHTTPGet Kind = iota - // KindTar is the kind of [Artifact] returned by [NewTar]. - KindTar - // KindExec is the kind of [Artifact] returned by [NewExec]. - KindExec - // KindExecNet is the kind of [Artifact] returned by [NewExec] but with a - // non-nil checksum. - KindExecNet - // KindFile is the kind of [Artifact] returned by [NewFile]. - KindFile - // KindDecompress is the kind of [Artifact] returned by [NewDecompress]. - KindDecompress - // KindArchive is the kind of [Artifact] returned by [NewArchive]. - KindArchive - - // _kindEnd is the total number of kinds and does not denote a kind. - _kindEnd - - // KindCustomOffset is the first [Kind] value reserved for implementations - // not from this package. - KindCustomOffset = 1 << 31 -) - -const ( - // kindCollection is the kind of [Collect]. It never cures successfully. - kindCollection Kind = KindCustomOffset - 1 - iota -) - -const ( - // fileLock is the lock file for exclusive access to the cache directory. - fileLock = "lock" - // fileVariant is a file holding the variant identification string set by a - // prior call to [SetExtension]. - fileVariant = "variant" - - // dirSubstitute holds symlinks to artifacts by checksum, named after their - // substitute identifier. - dirSubstitute = "substitute" - // dirIdentifier holds symlinks to artifacts by checksum, named after their - // IR-based identifier. - dirIdentifier = "identifier" - // dirChecksum holds artifacts named after their [Checksum]. - dirChecksum = "checksum" - // dirStatus holds artifact metadata and logs named after their IR-based - // identifier. For [FloodArtifact], the same file is also available under - // its substitute identifier. - dirStatus = "status" - // dirFault holds status files of faulted cures. - dirFault = "fault" - - // dirWork holds working pathnames set up during [Cache.Cure]. - dirWork = "work" - // dirTemp holds scratch space allocated during [Cache.Cure]. - dirTemp = "temp" - - // dirExecScratch is scratch space set up for the container started by - // [Cache.EnterExec]. Exclusivity via Cache.inExec. - dirExecScratch = "scratch" - - // checksumLinknamePrefix is prepended to the encoded [Checksum] value - // of an [Artifact] when creating a symbolic link to dirChecksum. - checksumLinknamePrefix = "../" + dirChecksum + "/" -) - -// cureRes are the non-error results returned by [Cache.Cure]. -type cureRes struct { - pathname *check.Absolute - checksum unique.Handle[Checksum] -} - -// A pendingArtifactDep is an input [Artifact] pending concurrent curing, -// subject to the cures limit. Values pointed to by result addresses are safe -// to access after the [sync.WaitGroup] associated with this pendingArtifactDep -// is done. pendingArtifactDep must not be reused or modified after it is sent -// to cure. -type pendingArtifactDep struct { - // Dependency artifact populated during [Cache.Cure]. - a Artifact - - // Address of result pathname populated during [Cache.Cure] and dereferenced - // if curing succeeds. - resP *cureRes - - // Address of result error map populated during [Cache.Cure], dereferenced - // after acquiring errsMu if curing fails. No additional action is taken, - // [Cache] and its caller are responsible for further error handling. - errs InputError - // Address of mutex synchronising access to errs. - errsMu *sync.Mutex - - // For synchronising access to result buffer. - *sync.WaitGroup -} - -const ( - // CValidateKnown arranges for [KnownChecksum] outcomes to be validated to - // match its intended checksum. - // - // A correct implementation of [KnownChecksum] does not successfully cure - // with output not matching its intended checksum. When an implementation - // fails to perform this validation correctly, the on-disk format enters - // an inconsistent state (correctable by [Cache.Scrub]). - // - // This flag causes [Cache.Cure] to always compute the checksum, and reject - // a cure if it does not match the intended checksum. - // - // This behaviour significantly reduces performance and is not recommended - // outside of testing a custom [Artifact] implementation. - CValidateKnown = 1 << iota - - // CSchedIdle arranges for the [ext.SCHED_IDLE] scheduling priority to be - // set for [KindExec] and [KindExecNet] containers. - CSchedIdle - - // CAssumeChecksum enables the use of [KnownChecksum] for duplicate function - // call suppression via the on-disk cache. - // - // This may cause incorrect cure outcome if an impossible checksum is - // specified that matches an output already present in the on-disk cache. - // This may be avoided by purposefully specifying a statistically - // unattainable checksum, like the zero value. - // - // While this optimisation might seem appealing, it is almost never - // applicable in real world use. Almost every time this path was taken, it - // was caused by an incorrect checksum accidentally left behind while - // bumping a package. Only enable this if you are really sure you need it. - CAssumeChecksum - - // CHostAbstract disables restriction of sandboxed processes from connecting - // to an abstract UNIX socket created by a host process. - // - // This is considered less secure in some systems, but does not introduce - // impurity due to [KindExecNet] being [KnownChecksum]. This flag exists - // to support kernels without Landlock LSM enabled. - CHostAbstract - - // CPromoteVariant allows [pkg.Open] to promote an unextended on-disk cache - // to the current extension variant. This is a one-way operation. - CPromoteVariant - - // CSuppressInit arranges for verbose output of the container init to be - // suppressed regardless of [message.Msg] state. - CSuppressInit - - // CIgnoreSubstitutes disables content-based input substitution. - CIgnoreSubstitutes - - // CExternShallow arranges for only non-flood inputs to be fetched when - // curing an [Artifact] available via the external cache. - CExternShallow - - // CColourOutput enables output colouring via ANSI control sequences. - CColourOutput -) - -// toplevel holds [context.WithCancel] over caller-supplied context, where all -// [Artifact] context are derived from. -type toplevel struct { - ctx context.Context - cancel context.CancelFunc -} - -// newToplevel returns the address of a new toplevel via ctx. -func newToplevel(ctx context.Context) *toplevel { - var t toplevel - t.ctx, t.cancel = context.WithCancel(ctx) - return &t -} - -// pendingCure provides synchronisation and cancellation for pending cures. -type pendingCure struct { - // Closed on cure completion. - done <-chan struct{} - // Error outcome, safe to access after done is closed. - err error - // Cancels the corresponding cure. - cancel context.CancelFunc -} - -// An External cache provides prepared [Artifact] cure outcomes. -type External interface { - // Artifact returns the address of the [Checksum] of the cure outcome of - // an [Artifact] corresponding to id, or nil if this [Artifact] is not - // available in the external cache. - Artifact(ctx context.Context, id unique.Handle[ID]) (*Checksum, error) - // Checksum returns an [Artifact] producing the specified checksum. - Checksum(checksum unique.Handle[Checksum]) Artifact - // Status returns [io.ReadCloser] of the status file of an [Artifact] - // corresponding to id, or nil if this [Artifact] is not available or a - // status file is not present. - Status(r *RContext, id unique.Handle[ID]) (io.ReadCloser, error) -} - -// Cache is a support layer that implementations of [Artifact] can use to store -// cured [Artifact] data in a content addressed fashion. -type Cache struct { - // Cures of any variant of [Artifact] sends to cures before entering the - // implementation and receives an equal amount of elements after. - cures chan struct{} - - // Parent context which toplevel was derived from. - parent context.Context - // For deriving curing context, must not be accessed directly. - toplevel atomic.Pointer[toplevel] - // For waiting on input curing goroutines. - wg sync.WaitGroup - // Reports new cures and passed to [Artifact]. - msg message.Msg - // Select graphics rendition sequences, populated by Open. - sgrRes, sgrIdent, sgrWarn, sgrErr string - - // Directory where all [Cache] related files are placed. - base *check.Absolute - // Immutable [CacheAttr] populated by [Open]. - attr CacheAttr - - // Must not be exposed directly. - irCache - - // Synchronises access to dirChecksum. - checksumMu sync.RWMutex - - // Presence of an alternative in the cache. Keys are not valid identifiers - // and must not be used as such. - substitute map[unique.Handle[ID]]unique.Handle[Checksum] - // Synchronises access to substitute and corresponding filesystem entries. - substituteMu sync.RWMutex - // Identifier to content pair cache. - ident map[unique.Handle[ID]]unique.Handle[Checksum] - // Identifier to error pair for unrecoverably faulted [Artifact]. - identErr map[unique.Handle[ID]]error - // Pending identifiers, accessed through Cure for entries not in ident. - identPending map[unique.Handle[ID]]*pendingCure - // Synchronises access to ident and corresponding filesystem entries. - identMu sync.RWMutex - // Synchronises entry into Abort and Cure. - abortMu sync.RWMutex - - // Synchronises entry into exclusive artifacts for the cure method. - exclMu sync.Mutex - // Buffered I/O free list, must not be accessed directly. - brPool, bwPool sync.Pool - - // Optional external cache implementation. - extern External - // Caches responses from extern. - externCache map[unique.Handle[ID]]unique.Handle[Checksum] - // Synchronises access to extern. - externMu sync.RWMutex - - // Unlocks the on-filesystem cache. Must only be called from Close. - unlock func() - // Whether [Cache] is considered closed. - closed bool - // Synchronises calls to Abort and Close. - closeMu sync.Mutex - - // Whether EnterExec has not yet returned. - inExec atomic.Bool -} - -// extIdent is a [Kind] concatenated with [ID]. -type extIdent [wordSize + len(ID{})]byte - -// getIdentBuf returns the address of an extIdent for Ident. -func (ic *irCache) getIdentBuf() *extIdent { return ic.identPool.Get().(*extIdent) } - -// putIdentBuf adds buf to identPool. -func (ic *irCache) putIdentBuf(buf *extIdent) { ic.identPool.Put(buf) } - -// storeIdent adds an [Artifact] to the artifact cache. -func (ic *irCache) storeIdent(a Artifact, buf *extIdent) unique.Handle[ID] { - idu := unique.Make(ID(buf[wordSize:])) - ic.artifact.Store(a, idu) - return idu -} - -// Ident returns the identifier of an [Artifact]. -func (ic *irCache) Ident(a Artifact) unique.Handle[ID] { - buf, idu := ic.unsafeIdent(a, false) - if buf != nil { - idu = ic.storeIdent(a, buf) - ic.putIdentBuf(buf) - } - return idu -} - -// unsafeIdent implements Ident but returns the underlying buffer for a newly -// computed identifier. Callers must return this buffer to identPool. encodeKind -// is only a hint, kind may still be encoded in the buffer. -func (ic *irCache) unsafeIdent(a Artifact, encodeKind bool) ( - buf *extIdent, - idu unique.Handle[ID], -) { - if id, ok := ic.artifact.Load(a); ok { - idu = id.(unique.Handle[ID]) - return - } - - if ki, ok := a.(KnownIdent); ok { - buf = ic.getIdentBuf() - if encodeKind { - binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind())) - } - *(*ID)(buf[wordSize:]) = ki.ID() - return - } - - buf = ic.getIdentBuf() - h := sha512.New384() - if err := ic.Encode(h, a); err != nil { - // unreachable - panic(err) - } - binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind())) - h.Sum(buf[wordSize:wordSize]) - return -} - -// getReader is like [bufio.NewReader] but for brPool. -func (c *Cache) getReader(r io.Reader) *bufio.Reader { - br := c.brPool.Get().(*bufio.Reader) - br.Reset(r) - return br -} - -// putReader adds br to brPool. -func (c *Cache) putReader(br *bufio.Reader) { c.brPool.Put(br) } - -// bufioReadCloser is the concrete type of value returned by Cache.getReaderRC. -type bufioReadCloser struct { - // Saved close error. - closeErr error - // Synchronises calls to Close. - closeOnce sync.Once - - // For backing freelist. - c *Cache - // Underlying reader. - r io.ReadCloser - // Allocated from c. - *bufio.Reader -} - -// Close closes the underlying reader, saves its return value, and returns the -// [bufio.Reader] instance to the backing [Cache]. -func (brc *bufioReadCloser) Close() error { - brc.closeOnce.Do(func() { - br := brc.Reader - brc.Reader = nil - brc.c.putReader(br) - brc.closeErr = brc.r.Close() - }) - return brc.closeErr -} - -// getReaderRC is like getReader, but returns an [io.ReadCloser]. -func (c *Cache) getReaderRC(r io.ReadCloser) io.ReadCloser { - return &bufioReadCloser{c: c, r: r, Reader: c.getReader(r)} -} - -// getWriter is like [bufio.NewWriter] but for bwPool. -func (c *Cache) getWriter(w io.Writer) *bufio.Writer { - bw := c.bwPool.Get().(*bufio.Writer) - bw.Reset(w) - return bw -} - -// putWriter adds bw to bwPool. -func (c *Cache) putWriter(bw *bufio.Writer) { c.bwPool.Put(bw) } - -// A ChecksumMismatchError describes an [Artifact] with unexpected content. -type ChecksumMismatchError struct { - // Actual and expected checksums. - Got, Want Checksum -} - -func (e *ChecksumMismatchError) Error() string { - return "got " + Encode(e.Got) + - " instead of " + Encode(e.Want) -} - -// LinknamePrefixError describes a malformed linkname to a [Checksum]. -type LinknamePrefixError string - -func (e LinknamePrefixError) Error() string { - return "linkname " + strconv.Quote(string(e)) + " missing prefix" -} - -// readlinkChecksum reads a symbolic link to a dirChecksum entry and saves the -// decoded [Checksum] to the value pointed to by buf. The checksumLinknamePrefix -// is required. -func readlinkChecksum(a *check.Absolute, buf *Checksum) error { - linkname, err := os.Readlink(a.String()) - if err != nil { - return nil - } - - if !strings.HasPrefix(linkname, checksumLinknamePrefix) { - return LinknamePrefixError(linkname) - } - return Decode(buf, linkname[len(checksumLinknamePrefix):]) -} - -// SetExternal sets e as the [External] implementation of c. -func (c *Cache) SetExternal(e External) { - c.externMu.Lock() - c.externCache = make(map[unique.Handle[ID]]unique.Handle[Checksum]) - c.extern = e - c.externMu.Unlock() -} - -// ScrubError describes the outcome of a [Cache.Scrub] call where errors were -// found and removed from the underlying storage of [Cache]. -type ScrubError struct { - // Content-addressed entries not matching their checksum. This can happen - // if an incorrect [FileArtifact] implementation was cured against - // a non-strict [Cache]. - ChecksumMismatches []ChecksumMismatchError - // Dangling identifier symlinks. This can happen if the content-addressed - // entry was removed while scrubbing due to a checksum mismatch. - DanglingIdentifiers []ID - // Dangling status files. This can happen if a dangling status symlink was - // removed while scrubbing. - DanglingStatus []ID - // Miscellaneous errors, including [os.ReadDir] on checksum and identifier - // directories, [Decode] on entry names and [os.RemoveAll] on inconsistent - // entries. - Errs map[unique.Handle[string]][]error -} - -// errs is a deterministic iterator over Errs. -func (e *ScrubError) errs(yield func(unique.Handle[string], []error) bool) { - keys := slices.AppendSeq( - make([]unique.Handle[string], 0, len(e.Errs)), - maps.Keys(e.Errs), - ) - slices.SortFunc(keys, func(a, b unique.Handle[string]) int { - return strings.Compare(a.Value(), b.Value()) - }) - for _, key := range keys { - if !yield(key, e.Errs[key]) { - break - } - } -} - -// Unwrap returns a concatenation of ChecksumMismatches and Errs. -func (e *ScrubError) Unwrap() []error { - s := make([]error, 0, len(e.ChecksumMismatches)+len(e.Errs)) - for _, err := range e.ChecksumMismatches { - s = append(s, &err) - } - for _, errs := range e.errs { - s = append(s, errs...) - } - return s -} - -// Error returns a multi-line representation of [ScrubError]. -func (e *ScrubError) Error() string { - var segments []string - var buf strings.Builder - - if len(e.ChecksumMismatches) > 0 { - buf.Reset() - buf.WriteString("checksum mismatches:\n") - for _, m := range e.ChecksumMismatches { - buf.WriteString(m.Error() + "\n") - } - segments = append(segments, buf.String()) - } - if len(e.DanglingIdentifiers) > 0 { - buf.Reset() - buf.WriteString("dangling identifiers:\n") - for _, id := range e.DanglingIdentifiers { - buf.WriteString(Encode(id) + "\n") - } - segments = append(segments, buf.String()) - } - if len(e.DanglingStatus) > 0 { - buf.Reset() - buf.WriteString("dangling status:\n") - for _, id := range e.DanglingStatus { - buf.WriteString(Encode(id) + "\n") - } - segments = append(segments, buf.String()) - } - if len(e.Errs) > 0 { - buf.Reset() - buf.WriteString("errors during scrub:\n") - for pathname, errs := range e.errs { - buf.WriteString(" " + pathname.Value() + ":\n") - for _, err := range errs { - buf.WriteString(" " + err.Error() + "\n") - } - } - segments = append(segments, buf.String()) - } - return strings.Join(segments, "\n") -} - -// Scrub frees internal in-memory identifier to content pair cache, verifies all -// cached artifacts against their checksums, checks for dangling identifier -// symlinks and removes them if found. -// -// This method is not safe for concurrent use with any other method. -func (c *Cache) Scrub(checks int) error { - if checks <= 0 { - checks = runtime.NumCPU() - } - - c.substituteMu.Lock() - defer c.substituteMu.Unlock() - c.identMu.Lock() - defer c.identMu.Unlock() - c.checksumMu.Lock() - defer c.checksumMu.Unlock() - - c.substitute = make(map[unique.Handle[ID]]unique.Handle[Checksum]) - c.ident = make(map[unique.Handle[ID]]unique.Handle[Checksum]) - c.identErr = make(map[unique.Handle[ID]]error) - c.artifact.Clear() - - var ( - se = ScrubError{Errs: make(map[unique.Handle[string]][]error)} - seMu sync.Mutex - - addErr = func(pathname *check.Absolute, err error) { - seMu.Lock() - se.Errs[pathname.Handle()] = append(se.Errs[pathname.Handle()], err) - seMu.Unlock() - } - ) - - type checkEntry struct { - ent os.DirEntry - check func(ent os.DirEntry, want *Checksum) bool - } - var ( - dir *check.Absolute - wg sync.WaitGroup - w = make(chan checkEntry, checks) - p = sync.Pool{New: func() any { return new(Checksum) }} - ) - condemn := func(ent os.DirEntry) { - pathname := dir.Append(ent.Name()) - chmodErr, removeErr := removeAll(pathname) - if chmodErr != nil { - addErr(pathname, chmodErr) - } - if removeErr != nil { - addErr(pathname, removeErr) - } - } - for i := 0; i < checks; i++ { - go func() { - for ce := range w { - want := p.Get().(*Checksum) - ent := ce.ent - if err := Decode(want, ent.Name()); err != nil { - addErr(dir.Append(ent.Name()), err) - wg.Go(func() { condemn(ent) }) - } else if !ce.check(ent, want) { - wg.Go(func() { condemn(ent) }) - } else { - c.msg.Verbosef( - "%s%s%s is consistent", - c.sgrIdent, ent.Name(), c.sgrRes, - ) - } - p.Put(want) - wg.Done() - } - }() - } - defer close(w) - - dir = c.base.Append(dirChecksum) - if entries, readdirErr := os.ReadDir(dir.String()); readdirErr != nil { - addErr(dir, readdirErr) - } else { - wg.Add(len(entries)) - for _, ent := range entries { - w <- checkEntry{ent, func(ent os.DirEntry, want *Checksum) bool { - got := p.Get().(*Checksum) - defer p.Put(got) - - pathname := dir.Append(ent.Name()) - if ent.IsDir() { - if err := SumDir(got, pathname); err != nil { - addErr(pathname, err) - return true - } - } else if ent.Type().IsRegular() { - h := sha512.New384() - - if r, err := os.Open(pathname.String()); err != nil { - addErr(pathname, err) - return true - } else { - _, err = io.Copy(h, r) - closeErr := r.Close() - if closeErr != nil { - addErr(pathname, closeErr) - } - if err != nil { - addErr(pathname, err) - } - } - h.Sum(got[:0]) - } else { - addErr(pathname, InvalidFileModeError(ent.Type())) - return false - } - - if *got != *want { - seMu.Lock() - se.ChecksumMismatches = append(se.ChecksumMismatches, - ChecksumMismatchError{Got: *got, Want: *want}, - ) - seMu.Unlock() - return false - } - return true - }} - } - wg.Wait() - } - - for _, suffix := range []string{ - dirSubstitute, - dirIdentifier, - } { - dir = c.base.Append(suffix) - if entries, readdirErr := os.ReadDir(dir.String()); readdirErr != nil { - addErr(dir, readdirErr) - } else { - wg.Add(len(entries)) - for _, ent := range entries { - w <- checkEntry{ent, func(ent os.DirEntry, want *Checksum) bool { - got := p.Get().(*Checksum) - defer p.Put(got) - - pathname := dir.Append(ent.Name()) - if linkname, err := os.Readlink( - pathname.String(), - ); err != nil { - seMu.Lock() - se.Errs[pathname.Handle()] = append(se.Errs[pathname.Handle()], err) - se.DanglingIdentifiers = append(se.DanglingIdentifiers, *want) - seMu.Unlock() - return false - } else if err = Decode(got, filepath.Base(linkname)); err != nil { - seMu.Lock() - lnp := dir.Append(linkname) - se.Errs[lnp.Handle()] = append(se.Errs[lnp.Handle()], err) - se.DanglingIdentifiers = append(se.DanglingIdentifiers, *want) - seMu.Unlock() - return false - } - - if _, err := os.Stat(pathname.String()); err != nil { - if !errors.Is(err, os.ErrNotExist) { - addErr(pathname, err) - } - seMu.Lock() - se.DanglingIdentifiers = append(se.DanglingIdentifiers, *want) - seMu.Unlock() - return false - } - return true - }} - } - wg.Wait() - } - } - - dir = c.base.Append(dirStatus) - if entries, readdirErr := os.ReadDir(dir.String()); readdirErr != nil { - if !errors.Is(readdirErr, os.ErrNotExist) { - addErr(dir, readdirErr) - } - } else { - wg.Add(len(entries)) - for _, ent := range entries { - w <- checkEntry{ent, func(ent os.DirEntry, want *Checksum) bool { - got := p.Get().(*Checksum) - defer p.Put(got) - - var ok bool - for _, name := range [...]string{ - dirIdentifier, - dirSubstitute, - } { - if _, err := os.Stat(c.base.Append( - name, - ent.Name(), - ).String()); err != nil { - if !errors.Is(err, os.ErrNotExist) { - addErr(dir.Append(ent.Name()), err) - } - continue - } - ok = true - } - if !ok { - seMu.Lock() - se.DanglingStatus = append(se.DanglingStatus, *want) - seMu.Unlock() - } - return ok - }} - } - wg.Wait() - } - - if len(c.identPending) > 0 { - addErr(c.base, errors.New( - "scrub began with pending artifacts", - )) - } else { - pathname := c.base.Append(dirWork) - chmodErr, removeErr := removeAll(pathname) - if chmodErr != nil { - addErr(pathname, chmodErr) - } - if removeErr != nil { - addErr(pathname, removeErr) - } - - if err := os.Mkdir(pathname.String(), 0700); err != nil { - addErr(pathname, err) - } - - pathname = c.base.Append(dirTemp) - chmodErr, removeErr = removeAll(pathname) - if chmodErr != nil { - addErr(pathname, chmodErr) - } - if removeErr != nil { - addErr(pathname, removeErr) - } - } - - if len(se.ChecksumMismatches) > 0 || - len(se.DanglingIdentifiers) > 0 || - len(se.DanglingStatus) > 0 || - len(se.Errs) > 0 { - slices.SortFunc(se.ChecksumMismatches, func(a, b ChecksumMismatchError) int { - return bytes.Compare(a.Want[:], b.Want[:]) - }) - slices.SortFunc(se.DanglingIdentifiers, func(a, b ID) int { - return bytes.Compare(a[:], b[:]) - }) - slices.SortFunc(se.DanglingStatus, func(a, b ID) int { - return bytes.Compare(a[:], b[:]) - }) - return &se - } else { - return nil - } -} - -// loadOrStoreIdent attempts to load a cached [Artifact] by its identifier or -// wait for a pending [Artifact] to cure. If neither is possible, the current -// identifier is stored in identPending and a non-nil channel is returned. -// -// Since identErr is treated as grow-only, loadOrStoreIdent must not be entered -// without holding a read lock on abortMu. -func (c *Cache) loadOrStoreIdent(id unique.Handle[ID]) ( - ctx context.Context, - done chan<- struct{}, - checksum unique.Handle[Checksum], - err error, -) { - var ok bool - - c.identMu.Lock() - if checksum, ok = c.ident[id]; ok { - c.identMu.Unlock() - return - } - if err, ok = c.identErr[id]; ok { - c.identMu.Unlock() - return - } - - var pending *pendingCure - if pending, ok = c.identPending[id]; ok { - c.identMu.Unlock() - <-pending.done - c.identMu.RLock() - if checksum, ok = c.ident[id]; !ok { - err = pending.err - } - c.identMu.RUnlock() - return - } - - d := make(chan struct{}) - pending = &pendingCure{done: d} - ctx, pending.cancel = context.WithCancel(c.toplevel.Load().ctx) - c.wg.Add(1) - c.identPending[id] = pending - c.identMu.Unlock() - done = d - return -} - -// finaliseIdent commits a checksum or error to ident for an identifier -// previously submitted to identPending. -func (c *Cache) finaliseIdent( - done chan<- struct{}, - id unique.Handle[ID], - checksum unique.Handle[Checksum], - err error, -) { - c.identMu.Lock() - if err != nil { - c.identPending[id].err = err - c.identErr[id] = err - } else { - c.ident[id] = checksum - } - delete(c.identPending, id) - c.identMu.Unlock() - c.wg.Done() - - close(done) -} - -// zeroChecksum is a zero [Checksum] handle, used for comparison only. -var zeroChecksum unique.Handle[Checksum] - -// loadSubstitute returns a checksum corresponding to a substitute identifier, -// or zeroChecksum if an alternative is not available. -func (c *Cache) loadSubstitute( - substitute unique.Handle[ID], -) (unique.Handle[Checksum], error) { - c.substituteMu.RLock() - if checksum, ok := c.substitute[substitute]; ok { - c.substituteMu.RUnlock() - return checksum, nil - } - - linkname, err := os.Readlink(c.base.Append( - dirSubstitute, - Encode(substitute.Value()), - ).String()) - c.substituteMu.RUnlock() - - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return zeroChecksum, err - } - - c.substituteMu.Lock() - c.substitute[substitute] = zeroChecksum - c.substituteMu.Unlock() - return zeroChecksum, nil - } - - var checksum unique.Handle[Checksum] - buf := c.getIdentBuf() - err = Decode((*Checksum)(buf[:]), filepath.Base(linkname)) - if err == nil { - checksum = unique.Make(Checksum(buf[:])) - - c.substituteMu.Lock() - c.substitute[substitute] = checksum - c.substituteMu.Unlock() - } - c.putIdentBuf(buf) - - return checksum, err -} - -// Done returns a channel that is closed when the ongoing cure of an [Artifact] -// referred to by the specified identifier completes. Done may return nil if -// no ongoing cure of the specified identifier exists. -func (c *Cache) Done(id unique.Handle[ID]) <-chan struct{} { - c.identMu.RLock() - pending, ok := c.identPending[id] - c.identMu.RUnlock() - if !ok || pending == nil { - return nil - } - return pending.done -} - -// Cancel cancels the ongoing cure of an [Artifact] referred to by the specified -// identifier. Cancel returns whether the [context.CancelFunc] has been killed. -// Cancel returns after the cure is complete. -func (c *Cache) Cancel(id unique.Handle[ID]) bool { - c.identMu.RLock() - pending, ok := c.identPending[id] - c.identMu.RUnlock() - if !ok || pending == nil || pending.cancel == nil { - return false - } - pending.cancel() - <-pending.done - - c.abortMu.Lock() - c.identMu.Lock() - delete(c.identErr, id) - c.identMu.Unlock() - c.abortMu.Unlock() - return true -} - -// openFile tries to load [FileArtifact] from [Cache], and if that fails, -// obtains it via [FileArtifact.Cure] instead. Notably, it does not cure -// [FileArtifact] to the filesystem. If err is nil, the caller is responsible -// for closing the resulting [io.ReadCloser]. -// -// The context must originate from loadOrStoreIdent to enable cancellation. -func (c *Cache) openFile( - ctx context.Context, - f FileArtifact, -) (r io.ReadCloser, err error) { - if kc, ok := f.(KnownChecksum); c.attr.Flags&CAssumeChecksum != 0 && ok { - c.checksumMu.RLock() - r, err = os.Open(c.base.Append( - dirChecksum, - Encode(kc.Checksum()), - ).String()) - c.checksumMu.RUnlock() - } else { - c.identMu.RLock() - r, err = os.Open(c.base.Append( - dirIdentifier, - Encode(c.Ident(f).Value()), - ).String()) - c.identMu.RUnlock() - } - - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return - } - id := c.Ident(f) - if c.msg.IsVerbose() { - rn := reportName(f, id) - c.msg.Verbosef("curing %s%s%s in memory...", c.sgrIdent, rn, c.sgrRes) - defer func() { - if err == nil { - c.msg.Verbosef("opened %s%s%s for reading", c.sgrIdent, rn, c.sgrRes) - } - }() - } - return f.Cure(&RContext{common{ctx, c}}) - } - return -} - -// InvalidFileModeError describes a [FloodArtifact.Cure] or -// [TrivialArtifact.Cure] that did not result in a regular file or directory -// located at the work pathname. -type InvalidFileModeError fs.FileMode - -// Error returns a constant string. -func (e InvalidFileModeError) Error() string { - return "artifact did not produce a regular file or directory" -} - -// NoOutputError describes a [FloodArtifact.Cure] or [TrivialArtifact.Cure] -// that did not populate its work pathname despite completing successfully. -type NoOutputError struct{} - -// Unwrap returns [os.ErrNotExist]. -func (NoOutputError) Unwrap() error { return os.ErrNotExist } - -// Error returns a constant string. -func (NoOutputError) Error() string { - return "artifact cured successfully but did not produce any output" -} - -// removeAll is similar to [os.RemoveAll] but is robust against any permissions. -func removeAll(pathname *check.Absolute) (chmodErr, removeErr error) { - chmodErr = filepath.WalkDir(pathname.String(), func( - path string, - d fs.DirEntry, - err error, - ) error { - if err != nil { - return err - } - if d.IsDir() { - return os.Chmod(path, 0700) - } - return nil - }) - if errors.Is(chmodErr, os.ErrNotExist) { - chmodErr = nil - } - removeErr = os.RemoveAll(pathname.String()) - return -} - -// zeroTimes zeroes atime and mtime for the named file. -func zeroTimes(path string) (err error) { - // include/uapi/linux/fcntl.h - const ( - AT_FDCWD = -100 - AT_SYMLINK_NOFOLLOW = 0x100 - ) - _AT_FDCWD := AT_FDCWD - - var _p0 *byte - _p0, err = syscall.BytePtrFromString(path) - if err != nil { - return - } - if _, _, errno := syscall.Syscall6( - syscall.SYS_UTIMENSAT, - uintptr(_AT_FDCWD), - uintptr(unsafe.Pointer(_p0)), - uintptr(unsafe.Pointer(new([2]syscall.Timespec))), - AT_SYMLINK_NOFOLLOW, - 0, 0, - ); errno != 0 { - return os.NewSyscallError("utimensat", errno) - } - return -} - -// overrideFileInfo overrides the permission bits of [fs.FileInfo] to 0500 and -// is the concrete type returned by overrideFile.Stat. -type overrideFileInfo struct{ fs.FileInfo } - -// Mode returns [fs.FileMode] with its permission bits set to 0500. -func (fi overrideFileInfo) Mode() fs.FileMode { - return fi.FileInfo.Mode()&(^fs.FileMode(0777)) | 0500 -} - -// Sys returns nil to avoid passing the original permission bits. -func (fi overrideFileInfo) Sys() any { return nil } - -// overrideFile overrides the permission bits of [fs.File] to 0500 and is the -// concrete type returned by dotOverrideFS for calls with "." passed as name. -type overrideFile struct{ fs.File } - -func (f overrideFile) Stat() (fi fs.FileInfo, err error) { - fi, err = f.File.Stat() - if err != nil { - return - } - fi = overrideFileInfo{fi} - return -} - -// dirFS is implemented by the concrete type of the return value of [os.DirFS]. -type dirFS interface { - fs.StatFS - fs.ReadFileFS - fs.ReadDirFS - fs.ReadLinkFS -} - -// dotOverrideFS overrides the permission bits of "." to 0500 to avoid the extra -// system calls to add and remove write bit from the target directory. -type dotOverrideFS struct{ dirFS } - -// Open wraps the underlying [fs.FS] with "." special case. -func (fsys dotOverrideFS) Open(name string) (f fs.File, err error) { - f, err = fsys.dirFS.Open(name) - if err != nil || name != "." { - return - } - f = overrideFile{f} - return -} - -// Stat wraps the underlying [fs.FS] with "." special case. -func (fsys dotOverrideFS) Stat(name string) (fi fs.FileInfo, err error) { - fi, err = fsys.dirFS.Stat(name) - if err != nil || name != "." { - return - } - fi = overrideFileInfo{fi} - return -} - -// InvalidArtifactError describes an artifact that does not implement a -// supported Cure method. -type InvalidArtifactError ID - -func (e InvalidArtifactError) Error() string { - return "artifact " + Encode(e) + " cannot be cured" -} - -// Cure cures the [Artifact] and returns its pathname and [Checksum]. Direct -// calls to Cure are not subject to the cures limit. -func (c *Cache) Cure(a Artifact) ( - pathname *check.Absolute, - checksum unique.Handle[Checksum], - err error, -) { - c.abortMu.RLock() - defer c.abortMu.RUnlock() - - if err = c.toplevel.Load().ctx.Err(); err != nil { - return - } - - pathname, checksum, _, err = c.cure(a, true, false) - return -} - -// CureWhence is like Cure, but returns the whence value. -func (c *Cache) CureWhence(a Artifact) ( - pathname *check.Absolute, - checksum unique.Handle[Checksum], - whence int, - err error, -) { - c.abortMu.RLock() - defer c.abortMu.RUnlock() - - if err = c.toplevel.Load().ctx.Err(); err != nil { - return - } - - return c.cure(a, true, false) -} - -// CureNew is like Cure, but always enters the implementation. -func (c *Cache) CureNew(a Artifact) ( - pathname *check.Absolute, - checksum unique.Handle[Checksum], - err error, -) { - c.abortMu.RLock() - defer c.abortMu.RUnlock() - - if err = c.toplevel.Load().ctx.Err(); err != nil { - return - } - - var whence int -retry: - pathname, checksum, whence, err = c.cure(a, true, true) - if err != nil || whence == WNew { - return - } - goto retry -} - -// An InputError describes inputs of a [FloodArtifact] which had failed to cure. -type InputError map[Artifact]error - -// unwrap returns an iterator over sorted, deduplicated [Artifact] and their -// corresponding identifier. -func (e InputError) unwrap() iter.Seq2[Artifact, unique.Handle[ID]] { - ir := NewIR() - - type input struct { - a Artifact - id unique.Handle[ID] - } - p := make([]input, 0, len(e)) - for a := range e { - p = append(p, input{a, ir.Ident(a)}) - } - - var identBuf [2]ID - slices.SortFunc(p, func(a, b input) int { - identBuf[0], identBuf[1] = a.id.Value(), b.id.Value() - return slices.Compare(identBuf[0][:], identBuf[1][:]) - }) - p = slices.CompactFunc(p, func(a, b input) bool { return a.id == b.id }) - - return func(yield func(Artifact, unique.Handle[ID]) bool) { - for _, i := range p { - if !yield(i.a, i.id) { - return - } - } - } -} - -// Error returns a user-facing, deterministic text representation of e. -func (e InputError) Error() string { - var buf strings.Builder - buf.WriteString("errors curing inputs:") - for a, id := range e.unwrap() { - buf.WriteString("\n\t") - buf.WriteString(reportName(a, id)) - buf.WriteString(": ") - buf.WriteString(e[a].Error()) - } - return buf.String() -} - -// Unwrap returns a slice of underlying errors sorted by identifier. -func (e InputError) Unwrap() []error { - errs := make([]error, 0, len(e)) - for a := range e.unwrap() { - errs = append(errs, e[a]) - } - return errs -} - -// enterCure must be called before entering an [Artifact] implementation. -func (c *Cache) enterCure(a Artifact, curesExempt bool) error { - if c.attr.Notify != nil { - c.attr.Notify <- true - } - - if a.IsExclusive() { - c.exclMu.Lock() - } - if curesExempt { - return nil - } - - ctx := c.toplevel.Load().ctx - select { - case c.cures <- struct{}{}: - return nil - - case <-ctx.Done(): - if a.IsExclusive() { - c.exclMu.Unlock() - } - return ctx.Err() - } -} - -// exitCure must be called after exiting an [Artifact] implementation. -func (c *Cache) exitCure(a Artifact, curesExempt bool) { - if c.attr.Notify != nil { - c.attr.Notify <- false - } - - if a.IsExclusive() { - c.exclMu.Unlock() - } - if curesExempt { - return - } - - <-c.cures -} - -// measuredReader implements [io.ReadCloser] and measures the checksum during -// Close. If the underlying reader is not read to EOF, Close blocks until all -// remaining data is consumed and validated. -type measuredReader struct { - // Underlying reader. Never exposed directly. - r io.ReadCloser - // For validating checksum. Never exposed directly. - h hash.Hash - // Buffers writes to h, initialised by [Cache]. Never exposed directly. - hbw *bufio.Writer - // Expected checksum, compared during Close. - want unique.Handle[Checksum] - - // For accessing free lists. - c *Cache - - // Set up via [io.TeeReader] by [Cache]. - io.Reader -} - -// Close reads the underlying [io.ReadCloser] to EOF, closes it and measures its -// outcome. It returns a [ChecksumMismatchError] for an unexpected checksum. -func (mr *measuredReader) Close() (err error) { - if mr.hbw == nil || mr.Reader == nil { - return os.ErrInvalid - } - err = mr.hbw.Flush() - mr.c.putWriter(mr.hbw) - mr.hbw, mr.Reader = nil, nil - if err != nil { - _ = mr.r.Close() - return - } - var n int64 - if n, err = io.Copy(mr.h, mr.r); err != nil { - _ = mr.r.Close() - return - } - - if n > 0 { - mr.c.msg.Verbosef( - "%smissed %d bytes on measured reader%s", - mr.c.sgrWarn, n, mr.c.sgrRes, - ) - } - - if err = mr.r.Close(); err != nil { - return - } - - buf := mr.c.getIdentBuf() - mr.h.Sum(buf[:0]) - - if got := Checksum(buf[:]); got != mr.want.Value() { - err = &ChecksumMismatchError{ - Got: got, - Want: mr.want.Value(), - } - } - - mr.c.putIdentBuf(buf) - return -} - -// newMeasuredReader implements [RContext.NewMeasuredReader]. -func (c *Cache) newMeasuredReader( - r io.ReadCloser, - checksum unique.Handle[Checksum], -) io.ReadCloser { - mr := measuredReader{r: r, h: sha512.New384(), want: checksum, c: c} - mr.hbw = c.getWriter(mr.h) - mr.Reader = io.TeeReader(r, mr.hbw) - return &mr -} - -// NewMeasuredReader returns an [io.ReadCloser] implementing behaviour required -// by [FileArtifact]. The resulting [io.ReadCloser] holds a buffer originating -// from [Cache] and must be closed to return this buffer. -func (r *RContext) NewMeasuredReader( - rc io.ReadCloser, - checksum unique.Handle[Checksum], -) io.ReadCloser { - return r.cache.newMeasuredReader(rc, checksum) -} - -// tryChecksum dereferences a symlink to a cure outcome. -func (c *Cache) tryChecksum(pathname *check.Absolute) ( - checksum unique.Handle[Checksum], - err error, -) { - _, err = os.Lstat(pathname.String()) - if err == nil { - var name string - if name, err = os.Readlink(pathname.String()); err != nil { - return - } - buf := c.getIdentBuf() - err = Decode((*Checksum)(buf[:]), filepath.Base(name)) - if err == nil { - checksum = unique.Make(Checksum(buf[:])) - } - c.putIdentBuf(buf) - } - return -} - -// tryLocal attempts to obtain an [Artifact] outcome from the filesystem. -func (c *Cache) tryLocal(id unique.Handle[ID]) (unique.Handle[Checksum], error) { - return c.tryChecksum(c.base.Append( - dirIdentifier, - Encode(id.Value()), - )) -} - -// tryExtern attempts to obtain an [Artifact] outcome from extern. -func (c *Cache) tryExtern(ctx context.Context, id unique.Handle[ID]) ( - unique.Handle[Checksum], - error, -) { - c.externMu.RLock() - defer c.externMu.RUnlock() - - checksum, ok := c.externCache[id] - if !ok { - if c.extern == nil { - return zeroChecksum, nil - } - - v, err := c.extern.Artifact(ctx, id) - if err != nil { - return zeroChecksum, err - } - if v == nil { - return zeroChecksum, nil - } - checksum = unique.Make(*v) - - var got unique.Handle[Checksum] - if _, got, err = c.Cure(c.extern.Checksum(checksum)); err != nil { - return checksum, err - } else if got != checksum { - return zeroChecksum, &ChecksumMismatchError{got.Value(), checksum.Value()} - } - } - return checksum, nil -} - -// cureMany concurrently collects outcome of multiple [Artifact]. -func (c *Cache) cureMany( - inputs []Artifact, - r map[Artifact]cureRes, - shallow bool, -) ([]bool, error) { - var wg sync.WaitGroup - wg.Add(len(inputs)) - var mask []bool - res := make([]cureRes, len(inputs)) - errs := make(InputError) - var errsMu sync.Mutex - if shallow { - mask = make([]bool, len(inputs)) - } - for i, d := range inputs { - if shallow { - if _, ok := d.(FloodArtifact); ok { - mask[i] = true - wg.Done() - continue - } - - if kc, ok := d.(KnownChecksum); ok { - res[i].checksum = unique.Make(kc.Checksum()) - wg.Done() - continue - } - } - pending := pendingArtifactDep{d, &res[i], errs, &errsMu, &wg} - go pending.cure(c) - } - wg.Wait() - - if len(errs) > 0 { - return mask, errs - } - for i, p := range res { - if shallow && mask[i] { - continue - } - r[inputs[i]] = p - } - return mask, nil -} - -// HangingInputError describes an input of an [Artifact] on a sparse cache -// without an outcome available locally or via [External]. -type HangingInputError unique.Handle[ID] - -func (e HangingInputError) Error() string { - return Encode(unique.Handle[ID](e).Value()) + " is unavailable" -} - -const ( - // WNew indicates a cure entering the implementation. - WNew = iota - // WCache indicates an [Artifact] present in the cache. - WCache - // WSubstitute indicates an [Artifact] hitting a content-based input - // substitution. - WSubstitute - // WExternal indicates a cure entering the external cache. - WExternal -) - -// WhenceString returns a printable string for a whence value. -func WhenceString(whence int) string { - switch whence { - case WNew: - return "new" - case WCache: - return "cache" - case WSubstitute: - return "substitute" - case WExternal: - return "external" - - default: - return "invalid whence " + strconv.Itoa(whence) - } -} - -// cure implements Cure without acquiring a read lock on abortMu. cure must not -// be entered during Abort. -func (c *Cache) cure(a Artifact, curesExempt, rebuild bool) ( - pathname *check.Absolute, - checksum unique.Handle[Checksum], - whence int, - err error, -) { - id := c.Ident(a) - if rebuild { - var v ID - _, _ = rand.Read(v[:]) - id = unique.Make(v) - } - - ids := Encode(id.Value()) - pathname = c.base.Append( - dirIdentifier, - ids, - ) - defer func() { - if err != nil { - pathname = nil - checksum = unique.Handle[Checksum]{} - } - }() - - if _, ok := a.(CuresExempt); ok { - curesExempt = true - } - - var ( - ctx context.Context - done chan<- struct{} - ) - ctx, done, checksum, err = c.loadOrStoreIdent(id) - if done == nil { - whence = WCache - return - } else { - defer func() { c.finaliseIdent(done, id, checksum, err) }() - } - - checksum, err = c.tryChecksum(pathname) - if err == nil || !errors.Is(err, os.ErrNotExist) { - whence = WCache - return - } - - var ( - checksums string - substitute unique.Handle[ID] - alternative *check.Absolute - ) - defer func() { - if err == nil && checksums != "" { - linkname := checksumLinknamePrefix + checksums - - err = os.Symlink( - linkname, - pathname.String(), - ) - if err == nil { - err = zeroTimes(pathname.String()) - } - - if err == nil && alternative != nil && substitute != id { - c.substituteMu.Lock() - err = os.Symlink( - linkname, - alternative.String(), - ) - if errors.Is(err, os.ErrExist) { - c.msg.Verbosef( - "creating alternative over %s%s%s for artifact %s%s%s", - c.sgrIdent, Encode(substitute.Value()), c.sgrRes, - c.sgrIdent, ids, c.sgrRes, - ) - err = nil - } - if err == nil { - err = zeroTimes(alternative.String()) - } - if err == nil && checksum != zeroChecksum { - c.substitute[substitute] = checksum - } - c.substituteMu.Unlock() - } - } - }() - - var checksumPathname *check.Absolute - var checksumFi os.FileInfo - if kc, ok := a.(KnownChecksum); ok { - checksum = unique.Make(kc.Checksum()) - checksums = Encode(checksum.Value()) - checksumPathname = c.base.Append( - dirChecksum, - checksums, - ) - - if c.attr.Flags&CAssumeChecksum != 0 { - c.checksumMu.RLock() - checksumFi, err = os.Stat(checksumPathname.String()) - c.checksumMu.RUnlock() - - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return - } - - checksumFi, err = nil, nil - } - } - } - - whence = WNew - if c.msg.IsVerbose() { - rn := reportName(a, id) - c.msg.Verbosef("curing %s%s%s...", c.sgrIdent, rn, c.sgrRes) - defer func() { - if err != nil { - return - } - if checksums != "" { - c.msg.Verbosef( - "cured %s%s%s checksum %s%s%s", - c.sgrIdent, rn, c.sgrRes, - c.sgrIdent, checksums, c.sgrRes) - } else { - c.msg.Verbosef("cured %s%s%s", c.sgrIdent, rn, c.sgrRes) - } - }() - } - - // cure FileArtifact outside type switch to skip TContext initialisation - if f, ok := a.(FileArtifact); ok { - if checksumFi != nil { - whence = WCache - if !checksumFi.Mode().IsRegular() { - // unreachable - err = InvalidFileModeError(checksumFi.Mode()) - } - return - } - - perm := os.FileMode(0400) - if f.IsExecutable() { - perm = 0500 - } - - work := c.base.Append(dirWork, ids) - var w *os.File - if w, err = os.OpenFile( - work.String(), - os.O_CREATE|os.O_EXCL|os.O_WRONLY, - perm, - ); err != nil { - return - } - defer func() { - closeErr := w.Close() - if err == nil { - err = closeErr - } - - removeErr := os.Remove(work.String()) - if err == nil && !errors.Is(removeErr, os.ErrNotExist) { - err = removeErr - } - }() - - var r io.ReadCloser - if err = c.enterCure(a, curesExempt); err != nil { - return - } - r, err = f.Cure(&RContext{common{ctx, c}}) - if err == nil { - if checksumPathname == nil || c.attr.Flags&CValidateKnown != 0 { - h := sha512.New384() - hbw := c.getWriter(h) - _, err = io.Copy(w, io.TeeReader(r, hbw)) - flushErr := hbw.Flush() - c.putWriter(hbw) - if err == nil { - err = flushErr - } - - if err == nil { - buf := c.getIdentBuf() - h.Sum(buf[:0]) - - if checksumPathname == nil { - checksum = unique.Make(Checksum(buf[:])) - checksums = Encode(Checksum(buf[:])) - } else if c.attr.Flags&CValidateKnown != 0 { - if got := Checksum(buf[:]); got != checksum.Value() { - err = &ChecksumMismatchError{ - Got: got, - Want: checksum.Value(), - } - } - } - - c.putIdentBuf(buf) - - if checksumPathname == nil { - checksumPathname = c.base.Append( - dirChecksum, - checksums, - ) - } - } - } else { - _, err = io.Copy(w, r) - } - - closeErr := r.Close() - if err == nil { - err = closeErr - } - } - c.exitCure(a, curesExempt) - if err != nil { - if c.msg.IsVerbose() { - c.msg.Verbosef( - "cure file %s%s%s: %s%v%s", - c.sgrIdent, reportName(f, id), c.sgrRes, - c.sgrErr, err, c.sgrRes, - ) - } - return - } - - c.checksumMu.Lock() - if err = os.Rename( - work.String(), - checksumPathname.String(), - ); err != nil { - c.checksumMu.Unlock() - return - } - timeErr := zeroTimes(checksumPathname.String()) - c.checksumMu.Unlock() - - if err == nil { - err = timeErr - } - return - } - - if checksumFi != nil { - whence = WCache - if !checksumFi.Mode().IsDir() { - // unreachable - err = InvalidFileModeError(checksumFi.Mode()) - } - return - } - - t := TContext{ - c.base.Append(dirWork, ids), - c.base.Append(dirTemp, ids), - ids, nil, nil, nil, nil, - common{ctx, c}, - } - switch ca := a.(type) { - case TrivialArtifact: - defer t.destroy(&err) - if err = c.enterCure(a, curesExempt); err != nil { - return - } - err = ca.Cure(&t) - c.exitCure(a, curesExempt) - if err != nil { - if c.msg.IsVerbose() { - c.msg.Verbosef( - "cure trivial %s%s%s: %s%v%s", - c.sgrIdent, reportName(ca, id), c.sgrRes, - c.sgrErr, err, c.sgrRes, - ) - } - return - } - break - - case FloodArtifact: - var externChecksum unique.Handle[Checksum] - if externChecksum, err = c.tryExtern(ctx, id); err != nil { - if c.msg.IsVerbose() { - c.msg.Verbosef( - "extern %s%s%s: %s%v%s", - c.sgrIdent, reportName(ca, id), c.sgrRes, - c.sgrErr, err, c.sgrRes, - ) - } - return - } - extern := externChecksum != zeroChecksum - shallow := extern && c.attr.Flags&CExternShallow != 0 - - inputs := a.Inputs() - f := FContext{t, make(map[Artifact]cureRes, len(inputs))} - var mask []bool - if mask, err = c.cureMany(inputs, f.inputs, shallow); err != nil { - return - } - - if shallow { - for i, d := range inputs { - if !mask[i] { - continue - } - - if kc, ok := d.(KnownChecksum); ok { - f.inputs[d] = cureRes{checksum: unique.Make(kc.Checksum())} - continue - } - - var sum unique.Handle[Checksum] - did := c.Ident(d) - sum, err = c.tryLocal(did) - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return - } - - sum, err = c.tryExtern(ctx, did) - if err != nil { - return - } - if sum == zeroChecksum { - if c.msg.IsVerbose() { - c.msg.Verbosef( - "input %s%s%s not available", - c.sgrIdent, reportName(d, did), c.sgrRes, - ) - } - err = HangingInputError(did) - return - } - } - - f.inputs[d] = cureRes{checksum: sum} - } - } - - sh := sha512.New384() - err = c.encode(sh, a, f.inputs) - if err != nil { - return - } - - buf := c.getIdentBuf() - sh.Sum(buf[wordSize:wordSize]) - substitute = unique.Make(ID(buf[wordSize:])) - substitutes := Encode(substitute.Value()) - c.putIdentBuf(buf) - alternative = c.base.Append( - dirSubstitute, - substitutes, - ) - - if !rebuild && c.attr.Flags&CIgnoreSubstitutes == 0 { - var substituteChecksum unique.Handle[Checksum] - substituteChecksum, err = c.loadSubstitute(substitute) - if err != nil { - return - } - if substituteChecksum != zeroChecksum { - whence = WSubstitute - checksum = substituteChecksum - checksums = Encode(checksum.Value()) - checksumPathname = c.base.Append( - dirChecksum, - checksums, - ) - if _, err = os.Lstat(c.base.Append( - dirStatus, - substitutes, - ).String()); err == nil { - err = os.Symlink(substitutes, c.base.Append( - dirStatus, - ids, - ).String()) - } else if errors.Is(err, os.ErrNotExist) { - err = nil - } - return - } - } - - defer f.destroy(&err) - - if extern { - whence = WExternal - if checksum != zeroChecksum && externChecksum != checksum { - err = &ChecksumMismatchError{externChecksum.Value(), checksum.Value()} - if c.msg.IsVerbose() { - c.msg.Verbosef( - "extern %s%s%s: %s%v%s", - c.sgrIdent, reportName(ca, id), c.sgrRes, - c.sgrErr, err, c.sgrRes, - ) - } - return - } - - var externStatus io.ReadCloser - c.externMu.RLock() - externStatus, err = c.extern.Status(&RContext{common{ctx, c}}, id) - c.externMu.RUnlock() - if err != nil { - return - } - - checksum = externChecksum - checksums = Encode(checksum.Value()) - checksumPathname = c.base.Append( - dirChecksum, - checksums, - ) - - if externStatus != nil { - if err = f.prepareStatus(false); err != nil { - _ = externStatus.Close() - return - } else if _, err = io.Copy(f.status, externStatus); err != nil { - _ = externStatus.Close() - return - } else if err = externStatus.Close(); err != nil { - return - } else if !rebuild { - if err = f.linkSubstitute(ids, substitutes); err != nil { - return - } - } - } - return - } - - if err = c.enterCure(a, curesExempt); err != nil { - return - } - err = ca.Cure(&f) - c.exitCure(a, curesExempt) - - if !rebuild && err == nil { - err = f.linkSubstitute(ids, substitutes) - } - if err != nil { - if c.msg.IsVerbose() { - c.msg.Verbosef( - "cure %s%s%s: %s%v%s", - c.sgrIdent, reportName(ca, id), c.sgrRes, - c.sgrErr, err, c.sgrRes, - ) - } - return - } - break - - default: - err = InvalidArtifactError(id.Value()) - return - } - t.cache = nil - - var fi os.FileInfo - if fi, err = os.Lstat(t.work.String()); err != nil { - if errors.Is(err, os.ErrNotExist) { - err = NoOutputError{} - } - return - } - - if !fi.IsDir() { - if !fi.Mode().IsRegular() { - err = InvalidFileModeError(fi.Mode()) - } else { - err = errors.New("non-file artifact produced regular file") - } - return - } - - var gotChecksum Checksum - if err = SumFS( - &gotChecksum, - dotOverrideFS{os.DirFS(t.work.String()).(dirFS)}, - ".", - ); err != nil { - return - } - - if checksumPathname == nil { - checksum = unique.Make(gotChecksum) - checksums = Encode(gotChecksum) - checksumPathname = c.base.Append( - dirChecksum, - checksums, - ) - } else if gotChecksum != checksum.Value() { - err = &ChecksumMismatchError{ - Got: gotChecksum, - Want: checksum.Value(), - } - if c.msg.IsVerbose() { - c.msg.Verbosef( - "validate %s%s%s: %s%v%s", - c.sgrIdent, reportName(a, id), c.sgrRes, - c.sgrErr, err, c.sgrRes, - ) - } - return - } - - if err = os.Chmod(t.work.String(), 0700); err != nil { - return - } - if err = filepath.WalkDir(t.work.String(), func(path string, _ fs.DirEntry, err error) error { - if err != nil { - return err - } - return zeroTimes(path) - }); err != nil { - return - } - c.checksumMu.Lock() - if err = os.Rename( - t.work.String(), - checksumPathname.String(), - ); err != nil { - if !errors.Is(err, os.ErrExist) { - c.checksumMu.Unlock() - return - } - // err is zeroed during deferred cleanup - } else { - err = os.Chmod(checksumPathname.String(), 0500) - } - c.checksumMu.Unlock() - return -} - -// cure cures the pending [Artifact], stores its result and notifies the caller. -func (pending *pendingArtifactDep) cure(c *Cache) { - defer pending.Done() - - var err error - pending.resP.pathname, pending.resP.checksum, _, err = c.cure(pending.a, false, false) - if err == nil { - return - } - - pending.errsMu.Lock() - if errs, ok := err.(InputError); ok { - maps.Copy(pending.errs, errs) - } else { - pending.errs[pending.a] = err - } - pending.errsMu.Unlock() -} - -// OpenStatus attempts to open the status file associated to an [Artifact]. If -// err is nil, the caller must close the resulting reader. -func (c *Cache) OpenStatus(a Artifact) (r io.ReadSeekCloser, err error) { - c.identMu.RLock() - r, err = os.Open(c.base.Append( - dirStatus, - Encode(c.Ident(a).Value())).String(), - ) - c.identMu.RUnlock() - return -} - -// Fault holds the pathname and termination time of an [Artifact] fault entry. -type Fault struct { - *check.Absolute - t uint64 -} - -// Time returns the instant in time where the fault occurred. -func (f Fault) Time() time.Time { return time.Unix(0, int64(f.t)) } - -// Open opens the underlying entry for reading. -func (f Fault) Open() (io.ReadCloser, error) { return os.Open(f.Absolute.String()) } - -// Destroy removes the underlying fault entry. -func (f Fault) Destroy() error { return os.Remove(f.Absolute.String()) } - -// ReadFaults returns fault entries for an [Artifact]. -func (c *Cache) ReadFaults(a Artifact) (faults []Fault, err error) { - prefix := Encode(c.Ident(a).Value()) + "." - var dents []os.DirEntry - if dents, err = os.ReadDir(c.base.Append(dirFault).String()); err != nil { - return - } - - for _, dent := range dents { - name := dent.Name() - if !strings.HasPrefix(name, prefix) { - continue - } - var t uint64 - t, err = strconv.ParseUint(name[len(prefix):], 10, 64) - if err != nil { - return - } - - faults = append(faults, Fault{c.base.Append( - dirFault, - name, - ), t}) - } - - slices.SortFunc(faults, func(a, b Fault) int { - return cmp.Compare(a.t, b.t) - }) - return -} - -// Abort cancels all pending cures and waits for them to clean up, but does not -// close the cache. -func (c *Cache) Abort() { - c.closeMu.Lock() - defer c.closeMu.Unlock() - - if c.closed { - return - } - - c.toplevel.Load().cancel() - c.abortMu.Lock() - defer c.abortMu.Unlock() - - // holding abortMu, identPending stays empty - c.wg.Wait() - c.identMu.Lock() - c.toplevel.Store(newToplevel(c.parent)) - clear(c.identErr) - c.identMu.Unlock() -} - -// Close cancels all pending cures and waits for them to clean up. -func (c *Cache) Close() { - c.closeMu.Lock() - defer c.closeMu.Unlock() - - if c.closed { - return - } - - c.closed = true - c.toplevel.Load().cancel() - c.wg.Wait() - close(c.cures) - c.unlock() - - if c.attr.Notify != nil { - close(c.attr.Notify) - } -} - -// UnsupportedVariantError describes an on-disk cache with an extension variant -// identification string that differs from the value returned by [Extension]. -type UnsupportedVariantError string - -func (e UnsupportedVariantError) Error() string { - return "unsupported variant " + strconv.Quote(string(e)) -} - -var ( - // ErrWouldPromote is returned by [Open] if the [CPromoteVariant] bit is not - // set and the on-disk cache requires variant promotion. - ErrWouldPromote = errors.New("operation would promote unextended cache") -) - -// CacheAttr holds the attributes that will be applied to a new [Cache] opened -// by [Open]. -type CacheAttr struct { - // Concurrent cures of a [FloodArtifact] dependency graph. - Cures int - // Options affecting [Cache] behaviour. - Flags int - // Preferred job count, when applicable. - Jobs int - // Preferred loadavg target, when applicable. - Load int - // Optional cure entry and exit notification. - Notify chan<- bool - - // Omit the [lockedfile] lock. - skipLock bool -} - -// Open returns the address of a newly opened instance of [Cache]. -// -// Concurrent cures of a [FloodArtifact] dependency graph is limited to the -// caller-supplied value, however direct calls to [Cache.Cure] is not subject -// to this limitation. -// -// A cures or jobs value of 0 or lower is equivalent to the value returned by -// [runtime.NumCPU]. -// -// A successful call to Open guarantees exclusive access to the on-filesystem -// cache for the resulting instance of [Cache]. The [Cache.Close] method cancels -// and waits for pending cures on [Cache] before releasing this lock and must be -// called once the [Cache] is no longer needed. -func Open( - ctx context.Context, - msg message.Msg, - base *check.Absolute, - attr *CacheAttr, -) (*Cache, error) { - openMu.Lock() - defer openMu.Unlock() - opened = true - - if extension == "" && len(irArtifact) != int(_kindEnd) { - panic("attempting to open cache with incomplete variant setup") - } - - var a CacheAttr - if attr != nil { - a = *attr - } - - if a.Cures < 1 { - a.Cures = runtime.NumCPU() - } - if a.Jobs < 1 { - a.Jobs = runtime.NumCPU() - } - if a.Load < 1 { - a.Load = runtime.NumCPU() + 2 - } - - for _, name := range []string{ - dirSubstitute, - dirIdentifier, - dirChecksum, - dirStatus, - dirFault, - dirWork, - } { - if err := os.MkdirAll( - base.Append(name).String(), - 0700, - ); err != nil && !errors.Is(err, os.ErrExist) { - return nil, err - } - } - - c := Cache{ - parent: ctx, - - cures: make(chan struct{}, a.Cures), - attr: a, - - msg: msg, - base: base, - - irCache: zeroIRCache(), - - substitute: make(map[unique.Handle[ID]]unique.Handle[Checksum]), - ident: make(map[unique.Handle[ID]]unique.Handle[Checksum]), - identErr: make(map[unique.Handle[ID]]error), - identPending: make(map[unique.Handle[ID]]*pendingCure), - - brPool: sync.Pool{New: func() any { return new(bufio.Reader) }}, - bwPool: sync.Pool{New: func() any { return new(bufio.Writer) }}, - } - c.toplevel.Store(newToplevel(ctx)) - - if !a.skipLock || !testing.Testing() { - if unlock, err := lockedfile.MutexAt( - base.Append(fileLock).String(), - ).Lock(); err != nil { - return nil, err - } else { - c.unlock = unlock - } - } else { - c.unlock = func() {} - } - - for _, name := range []string{ - dirWork, - dirTemp, - } { - dents, err := os.ReadDir(base.Append(name).String()) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - continue - } - c.unlock() - return nil, err - } - if len(dents) != 0 { - c.unlock() - return nil, fmt.Errorf( - "%s is not empty, scrub likely required", - name, - ) - } - } - - if _, err := os.ReadDir(base.Append( - dirExecScratch, - ).String()); !errors.Is(err, os.ErrNotExist) { - c.unlock() - if err != nil { - return nil, err - } - return nil, errors.New(dirExecScratch + " is present, scrub likely required") - } - - variantPath := base.Append(fileVariant).String() - if p, err := os.ReadFile(variantPath); err != nil { - if !errors.Is(err, os.ErrNotExist) { - c.unlock() - return nil, err - } - // nonexistence implies newly created cache, or a cache predating - // variant identification strings, in which case it is silently promoted - if err = os.WriteFile( - variantPath, - []byte(extension), - 0400, - ); err != nil { - c.unlock() - return nil, err - } - } else if s := string(p); s == "" { - if extension != "" { - if a.Flags&CPromoteVariant == 0 { - c.unlock() - return nil, ErrWouldPromote - } - if err = os.WriteFile( - variantPath, - []byte(extension), - 0400, - ); err != nil { - c.unlock() - return nil, err - } - } - } else if !ValidExtension(s) { - c.unlock() - return nil, ErrInvalidExtension - } else if s != extension { - c.unlock() - return nil, UnsupportedVariantError(s) - } - - if a.Flags&CColourOutput != 0 { - c.sgrRes = "\x1b[0m" - c.sgrIdent = "\x1b[1m" - c.sgrWarn = "\x1b[35m" - c.sgrErr = "\x1b[1;31m" - } - - return &c, nil -} - -// Collected is returned by [Collect.Cure] to indicate a successful collection. -type Collected struct{} - -// Error returns a constant string to satisfy error, but should never be seen -// by the user. -func (Collected) Error() string { return "artifacts successfully collected" } - -// IsCollected returns whether the underlying error contains that of the result -// of curing a [Collect] helper. -func IsCollected(err error) bool { return errors.As(err, new(Collected)) } - -// Collect implements [pkg.FloodArtifact] to concurrently cure multiple -// [pkg.Artifact]. It returns [Collected]. -type Collect []Artifact - -var _ Artifact = new(Collect) - -// Cure returns [Collected]. -func (*Collect) Cure(*FContext) error { return Collected{} } - -// Kind returns the hardcoded [pkg.Kind] value. -func (*Collect) Kind() Kind { return kindCollection } - -// Params is a noop: dependencies are already represented in the header. -func (*Collect) Params(*IContext) {} - -// Inputs returns [Collect] as is. -func (c *Collect) Inputs() []Artifact { return *c } - -// IsExclusive returns false: Cure is a noop. -func (*Collect) IsExclusive() bool { return false } diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go deleted file mode 100644 index b23948f8..00000000 --- a/internal/pkg/pkg_test.go +++ /dev/null @@ -1,2375 +0,0 @@ -package pkg_test - -import ( - "archive/tar" - "bytes" - "context" - "crypto/sha512" - "encoding/base64" - "errors" - "fmt" - "io" - "io/fs" - "log" - "net/http" - "os" - "path/filepath" - "reflect" - "strconv" - "strings" - "sync" - "syscall" - "testing" - "testing/fstest" - "time" - "unique" - "unsafe" - - "hakurei.app/check" - "hakurei.app/container" - "hakurei.app/fhs" - "hakurei.app/internal/info" - "hakurei.app/internal/landlock" - "hakurei.app/internal/pkg" - "hakurei.app/internal/stub" - "hakurei.app/message" -) - -var skipLock = func() pkg.CacheAttr { - var attr pkg.CacheAttr - *(*bool)(unsafe.Pointer(reflect.ValueOf(&attr). - Elem(). - FieldByName("skipLock"). - UnsafeAddr())) = true - return attr -}() - -var ( - // extension is a string uniquely identifying a set of custom [Artifact] - // implementations registered by calling [Register]. - // - //go:linkname extension hakurei.app/internal/pkg.extension - extension string - - // opened is false if [Open] was never called. - // - //go:linkname opened hakurei.app/internal/pkg.opened - opened bool - - // irArtifact refers to artifact IR interpretation functions and must not be - // written to directly. - // - //go:linkname irArtifact hakurei.app/internal/pkg.irArtifact - irArtifact map[pkg.Kind]pkg.IRReadFunc - - // statusHeader is the header written to all status files in dirStatus. - // - //go:linkname statusHeader hakurei.app/internal/pkg.statusHeader - statusHeader string -) - -// newRContext returns the address of a new [pkg.RContext] unsafely created for -// the specified [testing.TB]. -func newRContext(tb testing.TB, c *pkg.Cache) *pkg.RContext { - var r pkg.RContext - rContextVal := reflect.ValueOf(&r).Elem().FieldByName("ctx") - reflect.NewAt( - rContextVal.Type(), - unsafe.Pointer(rContextVal.UnsafeAddr()), - ).Elem().Set(reflect.ValueOf(tb.Context())) - rCacheVal := reflect.ValueOf(&r).Elem().FieldByName("cache") - reflect.NewAt( - rCacheVal.Type(), - unsafe.Pointer(rCacheVal.UnsafeAddr()), - ).Elem().Set(reflect.ValueOf(c)) - return &r -} - -// overrideIdent overrides the ID method of [Artifact]. -type overrideIdent struct { - id pkg.ID - pkg.TrivialArtifact -} - -func (a overrideIdent) ID() pkg.ID { return a.id } - -// overrideIdentFile overrides the ID method of [FileArtifact]. -type overrideIdentFile struct { - id pkg.ID - pkg.FileArtifact -} - -func (a overrideIdentFile) ID() pkg.ID { return a.id } - -// A knownIdentArtifact implements [pkg.KnownIdent] and [Artifact] -type knownIdentArtifact interface { - pkg.KnownIdent - pkg.TrivialArtifact -} - -// A knownIdentFile implements [pkg.KnownIdent] and [FileArtifact] -type knownIdentFile interface { - pkg.KnownIdent - pkg.FileArtifact -} - -// overrideChecksum overrides the Checksum method of [Artifact]. -type overrideChecksum struct { - checksum pkg.Checksum - knownIdentArtifact -} - -func (a overrideChecksum) Checksum() pkg.Checksum { return a.checksum } - -// overrideChecksumFile overrides the Checksum method of [FileArtifact]. -type overrideChecksumFile struct { - checksum pkg.Checksum - knownIdentFile -} - -func (a overrideChecksumFile) Checksum() pkg.Checksum { return a.checksum } - -// A stubArtifact implements [TrivialArtifact] with hardcoded behaviour. -type stubArtifact struct { - kind pkg.Kind - params []byte - deps []pkg.Artifact - - cure func(t *pkg.TContext) error -} - -func (a *stubArtifact) Kind() pkg.Kind { return a.kind } -func (a *stubArtifact) Params(ctx *pkg.IContext) { ctx.Write(a.params) } -func (a *stubArtifact) Inputs() []pkg.Artifact { return a.deps } -func (a *stubArtifact) Cure(t *pkg.TContext) error { return a.cure(t) } -func (*stubArtifact) IsExclusive() bool { return false } - -// A stubArtifactF implements [FloodArtifact] with hardcoded behaviour. -type stubArtifactF struct { - kind pkg.Kind - params []byte - deps []pkg.Artifact - excl bool - - cure func(f *pkg.FContext) error -} - -func (a *stubArtifactF) Kind() pkg.Kind { return a.kind } -func (a *stubArtifactF) Params(ctx *pkg.IContext) { ctx.Write(a.params) } -func (a *stubArtifactF) Inputs() []pkg.Artifact { return a.deps } -func (a *stubArtifactF) Cure(f *pkg.FContext) error { return a.cure(f) } -func (a *stubArtifactF) IsExclusive() bool { return a.excl } - -// A stubFile implements [FileArtifact] with hardcoded behaviour. -type stubFile struct { - data []byte - err error - - stubArtifact -} - -func (*stubFile) IsExecutable() bool { return false } - -func (a *stubFile) Cure(*pkg.RContext) (io.ReadCloser, error) { - return io.NopCloser(bytes.NewReader(a.data)), a.err -} - -// newStubFile returns an implementation of [pkg.File] with hardcoded behaviour. -func newStubFile( - kind pkg.Kind, - id pkg.ID, - sum *pkg.Checksum, - data []byte, - err error, -) pkg.FileArtifact { - f := overrideIdentFile{id, &stubFile{data, err, stubArtifact{ - kind, - nil, - nil, - func(*pkg.TContext) error { - panic("unreachable") - }, - }}} - if sum == nil { - return f - } else { - return overrideChecksumFile{*sum, f} - } -} - -// stubExtern implements [External] with hardcoded prepared outcomes. -type stubExtern struct { - artifact map[unique.Handle[pkg.ID]]pkg.Checksum - checksum map[unique.Handle[pkg.Checksum]]fstest.MapFS - status map[unique.Handle[pkg.ID]]string -} - -func (e stubExtern) Artifact(_ context.Context, id unique.Handle[pkg.ID]) (*pkg.Checksum, error) { - if checksum, ok := e.artifact[id]; ok { - return &checksum, nil - } - return nil, nil -} - -func (e stubExtern) Checksum(checksum unique.Handle[pkg.Checksum]) pkg.Artifact { - var buf bytes.Buffer - if err := pkg.Write(e.checksum[checksum], ".", &buf); err != nil { - panic(err) - } - return pkg.NewArchive(pkg.NewFile("", buf.Bytes())) -} - -func (e stubExtern) Status(_ *pkg.RContext, id unique.Handle[pkg.ID]) (io.ReadCloser, error) { - if status, ok := e.status[id]; ok { - return io.NopCloser(strings.NewReader(status)), nil - } - return nil, nil -} - -// destroyArtifact removes all traces of an [Artifact] from the on-disk cache. -// Do not use this in a test case without a very good reason to do so. -func destroyArtifact( - t *testing.T, - base *check.Absolute, - c *pkg.Cache, - a pkg.Artifact, -) { - if pathname, checksum, err := c.Cure(a); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if err = os.Remove(pathname.String()); err != nil { - t.Fatal(err) - } else { - p := base.Append( - "checksum", - pkg.Encode(checksum.Value()), - ) - if err = filepath.WalkDir(p.String(), func( - path string, - d fs.DirEntry, - err error, - ) error { - if err != nil { - return err - } - if d.IsDir() { - return os.Chmod(path, 0700) - } - return nil - }); err != nil && !errors.Is(err, os.ErrNotExist) { - t.Fatal(err) - } - if err = os.RemoveAll(p.String()); err != nil { - t.Fatal(err) - } - } -} - -// newDestroyArtifactFunc returns a function that calls destroyArtifact. -func newDestroyArtifactFunc(a pkg.Artifact) func( - t *testing.T, - base *check.Absolute, - c *pkg.Cache, -) { - return func( - t *testing.T, - base *check.Absolute, - c *pkg.Cache, - ) { - destroyArtifact(t, base, c, a) - } -} - -// destroyStatus counts non-substitution status entries and destroys them. -func destroyStatus(t *testing.T, base *check.Absolute, c, s int) { - dents, err := os.ReadDir(base.Append("status").String()) - if err != nil { - t.Fatal(err) - } - - var gotC, gotS int - for _, dent := range dents { - if err = os.Remove(base.Append( - "status", - dent.Name(), - ).String()); err != nil { - t.Fatal(err) - } - - if dent.Type().IsRegular() { - gotC++ - continue - } - if dent.Type()&fs.ModeSymlink == fs.ModeSymlink { - gotS++ - continue - } - t.Errorf("%s: %s", dent.Name(), dent.Type()) - } - - if gotC != c { - t.Errorf("status: c = %d, want %d", gotC, c) - } - if gotS != s { - t.Errorf("status: s = %d, want %d", gotS, s) - } -} - -func TestIdent(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - a pkg.Artifact - want unique.Handle[pkg.ID] - }{ - {"decompress", &stubArtifact{ - pkg.KindDecompress, - []byte{pkg.Gzip, 0, 0, 0, 0, 0, 0, 0}, - []pkg.Artifact{ - overrideIdent{pkg.ID{}, new(stubArtifact)}, - }, - nil, - }, unique.Make[pkg.ID](pkg.MustDecode( - "fkmge-H56Ph2LzNNbEUjTypes8LfOxrF_CEXg07kq6ho_nbTNvzQde9-bjB7gyB_", - ))}, - } - - msg := message.New(log.New(os.Stderr, "ident: ", 0)) - msg.SwapVerbose(true) - var cache *pkg.Cache - if a, err := check.NewAbs(t.TempDir()); err != nil { - t.Fatal(err) - } else if cache, err = pkg.Open(t.Context(), msg, a, nil); err != nil { - t.Fatal(err) - } - t.Cleanup(cache.Close) - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - if got := cache.Ident(tc.a); got != tc.want { - t.Errorf("Ident: %s, want %s", - pkg.Encode(got.Value()), - pkg.Encode(tc.want.Value()), - ) - } - }) - } -} - -// An expectsKnown describes an expected file or directory. -type expectsKnown interface { - // hash returns the checksum of the represented data. - hash() (checksum pkg.Checksum) -} - -// An expectsChecksum is a prepared checksum value. -type expectsChecksum pkg.Checksum - -// hash returns e. -func (e expectsChecksum) hash() pkg.Checksum { return e } - -// An expectsFile is the contents of a file expected by the test suite. -type expectsFile []byte - -// hash computes the checksum of e. -func (e expectsFile) hash() (checksum pkg.Checksum) { - h := sha512.New384() - h.Write(e) - h.Sum(checksum[:0]) - return -} - -// An expectsFS describes the state of a filesystem expected by the test suite. -type expectsFS fstest.MapFS - -// hash computes the checksum of e. -func (e expectsFS) hash() (checksum pkg.Checksum) { - if err := pkg.SumFS(&checksum, fstest.MapFS(e), "."); err != nil { - panic(err) - } - return -} - -// expectsFrom generates expectsFS for a filesystem directory. -func expectsFrom(pathname string) string { - var buf strings.Builder - buf.WriteString("expectsFS{\n") - if err := filepath.WalkDir(pathname, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - var rel string - if rel, err = filepath.Rel(pathname, path); err != nil { - return err - } - buf.WriteString("\t" + strconv.Quote(rel) + ": {Mode: ") - - var fi fs.FileInfo - if fi, err = d.Info(); err != nil { - return err - } - mode := fi.Mode() - - switch { - case mode.IsDir(): - buf.WriteString("fs.ModeDir | 0" + - strconv.FormatInt(int64(mode&^fs.ModeDir), 8)) - - case mode&fs.ModeSymlink != 0: - buf.WriteString("fs.ModeSymlink | 0" + - strconv.FormatInt(int64(mode&^fs.ModeSymlink), 8) + - ", Data: []byte(") - var linkname string - if linkname, err = os.Readlink(path); err != nil { - return err - } - buf.WriteString(strconv.Quote(linkname)) - buf.WriteByte(')') - - case mode.IsRegular(): - buf.WriteString("0" + strconv.FormatInt(int64(mode), 8)) - var p []byte - if p, err = os.ReadFile(path); err != nil { - return err - } - - if len(p) > 0 { - buf.WriteString(", Data: []byte(") - buf.WriteString(strconv.Quote(unsafe.String(unsafe.SliceData(p), len(p)))) - buf.WriteByte(')') - } - } - buf.WriteString("},\n") - return nil - }); err != nil { - panic(err) - } - buf.WriteString("}") - return buf.String() -} - -// cacheTestCase is a test case passed to checkWithCache where a new instance -// of [pkg.Cache] is prepared for the test case, and is validated and removed -// on test completion. -type cacheTestCase struct { - name string - flags int - early func(t *testing.T, base *check.Absolute) - f func(t *testing.T, base *check.Absolute, c *pkg.Cache) - want expectsFS -} - -const ( - // checkDestroySubstitutes arranges for substitutes to be destroyed before - // measurement during checkWithCache. - checkDestroySubstitutes = 1 << (iota + 32) -) - -// makeBase returns a [pkg.Cache] base directory created for tb. -func makeBase(tb testing.TB) (base *check.Absolute) { - tb.Helper() - - base = check.MustAbs(tb.TempDir()) - if err := os.Chmod(base.String(), 0700); err != nil { - tb.Fatal(err) - } - tb.Cleanup(func() { - if err := filepath.WalkDir(base.String(), func(path string, d fs.DirEntry, err error) error { - if err != nil { - tb.Error(err) - return nil - } - if !d.IsDir() { - return nil - } - return os.Chmod(path, 0700) - }); err != nil { - tb.Fatal(err) - } - }) - return -} - -// checkWithCache runs a slice of cacheTestCase. -func checkWithCache(t *testing.T, testCases []cacheTestCase) { - t.Helper() - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Helper() - t.Parallel() - - base := makeBase(t) - msg := message.New(log.New(os.Stderr, "cache: ", 0)) - msg.SwapVerbose(testing.Verbose()) - - flags := tc.flags | pkg.CSuppressInit - - if info.CanDegrade { - if _, err := landlock.GetABI(); err != nil { - if !errors.Is(err, syscall.ENOSYS) { - t.Fatalf("LandlockGetABI: error = %v", err) - } - flags |= pkg.CHostAbstract - t.Log("Landlock LSM is unavailable, setting CHostAbstract") - } - } - - var scrubFunc func() error // scrub after hashing - if c, err := pkg.Open(t.Context(), msg, base, &pkg.CacheAttr{ - Cures: 1 << 4, - Flags: flags, - }); err != nil { - t.Fatalf("Open: error = %v", err) - } else { - t.Cleanup(c.Close) - if tc.early != nil { - tc.early(t, base) - } - tc.f(t, base, c) - scrubFunc = func() error { - err = c.Scrub(1 << 7) - idents, checksums, cleanErr := c.Clean(false, false) - if len(idents) > 0 { - t.Errorf("destroyed %d idents", len(idents)) - } - if len(checksums) > 0 { - t.Errorf("destroyed %d checksums", len(checksums)) - } - return errors.Join(err, cleanErr) - } - } - - var restoreTemp bool - if _, err := os.Lstat(base.Append("temp").String()); err != nil { - if !errors.Is(err, os.ErrNotExist) { - t.Fatal(err) - } - } else { - restoreTemp = true - } - - // destroy lock and variant file to avoid changing cache checksums - for _, s := range []string{ - "lock", - "variant", - } { - pathname := base.Append(s) - if p, err := os.ReadFile(pathname.String()); err != nil { - t.Fatal(err) - } else if len(p) != 0 { - t.Fatalf("file %q: %q", s, string(p)) - } - if err := os.Remove(pathname.String()); err != nil { - t.Fatal(err) - } - } - - // destroy non-deterministic substitutes - if tc.flags&checkDestroySubstitutes != 0 { - substitute := base.Append("substitute") - if err := os.RemoveAll(substitute.String()); err != nil { - t.Fatal(err) - } else if err = os.Mkdir(substitute.String(), 0700); err != nil { - t.Fatal(err) - } - } - - // destroy empty status directory - if err := syscall.Rmdir(base.Append("status").String()); err != nil { - if !errors.Is(err, syscall.ENOTEMPTY) { - t.Fatal(err) - } - } - - // destroy empty fault directory - if err := os.Remove(base.Append("fault").String()); err != nil { - t.Fatal(err) - } - - want := tc.want.hash() - - var checksum pkg.Checksum - if err := pkg.SumDir(&checksum, base); err != nil { - t.Fatalf("SumDir: error = %v", err) - } else if checksum != want { - t.Fatal(expectsFrom(base.String())) - } - - if err := scrubFunc(); err != nil { - t.Fatal("cache contains inconsistencies\n\n" + err.Error()) - } - - if restoreTemp { - if err := os.Mkdir( - base.Append("temp").String(), - 0700, - ); err != nil { - t.Fatal(err) - } - } - - // validate again to make sure scrub did not condemn anything - if err := pkg.SumDir(&checksum, base); err != nil { - t.Fatalf("SumDir: error = %v", err) - } else if checksum != want { - t.Fatalf("(scrubbed) %s", expectsFrom(base.String())) - } - }) - } -} - -// A cureStep contains an [Artifact] to be cured, and the expected outcome. -type cureStep struct { - name string - - a pkg.Artifact - - pathname *check.Absolute - output expectsKnown - whence int - err error -} - -// ignorePathname is passed to cureMany to skip the pathname check. -var ignorePathname = check.MustAbs("/\x00") - -// cureMany cures many artifacts against a [Cache] and checks their outcomes. -func cureMany(t *testing.T, c *pkg.Cache, steps []cureStep) { - t.Helper() - - makeChecksumH := func(checksum pkg.Checksum) unique.Handle[pkg.Checksum] { - if checksum == (pkg.Checksum{}) { - return unique.Handle[pkg.Checksum]{} - } - return unique.Make(checksum) - } - - for _, step := range steps { - t.Log("cure step:", step.name) - if pathname, checksum, whence, err := c.CureWhence(step.a); !reflect.DeepEqual(err, step.err) { - faults, _err := c.ReadFaults(step.a) - if _err != nil { - t.Errorf("ReadFaults: error = %v", _err) - } - - var p []byte - for _, fault := range faults { - p, _err = os.ReadFile(fault.String()) - if _err != nil { - t.Error(_err) - continue - } - t.Log(string(p)) - t.Logf("faulting cure terminated %s ago", time.Since(faults[0].Time())) - } - t.Fatalf("Cure: error = %v, want %v", err, step.err) - } else if step.whence != whence { - t.Fatalf("Cure: whence = %s, want %s", pkg.WhenceString(whence), pkg.WhenceString(step.whence)) - } else if step.pathname != ignorePathname && !pathname.Is(step.pathname) { - t.Fatalf("Cure: pathname = %q, want %q", pathname, step.pathname) - } else if step.output == nil || checksum != makeChecksumH(step.output.hash()) { - if pathname != nil { - if name, _err := filepath.EvalSymlinks(pathname.String()); _err != nil { - t.Fatal(_err) - } else { - t.Fatal(expectsFrom(name)) - } - } else if checksum != (unique.Handle[pkg.Checksum]{}) { - t.Fatalf("Cure: unexpected checksum %s", pkg.Encode(checksum.Value())) - } - } else { - v := any(err) - if err == nil { - v = pathname - } - var checksumVal pkg.Checksum - if checksum != (unique.Handle[pkg.Checksum]{}) { - checksumVal = checksum.Value() - } - t.Log(pkg.Encode(checksumVal)+":", v) - } - } -} - -// newWantScrubError returns the address to a new [ScrubError] for base. -func newWantScrubError(base *check.Absolute) *pkg.ScrubError { - return &pkg.ScrubError{ - ChecksumMismatches: []pkg.ChecksumMismatchError{ - {Got: pkg.MustDecode( - "vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", - ), Want: pkg.Checksum{0xff, 0}}, - }, - DanglingIdentifiers: []pkg.ID{ - {0xfe, 0}, - {0xfe, 0xfe}, - {0xfe, 0xff}, - }, - Errs: map[unique.Handle[string]][]error{ - base.Append("checksum", "__8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA").Handle(): { - pkg.InvalidFileModeError(fs.ModeSymlink), - }, - - base.Append("checksum", "invalid").Handle(): { - base64.CorruptInputError(4), - }, - - base.Append("nonexistent").Handle(): { - base64.CorruptInputError(8), - }, - - base.Append("identifier", pkg.Encode(pkg.ID{0xfe, 0xff})).Handle(): { - &os.PathError{ - Op: "readlink", - Path: base.Append( - "identifier", - pkg.Encode(pkg.ID{0xfe, 0xff}), - ).String(), - Err: syscall.EINVAL, - }, - }, - base.Append("identifier", "invalid").Handle(): { - base64.CorruptInputError(4), - }, - }, - } -} - -func TestCache(t *testing.T) { - t.Parallel() - - testdata := expectsFile("" + - "\x00\x00\x00\x00" + - "\xad\x0b\x00" + - "\x04" + - "\xfe\xfe\x00\x00" + - "\xfe\xca\x00\x00") - - testCases := []cacheTestCase{ - {"file", pkg.CValidateKnown | pkg.CAssumeChecksum, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - identifier := (pkg.ID)(bytes.Repeat([]byte{ - 0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f, - }, 8)) - wantPathname := base.Append( - "identifier", - "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", - ) - identifier0 := (pkg.ID)(bytes.Repeat([]byte{ - 0x71, 0xa7, 0xde, 0x6d, 0xa6, 0xde, - }, 8)) - wantPathname0 := base.Append( - "identifier", - "cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe", - ) - failingFile := newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 3}, - nil, - nil, struct { - _ []byte - stub.UniqueError - }{UniqueError: 0xbad}, - ) - - cureMany(t, c, []cureStep{ - {"initial file", newStubFile( - pkg.KindHTTPGet, - identifier, - new(testdata.hash()), - testdata, nil, - ), wantPathname, testdata, pkg.WNew, nil}, - - {"identical content", newStubFile( - pkg.KindHTTPGet, - identifier0, - new(testdata.hash()), - testdata, nil, - ), wantPathname0, testdata, pkg.WCache, nil}, - - {"existing entry", newStubFile( - pkg.KindHTTPGet, - identifier, - new(testdata.hash()), - testdata, nil, - ), wantPathname, testdata, pkg.WCache, nil}, - - {"checksum mismatch", newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 0}, - new(pkg.Checksum), - testdata, nil, - ), nil, nil, pkg.WNew, &pkg.ChecksumMismatchError{ - Got: testdata.hash(), - }}, - - {"store without validation", newStubFile( - pkg.KindHTTPGet, - pkg.MustDecode("vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX"), - nil, - []byte{0}, nil, - ), base.Append( - "identifier", - "vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", - ), expectsChecksum{ - 0xbe, 0xc0, 0x21, 0xb4, 0xf3, 0x68, - 0xe3, 0x06, 0x91, 0x34, 0xe0, 0x12, - 0xc2, 0xb4, 0x30, 0x70, 0x83, 0xd3, - 0xa9, 0xbd, 0xd2, 0x06, 0xe2, 0x4e, - 0x5f, 0x0d, 0x86, 0xe1, 0x3d, 0x66, - 0x36, 0x65, 0x59, 0x33, 0xec, 0x2b, - 0x41, 0x34, 0x65, 0x96, 0x68, 0x17, - 0xa9, 0xc2, 0x08, 0xa1, 0x17, 0x17, - }, pkg.WNew, nil}, - - {"incomplete implementation", struct{ pkg.Artifact }{&stubArtifact{ - kind: pkg.KindExec, - params: []byte("artifact overridden to be incomplete"), - }}, nil, nil, pkg.WNew, pkg.InvalidArtifactError(pkg.MustDecode( - "poVrv3zNINbHd-WUJihwgWQwJpbHURFHfpY7Zux-ztto_bQhhS5PdpbNMjdkXv-V", - ))}, - - {"error passthrough", newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 1}, - nil, - nil, stub.UniqueError(0xcafe), - ), nil, nil, pkg.WNew, stub.UniqueError(0xcafe)}, - - {"error caching", newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 1}, - nil, - nil, nil, - ), nil, nil, pkg.WCache, stub.UniqueError(0xcafe)}, - - {"cache hit bad type", overrideChecksum{testdata.hash(), overrideIdent{pkg.ID{0xff, 2}, &stubArtifact{ - kind: pkg.KindTar, - }}}, nil, nil, pkg.WCache, pkg.InvalidFileModeError( - 0400, - )}, - - {"noncomparable error", &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("artifact with dependency returning noncomparable error"), - deps: []pkg.Artifact{failingFile}, - - cure: func(f *pkg.FContext) error { - panic("attempting to cure impossible artifact") - }, - }, nil, nil, pkg.WNew, pkg.InputError{ - failingFile: struct { - _ []byte - stub.UniqueError - }{UniqueError: 0xbad}, - }}, - }) - - if c0, err := pkg.Open( - t.Context(), - message.New(nil), - base, &skipLock, - ); err != nil { - t.Fatalf("open: error = %v", err) - } else { - t.Cleanup(c.Close) // check doubled cancel - cureMany(t, c0, []cureStep{ - {"cache hit ident", overrideIdent{ - id: identifier, - }, wantPathname, testdata, pkg.WCache, nil}, - - {"cache miss checksum match", newStubFile( - pkg.KindHTTPGet, - testdata.hash(), - nil, - testdata, - nil, - ), base.Append( - "identifier", - pkg.Encode(testdata.hash()), - ), testdata, pkg.WNew, nil}, - }) - - // cure after close - c.Close() - if _, _, err = c.Cure(&stubArtifactF{ - kind: pkg.KindExec, - params: []byte("unreachable artifact cured after cancel"), - deps: []pkg.Artifact{pkg.NewFile("", []byte("unreachable dependency"))}, - }); !reflect.DeepEqual(err, context.Canceled) { - t.Fatalf("(closed) Cure: error = %v", err) - } - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX": {Mode: 0400, Data: []byte{0}}, - "checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq": {Mode: 0400, Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX")}, - "identifier/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq")}, - "identifier/cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq")}, - "identifier/deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"directory", pkg.CAssumeChecksum, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - id := pkg.MustDecode( - "HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY", - ) - makeSample := func(t *pkg.TContext) error { - work := t.GetWorkDir() - if err := os.Mkdir(work.String(), 0700); err != nil { - return err - } - - if err := os.WriteFile( - work.Append("check").String(), - []byte{0, 0}, - 0400, - ); err != nil { - return err - } - - if err := os.MkdirAll(work.Append( - "lib", - "pkgconfig", - ).String(), 0700); err != nil { - return err - } - - return os.Symlink( - "/proc/nonexistent/libedac.so", - work.Append( - "lib", - "libedac.so", - ).String(), - ) - } - want := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "check": {Mode: 0400, Data: []byte{0, 0}}, - - "lib": {Mode: fs.ModeDir | 0700}, - "lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - - "lib/pkgconfig": {Mode: fs.ModeDir | 0700}, - } - wantPathname := base.Append( - "identifier", - pkg.Encode(id), - ) - - id0 := pkg.MustDecode( - "Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa", - ) - wantPathname0 := base.Append( - "identifier", - pkg.Encode(id0), - ) - - makeGarbage := func(work *check.Absolute, wantErr error) error { - if err := os.Mkdir(work.String(), 0700); err != nil { - return err - } - - mode := fs.FileMode(0) - if wantErr == nil { - mode = 0500 - } - - if err := os.MkdirAll(work.Append( - "lib", - "pkgconfig", - ).String(), 0700); err != nil { - return err - } - - if err := os.WriteFile(work.Append( - "lib", - "check", - ).String(), nil, 0400&mode); err != nil { - return err - } - - if err := os.Chmod(work.Append( - "lib", - "pkgconfig", - ).String(), 0500&mode); err != nil { - return err - } - if err := os.Chmod(work.Append( - "lib", - ).String(), 0500&mode); err != nil { - return err - } - - return wantErr - } - - cureMany(t, c, []cureStep{ - {"initial directory", overrideChecksum{want.hash(), overrideIdent{id, &stubArtifact{ - kind: pkg.KindTar, - cure: makeSample, - }}}, wantPathname, want, pkg.WNew, nil}, - - {"identical identifier", overrideChecksum{want.hash(), overrideIdent{id, &stubArtifact{ - kind: pkg.KindTar, - }}}, wantPathname, want, pkg.WCache, nil}, - - {"identical checksum", overrideIdent{id0, &stubArtifact{ - kind: pkg.KindTar, - cure: makeSample, - }}, wantPathname0, want, pkg.WNew, nil}, - - {"cure fault", overrideIdent{pkg.ID{0xff, 0}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return makeGarbage(t.GetWorkDir(), stub.UniqueError(0xcafe)) - }, - }}, nil, nil, pkg.WNew, stub.UniqueError(0xcafe)}, - - {"checksum mismatch", overrideChecksum{pkg.Checksum{}, overrideIdent{pkg.ID{0xff, 1}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return makeGarbage(t.GetWorkDir(), nil) - }, - }}}, nil, nil, pkg.WNew, &pkg.ChecksumMismatchError{ - Got: pkg.MustDecode( - "CUx-3hSbTWPsbMfDhgalG4Ni_GmR9TnVX8F99tY_P5GtkYvczg9RrF5zO0jX9XYT", - ), - }}, - - {"cache hit bad type", newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 2}, - new(want.hash()), - testdata, nil, - ), nil, nil, pkg.WCache, pkg.InvalidFileModeError( - fs.ModeDir | 0500, - )}, - - {"openFile directory", overrideIdent{pkg.ID{0xff, 3}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - r, err := t.Open(overrideChecksumFile{checksum: want.hash()}) - if err != nil { - panic(err) - } - _, err = io.ReadAll(r) - return err - }, - }}, nil, nil, pkg.WNew, &os.PathError{ - Op: "read", - Path: base.Append( - "checksum", - pkg.Encode(want.hash()), - ).String(), - Err: syscall.EISDIR, - }}, - - {"no output", overrideIdent{pkg.ID{0xff, 4}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return nil - }, - }}, nil, nil, pkg.WNew, pkg.NoOutputError{}}, - - {"file output", overrideIdent{pkg.ID{0xff, 5}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return os.WriteFile(t.GetWorkDir().String(), []byte{0}, 0400) - }, - }}, nil, nil, pkg.WNew, errors.New("non-file artifact produced regular file")}, - - {"symlink output", overrideIdent{pkg.ID{0xff, 6}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return os.Symlink( - t.GetWorkDir().String(), - t.GetWorkDir().String(), - ) - }, - }}, nil, nil, pkg.WNew, pkg.InvalidFileModeError( - fs.ModeSymlink | 0777, - )}, - - {"alternative", &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("substitutable artifact"), - deps: []pkg.Artifact{newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 8}, - nil, - []byte("substitutable dependency"), - nil, - )}, - - cure: func(f *pkg.FContext) error { - return makeSample(&f.TContext) - }, - }, base.Append( - "identifier", - "gIH5eti07nBAhVti3ZTn8EIGOBCZWWvNpxg5CmCZlDzIC4mBdomPmQLeEpf1IQg9", - ), want, pkg.WNew, nil}, - - {"substitutable", &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("substitutable artifact"), - deps: []pkg.Artifact{newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 10}, - nil, - []byte("substitutable dependency"), - nil, - )}, - - cure: func(f *pkg.FContext) error { - panic("substitution missed") - }, - }, base.Append( - "identifier", - "VGjpp7NVw3OYLpgOQumfM0KcJgY3suz6torykUczcg4bGT24ySkPn5hy-4dNqqkR", - ), want, pkg.WSubstitute, nil}, - }) - - if c0, err := pkg.Open( - t.Context(), - message.New(nil), - base, &skipLock, - ); err != nil { - t.Fatalf("open: error = %v", err) - } else { - t.Cleanup(c.Close) - cureMany(t, c0, []cureStep{ - {"substitutable", &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("substitutable artifact"), - deps: []pkg.Artifact{newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 0xff, 0xfd, 0xfd}, - nil, - []byte("substitutable dependency"), - nil, - )}, - - cure: func(f *pkg.FContext) error { - panic("substitution missed") - }, - }, base.Append( - "identifier", - "zeZ4JEZ0atAKl6Am67cBID6-KR18nZGgeYfU4qTuYaWhQC4eAn0dyTXdIBH9Bos8", - ), want, pkg.WSubstitute, nil}, - }) - } - - if dents, err := os.ReadDir(base.Append("status").String()); err != nil { - t.Fatal(err) - } else if len(dents) > 0 { - t.Errorf("ReadDir: %v", dents) - } - - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO": {Mode: 0400, Data: []byte("substitutable dependency")}, - "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b": {Mode: fs.ModeDir | 0500}, - "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/check": {Mode: 0400, Data: []byte{0, 0}}, - "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/lib": {Mode: fs.ModeDir | 0700}, - "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/lib/pkgconfig": {Mode: fs.ModeDir | 0700}, - "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")}, - "identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")}, - "identifier/zeZ4JEZ0atAKl6Am67cBID6-KR18nZGgeYfU4qTuYaWhQC4eAn0dyTXdIBH9Bos8": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")}, - "identifier/___9_QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO")}, - "identifier/_wgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO")}, - "identifier/_woAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO")}, - "identifier/VGjpp7NVw3OYLpgOQumfM0KcJgY3suz6torykUczcg4bGT24ySkPn5hy-4dNqqkR": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")}, - "identifier/gIH5eti07nBAhVti3ZTn8EIGOBCZWWvNpxg5CmCZlDzIC4mBdomPmQLeEpf1IQg9": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "substitute/nWPg893oqn_YNN8kgfZrs_LVLwP90e0mKxqU8UwoazQh-yBPwhoviD7of6YALx-M": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"pending", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - wantErr := stub.UniqueError(0xcafe) - n, ready := make(chan struct{}), make(chan struct{}) - go func() { - if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - close(ready) - <-n - return wantErr - }, - }}); !reflect.DeepEqual(err, wantErr) { - panic(fmt.Sprintf("Cure: error = %v, want %v", err, wantErr)) - } - }() - - <-ready - wCureDone := make(chan struct{}) - go func() { - if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, &stubArtifact{ - kind: pkg.KindTar, - }}); !reflect.DeepEqual(err, wantErr) { - panic(fmt.Sprintf("Cure: error = %v, want %v", err, wantErr)) - } - close(wCureDone) - }() - - // check cache activity while a cure is blocking - cureMany(t, c, []cureStep{ - {"error passthrough", newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 1}, - nil, - nil, stub.UniqueError(0xbad), - ), nil, nil, pkg.WNew, stub.UniqueError(0xbad)}, - - {"file output", overrideIdent{pkg.ID{0xff, 2}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return os.WriteFile( - t.GetWorkDir().String(), - []byte{0}, - 0400, - ) - }, - }}, nil, nil, pkg.WNew, errors.New( - "non-file artifact produced regular file", - )}, - }) - - wantErrScrub := &pkg.ScrubError{ - Errs: map[unique.Handle[string]][]error{ - base.Handle(): {errors.New("scrub began with pending artifacts")}, - }, - } - if err := c.Scrub(1 << 6); !reflect.DeepEqual(err, wantErrScrub) { - t.Fatalf("Scrub: error = %#v, want %#v", err, wantErrScrub) - } - - notify := c.Done(unique.Make(pkg.ID{0xff})) - go close(n) - if notify != nil { - <-notify - } - for c.Done(unique.Make(pkg.ID{0xff})) != nil { - } - <-wCureDone - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"cancel abort block", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - var wg sync.WaitGroup - defer wg.Wait() - - var started sync.WaitGroup - defer started.Wait() - - blockCures := func(d byte, e stub.UniqueError, n int) { - started.Add(n) - for i := range n { - wg.Go(func() { - if _, _, err := c.Cure(overrideIdent{pkg.ID{d, byte(i)}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - started.Done() - <-t.Unwrap().Done() - return e + stub.UniqueError(i) - }, - }}); !reflect.DeepEqual(err, e+stub.UniqueError(i)) { - panic(err) - } - }) - } - started.Wait() - } - - blockCures(0xfd, 0xbad, 16) - c.Abort() - wg.Wait() - - blockCures(0xfd, 0xcafe, 16) - c.Abort() - wg.Wait() - - blockCures(0xff, 0xbad, 1) - if !c.Cancel(unique.Make(pkg.ID{0xff})) { - t.Fatal("missed cancellation") - } - wg.Wait() - - blockCures(0xff, 0xcafe, 1) - if !c.Cancel(unique.Make(pkg.ID{0xff})) { - t.Fatal("missed cancellation") - } - wg.Wait() - - for c.Cancel(unique.Make(pkg.ID{0xff})) { - } - - c.Close() - c.Abort() - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"no assume checksum", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - makeGarbage := func(work *check.Absolute, wantErr error) error { - if err := os.Mkdir(work.String(), 0700); err != nil { - return err - } - - if err := os.WriteFile(work.Append( - "check", - ).String(), nil, 0400); err != nil { - return err - } - - return wantErr - } - - want := expectsChecksum(pkg.MustDecode( - "Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M", - )) - - cureMany(t, c, []cureStep{ - {"create", overrideChecksum{want.hash(), overrideIdent{pkg.ID{0xff, 0}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return makeGarbage(t.GetWorkDir(), nil) - }, - }}}, base.Append( - "identifier", - pkg.Encode(pkg.ID{0xff, 0}), - ), want, pkg.WNew, nil}, - - {"reject", overrideChecksum{want.hash(), overrideIdent{pkg.ID{0xfe, 1}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return makeGarbage(t.GetWorkDir(), stub.UniqueError(0xbad)) - }, - }}}, nil, nil, pkg.WNew, stub.UniqueError(0xbad)}, - - {"match", overrideChecksum{want.hash(), overrideIdent{pkg.ID{0xff, 1}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return makeGarbage(t.GetWorkDir(), nil) - }, - }}}, base.Append( - "identifier", - pkg.Encode(pkg.ID{0xff, 1}), - ), want, pkg.WNew, nil}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M": {Mode: fs.ModeDir | 0500}, - "checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M/check": {Mode: 0400, Data: []byte{}}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/_wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M")}, - "identifier/_wEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"scrub", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - cureMany(t, c, []cureStep{ - {"bad measured file", newStubFile( - pkg.KindHTTPGet, - pkg.Checksum{0xfe, 0}, - &pkg.Checksum{0xff, 0}, - []byte{0}, nil, - ), base.Append( - "identifier", - pkg.Encode(pkg.Checksum{0xfe, 0}), - ), expectsChecksum{0xff, 0}, pkg.WNew, nil}, - }) - - for _, p := range [][]string{ - {"identifier", "invalid"}, - {"identifier", pkg.Encode(pkg.ID{0xfe, 0xff})}, - {"checksum", "invalid"}, - } { - if err := os.WriteFile( - base.Append(p...).String(), - nil, - 0400, - ); err != nil { - t.Fatal(err) - } - } - - for _, p := range [][]string{ - {"../nonexistent", "checksum", pkg.Encode(pkg.Checksum{0xff, 0xff})}, - {"../nonexistent", "identifier", pkg.Encode(pkg.Checksum{0xfe, 0xfe})}, - } { - if err := os.Symlink( - p[0], - base.Append(p[1:]...).String(), - ); err != nil { - t.Fatal(err) - } - } - - wantErr := newWantScrubError(base) - if err := c.Scrub(1 << 6); !reflect.DeepEqual(err, wantErr) { - t.Fatalf("Scrub: error =\n%s\nwant\n%s", err, wantErr) - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"status substitute clean", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - destroyed := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("destroyed"), - deps: []pkg.Artifact{ - pkg.NewFile("destroyed-input", []byte("destroyed")), - }, - cure: func(f *pkg.FContext) error { - if w, err := f.GetStatusWriter(); err != nil { - return err - } else if _, err = w.Write([]byte("destroyed")); err != nil { - return err - } - - p := f.GetWorkDir() - if err := os.MkdirAll(p.String(), 0755); err != nil { - return err - } - return os.WriteFile(p.Append("result").String(), nil, 0444) - }, - } - substituted := new(*destroyed) - substituted.deps = []pkg.Artifact{ - pkg.NewFile("destroyed-input-0", []byte("destroyed")), - } - substituted.cure = func(*pkg.FContext) error { - panic("substitutable cure reached") - } - - cureMany(t, c, []cureStep{ - {"destroyed", destroyed, base.Append( - "identifier", - pkg.Encode(c.Ident(destroyed).Value()), - ), expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - "result": {Mode: 0444}, - }, pkg.WNew, nil}, - - {"substituted", substituted, base.Append( - "identifier", - pkg.Encode(c.Ident(substituted).Value()), - ), expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - "result": {Mode: 0444}, - }, pkg.WSubstitute, nil}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE": {Mode: fs.ModeDir | 0500}, - "checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE/result": {Mode: 0444}, - "checksum/wILUy2izpj2sgKJVhGUGIAde1XVuqvp5BpFMIQHanT5Q8R6jK4QPVSrJsjZh-njV": {Mode: 0400, Data: []byte("destroyed")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/CLyyxJWn1yUGFs_0KnJ4ZSU2hH4-elvGFRB36x14LrNrKBJPxvORhL9O4Rc7oeYU": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/wILUy2izpj2sgKJVhGUGIAde1XVuqvp5BpFMIQHanT5Q8R6jK4QPVSrJsjZh-njV")}, - "identifier/arv8a6z4i0x6pUb5Z2T-3iE4ETvyZj-uReHGKsS7uiBgMgKNEOzTGG3LIZb8vp0L": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/wILUy2izpj2sgKJVhGUGIAde1XVuqvp5BpFMIQHanT5Q8R6jK4QPVSrJsjZh-njV")}, - "identifier/heRgWhNJmhm898V68kw0ta76t4rP1o29tYW_TrdyBWXGfo37K3QWA7Z-JkLXAwcn": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")}, - "identifier/q-VoVBI3IjcWTqwIokpi3y05CP113CHgHslc-XWnhOvDULwzVc3q9Y5xOCm_YyLZ": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")}, - - "status": {Mode: fs.ModeDir | 0700}, - "status/q-VoVBI3IjcWTqwIokpi3y05CP113CHgHslc-XWnhOvDULwzVc3q9Y5xOCm_YyLZ": {Mode: fs.ModeSymlink | 0777, Data: []byte("dP-6_wIDRRouOaOF-nkBy-IaUbLdYHbOUTrptu7j_qfW01mnHSjYJ0oykUmvUd2x")}, - "status/heRgWhNJmhm898V68kw0ta76t4rP1o29tYW_TrdyBWXGfo37K3QWA7Z-JkLXAwcn": {Mode: 0400, Data: []byte(statusHeader + "destroyed")}, - "status/dP-6_wIDRRouOaOF-nkBy-IaUbLdYHbOUTrptu7j_qfW01mnHSjYJ0oykUmvUd2x": {Mode: 0400, Data: []byte(statusHeader + "destroyed")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "substitute/dP-6_wIDRRouOaOF-nkBy-IaUbLdYHbOUTrptu7j_qfW01mnHSjYJ0oykUmvUd2x": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"extern", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - a := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("extern"), - } - wantIdent := c.Ident(a) - wantOutput := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - "result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")}, - } - var wantChecksum pkg.Checksum - if err := pkg.SumFS( - &wantChecksum, - fstest.MapFS(wantOutput), - ".", - ); err != nil { - t.Fatal(err) - } - wantChecksumH := unique.Make(wantChecksum) - - _a := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("extern substitute"), - deps: []pkg.Artifact{pkg.NewFile("", nil)}, - } - _wantIdent := c.Ident(_a) - _wantOutput := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - } - var _wantChecksum pkg.Checksum - if err := pkg.SumFS( - &_wantChecksum, - fstest.MapFS(_wantOutput), - ".", - ); err != nil { - t.Fatal(err) - } - _wantChecksumH := unique.Make(_wantChecksum) - - kca := pkg.NewExec( - "", "", - new(pkg.Checksum), 0, false, false, - fhs.AbsRoot, nil, fhs.AbsRoot, nil, - ) - kcIdent := c.Ident(kca) - - c.SetExternal(stubExtern{ - artifact: map[unique.Handle[pkg.ID]]pkg.Checksum{ - wantIdent: wantChecksum, - _wantIdent: _wantChecksum, - kcIdent: wantChecksum, - }, - checksum: map[unique.Handle[pkg.Checksum]]fstest.MapFS{ - wantChecksumH: fstest.MapFS(wantOutput), - _wantChecksumH: fstest.MapFS(_wantOutput), - }, - status: map[unique.Handle[pkg.ID]]string{ - wantIdent: "\x00", - kcIdent: "unreachable", - }, - }) - - cureMany(t, c, []cureStep{ - {"extern", a, base.Append( - "identifier", - pkg.Encode(wantIdent.Value()), - ), wantOutput, pkg.WExternal, nil}, - - {"substitute", _a, base.Append( - "identifier", - pkg.Encode(_wantIdent.Value()), - ), _wantOutput, pkg.WExternal, nil}, - - {"mismatch", kca, nil, nil, pkg.WExternal, &pkg.ChecksumMismatchError{ - Got: wantChecksum, - }}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400}, - "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl": {Mode: fs.ModeDir | 0500}, - "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl/result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/k8yyhlBV3JpYzzywAnrRWkfp2TmgIDNC6FqMFwrrR2x0C3unBLveDNKgigOjG9MM": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/Xys8hPpJsKirZmTTuT7SwQ6661TCp_ZRu8iUjPPD7pC5E3ll7xw4zhNlZ2PVcz_-": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - "identifier/8sInO_dhPW9kweit7jVuH_vtMkRxlZT7XE4_yEKPZmpGkgVxkmxz0OpXgbMsH6ET": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/ofaXbB5gIza_7W4Rzo5XDe_8FwlCPKWn3TBcmGgFAkkTWMl5GXGmOzX89Jd2Mk2A": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")}, - "identifier/8VL-egZySw-RbnHZtxGWdf5Z2v9S9CihrdFwa54Bc5EYMQcEgWEem7QrdCTWlCb8": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - - "status": {Mode: fs.ModeDir | 0700}, - "status/8VL-egZySw-RbnHZtxGWdf5Z2v9S9CihrdFwa54Bc5EYMQcEgWEem7QrdCTWlCb8": {Mode: 0400, Data: []byte("\x00")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "substitute/socGRnhdsJlJ4Z01ZwHFZdLzFH0rWvoaJrylcUKtvhBdGsE7VoYl-vdUuvfK8B4I": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"extern shallow", pkg.CExternShallow, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - a := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("extern"), - } - wantIdent := c.Ident(a) - wantOutput := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - "result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")}, - } - var wantChecksum pkg.Checksum - if err := pkg.SumFS( - &wantChecksum, - fstest.MapFS(wantOutput), - ".", - ); err != nil { - t.Fatal(err) - } - wantChecksumH := unique.Make(wantChecksum) - - _a := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("extern substitute"), - deps: []pkg.Artifact{pkg.NewFile("", nil)}, - } - _wantIdent := c.Ident(_a) - _wantOutput := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - } - var _wantChecksum pkg.Checksum - if err := pkg.SumFS( - &_wantChecksum, - fstest.MapFS(_wantOutput), - ".", - ); err != nil { - t.Fatal(err) - } - _wantChecksumH := unique.Make(_wantChecksum) - - kca := pkg.NewExec( - "", "", - new(pkg.Checksum), 0, false, false, - fhs.AbsRoot, nil, fhs.AbsRoot, nil, - ) - kcIdent := c.Ident(kca) - - fia := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("flood input"), - deps: []pkg.Artifact{_a}, - } - fiaIdent := c.Ident(fia) - - flia := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("flood input local"), - deps: []pkg.Artifact{_a}, - } - fliaIdent := c.Ident(flia) - - h := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("hanging input"), - } - - fhia := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("flood input local"), - deps: []pkg.Artifact{h}, - } - fhiaIdent := c.Ident(fhia) - - c.SetExternal(stubExtern{ - artifact: map[unique.Handle[pkg.ID]]pkg.Checksum{ - wantIdent: wantChecksum, - _wantIdent: _wantChecksum, - kcIdent: wantChecksum, - fiaIdent: wantChecksum, - fliaIdent: wantChecksum, - fhiaIdent: wantChecksum, - }, - checksum: map[unique.Handle[pkg.Checksum]]fstest.MapFS{ - wantChecksumH: fstest.MapFS(wantOutput), - _wantChecksumH: fstest.MapFS(_wantOutput), - }, - status: map[unique.Handle[pkg.ID]]string{ - wantIdent: "\x00", - kcIdent: "unreachable", - }, - }) - - cureMany(t, c, []cureStep{ - {"extern", a, base.Append( - "identifier", - pkg.Encode(wantIdent.Value()), - ), wantOutput, pkg.WExternal, nil}, - - {"flood shallow extern", fia, base.Append( - "identifier", - pkg.Encode(fiaIdent.Value()), - ), wantOutput, pkg.WExternal, nil}, - - {"substitute", _a, base.Append( - "identifier", - pkg.Encode(_wantIdent.Value()), - ), _wantOutput, pkg.WExternal, nil}, - - {"flood shallow local", flia, base.Append( - "identifier", - pkg.Encode(fliaIdent.Value()), - ), wantOutput, pkg.WExternal, nil}, - - {"flood shallow hanging", fhia, nil, nil, - pkg.WNew, pkg.HangingInputError(c.Ident(h))}, - - {"mismatch", kca, nil, nil, pkg.WExternal, &pkg.ChecksumMismatchError{ - Got: wantChecksum, - }}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl": {Mode: fs.ModeDir | 0500}, - "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl/result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/3fYJjSiWhklt7TK9MPVLGk5_8AIU8ostTcFPXULJf_qeAVG6pmC-Cqqu4ZhEXRxn": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - "identifier/8VL-egZySw-RbnHZtxGWdf5Z2v9S9CihrdFwa54Bc5EYMQcEgWEem7QrdCTWlCb8": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - "identifier/8sInO_dhPW9kweit7jVuH_vtMkRxlZT7XE4_yEKPZmpGkgVxkmxz0OpXgbMsH6ET": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/Xys8hPpJsKirZmTTuT7SwQ6661TCp_ZRu8iUjPPD7pC5E3ll7xw4zhNlZ2PVcz_-": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - "identifier/k8yyhlBV3JpYzzywAnrRWkfp2TmgIDNC6FqMFwrrR2x0C3unBLveDNKgigOjG9MM": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/mXlNz_v77nW7jstn_rJn4RL7dCQlkakM0y-HjO6tkDubtzaj-wCmW1hIr2Gk64_O": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - - "status": {Mode: fs.ModeDir | 0700}, - "status/8VL-egZySw-RbnHZtxGWdf5Z2v9S9CihrdFwa54Bc5EYMQcEgWEem7QrdCTWlCb8": {Mode: 0400, Data: []byte("\x00")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "substitute/socGRnhdsJlJ4Z01ZwHFZdLzFH0rWvoaJrylcUKtvhBdGsE7VoYl-vdUuvfK8B4I": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "substitute/MccYezQ4pshkeBV4w_aAEIwvZdKHL3OAgAkR3ouZ43EF94Ur1HBHaN7njPfJAj50": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - "substitute/VV5W0_DiA1iylaqbKFThp1ZpV_y1ccVS1XyqDnDaXoHQ-EEyPI4XFrQHcHOF2hWG": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - } - checkWithCache(t, testCases) -} - -func TestErrors(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - err error - want string - }{ - {"InvalidLookupError", pkg.InvalidLookupError{ - 0xff, 0xf0, - }, "attempting to look up non-input artifact __AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}, - - {"InvalidArtifactError", pkg.InvalidArtifactError{ - 0xff, 0xfd, - }, "artifact __0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA cannot be cured"}, - - {"ChecksumMismatchError", &pkg.ChecksumMismatchError{ - Want: (pkg.Checksum)(bytes.Repeat([]byte{ - 0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f, - }, 8)), - }, "got AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + - " instead of deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"}, - - {"ResponseStatusError", pkg.ResponseStatusError( - http.StatusNotAcceptable, - ), "the requested URL returned non-OK status: Not Acceptable"}, - - {"DisallowedTypeflagError", pkg.DisallowedTypeflagError( - tar.TypeChar, - ), "disallowed typeflag '3'"}, - - {"InvalidFileModeError", pkg.InvalidFileModeError( - fs.ModeSymlink | 0777, - ), "artifact did not produce a regular file or directory"}, - - {"NoOutputError", pkg.NoOutputError{ - // empty struct - }, "artifact cured successfully but did not produce any output"}, - - {"IRKindError", &pkg.IRKindError{ - Got: pkg.IRKindEnd, - Want: pkg.IRKindIdent, - Ancillary: 0xcafebabe, - }, "got terminator IR value (0xcafebabe) instead of ident"}, - {"IRKindError invalid", &pkg.IRKindError{ - Got: 0xbeef, - Want: pkg.IRKindIdent, - Ancillary: 0xcafe, - }, "got invalid kind 48879 IR value (0xcafe) instead of ident"}, - - {"UnsupportedVariantError", pkg.UnsupportedVariantError( - "rosa", - ), `unsupported variant "rosa"`}, - - {"UnsupportedArchError zero", pkg.UnsupportedArchError(""), - "invalid architecture name"}, - {"UnsupportedArchError", pkg.UnsupportedArchError("riscv64"), - "unsupported architecture riscv64"}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - if got := tc.err.Error(); got != tc.want { - t.Errorf("Error: %q, want %q", got, tc.want) - } - }) - } -} - -func TestScrubError(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - err pkg.ScrubError - want string - unwrap []error - }{ - {"sample", *newWantScrubError( - fhs.AbsVarLib.Append("cure"), - ), `checksum mismatches: -got vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX instead of _wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA - -dangling identifiers: -_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -_v4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA - -errors during scrub: - /var/lib/cure/checksum/__8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: - artifact did not produce a regular file or directory - /var/lib/cure/checksum/invalid: - illegal base64 data at input byte 4 - /var/lib/cure/identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: - readlink /var/lib/cure/identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: invalid argument - /var/lib/cure/identifier/invalid: - illegal base64 data at input byte 4 - /var/lib/cure/nonexistent: - illegal base64 data at input byte 8 -`, []error{ - &pkg.ChecksumMismatchError{Got: pkg.MustDecode( - "vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", - ), Want: pkg.Checksum{0xff, 0}}, - - pkg.InvalidFileModeError(fs.ModeSymlink), - base64.CorruptInputError(4), - - &os.PathError{ - Op: "readlink", - Path: fhs.AbsVarLib.Append("cure").Append( - "identifier", - pkg.Encode(pkg.ID{0xfe, 0xff}), - ).String(), - Err: syscall.EINVAL, - }, - - base64.CorruptInputError(4), - base64.CorruptInputError(8), - }}, - - {"full", pkg.ScrubError{ - ChecksumMismatches: []pkg.ChecksumMismatchError{ - {Want: pkg.MustDecode("CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN")}, - }, - DanglingIdentifiers: []pkg.ID{ - (pkg.ID)(bytes.Repeat([]byte{0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f}, 8)), - (pkg.ID)(bytes.Repeat([]byte{0x71, 0xa7, 0xde, 0x6d, 0xa6, 0xde}, 8)), - }, - Errs: map[unique.Handle[string]][]error{ - unique.Make("/proc/nonexistent"): { - stub.UniqueError(0xcafe), - stub.UniqueError(0xbad), - stub.UniqueError(0xff), - }, - }, - }, `checksum mismatches: -got AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA instead of CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN - -dangling identifiers: -deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef -cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe - -errors during scrub: - /proc/nonexistent: - unique error 51966 injected by the test suite - unique error 2989 injected by the test suite - unique error 255 injected by the test suite -`, []error{ - &pkg.ChecksumMismatchError{Want: pkg.MustDecode("CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN")}, - stub.UniqueError(0xcafe), - stub.UniqueError(0xbad), - stub.UniqueError(0xff), - }}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - if got := tc.err.Error(); got != tc.want { - t.Errorf("Error:\n\n%s\n\nwant\n\n%s", got, tc.want) - } - - if unwrap := tc.err.Unwrap(); !reflect.DeepEqual(unwrap, tc.unwrap) { - t.Errorf("Unwrap: %#v, want %#v", unwrap, tc.unwrap) - } - }) - } -} - -func TestInputError(t *testing.T) { - t.Parallel() - - makeIdent := func(ident ...byte) pkg.Artifact { - var a overrideIdent - copy(a.id[:], ident) - // does not compare equal - a.TrivialArtifact = new(stubArtifact) - return a - } - - testCases := []struct { - name string - err pkg.InputError - want string - unwrap []error - }{ - {"simple", pkg.InputError{ - makeIdent(0xff, 9): stub.UniqueError(0xbad09), - makeIdent(0xff, 0): stub.UniqueError(0xbad00), - makeIdent(0xff, 0xf): stub.UniqueError(0xbad0f), - makeIdent(0xff, 1): stub.UniqueError(0xbad01), - }, `errors curing inputs: - _wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765184 injected by the test suite - _wEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765185 injected by the test suite - _wkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765193 injected by the test suite - _w8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765199 injected by the test suite`, []error{ - stub.UniqueError(0xbad00), - stub.UniqueError(0xbad01), - stub.UniqueError(0xbad09), - stub.UniqueError(0xbad0f), - }}, - - {"dedup", pkg.InputError{ - makeIdent(0xff, 9): stub.UniqueError(0xbad09), - makeIdent(0xff, 9): stub.UniqueError(0xbad09), - makeIdent(0xff, 9): stub.UniqueError(0xbad09), - makeIdent(0xff, 0): stub.UniqueError(0xbad00), - makeIdent(0xff, 0): stub.UniqueError(0xbad00), - makeIdent(0xff, 1): stub.UniqueError(0xbad01), - }, `errors curing inputs: - _wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765184 injected by the test suite - _wEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765185 injected by the test suite - _wkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765193 injected by the test suite`, []error{ - stub.UniqueError(0xbad00), - stub.UniqueError(0xbad01), - stub.UniqueError(0xbad09), - }}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - if got := tc.err.Error(); got != tc.want { - t.Errorf("Error:\n%s\nwant\n%s", got, tc.want) - } - - if unwrap := tc.err.Unwrap(); !reflect.DeepEqual(unwrap, tc.unwrap) { - t.Errorf("Unwrap: %#v, want %#v", unwrap, tc.unwrap) - } - }) - } -} - -// earlyFailureF is a [FloodArtifact] with a large dependency graph resulting in -// a large [DependencyCureError]. -type earlyFailureF int - -func (earlyFailureF) Kind() pkg.Kind { return pkg.KindExec } -func (earlyFailureF) Params(*pkg.IContext) {} -func (earlyFailureF) IsExclusive() bool { return false } - -func (a earlyFailureF) Inputs() []pkg.Artifact { - deps := make([]pkg.Artifact, a) - for i := range deps { - deps[i] = a - 1 - } - return deps -} - -func (a earlyFailureF) Cure(*pkg.FContext) error { - if a != 0 { - panic("unexpected cure on " + strconv.Itoa(int(a))) - } - return stub.UniqueError(0xcafe) -} - -func BenchmarkEarlyDCE(b *testing.B) { - msg := message.New(log.New(os.Stderr, "dce: ", 0)) - msg.SwapVerbose(testing.Verbose()) - - for b.Loop() { - b.StopTimer() - c, err := pkg.Open(b.Context(), msg, check.MustAbs(b.TempDir()), nil) - if err != nil { - b.Fatal(err) - } - b.StartTimer() - _, _, err = c.Cure(earlyFailureF(8)) - b.StopTimer() - if !errors.Is(err, stub.UniqueError(0xcafe)) { - b.Fatalf("Cure: error = %v", err) - } - c.Close() - b.StartTimer() - } -} - -func TestDependencyCureErrorEarly(t *testing.T) { - t.Parallel() - - checkWithCache(t, []cacheTestCase{ - {"early", 0, nil, func(t *testing.T, _ *check.Absolute, c *pkg.Cache) { - _, _, err := c.Cure(earlyFailureF(8)) - if !errors.Is(err, stub.UniqueError(0xcafe)) { - t.Fatalf("Cure: error = %v", err) - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} - -func TestOpen(t *testing.T) { - t.Parallel() - - t.Run("nonexistent", func(t *testing.T) { - t.Parallel() - - wantErr := &os.PathError{ - Op: "mkdir", - Path: container.Nonexistent, - Err: syscall.ENOENT, - } - if _, err := pkg.Open( - t.Context(), - message.New(nil), - check.MustAbs(container.Nonexistent), - nil, - ); !reflect.DeepEqual(err, wantErr) { - t.Errorf("Open: error = %#v, want %#v", err, wantErr) - } - }) - - t.Run("permission", func(t *testing.T) { - t.Parallel() - - tempDir := check.MustAbs(t.TempDir()) - if err := os.Chmod(tempDir.String(), 0); err != nil { - t.Fatal(err) - } else { - t.Cleanup(func() { - if err = os.Chmod(tempDir.String(), 0700); err != nil { - t.Fatal(err) - } - }) - } - - wantErr := &os.PathError{ - Op: "mkdir", - Path: tempDir.Append("cache").String(), - Err: syscall.EACCES, - } - if _, err := pkg.Open( - t.Context(), - message.New(nil), - tempDir.Append("cache"), - nil, - ); !reflect.DeepEqual(err, wantErr) { - t.Errorf("Open: error = %#v, want %#v", err, wantErr) - } - }) - - t.Run("dirty", func(t *testing.T) { - t.Parallel() - - tempDir := check.MustAbs(t.TempDir()) - if err := os.MkdirAll(tempDir.Append( - "cache", - "work", - "dirty", - ).String(), 0755); err != nil { - t.Fatal(err) - } - - wantErr := errors.New("work is not empty, scrub likely required") - if _, err := pkg.Open( - t.Context(), - message.New(nil), - tempDir.Append("cache"), - nil, - ); !reflect.DeepEqual(err, wantErr) { - t.Errorf("Open: error = %#v, want %#v", err, wantErr) - } - }) - - t.Run("scratch", func(t *testing.T) { - t.Parallel() - - tempDir := check.MustAbs(t.TempDir()) - if err := os.MkdirAll(tempDir.Append( - "cache", - "scratch", - ).String(), 0755); err != nil { - t.Fatal(err) - } - - wantErr := errors.New("scratch is present, scrub likely required") - if _, err := pkg.Open( - t.Context(), - message.New(nil), - tempDir.Append("cache"), nil, - ); !reflect.DeepEqual(err, wantErr) { - t.Errorf("Open: error = %#v, want %#v", err, wantErr) - } - }) -} - -func TestExtensionRegister(t *testing.T) { - extensionOld := extension - openedOld := opened - t.Cleanup(func() { extension = extensionOld; opened = openedOld }) - extension = "" - opened = false - - t.Run("set", func(t *testing.T) { - t.Cleanup(func() { extension = "" }) - - const want = "rosa" - pkg.SetExtension(want) - if got := pkg.Extension(); got != want { - t.Fatalf("Extension: %q, want %q", got, want) - } - }) - - t.Run("twice", func(t *testing.T) { - t.Cleanup(func() { extension = "" }) - - defer func() { - const wantPanic = "attempting to set extension twice" - if r := recover(); r != wantPanic { - t.Errorf("panic: %#v, want %q", r, wantPanic) - } - }() - pkg.SetExtension("rosa") - pkg.SetExtension("rosa") - }) - - t.Run("invalid", func(t *testing.T) { - defer func() { - var wantPanic = pkg.ErrInvalidExtension - if r := recover(); r != wantPanic { - t.Errorf("panic: %#v, want %#v", r, wantPanic) - } - }() - pkg.SetExtension(" ") - }) - - t.Run("opened", func(t *testing.T) { - t.Cleanup(func() { opened = false }) - - if _, err := pkg.Open( - t.Context(), - message.New(log.Default()), - check.MustAbs(container.Nonexistent), - nil, - ); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("Open: error = %v", err) - } - - t.Run("variant", func(t *testing.T) { - defer func() { - const wantPanic = "attempting to set extension after open" - if r := recover(); r != wantPanic { - t.Errorf("panic: %#v, want %q", r, wantPanic) - } - }() - pkg.SetExtension("rosa") - }) - - t.Run("register", func(t *testing.T) { - defer func() { - const wantPanic = "attempting to register after open" - if r := recover(); r != wantPanic { - t.Errorf("panic: %#v, want %q", r, wantPanic) - } - }() - pkg.Register(pkg.KindCustomOffset, nil) - }) - }) - - t.Run("incomplete", func(t *testing.T) { - t.Cleanup(func() { delete(irArtifact, pkg.KindCustomOffset) }) - - defer func() { - const wantPanic = "attempting to open cache with incomplete variant setup" - if r := recover(); r != wantPanic { - t.Errorf("panic: %#v, want %q", r, wantPanic) - } - }() - pkg.Register(pkg.KindCustomOffset, nil) - - t.Cleanup(func() { opened = false }) - _, _ = pkg.Open(nil, nil, nil, nil) - panic("unreachable") - }) - - t.Run("create", func(t *testing.T) { - t.Cleanup(func() { extension = "" }) - const want = "rosa" - pkg.SetExtension(want) - - base := check.MustAbs(t.TempDir()) - t.Cleanup(func() { opened = false }) - if c, err := pkg.Open(t.Context(), nil, base, nil); err != nil { - t.Fatal(err) - } else { - c.Close() - } - - if got, err := os.ReadFile(base.Append("variant").String()); err != nil { - t.Fatal(err) - } else if string(got) != want { - t.Fatalf("variant: %q", string(got)) - } - }) - - t.Run("access", func(t *testing.T) { - base := check.MustAbs(t.TempDir()) - t.Cleanup(func() { opened = false }) - - if err := os.WriteFile(base.Append("variant").String(), nil, 0); err != nil { - t.Fatal(err) - } - - wantErr := &os.PathError{ - Op: "open", - Path: base.Append("variant").String(), - Err: syscall.EACCES, - } - if _, err := pkg.Open( - t.Context(), nil, - base, nil, - ); !reflect.DeepEqual(err, wantErr) { - t.Fatalf("Open: error = %v, want %v", err, wantErr) - } - }) - - t.Run("promote", func(t *testing.T) { - t.Cleanup(func() { extension = "" }) - const want = "rosa" - pkg.SetExtension(want) - - base := check.MustAbs(t.TempDir()) - t.Cleanup(func() { opened = false }) - - variantPath := base.Append("variant") - if err := os.WriteFile(variantPath.String(), nil, 0600); err != nil { - t.Fatal(err) - } - - if _, err := pkg.Open( - t.Context(), nil, - base, - nil, - ); !reflect.DeepEqual(err, pkg.ErrWouldPromote) { - t.Fatalf("Open: error = %v", err) - } - - if p, err := os.ReadFile(variantPath.String()); err != nil { - t.Fatal(err) - } else if len(p) != 0 { - t.Fatalf("variant: %q", string(p)) - } - - if c, err := pkg.Open( - t.Context(), nil, - base, &pkg.CacheAttr{Flags: pkg.CPromoteVariant}, - ); err != nil { - t.Fatalf("Open: error = %v", err) - } else { - c.Close() - } - - if p, err := os.ReadFile(variantPath.String()); err != nil { - t.Fatal(err) - } else if string(p) != want { - t.Fatalf("variant: %q, want %q", string(p), want) - } - }) - - t.Run("open invalid", func(t *testing.T) { - base := check.MustAbs(t.TempDir()) - t.Cleanup(func() { opened = false }) - - variantPath := base.Append("variant") - if err := os.WriteFile(variantPath.String(), make([]byte, 129), 0400); err != nil { - t.Fatal(err) - } - - if _, err := pkg.Open( - t.Context(), nil, - base, nil, - ); !reflect.DeepEqual(err, pkg.ErrInvalidExtension) { - t.Fatalf("Open: error = %v", err) - } - }) - - t.Run("unsupported", func(t *testing.T) { - base := check.MustAbs(t.TempDir()) - t.Cleanup(func() { opened = false }) - - variantPath := base.Append("variant") - if err := os.WriteFile(variantPath.String(), []byte("rosa"), 0400); err != nil { - t.Fatal(err) - } - - if _, err := pkg.Open( - t.Context(), nil, - base, nil, - ); !reflect.DeepEqual(err, pkg.UnsupportedVariantError("rosa")) { - t.Fatalf("Open: error = %v", err) - } - }) -} diff --git a/internal/pkg/tar.go b/internal/pkg/tar.go deleted file mode 100644 index 9a8c4b76..00000000 --- a/internal/pkg/tar.go +++ /dev/null @@ -1,226 +0,0 @@ -package pkg - -import ( - "archive/tar" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" -) - -// A tarArtifact is an [Artifact] unpacking a tarball backed by a [FileArtifact]. -type tarArtifact struct { - // Caller-supplied backing tarball. - f Artifact -} - -var _ CuresExempt = new(tarArtifact) - -// tarArtifactNamed embeds tarArtifact for a [fmt.Stringer] tarball. -type tarArtifactNamed struct { - tarArtifact - // Copied from tarArtifact.f. - name string -} - -var _ fmt.Stringer = new(tarArtifactNamed) - -// String returns the name of the underlying [Artifact] prefixed with unpack. -func (a *tarArtifactNamed) String() string { return "unpack-" + a.name } - -// NewTar returns a new [Artifact] unpacking the tar stream produced by the -// backing [Artifact]. The source [Artifact] must be a [FileArtifact]. -func NewTar(a Artifact) Artifact { - ta := tarArtifact{a} - if s, ok := a.(fmt.Stringer); ok { - if name := s.String(); name != "" { - return &tarArtifactNamed{ta, name} - } - } - return &ta -} - -// Kind returns the hardcoded [Kind] constant. -func (a *tarArtifact) Kind() Kind { return KindTar } - -// Params is a noop. -func (a *tarArtifact) Params(*IContext) {} - -func init() { - register(KindTar, func(r *IRReader) Artifact { - a := NewTar(r.Next()) - if _, ok := r.Finalise(); ok { - panic(ErrUnexpectedChecksum) - } - return a - }) -} - -// Inputs returns a slice containing the backing file. -func (a *tarArtifact) Inputs() []Artifact { - return []Artifact{a.f} -} - -// IsExclusive returns false: decompressor and tar reader are fully sequential. -func (a *tarArtifact) IsExclusive() bool { return false } - -// A DisallowedTypeflagError describes a disallowed typeflag encountered while -// unpacking a tarball. -type DisallowedTypeflagError byte - -func (e DisallowedTypeflagError) Error() string { - return "disallowed typeflag '" + string(e) + "'" -} - -// Cure cures the [Artifact], producing a directory located at work. -func (a *tarArtifact) Cure(t *TContext) (err error) { - var r io.ReadCloser - if r, err = t.Open(a.f); err != nil { - return - } - - defer func() { - closeErr := r.Close() - if err == nil { - err = closeErr - } - }() - - type dirTargetPerm struct { - path string - mode fs.FileMode - } - var madeDirectories []dirTargetPerm - - if err = os.MkdirAll(t.GetTempDir().String(), 0700); err != nil { - return - } - var root *os.Root - if root, err = os.OpenRoot(t.GetTempDir().String()); err != nil { - return - } - defer func() { - closeErr := root.Close() - if err == nil { - err = closeErr - } - }() - - var header *tar.Header - tr := tar.NewReader(r) - for header, err = tr.Next(); err == nil; header, err = tr.Next() { - typeflag := header.Typeflag - if typeflag == 0 { - if len(header.Name) > 0 && header.Name[len(header.Name)-1] == '/' { - typeflag = tar.TypeDir - } else { - typeflag = tar.TypeReg - } - } - - if typeflag >= '0' && typeflag <= '9' && typeflag != tar.TypeDir { - if err = root.MkdirAll(filepath.Dir(header.Name), 0700); err != nil { - return - } - } - - switch typeflag { - case tar.TypeReg: - var f *os.File - if f, err = root.OpenFile( - header.Name, - os.O_CREATE|os.O_EXCL|os.O_WRONLY, - header.FileInfo().Mode()&0500, - ); err != nil { - return - } - if _, err = io.Copy(f, tr); err != nil { - _ = f.Close() - return - } else if err = f.Close(); err != nil { - return - } - break - - case tar.TypeLink: - if err = root.Link( - header.Linkname, - header.Name, - ); err != nil { - return - } - break - - case tar.TypeSymlink: - if err = root.Symlink( - header.Linkname, - header.Name, - ); err != nil { - return - } - break - - case tar.TypeDir: - madeDirectories = append(madeDirectories, dirTargetPerm{ - path: header.Name, - mode: header.FileInfo().Mode(), - }) - if err = root.MkdirAll(header.Name, 0700); err != nil { - return - } - break - - case tar.TypeChar: - t.GetMessage().Verbosef( - "%sskipping character device %d, %d %q%s", - t.cache.sgrWarn, header.Devmajor, header.Devminor, header.Name, t.cache.sgrRes, - ) - continue - - case tar.TypeXGlobalHeader: - continue // ignore - - default: - return DisallowedTypeflagError(typeflag) - } - } - if errors.Is(err, io.EOF) { - err = nil - } - if err == nil { - for _, e := range madeDirectories { - if err = root.Chmod(e.path, e.mode&0500); err != nil { - return - } - } - } else { - return - } - - temp := t.GetTempDir() - if err = os.Chmod(temp.String(), 0700); err != nil { - return - } - - var entries []os.DirEntry - if entries, err = os.ReadDir(temp.String()); err != nil { - return - } - - if len(entries) == 1 && entries[0].IsDir() { - p := temp.Append(entries[0].Name()) - if err = os.Chmod(p.String(), 0700); err != nil { - return - } - err = os.Rename(p.String(), t.GetWorkDir().String()) - } else { - err = os.Rename(temp.String(), t.GetWorkDir().String()) - } - return -} - -// CuresExempt exempts the cheap [KindTar] implementation often at the end of a -// [FileArtifact] pipeline. -func (*tarArtifact) CuresExempt() {} diff --git a/internal/pkg/tar_test.go b/internal/pkg/tar_test.go deleted file mode 100644 index 1f280e9c..00000000 --- a/internal/pkg/tar_test.go +++ /dev/null @@ -1,225 +0,0 @@ -package pkg_test - -import ( - "archive/tar" - "bytes" - "crypto/sha512" - "errors" - "io/fs" - "net/http" - "os" - "testing" - "testing/fstest" - - "hakurei.app/check" - "hakurei.app/internal/pkg" - "hakurei.app/internal/stub" -) - -func TestTar(t *testing.T) { - t.Parallel() - - want := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "checksum": {Mode: fs.ModeDir | 0500}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP": {Mode: fs.ModeDir | 0500}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check": {Mode: 0400, Data: []byte{0, 0}}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib": {Mode: fs.ModeDir | 0500}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig": {Mode: fs.ModeDir | 0500}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - - "identifier": {Mode: fs.ModeDir | 0500}, - "identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, - "identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, - - "work": {Mode: fs.ModeDir | 0500}, - } - wantEncode := pkg.Encode(want.hash()) - - wantExpand := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - } - wantExpandEncode := pkg.Encode(wantExpand.hash()) - - checkWithCache(t, []cacheTestCase{ - {"http", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - checkTarHTTP(t, base, c, fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP": {Mode: fs.ModeDir | 0700}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check": {Mode: 0400, Data: []byte{0, 0}}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib": {Mode: fs.ModeDir | 0700}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig": {Mode: fs.ModeDir | 0700}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, - "identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, - - "work": {Mode: fs.ModeDir | 0700}, - }, want) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantEncode + "/checksum": {Mode: fs.ModeDir | 0500}, - "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP": {Mode: fs.ModeDir | 0500}, - "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check": {Mode: 0400, Data: []byte{0, 0}}, - "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib": {Mode: fs.ModeDir | 0500}, - "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig": {Mode: fs.ModeDir | 0500}, - "checksum/" + wantEncode + "/identifier": {Mode: fs.ModeDir | 0500}, - "checksum/" + wantEncode + "/identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, - "checksum/" + wantEncode + "/identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, - "checksum/" + wantEncode + "/work": {Mode: fs.ModeDir | 0500}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/snWp53xxNdx6cV5KSJqzyVWzLlTsFd8udOisIu8Rgxly09mGcxufYCoQ5CQrHDyZ": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantEncode)}, - "identifier/v9DUj2R4YK_3ae0m9VuUYI-HJysDDCENu1k10thxioycRzFH1Qejw5bBGukm4IdL": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantEncode)}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"http expand", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - checkTarHTTP(t, base, c, fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "lib": {Mode: fs.ModeDir | 0700}, - "lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - }, wantExpand) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantExpandEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantExpandEncode + "/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/1SuvgGltcsgqps9tWqjULG8RCdpx_6KVWXdsMd5xPKvdtysOxqpE-3bLu4j0WpGk": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantExpandEncode)}, - "identifier/snWp53xxNdx6cV5KSJqzyVWzLlTsFd8udOisIu8Rgxly09mGcxufYCoQ5CQrHDyZ": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantExpandEncode)}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} - -func checkTarHTTP( - t *testing.T, - base *check.Absolute, - c *pkg.Cache, - testdataFsys fs.FS, - want expectsKnown, -) { - var testdata string - { - var buf bytes.Buffer - w := tar.NewWriter(&buf) - if err := w.AddFS(testdataFsys); err != nil { - t.Fatalf("AddFS: error = %v", err) - } - if err := w.Close(); err != nil { - t.Fatalf("Close: error = %v", err) - } - testdata = buf.String() - } - - testdataChecksum := func() pkg.Checksum { - h := sha512.New384() - h.Write([]byte(testdata)) - return (pkg.Checksum)(h.Sum(nil)) - }() - - var transport http.Transport - client := http.Client{Transport: &transport} - transport.RegisterProtocol("file", http.NewFileTransportFS(fstest.MapFS{ - "testdata": {Data: []byte(testdata), Mode: 0400}, - })) - - tarDir := stubArtifact{ - kind: pkg.KindExec, - params: []byte("directory containing a single regular file"), - cure: func(t *pkg.TContext) error { - work := t.GetWorkDir() - if err := os.MkdirAll(work.String(), 0700); err != nil { - return err - } - return os.WriteFile( - work.Append("sample.tar.gz").String(), - []byte(testdata), - 0400, - ) - }, - } - tarDirMulti := stubArtifact{ - kind: pkg.KindExec, - params: []byte("directory containing a multiple entries"), - cure: func(t *pkg.TContext) error { - work := t.GetWorkDir() - if err := os.MkdirAll(work.Append( - "garbage", - ).String(), 0700); err != nil { - return err - } - return os.WriteFile( - work.Append("sample.tar.gz").String(), - []byte(testdata), - 0400, - ) - }, - } - tarDirType := stubArtifact{ - kind: pkg.KindExec, - params: []byte("directory containing a symbolic link"), - cure: func(t *pkg.TContext) error { - work := t.GetWorkDir() - if err := os.MkdirAll(work.String(), 0700); err != nil { - return err - } - return os.Symlink( - work.String(), - work.Append("sample.tar.gz").String(), - ) - }, - } - // destroy these to avoid including it in flatten test case - defer newDestroyArtifactFunc(&tarDir)(t, base, c) - defer newDestroyArtifactFunc(&tarDirMulti)(t, base, c) - defer newDestroyArtifactFunc(&tarDirType)(t, base, c) - - cureMany(t, c, []cureStep{ - {"file", pkg.NewTar(pkg.NewHTTPGet( - &client, - "file:///testdata", - testdataChecksum, - )), ignorePathname, want, pkg.WNew, nil}, - - {"directory", pkg.NewTar(&tarDir), ignorePathname, want, pkg.WNew, nil}, - - {"multiple entries", pkg.NewTar(&tarDirMulti), nil, nil, pkg.WNew, errors.New( - "input directory does not contain a single regular file", - )}, - - {"bad type", pkg.NewTar(&tarDirType), nil, nil, pkg.WNew, errors.New( - "input directory does not contain a single regular file", - )}, - - {"error passthrough", pkg.NewTar(&stubArtifact{ - kind: pkg.KindExec, - params: []byte("doomed artifact"), - cure: func(t *pkg.TContext) error { - return stub.UniqueError(0xcafe) - }, - }), nil, nil, pkg.WNew, stub.UniqueError(0xcafe)}, - }) -} diff --git a/internal/rosa/builtins.go b/internal/rosa/builtins.go index dbafa397..edca2333 100644 --- a/internal/rosa/builtins.go +++ b/internal/rosa/builtins.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) const ( diff --git a/internal/rosa/etc.go b/internal/rosa/etc.go index 18c9750b..0af51830 100644 --- a/internal/rosa/etc.go +++ b/internal/rosa/etc.go @@ -8,7 +8,7 @@ import ( "syscall" "hakurei.app/fhs" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) // cureEtc contains deterministic elements of /etc, made available as part of diff --git a/internal/rosa/gentoo.go b/internal/rosa/gentoo.go index a0a8e525..a08fd705 100644 --- a/internal/rosa/gentoo.go +++ b/internal/rosa/gentoo.go @@ -6,7 +6,7 @@ import ( "os" "path/filepath" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) // gentooOverlay are symlinks on top of a Gentoo LLVM stage3 tarball for diff --git a/internal/rosa/git.go b/internal/rosa/git.go index bda380ce..70d5b6f8 100644 --- a/internal/rosa/git.go +++ b/internal/rosa/git.go @@ -4,7 +4,7 @@ import ( "path" "strings" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) var ( diff --git a/internal/rosa/go.go b/internal/rosa/go.go index 1b42447a..1127cf3d 100644 --- a/internal/rosa/go.go +++ b/internal/rosa/go.go @@ -3,7 +3,7 @@ package rosa import ( "slices" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) // newGo returns a specific version of the Go toolchain. diff --git a/internal/rosa/llvm.go b/internal/rosa/llvm.go index dddd6388..0882e7d8 100644 --- a/internal/rosa/llvm.go +++ b/internal/rosa/llvm.go @@ -5,7 +5,7 @@ import ( "slices" "strings" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) var ( diff --git a/internal/rosa/llvm_test.go b/internal/rosa/llvm_test.go index 52562579..3152107c 100644 --- a/internal/rosa/llvm_test.go +++ b/internal/rosa/llvm_test.go @@ -3,8 +3,8 @@ package rosa_test import ( "testing" - "hakurei.app/internal/pkg" "hakurei.app/internal/rosa" + "hakurei.app/pkg" ) func TestLLVMInputs(t *testing.T) { diff --git a/internal/rosa/mirror.go b/internal/rosa/mirror.go index 80d4509c..4af931c7 100644 --- a/internal/rosa/mirror.go +++ b/internal/rosa/mirror.go @@ -17,8 +17,8 @@ import ( "syscall" "unique" - "hakurei.app/internal/pkg" "hakurei.app/message" + "hakurei.app/pkg" ) // Remote is an authenticated cache mirror. diff --git a/internal/rosa/mirror_test.go b/internal/rosa/mirror_test.go index 6c3ec92e..d45f9ae8 100644 --- a/internal/rosa/mirror_test.go +++ b/internal/rosa/mirror_test.go @@ -13,9 +13,9 @@ import ( "hakurei.app/check" "hakurei.app/fhs" - "hakurei.app/internal/pkg" "hakurei.app/internal/rosa" "hakurei.app/message" + "hakurei.app/pkg" ) func TestMirror(t *testing.T) { diff --git a/internal/rosa/report.go b/internal/rosa/report.go index 7792d9e9..7a1f4521 100644 --- a/internal/rosa/report.go +++ b/internal/rosa/report.go @@ -14,8 +14,8 @@ import ( "unique" "unsafe" - "hakurei.app/internal/pkg" "hakurei.app/message" + "hakurei.app/pkg" ) // wordSize is the boundary which binary segments are always aligned to. diff --git a/internal/rosa/report_test.go b/internal/rosa/report_test.go index 2ca8ac57..8154bc5a 100644 --- a/internal/rosa/report_test.go +++ b/internal/rosa/report_test.go @@ -8,8 +8,8 @@ import ( "testing" "unique" - "hakurei.app/internal/pkg" "hakurei.app/internal/rosa" + "hakurei.app/pkg" ) func TestReportZeroLength(t *testing.T) { diff --git a/internal/rosa/rosa.go b/internal/rosa/rosa.go index 7b06700c..1ad15100 100644 --- a/internal/rosa/rosa.go +++ b/internal/rosa/rosa.go @@ -10,7 +10,7 @@ import ( "unsafe" "hakurei.app/fhs" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) // Extension is the variant identification string of custom artifact diff --git a/internal/rosa/rosa_test.go b/internal/rosa/rosa_test.go index 76e8a53e..ce3dad42 100644 --- a/internal/rosa/rosa_test.go +++ b/internal/rosa/rosa_test.go @@ -10,9 +10,9 @@ import ( "testing" "hakurei.app/check" - "hakurei.app/internal/pkg" "hakurei.app/internal/rosa" "hakurei.app/message" + "hakurei.app/pkg" ) var ( diff --git a/internal/rosa/state.go b/internal/rosa/state.go index ba17ea23..fca1aa48 100644 --- a/internal/rosa/state.go +++ b/internal/rosa/state.go @@ -25,8 +25,8 @@ import ( "hakurei.app/check" "hakurei.app/container" - "hakurei.app/internal/pkg" "hakurei.app/internal/rosa/azalea" + "hakurei.app/pkg" ) // ArtifactH is a handle of the unique name of a prepared [pkg.Artifact]. diff --git a/internal/uevent/coldboot_test.go b/internal/uevent/coldboot_test.go index 0a50ead4..aa6428a1 100644 --- a/internal/uevent/coldboot_test.go +++ b/internal/uevent/coldboot_test.go @@ -11,8 +11,8 @@ import ( "testing" "hakurei.app/check" - "hakurei.app/internal/pkg" "hakurei.app/internal/uevent" + "hakurei.app/pkg" ) func TestColdboot(t *testing.T) { |