diff options
Diffstat (limited to 'internal')
174 files changed, 1477 insertions, 11177 deletions
diff --git a/internal/outcome/process.go b/internal/outcome/process.go index 614b22fc..599af8f2 100644 --- a/internal/outcome/process.go +++ b/internal/outcome/process.go @@ -303,7 +303,7 @@ func (k *outcome) main(msg message.Msg, identifierFd int) { ) for eh := range entries { var et hst.Enablements - if et, err = eh.Load(nil); err != nil { + if et, err = eh.Load(nil, syscall.Kill); err != nil { perror(err, "read state header of instance "+eh.ID.String()) } else { rt |= et diff --git a/internal/pkg/archive.go b/internal/pkg/archive.go deleted file mode 100644 index ea4017d9..00000000 --- a/internal/pkg/archive.go +++ /dev/null @@ -1,411 +0,0 @@ -package pkg - -import ( - "crypto/sha512" - "encoding/binary" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "unsafe" - - "hakurei.app/check" -) - -/* -| mode uint32 | path_sz uint32 | -| data_sz uint64 | -| path string | -| data []byte | -*/ - -// An ArchiveHeader represents a single header in an archive. -type ArchiveHeader struct { - Mode fs.FileMode // file mode bits - Path string // pathname of the file - Size uint64 // size of data segment -} - -// Writer implements sequential writing of an archive. [Writer.WriteHeader] -// begins a new file with the provided [ArchiveHeader], and then Writer can be -// treated as an [io.Writer] to supply that file's data. -// -// It is the caller's responsibility to write entries in lexical order. -type Writer struct { - // Underlying writer. - w io.Writer - // Current header. - h ArchiveHeader - // Fixed-size header segment. - buf [wordSize * 2]byte - // Current position in data segment. - n uint64 -} - -// NewWriter returns the address of a new [Writer] writing to w. -func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } - -var zero [wordSize]byte - -// padSize returns the padding size for aligning sz. -func padSize[T int | uint64](sz T) T { - return (wordSize - (sz)%wordSize) % wordSize -} - -// flush concludes writing to the current file and writes padding. -func (aw *Writer) flush() error { - if aw.h.Size > aw.n { - return fmt.Errorf("missed writing %d bytes", aw.h.Size-aw.n) - } else if aw.h.Size < aw.n { - return fmt.Errorf("wrote %d bytes beyond end of file", aw.n-aw.h.Size) - } - - if psz := padSize(aw.h.Size); psz != 0 { - if _, err := aw.w.Write(zero[:psz]); err != nil { - return err - } - } - - aw.n = 0 - return nil -} - -// WriteHeader writes h and begins accepting its corresponding file. -func (aw *Writer) WriteHeader(h *ArchiveHeader) error { - if err := aw.flush(); err != nil { - return err - } - - aw.h = *h - binary.LittleEndian.PutUint32(aw.buf[:], uint32(aw.h.Mode)) - binary.LittleEndian.PutUint32(aw.buf[wordSize/2:], uint32(len(aw.h.Path))) - binary.LittleEndian.PutUint64(aw.buf[wordSize:], aw.h.Size) - if _, err := aw.w.Write(aw.buf[:]); err != nil { - return err - } else if _, err = aw.w.Write( - unsafe.Slice(unsafe.StringData(aw.h.Path), len(aw.h.Path)), - ); err != nil { - return err - } else if psz := padSize(len(aw.h.Path)); psz != 0 { - if _, err = aw.w.Write(zero[:psz]); err != nil { - return err - } - } - return nil -} - -// Write writes p to the underlying writer and records the new position. Invalid -// positions are reported by WriteHeader and Close. -func (aw *Writer) Write(p []byte) (n int, err error) { - n, err = aw.w.Write(p) - aw.n += uint64(n) - return -} - -// Close concludes writing to the archive stream. -func (aw *Writer) Close() (err error) { - err = aw.flush() - aw.w = nil - return -} - -// ErrInsecurePath is returned by [FlatEntry.Decode] if validation is requested -// and a nonlocal path is encountered in the stream. -var ErrInsecurePath = errors.New("insecure file path") - -// Reader implements sequential reading of an archive. [Reader.Next] advances to -// the next file in the archive (including the first), and then Reader can be -// treated as an [io.Reader] to access the file's data. -type Reader struct { - // Underlying reader. - r io.Reader - // Fixed-size header segment. - buf [wordSize * 2]byte - // Remaining bytes in current data segment. - n, pad uint64 -} - -// NewReader returns the address of a new [Reader] reading from r. -func NewReader(r io.Reader) *Reader { return &Reader{r: r} } - -// Next advances ar to the next entry. Remaining bytes of the current data -// segment are discarded. Advancing beyond the final entry returns [io.EOF]. -func (ar *Reader) Next() (*ArchiveHeader, error) { - if dsz := int64(ar.n + ar.pad); dsz > 0 { - if n, err := io.CopyN(io.Discard, ar.r, dsz); err != nil { - if errors.Is(err, io.EOF) && n != dsz { - err = io.ErrUnexpectedEOF - } - return nil, err - } - } - - if _, err := io.ReadFull(ar.r, ar.buf[:]); err != nil { - return nil, err - } - - h := ArchiveHeader{ - Mode: fs.FileMode(binary.LittleEndian.Uint32(ar.buf[:])), - Size: binary.LittleEndian.Uint64(ar.buf[wordSize:]), - } - pathSize := int(binary.LittleEndian.Uint32(ar.buf[wordSize/2:])) - pPathSize := alignSize(pathSize) - - buf := make([]byte, pPathSize) - if _, err := io.ReadFull(ar.r, buf); err != nil { - if errors.Is(err, io.EOF) { - err = io.ErrUnexpectedEOF - } - return nil, err - } - - h.Path = unsafe.String(unsafe.SliceData(buf), pathSize) - if !filepath.IsLocal(h.Path) { - return &h, ErrInsecurePath - } - - ar.n = h.Size - ar.pad = padSize(h.Size) - return &h, nil -} - -// Read implements [io.Reader] for the data segment of the current entry. -func (ar *Reader) Read(p []byte) (n int, err error) { - if uint64(len(p)) > ar.n { - p = p[:ar.n] - } - - if len(p) > 0 { - n, err = ar.r.Read(p) - ar.n -= uint64(n) - } - - switch err { - case io.EOF: - if ar.n > 0 { - return n, io.ErrUnexpectedEOF - } - - case nil: - if ar.n == 0 { - return n, io.EOF - } - } - return -} - -// Write writes a deterministic representation of the contents of fsys to w. -// The resulting data can be hashed to produce a deterministic checksum for the -// directory. -func Write(fsys fs.FS, root string, w io.Writer) error { - aw := NewWriter(w) - if err := fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - var fi fs.FileInfo - fi, err = d.Info() - if err != nil { - return err - } - - h := ArchiveHeader{ - Path: path, - Mode: fi.Mode(), - } - if h.Mode.IsRegular() { - h.Size = uint64(fi.Size()) - if err = aw.WriteHeader(&h); err != nil { - return err - } - - var r fs.File - r, err = fsys.Open(path) - if err != nil { - return err - } - _, err = io.Copy(aw, r) - if _err := r.Close(); err == nil { - err = _err - } - return err - } else if h.Mode&fs.ModeSymlink != 0 { - var newpath string - if newpath, err = fs.ReadLink(fsys, path); err != nil { - return err - } - - h.Size = uint64(len(newpath)) - if err = aw.WriteHeader(&h); err != nil { - return err - } - - _, err = aw.Write(unsafe.Slice(unsafe.StringData(newpath), len(newpath))) - return err - } else if !h.Mode.IsDir() { - return InvalidFileModeError(h.Mode) - } - return aw.WriteHeader(&h) - }); err != nil { - return err - } - return aw.Close() -} - -// SumFS saves checksum of the archive of fsys to the value pointed to by buf. -func SumFS(buf *Checksum, fsys fs.FS, root string) error { - h := sha512.New384() - if err := Write(fsys, root, h); err != nil { - return err - } - h.Sum(buf[:0]) - return nil -} - -// SumDir saves checksum of the archive of directory at pathname to the value -// pointed to by buf. -func SumDir(buf *Checksum, pathname *check.Absolute) error { - return SumFS(buf, os.DirFS(pathname.String()), ".") -} - -// archiveArtifact is an [Artifact] unpacking an archive supported by [Reader] -// backed by a [FileArtifact]. -type archiveArtifact struct { - // Caller-supplied backing archive. - f Artifact -} - -var _ CuresExempt = archiveArtifact{} - -// NewArchive returns a new [Artifact] backed by the supplied [Artifact]. The -// source [Artifact] must be a [FileArtifact] and produce a stream compatible -// with [Reader]. -func NewArchive(a Artifact) Artifact { - return archiveArtifact{a} -} - -// Kind returns the hardcoded [Kind] constant. -func (archiveArtifact) Kind() Kind { return KindArchive } - -// Params is a noop. -func (archiveArtifact) Params(*IContext) {} - -func init() { - register(KindArchive, func(r *IRReader) Artifact { - a := NewArchive(r.Next()) - if _, ok := r.Finalise(); ok { - panic(ErrUnexpectedChecksum) - } - return a - }) -} - -// Inputs returns a slice containing the backing file. -func (a archiveArtifact) Inputs() []Artifact { - return []Artifact{a.f} -} - -// IsExclusive returns false: [Reader] is fully sequential. -func (archiveArtifact) IsExclusive() bool { return false } - -// Cure cures the [Artifact], producing a directory located at work. -func (a archiveArtifact) Cure(t *TContext) (err error) { - var r io.ReadCloser - if r, err = t.Open(a.f); err != nil { - return - } - - defer func() { - closeErr := r.Close() - if err == nil { - err = closeErr - } - }() - - type dirTargetPerm struct { - path string - mode fs.FileMode - } - var madeDirectories []dirTargetPerm - - if err = os.MkdirAll(t.GetWorkDir().String(), 0700); err != nil { - return - } - var root *os.Root - if root, err = os.OpenRoot(t.GetWorkDir().String()); err != nil { - return - } - defer func() { - closeErr := root.Close() - if err == nil { - err = closeErr - } - }() - - var header *ArchiveHeader - ar := NewReader(r) - for header, err = ar.Next(); err == nil; header, err = ar.Next() { - if header.Mode.IsRegular() { - var f *os.File - if f, err = root.OpenFile( - header.Path, - os.O_CREATE|os.O_EXCL|os.O_WRONLY, - header.Mode.Perm(), - ); err != nil { - return - } - if _, err = io.Copy(f, ar); err != nil { - _ = f.Close() - return - } else if err = f.Close(); err != nil { - return - } - } else if header.Mode&fs.ModeSymlink != 0 { - var p []byte - if p, err = io.ReadAll(ar); err != nil { - return - } - - if err = root.Symlink( - unsafe.String(unsafe.SliceData(p), len(p)), - header.Path, - ); err != nil { - return - } - } else if header.Mode.IsDir() { - if header.Path == "." { - continue - } - - madeDirectories = append(madeDirectories, dirTargetPerm{ - path: header.Path, - mode: header.Mode, - }) - if err = root.Mkdir(header.Path, 0700); err != nil { - return - } - } else { - return InvalidFileModeError(header.Mode) - } - } - if errors.Is(err, io.EOF) { - err = nil - } - if err == nil { - for _, e := range madeDirectories { - if err = root.Chmod(e.path, e.mode.Perm()); err != nil { - return - } - } - } else { - return - } - return -} - -// CuresExempt exempts the cheap [KindArchive] implementation often found at -// the end of a [FileArtifact] pipeline. -func (archiveArtifact) CuresExempt() {} diff --git a/internal/pkg/archive_test.go b/internal/pkg/archive_test.go deleted file mode 100644 index 1cb4bea9..00000000 --- a/internal/pkg/archive_test.go +++ /dev/null @@ -1,240 +0,0 @@ -package pkg_test - -import ( - "bytes" - "io" - "io/fs" - "maps" - "reflect" - "testing" - "testing/fstest" - "unsafe" - - "hakurei.app/check" - "hakurei.app/internal/pkg" -) - -func TestArchive(t *testing.T) { - t.Parallel() - - type entry struct { - path string - mode fs.FileMode - data string - } - testCases := []struct { - name string - fsys fs.FS - entries []entry - sum pkg.Checksum - err error - }{ - {"bad type", fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0700}, - "invalid": {Mode: fs.ModeCharDevice | 0400}, - }, nil, pkg.Checksum{}, pkg.InvalidFileModeError( - fs.ModeCharDevice | 0400, - )}, - - {"coldboot", fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "devices": {Mode: fs.ModeDir | 0700}, - "devices/uevent": {Mode: 0600, Data: []byte("add")}, - "devices/empty": {Mode: fs.ModeDir | 0700}, - - "devices/sub": {Mode: fs.ModeDir | 0700}, - "devices/sub/uevent": {Mode: 0600, Data: []byte("add")}, - - "block": {Mode: fs.ModeDir | 0700}, - "block/uevent": {Mode: 0600}, - }, []entry{ - {".", fs.ModeDir | 0700, ""}, - - {"block", fs.ModeDir | 0700, ""}, - {"block/uevent", 0600, ""}, - - {"devices", fs.ModeDir | 0700, ""}, - {"devices/empty", fs.ModeDir | 0700, ""}, - {"devices/sub", fs.ModeDir | 0700, ""}, - {"devices/sub/uevent", 0600, "add"}, - {"devices/uevent", 0600, "add"}, - }, pkg.MustDecode("mEy_Lf5KotThm7OwMx7yTKZh5HCCyaB41pVAvI9uDMgVQFM91iosBLYsRm8bDsX8"), nil}, - - {"empty", fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, []entry{ - {".", fs.ModeDir | 0700, ""}, - {"checksum", fs.ModeDir | 0700, ""}, - {"identifier", fs.ModeDir | 0700, ""}, - {"work", fs.ModeDir | 0700, ""}, - }, pkg.MustDecode("E4vEZKhCcL2gPZ2Tt59FS3lDng-d_2SKa2i5G_RbDfwGn6EemptFaGLPUDiOa94C"), nil}, - - {"sample directory step garbage", fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "lib": {Mode: fs.ModeDir | 0500}, - "lib/check": {Mode: 0400}, - - "lib/pkgconfig": {Mode: fs.ModeDir | 0500}, - }, []entry{ - {".", fs.ModeDir | 0500, ""}, - - {"lib", fs.ModeDir | 0500, ""}, - {"lib/check", 0400, ""}, - - {"lib/pkgconfig", fs.ModeDir | 0500, ""}, - }, pkg.MustDecode("CUx-3hSbTWPsbMfDhgalG4Ni_GmR9TnVX8F99tY_P5GtkYvczg9RrF5zO0jX9XYT"), nil}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - t.Run("roundtrip", func(t *testing.T) { - t.Parallel() - - var buf bytes.Buffer - if err := pkg.Write( - tc.fsys, - ".", - &buf, - ); !reflect.DeepEqual(err, tc.err) { - t.Fatalf("Flatten: error = %v, want %v", err, tc.err) - } else if tc.err != nil { - return - } - - r := pkg.NewReader(bytes.NewReader(buf.Bytes())) - var got []entry - for { - h, err := r.Next() - if err != nil { - if err == io.EOF { - break - } - t.Fatalf("Next: error = %v", err) - } - - var data []byte - if data, err = io.ReadAll(r); err != nil { - t.Fatalf("Read: error = %v", err) - } - - got = append(got, entry{ - path: h.Path, - mode: h.Mode, - data: unsafe.String(unsafe.SliceData(data), len(data)), - }) - } - - if !reflect.DeepEqual(got, tc.entries) { - t.Fatalf("Reader: %#v, want %#v", got, tc.entries) - } - }) - - if tc.err != nil { - return - } - - t.Run("hash", func(t *testing.T) { - t.Parallel() - - var got pkg.Checksum - if err := pkg.SumFS(&got, tc.fsys, "."); err != nil { - t.Fatalf("SumFS: error = %v", err) - } else if got != tc.sum { - t.Fatalf("SumFS: %v", &pkg.ChecksumMismatchError{ - Got: got, - Want: tc.sum, - }) - } - }) - }) - } -} - -var archiveTestdata = fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "devices": {Mode: fs.ModeDir | 0700}, - "devices/uevent": {Mode: 0600, Data: []byte("add")}, - "devices/empty": {Mode: fs.ModeDir | 0700}, - - "devices/sub": {Mode: fs.ModeDir | 0700}, - "devices/sub/uevent": {Mode: 0600, Data: []byte("add")}, - - "block": {Mode: fs.ModeDir | 0700}, - "block/uevent": {Mode: 0600}, -} - -func TestArchiveArtifact(t *testing.T) { - t.Parallel() - - want := maps.Clone(archiveTestdata) - want["."].Mode = fs.ModeDir | 0500 - - checkWithCache(t, []cacheTestCase{ - {"unpack", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - var buf bytes.Buffer - if err := pkg.Write(archiveTestdata, ".", &buf); err != nil { - t.Fatal(err) - } - - cureMany(t, c, []cureStep{ - {"sample", pkg.NewArchive( - pkg.NewFile("", buf.Bytes()), - ), ignorePathname, expectsFS(want), pkg.WNew, nil}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F": {Mode: fs.ModeDir | 0500}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/block": {Mode: fs.ModeDir | 0700}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/block/uevent": {Mode: 0600}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices": {Mode: fs.ModeDir | 0700}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices/empty": {Mode: fs.ModeDir | 0700}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices/sub": {Mode: fs.ModeDir | 0700}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices/sub/uevent": {Mode: 0600, Data: []byte("add")}, - "checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F/devices/uevent": {Mode: 0600, Data: []byte("add")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/-zXKfphyWM2Ko7VMVURaEMSuixNokFSe0xFbnwR2RtRtvMrcPGWalV0LIn45PXTY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/CBPcoVHuVUTVRCMbRl8J30RSSzm_tyfuXaZ-HlZsanY1sY50meOVmgaWDrGKbx9F")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} - -func BenchmarkArchiveRead(b *testing.B) { - var buf bytes.Buffer - if err := pkg.Write(archiveTestdata, ".", &buf); err != nil { - b.Fatal(err) - } - testdata := buf.Bytes() - - for b.Loop() { - r := pkg.NewReader(bytes.NewReader(testdata)) - for { - _, err := r.Next() - if err != nil { - if err == io.EOF { - break - } - b.Fatal(err) - } - } - } -} - -func BenchmarkArchiveWrite(b *testing.B) { - for b.Loop() { - if err := pkg.Write(archiveTestdata, ".", io.Discard); err != nil { - b.Fatal(err) - } - } -} diff --git a/internal/pkg/clean.go b/internal/pkg/clean.go deleted file mode 100644 index f07706cf..00000000 --- a/internal/pkg/clean.go +++ /dev/null @@ -1,161 +0,0 @@ -package pkg - -import ( - "errors" - "os" - "unique" -) - -// Clean destroys checksum backing entries without any identifier or substitute -// entry referring to it. If at least one keep [Artifact] is specified, -// identifier and substitute entries not kept alive by them are destroyed first. -func (c *Cache) Clean(dry, inputs bool, keep ...Artifact) ( - []unique.Handle[ID], - []unique.Handle[Checksum], - error, -) { - c.identMu.Lock() - defer c.identMu.Unlock() - - c.checksumMu.Lock() - defer c.checksumMu.Unlock() - - dents, err := os.ReadDir(c.base.Append(dirChecksum).String()) - if err != nil { - return nil, nil, err - } - checksums := make(map[unique.Handle[Checksum]]string, len(dents)) - var buf Checksum - for _, dent := range dents { - name := dent.Name() - if err = Decode(&buf, name); err != nil { - return nil, nil, err - } - checksums[unique.Make(buf)] = name - } - - type identPair struct { - id unique.Handle[ID] - name string - } - dents, err = os.ReadDir(c.base.Append(dirIdentifier).String()) - if err != nil { - return nil, nil, err - } - - keepIdents := make(map[unique.Handle[ID]]struct{}) - if inputs { - for _, id := range Inputs((*Collect)(&keep)) { - keepIdents[id] = struct{}{} - } - } else { - for _, a := range keep { - keepIdents[c.Ident(a)] = struct{}{} - } - } - - idents := make([]identPair, 0, len(dents)) - for _, dent := range dents { - name := dent.Name() - if err = Decode(&buf, name); err != nil { - return nil, nil, err - } - id := unique.Make(ID(buf)) - - if _, ok := keepIdents[id]; len(keep) == 0 || ok { - if err = readlinkChecksum(c.base.Append( - dirIdentifier, - name, - ), &buf); err != nil { - return nil, nil, err - } - delete(checksums, unique.Make(buf)) - continue - } - - c.msg.Verbosef( - "arranging for destruction of %s%s%s...", - c.sgrIdent, name, c.sgrRes, - ) - idents = append(idents, identPair{id, name}) - } - - destroyedIdents := make([]unique.Handle[ID], 0, len(idents)) - for _, pair := range idents { - if !dry { - if err = os.Remove(c.base.Append( - dirStatus, - pair.name, - ).String()); err != nil && !errors.Is(err, os.ErrNotExist) { - return destroyedIdents, nil, err - } - - if err = os.Remove(c.base.Append( - dirIdentifier, - pair.name, - ).String()); err != nil { - return destroyedIdents, nil, err - } - } - destroyedIdents = append(destroyedIdents, pair.id) - } - - destroyedChecksums := make([]unique.Handle[Checksum], 0, len(checksums)) - for checksum, name := range checksums { - if err = c.parent.Err(); err != nil { - return destroyedIdents, destroyedChecksums, err - } - c.msg.Verbosef( - "destroying checksum %s%s%s...", - c.sgrIdent, name, c.sgrRes, - ) - if !dry { - if err = errors.Join(removeAll(c.base.Append( - dirChecksum, - name, - ))); err != nil { - return destroyedIdents, destroyedChecksums, err - } - } - destroyedChecksums = append(destroyedChecksums, checksum) - } - - dents, err = os.ReadDir(c.base.Append(dirSubstitute).String()) - if err != nil { - return destroyedIdents, destroyedChecksums, err - } - for _, dent := range dents { - name := dent.Name() - if err = readlinkChecksum(c.base.Append( - dirSubstitute, - name, - ), &buf); err != nil { - return destroyedIdents, destroyedChecksums, err - } - if _, ok := checksums[unique.Make(buf)]; !ok { - continue - } - - c.msg.Verbosef( - "destroying substitute %s%s%s...", - c.sgrIdent, name, c.sgrRes, - ) - if !dry { - if err = os.Remove(c.base.Append( - dirStatus, - name, - ).String()); err != nil && !errors.Is(err, os.ErrNotExist) { - return destroyedIdents, nil, err - } - - if err = os.Remove(c.base.Append( - dirSubstitute, - name, - ).String()); err != nil { - return destroyedIdents, destroyedChecksums, err - } - } - } - - return destroyedIdents, destroyedChecksums, nil -} diff --git a/internal/pkg/clean_test.go b/internal/pkg/clean_test.go deleted file mode 100644 index 67dceb09..00000000 --- a/internal/pkg/clean_test.go +++ /dev/null @@ -1,293 +0,0 @@ -package pkg_test - -import ( - "bytes" - "crypto/sha512" - "io/fs" - "log" - "os" - "slices" - "strings" - "testing" - "unique" - - "hakurei.app/internal/pkg" - "hakurei.app/message" -) - -// formatHandles returns a user-facing string representing h. -func formatHandles[T pkg.ID | pkg.Checksum](handles ...unique.Handle[T]) string { - var buf strings.Builder - for _, h := range handles { - buf.WriteString(pkg.Encode(pkg.Checksum(h.Value()))) - buf.WriteString(", ") - } - return strings.TrimSuffix(buf.String(), ", ") -} - -func TestClean(t *testing.T) { - t.Parallel() - ic := pkg.NewIR() - - testCases := []struct { - name string - a []pkg.Artifact - keep []pkg.Artifact - inputs bool - want expectsFS - - wantIdents []unique.Handle[pkg.ID] - wantChecksums []unique.Handle[pkg.Checksum] - }{ - {"simple", []pkg.Artifact{ - pkg.NewFile("file", nil), - }, nil, false, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/h6qx7m0vsyhO6oLEJgY2rKa1c6RZuIZ4QXxZSh__NZJ9VU0MIj5bFB-KuaPcsfpE": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")}, - - "lock": {Mode: 0644}, - "variant": {Mode: 0400}, - "status": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "fault": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, nil, nil}, - - {"keep", []pkg.Artifact{ - pkg.NewFile("removed-file", []byte("removed file")), - }, []pkg.Artifact{ - pkg.NewFile("file", []byte("\xfd")), - }, false, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/KgZ-FjbGuU-XP2QEHInpgv-2Zn0cTH5NqFMgTU0XrSdKmSwyC-3baVs1BMCP5spk": {Mode: 0400, Data: []byte("\xfd")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/tkrOYhQfTRaArk4nNYNjTmHQUmuJMVIoDeAKIQBm3I4uuMimBKXKMIY2gGZqwQg4": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/KgZ-FjbGuU-XP2QEHInpgv-2Zn0cTH5NqFMgTU0XrSdKmSwyC-3baVs1BMCP5spk")}, - - "lock": {Mode: 0644}, - "variant": {Mode: 0400}, - "status": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "fault": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, []unique.Handle[pkg.ID]{ - ic.Ident(pkg.NewFile("removed-file", []byte("removed file"))), - }, []unique.Handle[pkg.Checksum]{ - unique.Make(sha512.Sum384([]byte("removed file"))), - }}, - - {"inputs anchored substitute", []pkg.Artifact{ - &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("destroyed"), - deps: []pkg.Artifact{ - pkg.NewFile("destroyed-input", []byte("destroyed")), - }, - cure: func(f *pkg.FContext) error { - p := f.GetWorkDir() - if err := os.MkdirAll(p.String(), 0755); err != nil { - return err - } - return os.WriteFile(p.Append("result").String(), nil, 0444) - }, - }, - }, []pkg.Artifact{ - &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("kept"), - deps: []pkg.Artifact{ - pkg.NewFile("kept-input", []byte("kept")), - }, - cure: func(f *pkg.FContext) error { - p := f.GetWorkDir() - if err := os.MkdirAll(p.String(), 0755); err != nil { - return err - } - return os.WriteFile(p.Append("result").String(), nil, 0444) - }, - }, - }, true, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/H-eSiCo227-xdqyNl2R-5G3eqXPtbb8XegAB70I5OQb2majeZXJoCxTq9wJy5qqv": {Mode: 0400, Data: []byte("kept")}, - "checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE": {Mode: fs.ModeDir | 0500}, - "checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE/result": {Mode: 0444}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/KpxTUsUfGbsX6C6HU2j9cd9He3_tt2o2m2MgH4m0zial8lmoqSWY-Gc3KrS0k-UK": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/H-eSiCo227-xdqyNl2R-5G3eqXPtbb8XegAB70I5OQb2majeZXJoCxTq9wJy5qqv")}, - "identifier/xxQ43VMIS1WHMSEFnUdBNd6pulbtgiTorP_mtJpEsZENYhGBiDEL6Y2D8yqn0Sra": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")}, - - "lock": {Mode: 0644}, - "variant": {Mode: 0400}, - "status": {Mode: fs.ModeDir | 0700}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "substitute/dP-6_wIDRRouOaOF-nkBy-IaUbLdYHbOUTrptu7j_qfW01mnHSjYJ0oykUmvUd2x": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")}, - "substitute/td1tf1kb3z8iUFg2k1xlHeTTEz3xR4e77WSWdv2JhjmZyY6iuvodhdT8BblDOi0E": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")}, - - "fault": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, []unique.Handle[pkg.ID]{ - ic.Ident(pkg.NewFile("destroyed-input", []byte("destroyed"))), - ic.Ident(&stubArtifactF{ - kind: pkg.KindExec, - params: []byte("destroyed"), - deps: []pkg.Artifact{ - pkg.NewFile("destroyed-input", []byte("destroyed")), - }, - }), - }, []unique.Handle[pkg.Checksum]{ - unique.Make(sha512.Sum384([]byte("destroyed"))), - }}, - - {"inputs", []pkg.Artifact{ - &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("destroyed"), - deps: []pkg.Artifact{ - pkg.NewFile("destroyed-input", []byte("destroyed")), - }, - cure: func(f *pkg.FContext) error { - if w, err := f.GetStatusWriter(); err != nil { - return err - } else if _, err = w.Write([]byte("destroyed")); err != nil { - return err - } - - p := f.GetWorkDir() - if err := os.MkdirAll(p.String(), 0755); err != nil { - return err - } - return os.WriteFile(p.Append("result").String(), nil, 0444) - }, - }, - }, []pkg.Artifact{ - &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("kept"), - deps: []pkg.Artifact{ - pkg.NewFile("kept-input", []byte("kept")), - }, - cure: func(f *pkg.FContext) error { - if w, err := f.GetStatusWriter(); err != nil { - return err - } else if _, err = w.Write([]byte("kept")); err != nil { - return err - } - - p := f.GetWorkDir() - if err := os.MkdirAll(p.String(), 0755); err != nil { - return err - } - return os.WriteFile(p.Append("result").String(), []byte{0}, 0444) - }, - }, - }, true, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/CyDnDvF-LaeGPcSW70tPosNCoclByWkTjznUUF1DcgzlIwkN9yzz1ZFME1TlPj6W": {Mode: fs.ModeDir | 0500}, - "checksum/CyDnDvF-LaeGPcSW70tPosNCoclByWkTjznUUF1DcgzlIwkN9yzz1ZFME1TlPj6W/result": {Mode: 0444, Data: []byte("\x00")}, - "checksum/H-eSiCo227-xdqyNl2R-5G3eqXPtbb8XegAB70I5OQb2majeZXJoCxTq9wJy5qqv": {Mode: 0400, Data: []byte("kept")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/KpxTUsUfGbsX6C6HU2j9cd9He3_tt2o2m2MgH4m0zial8lmoqSWY-Gc3KrS0k-UK": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/H-eSiCo227-xdqyNl2R-5G3eqXPtbb8XegAB70I5OQb2majeZXJoCxTq9wJy5qqv")}, - "identifier/xxQ43VMIS1WHMSEFnUdBNd6pulbtgiTorP_mtJpEsZENYhGBiDEL6Y2D8yqn0Sra": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/CyDnDvF-LaeGPcSW70tPosNCoclByWkTjznUUF1DcgzlIwkN9yzz1ZFME1TlPj6W")}, - - "lock": {Mode: 0644}, - "variant": {Mode: 0400}, - - "status": {Mode: fs.ModeDir | 0700}, - "status/xxQ43VMIS1WHMSEFnUdBNd6pulbtgiTorP_mtJpEsZENYhGBiDEL6Y2D8yqn0Sra": {Mode: 0400, Data: []byte(statusHeader + "kept")}, - "status/td1tf1kb3z8iUFg2k1xlHeTTEz3xR4e77WSWdv2JhjmZyY6iuvodhdT8BblDOi0E": {Mode: 0400, Data: []byte(statusHeader + "kept")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "substitute/td1tf1kb3z8iUFg2k1xlHeTTEz3xR4e77WSWdv2JhjmZyY6iuvodhdT8BblDOi0E": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/CyDnDvF-LaeGPcSW70tPosNCoclByWkTjznUUF1DcgzlIwkN9yzz1ZFME1TlPj6W")}, - - "fault": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, []unique.Handle[pkg.ID]{ - ic.Ident(pkg.NewFile("destroyed-input", []byte("destroyed"))), - ic.Ident(&stubArtifactF{ - kind: pkg.KindExec, - params: []byte("destroyed"), - deps: []pkg.Artifact{ - pkg.NewFile("destroyed-input", []byte("destroyed")), - }, - }), - }, []unique.Handle[pkg.Checksum]{ - unique.Make(expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - "result": {Mode: 0444}, - }.hash()), - unique.Make(sha512.Sum384([]byte("destroyed"))), - }}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - base := makeBase(t) - msg := message.New(log.New(os.Stderr, "clean: ", 0)) - msg.SwapVerbose(testing.Verbose()) - c, err := pkg.Open(t.Context(), msg, base, nil) - if err != nil { - t.Fatal(err) - } - t.Cleanup(c.Close) - - all := pkg.Collect(slices.Concat(tc.a, tc.keep)) - if _, _, err = c.Cure(&all); !pkg.IsCollected(err) { - t.Fatal(err) - } - - var ( - idents []unique.Handle[pkg.ID] - checksums []unique.Handle[pkg.Checksum] - ) - idents, checksums, err = c.Clean(false, tc.inputs, tc.keep...) - if err != nil { - t.Fatalf("Clean: error = %v", err) - } - var buf [2]pkg.Checksum - - slices.SortFunc(idents, func(a, b unique.Handle[pkg.ID]) int { - buf[0], buf[1] = a.Value(), b.Value() - return bytes.Compare(buf[0][:], buf[1][:]) - }) - slices.SortFunc(checksums, func(a, b unique.Handle[pkg.Checksum]) int { - buf[0], buf[1] = a.Value(), b.Value() - return bytes.Compare(buf[0][:], buf[1][:]) - }) - - if !slices.Equal(idents, tc.wantIdents) { - t.Errorf( - "Clean: idents = %s, want %s", - formatHandles(idents...), formatHandles(tc.wantIdents...), - ) - } - if !slices.Equal(checksums, tc.wantChecksums) { - t.Errorf( - "Clean: checksums = %s, want %s", - formatHandles(checksums...), formatHandles(tc.wantChecksums...), - ) - } - - want := tc.want.hash() - var checksum pkg.Checksum - if err = pkg.SumDir(&checksum, base); err != nil { - t.Fatalf("SumDir: error = %v", err) - } else if checksum != want { - t.Error(expectsFrom(base.String())) - } - }) - } -} diff --git a/internal/pkg/compress.go b/internal/pkg/compress.go deleted file mode 100644 index 22f990f1..00000000 --- a/internal/pkg/compress.go +++ /dev/null @@ -1,151 +0,0 @@ -package pkg - -import ( - "compress/bzip2" - "compress/gzip" - "fmt" - "io" - "os" - - "hakurei.app/internal/xz" - "hakurei.app/internal/zstd" -) - -const ( - // Gzip denotes a stream compressed via [gzip]. - Gzip = iota - // Bzip2 denotes a stream compressed via [bzip2]. - Bzip2 - // Zstd denotes a stream compressed via [zstd]. - Zstd - // XZ denotes a stream compressed via [xz]. - XZ -) - -// A decompressArtifact is a [FileArtifact] decompressing a backing -// [FileArtifact] stream. -type decompressArtifact struct { - // Caller-supplied backing stream. - f Artifact - // Compression on top of the stream. - compress uint32 -} - -var _ FileArtifact = new(decompressArtifact) -var _ CuresExempt = new(decompressArtifact) - -// decompressArtifactNamed embeds decompressArtifact for a [fmt.Stringer] stream. -type decompressArtifactNamed struct { - decompressArtifact - // Copied from decompressArtifact.f. - name string -} - -var _ fmt.Stringer = new(decompressArtifactNamed) - -// NewDecompress returns a [FileArtifact] decompressing the supplied [Artifact]. -func NewDecompress(a Artifact, compress uint32) FileArtifact { - da := decompressArtifact{a, compress} - if s, ok := a.(fmt.Stringer); ok { - if name := s.String(); name != "" { - return &decompressArtifactNamed{da, name} - } - } - return &da -} - -// String returns the name of the underlying [Artifact] prefixed with decompress. -func (a *decompressArtifactNamed) String() string { return "decompress-" + a.name } - -// Kind returns the hardcoded [Kind] constant. -func (a *decompressArtifact) Kind() Kind { return KindDecompress } - -// Params writes value of compression enum. -func (a *decompressArtifact) Params(ctx *IContext) { ctx.WriteUint32(a.compress) } - -func init() { - register(KindDecompress, func(r *IRReader) Artifact { - a := NewDecompress(r.Next(), r.ReadUint32()) - if _, ok := r.Finalise(); ok { - panic(ErrUnexpectedChecksum) - } - return a - }) -} - -// Inputs returns a slice containing the backing file. -func (a *decompressArtifact) Inputs() []Artifact { - return []Artifact{a.f} -} - -// IsExclusive returns false: decompressor is fully sequential. -func (a *decompressArtifact) IsExclusive() bool { return false } - -// compoundCloser is an [io.ReadCloser] with an additional [io.Closer] attached. -type compoundCloser struct { - io.ReadCloser - c io.Closer -} - -// Close closes [io.ReadCloser] and the additional [io.Closer]. It returns the -// non-nil error returned by the underlying [io.ReadCloser], otherwise it -// returns the error returned by the additional [io.Closer]. -func (c compoundCloser) Close() error { - err := c.ReadCloser.Close() - if _err := c.c.Close(); err == nil { - err = _err - } - return err -} - -// IsExecutable returns false. -func (*decompressArtifact) IsExecutable() bool { return false } - -// Cure returns a decompressor [io.ReadCloser]. -func (a *decompressArtifact) Cure(r *RContext) (io.ReadCloser, error) { - sr, err := r.Open(a.f) - if err != nil { - return nil, err - } - br := r.cache.getReaderRC(sr) - - var dr io.ReadCloser - switch a.compress { - case Gzip: - if dr, err = gzip.NewReader(br); err != nil { - _ = br.Close() - return nil, err - } - return compoundCloser{dr, br}, nil - - case Bzip2: - return struct { - io.Reader - io.Closer - }{bzip2.NewReader(br), br}, nil - - case Zstd: - return struct { - io.Reader - io.Closer - }{zstd.NewReader(br), br}, nil - - case XZ: - var _dr io.Reader - if _dr, err = xz.NewReader(br, 0); err != nil { - _ = br.Close() - return nil, err - } - return struct { - io.Reader - io.Closer - }{_dr, br}, nil - - default: - return nil, os.ErrInvalid - } -} - -// CuresExempt exempts the cheap [KindDecompress] implementation often part of -// a [FileArtifact] pipeline. -func (*decompressArtifact) CuresExempt() {} diff --git a/internal/pkg/compress_test.go b/internal/pkg/compress_test.go deleted file mode 100644 index 7ce00edf..00000000 --- a/internal/pkg/compress_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package pkg_test - -import ( - "bytes" - "compress/gzip" - "crypto/sha512" - "io/fs" - "net/http" - "testing" - "testing/fstest" - - "hakurei.app/check" - "hakurei.app/internal/pkg" -) - -func TestDecompress(t *testing.T) { - t.Parallel() - - var buf bytes.Buffer - gw := gzip.NewWriter(&buf) - if _, err := gw.Write([]byte{0}); err != nil { - t.Fatal(err) - } else if err = gw.Close(); err != nil { - t.Fatal(err) - } - testdata := buf.String() - - var transport http.Transport - client := http.Client{Transport: &transport} - transport.RegisterProtocol("file", http.NewFileTransportFS(fstest.MapFS{ - "testdata": {Data: []byte(testdata), Mode: 0400}, - })) - testdataChecksum := func() pkg.Checksum { - h := sha512.New384() - h.Write([]byte(testdata)) - return (pkg.Checksum)(h.Sum(nil)) - }() - - gh := pkg.NewDecompress(pkg.NewHTTPGet( - &client, - "file:///testdata", - testdataChecksum, - ), pkg.Gzip) - - checkWithCache(t, []cacheTestCase{ - {"decompress", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - cureMany(t, c, []cureStep{ - {"close", pkg.NewDecompress(pkg.NewHTTPGet( - &client, - "file:///testdata", - pkg.Checksum{0xfd}, - ), pkg.Gzip), nil, nil, pkg.WNew, &pkg.ChecksumMismatchError{ - Got: testdataChecksum, - Want: pkg.Checksum{0xfd}, - }}, - - {"gzip", gh, ignorePathname, expectsChecksum(sha512.Sum384([]byte{0})), pkg.WNew, nil}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + pkg.Encode(sha512.Sum384([]byte{0})): {Mode: 0400, Data: []byte{0}}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/" + pkg.Encode(pkg.NewIR().Ident(gh).Value()): {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} diff --git a/internal/pkg/exec.go b/internal/pkg/exec.go deleted file mode 100644 index b40ae84b..00000000 --- a/internal/pkg/exec.go +++ /dev/null @@ -1,808 +0,0 @@ -package pkg - -import ( - "bufio" - "bytes" - "context" - "errors" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "runtime" - "slices" - "strconv" - "sync" - "syscall" - "time" - "unique" - - "hakurei.app/check" - "hakurei.app/container" - "hakurei.app/container/seccomp" - "hakurei.app/container/std" - "hakurei.app/ext" - "hakurei.app/fhs" - "hakurei.app/message" -) - -// AbsWork is the container pathname [TContext.GetWorkDir] is mounted on. -var AbsWork = fhs.AbsRoot.Append("work/") - -const ( - // EnvJobs is the name of the environment variable holding a decimal - // representation of the preferred job count. Its value must not affect cure - // outcome. - EnvJobs = "CURE_JOBS" - // EnvLoad is the name of the environment variable holding a decimal - // representation of the preferred loadavg target. Its value must not affect - // cure outcome. - EnvLoad = "CURE_LOAD" -) - -// ExecPath is a slice of [Artifact] and the [check.Absolute] pathname to make -// it available at under in the container. -type ExecPath struct { - // Pathname in the container mount namespace. - P *check.Absolute - // Artifacts to mount on the pathname, must contain at least one [Artifact]. - // If there are multiple entries or W is true, P is set up as an overlay - // mount, and entries of A must not implement [FileArtifact]. - A []Artifact - // Whether to make the mount point writable via the temp directory. - W bool -} - -// GetArtifactFunc is the function signature of [FContext.GetArtifact]. -type GetArtifactFunc func(Artifact) (*check.Absolute, unique.Handle[Checksum]) - -// PromoteLayers returns artifacts with identical-by-content layers promoted to -// the highest priority instance, as if mounted via [ExecPath]. -func PromoteLayers( - artifacts []Artifact, - getArtifact GetArtifactFunc, - report func(i int, d Artifact), -) []*check.Absolute { - layers := make([]*check.Absolute, 0, len(artifacts)) - checksums := make(map[unique.Handle[Checksum]]struct{}, len(artifacts)) - for i := range artifacts { - d := artifacts[len(artifacts)-1-i] - pathname, checksum := getArtifact(d) - if _, ok := checksums[checksum]; ok { - report(len(artifacts)-1-i, d) - continue - } - checksums[checksum] = struct{}{} - layers = append(layers, pathname) - } - slices.Reverse(layers) - return layers -} - -// layers returns pathnames collected from A deduplicated via [PromoteLayers]. -func (p *ExecPath) layers( - msg message.Msg, - getArtifact GetArtifactFunc, - ident func(a Artifact) unique.Handle[ID], -) []*check.Absolute { - return PromoteLayers(p.A, getArtifact, func(i int, d Artifact) { - if msg.IsVerbose() { - msg.Verbosef("promoted layer %d as %s", i, reportName(d, ident(d))) - } - }) -} - -// Path returns a populated [ExecPath]. -func Path(pathname *check.Absolute, writable bool, a ...Artifact) ExecPath { - return ExecPath{pathname, a, writable} -} - -// MustPath is like [Path], but takes a string pathname via [check.MustAbs]. -func MustPath(pathname string, writable bool, a ...Artifact) ExecPath { - return ExecPath{check.MustAbs(pathname), a, writable} -} - -var ( - binfmt map[string]container.BinfmtEntry - binfmtMu sync.RWMutex -) - -// RegisterArch arranges for [KindExec] and [KindExecNet] to support a new -// architecture via a binfmt_misc entry. Each architecture must be registered -// at most once. -func RegisterArch(arch string, e container.BinfmtEntry) { - if arch == "" { - panic(UnsupportedArchError(arch)) - } - - binfmtMu.Lock() - defer binfmtMu.Unlock() - - if binfmt == nil { - binfmt = make(map[string]container.BinfmtEntry) - } - - if _, ok := binfmt[arch]; ok { - panic("attempting to register " + strconv.Quote(arch) + " twice") - } - binfmt[arch] = e -} - -const ( - // ExecTimeoutDefault replaces out of range [NewExec] timeout values. - ExecTimeoutDefault = 15 * time.Minute - // ExecTimeoutMax is the arbitrary upper bound of [NewExec] timeout. - ExecTimeoutMax = 48 * time.Hour -) - -// An execArtifact is an [Artifact] that produces output by running a program -// part of another [Artifact] in a [container] to produce its output. -// -// Methods of execArtifact does not modify any struct field or underlying arrays -// referred to by slices. -type execArtifact struct { - // Caller-supplied user-facing reporting name, guaranteed to be nonzero - // during initialisation. - name string - // Target architecture. - arch string - // Caller-supplied inner mount points. - paths []ExecPath - - // Passed through to [container.Params]. - dir *check.Absolute - // Passed through to [container.Params]. - env []string - // Passed through to [container.Params]. - path *check.Absolute - // Passed through to [container.Params]. - args []string - - // Duration the initial process is allowed to run. The zero value is - // equivalent to [ExecTimeoutDefault]. - timeout time.Duration - - // Caller-supplied exclusivity value, returned as is by IsExclusive. - exclusive bool -} - -var _ fmt.Stringer = new(execArtifact) - -// execMeasuredArtifact is like execArtifact but implements [KnownChecksum] and -// has its resulting container optionally keep the host net namespace. -type execMeasuredArtifact struct { - checksum Checksum - - // Whether to keep host net namespace. - hostNet bool - - execArtifact -} - -var _ KnownChecksum = new(execMeasuredArtifact) - -// Checksum returns the caller-supplied checksum. -func (a *execMeasuredArtifact) Checksum() Checksum { return a.checksum } - -// Kind returns [KindExecNet], or [KindExec] if hostNet is false. -func (a *execMeasuredArtifact) Kind() Kind { - if a == nil || a.hostNet { - return KindExecNet - } - return KindExec -} - -// Cure cures the [Artifact] in the container described by the caller. The -// container optionally retains host networking. -func (a *execMeasuredArtifact) Cure(f *FContext) error { - return a.cure(f, a.hostNet) -} - -// ErrNetChecksum is panicked by [NewExec] if host net namespace is requested -// with a nil checksum. -var ErrNetChecksum = errors.New("attempting to keep net namespace without checksum") - -// NewExec returns a new [Artifact] that executes the program path in a -// container with specified paths bind mounted read-only in order. A private -// instance of /proc and /dev is made available to the container. -// -// The working and temporary directories are both created and mounted writable -// on [AbsWork] and [fhs.AbsTmp] respectively. If one or more paths target -// [AbsWork], the final entry is set up as a writable overlay mount on /work for -// which the upperdir is the host side work directory. In this configuration, -// the W field is ignored, and the program must avoid causing whiteout files to -// be created. Cure fails if upperdir ends up with entries other than directory, -// regular or symlink. -// -// If checksum is non-nil, the resulting [Artifact] implements [KnownChecksum] -// and its container optionally runs in the host net namespace. -// -// The container is allowed to run for the specified duration before the initial -// process and all processes originating from it is terminated. A zero or -// negative timeout value is equivalent tp [ExecTimeoutDefault], a timeout value -// greater than [ExecTimeoutMax] is equivalent to [ExecTimeoutMax]. -// -// The user-facing name and exclusivity value are not accessible from the -// container and does not affect curing outcome. Because of this, it is omitted -// from parameter data for computing identifier. -func NewExec( - name, arch string, - checksum *Checksum, - timeout time.Duration, - hostNet, exclusive bool, - - dir *check.Absolute, - env []string, - pathname *check.Absolute, - args []string, - - paths ...ExecPath, -) Artifact { - if name == "" { - name = "exec-" + filepath.Base(pathname.String()) - } - if arch == "" { - arch = runtime.GOARCH - } - if timeout <= 0 { - timeout = ExecTimeoutDefault - } - if timeout > ExecTimeoutMax { - timeout = ExecTimeoutMax - } - a := execArtifact{name, arch, paths, dir, env, pathname, args, timeout, exclusive} - if checksum == nil { - if hostNet { - panic(ErrNetChecksum) - } - return &a - } - return &execMeasuredArtifact{*checksum, hostNet, a} -} - -// Kind returns the hardcoded [Kind] constant. -func (*execArtifact) Kind() Kind { return KindExec } - -// Params writes paths, executable pathname and args. -func (a *execArtifact) Params(ctx *IContext) { - ctx.WriteString(a.arch) - ctx.WriteString(a.name) - - ctx.WriteUint32(uint32(len(a.paths))) - for _, p := range a.paths { - if p.P != nil { - ctx.WriteString(p.P.String()) - } else { - ctx.WriteString("invalid P\x00") - } - - ctx.WriteUint32(uint32(len(p.A))) - for _, d := range p.A { - ctx.WriteIdent(d) - } - - if p.W { - ctx.WriteUint32(1) - } else { - ctx.WriteUint32(0) - } - } - - ctx.WriteString(a.dir.String()) - - ctx.WriteUint32(uint32(len(a.env))) - for _, e := range a.env { - ctx.WriteString(e) - } - - ctx.WriteString(a.path.String()) - - ctx.WriteUint32(uint32(len(a.args))) - for _, arg := range a.args { - ctx.WriteString(arg) - } - - ctx.WriteUint32(uint32(a.timeout & 0xffffffff)) - ctx.WriteUint32(uint32(a.timeout >> 32)) - - if a.exclusive { - ctx.WriteUint32(1) - } else { - ctx.WriteUint32(0) - } -} - -// UnsupportedArchError describes an unsupported or invalid architecture. -type UnsupportedArchError string - -func (e UnsupportedArchError) Error() string { - if e == "" { - return "invalid architecture name" - } - return "unsupported architecture " + string(e) -} - -// readExecArtifact interprets IR values and returns the address of execArtifact -// or execNetArtifact. -func readExecArtifact(r *IRReader, net bool) Artifact { - r.DiscardAll() - - arch := r.ReadString() - if arch == "" { - panic(UnsupportedArchError(arch)) - } - - name := r.ReadString() - - sz := r.ReadUint32() - if sz > irMaxDeps { - panic(ErrIRDepend) - } - paths := make([]ExecPath, sz) - for i := range paths { - paths[i].P = check.MustAbs(r.ReadString()) - - sz = r.ReadUint32() - if sz > irMaxDeps { - panic(ErrIRDepend) - } - paths[i].A = make([]Artifact, sz) - for j := range paths[i].A { - paths[i].A[j] = r.ReadIdent() - } - - paths[i].W = r.ReadUint32() != 0 - } - - dir := check.MustAbs(r.ReadString()) - - sz = r.ReadUint32() - if sz > irMaxValues { - panic(ErrIRValues) - } - env := make([]string, sz) - for i := range env { - env[i] = r.ReadString() - } - - pathname := check.MustAbs(r.ReadString()) - - sz = r.ReadUint32() - if sz > irMaxValues { - panic(ErrIRValues) - } - args := make([]string, sz) - for i := range args { - args[i] = r.ReadString() - } - - timeout := time.Duration(r.ReadUint32()) - timeout |= time.Duration(r.ReadUint32()) << 32 - - exclusive := r.ReadUint32() != 0 - - checksum, ok := r.Finalise() - var checksumP *Checksum - if ok { - checksumP = new(checksum.Value()) - } - - if net && !ok { - panic(ErrExpectedChecksum) - } - - return NewExec( - name, arch, checksumP, timeout, net, exclusive, dir, env, pathname, args, paths..., - ) -} - -func init() { - register(KindExec, - func(r *IRReader) Artifact { return readExecArtifact(r, false) }) - register(KindExecNet, - func(r *IRReader) Artifact { return readExecArtifact(r, true) }) -} - -// Inputs returns a slice of all artifacts collected from caller-supplied -// [ExecPath]. -func (a *execArtifact) Inputs() []Artifact { - artifacts := make([][]Artifact, 0, len(a.paths)) - for _, p := range a.paths { - artifacts = append(artifacts, p.A) - } - return slices.Concat(artifacts...) -} - -// IsExclusive returns the caller-supplied exclusivity value. -func (a *execArtifact) IsExclusive() bool { return a.exclusive } - -// String returns the caller-supplied reporting name. -func (a *execArtifact) String() string { return a.name } - -// Cure cures the [Artifact] in the container described by the caller. -func (a *execArtifact) Cure(f *FContext) (err error) { - return a.cure(f, false) -} - -const ( - // execWaitDelay is passed through to [container.Params]. - execWaitDelay = time.Nanosecond -) - -// scanLinesCR is like [bufio.ScanLines], but also treats a bare \r as an -// end-of-line marker. -func scanLinesCR(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF && len(data) == 0 { - return 0, nil, nil - } - ri, ni := bytes.IndexByte(data, '\r'), bytes.IndexByte(data, '\n') - - if ri >= 0 && (ni < 0 || ri < ni) { - if ri+1 == ni { - // We have a full \r\n-terminated line. - return ri + 2, data[:ri], nil - } - // We have a bare \r, probably some kind of progress indicator. - return ri + 1, data[:ri], nil - } - if ni >= 0 && (ri < 0 || ni < ri) { - // We have a full newline-terminated line. - return ni + 1, data[:ni], nil - } - // If we're at EOF, we have a final, non-terminated line. Return it. - if atEOF { - return len(data), data, nil - } - // Request more data. - return 0, nil, nil -} - -// scanVerbose prefixes program output for a verbose [message.Msg]. -func scanVerbose( - msg message.Msg, - cancel context.CancelFunc, - done chan<- struct{}, - prefix, suffix string, - r io.Reader, -) { - defer close(done) - s := bufio.NewScanner(r) - s.Split(scanLinesCR) - s.Buffer( - make([]byte, bufio.MaxScanTokenSize), - bufio.MaxScanTokenSize<<12, - ) - for s.Scan() { - msg.Verbose(prefix, s.Text()+suffix) - } - if err := s.Err(); err != nil && !errors.Is(err, os.ErrClosed) { - cancel() - msg.Verbose("*"+prefix, err.Error()+suffix) - } -} - -var ( - // ErrInvalidPaths is returned for an [Artifact] of [KindExec] or - // [KindExecNet] specified with invalid paths. - ErrInvalidPaths = errors.New("invalid mount point") -) - -// SeccompPresets is the [seccomp] presets used by exec artifacts. -const SeccompPresets = std.PresetStrict & - ^(std.PresetDenyNS | std.PresetDenyDevel) - -// makeContainer sets up the specified temp and work directories and returns the -// corresponding [container.Container] that would have run for cure. -func (a *execArtifact) makeContainer( - ctx context.Context, - msg message.Msg, - flags, jobs, load int, - hostNet bool, - temp, work *check.Absolute, - getArtifact GetArtifactFunc, - ident func(a Artifact) unique.Handle[ID], -) (z *container.Container, err error) { - overlayWorkIndex := -1 - for i, p := range a.paths { - if p.P == nil || len(p.A) == 0 { - return nil, ErrInvalidPaths - } - if p.P.Is(AbsWork) { - overlayWorkIndex = i - } - } - - var artifactCount int - for _, p := range a.paths { - artifactCount += len(p.A) - } - - z = container.New(ctx, msg) - z.WaitDelay = execWaitDelay - z.SeccompPresets = SeccompPresets - z.SeccompFlags |= seccomp.AllowMultiarch - z.ParentPerm = 0700 - z.HostNet = hostNet - z.HostAbstract = flags&CHostAbstract != 0 - z.Hostname = "cure" - z.SetScheduler = flags&CSchedIdle != 0 - z.SchedPolicy = ext.SCHED_IDLE - if z.HostNet { - z.Hostname = "cure-net" - } - z.Quiet = flags&CSuppressInit != 0 - z.Uid, z.Gid = (1<<10)-1, (1<<10)-1 - z.Dir, z.Path, z.Args = a.dir, a.path, a.args - z.Env = slices.Concat(a.env, []string{ - EnvJobs + "=" + strconv.Itoa(jobs), - EnvLoad + "=" + strconv.Itoa(load), - }) - z.Grow(len(a.paths) + 4) - - if a.arch != runtime.GOARCH { - binfmtMu.RLock() - e, ok := binfmt[a.arch] - binfmtMu.RUnlock() - if !ok { - return nil, UnsupportedArchError(a.arch) - } - z.Binfmt = []container.BinfmtEntry{e} - z.InitAsRoot = true - } - - for i, b := range a.paths { - if i == overlayWorkIndex { - if err = os.MkdirAll(work.String(), 0700); err != nil { - return - } - tempWork := temp.Append(".work") - if err = os.MkdirAll(tempWork.String(), 0700); err != nil { - return - } - z.Overlay( - AbsWork, - work, - tempWork, - b.layers(msg, getArtifact, ident)..., - ) - continue - } - - if a.paths[i].W { - tempUpper, tempWork := temp.Append( - ".upper", strconv.Itoa(i), - ), temp.Append( - ".work", strconv.Itoa(i), - ) - if err = os.MkdirAll(tempUpper.String(), 0700); err != nil { - return - } - if err = os.MkdirAll(tempWork.String(), 0700); err != nil { - return - } - z.Overlay(b.P, tempUpper, tempWork, b.layers(msg, getArtifact, ident)...) - } else if len(b.A) == 1 { - pathname, _ := getArtifact(b.A[0]) - z.Bind(pathname, b.P, 0) - } else { - z.OverlayReadonly(b.P, b.layers(msg, getArtifact, ident)...) - } - } - if overlayWorkIndex < 0 { - z.Bind( - work, - AbsWork, - std.BindWritable|std.BindEnsure, - ) - } - z.Bind( - temp, - fhs.AbsTmp, - std.BindWritable|std.BindEnsure, - ) - z.Proc(fhs.AbsProc).Dev(fhs.AbsDev, true) - return -} - -var ( - // ErrExecBusy is returned entering [Cache.EnterExec] while another - // goroutine has not yet returned from it. - ErrExecBusy = errors.New("scratch directories in use") - // ErrNotExec is returned for unsupported implementations of [Artifact] - // passed to [Cache.EnterExec]. - ErrNotExec = errors.New("attempting to run a non-exec artifact") -) - -// EnterExec runs the container of an [Artifact] of [KindExec] or [KindExecNet] -// with its entry point, argument, and standard streams replaced with values -// supplied by the caller. -func (c *Cache) EnterExec( - ctx context.Context, - a Artifact, - hostname string, - retainSession bool, - stdin io.Reader, - stdout, stderr io.Writer, - path *check.Absolute, - args ...string, -) (err error) { - if !c.inExec.CompareAndSwap(false, true) { - return ErrExecBusy - } - defer c.inExec.Store(false) - - var hostNet bool - var e *execArtifact - switch f := a.(type) { - case *execArtifact: - e = f - - case *execMeasuredArtifact: - e = &f.execArtifact - hostNet = f.hostNet - - default: - return ErrNotExec - } - - deps := Collect(a.Inputs()) - if _, _, err = c.Cure(&deps); err == nil { - return errors.New("unreachable") - } else if !IsCollected(err) { - return - } - - dm := make(map[Artifact]cureRes) - for i, p := range deps { - var res cureRes - res.pathname, res.checksum, err = c.Cure(p) - if err != nil { - return - } - dm[deps[i]] = res - } - - scratch := c.base.Append(dirExecScratch) - temp, work := scratch.Append("temp"), scratch.Append("work") - // work created during makeContainer - if err = os.MkdirAll(temp.String(), 0700); err != nil { - return - } - defer func() { - if chmodErr, removeErr := removeAll(scratch); chmodErr != nil || removeErr != nil { - err = errors.Join(err, chmodErr, removeErr) - } - }() - - var z *container.Container - z, err = e.makeContainer( - ctx, c.msg, - c.attr.Flags, - c.attr.Jobs, - c.attr.Load, - hostNet, - temp, work, - func(a Artifact) (*check.Absolute, unique.Handle[Checksum]) { - if res, ok := dm[a]; ok { - return res.pathname, res.checksum - } - panic(InvalidLookupError(c.Ident(a).Value())) - }, - c.Ident, - ) - if err != nil { - return - } - z.Stdin, z.Stdout, z.Stderr = stdin, stdout, stderr - z.Path, z.Args = path, args - z.RetainSession = retainSession - if stdin == os.Stdin { - if s, ok := os.LookupEnv("TERM"); ok { - z.Env = append(z.Env, "TERM="+s) - } - } - if hostname != "" { - z.Hostname = hostname - } - - if err = z.Start(); err != nil { - return - } - if err = z.Serve(); err != nil { - return - } - return z.Wait() -} - -// cure is like Cure but allows optional host net namespace. -func (a *execArtifact) cure(f *FContext, hostNet bool) (err error) { - ctx, cancel := context.WithTimeout(f.Unwrap(), a.timeout) - defer cancel() - - msg := f.GetMessage() - var z *container.Container - if z, err = a.makeContainer( - ctx, msg, f.cache.attr.Flags, f.GetJobs(), f.GetLoad(), hostNet, - f.GetTempDir(), f.GetWorkDir(), - f.GetArtifact, - f.cache.Ident, - ); err != nil { - return - } - - var status io.Writer - if status, err = f.GetStatusWriter(); err != nil { - return - } - - if msg.IsVerbose() { - var stdout, stderr io.ReadCloser - if stdout, err = z.StdoutPipe(); err != nil { - return - } - if stderr, err = z.StderrPipe(); err != nil { - _ = stdout.Close() - return - } - - brStdout, brStderr := f.cache.getReader(stdout), f.cache.getReader(stderr) - stdoutDone, stderrDone := make(chan struct{}), make(chan struct{}) - - var suffix, prefixO, prefixE string - if f.cache.attr.Flags&CColourOutput != 0 { - suffix = "\x1b[0m" - prefixO = "\x1b[1;37m(" + a.name + ")\x1b[0m" - prefixE = "\x1b[1;97m(" + a.name + ")\x1b[0m" - } else { - prefixO = "(" + a.name + ":1)" - prefixE = "(" + a.name + ":2)" - } - - go scanVerbose( - msg, cancel, stdoutDone, - prefixO, suffix, - io.TeeReader(brStdout, status), - ) - go scanVerbose( - msg, cancel, stderrDone, - prefixE, suffix, - io.TeeReader(brStderr, status), - ) - defer func() { - if err != nil && !errors.As(err, new(*exec.ExitError)) { - _ = stdout.Close() - _ = stderr.Close() - } - - <-stdoutDone - <-stderrDone - f.cache.putReader(brStdout) - f.cache.putReader(brStderr) - }() - } else { - z.Stdout, z.Stderr = status, status - } - - if err = z.Start(); err != nil { - return - } - if err = z.Serve(); err != nil { - return - } - if err = z.Wait(); err != nil { - return - } - - // do not allow empty directories to succeed - for { - err = syscall.Rmdir(f.GetWorkDir().String()) - if err != syscall.EINTR { - break - } - } - if err != nil && errors.Is(err, syscall.ENOTEMPTY) { - err = nil - } - return -} diff --git a/internal/pkg/exec_test.go b/internal/pkg/exec_test.go deleted file mode 100644 index 408d6257..00000000 --- a/internal/pkg/exec_test.go +++ /dev/null @@ -1,670 +0,0 @@ -package pkg_test - -import ( - "bufio" - "bytes" - _ "embed" - "encoding/gob" - "errors" - "io/fs" - "net" - "os" - "os/exec" - "path/filepath" - "slices" - "strings" - "testing" - _ "unsafe" // for go:linkname - - "hakurei.app/check" - "hakurei.app/container" - "hakurei.app/hst" - "hakurei.app/internal/info" - "hakurei.app/internal/pkg" - "hakurei.app/internal/stub" - - "hakurei.app/internal/pkg/internal/testtool/expected" -) - -// testtoolBin is the container test tool binary made available to the -// execArtifact for testing its curing environment. -// -//go:generate env CGO_ENABLED=0 go build -tags testtool -o internal/testtool ./internal/testtool -//go:embed internal/testtool/testtool -var testtoolBin []byte - -func init() { - pathname, err := filepath.Abs("internal/testtool/testtool") - if err != nil { - panic(err) - } - pkg.RegisterArch("cafe", container.BinfmtEntry{ - Magic: expected.Magic, - Interpreter: check.MustAbs(pathname), - }) -} - -// scanLinesCR is like [bufio.ScanLines], but also treats a bare \r as an -// end-of-line marker. -// -//go:linkname scanLinesCR hakurei.app/internal/pkg.scanLinesCR -func scanLinesCR(data []byte, atEOF bool) (advance int, token []byte, err error) - -func TestScan(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - data string - want []string - }{ - {"progress indicator", "Updating files: 76% (8655/11256)\r" + - "Updating files: 77% (8668/11256)\r" + - "Updating files: 78% (8780/11256)\r" + - "Updating files: 79% (8893/11256)\r" + - "Updating files: 80% (9005/11256)\r" + - "Updating files: 81% (9118/11256)\r" + - "Updating files: 82% (9230/11256)\r" + - "Updating files: 83% (9343/11256)\r" + - "Updating files: 84% (9456/11256)\r" + - "Updating files: 85% (9568/11256)\r" + - "Updating files: 86% (9681/11256)\r" + - "Updating files: 87% (9793/11256)\r" + - "Updating files: 88% (9906/11256)\r" + - "Updating files: 89% (10018/11256)\r" + - "Updating files: 90% (10131/11256)\r" + - "Updating files: 91% (10243/11256)\r" + - "Updating files: 92% (10356/11256)\r" + - "Updating files: 93% (10469/11256)\r" + - "Updating files: 94% (10581/11256)\r" + - "Updating files: 95% (10694/11256)\r" + - "Updating files: 96% (10806/11256)\r" + - "Updating files: 97% (10919/11256)\r" + - "Updating files: 98% (11031/11256)\r" + - "Updating files: 99% (11144/11256)\r" + - "Updating files: 100% (11256/11256)\r" + - "Updating files: 100% (11256/11256), done.\n", []string{ - "Updating files: 76% (8655/11256)", - "Updating files: 77% (8668/11256)", - "Updating files: 78% (8780/11256)", - "Updating files: 79% (8893/11256)", - "Updating files: 80% (9005/11256)", - "Updating files: 81% (9118/11256)", - "Updating files: 82% (9230/11256)", - "Updating files: 83% (9343/11256)", - "Updating files: 84% (9456/11256)", - "Updating files: 85% (9568/11256)", - "Updating files: 86% (9681/11256)", - "Updating files: 87% (9793/11256)", - "Updating files: 88% (9906/11256)", - "Updating files: 89% (10018/11256)", - "Updating files: 90% (10131/11256)", - "Updating files: 91% (10243/11256)", - "Updating files: 92% (10356/11256)", - "Updating files: 93% (10469/11256)", - "Updating files: 94% (10581/11256)", - "Updating files: 95% (10694/11256)", - "Updating files: 96% (10806/11256)", - "Updating files: 97% (10919/11256)", - "Updating files: 98% (11031/11256)", - "Updating files: 99% (11144/11256)", - "Updating files: 100% (11256/11256)", - "Updating files: 100% (11256/11256), done.", - }}, - - {"crlf", "0\r1\n2\n3\r\n4\n5\r6\r\n7\n", []string{ - "0", "1", "2", "3", "4", "5", "6", "7", - }}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - s := bufio.NewScanner(strings.NewReader(tc.data)) - s.Split(scanLinesCR) - got := make([]string, 0, len(tc.want)) - for s.Scan() { - got = append(got, s.Text()) - } - if err := s.Err(); err != nil { - t.Fatal(err) - } - if !slices.Equal(got, tc.want) { - t.Fatalf("Scan: %q, want %q", got, tc.want) - } - }) - } -} - -func TestExec(t *testing.T) { - t.Parallel() - - wantOffline := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "check": {Mode: 0400, Data: []byte{0}}, - } - wantOfflineEncode := pkg.Encode(wantOffline.hash()) - failingArtifact := &stubArtifact{ - kind: pkg.KindTar, - params: []byte("doomed artifact"), - cure: func(t *pkg.TContext) error { - return stub.UniqueError(0xcafe) - }, - } - - checkWithCache(t, []cacheTestCase{ - {"offline", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - testtool, testtoolDestroy := newTesttool() - - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-offline", "", new(wantOffline.hash()), 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - - pkg.MustPath("/file", false, newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xfe, 0}, - nil, - nil, nil, - )), - pkg.MustPath("/.hakurei", false, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), - pkg.MustPath("/opt", false, testtool), - ), ignorePathname, wantOffline, pkg.WNew, nil}, - - {"substitution", pkg.NewExec( - "exec-offline", "", new(wantOffline.hash()), 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - - pkg.MustPath("/file", false, newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xfe, 0}, - nil, - nil, nil, - )), - // substitution miss fails in testtool due to differing idents - pkg.MustPath("/.hakurei", false, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory (substituted)"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), - pkg.MustPath("/opt", false, testtool), - ), ignorePathname, wantOffline, pkg.WSubstitute, nil}, - - {"error passthrough", pkg.NewExec( - "", "", nil, 0, false, true, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - - pkg.MustPath("/proc/nonexistent", false, failingArtifact), - ), nil, nil, pkg.WNew, pkg.InputError{ - failingArtifact: stub.UniqueError(0xcafe), - }}, - - {"invalid paths", pkg.NewExec( - "", "", nil, 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - - pkg.ExecPath{}, - ), nil, nil, pkg.WNew, pkg.ErrInvalidPaths}, - }) - - // check init failure passthrough - initFailureArtifact := pkg.NewExec( - "", "", nil, 0, false, false, - pkg.AbsWork, - nil, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - ) - var exitError *exec.ExitError - if _, _, err := c.Cure(initFailureArtifact); !errors.As(err, &exitError) || - exitError.ExitCode() != hst.ExitFailure { - t.Fatalf("Cure: error = %v, want init exit status 1", err) - } - - var faultStatus []byte - if faults, err := c.ReadFaults(initFailureArtifact); err != nil { - t.Fatal(err) - } else if len(faults) != 1 { - t.Fatalf("ReadFaults: %v", faults) - } else if faultStatus, err = os.ReadFile(faults[0].String()); err != nil { - t.Fatal(err) - } else if err = faults[0].Destroy(); err != nil { - t.Fatal(err) - } else { - t.Logf("destroyed expected fault at %s", faults[0].Time().UTC()) - } - - if !bytes.HasPrefix(faultStatus, []byte( - "internal/pkg ", - )) || !bytes.Contains(faultStatus, []byte( - "\ninit: fork/exec /opt/bin/testtool: no such file or directory\n", - )) { - t.Errorf("unexpected status:\n%s", string(faultStatus)) - } - - destroyStatus(t, base, 2, 1) - testtoolDestroy(t, base, c) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400, Data: []byte{}}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")}, - "identifier/" + expected.Offline: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)}, - "identifier/" + expected.OfflineS: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)}, - "identifier/_OHzj_xdmfpaWfFN-_D9JpGpPlIjh84873E3Ui9xwv9PPPIt0BlzvH-cnPNeRsTl": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"net", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - testtool, testtoolDestroy := newTesttool() - - wantNet := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "check": {Mode: 0400, Data: []byte("net")}, - } - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-net", "", new(wantNet.hash()), 0, true, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool", "net"}, - - pkg.MustPath("/file", false, newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xfe, 0}, - nil, - nil, nil, - )), - pkg.MustPath("/.hakurei", false, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), - pkg.MustPath("/opt", false, testtool), - ), ignorePathname, wantNet, pkg.WNew, nil}, - }) - - destroyStatus(t, base, 2, 0) - testtoolDestroy(t, base, c) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400, Data: []byte{}}, - "checksum/a1F_i9PVQI4qMcoHgTQkORuyWLkC1GLIxOhDt2JpU1NGAxWc5VJzdlfRK-PYBh3W": {Mode: fs.ModeDir | 0500}, - "checksum/a1F_i9PVQI4qMcoHgTQkORuyWLkC1GLIxOhDt2JpU1NGAxWc5VJzdlfRK-PYBh3W/check": {Mode: 0400, Data: []byte("net")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/" + expected.Net: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/a1F_i9PVQI4qMcoHgTQkORuyWLkC1GLIxOhDt2JpU1NGAxWc5VJzdlfRK-PYBh3W")}, - "identifier/_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"overlay root", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - testtool, testtoolDestroy := newTesttool() - - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-overlay-root", "", nil, 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - - pkg.MustPath("/", true, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), - pkg.MustPath("/opt", false, testtool), - ), ignorePathname, wantOffline, pkg.WNew, nil}, - }) - - destroyStatus(t, base, 2, 0) - testtoolDestroy(t, base, c) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/" + expected.OvlRoot: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"overlay work", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - testtool, testtoolDestroy := newTesttool() - - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-overlay-work", "", nil, 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"}, - check.MustAbs("/work/bin/testtool"), - []string{"testtool"}, - - pkg.MustPath("/", true, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), pkg.MustPath("/work/", false, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), pkg.Path(pkg.AbsWork, false /* ignored */, testtool), - ), ignorePathname, wantOffline, pkg.WNew, nil}, - }) - - destroyStatus(t, base, 2, 0) - testtoolDestroy(t, base, c) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/" + expected.Work: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"multiple layers", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - testtool, testtoolDestroy := newTesttool() - - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-multiple-layers", "", nil, 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool", "layers"}, - - pkg.MustPath("/", true, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }, &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("test sample with dependencies"), - - deps: slices.Repeat([]pkg.Artifact{newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xfe, 0}, - nil, - nil, nil, - ), &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - - // this is queued and might run instead of the other - // one so do not leave it as nil - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }}, 1<<5 /* concurrent cache hits */), - - cure: func(f *pkg.FContext) error { - work := f.GetWorkDir() - if err := os.MkdirAll(work.String(), 0700); err != nil { - return err - } - return os.WriteFile(work.Append("check").String(), []byte("layers"), 0400) - }, - }), - pkg.MustPath("/opt", false, testtool), - ), ignorePathname, wantOffline, pkg.WNew, nil}, - }) - - destroyStatus(t, base, 2, 0) - testtoolDestroy(t, base, c) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400, Data: []byte{}}, - "checksum/nY_CUdiaUM1OL4cPr5TS92FCJ3rCRV7Hm5oVTzAvMXwC03_QnTRfQ5PPs7mOU9fK": {Mode: fs.ModeDir | 0500}, - "checksum/nY_CUdiaUM1OL4cPr5TS92FCJ3rCRV7Hm5oVTzAvMXwC03_QnTRfQ5PPs7mOU9fK/check": {Mode: 0400, Data: []byte("layers")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")}, - "identifier/2uIvmnqtzAx4Ed4PWV8xjZCWeJCwfE2IcCc0evM-Nd8mDUlgSU6nLtPLcpPUcMlG": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/nY_CUdiaUM1OL4cPr5TS92FCJ3rCRV7Hm5oVTzAvMXwC03_QnTRfQ5PPs7mOU9fK")}, - "identifier/" + expected.Layers: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"overlay layer promotion", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - testtool, testtoolDestroy := newTesttool() - - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-layer-promotion", "", nil, 0, false, true, - pkg.AbsWork, - []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool", "promote"}, - - pkg.MustPath("/", true, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("another empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), - pkg.MustPath("/opt", false, testtool), - ), ignorePathname, wantOffline, pkg.WNew, nil}, - }) - - destroyStatus(t, base, 2, 0) - testtoolDestroy(t, base, c) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantOfflineEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantOfflineEncode + "/check": {Mode: 0400, Data: []byte{0}}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/Vzj9qaSVBtNyBQv_PW6uojFOWTwraHoaC0C6xdaGG1q_HTvSh4KLVxRcEQU1jd2Y": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/" + expected.Promote: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantOfflineEncode)}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"binfmt", pkg.CValidateKnown | checkDestroySubstitutes, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - if info.CanDegrade && os.Getenv("ROSA_SKIP_BINFMT") != "" { - t.Skip("binfmt_misc test explicitly skipped") - } - - cureMany(t, c, []cureStep{ - {"container", pkg.NewExec( - "exec-binfmt", "cafe", nil, 0, false, true, - pkg.AbsWork, - []string{"HAKUREI_TEST=1", "HAKUREI_BINFMT=1"}, - check.MustAbs("/opt/bin/sample"), - []string{"sample"}, - - pkg.MustPath("/", true, &stubArtifact{ - kind: pkg.KindTar, - params: []byte("empty directory"), - cure: func(t *pkg.TContext) error { - return os.MkdirAll(t.GetWorkDir().String(), 0700) - }, - }), - pkg.MustPath("/opt", false, overrideIdent{pkg.ID{0xfe, 0xff}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - work := t.GetWorkDir() - if err := os.MkdirAll( - work.Append("bin").String(), - 0700, - ); err != nil { - return err - } - - return os.WriteFile(t.GetWorkDir().Append( - "bin", - "sample", - ).String(), []byte(expected.Full), 0500) - }, - }}), - ), ignorePathname, expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "check": {Mode: 0400, Data: []byte("binfmt")}, - }, pkg.WNew, nil}, - }) - - destroyStatus(t, base, 2, 0) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/5aevg3YpDxjqQZ-pdvXK7YqgkL5JKqcoStYQxeD96kuYar6K2mRQWMHib6NQRnpV": {Mode: fs.ModeDir | 0500}, - "checksum/5aevg3YpDxjqQZ-pdvXK7YqgkL5JKqcoStYQxeD96kuYar6K2mRQWMHib6NQRnpV/bin": {Mode: fs.ModeDir | 0700}, - "checksum/5aevg3YpDxjqQZ-pdvXK7YqgkL5JKqcoStYQxeD96kuYar6K2mRQWMHib6NQRnpV/bin/sample": {Mode: 0500, Data: []byte("\xca\xfe\xba\xbe\xfd\xfd:3")}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - "checksum/UnDo4B5KneEUY5b4vRUk_y9MWgkWuw2N8f8a2XayO686xXur-aZmX2-7n_8tKMe3": {Mode: fs.ModeDir | 0500}, - "checksum/UnDo4B5KneEUY5b4vRUk_y9MWgkWuw2N8f8a2XayO686xXur-aZmX2-7n_8tKMe3/check": {Mode: 0400, Data: []byte("binfmt")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/c-HodmoEG0f_WGdst9aN-duLMH7QDDXOUi7UcyArXMeM1fc9lYKKQlTCW8ktaK8X": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UnDo4B5KneEUY5b4vRUk_y9MWgkWuw2N8f8a2XayO686xXur-aZmX2-7n_8tKMe3")}, - "identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/5aevg3YpDxjqQZ-pdvXK7YqgkL5JKqcoStYQxeD96kuYar6K2mRQWMHib6NQRnpV")}, - "identifier/MEaM1v6MLRQ60JQOOKbfDzfJOdC3L8i30IgOpYbQdX68enwF0FN-P36_M4RX_0Vd": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} - -// newTesttool returns an [Artifact] that cures into testtoolBin. The returned -// function must be called at the end of the test but not deferred. -func newTesttool() ( - testtool pkg.Artifact, - testtoolDestroy func(t *testing.T, base *check.Absolute, c *pkg.Cache), -) { - // testtoolBin is built during go:generate and is not deterministic - testtool = overrideIdent{pkg.ID{0xfe, 0xff}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - work := t.GetWorkDir() - if err := os.MkdirAll( - work.Append("bin").String(), - 0700, - ); err != nil { - return err - } - - if ift, err := net.Interfaces(); err != nil { - return err - } else { - var f *os.File - if f, err = os.Create(t.GetWorkDir().Append( - "ift", - ).String()); err != nil { - return err - } else { - err = gob.NewEncoder(f).Encode(ift) - closeErr := f.Close() - if err != nil { - return err - } - if closeErr != nil { - return closeErr - } - } - } - - return os.WriteFile(t.GetWorkDir().Append( - "bin", - "testtool", - ).String(), testtoolBin, 0500) - }, - }} - testtoolDestroy = newDestroyArtifactFunc(testtool) - return -} diff --git a/internal/pkg/file.go b/internal/pkg/file.go deleted file mode 100644 index 14467153..00000000 --- a/internal/pkg/file.go +++ /dev/null @@ -1,100 +0,0 @@ -package pkg - -import ( - "bytes" - "crypto/sha512" - "fmt" - "io" -) - -// A fileArtifact is an [Artifact] that cures into data known ahead of time. -type fileArtifact []byte - -var _ KnownChecksum = new(fileArtifact) -var _ CuresExempt = new(fileArtifact) -var _ RevisionArtifact = new(fileArtifact) - -// fileArtifactNamed embeds fileArtifact alongside a caller-supplied name. -type fileArtifactNamed struct { - fileArtifact - // Caller-supplied user-facing reporting name. - name string -} - -var _ fmt.Stringer = new(fileArtifactNamed) -var _ KnownChecksum = new(fileArtifactNamed) - -// String returns the caller-supplied reporting name. -func (a *fileArtifactNamed) String() string { return a.name } - -// Params writes the caller-supplied reporting name and the file body. -func (a *fileArtifactNamed) Params(ctx *IContext) { - ctx.WriteString(a.name) - ctx.Write(a.fileArtifact) -} - -// NewFile returns a [FileArtifact] that cures into a caller-supplied byte slice. -// -// Caller must not modify data after NewFile returns. -func NewFile(name string, data []byte) FileArtifact { - f := fileArtifact(data) - if name != "" { - return &fileArtifactNamed{f, name} - } - return &f -} - -// Kind returns the hardcoded [Kind] constant. -func (*fileArtifact) Kind() Kind { return KindFile } - -// Params writes an empty string and the file body. -func (a *fileArtifact) Params(ctx *IContext) { - ctx.WriteString("") - ctx.Write(*a) -} - -func init() { - register(KindFile, func(r *IRReader) Artifact { - name := r.ReadString() - data := r.ReadStringBytes() - if _, ok := r.Finalise(); !ok { - panic(ErrExpectedChecksum) - } - return NewFile(name, data) - }) -} - -// Inputs returns a nil slice. -func (*fileArtifact) Inputs() []Artifact { return nil } - -// IsExclusive returns false: Cure returns a prepopulated buffer. -func (*fileArtifact) IsExclusive() bool { return false } - -// Checksum computes and returns the checksum of caller-supplied data. -func (a *fileArtifact) Checksum() Checksum { - h := sha512.New384() - h.Write(*a) - return Checksum(h.Sum(nil)) -} - -// Revision satisfies [RevisionArtifact] for incompatible IsExecutable behaviour. -func (*fileArtifact) Revision() uint64 { return 0 } - -// IsExecutable returns whether the contents begin with shebang or the ELF -// magic number. -func (a *fileArtifact) IsExecutable() bool { - return a != nil && - (bytes.HasPrefix( - *a, []byte{'#', '!'}, - ) || bytes.HasPrefix( - *a, []byte{0x7f, 'E', 'L', 'F'}, - )) -} - -// Cure returns the caller-supplied data. -func (a *fileArtifact) Cure(*RContext) (io.ReadCloser, error) { - return io.NopCloser(bytes.NewReader(*a)), nil -} - -// CuresExempt exempts the cheap [KindFile] implementation. -func (*fileArtifact) CuresExempt() {} diff --git a/internal/pkg/file_test.go b/internal/pkg/file_test.go deleted file mode 100644 index f6fcc96f..00000000 --- a/internal/pkg/file_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package pkg_test - -import ( - "io/fs" - "testing" - - "hakurei.app/check" - "hakurei.app/internal/pkg" -) - -func TestFile(t *testing.T) { - t.Parallel() - - want := expectsFile{0} - const ident = "N7M6W8-e55Wky_1q1a6qRKFVDw1S5XtPdB4HE2YZObG8OFum8HittyJgZUG5Ru2u" - wantShebang := expectsFile("#!/bin/sh\n") - const identShebang = "oKTGZb_TRtJf5G1iGGj4SOx1WUbI6YhmsUeEsCNvmK9tZYRwdwQf-DLFUxRPx5qm" - wantELF := expectsFile("\x7fELF\xde\xad\xbe\xef") - const identELF = "LjeHmIJgD1SiXIH8ewpGRed2_LU84NLapstnOAa_0hxLd8YY7x4KM0aApsj-RwCh" - checkWithCache(t, []cacheTestCase{ - {"file", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - cureMany(t, c, []cureStep{ - {"short", pkg.NewFile("null", want), base.Append( - "identifier", - ident, - ), want, pkg.WNew, nil}, - - {"shebang", pkg.NewFile("shebang", wantShebang), base.Append( - "identifier", - identShebang, - ), wantShebang, pkg.WNew, nil}, - - {"elf", pkg.NewFile("ELF", wantELF), base.Append( - "identifier", - identELF, - ), wantELF, pkg.WNew, nil}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + pkg.Encode(want.hash()): {Mode: 0400, Data: want}, - "checksum/" + pkg.Encode(wantShebang.hash()): {Mode: 0500, Data: wantShebang}, - "checksum/" + pkg.Encode(wantELF.hash()): {Mode: 0500, Data: wantELF}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/" + ident: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + pkg.Encode(want.hash()))}, - "identifier/" + identShebang: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + pkg.Encode(wantShebang.hash()))}, - "identifier/" + identELF: {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + pkg.Encode(wantELF.hash()))}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} diff --git a/internal/pkg/internal/testtool/expected/expected.go b/internal/pkg/internal/testtool/expected/expected.go deleted file mode 100644 index c0e8e4a2..00000000 --- a/internal/pkg/internal/testtool/expected/expected.go +++ /dev/null @@ -1,9 +0,0 @@ -// Package expected contains data shared between test helper and test harness. -package expected - -const ( - // Magic are magic bytes in the binfmt test case. - Magic = "\xca\xfe\xba\xbe\xfd\xfd" - // Full is the full content of the binfmt test case executable. - Full = Magic + ":3" -) diff --git a/internal/pkg/internal/testtool/expected/sum_amd64.go b/internal/pkg/internal/testtool/expected/sum_amd64.go deleted file mode 100644 index 354931be..00000000 --- a/internal/pkg/internal/testtool/expected/sum_amd64.go +++ /dev/null @@ -1,11 +0,0 @@ -package expected - -const ( - Offline = "IKmVhALejtYA3JhN1PCfjKF-Dio1XVcOeMWDHhfSHCbb4iBNhGjFy2UspZveZvb8" - OfflineS = "6ZubjSyCnz4-TKgQZi_BbY-JUwsStYCtD_f-BTDOOotfEbqkm47W20AazHRaLviz" - OvlRoot = "Yox7WpG8B_OqIXWF4HWFZkUfsEl7DGzpFyZM0qtd9T8nQ-7SAkEUurlAe6_DJ6Iu" - Layers = "aAwfxDB-efVo379FNl178jK7HEKU3LEjHbZdaVVjk7GnXGZGEANNsXnB3SCUxM9A" - Net = "IU3GwU2jTBd7HjotvFo2wsDWBQGVGpamwiSKdc2G1EnhM-ZaacxAuK6q0wbTVxra" - Promote = "rOjetGHzT_jPErrBR24IAZKDj8Q9dLLqxFQqloMgjsULss0VXDI--n6LxuMBHvmZ" - Work = "7f9XDeREfvJIjCe0qesND0s0tasExAIcc-OH33-eq9tsrQPUXhJp4KEkpsYH4hCD" -) diff --git a/internal/pkg/internal/testtool/expected/sum_arm64.go b/internal/pkg/internal/testtool/expected/sum_arm64.go deleted file mode 100644 index 2e6f7cbf..00000000 --- a/internal/pkg/internal/testtool/expected/sum_arm64.go +++ /dev/null @@ -1,11 +0,0 @@ -package expected - -const ( - Offline = "MTgCSzwPrzGigp9phvDPRwUTNGLm1KM2EsEtbz-l2cNieqjaGtnzm65CDXlSxXrl" - OfflineS = "p89bQg7_jR4nwaSbpEd08ne1jHOEw7Wlnlyv5FHq7ba1Kml3HSBZwEKRzu0gCqEb" - OvlRoot = "9ifwO6MAaV9kTqRR2O5PJzm8XhfXbr50mFLWyEHQm8OHHcJtAEUgXZzJYoZt9YVV" - Layers = "INuGActFPwJtS7mAR9Bkt7FD4yoHlXHSJ9CaSAQJrpkUBL_k815Cs_jvRfxy5-a7" - Net = "kJvy24uk4ePvHoxSKf4vMOhR4NxLbmyWlA9bTS7d3z3fE9WF1Tvpao9g-QuYU4vm" - Promote = "lYqyYrfjPawIbld5Qz0NbKvd_bmI3nJC1eEnlIQWzsjWE9uA1HuKPAaClnDeyG6L" - Work = "V6k4FqOBMRCmHiqchs5-joVGZ_MhNWZyXm3gdbU0S7S5eMze7G4SwYMTf66ojKQL" -) diff --git a/internal/pkg/internal/testtool/expected/sum_riscv64.go b/internal/pkg/internal/testtool/expected/sum_riscv64.go deleted file mode 100644 index 53872c08..00000000 --- a/internal/pkg/internal/testtool/expected/sum_riscv64.go +++ /dev/null @@ -1,11 +0,0 @@ -package expected - -const ( - Offline = "LlieSHVDe1lnKvfI8t4oGRg_2hPxwyy1drmaqc-EKCOENuS9C2YHrp2VY5pSyhbg" - OfflineS = "qccjM8oYjUuEYPKFFzAxDFVzMerSUKUZc18NdVQoiPWf-70XOsNK8NaUQqjTgTAD" - OvlRoot = "lpV2bb_KGYFHPiOn-jtGop8K_ew0y1p3BezRp0ebM93_a-T9J-vkqGD5uokda1aN" - Layers = "sM3DJm210CHPGHkAquIbrXhspB81FpVC60ApQrvJGD5El2i7cSONj7LWf7EWqKpT" - Net = "JihQUwsVETRYjb2-yIXLYZ6WVsaFebwl8btSYAsNQ2QapS5MZVaI-dYwrVoySycR" - Promote = "lGqwR4W2OjylSVitMQP1lc6wApOBkbMjsmIqkxQbfEZ3DHEQBbJr84ErW6Xb54zo" - Work = "aZgtyv6aMtoHFkHAaJr6nLaA7N_SHjShgWPyQ5UdzaIqYSiVqvCrZqSokGTCcTii" -) diff --git a/internal/pkg/internal/testtool/main.go b/internal/pkg/internal/testtool/main.go deleted file mode 100644 index 96a85058..00000000 --- a/internal/pkg/internal/testtool/main.go +++ /dev/null @@ -1,277 +0,0 @@ -//go:build testtool - -package main - -import ( - "encoding/gob" - "log" - "net" - "os" - "path/filepath" - "reflect" - "runtime" - "slices" - "strconv" - "strings" - - "hakurei.app/fhs" - "hakurei.app/vfs" - - "hakurei.app/internal/pkg/internal/testtool/expected" -) - -func main() { - log.SetFlags(0) - log.SetPrefix("testtool: ") - - if os.Getenv("HAKUREI_BINFMT") == "1" { - wantArgs := []string{"/interpreter", "/opt/bin/sample"} - if !slices.Equal(os.Args, wantArgs) { - log.Fatalf("Args: %q, want %q", os.Args, wantArgs) - } - - if err := os.WriteFile("check", []byte("binfmt"), 0400); err != nil { - log.Fatal(err) - } - - return - } - - environ := slices.DeleteFunc(slices.Clone(os.Environ()), func(s string) bool { - for _, t := range []string{ - "CURE_JOBS=" + strconv.Itoa(runtime.NumCPU()), - "CURE_LOAD=" + strconv.Itoa(runtime.NumCPU()+2), - } { - if s == t { - return true - } - } - return false - }) - - var hostNet, layers, promote bool - if len(os.Args) == 2 && os.Args[0] == "testtool" { - switch os.Args[1] { - case "net": - hostNet = true - log.SetPrefix("testtool(net): ") - break - - case "layers": - layers = true - log.SetPrefix("testtool(layers): ") - break - - case "promote": - promote = true - log.SetPrefix("testtool(promote): ") - - default: - log.Fatalf("Args: %q", os.Args) - return - } - } else if wantArgs := []string{"testtool"}; !slices.Equal(os.Args, wantArgs) { - log.Fatalf("Args: %q, want %q", os.Args, wantArgs) - } - - var overlayRoot bool - wantEnv := []string{"HAKUREI_TEST=1"} - if len(environ) == 2 { - overlayRoot = true - if !layers && !promote { - log.SetPrefix("testtool(overlay root): ") - } - wantEnv = []string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"} - } - if !slices.Equal(wantEnv, environ) { - log.Fatalf("Environ: %q, want %q", environ, wantEnv) - } - - var overlayWork bool - const ( - wantExec = "/opt/bin/testtool" - wantExecWork = "/work/bin/testtool" - ) - var iftPath string - if got, err := os.Executable(); err != nil { - log.Fatalf("Executable: error = %v", err) - } else { - iftPath = filepath.Join(filepath.Dir(filepath.Dir(got)), "ift") - - if got != wantExec { - switch got { - case wantExecWork: - overlayWork = true - log.SetPrefix("testtool(overlay work): ") - - default: - log.Fatalf("Executable: %q, want %q", got, wantExec) - } - } - } - - wantHostname := "cure" - if hostNet { - wantHostname += "-net" - } - - if hostname, err := os.Hostname(); err != nil { - log.Fatalf("Hostname: error = %v", err) - } else if hostname != wantHostname { - log.Fatalf("Hostname: %q, want %q", hostname, wantHostname) - } - - var m *vfs.MountInfo - if f, err := os.Open(fhs.Proc + "self/mountinfo"); err != nil { - log.Fatalf("Open: error = %v", err) - } else { - err = vfs.NewMountInfoDecoder(f).Decode(&m) - closeErr := f.Close() - if err != nil { - log.Fatalf("Decode: error = %v", err) - } - if closeErr != nil { - log.Fatalf("Close: error = %v", err) - } - } - - if ift, err := net.Interfaces(); err != nil { - log.Fatal(err) - } else if !hostNet { - if len(ift) != 1 || ift[0].Name != "lo" { - log.Fatalln("got interfaces", strings.Join(slices.Collect(func(yield func(ifn string) bool) { - for _, ifi := range ift { - if !yield(ifi.Name) { - break - } - } - }), ", ")) - } - } else { - var iftParent []net.Interface - - var r *os.File - if r, err = os.Open(iftPath); err != nil { - log.Fatal(err) - } else { - err = gob.NewDecoder(r).Decode(&iftParent) - closeErr := r.Close() - if err != nil { - log.Fatal(err) - } - if closeErr != nil { - log.Fatal(closeErr) - } - } - - if !reflect.DeepEqual(ift, iftParent) { - log.Fatalf("Interfaces: %#v, want %#v", ift, iftParent) - } - } - - const checksumEmptyDir = "MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU" - ident := expected.Offline - log.Println(m) - next := func() { m = m.Next; log.Println(m) } - - if overlayRoot { - ident = expected.OvlRoot - - if m.Root != "/" || m.Target != "/" || - m.Source != "overlay" || m.FsType != "overlay" { - log.Fatal("unexpected root mount entry") - } - var lowerdir []string - for _, o := range strings.Split(m.FsOptstr, ",") { - const lowerdirKey = "lowerdir+=" - if strings.HasPrefix(o, lowerdirKey) { - lowerdir = append(lowerdir, o[len(lowerdirKey):]) - } - } - if !layers { - if len(lowerdir) != 1 || filepath.Base(lowerdir[0]) != checksumEmptyDir { - log.Fatal("unexpected artifact checksum") - } - } else { - ident = expected.Layers - - if len(lowerdir) != 2 || - filepath.Base(lowerdir[0]) != "MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU" || - filepath.Base(lowerdir[1]) != "nY_CUdiaUM1OL4cPr5TS92FCJ3rCRV7Hm5oVTzAvMXwC03_QnTRfQ5PPs7mOU9fK" { - log.Fatalf("unexpected lowerdirs %s", strings.Join(lowerdir, ", ")) - } - } - } else { - if hostNet { - ident = expected.Net - } - - if m.Root != "/sysroot" || m.Target != "/" { - log.Fatal("unexpected root mount entry") - } - - next() - if filepath.Base(m.Root) != "OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb" { - log.Fatal("unexpected file artifact checksum") - } - - next() - if filepath.Base(m.Root) != checksumEmptyDir { - log.Fatal("unexpected artifact checksum") - } - } - - if promote { - ident = expected.Promote - } - - next() // testtool artifact - - next() - if overlayWork { - ident = expected.Work - if m.Root != "/" || m.Target != "/work" || - m.Source != "overlay" || m.FsType != "overlay" { - log.Fatal("unexpected work mount entry") - } - } else { - if filepath.Base(m.Root) != ident || m.Target != "/work" { - log.Fatal("unexpected work mount entry") - } - } - - next() - if filepath.Base(m.Root) != ident || m.Target != "/tmp" { - log.Fatal("unexpected temp mount entry") - } - - next() - if m.Root != "/" || m.Target != "/proc" || m.Source != "proc" || m.FsType != "proc" { - log.Fatal("unexpected proc mount entry") - } - - next() - if m.Root != "/" || m.Target != "/dev" || m.Source != "devtmpfs" || m.FsType != "tmpfs" { - log.Fatal("unexpected dev mount entry") - } - - for i := 0; i < 9; i++ { // private /dev entries - next() - } - - if m.Next != nil { - log.Println("unexpected extra mount entries") - for m.Next != nil { - next() - } - os.Exit(1) - } - - checkData := []byte{0} - if hostNet { - checkData = []byte("net") - } - if err := os.WriteFile("check", checkData, 0400); err != nil { - log.Fatal(err) - } -} diff --git a/internal/pkg/ir.go b/internal/pkg/ir.go deleted file mode 100644 index d9f984a7..00000000 --- a/internal/pkg/ir.go +++ /dev/null @@ -1,865 +0,0 @@ -package pkg - -import ( - "bufio" - "bytes" - "crypto/sha512" - "encoding/binary" - "errors" - "fmt" - "io" - "iter" - "slices" - "strconv" - "sync" - "syscall" - "unique" - "unsafe" -) - -// wordSize is the boundary which binary segments are always aligned to. -const wordSize = 8 - -// alignSize returns the padded size for aligning sz. -func alignSize[T int | uint64](sz T) T { - return sz + (wordSize-(sz)%wordSize)%wordSize -} - -// panicToError recovers from a panic and replaces a nil error with the panicked -// error value. If the value does not implement error, it is re-panicked. -func panicToError(errP *error) { - r := recover() - if r == nil { - return - } - - if err, ok := r.(error); !ok { - panic(r) - } else if *errP == nil { - *errP = err - } -} - -// irCache implements [IRCache]. -type irCache struct { - // Artifact to [unique.Handle] of identifier cache. - artifact sync.Map - // Identifier free list, must not be accessed directly. - identPool sync.Pool -} - -// zeroIRCache returns the initialised value of irCache. -func zeroIRCache() irCache { - return irCache{ - identPool: sync.Pool{New: func() any { return new(extIdent) }}, - } -} - -// IRCache provides memory management and caching primitives for IR and -// identifier operations against [Artifact] implementations. -// -// The zero value is not safe for use. -type IRCache struct{ irCache } - -// NewIR returns the address of a new [IRCache]. -func NewIR() *IRCache { - return &IRCache{zeroIRCache()} -} - -// Inputs returns an iterator over direct and transitive inputs of an [Artifact] -// in randomised order. -func Inputs(a Artifact) iter.Seq2[Artifact, unique.Handle[ID]] { - ic := NewIR() - ic.Ident(a) - return func(yield func(Artifact, unique.Handle[ID]) bool) { - ic.artifact.Range(func(key, value any) bool { - return yield(key.(Artifact), value.(unique.Handle[ID])) - }) - } -} - -// IContext is passed to [Artifact.Params] and provides methods for writing -// values to the IR writer. It does not expose the underlying [io.Writer]. -// -// IContext is valid until [Artifact.Params] returns. -type IContext struct { - // Address of underlying irCache, should be zeroed or made unusable after - // [Artifact.Params] returns and must not be exposed directly. - ic *irCache - // Written to by various methods, should be zeroed after [Artifact.Params] - // returns and must not be exposed directly. - w io.Writer - // Optional [Artifact] to cureRes cache, replaces [IRKindIdent] with - // checksum values if non-nil. The pathname field may not be populated. - inputs map[Artifact]cureRes -} - -// irZero is a zero IR word. -var irZero [wordSize]byte - -// IRValueKind denotes the kind of encoded value. -type IRValueKind uint32 - -const ( - // IRKindEnd denotes the end of the current parameters stream. The ancillary - // value is interpreted as [IREndFlag]. - IRKindEnd IRValueKind = iota - // IRKindIdent denotes the identifier of a dependency [Artifact]. The - // ancillary value is reserved for future use. - IRKindIdent - // IRKindUint32 denotes an inlined uint32 value. - IRKindUint32 - // IRKindString denotes a string with its true length encoded in header - // ancillary data. Its wire length is always aligned to 8 byte boundary. - IRKindString - - irHeaderShift = 32 - irHeaderMask = 0xffffffff -) - -// String returns a user-facing name of k. -func (k IRValueKind) String() string { - switch k { - case IRKindEnd: - return "terminator" - case IRKindIdent: - return "ident" - case IRKindUint32: - return "uint32" - case IRKindString: - return "string" - default: - return "invalid kind " + strconv.Itoa(int(k)) - } -} - -// irValueHeader encodes [IRValueKind] and a 32-bit ancillary value. -type irValueHeader uint64 - -// encodeHeader returns irValueHeader encoding [IRValueKind] and ancillary data. -func (k IRValueKind) encodeHeader(v uint32) irValueHeader { - return irValueHeader(v)<<irHeaderShift | irValueHeader(k) -} - -// put stores h in b[0:8]. -func (h irValueHeader) put(b []byte) { - binary.LittleEndian.PutUint64(b[:], uint64(h)) -} - -// append appends the bytes of h to b and returns the appended slice. -func (h irValueHeader) append(b []byte) []byte { - return binary.LittleEndian.AppendUint64(b, uint64(h)) -} - -// IREndFlag is ancillary data encoded in the header of an [IRKindEnd] value and -// specifies the presence of optional fields in the remaining [IRKindEnd] data. -// Order of present fields is the order of their corresponding constants defined -// below. -type IREndFlag uint32 - -const ( - // IREndKnownChecksum denotes a [KnownChecksum] artifact. For an [IRKindEnd] - // value with this flag set, the remaining data contains the [Checksum]. - IREndKnownChecksum IREndFlag = 1 << iota -) - -// mustWrite writes to IContext.w and panics on error. The panic is recovered -// from by the caller and used as the return value. -func (i *IContext) mustWrite(p []byte) { - if _, err := i.w.Write(p); err != nil { - panic(err) - } -} - -// WriteIdent writes the identifier of [Artifact] to the IR. The behaviour of -// WriteIdent is not defined for an [Artifact] not part of the slice returned by -// [Artifact.Inputs]. -func (i *IContext) WriteIdent(a Artifact) { - buf := i.ic.getIdentBuf() - defer i.ic.putIdentBuf(buf) - - IRKindIdent.encodeHeader(0).put(buf[:]) - if i.inputs != nil { - res, ok := i.inputs[a] - if !ok { - panic(InvalidLookupError(i.ic.Ident(a).Value())) - } - *(*ID)(buf[wordSize:]) = res.checksum.Value() - } else { - *(*ID)(buf[wordSize:]) = i.ic.Ident(a).Value() - } - i.mustWrite(buf[:]) -} - -// WriteUint32 writes a uint32 value to the IR. -func (i *IContext) WriteUint32(v uint32) { - i.mustWrite(IRKindUint32.encodeHeader(v).append(nil)) -} - -// irMaxStringLength is the maximum acceptable wire size of [IRKindString]. -const irMaxStringLength = 1 << 24 - -// IRStringError is a string value too big to encode in IR. -type IRStringError string - -func (IRStringError) Error() string { - return "params value too big to encode in IR" -} - -// Write writes p as a string value to the IR. -func (i *IContext) Write(p []byte) { - sz := alignSize(len(p)) - if len(p) > irMaxStringLength || sz > irMaxStringLength { - panic(IRStringError(p)) - } - - i.mustWrite(IRKindString.encodeHeader(uint32(len(p))).append(nil)) - i.mustWrite(p) - - psz := sz - len(p) - if psz > 0 { - i.mustWrite(irZero[:psz]) - } -} - -// WriteString writes s as a string value to the IR. -func (i *IContext) WriteString(s string) { - p := unsafe.Slice(unsafe.StringData(s), len(s)) - i.Write(p) -} - -// Encode writes a deterministic, efficient representation of a to w and returns -// the first non-nil error encountered while writing to w. -func (ic *irCache) Encode(w io.Writer, a Artifact) (err error) { - return ic.encode(w, a, nil) -} - -// encode implements Encode but replaces identifiers with their cured checksums -// for a non-nil ident. The pathname field is unused. Caller must acquire -// Cache.identMu. -func (ic *irCache) encode( - w io.Writer, - a Artifact, - inputs map[Artifact]cureRes, -) (err error) { - deps := a.Inputs() - idents := make([]*extIdent, len(deps)) - if inputs == nil { - for i, d := range deps { - dbuf, did := ic.unsafeIdent(d, true) - if dbuf == nil { - dbuf = ic.getIdentBuf() - binary.LittleEndian.PutUint64(dbuf[:], uint64(d.Kind())) - *(*ID)(dbuf[wordSize:]) = did.Value() - } else { - ic.storeIdent(d, dbuf) - } - defer ic.putIdentBuf(dbuf) - idents[i] = dbuf - } - } else { - for i, d := range deps { - res, ok := inputs[d] - if !ok { - return InvalidLookupError(ic.Ident(d).Value()) - } - - dbuf := ic.getIdentBuf() - binary.LittleEndian.PutUint64(dbuf[:], uint64(d.Kind())) - *(*ID)(dbuf[wordSize:]) = res.checksum.Value() - defer ic.putIdentBuf(dbuf) - idents[i] = dbuf - } - } - slices.SortFunc(idents, func(a, b *extIdent) int { - return bytes.Compare(a[:], b[:]) - }) - idents = slices.CompactFunc(idents, func(a, b *extIdent) bool { - return *a == *b - }) - - // kind uint64 | rev uint64 | deps_sz uint64 - var buf [wordSize * 3]byte - binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind())) - binary.LittleEndian.PutUint64(buf[wordSize:], GetRevision(a)) - binary.LittleEndian.PutUint64(buf[wordSize*2:], uint64(len(idents))) - if _, err = w.Write(buf[:]); err != nil { - return - } - - for _, dn := range idents { - // kind uint64 | ident ID - if _, err = w.Write(dn[:]); err != nil { - return - } - } - - func() { - i := IContext{ic, w, inputs} - - defer panicToError(&err) - defer func() { i.ic, i.w = nil, nil }() - - a.Params(&i) - }() - if err != nil { - return - } - - var f IREndFlag - kcBuf := ic.getIdentBuf() - sz := wordSize - if kc, ok := a.(KnownChecksum); ok { - f |= IREndKnownChecksum - *(*Checksum)(kcBuf[wordSize:]) = kc.Checksum() - sz += len(Checksum{}) - } - IRKindEnd.encodeHeader(uint32(f)).put(kcBuf[:]) - - _, err = w.Write(kcBuf[:sz]) - ic.putIdentBuf(kcBuf) - return -} - -// encodeAll implements EncodeAll by recursively encoding dependencies and -// performs deduplication by value via the encoded map. -func (ic *irCache) encodeAll( - w io.Writer, - a Artifact, - encoded map[Artifact]struct{}, -) (err error) { - if _, ok := encoded[a]; ok { - return - } - - for _, d := range a.Inputs() { - if err = ic.encodeAll(w, d, encoded); err != nil { - return - } - } - - encoded[a] = struct{}{} - return ic.Encode(w, a) -} - -// EncodeAll writes a self-describing IR stream of a to w and returns the first -// non-nil error encountered while writing to w. -// -// EncodeAll tries to avoid encoding the same [Artifact] more than once, however -// it will fail to do so if they do not compare equal by value, as that will -// require buffering and greatly reduce performance. It is therefore up to the -// caller to avoid causing dependencies to be represented in a way such that -// two equivalent artifacts do not compare equal. While an IR stream with -// repeated artifacts is valid, it is somewhat inefficient, and the reference -// [IRDecoder] implementation produces a warning for it. -// -// Note that while EncodeAll makes use of the ident free list, it does not use -// the ident cache, nor does it contribute identifiers it computes back to the -// ident cache. Because of this, multiple invocations of EncodeAll will have -// similar cost and does not amortise when combined with a call to Cure. -func (ic *irCache) EncodeAll(w io.Writer, a Artifact) error { - return ic.encodeAll(w, a, make(map[Artifact]struct{})) -} - -// ErrRemainingIR is returned for a [IRReadFunc] that failed to call -// [IRReader.Finalise] before returning. -var ErrRemainingIR = errors.New("implementation did not consume final value") - -// DanglingIdentError is an identifier in a [IRKindIdent] value that was never -// described in the IR stream before it was encountered. -type DanglingIdentError unique.Handle[ID] - -func (e DanglingIdentError) Error() string { - return "artifact " + Encode(unique.Handle[ID](e).Value()) + - " was never described" -} - -type ( - // IRDecoder decodes [Artifact] from an IR stream. The stream is read to - // EOF and the final [Artifact] is returned. Previous artifacts may be - // looked up by their identifier. - // - // An [Artifact] may appear more than once in the same IR stream. A - // repeating [Artifact] generates a warning via [Cache] and will appear if - // verbose logging is enabled. Artifacts may only depend on artifacts - // previously described in the IR stream. - // - // IRDecoder rejects an IR stream on the first decoding error, it does not - // check against nonzero reserved ancillary data or incorrectly ordered or - // redundant unstructured dependencies. An invalid IR stream as such will - // yield [Artifact] values with identifiers disagreeing with those computed - // by IRDecoder. For this reason, IRDecoder does not access the ident cache - // to avoid putting [Cache] into an inconsistent state. - // - // Methods of IRDecoder are not safe for concurrent use. - IRDecoder struct { - // Address of underlying [Cache], must not be exposed directly. - c *Cache - - // Underlying IR reader. Methods of [IRReader] must not use this as it - // bypasses ident measurement. - r io.Reader - // Artifacts already seen in the IR stream. - ident map[unique.Handle[ID]]Artifact - - // Whether Decode returned, and the entire IR stream was decoded. - done, ok bool - } - - // IRReader provides methods to decode the IR wire format and read values - // from the reader embedded in the underlying [IRDecoder]. It is - // deliberately impossible to obtain the [IRValueKind] of the next value, - // and callers must never recover from panics in any read method. - // - // It is the responsibility of the caller to call Finalise after all IR - // values have been read. Failure to call Finalise causes the resulting - // [Artifact] to be rejected with [ErrRemainingIR]. - // - // For an [Artifact] expected to have dependencies, the caller must consume - // all dependencies by calling Next until all dependencies are depleted, or - // call DiscardAll to explicitly discard them and rely on values encoded as - // [IRKindIdent] instead. Failure to consume all unstructured dependencies - // causes the resulting [Artifact] to be rejected with [MissedDependencyError]. - // - // Requesting the value of an unstructured dependency not yet described in - // the IR stream via Next, or reading an [IRKindIdent] value not part of - // unstructured dependencies via ReadIdent may cause the resulting - // [Artifact] to be rejected with [DanglingIdentError], however either - // method may return a non-nil [Artifact] implementation of unspecified - // value. - IRReader struct { - // Address of underlying [IRDecoder], should be zeroed or made unusable - // after finalisation and must not be exposed directly. - d *IRDecoder - // Common buffer for word-sized reads. - buf [wordSize]byte - - // Inputs sent before params, sorted by identifier. Resliced on - // each call to Next and checked to be depleted during Finalise. - inputs []*extIdent - - // Number of values already read, -1 denotes a finalised IRReader. - count int - // Header of value currently being read. - h irValueHeader - - // Measured IR reader. All reads for the current [Artifact] must go - // through this to produce a correct ident. - r io.Reader - // Buffers measure writes. Flushed and returned to d during Finalise. - ibw *bufio.Writer - } - - // IRReadFunc reads IR values written by [Artifact.Params] to produce an - // instance of [Artifact] identical to the one to produce these values. - IRReadFunc func(r *IRReader) Artifact -) - -// kind returns the [IRValueKind] encoded in h. -func (h irValueHeader) kind() IRValueKind { - return IRValueKind(h & irHeaderMask) -} - -// value returns ancillary data encoded in h. -func (h irValueHeader) value() uint32 { - return uint32(h >> irHeaderShift) -} - -// irArtifact refers to artifact IR interpretation functions and must not be -// written to directly. -var irArtifact = make(map[Kind]IRReadFunc) - -// InvalidKindError is an unregistered [Kind] value. -type InvalidKindError Kind - -func (e InvalidKindError) Error() string { - return "invalid artifact kind " + strconv.Itoa(int(e)) -} - -// register records the [IRReadFunc] of an implementation of [Artifact] under -// the specified [Kind]. Expecting to be used only during initialization, it -// panics if the mapping between [Kind] and [IRReadFunc] is not a bijection. -// -// register is not safe for concurrent use. register must not be called after -// the first instance of [Cache] has been opened. -func register(k Kind, f IRReadFunc) { - openMu.Lock() - defer openMu.Unlock() - - if opened { - panic("attempting to register after open") - } - if _, ok := irArtifact[k]; ok { - panic("attempting to register " + strconv.Itoa(int(k)) + " twice") - } - irArtifact[k] = f -} - -// Register records the [IRReadFunc] of a custom implementation of [Artifact] -// under the specified [Kind]. Expecting to be used only during initialization, -// it panics if the mapping between [Kind] and [IRReadFunc] is not a bijection, -// or the specified [Kind] is below [KindCustomOffset]. -// -// Register is not safe for concurrent use. Register must not be called after -// the first instance of [Cache] has been opened. -func Register(k Kind, f IRReadFunc) { - if k < KindCustomOffset { - panic("attempting to register within internal kind range") - } - register(k, f) -} - -// NewDecoder returns a new [IRDecoder] that reads from the [io.Reader]. -func (c *Cache) NewDecoder(r io.Reader) *IRDecoder { - return &IRDecoder{c, r, make(map[unique.Handle[ID]]Artifact), false, false} -} - -const ( - // irMaxValues is the arbitrary maximum number of values allowed to be - // written by [Artifact.Params] and subsequently read via [IRReader]. - irMaxValues = 1 << 12 - - // irMaxDeps is the arbitrary maximum number of direct dependencies allowed - // to be returned by [Artifact.Inputs] and subsequently decoded by - // [IRDecoder]. - irMaxDeps = 1 << 10 -) - -var ( - // ErrIRValues is returned for an [Artifact] with too many parameter values. - ErrIRValues = errors.New("artifact has too many IR parameter values") - - // ErrIRDepend is returned for an [Artifact] with too many dependencies. - ErrIRDepend = errors.New("artifact has too many dependencies") - - // ErrAlreadyFinalised is returned when attempting to use an [IRReader] that - // has already been finalised. - ErrAlreadyFinalised = errors.New("reader has already finalised") -) - -// enterReader panics with an appropriate error for an out-of-bounds count and -// must be called at some point in any exported method. -func (ir *IRReader) enterReader(read bool) { - if ir.count < 0 { - panic(ErrAlreadyFinalised) - } - if ir.count >= irMaxValues { - panic(ErrIRValues) - } - - if read { - ir.count++ - } -} - -// IRKindError describes an attempt to read an IR value of unexpected kind. -type IRKindError struct { - Got, Want IRValueKind - Ancillary uint32 -} - -func (e *IRKindError) Error() string { - return fmt.Sprintf( - "got %s IR value (%#x) instead of %s", - e.Got, e.Ancillary, e.Want, - ) -} - -// readFull reads until either p is filled or an error is encountered. -func (ir *IRReader) readFull(p []byte) (n int, err error) { - for n < len(p) && err == nil { - var nn int - nn, err = ir.r.Read(p[n:]) - n += nn - } - return -} - -// mustRead reads from the underlying measured reader and panics on error. If -// an [io.EOF] is encountered and n != len(p), the error is promoted to a -// [io.ErrUnexpectedEOF], if n == 0, [io.EOF] is kept as is, otherwise it is -// zeroed. -func (ir *IRReader) mustRead(p []byte) { - n, err := ir.readFull(p) - if err == nil { - return - } - - if errors.Is(err, io.EOF) { - if n == len(p) { - return - } - err = io.ErrUnexpectedEOF - } - panic(err) -} - -// mustReadHeader reads the next header via d and checks its kind. -func (ir *IRReader) mustReadHeader(k IRValueKind) { - ir.mustRead(ir.buf[:]) - ir.h = irValueHeader(binary.LittleEndian.Uint64(ir.buf[:])) - if wk := ir.h.kind(); wk != k { - panic(&IRKindError{wk, k, ir.h.value()}) - } -} - -// putAll returns all dependency buffers to the underlying [Cache]. -func (ir *IRReader) putAll() { - for _, buf := range ir.inputs { - ir.d.c.putIdentBuf(buf) - } - ir.inputs = nil -} - -// DiscardAll discards all unstructured dependencies. This is useful to -// implementations that encode dependencies as [IRKindIdent] which are read back -// via ReadIdent. -func (ir *IRReader) DiscardAll() { - if ir.inputs == nil { - panic("attempting to discard dependencies twice") - } - ir.putAll() -} - -// ErrDependencyDepleted is returned when attempting to advance to the next -// unstructured dependency when there are none left. -var ErrDependencyDepleted = errors.New("reading past end of dependencies") - -// Next returns the next unstructured dependency. -func (ir *IRReader) Next() Artifact { - if len(ir.inputs) == 0 { - panic(ErrDependencyDepleted) - } - - id := unique.Make(ID(ir.inputs[0][wordSize:])) - ir.d.c.putIdentBuf(ir.inputs[0]) - ir.inputs = ir.inputs[1:] - - if a, ok := ir.d.ident[id]; !ok { - ir.putAll() - panic(DanglingIdentError(id)) - } else { - return a - } -} - -// MissedDependencyError is the number of unstructured dependencies remaining -// in [IRReader] that was never requested or explicitly discarded before -// finalisation. -type MissedDependencyError int - -func (e MissedDependencyError) Error() string { - return "missed " + strconv.Itoa(int(e)) + " unstructured dependencies" -} - -var ( - // ErrUnexpectedChecksum is returned by a [IRReadFunc] that does not expect - // a checksum but received one in [IRKindEnd] anyway. - ErrUnexpectedChecksum = errors.New("checksum specified on unsupported artifact") - // ErrExpectedChecksum is returned by a [IRReadFunc] that expects a checksum - // but did not receive one in [IRKindEnd]. - ErrExpectedChecksum = errors.New("checksum required but not specified") -) - -// Finalise reads the final [IRKindEnd] value and marks r as finalised. Methods -// of r are invalid upon entry into Finalise. If a [Checksum] is available via -// [IREndKnownChecksum], its handle is returned and the caller must store its -// value in the resulting [Artifact]. -func (ir *IRReader) Finalise() (checksum unique.Handle[Checksum], ok bool) { - ir.enterReader(true) - ir.count = -1 - - ir.mustReadHeader(IRKindEnd) - f := IREndFlag(ir.h.value()) - - if f&IREndKnownChecksum != 0 { - buf := ir.d.c.getIdentBuf() - defer ir.d.c.putIdentBuf(buf) - - ir.mustRead(buf[wordSize:]) - checksum = unique.Make(Checksum(buf[wordSize:])) - ok = true - } - - if err := ir.ibw.Flush(); err != nil { - panic(err) - } - ir.r, ir.ibw = nil, nil - - if len(ir.inputs) != 0 { - panic(MissedDependencyError(len(ir.inputs))) - } - - return -} - -// ReadIdent reads the next value as [IRKindIdent]. -func (ir *IRReader) ReadIdent() Artifact { - ir.enterReader(true) - ir.mustReadHeader(IRKindIdent) - - buf := ir.d.c.getIdentBuf() - defer ir.d.c.putIdentBuf(buf) - - ir.mustRead(buf[wordSize:]) - id := unique.Make(ID(buf[wordSize:])) - - if a, ok := ir.d.ident[id]; !ok { - panic(DanglingIdentError(id)) - } else { - return a - } -} - -// ReadUint32 reads the next value as [IRKindUint32]. -func (ir *IRReader) ReadUint32() uint32 { - ir.enterReader(true) - ir.mustReadHeader(IRKindUint32) - return ir.h.value() -} - -// ReadStringBytes reads the next value as [IRKindString] but returns it as a -// byte slice instead. -func (ir *IRReader) ReadStringBytes() []byte { - ir.enterReader(true) - ir.mustReadHeader(IRKindString) - - sz := int(ir.h.value()) - szWire := alignSize(sz) - if szWire > irMaxStringLength { - panic(IRStringError("\x00")) - } - - p := make([]byte, szWire) - ir.mustRead(p) - return p[:sz] -} - -// ReadString reads the next value as [IRKindString]. -func (ir *IRReader) ReadString() string { - p := ir.ReadStringBytes() - return unsafe.String(unsafe.SliceData(p), len(p)) -} - -// A RevisionError describes the first [Artifact] in an IR stream claiming a -// revision not supported by the registered implementation. -type RevisionError [2]uint64 - -func (e RevisionError) Error() string { - return "claimed revision " + strconv.FormatUint(e[0], 10) + - " differs from implementation value " + strconv.FormatUint(e[1], 10) -} - -// decode decodes the next [Artifact] in the IR stream and returns any buffer -// originating from [Cache] before returning. decode returns [io.EOF] if and -// only if the underlying [io.Reader] is already read to EOF. -func (d *IRDecoder) decode() (a Artifact, err error) { - defer panicToError(&err) - var ir IRReader - - defer func() { ir.d = nil }() - ir.d = d - - h := sha512.New384() - ir.ibw = d.c.getWriter(h) - defer d.c.putWriter(ir.ibw) - ir.r = io.TeeReader(d.r, ir.ibw) - - if n, _err := ir.readFull(ir.buf[:]); _err != nil { - if errors.Is(_err, io.EOF) { - if n != 0 { - _err = io.ErrUnexpectedEOF - } - } - - err = _err - return - } - ak := Kind(binary.LittleEndian.Uint64(ir.buf[:])) - f, ok := irArtifact[ak] - if !ok { - err = InvalidKindError(ak) - return - } - - ir.mustRead(ir.buf[:]) - rev := binary.LittleEndian.Uint64(ir.buf[:]) - - defer ir.putAll() - ir.mustRead(ir.buf[:]) - sz := binary.LittleEndian.Uint64(ir.buf[:]) - if sz > irMaxDeps { - err = ErrIRDepend - return - } - ir.inputs = make([]*extIdent, sz) - for i := range ir.inputs { - ir.inputs[i] = d.c.getIdentBuf() - } - for _, buf := range ir.inputs { - ir.mustRead(buf[:]) - } - - a = f(&ir) - if a == nil { - err = syscall.ENOTRECOVERABLE - return - } - if _rev := GetRevision(a); _rev != rev { - err = RevisionError{rev, _rev} - return - } - - if ir.count != -1 { - err = ErrRemainingIR - return - } - - buf := d.c.getIdentBuf() - h.Sum(buf[wordSize:wordSize]) - id := unique.Make(ID(buf[wordSize:])) - d.c.putIdentBuf(buf) - if _, ok = d.ident[id]; !ok { - d.ident[id] = a - } else { - d.c.msg.Verbosef( - "artifact %s%s%s appeared more than once in IR stream", - d.c.sgrIdent, Encode(id.Value()), d.c.sgrRes, - ) - } - - return -} - -// Decode consumes the IR stream to EOF and returns the final [Artifact]. After -// Decode returns, Lookup is available and Decode must not be called again. -func (d *IRDecoder) Decode() (a Artifact, err error) { - if d.done { - panic("attempting to decode an IR stream twice") - } - defer func() { d.done = true }() - - var cur Artifact -next: - a, err = d.decode() - - if err == nil { - cur = a - goto next - } - - if errors.Is(err, io.EOF) { - a, err = cur, nil - d.ok = true - } - return -} - -// Lookup looks up an [Artifact] described by the IR stream by its identifier. -func (d *IRDecoder) Lookup(id unique.Handle[ID]) (a Artifact, ok bool) { - if !d.ok { - panic("attempting to look up artifact without full IR stream") - } - a, ok = d.ident[id] - return -} diff --git a/internal/pkg/ir_test.go b/internal/pkg/ir_test.go deleted file mode 100644 index 312474bc..00000000 --- a/internal/pkg/ir_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package pkg_test - -import ( - "bytes" - "io" - "io/fs" - "reflect" - "testing" - - "hakurei.app/check" - "hakurei.app/internal/pkg" -) - -func TestIRRoundtrip(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - a pkg.Artifact - }{ - {"http get aligned", pkg.NewHTTPGet( - nil, "file:///testdata", - pkg.Checksum(bytes.Repeat([]byte{0xfd}, len(pkg.Checksum{}))), - )}, - {"http get unaligned", pkg.NewHTTPGet( - nil, "https://hakurei.app", - pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))), - )}, - - {"http get tar", pkg.NewTar(pkg.NewDecompress(pkg.NewHTTPGet( - nil, "file:///testdata", - pkg.Checksum(bytes.Repeat([]byte{0xff}, len(pkg.Checksum{}))), - ), pkg.Bzip2))}, - {"http get tar unaligned", pkg.NewTar(pkg.NewHTTPGet( - nil, "https://hakurei.app", - pkg.Checksum(bytes.Repeat([]byte{0xfe}, len(pkg.Checksum{}))), - ))}, - - {"exec offline", pkg.NewExec( - "exec-offline", "", nil, 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool"}, - - pkg.MustPath("/file", false, pkg.NewFile("file", []byte( - "stub file", - ))), pkg.MustPath("/.hakurei", false, pkg.NewTar(pkg.NewHTTPGet( - nil, "file:///hakurei.tar", - pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))), - ))), pkg.MustPath("/opt", false, pkg.NewTar(pkg.NewDecompress(pkg.NewHTTPGet( - nil, "file:///testtool.tar.gz", - pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))), - ), pkg.Gzip))), - )}, - - {"exec net", pkg.NewExec( - "exec-net", "", - (*pkg.Checksum)(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))), - 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool", "net"}, - - pkg.MustPath("/file", false, pkg.NewFile("file", []byte( - "stub file", - ))), pkg.MustPath("/.hakurei", false, pkg.NewTar(pkg.NewHTTPGet( - nil, "file:///hakurei.tar", - pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))), - ))), pkg.MustPath("/opt", false, pkg.NewTar(pkg.NewDecompress(pkg.NewHTTPGet( - nil, "file:///testtool.tar.gz", - pkg.Checksum(bytes.Repeat([]byte{0xfc}, len(pkg.Checksum{}))), - ), pkg.Gzip))), - )}, - - {"exec measured", pkg.NewExec( - "exec-measured", "", - (*pkg.Checksum)(bytes.Repeat([]byte{0xfd}, len(pkg.Checksum{}))), - 0, false, false, - pkg.AbsWork, - []string{"HAKUREI_TEST=1"}, - check.MustAbs("/opt/bin/testtool"), - []string{"testtool", "measured"}, - - pkg.MustPath("/file", false, pkg.NewFile("file", []byte( - "stub file", - ))), pkg.MustPath("/.hakurei", false, pkg.NewTar(pkg.NewHTTPGet( - nil, "file:///hakurei.tar", - pkg.Checksum(bytes.Repeat([]byte{0xfd}, len(pkg.Checksum{}))), - ))), pkg.MustPath("/opt", false, pkg.NewTar(pkg.NewDecompress(pkg.NewHTTPGet( - nil, "file:///testtool.tar.gz", - pkg.Checksum(bytes.Repeat([]byte{0xfd}, len(pkg.Checksum{}))), - ), pkg.Gzip))), - )}, - - {"file anonymous", pkg.NewFile("", []byte{0})}, - {"file", pkg.NewFile("stub", []byte("stub"))}, - - {"decompress", pkg.NewDecompress(pkg.NewFile("", []byte{0}), pkg.Bzip2)}, - - {"archive", pkg.NewArchive(pkg.NewFile("", []byte{0}))}, - } - testCasesCache := make([]cacheTestCase, len(testCases)) - for i, tc := range testCases { - want := tc.a - testCasesCache[i] = cacheTestCase{tc.name, 0, nil, - func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - r, w := io.Pipe() - - done := make(chan error, 1) - go func() { - t.Helper() - done <- c.EncodeAll(w, want) - _ = w.Close() - }() - - if got, err := c.NewDecoder(r).Decode(); err != nil { - t.Fatalf("Decode: error = %v", err) - } else if !reflect.DeepEqual(got, want) { - t.Fatalf("Decode: %#v, want %#v", got, want) - } - - if err := <-done; err != nil { - t.Fatalf("EncodeAll: error = %v", err) - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, - } - } - checkWithCache(t, testCasesCache) -} - -func TestRevision(t *testing.T) { - checkWithCache(t, []cacheTestCase{{"revision", 0, nil, - func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - a := pkg.NewFile("", nil) - var buf bytes.Buffer - if err := c.EncodeAll(&buf, a); err != nil { - t.Fatal(err) - } - - if _, err := c.NewDecoder( - bytes.NewReader(buf.Bytes()), - ).Decode(); err != nil { - t.Fatalf("Decode: error = %v", err) - } - - p := buf.Bytes() - p[8] = 0xfd - wantErr := pkg.RevisionError{0xfd, 0} - if _, err := c.NewDecoder( - bytes.NewReader(p), - ).Decode(); !reflect.DeepEqual(err, wantErr) { - t.Fatalf("Decode: error = %v, want %v", err, wantErr) - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }, - }}) -} diff --git a/internal/pkg/net.go b/internal/pkg/net.go deleted file mode 100644 index cd30e3bb..00000000 --- a/internal/pkg/net.go +++ /dev/null @@ -1,109 +0,0 @@ -package pkg - -import ( - "fmt" - "io" - "net/http" - "path" - "unique" -) - -// An httpArtifact is an [Artifact] backed by a [http] url string. The method is -// hardcoded as [http.MethodGet]. Request body is not allowed because it cannot -// be deterministically represented by Params. -type httpArtifact struct { - // Caller-supplied url string. - url string - - // Caller-supplied checksum of the response body. This is validated when - // closing the [io.ReadCloser] returned by Cure. - checksum unique.Handle[Checksum] - - // client is the address of the caller-supplied [http.Client]. - client *http.Client -} - -var _ KnownChecksum = new(httpArtifact) -var _ fmt.Stringer = new(httpArtifact) - -// NewHTTPGet returns a new [FileArtifact] backed by the supplied client. A GET -// request is set up for url. If c is nil, [http.DefaultClient] is used instead. -func NewHTTPGet( - c *http.Client, - url string, - checksum Checksum, -) FileArtifact { - return &httpArtifact{url: url, checksum: unique.Make(checksum), client: c} -} - -// Kind returns the hardcoded [Kind] constant. -func (*httpArtifact) Kind() Kind { return KindHTTPGet } - -// Params writes the backing url string. Client is not represented as it does -// not affect [Cache.Cure] outcome. -func (a *httpArtifact) Params(ctx *IContext) { ctx.WriteString(a.url) } - -func init() { - register(KindHTTPGet, func(r *IRReader) Artifact { - url := r.ReadString() - checksum, ok := r.Finalise() - if !ok { - panic(ErrExpectedChecksum) - } - return NewHTTPGet(nil, url, checksum.Value()) - }) -} - -// Inputs returns a nil slice. -func (*httpArtifact) Inputs() []Artifact { return nil } - -// IsExclusive returns false: Cure returns as soon as a response is received. -func (*httpArtifact) IsExclusive() bool { return false } - -// Checksum returns the caller-supplied checksum. -func (a *httpArtifact) Checksum() Checksum { return a.checksum.Value() } - -// String returns [path.Base] over the backing url. -func (a *httpArtifact) String() string { return path.Base(a.url) } - -// ResponseStatusError is returned for a response returned by an [http.Client] -// with a status code other than [http.StatusOK]. -type ResponseStatusError int - -func (e ResponseStatusError) Error() string { - return "the requested URL returned non-OK status: " + http.StatusText(int(e)) -} - -// IsExecutable returns false. -func (*httpArtifact) IsExecutable() bool { return false } - -// Cure sends the http request and returns the resulting response body reader -// wrapped to perform checksum validation. It is valid but not encouraged to -// close the resulting [io.ReadCloser] before it is read to EOF, as that causes -// Close to block until all remaining data is consumed and validated. -func (a *httpArtifact) Cure(r *RContext) (rc io.ReadCloser, err error) { - var req *http.Request - req, err = http.NewRequestWithContext(r.Unwrap(), http.MethodGet, a.url, nil) - if err != nil { - return - } - req.Header.Set("User-Agent", "Hakurei/1.1") - - c := a.client - if c == nil { - c = http.DefaultClient - } - - var resp *http.Response - if resp, err = c.Do(req); err != nil { - return - } - - if resp.StatusCode != http.StatusOK { - _ = resp.Body.Close() - return nil, ResponseStatusError(resp.StatusCode) - } - - rc = r.NewMeasuredReader(resp.Body, a.checksum) - return -} diff --git a/internal/pkg/net_test.go b/internal/pkg/net_test.go deleted file mode 100644 index d809d35f..00000000 --- a/internal/pkg/net_test.go +++ /dev/null @@ -1,168 +0,0 @@ -package pkg_test - -import ( - "crypto/sha512" - "io" - "io/fs" - "net/http" - "reflect" - "testing" - "testing/fstest" - "unique" - - "hakurei.app/check" - "hakurei.app/internal/pkg" -) - -func TestHTTPGet(t *testing.T) { - t.Parallel() - - const testdata = "\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69" - - testdataChecksum := func() unique.Handle[pkg.Checksum] { - h := sha512.New384() - h.Write([]byte(testdata)) - return unique.Make(pkg.Checksum(h.Sum(nil))) - }() - - var transport http.Transport - client := http.Client{Transport: &transport} - transport.RegisterProtocol("file", http.NewFileTransportFS(fstest.MapFS{ - "testdata": {Data: []byte(testdata), Mode: 0400}, - })) - - checkWithCache(t, []cacheTestCase{ - {"direct", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - r := newRContext(t, c) - f := pkg.NewHTTPGet( - &client, - "file:///testdata", - testdataChecksum.Value(), - ) - var got []byte - if rc, err := f.Cure(r); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if got, err = io.ReadAll(rc); err != nil { - t.Fatalf("ReadAll: error = %v", err) - } else if string(got) != testdata { - t.Fatalf("Cure: %x, want %x", got, testdata) - } else if err = rc.Close(); err != nil { - t.Fatalf("Close: error = %v", err) - } - - // check direct validation - f = pkg.NewHTTPGet( - &client, - "file:///testdata", - pkg.Checksum{}, - ) - wantErrMismatch := &pkg.ChecksumMismatchError{ - Got: testdataChecksum.Value(), - } - if rc, err := f.Cure(r); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if got, err = io.ReadAll(rc); err != nil { - t.Fatalf("ReadAll: error = %v", err) - } else if string(got) != testdata { - t.Fatalf("Cure: %x, want %x", got, testdata) - } else if err = rc.Close(); !reflect.DeepEqual(err, wantErrMismatch) { - t.Fatalf("Close: error = %#v, want %#v", err, wantErrMismatch) - } - - // check fallback validation - if rc, err := f.Cure(r); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if err = rc.Close(); !reflect.DeepEqual(err, wantErrMismatch) { - t.Fatalf("Close: error = %#v, want %#v", err, wantErrMismatch) - } - - // check direct response error - f = pkg.NewHTTPGet( - &client, - "file:///nonexistent", - pkg.Checksum{}, - ) - wantErrNotFound := pkg.ResponseStatusError(http.StatusNotFound) - if _, err := f.Cure(r); !reflect.DeepEqual(err, wantErrNotFound) { - t.Fatalf("Cure: error = %#v, want %#v", err, wantErrNotFound) - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"cure", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - r := newRContext(t, c) - - f := pkg.NewHTTPGet( - &client, - "file:///testdata", - testdataChecksum.Value(), - ) - wantPathname := base.Append( - "identifier", - "COBRUwkxpIvZWJggpRxn7HVXiw6epXZOex8jGRJKpowpwukV7GRBY_LBvOJNQHue", - ) - if pathname, checksum, err := c.Cure(f); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if !pathname.Is(wantPathname) { - t.Fatalf("Cure: %q, want %q", pathname, wantPathname) - } else if checksum != testdataChecksum { - t.Fatalf("Cure: %x, want %x", checksum.Value(), testdataChecksum.Value()) - } - - var got []byte - if rc, err := f.Cure(r); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if got, err = io.ReadAll(rc); err != nil { - t.Fatalf("ReadAll: error = %v", err) - } else if string(got) != testdata { - t.Fatalf("Cure: %x, want %x", got, testdata) - } else if err = rc.Close(); err != nil { - t.Fatalf("Close: error = %v", err) - } - - // check load from cache - f = pkg.NewHTTPGet( - &client, - "file:///testdata", - testdataChecksum.Value(), - ) - if rc, err := f.Cure(r); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if got, err = io.ReadAll(rc); err != nil { - t.Fatalf("ReadAll: error = %v", err) - } else if string(got) != testdata { - t.Fatalf("Cure: %x, want %x", got, testdata) - } else if err = rc.Close(); err != nil { - t.Fatalf("Close: error = %v", err) - } - - // check error passthrough - f = pkg.NewHTTPGet( - &client, - "file:///nonexistent", - pkg.Checksum{}, - ) - wantErrNotFound := pkg.ResponseStatusError(http.StatusNotFound) - if _, _, err := c.Cure(f); !reflect.DeepEqual(err, wantErrNotFound) { - t.Fatalf("Pathname: error = %#v, want %#v", err, wantErrNotFound) - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU": {Mode: 0400, Data: []byte("\x7f\xe1\x69\xa2\xdd\x63\x96\x26\x83\x79\x61\x8b\xf0\x3f\xd5\x16\x9a\x39\x3a\xdb\xcf\xb1\xbc\x8d\x33\xff\x75\xee\x62\x56\xa9\xf0\x27\xac\x13\x94\x69")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/COBRUwkxpIvZWJggpRxn7HVXiw6epXZOex8jGRJKpowpwukV7GRBY_LBvOJNQHue": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fLYGIMHgN1louE-JzITJZJo2SDniPu-IHBXubtvQWFO-hXnDVKNuscV7-zlyr5fU")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} diff --git a/internal/pkg/pkg.go b/internal/pkg/pkg.go deleted file mode 100644 index 268d0248..00000000 --- a/internal/pkg/pkg.go +++ /dev/null @@ -1,2947 +0,0 @@ -// Package pkg provides utilities for packaging software. -package pkg - -import ( - "bufio" - "bytes" - "cmp" - "context" - "crypto/rand" - "crypto/sha512" - "encoding/base64" - "encoding/binary" - "errors" - "fmt" - "hash" - "io" - "io/fs" - "iter" - "maps" - "math" - "os" - "path/filepath" - "runtime" - "slices" - "strconv" - "strings" - "sync" - "sync/atomic" - "syscall" - "testing" - "time" - "unique" - "unsafe" - - "hakurei.app/check" - "hakurei.app/internal/info" - "hakurei.app/internal/lockedfile" - "hakurei.app/message" -) - -const ( - // programName is the string identifying this build system. - programName = "internal/pkg" -) - -type ( - // A Checksum is a SHA-384 checksum computed for a cured [Artifact]. - Checksum = [sha512.Size384]byte - - // An ID is a unique identifier returned by [KnownIdent.ID]. This value must - // be deterministically determined ahead of time. - ID Checksum -) - -// Encode is abbreviation for base64.URLEncoding.EncodeToString(checksum[:]). -func Encode(checksum Checksum) string { - return base64.URLEncoding.EncodeToString(checksum[:]) -} - -// Decode is abbreviation for base64.URLEncoding.Decode(checksum[:], []byte(s)). -func Decode(buf *Checksum, s string) (err error) { - var n int - n, err = base64.URLEncoding.Decode(buf[:], []byte(s)) - if err == nil && n != len(buf) { - err = io.ErrUnexpectedEOF - } - return -} - -// MustDecode decodes a string representation of [Checksum] and panics if there -// is a decoding error or the resulting data is too short. -func MustDecode(s string) (checksum Checksum) { - if err := Decode(&checksum, s); err != nil { - panic(err) - } - return -} - -var ( - // extension is a string uniquely identifying a set of custom [Artifact] - // implementations registered by calling [Register]. - extension string - - // openMu synchronises access to global state for initialisation. - openMu sync.Mutex - // opened is false if [Open] was never called. - opened bool -) - -// Extension returns a string uniquely identifying the currently registered set -// of custom [Artifact], or the zero value if none was registered. -func Extension() string { return extension } - -// ValidExtension returns whether s is valid for use in a call to SetExtension. -func ValidExtension(s string) bool { - if l := len(s); l == 0 || l > 128 { - return false - } - for _, v := range s { - if v < 'a' || v > 'z' { - return false - } - } - return true -} - -// ErrInvalidExtension is returned for a variant identification string for which -// [ValidExtension] returns false. -var ErrInvalidExtension = errors.New("invalid extension variant identification string") - -// SetExtension sets the extension variant identification string. SetExtension -// must be called before [Open] if custom [Artifact] implementations had been -// recorded by calling [Register]. -// -// The variant identification string must be between 1 and 128 bytes long and -// consists of only bytes between 'a' and 'z'. -// -// SetExtension is not safe for concurrent use. SetExtension is called at most -// once and must not be called after the first instance of Cache has been opened. -func SetExtension(s string) { - openMu.Lock() - defer openMu.Unlock() - - if opened { - panic("attempting to set extension after open") - } - if extension != "" { - panic("attempting to set extension twice") - } - if !ValidExtension(s) { - panic(ErrInvalidExtension) - } - extension = s - statusHeader = makeStatusHeader(s) -} - -// common holds elements and receives methods shared between different contexts. -type common struct { - // Context specific to this [Artifact]. The toplevel context in [Cache] must - // not be exposed directly. - ctx context.Context - - // Address of underlying [Cache], should be zeroed or made unusable after - // Cure returns and must not be exposed directly. - cache *Cache -} - -// TContext is passed to [TrivialArtifact.Cure] and provides information and -// methods required for curing the [TrivialArtifact]. -// -// Methods of TContext are safe for concurrent use. TContext is valid -// until [TrivialArtifact.Cure] returns. -type TContext struct { - // Populated during [Cache.Cure]. - work, temp *check.Absolute - - // Target [Artifact] encoded identifier. - ids string - // Pathname status was created at. - statusPath, statusSPath *check.Absolute - // File statusHeader and logs are written to. - status *os.File - // Error value during prepareStatus. - statusErr error - - common -} - -// makeStatusHeader creates the header written to every status file. This should -// not be called directly, its result is stored in statusHeader and will not -// change after the first [Cache] is opened. -func makeStatusHeader(extension string) string { - s := programName - if v := info.Version(); v != info.FallbackVersion { - s += " " + v - } - if extension != "" { - s += " with " + extension + " extensions" - } - s += " (" + runtime.GOARCH + ")" - if name, err := os.Hostname(); err == nil { - s += " on " + name - } - s += "\n\n" - return s -} - -// statusHeader is the header written to all status files in dirStatus. -var statusHeader = makeStatusHeader("") - -// prepareStatus initialises the status file once. -func (t *TContext) prepareStatus(writeHeader bool) error { - if t.statusPath != nil || t.status != nil { - return t.statusErr - } - - t.statusPath = t.cache.base.Append( - dirStatus, - t.ids, - ) - if t.status, t.statusErr = os.OpenFile( - t.statusPath.String(), - syscall.O_CREAT|syscall.O_EXCL|syscall.O_WRONLY, - 0400, - ); t.statusErr != nil { - return t.statusErr - } - - if writeHeader { - _, t.statusErr = t.status.WriteString(statusHeader) - } - return t.statusErr -} - -// GetStatusWriter returns a [io.Writer] for build logs. The caller must not -// seek this writer before the position it was first returned in. -func (t *TContext) GetStatusWriter() (io.Writer, error) { - err := t.prepareStatus(true) - return t.status, err -} - -// destroy destroys the temporary directory and joins its errors with the error -// referred to by errP. If the error referred to by errP is non-nil, the work -// directory is removed similarly. [Cache] is responsible for making sure work -// is never left behind for a successful [Cache.Cure]. -// -// If implementation had requested status, it is closed with error joined with -// the error referred to by errP. If the error referred to by errP is non-nil, -// the status file is removed from the filesystem. -// -// destroy must be deferred by [Cache.Cure] if [TContext] is passed to any Cure -// implementation. It should not be called prior to that point. -func (t *TContext) destroy(errP *error) { - if chmodErr, removeErr := removeAll(t.temp); chmodErr != nil || removeErr != nil { - *errP = errors.Join(*errP, chmodErr, removeErr) - } - - if *errP != nil { - chmodErr, removeErr := removeAll(t.work) - if chmodErr != nil || removeErr != nil { - *errP = errors.Join(*errP, chmodErr, removeErr) - } else if errors.Is(*errP, os.ErrExist) { - if linkError, ok := errors.AsType[*os.LinkError](*errP); ok && - linkError != nil && - linkError.Op == "rename" { - // two artifacts may be backed by the same file - *errP = nil - } - } - } - - if t.status != nil { - if err := t.status.Close(); err != nil { - *errP = errors.Join(*errP, err) - } - if *errP != nil { - *errP = errors.Join(*errP, os.Rename( - t.statusPath.String(), t.cache.base.Append( - dirFault, - t.ids+"."+strconv.FormatUint(uint64( - time.Now().UnixNano(), - ), 10), - ).String(), - )) - if t.statusSPath != nil { - t.cache.checksumMu.Lock() - *errP = errors.Join(*errP, os.Remove(t.statusSPath.String())) - t.cache.checksumMu.Unlock() - } - } - t.status = nil - } -} - -// Unwrap returns the underlying [context.Context]. -func (c *common) Unwrap() context.Context { return c.ctx } - -// GetMessage returns [message.Msg] held by the underlying [Cache]. -func (c *common) GetMessage() message.Msg { return c.cache.msg } - -// GetJobs returns the preferred number of jobs to run, when applicable. Its -// value must not affect cure outcome. -func (c *common) GetJobs() int { return c.cache.attr.Jobs } - -// GetLoad returns the preferred load average target, when applicable. Its -// value must not affect cure outcome. -func (c *common) GetLoad() int { return c.cache.attr.Load } - -// GetWorkDir returns a pathname to a directory which [Artifact] is expected to -// write its output to. This is not the final resting place of the [Artifact] -// and this pathname should not be directly referred to in the final contents. -func (t *TContext) GetWorkDir() *check.Absolute { return t.work } - -// GetTempDir returns a pathname which implementations may use as scratch space. -// A directory is not created automatically, implementations are expected to -// create it if they wish to use it, using [os.MkdirAll]. -func (t *TContext) GetTempDir() *check.Absolute { return t.temp } - -// Open tries to open [Artifact] for reading. If a implements [FileArtifact], -// its reader might be used directly, eliminating the roundtrip to vfs. -// Otherwise, it must cure into a directory containing a single regular file. -// -// If err is nil, the caller must close the resulting [io.ReadCloser] and return -// its error, if any. Failure to read r to EOF may result in a spurious -// [ChecksumMismatchError], or the underlying implementation may block on Close. -func (c *common) Open(a Artifact) (r io.ReadCloser, err error) { - if f, ok := a.(FileArtifact); ok { - return c.cache.openFile(c.ctx, f) - } - - var pathname *check.Absolute - if pathname, _, _, err = c.cache.cure(a, true, false); err != nil { - return - } - - var entries []os.DirEntry - if entries, err = os.ReadDir(pathname.String()); err != nil { - return - } - - if len(entries) != 1 || !entries[0].Type().IsRegular() { - err = errors.New( - "input directory does not contain a single regular file", - ) - return - } else { - return os.Open(pathname.Append(entries[0].Name()).String()) - } -} - -// FContext is passed to [FloodArtifact.Cure] and provides information and -// methods required for curing the [FloodArtifact]. -// -// Methods of FContext are safe for concurrent use. FContext is valid -// until [FloodArtifact.Cure] returns. -type FContext struct { - TContext - - // Cured top-level inputs looked up by Pathname. - inputs map[Artifact]cureRes -} - -// linkSubstitute links status for substitute if populated. -func (f *FContext) linkSubstitute(ids, substitutes string) (err error) { - if f.status == nil || ids == substitutes { - return - } - - statusS := f.cache.base.Append( - dirStatus, - substitutes, - ) - f.cache.checksumMu.Lock() - err = os.Link(f.cache.base.Append( - dirStatus, - ids, - ).String(), statusS.String()) - f.cache.checksumMu.Unlock() - if err == nil { - f.statusSPath = statusS - } - return -} - -// InvalidLookupError is the identifier of non-input [Artifact] looked up -// via [FContext.GetArtifact] by a misbehaving [Artifact] implementation. -type InvalidLookupError ID - -func (e InvalidLookupError) Error() string { - return "attempting to look up non-input artifact " + Encode(e) -} - -var _ error = InvalidLookupError{} - -// GetArtifact returns the identifier pathname and checksum of an [Artifact]. -// Calling Pathname with an [Artifact] not part of the slice returned by -// [Artifact.Inputs] panics. -func (f *FContext) GetArtifact(a Artifact) ( - pathname *check.Absolute, - checksum unique.Handle[Checksum], -) { - if res, ok := f.inputs[a]; ok { - return res.pathname, res.checksum - } - panic(InvalidLookupError(f.cache.Ident(a).Value())) -} - -// RContext is passed to [FileArtifact.Cure] and provides helper methods useful -// for curing the [FileArtifact]. -// -// Methods of RContext are safe for concurrent use. RContext is valid -// until [FileArtifact.Cure] returns. -type RContext struct{ common } - -// An Artifact is a read-only reference to a piece of data that may be created -// deterministically but might not currently be available in memory or on the -// filesystem. -type Artifact interface { - // Kind returns the [Kind] of artifact. This is usually unique to the - // concrete type but two functionally identical implementations of - // [Artifact] is allowed to return the same [Kind] value. - Kind() Kind - - // Params writes deterministic values describing [Artifact]. Implementations - // must guarantee that these values are unique among differing instances - // of the same implementation with identical dependencies and conveys enough - // information to create another instance of [Artifact] identical to the - // instance emitting these values. The new instance created via [IRReadFunc] - // from these values must then produce identical IR values. - // - // Result must remain identical across multiple invocations. - Params(ctx *IContext) - - // Inputs returns a slice of [Artifact] the current instance has access to - // while producing its output. - // - // Callers must not modify the retuned slice. - // - // Result must remain identical across multiple invocations. - Inputs() []Artifact - - // IsExclusive returns whether the [Artifact] is exclusive. Exclusive - // artifacts might not run in parallel with each other, and are still - // subject to the cures limit. - // - // Some implementations may saturate the CPU for a nontrivial amount of - // time. Curing multiple such implementations simultaneously causes - // significant CPU scheduler overhead. An exclusive artifact will generally - // not be cured alongside another exclusive artifact, thus alleviating this - // overhead. - // - // Note that [Cache] reserves the right to still cure exclusive - // artifacts concurrently as this is not a synchronisation primitive but - // an optimisation one. Implementations are forbidden from accessing global - // state regardless of exclusivity. - // - // Result must remain identical across multiple invocations. - IsExclusive() bool -} - -// FloodArtifact refers to an [Artifact] requiring its entire dependency graph -// to be cured prior to curing itself. -type FloodArtifact interface { - // Cure cures the current [Artifact] to the working directory obtained via - // [TContext.GetWorkDir] embedded in [FContext]. - // - // Implementations must not retain c. - Cure(f *FContext) (err error) - - Artifact -} - -// TrivialArtifact refers to an [Artifact] that cures without requiring that -// any other [Artifact] is cured before it. Its dependency tree is ignored after -// computing its identifier. -// -// TrivialArtifact is unable to cure any other [Artifact] and it cannot access -// pathnames. This type of [Artifact] is primarily intended for dependency-less -// artifacts or direct dependencies that only consists of [FileArtifact]. -type TrivialArtifact interface { - // Cure cures the current [Artifact] to the working directory obtained via - // [TContext.GetWorkDir]. - // - // Implementations must not retain c. - Cure(t *TContext) (err error) - - Artifact -} - -// KnownIdent is optionally implemented by [Artifact] and is used instead of -// [Cache.Ident] when it is available. -// -// This is very subtle to use correctly. The implementation must ensure that -// this value is globally unique, otherwise [Cache] can enter an inconsistent -// state. This should not be implemented outside of testing. -type KnownIdent interface { - // ID returns a globally unique identifier referring to the current - // [Artifact]. This value must be known ahead of time and guaranteed to be - // unique without having obtained the full contents of the [Artifact]. - ID() ID - - Artifact -} - -// KnownChecksum is optionally implemented by [Artifact] for an artifact with -// output known ahead of time. -type KnownChecksum interface { - // Checksum returns the address of a known checksum. - // - // Callers must not modify the [Checksum]. - // - // Result must remain identical across multiple invocations. - Checksum() Checksum - - Artifact -} - -// CuresExempt is optionally implemented for an artifact exempt to the -// cache-wide cures counter and limit. -type CuresExempt interface { - Artifact - - // CuresExempt is a no-op function but serves to distinguish implementations - // that are cures-exempt. - CuresExempt() -} - -// FileArtifact refers to an [Artifact] backed by a single file. -// -// FileArtifact does not support fine-grained cancellation. Its context is -// inherited from the first [TrivialArtifact] or [FloodArtifact] that opens it. -type FileArtifact interface { - // IsExecutable returns whether the resulting filesystem entry should be made - // executable, if the [FileArtifact] is cured to the on-disk cache. - // - // Result must remain identical across multiple invocations. - IsExecutable() bool - - // Cure returns [io.ReadCloser] of the full contents of [FileArtifact]. If - // [FileArtifact] implements [KnownChecksum], Cure is responsible for - // validating any data it produces and must return [ChecksumMismatchError] - // if validation fails. This error is conventionally returned during the - // first call to Close, but may be returned during any call to Read before - // EOF, or by Cure itself. - // - // Callers are responsible for closing the resulting [io.ReadCloser]. - // - // The resulting [io.ReadCloser] across multiple invocations must have - // identical behaviour. - Cure(r *RContext) (io.ReadCloser, error) - - Artifact -} - -// RevisionArtifact is optionally implemented by an artifact that had undergone -// internal changes affecting its behaviour while retaining its IR structure. -type RevisionArtifact interface { - // Revision returns the revision number of [Artifact]. This value is always - // represented in the IR. An IR stream produced from the same [Kind] with - // differing revision is rejected. - // - // Result must remain identical across multiple invocations. - Revision() uint64 - - Artifact -} - -// GetRevision returns the revision number of an [Artifact]. -func GetRevision(a Artifact) (revision uint64) { - revision = math.MaxUint64 - if r, ok := a.(RevisionArtifact); ok { - revision = r.Revision() - } - return -} - -// reportName returns a string describing [Artifact] presented to the user. -func reportName(a Artifact, id unique.Handle[ID]) string { - r := Encode(id.Value()) - if s, ok := a.(fmt.Stringer); ok { - if name := s.String(); name != "" { - r += "-" + name - } - } - return r -} - -// Kind corresponds to the concrete type of [Artifact] and is used to create -// identifier for an [Artifact] with dependencies. -type Kind uint64 - -const ( - // KindHTTPGet is the kind of [Artifact] returned by [NewHTTPGet]. - KindHTTPGet Kind = iota - // KindTar is the kind of [Artifact] returned by [NewTar]. - KindTar - // KindExec is the kind of [Artifact] returned by [NewExec]. - KindExec - // KindExecNet is the kind of [Artifact] returned by [NewExec] but with a - // non-nil checksum. - KindExecNet - // KindFile is the kind of [Artifact] returned by [NewFile]. - KindFile - // KindDecompress is the kind of [Artifact] returned by [NewDecompress]. - KindDecompress - // KindArchive is the kind of [Artifact] returned by [NewArchive]. - KindArchive - - // _kindEnd is the total number of kinds and does not denote a kind. - _kindEnd - - // KindCustomOffset is the first [Kind] value reserved for implementations - // not from this package. - KindCustomOffset = 1 << 31 -) - -const ( - // kindCollection is the kind of [Collect]. It never cures successfully. - kindCollection Kind = KindCustomOffset - 1 - iota -) - -const ( - // fileLock is the lock file for exclusive access to the cache directory. - fileLock = "lock" - // fileVariant is a file holding the variant identification string set by a - // prior call to [SetExtension]. - fileVariant = "variant" - - // dirSubstitute holds symlinks to artifacts by checksum, named after their - // substitute identifier. - dirSubstitute = "substitute" - // dirIdentifier holds symlinks to artifacts by checksum, named after their - // IR-based identifier. - dirIdentifier = "identifier" - // dirChecksum holds artifacts named after their [Checksum]. - dirChecksum = "checksum" - // dirStatus holds artifact metadata and logs named after their IR-based - // identifier. For [FloodArtifact], the same file is also available under - // its substitute identifier. - dirStatus = "status" - // dirFault holds status files of faulted cures. - dirFault = "fault" - - // dirWork holds working pathnames set up during [Cache.Cure]. - dirWork = "work" - // dirTemp holds scratch space allocated during [Cache.Cure]. - dirTemp = "temp" - - // dirExecScratch is scratch space set up for the container started by - // [Cache.EnterExec]. Exclusivity via Cache.inExec. - dirExecScratch = "scratch" - - // checksumLinknamePrefix is prepended to the encoded [Checksum] value - // of an [Artifact] when creating a symbolic link to dirChecksum. - checksumLinknamePrefix = "../" + dirChecksum + "/" -) - -// cureRes are the non-error results returned by [Cache.Cure]. -type cureRes struct { - pathname *check.Absolute - checksum unique.Handle[Checksum] -} - -// A pendingArtifactDep is an input [Artifact] pending concurrent curing, -// subject to the cures limit. Values pointed to by result addresses are safe -// to access after the [sync.WaitGroup] associated with this pendingArtifactDep -// is done. pendingArtifactDep must not be reused or modified after it is sent -// to cure. -type pendingArtifactDep struct { - // Dependency artifact populated during [Cache.Cure]. - a Artifact - - // Address of result pathname populated during [Cache.Cure] and dereferenced - // if curing succeeds. - resP *cureRes - - // Address of result error map populated during [Cache.Cure], dereferenced - // after acquiring errsMu if curing fails. No additional action is taken, - // [Cache] and its caller are responsible for further error handling. - errs InputError - // Address of mutex synchronising access to errs. - errsMu *sync.Mutex - - // For synchronising access to result buffer. - *sync.WaitGroup -} - -const ( - // CValidateKnown arranges for [KnownChecksum] outcomes to be validated to - // match its intended checksum. - // - // A correct implementation of [KnownChecksum] does not successfully cure - // with output not matching its intended checksum. When an implementation - // fails to perform this validation correctly, the on-disk format enters - // an inconsistent state (correctable by [Cache.Scrub]). - // - // This flag causes [Cache.Cure] to always compute the checksum, and reject - // a cure if it does not match the intended checksum. - // - // This behaviour significantly reduces performance and is not recommended - // outside of testing a custom [Artifact] implementation. - CValidateKnown = 1 << iota - - // CSchedIdle arranges for the [ext.SCHED_IDLE] scheduling priority to be - // set for [KindExec] and [KindExecNet] containers. - CSchedIdle - - // CAssumeChecksum enables the use of [KnownChecksum] for duplicate function - // call suppression via the on-disk cache. - // - // This may cause incorrect cure outcome if an impossible checksum is - // specified that matches an output already present in the on-disk cache. - // This may be avoided by purposefully specifying a statistically - // unattainable checksum, like the zero value. - // - // While this optimisation might seem appealing, it is almost never - // applicable in real world use. Almost every time this path was taken, it - // was caused by an incorrect checksum accidentally left behind while - // bumping a package. Only enable this if you are really sure you need it. - CAssumeChecksum - - // CHostAbstract disables restriction of sandboxed processes from connecting - // to an abstract UNIX socket created by a host process. - // - // This is considered less secure in some systems, but does not introduce - // impurity due to [KindExecNet] being [KnownChecksum]. This flag exists - // to support kernels without Landlock LSM enabled. - CHostAbstract - - // CPromoteVariant allows [pkg.Open] to promote an unextended on-disk cache - // to the current extension variant. This is a one-way operation. - CPromoteVariant - - // CSuppressInit arranges for verbose output of the container init to be - // suppressed regardless of [message.Msg] state. - CSuppressInit - - // CIgnoreSubstitutes disables content-based input substitution. - CIgnoreSubstitutes - - // CExternShallow arranges for only non-flood inputs to be fetched when - // curing an [Artifact] available via the external cache. - CExternShallow - - // CColourOutput enables output colouring via ANSI control sequences. - CColourOutput -) - -// toplevel holds [context.WithCancel] over caller-supplied context, where all -// [Artifact] context are derived from. -type toplevel struct { - ctx context.Context - cancel context.CancelFunc -} - -// newToplevel returns the address of a new toplevel via ctx. -func newToplevel(ctx context.Context) *toplevel { - var t toplevel - t.ctx, t.cancel = context.WithCancel(ctx) - return &t -} - -// pendingCure provides synchronisation and cancellation for pending cures. -type pendingCure struct { - // Closed on cure completion. - done <-chan struct{} - // Error outcome, safe to access after done is closed. - err error - // Cancels the corresponding cure. - cancel context.CancelFunc -} - -// An External cache provides prepared [Artifact] cure outcomes. -type External interface { - // Artifact returns the address of the [Checksum] of the cure outcome of - // an [Artifact] corresponding to id, or nil if this [Artifact] is not - // available in the external cache. - Artifact(ctx context.Context, id unique.Handle[ID]) (*Checksum, error) - // Checksum returns an [Artifact] producing the specified checksum. - Checksum(checksum unique.Handle[Checksum]) Artifact - // Status returns [io.ReadCloser] of the status file of an [Artifact] - // corresponding to id, or nil if this [Artifact] is not available or a - // status file is not present. - Status(r *RContext, id unique.Handle[ID]) (io.ReadCloser, error) -} - -// Cache is a support layer that implementations of [Artifact] can use to store -// cured [Artifact] data in a content addressed fashion. -type Cache struct { - // Cures of any variant of [Artifact] sends to cures before entering the - // implementation and receives an equal amount of elements after. - cures chan struct{} - - // Parent context which toplevel was derived from. - parent context.Context - // For deriving curing context, must not be accessed directly. - toplevel atomic.Pointer[toplevel] - // For waiting on input curing goroutines. - wg sync.WaitGroup - // Reports new cures and passed to [Artifact]. - msg message.Msg - // Select graphics rendition sequences, populated by Open. - sgrRes, sgrIdent, sgrWarn, sgrErr string - - // Directory where all [Cache] related files are placed. - base *check.Absolute - // Immutable [CacheAttr] populated by [Open]. - attr CacheAttr - - // Must not be exposed directly. - irCache - - // Synchronises access to dirChecksum. - checksumMu sync.RWMutex - - // Presence of an alternative in the cache. Keys are not valid identifiers - // and must not be used as such. - substitute map[unique.Handle[ID]]unique.Handle[Checksum] - // Synchronises access to substitute and corresponding filesystem entries. - substituteMu sync.RWMutex - // Identifier to content pair cache. - ident map[unique.Handle[ID]]unique.Handle[Checksum] - // Identifier to error pair for unrecoverably faulted [Artifact]. - identErr map[unique.Handle[ID]]error - // Pending identifiers, accessed through Cure for entries not in ident. - identPending map[unique.Handle[ID]]*pendingCure - // Synchronises access to ident and corresponding filesystem entries. - identMu sync.RWMutex - // Synchronises entry into Abort and Cure. - abortMu sync.RWMutex - - // Synchronises entry into exclusive artifacts for the cure method. - exclMu sync.Mutex - // Buffered I/O free list, must not be accessed directly. - brPool, bwPool sync.Pool - - // Optional external cache implementation. - extern External - // Caches responses from extern. - externCache map[unique.Handle[ID]]unique.Handle[Checksum] - // Synchronises access to extern. - externMu sync.RWMutex - - // Unlocks the on-filesystem cache. Must only be called from Close. - unlock func() - // Whether [Cache] is considered closed. - closed bool - // Synchronises calls to Abort and Close. - closeMu sync.Mutex - - // Whether EnterExec has not yet returned. - inExec atomic.Bool -} - -// extIdent is a [Kind] concatenated with [ID]. -type extIdent [wordSize + len(ID{})]byte - -// getIdentBuf returns the address of an extIdent for Ident. -func (ic *irCache) getIdentBuf() *extIdent { return ic.identPool.Get().(*extIdent) } - -// putIdentBuf adds buf to identPool. -func (ic *irCache) putIdentBuf(buf *extIdent) { ic.identPool.Put(buf) } - -// storeIdent adds an [Artifact] to the artifact cache. -func (ic *irCache) storeIdent(a Artifact, buf *extIdent) unique.Handle[ID] { - idu := unique.Make(ID(buf[wordSize:])) - ic.artifact.Store(a, idu) - return idu -} - -// Ident returns the identifier of an [Artifact]. -func (ic *irCache) Ident(a Artifact) unique.Handle[ID] { - buf, idu := ic.unsafeIdent(a, false) - if buf != nil { - idu = ic.storeIdent(a, buf) - ic.putIdentBuf(buf) - } - return idu -} - -// unsafeIdent implements Ident but returns the underlying buffer for a newly -// computed identifier. Callers must return this buffer to identPool. encodeKind -// is only a hint, kind may still be encoded in the buffer. -func (ic *irCache) unsafeIdent(a Artifact, encodeKind bool) ( - buf *extIdent, - idu unique.Handle[ID], -) { - if id, ok := ic.artifact.Load(a); ok { - idu = id.(unique.Handle[ID]) - return - } - - if ki, ok := a.(KnownIdent); ok { - buf = ic.getIdentBuf() - if encodeKind { - binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind())) - } - *(*ID)(buf[wordSize:]) = ki.ID() - return - } - - buf = ic.getIdentBuf() - h := sha512.New384() - if err := ic.Encode(h, a); err != nil { - // unreachable - panic(err) - } - binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind())) - h.Sum(buf[wordSize:wordSize]) - return -} - -// getReader is like [bufio.NewReader] but for brPool. -func (c *Cache) getReader(r io.Reader) *bufio.Reader { - br := c.brPool.Get().(*bufio.Reader) - br.Reset(r) - return br -} - -// putReader adds br to brPool. -func (c *Cache) putReader(br *bufio.Reader) { c.brPool.Put(br) } - -// bufioReadCloser is the concrete type of value returned by Cache.getReaderRC. -type bufioReadCloser struct { - // Saved close error. - closeErr error - // Synchronises calls to Close. - closeOnce sync.Once - - // For backing freelist. - c *Cache - // Underlying reader. - r io.ReadCloser - // Allocated from c. - *bufio.Reader -} - -// Close closes the underlying reader, saves its return value, and returns the -// [bufio.Reader] instance to the backing [Cache]. -func (brc *bufioReadCloser) Close() error { - brc.closeOnce.Do(func() { - br := brc.Reader - brc.Reader = nil - brc.c.putReader(br) - brc.closeErr = brc.r.Close() - }) - return brc.closeErr -} - -// getReaderRC is like getReader, but returns an [io.ReadCloser]. -func (c *Cache) getReaderRC(r io.ReadCloser) io.ReadCloser { - return &bufioReadCloser{c: c, r: r, Reader: c.getReader(r)} -} - -// getWriter is like [bufio.NewWriter] but for bwPool. -func (c *Cache) getWriter(w io.Writer) *bufio.Writer { - bw := c.bwPool.Get().(*bufio.Writer) - bw.Reset(w) - return bw -} - -// putWriter adds bw to bwPool. -func (c *Cache) putWriter(bw *bufio.Writer) { c.bwPool.Put(bw) } - -// A ChecksumMismatchError describes an [Artifact] with unexpected content. -type ChecksumMismatchError struct { - // Actual and expected checksums. - Got, Want Checksum -} - -func (e *ChecksumMismatchError) Error() string { - return "got " + Encode(e.Got) + - " instead of " + Encode(e.Want) -} - -// LinknamePrefixError describes a malformed linkname to a [Checksum]. -type LinknamePrefixError string - -func (e LinknamePrefixError) Error() string { - return "linkname " + strconv.Quote(string(e)) + " missing prefix" -} - -// readlinkChecksum reads a symbolic link to a dirChecksum entry and saves the -// decoded [Checksum] to the value pointed to by buf. The checksumLinknamePrefix -// is required. -func readlinkChecksum(a *check.Absolute, buf *Checksum) error { - linkname, err := os.Readlink(a.String()) - if err != nil { - return nil - } - - if !strings.HasPrefix(linkname, checksumLinknamePrefix) { - return LinknamePrefixError(linkname) - } - return Decode(buf, linkname[len(checksumLinknamePrefix):]) -} - -// SetExternal sets e as the [External] implementation of c. -func (c *Cache) SetExternal(e External) { - c.externMu.Lock() - c.externCache = make(map[unique.Handle[ID]]unique.Handle[Checksum]) - c.extern = e - c.externMu.Unlock() -} - -// ScrubError describes the outcome of a [Cache.Scrub] call where errors were -// found and removed from the underlying storage of [Cache]. -type ScrubError struct { - // Content-addressed entries not matching their checksum. This can happen - // if an incorrect [FileArtifact] implementation was cured against - // a non-strict [Cache]. - ChecksumMismatches []ChecksumMismatchError - // Dangling identifier symlinks. This can happen if the content-addressed - // entry was removed while scrubbing due to a checksum mismatch. - DanglingIdentifiers []ID - // Dangling status files. This can happen if a dangling status symlink was - // removed while scrubbing. - DanglingStatus []ID - // Miscellaneous errors, including [os.ReadDir] on checksum and identifier - // directories, [Decode] on entry names and [os.RemoveAll] on inconsistent - // entries. - Errs map[unique.Handle[string]][]error -} - -// errs is a deterministic iterator over Errs. -func (e *ScrubError) errs(yield func(unique.Handle[string], []error) bool) { - keys := slices.AppendSeq( - make([]unique.Handle[string], 0, len(e.Errs)), - maps.Keys(e.Errs), - ) - slices.SortFunc(keys, func(a, b unique.Handle[string]) int { - return strings.Compare(a.Value(), b.Value()) - }) - for _, key := range keys { - if !yield(key, e.Errs[key]) { - break - } - } -} - -// Unwrap returns a concatenation of ChecksumMismatches and Errs. -func (e *ScrubError) Unwrap() []error { - s := make([]error, 0, len(e.ChecksumMismatches)+len(e.Errs)) - for _, err := range e.ChecksumMismatches { - s = append(s, &err) - } - for _, errs := range e.errs { - s = append(s, errs...) - } - return s -} - -// Error returns a multi-line representation of [ScrubError]. -func (e *ScrubError) Error() string { - var segments []string - var buf strings.Builder - - if len(e.ChecksumMismatches) > 0 { - buf.Reset() - buf.WriteString("checksum mismatches:\n") - for _, m := range e.ChecksumMismatches { - buf.WriteString(m.Error() + "\n") - } - segments = append(segments, buf.String()) - } - if len(e.DanglingIdentifiers) > 0 { - buf.Reset() - buf.WriteString("dangling identifiers:\n") - for _, id := range e.DanglingIdentifiers { - buf.WriteString(Encode(id) + "\n") - } - segments = append(segments, buf.String()) - } - if len(e.DanglingStatus) > 0 { - buf.Reset() - buf.WriteString("dangling status:\n") - for _, id := range e.DanglingStatus { - buf.WriteString(Encode(id) + "\n") - } - segments = append(segments, buf.String()) - } - if len(e.Errs) > 0 { - buf.Reset() - buf.WriteString("errors during scrub:\n") - for pathname, errs := range e.errs { - buf.WriteString(" " + pathname.Value() + ":\n") - for _, err := range errs { - buf.WriteString(" " + err.Error() + "\n") - } - } - segments = append(segments, buf.String()) - } - return strings.Join(segments, "\n") -} - -// Scrub frees internal in-memory identifier to content pair cache, verifies all -// cached artifacts against their checksums, checks for dangling identifier -// symlinks and removes them if found. -// -// This method is not safe for concurrent use with any other method. -func (c *Cache) Scrub(checks int) error { - if checks <= 0 { - checks = runtime.NumCPU() - } - - c.substituteMu.Lock() - defer c.substituteMu.Unlock() - c.identMu.Lock() - defer c.identMu.Unlock() - c.checksumMu.Lock() - defer c.checksumMu.Unlock() - - c.substitute = make(map[unique.Handle[ID]]unique.Handle[Checksum]) - c.ident = make(map[unique.Handle[ID]]unique.Handle[Checksum]) - c.identErr = make(map[unique.Handle[ID]]error) - c.artifact.Clear() - - var ( - se = ScrubError{Errs: make(map[unique.Handle[string]][]error)} - seMu sync.Mutex - - addErr = func(pathname *check.Absolute, err error) { - seMu.Lock() - se.Errs[pathname.Handle()] = append(se.Errs[pathname.Handle()], err) - seMu.Unlock() - } - ) - - type checkEntry struct { - ent os.DirEntry - check func(ent os.DirEntry, want *Checksum) bool - } - var ( - dir *check.Absolute - wg sync.WaitGroup - w = make(chan checkEntry, checks) - p = sync.Pool{New: func() any { return new(Checksum) }} - ) - condemn := func(ent os.DirEntry) { - pathname := dir.Append(ent.Name()) - chmodErr, removeErr := removeAll(pathname) - if chmodErr != nil { - addErr(pathname, chmodErr) - } - if removeErr != nil { - addErr(pathname, removeErr) - } - } - for i := 0; i < checks; i++ { - go func() { - for ce := range w { - want := p.Get().(*Checksum) - ent := ce.ent - if err := Decode(want, ent.Name()); err != nil { - addErr(dir.Append(ent.Name()), err) - wg.Go(func() { condemn(ent) }) - } else if !ce.check(ent, want) { - wg.Go(func() { condemn(ent) }) - } else { - c.msg.Verbosef( - "%s%s%s is consistent", - c.sgrIdent, ent.Name(), c.sgrRes, - ) - } - p.Put(want) - wg.Done() - } - }() - } - defer close(w) - - dir = c.base.Append(dirChecksum) - if entries, readdirErr := os.ReadDir(dir.String()); readdirErr != nil { - addErr(dir, readdirErr) - } else { - wg.Add(len(entries)) - for _, ent := range entries { - w <- checkEntry{ent, func(ent os.DirEntry, want *Checksum) bool { - got := p.Get().(*Checksum) - defer p.Put(got) - - pathname := dir.Append(ent.Name()) - if ent.IsDir() { - if err := SumDir(got, pathname); err != nil { - addErr(pathname, err) - return true - } - } else if ent.Type().IsRegular() { - h := sha512.New384() - - if r, err := os.Open(pathname.String()); err != nil { - addErr(pathname, err) - return true - } else { - _, err = io.Copy(h, r) - closeErr := r.Close() - if closeErr != nil { - addErr(pathname, closeErr) - } - if err != nil { - addErr(pathname, err) - } - } - h.Sum(got[:0]) - } else { - addErr(pathname, InvalidFileModeError(ent.Type())) - return false - } - - if *got != *want { - seMu.Lock() - se.ChecksumMismatches = append(se.ChecksumMismatches, - ChecksumMismatchError{Got: *got, Want: *want}, - ) - seMu.Unlock() - return false - } - return true - }} - } - wg.Wait() - } - - for _, suffix := range []string{ - dirSubstitute, - dirIdentifier, - } { - dir = c.base.Append(suffix) - if entries, readdirErr := os.ReadDir(dir.String()); readdirErr != nil { - addErr(dir, readdirErr) - } else { - wg.Add(len(entries)) - for _, ent := range entries { - w <- checkEntry{ent, func(ent os.DirEntry, want *Checksum) bool { - got := p.Get().(*Checksum) - defer p.Put(got) - - pathname := dir.Append(ent.Name()) - if linkname, err := os.Readlink( - pathname.String(), - ); err != nil { - seMu.Lock() - se.Errs[pathname.Handle()] = append(se.Errs[pathname.Handle()], err) - se.DanglingIdentifiers = append(se.DanglingIdentifiers, *want) - seMu.Unlock() - return false - } else if err = Decode(got, filepath.Base(linkname)); err != nil { - seMu.Lock() - lnp := dir.Append(linkname) - se.Errs[lnp.Handle()] = append(se.Errs[lnp.Handle()], err) - se.DanglingIdentifiers = append(se.DanglingIdentifiers, *want) - seMu.Unlock() - return false - } - - if _, err := os.Stat(pathname.String()); err != nil { - if !errors.Is(err, os.ErrNotExist) { - addErr(pathname, err) - } - seMu.Lock() - se.DanglingIdentifiers = append(se.DanglingIdentifiers, *want) - seMu.Unlock() - return false - } - return true - }} - } - wg.Wait() - } - } - - dir = c.base.Append(dirStatus) - if entries, readdirErr := os.ReadDir(dir.String()); readdirErr != nil { - if !errors.Is(readdirErr, os.ErrNotExist) { - addErr(dir, readdirErr) - } - } else { - wg.Add(len(entries)) - for _, ent := range entries { - w <- checkEntry{ent, func(ent os.DirEntry, want *Checksum) bool { - got := p.Get().(*Checksum) - defer p.Put(got) - - var ok bool - for _, name := range [...]string{ - dirIdentifier, - dirSubstitute, - } { - if _, err := os.Stat(c.base.Append( - name, - ent.Name(), - ).String()); err != nil { - if !errors.Is(err, os.ErrNotExist) { - addErr(dir.Append(ent.Name()), err) - } - continue - } - ok = true - } - if !ok { - seMu.Lock() - se.DanglingStatus = append(se.DanglingStatus, *want) - seMu.Unlock() - } - return ok - }} - } - wg.Wait() - } - - if len(c.identPending) > 0 { - addErr(c.base, errors.New( - "scrub began with pending artifacts", - )) - } else { - pathname := c.base.Append(dirWork) - chmodErr, removeErr := removeAll(pathname) - if chmodErr != nil { - addErr(pathname, chmodErr) - } - if removeErr != nil { - addErr(pathname, removeErr) - } - - if err := os.Mkdir(pathname.String(), 0700); err != nil { - addErr(pathname, err) - } - - pathname = c.base.Append(dirTemp) - chmodErr, removeErr = removeAll(pathname) - if chmodErr != nil { - addErr(pathname, chmodErr) - } - if removeErr != nil { - addErr(pathname, removeErr) - } - } - - if len(se.ChecksumMismatches) > 0 || - len(se.DanglingIdentifiers) > 0 || - len(se.DanglingStatus) > 0 || - len(se.Errs) > 0 { - slices.SortFunc(se.ChecksumMismatches, func(a, b ChecksumMismatchError) int { - return bytes.Compare(a.Want[:], b.Want[:]) - }) - slices.SortFunc(se.DanglingIdentifiers, func(a, b ID) int { - return bytes.Compare(a[:], b[:]) - }) - slices.SortFunc(se.DanglingStatus, func(a, b ID) int { - return bytes.Compare(a[:], b[:]) - }) - return &se - } else { - return nil - } -} - -// loadOrStoreIdent attempts to load a cached [Artifact] by its identifier or -// wait for a pending [Artifact] to cure. If neither is possible, the current -// identifier is stored in identPending and a non-nil channel is returned. -// -// Since identErr is treated as grow-only, loadOrStoreIdent must not be entered -// without holding a read lock on abortMu. -func (c *Cache) loadOrStoreIdent(id unique.Handle[ID]) ( - ctx context.Context, - done chan<- struct{}, - checksum unique.Handle[Checksum], - err error, -) { - var ok bool - - c.identMu.Lock() - if checksum, ok = c.ident[id]; ok { - c.identMu.Unlock() - return - } - if err, ok = c.identErr[id]; ok { - c.identMu.Unlock() - return - } - - var pending *pendingCure - if pending, ok = c.identPending[id]; ok { - c.identMu.Unlock() - <-pending.done - c.identMu.RLock() - if checksum, ok = c.ident[id]; !ok { - err = pending.err - } - c.identMu.RUnlock() - return - } - - d := make(chan struct{}) - pending = &pendingCure{done: d} - ctx, pending.cancel = context.WithCancel(c.toplevel.Load().ctx) - c.wg.Add(1) - c.identPending[id] = pending - c.identMu.Unlock() - done = d - return -} - -// finaliseIdent commits a checksum or error to ident for an identifier -// previously submitted to identPending. -func (c *Cache) finaliseIdent( - done chan<- struct{}, - id unique.Handle[ID], - checksum unique.Handle[Checksum], - err error, -) { - c.identMu.Lock() - if err != nil { - c.identPending[id].err = err - c.identErr[id] = err - } else { - c.ident[id] = checksum - } - delete(c.identPending, id) - c.identMu.Unlock() - c.wg.Done() - - close(done) -} - -// zeroChecksum is a zero [Checksum] handle, used for comparison only. -var zeroChecksum unique.Handle[Checksum] - -// loadSubstitute returns a checksum corresponding to a substitute identifier, -// or zeroChecksum if an alternative is not available. -func (c *Cache) loadSubstitute( - substitute unique.Handle[ID], -) (unique.Handle[Checksum], error) { - c.substituteMu.RLock() - if checksum, ok := c.substitute[substitute]; ok { - c.substituteMu.RUnlock() - return checksum, nil - } - - linkname, err := os.Readlink(c.base.Append( - dirSubstitute, - Encode(substitute.Value()), - ).String()) - c.substituteMu.RUnlock() - - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return zeroChecksum, err - } - - c.substituteMu.Lock() - c.substitute[substitute] = zeroChecksum - c.substituteMu.Unlock() - return zeroChecksum, nil - } - - var checksum unique.Handle[Checksum] - buf := c.getIdentBuf() - err = Decode((*Checksum)(buf[:]), filepath.Base(linkname)) - if err == nil { - checksum = unique.Make(Checksum(buf[:])) - - c.substituteMu.Lock() - c.substitute[substitute] = checksum - c.substituteMu.Unlock() - } - c.putIdentBuf(buf) - - return checksum, err -} - -// Done returns a channel that is closed when the ongoing cure of an [Artifact] -// referred to by the specified identifier completes. Done may return nil if -// no ongoing cure of the specified identifier exists. -func (c *Cache) Done(id unique.Handle[ID]) <-chan struct{} { - c.identMu.RLock() - pending, ok := c.identPending[id] - c.identMu.RUnlock() - if !ok || pending == nil { - return nil - } - return pending.done -} - -// Cancel cancels the ongoing cure of an [Artifact] referred to by the specified -// identifier. Cancel returns whether the [context.CancelFunc] has been killed. -// Cancel returns after the cure is complete. -func (c *Cache) Cancel(id unique.Handle[ID]) bool { - c.identMu.RLock() - pending, ok := c.identPending[id] - c.identMu.RUnlock() - if !ok || pending == nil || pending.cancel == nil { - return false - } - pending.cancel() - <-pending.done - - c.abortMu.Lock() - c.identMu.Lock() - delete(c.identErr, id) - c.identMu.Unlock() - c.abortMu.Unlock() - return true -} - -// openFile tries to load [FileArtifact] from [Cache], and if that fails, -// obtains it via [FileArtifact.Cure] instead. Notably, it does not cure -// [FileArtifact] to the filesystem. If err is nil, the caller is responsible -// for closing the resulting [io.ReadCloser]. -// -// The context must originate from loadOrStoreIdent to enable cancellation. -func (c *Cache) openFile( - ctx context.Context, - f FileArtifact, -) (r io.ReadCloser, err error) { - if kc, ok := f.(KnownChecksum); c.attr.Flags&CAssumeChecksum != 0 && ok { - c.checksumMu.RLock() - r, err = os.Open(c.base.Append( - dirChecksum, - Encode(kc.Checksum()), - ).String()) - c.checksumMu.RUnlock() - } else { - c.identMu.RLock() - r, err = os.Open(c.base.Append( - dirIdentifier, - Encode(c.Ident(f).Value()), - ).String()) - c.identMu.RUnlock() - } - - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return - } - id := c.Ident(f) - if c.msg.IsVerbose() { - rn := reportName(f, id) - c.msg.Verbosef("curing %s%s%s in memory...", c.sgrIdent, rn, c.sgrRes) - defer func() { - if err == nil { - c.msg.Verbosef("opened %s%s%s for reading", c.sgrIdent, rn, c.sgrRes) - } - }() - } - return f.Cure(&RContext{common{ctx, c}}) - } - return -} - -// InvalidFileModeError describes a [FloodArtifact.Cure] or -// [TrivialArtifact.Cure] that did not result in a regular file or directory -// located at the work pathname. -type InvalidFileModeError fs.FileMode - -// Error returns a constant string. -func (e InvalidFileModeError) Error() string { - return "artifact did not produce a regular file or directory" -} - -// NoOutputError describes a [FloodArtifact.Cure] or [TrivialArtifact.Cure] -// that did not populate its work pathname despite completing successfully. -type NoOutputError struct{} - -// Unwrap returns [os.ErrNotExist]. -func (NoOutputError) Unwrap() error { return os.ErrNotExist } - -// Error returns a constant string. -func (NoOutputError) Error() string { - return "artifact cured successfully but did not produce any output" -} - -// removeAll is similar to [os.RemoveAll] but is robust against any permissions. -func removeAll(pathname *check.Absolute) (chmodErr, removeErr error) { - chmodErr = filepath.WalkDir(pathname.String(), func( - path string, - d fs.DirEntry, - err error, - ) error { - if err != nil { - return err - } - if d.IsDir() { - return os.Chmod(path, 0700) - } - return nil - }) - if errors.Is(chmodErr, os.ErrNotExist) { - chmodErr = nil - } - removeErr = os.RemoveAll(pathname.String()) - return -} - -// zeroTimes zeroes atime and mtime for the named file. -func zeroTimes(path string) (err error) { - // include/uapi/linux/fcntl.h - const ( - AT_FDCWD = -100 - AT_SYMLINK_NOFOLLOW = 0x100 - ) - _AT_FDCWD := AT_FDCWD - - var _p0 *byte - _p0, err = syscall.BytePtrFromString(path) - if err != nil { - return - } - if _, _, errno := syscall.Syscall6( - syscall.SYS_UTIMENSAT, - uintptr(_AT_FDCWD), - uintptr(unsafe.Pointer(_p0)), - uintptr(unsafe.Pointer(new([2]syscall.Timespec))), - AT_SYMLINK_NOFOLLOW, - 0, 0, - ); errno != 0 { - return os.NewSyscallError("utimensat", errno) - } - return -} - -// overrideFileInfo overrides the permission bits of [fs.FileInfo] to 0500 and -// is the concrete type returned by overrideFile.Stat. -type overrideFileInfo struct{ fs.FileInfo } - -// Mode returns [fs.FileMode] with its permission bits set to 0500. -func (fi overrideFileInfo) Mode() fs.FileMode { - return fi.FileInfo.Mode()&(^fs.FileMode(0777)) | 0500 -} - -// Sys returns nil to avoid passing the original permission bits. -func (fi overrideFileInfo) Sys() any { return nil } - -// overrideFile overrides the permission bits of [fs.File] to 0500 and is the -// concrete type returned by dotOverrideFS for calls with "." passed as name. -type overrideFile struct{ fs.File } - -func (f overrideFile) Stat() (fi fs.FileInfo, err error) { - fi, err = f.File.Stat() - if err != nil { - return - } - fi = overrideFileInfo{fi} - return -} - -// dirFS is implemented by the concrete type of the return value of [os.DirFS]. -type dirFS interface { - fs.StatFS - fs.ReadFileFS - fs.ReadDirFS - fs.ReadLinkFS -} - -// dotOverrideFS overrides the permission bits of "." to 0500 to avoid the extra -// system calls to add and remove write bit from the target directory. -type dotOverrideFS struct{ dirFS } - -// Open wraps the underlying [fs.FS] with "." special case. -func (fsys dotOverrideFS) Open(name string) (f fs.File, err error) { - f, err = fsys.dirFS.Open(name) - if err != nil || name != "." { - return - } - f = overrideFile{f} - return -} - -// Stat wraps the underlying [fs.FS] with "." special case. -func (fsys dotOverrideFS) Stat(name string) (fi fs.FileInfo, err error) { - fi, err = fsys.dirFS.Stat(name) - if err != nil || name != "." { - return - } - fi = overrideFileInfo{fi} - return -} - -// InvalidArtifactError describes an artifact that does not implement a -// supported Cure method. -type InvalidArtifactError ID - -func (e InvalidArtifactError) Error() string { - return "artifact " + Encode(e) + " cannot be cured" -} - -// Cure cures the [Artifact] and returns its pathname and [Checksum]. Direct -// calls to Cure are not subject to the cures limit. -func (c *Cache) Cure(a Artifact) ( - pathname *check.Absolute, - checksum unique.Handle[Checksum], - err error, -) { - c.abortMu.RLock() - defer c.abortMu.RUnlock() - - if err = c.toplevel.Load().ctx.Err(); err != nil { - return - } - - pathname, checksum, _, err = c.cure(a, true, false) - return -} - -// CureWhence is like Cure, but returns the whence value. -func (c *Cache) CureWhence(a Artifact) ( - pathname *check.Absolute, - checksum unique.Handle[Checksum], - whence int, - err error, -) { - c.abortMu.RLock() - defer c.abortMu.RUnlock() - - if err = c.toplevel.Load().ctx.Err(); err != nil { - return - } - - return c.cure(a, true, false) -} - -// CureNew is like Cure, but always enters the implementation. -func (c *Cache) CureNew(a Artifact) ( - pathname *check.Absolute, - checksum unique.Handle[Checksum], - err error, -) { - c.abortMu.RLock() - defer c.abortMu.RUnlock() - - if err = c.toplevel.Load().ctx.Err(); err != nil { - return - } - - var whence int -retry: - pathname, checksum, whence, err = c.cure(a, true, true) - if err != nil || whence == WNew { - return - } - goto retry -} - -// An InputError describes inputs of a [FloodArtifact] which had failed to cure. -type InputError map[Artifact]error - -// unwrap returns an iterator over sorted, deduplicated [Artifact] and their -// corresponding identifier. -func (e InputError) unwrap() iter.Seq2[Artifact, unique.Handle[ID]] { - ir := NewIR() - - type input struct { - a Artifact - id unique.Handle[ID] - } - p := make([]input, 0, len(e)) - for a := range e { - p = append(p, input{a, ir.Ident(a)}) - } - - var identBuf [2]ID - slices.SortFunc(p, func(a, b input) int { - identBuf[0], identBuf[1] = a.id.Value(), b.id.Value() - return slices.Compare(identBuf[0][:], identBuf[1][:]) - }) - p = slices.CompactFunc(p, func(a, b input) bool { return a.id == b.id }) - - return func(yield func(Artifact, unique.Handle[ID]) bool) { - for _, i := range p { - if !yield(i.a, i.id) { - return - } - } - } -} - -// Error returns a user-facing, deterministic text representation of e. -func (e InputError) Error() string { - var buf strings.Builder - buf.WriteString("errors curing inputs:") - for a, id := range e.unwrap() { - buf.WriteString("\n\t") - buf.WriteString(reportName(a, id)) - buf.WriteString(": ") - buf.WriteString(e[a].Error()) - } - return buf.String() -} - -// Unwrap returns a slice of underlying errors sorted by identifier. -func (e InputError) Unwrap() []error { - errs := make([]error, 0, len(e)) - for a := range e.unwrap() { - errs = append(errs, e[a]) - } - return errs -} - -// enterCure must be called before entering an [Artifact] implementation. -func (c *Cache) enterCure(a Artifact, curesExempt bool) error { - if c.attr.Notify != nil { - c.attr.Notify <- true - } - - if a.IsExclusive() { - c.exclMu.Lock() - } - if curesExempt { - return nil - } - - ctx := c.toplevel.Load().ctx - select { - case c.cures <- struct{}{}: - return nil - - case <-ctx.Done(): - if a.IsExclusive() { - c.exclMu.Unlock() - } - return ctx.Err() - } -} - -// exitCure must be called after exiting an [Artifact] implementation. -func (c *Cache) exitCure(a Artifact, curesExempt bool) { - if c.attr.Notify != nil { - c.attr.Notify <- false - } - - if a.IsExclusive() { - c.exclMu.Unlock() - } - if curesExempt { - return - } - - <-c.cures -} - -// measuredReader implements [io.ReadCloser] and measures the checksum during -// Close. If the underlying reader is not read to EOF, Close blocks until all -// remaining data is consumed and validated. -type measuredReader struct { - // Underlying reader. Never exposed directly. - r io.ReadCloser - // For validating checksum. Never exposed directly. - h hash.Hash - // Buffers writes to h, initialised by [Cache]. Never exposed directly. - hbw *bufio.Writer - // Expected checksum, compared during Close. - want unique.Handle[Checksum] - - // For accessing free lists. - c *Cache - - // Set up via [io.TeeReader] by [Cache]. - io.Reader -} - -// Close reads the underlying [io.ReadCloser] to EOF, closes it and measures its -// outcome. It returns a [ChecksumMismatchError] for an unexpected checksum. -func (mr *measuredReader) Close() (err error) { - if mr.hbw == nil || mr.Reader == nil { - return os.ErrInvalid - } - err = mr.hbw.Flush() - mr.c.putWriter(mr.hbw) - mr.hbw, mr.Reader = nil, nil - if err != nil { - _ = mr.r.Close() - return - } - var n int64 - if n, err = io.Copy(mr.h, mr.r); err != nil { - _ = mr.r.Close() - return - } - - if n > 0 { - mr.c.msg.Verbosef( - "%smissed %d bytes on measured reader%s", - mr.c.sgrWarn, n, mr.c.sgrRes, - ) - } - - if err = mr.r.Close(); err != nil { - return - } - - buf := mr.c.getIdentBuf() - mr.h.Sum(buf[:0]) - - if got := Checksum(buf[:]); got != mr.want.Value() { - err = &ChecksumMismatchError{ - Got: got, - Want: mr.want.Value(), - } - } - - mr.c.putIdentBuf(buf) - return -} - -// newMeasuredReader implements [RContext.NewMeasuredReader]. -func (c *Cache) newMeasuredReader( - r io.ReadCloser, - checksum unique.Handle[Checksum], -) io.ReadCloser { - mr := measuredReader{r: r, h: sha512.New384(), want: checksum, c: c} - mr.hbw = c.getWriter(mr.h) - mr.Reader = io.TeeReader(r, mr.hbw) - return &mr -} - -// NewMeasuredReader returns an [io.ReadCloser] implementing behaviour required -// by [FileArtifact]. The resulting [io.ReadCloser] holds a buffer originating -// from [Cache] and must be closed to return this buffer. -func (r *RContext) NewMeasuredReader( - rc io.ReadCloser, - checksum unique.Handle[Checksum], -) io.ReadCloser { - return r.cache.newMeasuredReader(rc, checksum) -} - -// tryChecksum dereferences a symlink to a cure outcome. -func (c *Cache) tryChecksum(pathname *check.Absolute) ( - checksum unique.Handle[Checksum], - err error, -) { - _, err = os.Lstat(pathname.String()) - if err == nil { - var name string - if name, err = os.Readlink(pathname.String()); err != nil { - return - } - buf := c.getIdentBuf() - err = Decode((*Checksum)(buf[:]), filepath.Base(name)) - if err == nil { - checksum = unique.Make(Checksum(buf[:])) - } - c.putIdentBuf(buf) - } - return -} - -// tryLocal attempts to obtain an [Artifact] outcome from the filesystem. -func (c *Cache) tryLocal(id unique.Handle[ID]) (unique.Handle[Checksum], error) { - return c.tryChecksum(c.base.Append( - dirIdentifier, - Encode(id.Value()), - )) -} - -// tryExtern attempts to obtain an [Artifact] outcome from extern. -func (c *Cache) tryExtern(ctx context.Context, id unique.Handle[ID]) ( - unique.Handle[Checksum], - error, -) { - c.externMu.RLock() - defer c.externMu.RUnlock() - - checksum, ok := c.externCache[id] - if !ok { - if c.extern == nil { - return zeroChecksum, nil - } - - v, err := c.extern.Artifact(ctx, id) - if err != nil { - return zeroChecksum, err - } - if v == nil { - return zeroChecksum, nil - } - checksum = unique.Make(*v) - - var got unique.Handle[Checksum] - if _, got, err = c.Cure(c.extern.Checksum(checksum)); err != nil { - return checksum, err - } else if got != checksum { - return zeroChecksum, &ChecksumMismatchError{got.Value(), checksum.Value()} - } - } - return checksum, nil -} - -// cureMany concurrently collects outcome of multiple [Artifact]. -func (c *Cache) cureMany( - inputs []Artifact, - r map[Artifact]cureRes, - shallow bool, -) ([]bool, error) { - var wg sync.WaitGroup - wg.Add(len(inputs)) - var mask []bool - res := make([]cureRes, len(inputs)) - errs := make(InputError) - var errsMu sync.Mutex - if shallow { - mask = make([]bool, len(inputs)) - } - for i, d := range inputs { - if shallow { - if _, ok := d.(FloodArtifact); ok { - mask[i] = true - wg.Done() - continue - } - - if kc, ok := d.(KnownChecksum); ok { - res[i].checksum = unique.Make(kc.Checksum()) - wg.Done() - continue - } - } - pending := pendingArtifactDep{d, &res[i], errs, &errsMu, &wg} - go pending.cure(c) - } - wg.Wait() - - if len(errs) > 0 { - return mask, errs - } - for i, p := range res { - if shallow && mask[i] { - continue - } - r[inputs[i]] = p - } - return mask, nil -} - -// HangingInputError describes an input of an [Artifact] on a sparse cache -// without an outcome available locally or via [External]. -type HangingInputError unique.Handle[ID] - -func (e HangingInputError) Error() string { - return Encode(unique.Handle[ID](e).Value()) + " is unavailable" -} - -const ( - // WNew indicates a cure entering the implementation. - WNew = iota - // WCache indicates an [Artifact] present in the cache. - WCache - // WSubstitute indicates an [Artifact] hitting a content-based input - // substitution. - WSubstitute - // WExternal indicates a cure entering the external cache. - WExternal -) - -// WhenceString returns a printable string for a whence value. -func WhenceString(whence int) string { - switch whence { - case WNew: - return "new" - case WCache: - return "cache" - case WSubstitute: - return "substitute" - case WExternal: - return "external" - - default: - return "invalid whence " + strconv.Itoa(whence) - } -} - -// cure implements Cure without acquiring a read lock on abortMu. cure must not -// be entered during Abort. -func (c *Cache) cure(a Artifact, curesExempt, rebuild bool) ( - pathname *check.Absolute, - checksum unique.Handle[Checksum], - whence int, - err error, -) { - id := c.Ident(a) - if rebuild { - var v ID - _, _ = rand.Read(v[:]) - id = unique.Make(v) - } - - ids := Encode(id.Value()) - pathname = c.base.Append( - dirIdentifier, - ids, - ) - defer func() { - if err != nil { - pathname = nil - checksum = unique.Handle[Checksum]{} - } - }() - - if _, ok := a.(CuresExempt); ok { - curesExempt = true - } - - var ( - ctx context.Context - done chan<- struct{} - ) - ctx, done, checksum, err = c.loadOrStoreIdent(id) - if done == nil { - whence = WCache - return - } else { - defer func() { c.finaliseIdent(done, id, checksum, err) }() - } - - checksum, err = c.tryChecksum(pathname) - if err == nil || !errors.Is(err, os.ErrNotExist) { - whence = WCache - return - } - - var ( - checksums string - substitute unique.Handle[ID] - alternative *check.Absolute - ) - defer func() { - if err == nil && checksums != "" { - linkname := checksumLinknamePrefix + checksums - - err = os.Symlink( - linkname, - pathname.String(), - ) - if err == nil { - err = zeroTimes(pathname.String()) - } - - if err == nil && alternative != nil && substitute != id { - c.substituteMu.Lock() - err = os.Symlink( - linkname, - alternative.String(), - ) - if errors.Is(err, os.ErrExist) { - c.msg.Verbosef( - "creating alternative over %s%s%s for artifact %s%s%s", - c.sgrIdent, Encode(substitute.Value()), c.sgrRes, - c.sgrIdent, ids, c.sgrRes, - ) - err = nil - } - if err == nil { - err = zeroTimes(alternative.String()) - } - if err == nil && checksum != zeroChecksum { - c.substitute[substitute] = checksum - } - c.substituteMu.Unlock() - } - } - }() - - var checksumPathname *check.Absolute - var checksumFi os.FileInfo - if kc, ok := a.(KnownChecksum); ok { - checksum = unique.Make(kc.Checksum()) - checksums = Encode(checksum.Value()) - checksumPathname = c.base.Append( - dirChecksum, - checksums, - ) - - if c.attr.Flags&CAssumeChecksum != 0 { - c.checksumMu.RLock() - checksumFi, err = os.Stat(checksumPathname.String()) - c.checksumMu.RUnlock() - - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return - } - - checksumFi, err = nil, nil - } - } - } - - whence = WNew - if c.msg.IsVerbose() { - rn := reportName(a, id) - c.msg.Verbosef("curing %s%s%s...", c.sgrIdent, rn, c.sgrRes) - defer func() { - if err != nil { - return - } - if checksums != "" { - c.msg.Verbosef( - "cured %s%s%s checksum %s%s%s", - c.sgrIdent, rn, c.sgrRes, - c.sgrIdent, checksums, c.sgrRes) - } else { - c.msg.Verbosef("cured %s%s%s", c.sgrIdent, rn, c.sgrRes) - } - }() - } - - // cure FileArtifact outside type switch to skip TContext initialisation - if f, ok := a.(FileArtifact); ok { - if checksumFi != nil { - whence = WCache - if !checksumFi.Mode().IsRegular() { - // unreachable - err = InvalidFileModeError(checksumFi.Mode()) - } - return - } - - perm := os.FileMode(0400) - if f.IsExecutable() { - perm = 0500 - } - - work := c.base.Append(dirWork, ids) - var w *os.File - if w, err = os.OpenFile( - work.String(), - os.O_CREATE|os.O_EXCL|os.O_WRONLY, - perm, - ); err != nil { - return - } - defer func() { - closeErr := w.Close() - if err == nil { - err = closeErr - } - - removeErr := os.Remove(work.String()) - if err == nil && !errors.Is(removeErr, os.ErrNotExist) { - err = removeErr - } - }() - - var r io.ReadCloser - if err = c.enterCure(a, curesExempt); err != nil { - return - } - r, err = f.Cure(&RContext{common{ctx, c}}) - if err == nil { - if checksumPathname == nil || c.attr.Flags&CValidateKnown != 0 { - h := sha512.New384() - hbw := c.getWriter(h) - _, err = io.Copy(w, io.TeeReader(r, hbw)) - flushErr := hbw.Flush() - c.putWriter(hbw) - if err == nil { - err = flushErr - } - - if err == nil { - buf := c.getIdentBuf() - h.Sum(buf[:0]) - - if checksumPathname == nil { - checksum = unique.Make(Checksum(buf[:])) - checksums = Encode(Checksum(buf[:])) - } else if c.attr.Flags&CValidateKnown != 0 { - if got := Checksum(buf[:]); got != checksum.Value() { - err = &ChecksumMismatchError{ - Got: got, - Want: checksum.Value(), - } - } - } - - c.putIdentBuf(buf) - - if checksumPathname == nil { - checksumPathname = c.base.Append( - dirChecksum, - checksums, - ) - } - } - } else { - _, err = io.Copy(w, r) - } - - closeErr := r.Close() - if err == nil { - err = closeErr - } - } - c.exitCure(a, curesExempt) - if err != nil { - if c.msg.IsVerbose() { - c.msg.Verbosef( - "cure file %s%s%s: %s%v%s", - c.sgrIdent, reportName(f, id), c.sgrRes, - c.sgrErr, err, c.sgrRes, - ) - } - return - } - - c.checksumMu.Lock() - if err = os.Rename( - work.String(), - checksumPathname.String(), - ); err != nil { - c.checksumMu.Unlock() - return - } - timeErr := zeroTimes(checksumPathname.String()) - c.checksumMu.Unlock() - - if err == nil { - err = timeErr - } - return - } - - if checksumFi != nil { - whence = WCache - if !checksumFi.Mode().IsDir() { - // unreachable - err = InvalidFileModeError(checksumFi.Mode()) - } - return - } - - t := TContext{ - c.base.Append(dirWork, ids), - c.base.Append(dirTemp, ids), - ids, nil, nil, nil, nil, - common{ctx, c}, - } - switch ca := a.(type) { - case TrivialArtifact: - defer t.destroy(&err) - if err = c.enterCure(a, curesExempt); err != nil { - return - } - err = ca.Cure(&t) - c.exitCure(a, curesExempt) - if err != nil { - if c.msg.IsVerbose() { - c.msg.Verbosef( - "cure trivial %s%s%s: %s%v%s", - c.sgrIdent, reportName(ca, id), c.sgrRes, - c.sgrErr, err, c.sgrRes, - ) - } - return - } - break - - case FloodArtifact: - var externChecksum unique.Handle[Checksum] - if externChecksum, err = c.tryExtern(ctx, id); err != nil { - if c.msg.IsVerbose() { - c.msg.Verbosef( - "extern %s%s%s: %s%v%s", - c.sgrIdent, reportName(ca, id), c.sgrRes, - c.sgrErr, err, c.sgrRes, - ) - } - return - } - extern := externChecksum != zeroChecksum - shallow := extern && c.attr.Flags&CExternShallow != 0 - - inputs := a.Inputs() - f := FContext{t, make(map[Artifact]cureRes, len(inputs))} - var mask []bool - if mask, err = c.cureMany(inputs, f.inputs, shallow); err != nil { - return - } - - if shallow { - for i, d := range inputs { - if !mask[i] { - continue - } - - if kc, ok := d.(KnownChecksum); ok { - f.inputs[d] = cureRes{checksum: unique.Make(kc.Checksum())} - continue - } - - var sum unique.Handle[Checksum] - did := c.Ident(d) - sum, err = c.tryLocal(did) - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return - } - - sum, err = c.tryExtern(ctx, did) - if err != nil { - return - } - if sum == zeroChecksum { - if c.msg.IsVerbose() { - c.msg.Verbosef( - "input %s%s%s not available", - c.sgrIdent, reportName(d, did), c.sgrRes, - ) - } - err = HangingInputError(did) - return - } - } - - f.inputs[d] = cureRes{checksum: sum} - } - } - - sh := sha512.New384() - err = c.encode(sh, a, f.inputs) - if err != nil { - return - } - - buf := c.getIdentBuf() - sh.Sum(buf[wordSize:wordSize]) - substitute = unique.Make(ID(buf[wordSize:])) - substitutes := Encode(substitute.Value()) - c.putIdentBuf(buf) - alternative = c.base.Append( - dirSubstitute, - substitutes, - ) - - if !rebuild && c.attr.Flags&CIgnoreSubstitutes == 0 { - var substituteChecksum unique.Handle[Checksum] - substituteChecksum, err = c.loadSubstitute(substitute) - if err != nil { - return - } - if substituteChecksum != zeroChecksum { - whence = WSubstitute - checksum = substituteChecksum - checksums = Encode(checksum.Value()) - checksumPathname = c.base.Append( - dirChecksum, - checksums, - ) - if _, err = os.Lstat(c.base.Append( - dirStatus, - substitutes, - ).String()); err == nil { - err = os.Symlink(substitutes, c.base.Append( - dirStatus, - ids, - ).String()) - } else if errors.Is(err, os.ErrNotExist) { - err = nil - } - return - } - } - - defer f.destroy(&err) - - if extern { - whence = WExternal - if checksum != zeroChecksum && externChecksum != checksum { - err = &ChecksumMismatchError{externChecksum.Value(), checksum.Value()} - if c.msg.IsVerbose() { - c.msg.Verbosef( - "extern %s%s%s: %s%v%s", - c.sgrIdent, reportName(ca, id), c.sgrRes, - c.sgrErr, err, c.sgrRes, - ) - } - return - } - - var externStatus io.ReadCloser - c.externMu.RLock() - externStatus, err = c.extern.Status(&RContext{common{ctx, c}}, id) - c.externMu.RUnlock() - if err != nil { - return - } - - checksum = externChecksum - checksums = Encode(checksum.Value()) - checksumPathname = c.base.Append( - dirChecksum, - checksums, - ) - - if externStatus != nil { - if err = f.prepareStatus(false); err != nil { - _ = externStatus.Close() - return - } else if _, err = io.Copy(f.status, externStatus); err != nil { - _ = externStatus.Close() - return - } else if err = externStatus.Close(); err != nil { - return - } else if !rebuild { - if err = f.linkSubstitute(ids, substitutes); err != nil { - return - } - } - } - return - } - - if err = c.enterCure(a, curesExempt); err != nil { - return - } - err = ca.Cure(&f) - c.exitCure(a, curesExempt) - - if !rebuild && err == nil { - err = f.linkSubstitute(ids, substitutes) - } - if err != nil { - if c.msg.IsVerbose() { - c.msg.Verbosef( - "cure %s%s%s: %s%v%s", - c.sgrIdent, reportName(ca, id), c.sgrRes, - c.sgrErr, err, c.sgrRes, - ) - } - return - } - break - - default: - err = InvalidArtifactError(id.Value()) - return - } - t.cache = nil - - var fi os.FileInfo - if fi, err = os.Lstat(t.work.String()); err != nil { - if errors.Is(err, os.ErrNotExist) { - err = NoOutputError{} - } - return - } - - if !fi.IsDir() { - if !fi.Mode().IsRegular() { - err = InvalidFileModeError(fi.Mode()) - } else { - err = errors.New("non-file artifact produced regular file") - } - return - } - - var gotChecksum Checksum - if err = SumFS( - &gotChecksum, - dotOverrideFS{os.DirFS(t.work.String()).(dirFS)}, - ".", - ); err != nil { - return - } - - if checksumPathname == nil { - checksum = unique.Make(gotChecksum) - checksums = Encode(gotChecksum) - checksumPathname = c.base.Append( - dirChecksum, - checksums, - ) - } else if gotChecksum != checksum.Value() { - err = &ChecksumMismatchError{ - Got: gotChecksum, - Want: checksum.Value(), - } - if c.msg.IsVerbose() { - c.msg.Verbosef( - "validate %s%s%s: %s%v%s", - c.sgrIdent, reportName(a, id), c.sgrRes, - c.sgrErr, err, c.sgrRes, - ) - } - return - } - - if err = os.Chmod(t.work.String(), 0700); err != nil { - return - } - if err = filepath.WalkDir(t.work.String(), func(path string, _ fs.DirEntry, err error) error { - if err != nil { - return err - } - return zeroTimes(path) - }); err != nil { - return - } - c.checksumMu.Lock() - if err = os.Rename( - t.work.String(), - checksumPathname.String(), - ); err != nil { - if !errors.Is(err, os.ErrExist) { - c.checksumMu.Unlock() - return - } - // err is zeroed during deferred cleanup - } else { - err = os.Chmod(checksumPathname.String(), 0500) - } - c.checksumMu.Unlock() - return -} - -// cure cures the pending [Artifact], stores its result and notifies the caller. -func (pending *pendingArtifactDep) cure(c *Cache) { - defer pending.Done() - - var err error - pending.resP.pathname, pending.resP.checksum, _, err = c.cure(pending.a, false, false) - if err == nil { - return - } - - pending.errsMu.Lock() - if errs, ok := err.(InputError); ok { - maps.Copy(pending.errs, errs) - } else { - pending.errs[pending.a] = err - } - pending.errsMu.Unlock() -} - -// OpenStatus attempts to open the status file associated to an [Artifact]. If -// err is nil, the caller must close the resulting reader. -func (c *Cache) OpenStatus(a Artifact) (r io.ReadSeekCloser, err error) { - c.identMu.RLock() - r, err = os.Open(c.base.Append( - dirStatus, - Encode(c.Ident(a).Value())).String(), - ) - c.identMu.RUnlock() - return -} - -// Fault holds the pathname and termination time of an [Artifact] fault entry. -type Fault struct { - *check.Absolute - t uint64 -} - -// Time returns the instant in time where the fault occurred. -func (f Fault) Time() time.Time { return time.Unix(0, int64(f.t)) } - -// Open opens the underlying entry for reading. -func (f Fault) Open() (io.ReadCloser, error) { return os.Open(f.Absolute.String()) } - -// Destroy removes the underlying fault entry. -func (f Fault) Destroy() error { return os.Remove(f.Absolute.String()) } - -// ReadFaults returns fault entries for an [Artifact]. -func (c *Cache) ReadFaults(a Artifact) (faults []Fault, err error) { - prefix := Encode(c.Ident(a).Value()) + "." - var dents []os.DirEntry - if dents, err = os.ReadDir(c.base.Append(dirFault).String()); err != nil { - return - } - - for _, dent := range dents { - name := dent.Name() - if !strings.HasPrefix(name, prefix) { - continue - } - var t uint64 - t, err = strconv.ParseUint(name[len(prefix):], 10, 64) - if err != nil { - return - } - - faults = append(faults, Fault{c.base.Append( - dirFault, - name, - ), t}) - } - - slices.SortFunc(faults, func(a, b Fault) int { - return cmp.Compare(a.t, b.t) - }) - return -} - -// Abort cancels all pending cures and waits for them to clean up, but does not -// close the cache. -func (c *Cache) Abort() { - c.closeMu.Lock() - defer c.closeMu.Unlock() - - if c.closed { - return - } - - c.toplevel.Load().cancel() - c.abortMu.Lock() - defer c.abortMu.Unlock() - - // holding abortMu, identPending stays empty - c.wg.Wait() - c.identMu.Lock() - c.toplevel.Store(newToplevel(c.parent)) - clear(c.identErr) - c.identMu.Unlock() -} - -// Close cancels all pending cures and waits for them to clean up. -func (c *Cache) Close() { - c.closeMu.Lock() - defer c.closeMu.Unlock() - - if c.closed { - return - } - - c.closed = true - c.toplevel.Load().cancel() - c.wg.Wait() - close(c.cures) - c.unlock() - - if c.attr.Notify != nil { - close(c.attr.Notify) - } -} - -// UnsupportedVariantError describes an on-disk cache with an extension variant -// identification string that differs from the value returned by [Extension]. -type UnsupportedVariantError string - -func (e UnsupportedVariantError) Error() string { - return "unsupported variant " + strconv.Quote(string(e)) -} - -var ( - // ErrWouldPromote is returned by [Open] if the [CPromoteVariant] bit is not - // set and the on-disk cache requires variant promotion. - ErrWouldPromote = errors.New("operation would promote unextended cache") -) - -// CacheAttr holds the attributes that will be applied to a new [Cache] opened -// by [Open]. -type CacheAttr struct { - // Concurrent cures of a [FloodArtifact] dependency graph. - Cures int - // Options affecting [Cache] behaviour. - Flags int - // Preferred job count, when applicable. - Jobs int - // Preferred loadavg target, when applicable. - Load int - // Optional cure entry and exit notification. - Notify chan<- bool - - // Omit the [lockedfile] lock. - skipLock bool -} - -// Open returns the address of a newly opened instance of [Cache]. -// -// Concurrent cures of a [FloodArtifact] dependency graph is limited to the -// caller-supplied value, however direct calls to [Cache.Cure] is not subject -// to this limitation. -// -// A cures or jobs value of 0 or lower is equivalent to the value returned by -// [runtime.NumCPU]. -// -// A successful call to Open guarantees exclusive access to the on-filesystem -// cache for the resulting instance of [Cache]. The [Cache.Close] method cancels -// and waits for pending cures on [Cache] before releasing this lock and must be -// called once the [Cache] is no longer needed. -func Open( - ctx context.Context, - msg message.Msg, - base *check.Absolute, - attr *CacheAttr, -) (*Cache, error) { - openMu.Lock() - defer openMu.Unlock() - opened = true - - if extension == "" && len(irArtifact) != int(_kindEnd) { - panic("attempting to open cache with incomplete variant setup") - } - - var a CacheAttr - if attr != nil { - a = *attr - } - - if a.Cures < 1 { - a.Cures = runtime.NumCPU() - } - if a.Jobs < 1 { - a.Jobs = runtime.NumCPU() - } - if a.Load < 1 { - a.Load = runtime.NumCPU() + 2 - } - - for _, name := range []string{ - dirSubstitute, - dirIdentifier, - dirChecksum, - dirStatus, - dirFault, - dirWork, - } { - if err := os.MkdirAll( - base.Append(name).String(), - 0700, - ); err != nil && !errors.Is(err, os.ErrExist) { - return nil, err - } - } - - c := Cache{ - parent: ctx, - - cures: make(chan struct{}, a.Cures), - attr: a, - - msg: msg, - base: base, - - irCache: zeroIRCache(), - - substitute: make(map[unique.Handle[ID]]unique.Handle[Checksum]), - ident: make(map[unique.Handle[ID]]unique.Handle[Checksum]), - identErr: make(map[unique.Handle[ID]]error), - identPending: make(map[unique.Handle[ID]]*pendingCure), - - brPool: sync.Pool{New: func() any { return new(bufio.Reader) }}, - bwPool: sync.Pool{New: func() any { return new(bufio.Writer) }}, - } - c.toplevel.Store(newToplevel(ctx)) - - if !a.skipLock || !testing.Testing() { - if unlock, err := lockedfile.MutexAt( - base.Append(fileLock).String(), - ).Lock(); err != nil { - return nil, err - } else { - c.unlock = unlock - } - } else { - c.unlock = func() {} - } - - for _, name := range []string{ - dirWork, - dirTemp, - } { - dents, err := os.ReadDir(base.Append(name).String()) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - continue - } - c.unlock() - return nil, err - } - if len(dents) != 0 { - c.unlock() - return nil, fmt.Errorf( - "%s is not empty, scrub likely required", - name, - ) - } - } - - if _, err := os.ReadDir(base.Append( - dirExecScratch, - ).String()); !errors.Is(err, os.ErrNotExist) { - c.unlock() - if err != nil { - return nil, err - } - return nil, errors.New(dirExecScratch + " is present, scrub likely required") - } - - variantPath := base.Append(fileVariant).String() - if p, err := os.ReadFile(variantPath); err != nil { - if !errors.Is(err, os.ErrNotExist) { - c.unlock() - return nil, err - } - // nonexistence implies newly created cache, or a cache predating - // variant identification strings, in which case it is silently promoted - if err = os.WriteFile( - variantPath, - []byte(extension), - 0400, - ); err != nil { - c.unlock() - return nil, err - } - } else if s := string(p); s == "" { - if extension != "" { - if a.Flags&CPromoteVariant == 0 { - c.unlock() - return nil, ErrWouldPromote - } - if err = os.WriteFile( - variantPath, - []byte(extension), - 0400, - ); err != nil { - c.unlock() - return nil, err - } - } - } else if !ValidExtension(s) { - c.unlock() - return nil, ErrInvalidExtension - } else if s != extension { - c.unlock() - return nil, UnsupportedVariantError(s) - } - - if a.Flags&CColourOutput != 0 { - c.sgrRes = "\x1b[0m" - c.sgrIdent = "\x1b[1m" - c.sgrWarn = "\x1b[35m" - c.sgrErr = "\x1b[1;31m" - } - - return &c, nil -} - -// Collected is returned by [Collect.Cure] to indicate a successful collection. -type Collected struct{} - -// Error returns a constant string to satisfy error, but should never be seen -// by the user. -func (Collected) Error() string { return "artifacts successfully collected" } - -// IsCollected returns whether the underlying error contains that of the result -// of curing a [Collect] helper. -func IsCollected(err error) bool { return errors.As(err, new(Collected)) } - -// Collect implements [pkg.FloodArtifact] to concurrently cure multiple -// [pkg.Artifact]. It returns [Collected]. -type Collect []Artifact - -var _ Artifact = new(Collect) - -// Cure returns [Collected]. -func (*Collect) Cure(*FContext) error { return Collected{} } - -// Kind returns the hardcoded [pkg.Kind] value. -func (*Collect) Kind() Kind { return kindCollection } - -// Params is a noop: dependencies are already represented in the header. -func (*Collect) Params(*IContext) {} - -// Inputs returns [Collect] as is. -func (c *Collect) Inputs() []Artifact { return *c } - -// IsExclusive returns false: Cure is a noop. -func (*Collect) IsExclusive() bool { return false } diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go deleted file mode 100644 index b23948f8..00000000 --- a/internal/pkg/pkg_test.go +++ /dev/null @@ -1,2375 +0,0 @@ -package pkg_test - -import ( - "archive/tar" - "bytes" - "context" - "crypto/sha512" - "encoding/base64" - "errors" - "fmt" - "io" - "io/fs" - "log" - "net/http" - "os" - "path/filepath" - "reflect" - "strconv" - "strings" - "sync" - "syscall" - "testing" - "testing/fstest" - "time" - "unique" - "unsafe" - - "hakurei.app/check" - "hakurei.app/container" - "hakurei.app/fhs" - "hakurei.app/internal/info" - "hakurei.app/internal/landlock" - "hakurei.app/internal/pkg" - "hakurei.app/internal/stub" - "hakurei.app/message" -) - -var skipLock = func() pkg.CacheAttr { - var attr pkg.CacheAttr - *(*bool)(unsafe.Pointer(reflect.ValueOf(&attr). - Elem(). - FieldByName("skipLock"). - UnsafeAddr())) = true - return attr -}() - -var ( - // extension is a string uniquely identifying a set of custom [Artifact] - // implementations registered by calling [Register]. - // - //go:linkname extension hakurei.app/internal/pkg.extension - extension string - - // opened is false if [Open] was never called. - // - //go:linkname opened hakurei.app/internal/pkg.opened - opened bool - - // irArtifact refers to artifact IR interpretation functions and must not be - // written to directly. - // - //go:linkname irArtifact hakurei.app/internal/pkg.irArtifact - irArtifact map[pkg.Kind]pkg.IRReadFunc - - // statusHeader is the header written to all status files in dirStatus. - // - //go:linkname statusHeader hakurei.app/internal/pkg.statusHeader - statusHeader string -) - -// newRContext returns the address of a new [pkg.RContext] unsafely created for -// the specified [testing.TB]. -func newRContext(tb testing.TB, c *pkg.Cache) *pkg.RContext { - var r pkg.RContext - rContextVal := reflect.ValueOf(&r).Elem().FieldByName("ctx") - reflect.NewAt( - rContextVal.Type(), - unsafe.Pointer(rContextVal.UnsafeAddr()), - ).Elem().Set(reflect.ValueOf(tb.Context())) - rCacheVal := reflect.ValueOf(&r).Elem().FieldByName("cache") - reflect.NewAt( - rCacheVal.Type(), - unsafe.Pointer(rCacheVal.UnsafeAddr()), - ).Elem().Set(reflect.ValueOf(c)) - return &r -} - -// overrideIdent overrides the ID method of [Artifact]. -type overrideIdent struct { - id pkg.ID - pkg.TrivialArtifact -} - -func (a overrideIdent) ID() pkg.ID { return a.id } - -// overrideIdentFile overrides the ID method of [FileArtifact]. -type overrideIdentFile struct { - id pkg.ID - pkg.FileArtifact -} - -func (a overrideIdentFile) ID() pkg.ID { return a.id } - -// A knownIdentArtifact implements [pkg.KnownIdent] and [Artifact] -type knownIdentArtifact interface { - pkg.KnownIdent - pkg.TrivialArtifact -} - -// A knownIdentFile implements [pkg.KnownIdent] and [FileArtifact] -type knownIdentFile interface { - pkg.KnownIdent - pkg.FileArtifact -} - -// overrideChecksum overrides the Checksum method of [Artifact]. -type overrideChecksum struct { - checksum pkg.Checksum - knownIdentArtifact -} - -func (a overrideChecksum) Checksum() pkg.Checksum { return a.checksum } - -// overrideChecksumFile overrides the Checksum method of [FileArtifact]. -type overrideChecksumFile struct { - checksum pkg.Checksum - knownIdentFile -} - -func (a overrideChecksumFile) Checksum() pkg.Checksum { return a.checksum } - -// A stubArtifact implements [TrivialArtifact] with hardcoded behaviour. -type stubArtifact struct { - kind pkg.Kind - params []byte - deps []pkg.Artifact - - cure func(t *pkg.TContext) error -} - -func (a *stubArtifact) Kind() pkg.Kind { return a.kind } -func (a *stubArtifact) Params(ctx *pkg.IContext) { ctx.Write(a.params) } -func (a *stubArtifact) Inputs() []pkg.Artifact { return a.deps } -func (a *stubArtifact) Cure(t *pkg.TContext) error { return a.cure(t) } -func (*stubArtifact) IsExclusive() bool { return false } - -// A stubArtifactF implements [FloodArtifact] with hardcoded behaviour. -type stubArtifactF struct { - kind pkg.Kind - params []byte - deps []pkg.Artifact - excl bool - - cure func(f *pkg.FContext) error -} - -func (a *stubArtifactF) Kind() pkg.Kind { return a.kind } -func (a *stubArtifactF) Params(ctx *pkg.IContext) { ctx.Write(a.params) } -func (a *stubArtifactF) Inputs() []pkg.Artifact { return a.deps } -func (a *stubArtifactF) Cure(f *pkg.FContext) error { return a.cure(f) } -func (a *stubArtifactF) IsExclusive() bool { return a.excl } - -// A stubFile implements [FileArtifact] with hardcoded behaviour. -type stubFile struct { - data []byte - err error - - stubArtifact -} - -func (*stubFile) IsExecutable() bool { return false } - -func (a *stubFile) Cure(*pkg.RContext) (io.ReadCloser, error) { - return io.NopCloser(bytes.NewReader(a.data)), a.err -} - -// newStubFile returns an implementation of [pkg.File] with hardcoded behaviour. -func newStubFile( - kind pkg.Kind, - id pkg.ID, - sum *pkg.Checksum, - data []byte, - err error, -) pkg.FileArtifact { - f := overrideIdentFile{id, &stubFile{data, err, stubArtifact{ - kind, - nil, - nil, - func(*pkg.TContext) error { - panic("unreachable") - }, - }}} - if sum == nil { - return f - } else { - return overrideChecksumFile{*sum, f} - } -} - -// stubExtern implements [External] with hardcoded prepared outcomes. -type stubExtern struct { - artifact map[unique.Handle[pkg.ID]]pkg.Checksum - checksum map[unique.Handle[pkg.Checksum]]fstest.MapFS - status map[unique.Handle[pkg.ID]]string -} - -func (e stubExtern) Artifact(_ context.Context, id unique.Handle[pkg.ID]) (*pkg.Checksum, error) { - if checksum, ok := e.artifact[id]; ok { - return &checksum, nil - } - return nil, nil -} - -func (e stubExtern) Checksum(checksum unique.Handle[pkg.Checksum]) pkg.Artifact { - var buf bytes.Buffer - if err := pkg.Write(e.checksum[checksum], ".", &buf); err != nil { - panic(err) - } - return pkg.NewArchive(pkg.NewFile("", buf.Bytes())) -} - -func (e stubExtern) Status(_ *pkg.RContext, id unique.Handle[pkg.ID]) (io.ReadCloser, error) { - if status, ok := e.status[id]; ok { - return io.NopCloser(strings.NewReader(status)), nil - } - return nil, nil -} - -// destroyArtifact removes all traces of an [Artifact] from the on-disk cache. -// Do not use this in a test case without a very good reason to do so. -func destroyArtifact( - t *testing.T, - base *check.Absolute, - c *pkg.Cache, - a pkg.Artifact, -) { - if pathname, checksum, err := c.Cure(a); err != nil { - t.Fatalf("Cure: error = %v", err) - } else if err = os.Remove(pathname.String()); err != nil { - t.Fatal(err) - } else { - p := base.Append( - "checksum", - pkg.Encode(checksum.Value()), - ) - if err = filepath.WalkDir(p.String(), func( - path string, - d fs.DirEntry, - err error, - ) error { - if err != nil { - return err - } - if d.IsDir() { - return os.Chmod(path, 0700) - } - return nil - }); err != nil && !errors.Is(err, os.ErrNotExist) { - t.Fatal(err) - } - if err = os.RemoveAll(p.String()); err != nil { - t.Fatal(err) - } - } -} - -// newDestroyArtifactFunc returns a function that calls destroyArtifact. -func newDestroyArtifactFunc(a pkg.Artifact) func( - t *testing.T, - base *check.Absolute, - c *pkg.Cache, -) { - return func( - t *testing.T, - base *check.Absolute, - c *pkg.Cache, - ) { - destroyArtifact(t, base, c, a) - } -} - -// destroyStatus counts non-substitution status entries and destroys them. -func destroyStatus(t *testing.T, base *check.Absolute, c, s int) { - dents, err := os.ReadDir(base.Append("status").String()) - if err != nil { - t.Fatal(err) - } - - var gotC, gotS int - for _, dent := range dents { - if err = os.Remove(base.Append( - "status", - dent.Name(), - ).String()); err != nil { - t.Fatal(err) - } - - if dent.Type().IsRegular() { - gotC++ - continue - } - if dent.Type()&fs.ModeSymlink == fs.ModeSymlink { - gotS++ - continue - } - t.Errorf("%s: %s", dent.Name(), dent.Type()) - } - - if gotC != c { - t.Errorf("status: c = %d, want %d", gotC, c) - } - if gotS != s { - t.Errorf("status: s = %d, want %d", gotS, s) - } -} - -func TestIdent(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - a pkg.Artifact - want unique.Handle[pkg.ID] - }{ - {"decompress", &stubArtifact{ - pkg.KindDecompress, - []byte{pkg.Gzip, 0, 0, 0, 0, 0, 0, 0}, - []pkg.Artifact{ - overrideIdent{pkg.ID{}, new(stubArtifact)}, - }, - nil, - }, unique.Make[pkg.ID](pkg.MustDecode( - "fkmge-H56Ph2LzNNbEUjTypes8LfOxrF_CEXg07kq6ho_nbTNvzQde9-bjB7gyB_", - ))}, - } - - msg := message.New(log.New(os.Stderr, "ident: ", 0)) - msg.SwapVerbose(true) - var cache *pkg.Cache - if a, err := check.NewAbs(t.TempDir()); err != nil { - t.Fatal(err) - } else if cache, err = pkg.Open(t.Context(), msg, a, nil); err != nil { - t.Fatal(err) - } - t.Cleanup(cache.Close) - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - if got := cache.Ident(tc.a); got != tc.want { - t.Errorf("Ident: %s, want %s", - pkg.Encode(got.Value()), - pkg.Encode(tc.want.Value()), - ) - } - }) - } -} - -// An expectsKnown describes an expected file or directory. -type expectsKnown interface { - // hash returns the checksum of the represented data. - hash() (checksum pkg.Checksum) -} - -// An expectsChecksum is a prepared checksum value. -type expectsChecksum pkg.Checksum - -// hash returns e. -func (e expectsChecksum) hash() pkg.Checksum { return e } - -// An expectsFile is the contents of a file expected by the test suite. -type expectsFile []byte - -// hash computes the checksum of e. -func (e expectsFile) hash() (checksum pkg.Checksum) { - h := sha512.New384() - h.Write(e) - h.Sum(checksum[:0]) - return -} - -// An expectsFS describes the state of a filesystem expected by the test suite. -type expectsFS fstest.MapFS - -// hash computes the checksum of e. -func (e expectsFS) hash() (checksum pkg.Checksum) { - if err := pkg.SumFS(&checksum, fstest.MapFS(e), "."); err != nil { - panic(err) - } - return -} - -// expectsFrom generates expectsFS for a filesystem directory. -func expectsFrom(pathname string) string { - var buf strings.Builder - buf.WriteString("expectsFS{\n") - if err := filepath.WalkDir(pathname, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - var rel string - if rel, err = filepath.Rel(pathname, path); err != nil { - return err - } - buf.WriteString("\t" + strconv.Quote(rel) + ": {Mode: ") - - var fi fs.FileInfo - if fi, err = d.Info(); err != nil { - return err - } - mode := fi.Mode() - - switch { - case mode.IsDir(): - buf.WriteString("fs.ModeDir | 0" + - strconv.FormatInt(int64(mode&^fs.ModeDir), 8)) - - case mode&fs.ModeSymlink != 0: - buf.WriteString("fs.ModeSymlink | 0" + - strconv.FormatInt(int64(mode&^fs.ModeSymlink), 8) + - ", Data: []byte(") - var linkname string - if linkname, err = os.Readlink(path); err != nil { - return err - } - buf.WriteString(strconv.Quote(linkname)) - buf.WriteByte(')') - - case mode.IsRegular(): - buf.WriteString("0" + strconv.FormatInt(int64(mode), 8)) - var p []byte - if p, err = os.ReadFile(path); err != nil { - return err - } - - if len(p) > 0 { - buf.WriteString(", Data: []byte(") - buf.WriteString(strconv.Quote(unsafe.String(unsafe.SliceData(p), len(p)))) - buf.WriteByte(')') - } - } - buf.WriteString("},\n") - return nil - }); err != nil { - panic(err) - } - buf.WriteString("}") - return buf.String() -} - -// cacheTestCase is a test case passed to checkWithCache where a new instance -// of [pkg.Cache] is prepared for the test case, and is validated and removed -// on test completion. -type cacheTestCase struct { - name string - flags int - early func(t *testing.T, base *check.Absolute) - f func(t *testing.T, base *check.Absolute, c *pkg.Cache) - want expectsFS -} - -const ( - // checkDestroySubstitutes arranges for substitutes to be destroyed before - // measurement during checkWithCache. - checkDestroySubstitutes = 1 << (iota + 32) -) - -// makeBase returns a [pkg.Cache] base directory created for tb. -func makeBase(tb testing.TB) (base *check.Absolute) { - tb.Helper() - - base = check.MustAbs(tb.TempDir()) - if err := os.Chmod(base.String(), 0700); err != nil { - tb.Fatal(err) - } - tb.Cleanup(func() { - if err := filepath.WalkDir(base.String(), func(path string, d fs.DirEntry, err error) error { - if err != nil { - tb.Error(err) - return nil - } - if !d.IsDir() { - return nil - } - return os.Chmod(path, 0700) - }); err != nil { - tb.Fatal(err) - } - }) - return -} - -// checkWithCache runs a slice of cacheTestCase. -func checkWithCache(t *testing.T, testCases []cacheTestCase) { - t.Helper() - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Helper() - t.Parallel() - - base := makeBase(t) - msg := message.New(log.New(os.Stderr, "cache: ", 0)) - msg.SwapVerbose(testing.Verbose()) - - flags := tc.flags | pkg.CSuppressInit - - if info.CanDegrade { - if _, err := landlock.GetABI(); err != nil { - if !errors.Is(err, syscall.ENOSYS) { - t.Fatalf("LandlockGetABI: error = %v", err) - } - flags |= pkg.CHostAbstract - t.Log("Landlock LSM is unavailable, setting CHostAbstract") - } - } - - var scrubFunc func() error // scrub after hashing - if c, err := pkg.Open(t.Context(), msg, base, &pkg.CacheAttr{ - Cures: 1 << 4, - Flags: flags, - }); err != nil { - t.Fatalf("Open: error = %v", err) - } else { - t.Cleanup(c.Close) - if tc.early != nil { - tc.early(t, base) - } - tc.f(t, base, c) - scrubFunc = func() error { - err = c.Scrub(1 << 7) - idents, checksums, cleanErr := c.Clean(false, false) - if len(idents) > 0 { - t.Errorf("destroyed %d idents", len(idents)) - } - if len(checksums) > 0 { - t.Errorf("destroyed %d checksums", len(checksums)) - } - return errors.Join(err, cleanErr) - } - } - - var restoreTemp bool - if _, err := os.Lstat(base.Append("temp").String()); err != nil { - if !errors.Is(err, os.ErrNotExist) { - t.Fatal(err) - } - } else { - restoreTemp = true - } - - // destroy lock and variant file to avoid changing cache checksums - for _, s := range []string{ - "lock", - "variant", - } { - pathname := base.Append(s) - if p, err := os.ReadFile(pathname.String()); err != nil { - t.Fatal(err) - } else if len(p) != 0 { - t.Fatalf("file %q: %q", s, string(p)) - } - if err := os.Remove(pathname.String()); err != nil { - t.Fatal(err) - } - } - - // destroy non-deterministic substitutes - if tc.flags&checkDestroySubstitutes != 0 { - substitute := base.Append("substitute") - if err := os.RemoveAll(substitute.String()); err != nil { - t.Fatal(err) - } else if err = os.Mkdir(substitute.String(), 0700); err != nil { - t.Fatal(err) - } - } - - // destroy empty status directory - if err := syscall.Rmdir(base.Append("status").String()); err != nil { - if !errors.Is(err, syscall.ENOTEMPTY) { - t.Fatal(err) - } - } - - // destroy empty fault directory - if err := os.Remove(base.Append("fault").String()); err != nil { - t.Fatal(err) - } - - want := tc.want.hash() - - var checksum pkg.Checksum - if err := pkg.SumDir(&checksum, base); err != nil { - t.Fatalf("SumDir: error = %v", err) - } else if checksum != want { - t.Fatal(expectsFrom(base.String())) - } - - if err := scrubFunc(); err != nil { - t.Fatal("cache contains inconsistencies\n\n" + err.Error()) - } - - if restoreTemp { - if err := os.Mkdir( - base.Append("temp").String(), - 0700, - ); err != nil { - t.Fatal(err) - } - } - - // validate again to make sure scrub did not condemn anything - if err := pkg.SumDir(&checksum, base); err != nil { - t.Fatalf("SumDir: error = %v", err) - } else if checksum != want { - t.Fatalf("(scrubbed) %s", expectsFrom(base.String())) - } - }) - } -} - -// A cureStep contains an [Artifact] to be cured, and the expected outcome. -type cureStep struct { - name string - - a pkg.Artifact - - pathname *check.Absolute - output expectsKnown - whence int - err error -} - -// ignorePathname is passed to cureMany to skip the pathname check. -var ignorePathname = check.MustAbs("/\x00") - -// cureMany cures many artifacts against a [Cache] and checks their outcomes. -func cureMany(t *testing.T, c *pkg.Cache, steps []cureStep) { - t.Helper() - - makeChecksumH := func(checksum pkg.Checksum) unique.Handle[pkg.Checksum] { - if checksum == (pkg.Checksum{}) { - return unique.Handle[pkg.Checksum]{} - } - return unique.Make(checksum) - } - - for _, step := range steps { - t.Log("cure step:", step.name) - if pathname, checksum, whence, err := c.CureWhence(step.a); !reflect.DeepEqual(err, step.err) { - faults, _err := c.ReadFaults(step.a) - if _err != nil { - t.Errorf("ReadFaults: error = %v", _err) - } - - var p []byte - for _, fault := range faults { - p, _err = os.ReadFile(fault.String()) - if _err != nil { - t.Error(_err) - continue - } - t.Log(string(p)) - t.Logf("faulting cure terminated %s ago", time.Since(faults[0].Time())) - } - t.Fatalf("Cure: error = %v, want %v", err, step.err) - } else if step.whence != whence { - t.Fatalf("Cure: whence = %s, want %s", pkg.WhenceString(whence), pkg.WhenceString(step.whence)) - } else if step.pathname != ignorePathname && !pathname.Is(step.pathname) { - t.Fatalf("Cure: pathname = %q, want %q", pathname, step.pathname) - } else if step.output == nil || checksum != makeChecksumH(step.output.hash()) { - if pathname != nil { - if name, _err := filepath.EvalSymlinks(pathname.String()); _err != nil { - t.Fatal(_err) - } else { - t.Fatal(expectsFrom(name)) - } - } else if checksum != (unique.Handle[pkg.Checksum]{}) { - t.Fatalf("Cure: unexpected checksum %s", pkg.Encode(checksum.Value())) - } - } else { - v := any(err) - if err == nil { - v = pathname - } - var checksumVal pkg.Checksum - if checksum != (unique.Handle[pkg.Checksum]{}) { - checksumVal = checksum.Value() - } - t.Log(pkg.Encode(checksumVal)+":", v) - } - } -} - -// newWantScrubError returns the address to a new [ScrubError] for base. -func newWantScrubError(base *check.Absolute) *pkg.ScrubError { - return &pkg.ScrubError{ - ChecksumMismatches: []pkg.ChecksumMismatchError{ - {Got: pkg.MustDecode( - "vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", - ), Want: pkg.Checksum{0xff, 0}}, - }, - DanglingIdentifiers: []pkg.ID{ - {0xfe, 0}, - {0xfe, 0xfe}, - {0xfe, 0xff}, - }, - Errs: map[unique.Handle[string]][]error{ - base.Append("checksum", "__8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA").Handle(): { - pkg.InvalidFileModeError(fs.ModeSymlink), - }, - - base.Append("checksum", "invalid").Handle(): { - base64.CorruptInputError(4), - }, - - base.Append("nonexistent").Handle(): { - base64.CorruptInputError(8), - }, - - base.Append("identifier", pkg.Encode(pkg.ID{0xfe, 0xff})).Handle(): { - &os.PathError{ - Op: "readlink", - Path: base.Append( - "identifier", - pkg.Encode(pkg.ID{0xfe, 0xff}), - ).String(), - Err: syscall.EINVAL, - }, - }, - base.Append("identifier", "invalid").Handle(): { - base64.CorruptInputError(4), - }, - }, - } -} - -func TestCache(t *testing.T) { - t.Parallel() - - testdata := expectsFile("" + - "\x00\x00\x00\x00" + - "\xad\x0b\x00" + - "\x04" + - "\xfe\xfe\x00\x00" + - "\xfe\xca\x00\x00") - - testCases := []cacheTestCase{ - {"file", pkg.CValidateKnown | pkg.CAssumeChecksum, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - identifier := (pkg.ID)(bytes.Repeat([]byte{ - 0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f, - }, 8)) - wantPathname := base.Append( - "identifier", - "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", - ) - identifier0 := (pkg.ID)(bytes.Repeat([]byte{ - 0x71, 0xa7, 0xde, 0x6d, 0xa6, 0xde, - }, 8)) - wantPathname0 := base.Append( - "identifier", - "cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe", - ) - failingFile := newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 3}, - nil, - nil, struct { - _ []byte - stub.UniqueError - }{UniqueError: 0xbad}, - ) - - cureMany(t, c, []cureStep{ - {"initial file", newStubFile( - pkg.KindHTTPGet, - identifier, - new(testdata.hash()), - testdata, nil, - ), wantPathname, testdata, pkg.WNew, nil}, - - {"identical content", newStubFile( - pkg.KindHTTPGet, - identifier0, - new(testdata.hash()), - testdata, nil, - ), wantPathname0, testdata, pkg.WCache, nil}, - - {"existing entry", newStubFile( - pkg.KindHTTPGet, - identifier, - new(testdata.hash()), - testdata, nil, - ), wantPathname, testdata, pkg.WCache, nil}, - - {"checksum mismatch", newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 0}, - new(pkg.Checksum), - testdata, nil, - ), nil, nil, pkg.WNew, &pkg.ChecksumMismatchError{ - Got: testdata.hash(), - }}, - - {"store without validation", newStubFile( - pkg.KindHTTPGet, - pkg.MustDecode("vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX"), - nil, - []byte{0}, nil, - ), base.Append( - "identifier", - "vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", - ), expectsChecksum{ - 0xbe, 0xc0, 0x21, 0xb4, 0xf3, 0x68, - 0xe3, 0x06, 0x91, 0x34, 0xe0, 0x12, - 0xc2, 0xb4, 0x30, 0x70, 0x83, 0xd3, - 0xa9, 0xbd, 0xd2, 0x06, 0xe2, 0x4e, - 0x5f, 0x0d, 0x86, 0xe1, 0x3d, 0x66, - 0x36, 0x65, 0x59, 0x33, 0xec, 0x2b, - 0x41, 0x34, 0x65, 0x96, 0x68, 0x17, - 0xa9, 0xc2, 0x08, 0xa1, 0x17, 0x17, - }, pkg.WNew, nil}, - - {"incomplete implementation", struct{ pkg.Artifact }{&stubArtifact{ - kind: pkg.KindExec, - params: []byte("artifact overridden to be incomplete"), - }}, nil, nil, pkg.WNew, pkg.InvalidArtifactError(pkg.MustDecode( - "poVrv3zNINbHd-WUJihwgWQwJpbHURFHfpY7Zux-ztto_bQhhS5PdpbNMjdkXv-V", - ))}, - - {"error passthrough", newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 1}, - nil, - nil, stub.UniqueError(0xcafe), - ), nil, nil, pkg.WNew, stub.UniqueError(0xcafe)}, - - {"error caching", newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 1}, - nil, - nil, nil, - ), nil, nil, pkg.WCache, stub.UniqueError(0xcafe)}, - - {"cache hit bad type", overrideChecksum{testdata.hash(), overrideIdent{pkg.ID{0xff, 2}, &stubArtifact{ - kind: pkg.KindTar, - }}}, nil, nil, pkg.WCache, pkg.InvalidFileModeError( - 0400, - )}, - - {"noncomparable error", &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("artifact with dependency returning noncomparable error"), - deps: []pkg.Artifact{failingFile}, - - cure: func(f *pkg.FContext) error { - panic("attempting to cure impossible artifact") - }, - }, nil, nil, pkg.WNew, pkg.InputError{ - failingFile: struct { - _ []byte - stub.UniqueError - }{UniqueError: 0xbad}, - }}, - }) - - if c0, err := pkg.Open( - t.Context(), - message.New(nil), - base, &skipLock, - ); err != nil { - t.Fatalf("open: error = %v", err) - } else { - t.Cleanup(c.Close) // check doubled cancel - cureMany(t, c0, []cureStep{ - {"cache hit ident", overrideIdent{ - id: identifier, - }, wantPathname, testdata, pkg.WCache, nil}, - - {"cache miss checksum match", newStubFile( - pkg.KindHTTPGet, - testdata.hash(), - nil, - testdata, - nil, - ), base.Append( - "identifier", - pkg.Encode(testdata.hash()), - ), testdata, pkg.WNew, nil}, - }) - - // cure after close - c.Close() - if _, _, err = c.Cure(&stubArtifactF{ - kind: pkg.KindExec, - params: []byte("unreachable artifact cured after cancel"), - deps: []pkg.Artifact{pkg.NewFile("", []byte("unreachable dependency"))}, - }); !reflect.DeepEqual(err, context.Canceled) { - t.Fatalf("(closed) Cure: error = %v", err) - } - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX": {Mode: 0400, Data: []byte{0}}, - "checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq": {Mode: 0400, Data: []byte{0, 0, 0, 0, 0xad, 0xb, 0, 4, 0xfe, 0xfe, 0, 0, 0xfe, 0xca, 0, 0}}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX")}, - "identifier/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq")}, - "identifier/cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq")}, - "identifier/deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"directory", pkg.CAssumeChecksum, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - id := pkg.MustDecode( - "HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY", - ) - makeSample := func(t *pkg.TContext) error { - work := t.GetWorkDir() - if err := os.Mkdir(work.String(), 0700); err != nil { - return err - } - - if err := os.WriteFile( - work.Append("check").String(), - []byte{0, 0}, - 0400, - ); err != nil { - return err - } - - if err := os.MkdirAll(work.Append( - "lib", - "pkgconfig", - ).String(), 0700); err != nil { - return err - } - - return os.Symlink( - "/proc/nonexistent/libedac.so", - work.Append( - "lib", - "libedac.so", - ).String(), - ) - } - want := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "check": {Mode: 0400, Data: []byte{0, 0}}, - - "lib": {Mode: fs.ModeDir | 0700}, - "lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - - "lib/pkgconfig": {Mode: fs.ModeDir | 0700}, - } - wantPathname := base.Append( - "identifier", - pkg.Encode(id), - ) - - id0 := pkg.MustDecode( - "Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa", - ) - wantPathname0 := base.Append( - "identifier", - pkg.Encode(id0), - ) - - makeGarbage := func(work *check.Absolute, wantErr error) error { - if err := os.Mkdir(work.String(), 0700); err != nil { - return err - } - - mode := fs.FileMode(0) - if wantErr == nil { - mode = 0500 - } - - if err := os.MkdirAll(work.Append( - "lib", - "pkgconfig", - ).String(), 0700); err != nil { - return err - } - - if err := os.WriteFile(work.Append( - "lib", - "check", - ).String(), nil, 0400&mode); err != nil { - return err - } - - if err := os.Chmod(work.Append( - "lib", - "pkgconfig", - ).String(), 0500&mode); err != nil { - return err - } - if err := os.Chmod(work.Append( - "lib", - ).String(), 0500&mode); err != nil { - return err - } - - return wantErr - } - - cureMany(t, c, []cureStep{ - {"initial directory", overrideChecksum{want.hash(), overrideIdent{id, &stubArtifact{ - kind: pkg.KindTar, - cure: makeSample, - }}}, wantPathname, want, pkg.WNew, nil}, - - {"identical identifier", overrideChecksum{want.hash(), overrideIdent{id, &stubArtifact{ - kind: pkg.KindTar, - }}}, wantPathname, want, pkg.WCache, nil}, - - {"identical checksum", overrideIdent{id0, &stubArtifact{ - kind: pkg.KindTar, - cure: makeSample, - }}, wantPathname0, want, pkg.WNew, nil}, - - {"cure fault", overrideIdent{pkg.ID{0xff, 0}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return makeGarbage(t.GetWorkDir(), stub.UniqueError(0xcafe)) - }, - }}, nil, nil, pkg.WNew, stub.UniqueError(0xcafe)}, - - {"checksum mismatch", overrideChecksum{pkg.Checksum{}, overrideIdent{pkg.ID{0xff, 1}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return makeGarbage(t.GetWorkDir(), nil) - }, - }}}, nil, nil, pkg.WNew, &pkg.ChecksumMismatchError{ - Got: pkg.MustDecode( - "CUx-3hSbTWPsbMfDhgalG4Ni_GmR9TnVX8F99tY_P5GtkYvczg9RrF5zO0jX9XYT", - ), - }}, - - {"cache hit bad type", newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 2}, - new(want.hash()), - testdata, nil, - ), nil, nil, pkg.WCache, pkg.InvalidFileModeError( - fs.ModeDir | 0500, - )}, - - {"openFile directory", overrideIdent{pkg.ID{0xff, 3}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - r, err := t.Open(overrideChecksumFile{checksum: want.hash()}) - if err != nil { - panic(err) - } - _, err = io.ReadAll(r) - return err - }, - }}, nil, nil, pkg.WNew, &os.PathError{ - Op: "read", - Path: base.Append( - "checksum", - pkg.Encode(want.hash()), - ).String(), - Err: syscall.EISDIR, - }}, - - {"no output", overrideIdent{pkg.ID{0xff, 4}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return nil - }, - }}, nil, nil, pkg.WNew, pkg.NoOutputError{}}, - - {"file output", overrideIdent{pkg.ID{0xff, 5}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return os.WriteFile(t.GetWorkDir().String(), []byte{0}, 0400) - }, - }}, nil, nil, pkg.WNew, errors.New("non-file artifact produced regular file")}, - - {"symlink output", overrideIdent{pkg.ID{0xff, 6}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return os.Symlink( - t.GetWorkDir().String(), - t.GetWorkDir().String(), - ) - }, - }}, nil, nil, pkg.WNew, pkg.InvalidFileModeError( - fs.ModeSymlink | 0777, - )}, - - {"alternative", &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("substitutable artifact"), - deps: []pkg.Artifact{newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 8}, - nil, - []byte("substitutable dependency"), - nil, - )}, - - cure: func(f *pkg.FContext) error { - return makeSample(&f.TContext) - }, - }, base.Append( - "identifier", - "gIH5eti07nBAhVti3ZTn8EIGOBCZWWvNpxg5CmCZlDzIC4mBdomPmQLeEpf1IQg9", - ), want, pkg.WNew, nil}, - - {"substitutable", &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("substitutable artifact"), - deps: []pkg.Artifact{newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 10}, - nil, - []byte("substitutable dependency"), - nil, - )}, - - cure: func(f *pkg.FContext) error { - panic("substitution missed") - }, - }, base.Append( - "identifier", - "VGjpp7NVw3OYLpgOQumfM0KcJgY3suz6torykUczcg4bGT24ySkPn5hy-4dNqqkR", - ), want, pkg.WSubstitute, nil}, - }) - - if c0, err := pkg.Open( - t.Context(), - message.New(nil), - base, &skipLock, - ); err != nil { - t.Fatalf("open: error = %v", err) - } else { - t.Cleanup(c.Close) - cureMany(t, c0, []cureStep{ - {"substitutable", &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("substitutable artifact"), - deps: []pkg.Artifact{newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 0xff, 0xfd, 0xfd}, - nil, - []byte("substitutable dependency"), - nil, - )}, - - cure: func(f *pkg.FContext) error { - panic("substitution missed") - }, - }, base.Append( - "identifier", - "zeZ4JEZ0atAKl6Am67cBID6-KR18nZGgeYfU4qTuYaWhQC4eAn0dyTXdIBH9Bos8", - ), want, pkg.WSubstitute, nil}, - }) - } - - if dents, err := os.ReadDir(base.Append("status").String()); err != nil { - t.Fatal(err) - } else if len(dents) > 0 { - t.Errorf("ReadDir: %v", dents) - } - - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO": {Mode: 0400, Data: []byte("substitutable dependency")}, - "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b": {Mode: fs.ModeDir | 0500}, - "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/check": {Mode: 0400, Data: []byte{0, 0}}, - "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/lib": {Mode: fs.ModeDir | 0700}, - "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/lib/pkgconfig": {Mode: fs.ModeDir | 0700}, - "checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")}, - "identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")}, - "identifier/zeZ4JEZ0atAKl6Am67cBID6-KR18nZGgeYfU4qTuYaWhQC4eAn0dyTXdIBH9Bos8": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")}, - "identifier/___9_QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO")}, - "identifier/_wgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO")}, - "identifier/_woAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO")}, - "identifier/VGjpp7NVw3OYLpgOQumfM0KcJgY3suz6torykUczcg4bGT24ySkPn5hy-4dNqqkR": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")}, - "identifier/gIH5eti07nBAhVti3ZTn8EIGOBCZWWvNpxg5CmCZlDzIC4mBdomPmQLeEpf1IQg9": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "substitute/nWPg893oqn_YNN8kgfZrs_LVLwP90e0mKxqU8UwoazQh-yBPwhoviD7of6YALx-M": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"pending", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - wantErr := stub.UniqueError(0xcafe) - n, ready := make(chan struct{}), make(chan struct{}) - go func() { - if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - close(ready) - <-n - return wantErr - }, - }}); !reflect.DeepEqual(err, wantErr) { - panic(fmt.Sprintf("Cure: error = %v, want %v", err, wantErr)) - } - }() - - <-ready - wCureDone := make(chan struct{}) - go func() { - if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, &stubArtifact{ - kind: pkg.KindTar, - }}); !reflect.DeepEqual(err, wantErr) { - panic(fmt.Sprintf("Cure: error = %v, want %v", err, wantErr)) - } - close(wCureDone) - }() - - // check cache activity while a cure is blocking - cureMany(t, c, []cureStep{ - {"error passthrough", newStubFile( - pkg.KindHTTPGet, - pkg.ID{0xff, 1}, - nil, - nil, stub.UniqueError(0xbad), - ), nil, nil, pkg.WNew, stub.UniqueError(0xbad)}, - - {"file output", overrideIdent{pkg.ID{0xff, 2}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return os.WriteFile( - t.GetWorkDir().String(), - []byte{0}, - 0400, - ) - }, - }}, nil, nil, pkg.WNew, errors.New( - "non-file artifact produced regular file", - )}, - }) - - wantErrScrub := &pkg.ScrubError{ - Errs: map[unique.Handle[string]][]error{ - base.Handle(): {errors.New("scrub began with pending artifacts")}, - }, - } - if err := c.Scrub(1 << 6); !reflect.DeepEqual(err, wantErrScrub) { - t.Fatalf("Scrub: error = %#v, want %#v", err, wantErrScrub) - } - - notify := c.Done(unique.Make(pkg.ID{0xff})) - go close(n) - if notify != nil { - <-notify - } - for c.Done(unique.Make(pkg.ID{0xff})) != nil { - } - <-wCureDone - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"cancel abort block", pkg.CValidateKnown, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - var wg sync.WaitGroup - defer wg.Wait() - - var started sync.WaitGroup - defer started.Wait() - - blockCures := func(d byte, e stub.UniqueError, n int) { - started.Add(n) - for i := range n { - wg.Go(func() { - if _, _, err := c.Cure(overrideIdent{pkg.ID{d, byte(i)}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - started.Done() - <-t.Unwrap().Done() - return e + stub.UniqueError(i) - }, - }}); !reflect.DeepEqual(err, e+stub.UniqueError(i)) { - panic(err) - } - }) - } - started.Wait() - } - - blockCures(0xfd, 0xbad, 16) - c.Abort() - wg.Wait() - - blockCures(0xfd, 0xcafe, 16) - c.Abort() - wg.Wait() - - blockCures(0xff, 0xbad, 1) - if !c.Cancel(unique.Make(pkg.ID{0xff})) { - t.Fatal("missed cancellation") - } - wg.Wait() - - blockCures(0xff, 0xcafe, 1) - if !c.Cancel(unique.Make(pkg.ID{0xff})) { - t.Fatal("missed cancellation") - } - wg.Wait() - - for c.Cancel(unique.Make(pkg.ID{0xff})) { - } - - c.Close() - c.Abort() - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"no assume checksum", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - makeGarbage := func(work *check.Absolute, wantErr error) error { - if err := os.Mkdir(work.String(), 0700); err != nil { - return err - } - - if err := os.WriteFile(work.Append( - "check", - ).String(), nil, 0400); err != nil { - return err - } - - return wantErr - } - - want := expectsChecksum(pkg.MustDecode( - "Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M", - )) - - cureMany(t, c, []cureStep{ - {"create", overrideChecksum{want.hash(), overrideIdent{pkg.ID{0xff, 0}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return makeGarbage(t.GetWorkDir(), nil) - }, - }}}, base.Append( - "identifier", - pkg.Encode(pkg.ID{0xff, 0}), - ), want, pkg.WNew, nil}, - - {"reject", overrideChecksum{want.hash(), overrideIdent{pkg.ID{0xfe, 1}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return makeGarbage(t.GetWorkDir(), stub.UniqueError(0xbad)) - }, - }}}, nil, nil, pkg.WNew, stub.UniqueError(0xbad)}, - - {"match", overrideChecksum{want.hash(), overrideIdent{pkg.ID{0xff, 1}, &stubArtifact{ - kind: pkg.KindTar, - cure: func(t *pkg.TContext) error { - return makeGarbage(t.GetWorkDir(), nil) - }, - }}}, base.Append( - "identifier", - pkg.Encode(pkg.ID{0xff, 1}), - ), want, pkg.WNew, nil}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M": {Mode: fs.ModeDir | 0500}, - "checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M/check": {Mode: 0400, Data: []byte{}}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/_wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M")}, - "identifier/_wEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"scrub", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - cureMany(t, c, []cureStep{ - {"bad measured file", newStubFile( - pkg.KindHTTPGet, - pkg.Checksum{0xfe, 0}, - &pkg.Checksum{0xff, 0}, - []byte{0}, nil, - ), base.Append( - "identifier", - pkg.Encode(pkg.Checksum{0xfe, 0}), - ), expectsChecksum{0xff, 0}, pkg.WNew, nil}, - }) - - for _, p := range [][]string{ - {"identifier", "invalid"}, - {"identifier", pkg.Encode(pkg.ID{0xfe, 0xff})}, - {"checksum", "invalid"}, - } { - if err := os.WriteFile( - base.Append(p...).String(), - nil, - 0400, - ); err != nil { - t.Fatal(err) - } - } - - for _, p := range [][]string{ - {"../nonexistent", "checksum", pkg.Encode(pkg.Checksum{0xff, 0xff})}, - {"../nonexistent", "identifier", pkg.Encode(pkg.Checksum{0xfe, 0xfe})}, - } { - if err := os.Symlink( - p[0], - base.Append(p[1:]...).String(), - ); err != nil { - t.Fatal(err) - } - } - - wantErr := newWantScrubError(base) - if err := c.Scrub(1 << 6); !reflect.DeepEqual(err, wantErr) { - t.Fatalf("Scrub: error =\n%s\nwant\n%s", err, wantErr) - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"status substitute clean", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - destroyed := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("destroyed"), - deps: []pkg.Artifact{ - pkg.NewFile("destroyed-input", []byte("destroyed")), - }, - cure: func(f *pkg.FContext) error { - if w, err := f.GetStatusWriter(); err != nil { - return err - } else if _, err = w.Write([]byte("destroyed")); err != nil { - return err - } - - p := f.GetWorkDir() - if err := os.MkdirAll(p.String(), 0755); err != nil { - return err - } - return os.WriteFile(p.Append("result").String(), nil, 0444) - }, - } - substituted := new(*destroyed) - substituted.deps = []pkg.Artifact{ - pkg.NewFile("destroyed-input-0", []byte("destroyed")), - } - substituted.cure = func(*pkg.FContext) error { - panic("substitutable cure reached") - } - - cureMany(t, c, []cureStep{ - {"destroyed", destroyed, base.Append( - "identifier", - pkg.Encode(c.Ident(destroyed).Value()), - ), expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - "result": {Mode: 0444}, - }, pkg.WNew, nil}, - - {"substituted", substituted, base.Append( - "identifier", - pkg.Encode(c.Ident(substituted).Value()), - ), expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - "result": {Mode: 0444}, - }, pkg.WSubstitute, nil}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE": {Mode: fs.ModeDir | 0500}, - "checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE/result": {Mode: 0444}, - "checksum/wILUy2izpj2sgKJVhGUGIAde1XVuqvp5BpFMIQHanT5Q8R6jK4QPVSrJsjZh-njV": {Mode: 0400, Data: []byte("destroyed")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/CLyyxJWn1yUGFs_0KnJ4ZSU2hH4-elvGFRB36x14LrNrKBJPxvORhL9O4Rc7oeYU": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/wILUy2izpj2sgKJVhGUGIAde1XVuqvp5BpFMIQHanT5Q8R6jK4QPVSrJsjZh-njV")}, - "identifier/arv8a6z4i0x6pUb5Z2T-3iE4ETvyZj-uReHGKsS7uiBgMgKNEOzTGG3LIZb8vp0L": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/wILUy2izpj2sgKJVhGUGIAde1XVuqvp5BpFMIQHanT5Q8R6jK4QPVSrJsjZh-njV")}, - "identifier/heRgWhNJmhm898V68kw0ta76t4rP1o29tYW_TrdyBWXGfo37K3QWA7Z-JkLXAwcn": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")}, - "identifier/q-VoVBI3IjcWTqwIokpi3y05CP113CHgHslc-XWnhOvDULwzVc3q9Y5xOCm_YyLZ": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")}, - - "status": {Mode: fs.ModeDir | 0700}, - "status/q-VoVBI3IjcWTqwIokpi3y05CP113CHgHslc-XWnhOvDULwzVc3q9Y5xOCm_YyLZ": {Mode: fs.ModeSymlink | 0777, Data: []byte("dP-6_wIDRRouOaOF-nkBy-IaUbLdYHbOUTrptu7j_qfW01mnHSjYJ0oykUmvUd2x")}, - "status/heRgWhNJmhm898V68kw0ta76t4rP1o29tYW_TrdyBWXGfo37K3QWA7Z-JkLXAwcn": {Mode: 0400, Data: []byte(statusHeader + "destroyed")}, - "status/dP-6_wIDRRouOaOF-nkBy-IaUbLdYHbOUTrptu7j_qfW01mnHSjYJ0oykUmvUd2x": {Mode: 0400, Data: []byte(statusHeader + "destroyed")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "substitute/dP-6_wIDRRouOaOF-nkBy-IaUbLdYHbOUTrptu7j_qfW01mnHSjYJ0oykUmvUd2x": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/UjZSrgz7_B7XMd9fHU7jM33UZhWlFgX0rz7JZbCBYR28bCS7jr_CAJdcDhi52ruE")}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"extern", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - a := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("extern"), - } - wantIdent := c.Ident(a) - wantOutput := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - "result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")}, - } - var wantChecksum pkg.Checksum - if err := pkg.SumFS( - &wantChecksum, - fstest.MapFS(wantOutput), - ".", - ); err != nil { - t.Fatal(err) - } - wantChecksumH := unique.Make(wantChecksum) - - _a := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("extern substitute"), - deps: []pkg.Artifact{pkg.NewFile("", nil)}, - } - _wantIdent := c.Ident(_a) - _wantOutput := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - } - var _wantChecksum pkg.Checksum - if err := pkg.SumFS( - &_wantChecksum, - fstest.MapFS(_wantOutput), - ".", - ); err != nil { - t.Fatal(err) - } - _wantChecksumH := unique.Make(_wantChecksum) - - kca := pkg.NewExec( - "", "", - new(pkg.Checksum), 0, false, false, - fhs.AbsRoot, nil, fhs.AbsRoot, nil, - ) - kcIdent := c.Ident(kca) - - c.SetExternal(stubExtern{ - artifact: map[unique.Handle[pkg.ID]]pkg.Checksum{ - wantIdent: wantChecksum, - _wantIdent: _wantChecksum, - kcIdent: wantChecksum, - }, - checksum: map[unique.Handle[pkg.Checksum]]fstest.MapFS{ - wantChecksumH: fstest.MapFS(wantOutput), - _wantChecksumH: fstest.MapFS(_wantOutput), - }, - status: map[unique.Handle[pkg.ID]]string{ - wantIdent: "\x00", - kcIdent: "unreachable", - }, - }) - - cureMany(t, c, []cureStep{ - {"extern", a, base.Append( - "identifier", - pkg.Encode(wantIdent.Value()), - ), wantOutput, pkg.WExternal, nil}, - - {"substitute", _a, base.Append( - "identifier", - pkg.Encode(_wantIdent.Value()), - ), _wantOutput, pkg.WExternal, nil}, - - {"mismatch", kca, nil, nil, pkg.WExternal, &pkg.ChecksumMismatchError{ - Got: wantChecksum, - }}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - "checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb": {Mode: 0400}, - "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl": {Mode: fs.ModeDir | 0500}, - "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl/result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/k8yyhlBV3JpYzzywAnrRWkfp2TmgIDNC6FqMFwrrR2x0C3unBLveDNKgigOjG9MM": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/Xys8hPpJsKirZmTTuT7SwQ6661TCp_ZRu8iUjPPD7pC5E3ll7xw4zhNlZ2PVcz_-": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - "identifier/8sInO_dhPW9kweit7jVuH_vtMkRxlZT7XE4_yEKPZmpGkgVxkmxz0OpXgbMsH6ET": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/ofaXbB5gIza_7W4Rzo5XDe_8FwlCPKWn3TBcmGgFAkkTWMl5GXGmOzX89Jd2Mk2A": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/OLBgp1GsljhM2TJ-sbHjaiH9txEUvgdDTAzHv2P24donTt6_529l-9Ua0vFImLlb")}, - "identifier/8VL-egZySw-RbnHZtxGWdf5Z2v9S9CihrdFwa54Bc5EYMQcEgWEem7QrdCTWlCb8": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - - "status": {Mode: fs.ModeDir | 0700}, - "status/8VL-egZySw-RbnHZtxGWdf5Z2v9S9CihrdFwa54Bc5EYMQcEgWEem7QrdCTWlCb8": {Mode: 0400, Data: []byte("\x00")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "substitute/socGRnhdsJlJ4Z01ZwHFZdLzFH0rWvoaJrylcUKtvhBdGsE7VoYl-vdUuvfK8B4I": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"extern shallow", pkg.CExternShallow, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - a := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("extern"), - } - wantIdent := c.Ident(a) - wantOutput := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - "result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")}, - } - var wantChecksum pkg.Checksum - if err := pkg.SumFS( - &wantChecksum, - fstest.MapFS(wantOutput), - ".", - ); err != nil { - t.Fatal(err) - } - wantChecksumH := unique.Make(wantChecksum) - - _a := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("extern substitute"), - deps: []pkg.Artifact{pkg.NewFile("", nil)}, - } - _wantIdent := c.Ident(_a) - _wantOutput := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - } - var _wantChecksum pkg.Checksum - if err := pkg.SumFS( - &_wantChecksum, - fstest.MapFS(_wantOutput), - ".", - ); err != nil { - t.Fatal(err) - } - _wantChecksumH := unique.Make(_wantChecksum) - - kca := pkg.NewExec( - "", "", - new(pkg.Checksum), 0, false, false, - fhs.AbsRoot, nil, fhs.AbsRoot, nil, - ) - kcIdent := c.Ident(kca) - - fia := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("flood input"), - deps: []pkg.Artifact{_a}, - } - fiaIdent := c.Ident(fia) - - flia := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("flood input local"), - deps: []pkg.Artifact{_a}, - } - fliaIdent := c.Ident(flia) - - h := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("hanging input"), - } - - fhia := &stubArtifactF{ - kind: pkg.KindExec, - params: []byte("flood input local"), - deps: []pkg.Artifact{h}, - } - fhiaIdent := c.Ident(fhia) - - c.SetExternal(stubExtern{ - artifact: map[unique.Handle[pkg.ID]]pkg.Checksum{ - wantIdent: wantChecksum, - _wantIdent: _wantChecksum, - kcIdent: wantChecksum, - fiaIdent: wantChecksum, - fliaIdent: wantChecksum, - fhiaIdent: wantChecksum, - }, - checksum: map[unique.Handle[pkg.Checksum]]fstest.MapFS{ - wantChecksumH: fstest.MapFS(wantOutput), - _wantChecksumH: fstest.MapFS(_wantOutput), - }, - status: map[unique.Handle[pkg.ID]]string{ - wantIdent: "\x00", - kcIdent: "unreachable", - }, - }) - - cureMany(t, c, []cureStep{ - {"extern", a, base.Append( - "identifier", - pkg.Encode(wantIdent.Value()), - ), wantOutput, pkg.WExternal, nil}, - - {"flood shallow extern", fia, base.Append( - "identifier", - pkg.Encode(fiaIdent.Value()), - ), wantOutput, pkg.WExternal, nil}, - - {"substitute", _a, base.Append( - "identifier", - pkg.Encode(_wantIdent.Value()), - ), _wantOutput, pkg.WExternal, nil}, - - {"flood shallow local", flia, base.Append( - "identifier", - pkg.Encode(fliaIdent.Value()), - ), wantOutput, pkg.WExternal, nil}, - - {"flood shallow hanging", fhia, nil, nil, - pkg.WNew, pkg.HangingInputError(c.Ident(h))}, - - {"mismatch", kca, nil, nil, pkg.WExternal, &pkg.ChecksumMismatchError{ - Got: wantChecksum, - }}, - }) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500}, - "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl": {Mode: fs.ModeDir | 0500}, - "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl/result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/3fYJjSiWhklt7TK9MPVLGk5_8AIU8ostTcFPXULJf_qeAVG6pmC-Cqqu4ZhEXRxn": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - "identifier/8VL-egZySw-RbnHZtxGWdf5Z2v9S9CihrdFwa54Bc5EYMQcEgWEem7QrdCTWlCb8": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - "identifier/8sInO_dhPW9kweit7jVuH_vtMkRxlZT7XE4_yEKPZmpGkgVxkmxz0OpXgbMsH6ET": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/Xys8hPpJsKirZmTTuT7SwQ6661TCp_ZRu8iUjPPD7pC5E3ll7xw4zhNlZ2PVcz_-": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - "identifier/k8yyhlBV3JpYzzywAnrRWkfp2TmgIDNC6FqMFwrrR2x0C3unBLveDNKgigOjG9MM": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "identifier/mXlNz_v77nW7jstn_rJn4RL7dCQlkakM0y-HjO6tkDubtzaj-wCmW1hIr2Gk64_O": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - - "status": {Mode: fs.ModeDir | 0700}, - "status/8VL-egZySw-RbnHZtxGWdf5Z2v9S9CihrdFwa54Bc5EYMQcEgWEem7QrdCTWlCb8": {Mode: 0400, Data: []byte("\x00")}, - - "substitute": {Mode: fs.ModeDir | 0700}, - "substitute/socGRnhdsJlJ4Z01ZwHFZdLzFH0rWvoaJrylcUKtvhBdGsE7VoYl-vdUuvfK8B4I": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")}, - "substitute/MccYezQ4pshkeBV4w_aAEIwvZdKHL3OAgAkR3ouZ43EF94Ur1HBHaN7njPfJAj50": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - "substitute/VV5W0_DiA1iylaqbKFThp1ZpV_y1ccVS1XyqDnDaXoHQ-EEyPI4XFrQHcHOF2hWG": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")}, - - "work": {Mode: fs.ModeDir | 0700}, - }}, - } - checkWithCache(t, testCases) -} - -func TestErrors(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - err error - want string - }{ - {"InvalidLookupError", pkg.InvalidLookupError{ - 0xff, 0xf0, - }, "attempting to look up non-input artifact __AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}, - - {"InvalidArtifactError", pkg.InvalidArtifactError{ - 0xff, 0xfd, - }, "artifact __0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA cannot be cured"}, - - {"ChecksumMismatchError", &pkg.ChecksumMismatchError{ - Want: (pkg.Checksum)(bytes.Repeat([]byte{ - 0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f, - }, 8)), - }, "got AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + - " instead of deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"}, - - {"ResponseStatusError", pkg.ResponseStatusError( - http.StatusNotAcceptable, - ), "the requested URL returned non-OK status: Not Acceptable"}, - - {"DisallowedTypeflagError", pkg.DisallowedTypeflagError( - tar.TypeChar, - ), "disallowed typeflag '3'"}, - - {"InvalidFileModeError", pkg.InvalidFileModeError( - fs.ModeSymlink | 0777, - ), "artifact did not produce a regular file or directory"}, - - {"NoOutputError", pkg.NoOutputError{ - // empty struct - }, "artifact cured successfully but did not produce any output"}, - - {"IRKindError", &pkg.IRKindError{ - Got: pkg.IRKindEnd, - Want: pkg.IRKindIdent, - Ancillary: 0xcafebabe, - }, "got terminator IR value (0xcafebabe) instead of ident"}, - {"IRKindError invalid", &pkg.IRKindError{ - Got: 0xbeef, - Want: pkg.IRKindIdent, - Ancillary: 0xcafe, - }, "got invalid kind 48879 IR value (0xcafe) instead of ident"}, - - {"UnsupportedVariantError", pkg.UnsupportedVariantError( - "rosa", - ), `unsupported variant "rosa"`}, - - {"UnsupportedArchError zero", pkg.UnsupportedArchError(""), - "invalid architecture name"}, - {"UnsupportedArchError", pkg.UnsupportedArchError("riscv64"), - "unsupported architecture riscv64"}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - if got := tc.err.Error(); got != tc.want { - t.Errorf("Error: %q, want %q", got, tc.want) - } - }) - } -} - -func TestScrubError(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - err pkg.ScrubError - want string - unwrap []error - }{ - {"sample", *newWantScrubError( - fhs.AbsVarLib.Append("cure"), - ), `checksum mismatches: -got vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX instead of _wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA - -dangling identifiers: -_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -_v4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA - -errors during scrub: - /var/lib/cure/checksum/__8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: - artifact did not produce a regular file or directory - /var/lib/cure/checksum/invalid: - illegal base64 data at input byte 4 - /var/lib/cure/identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: - readlink /var/lib/cure/identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: invalid argument - /var/lib/cure/identifier/invalid: - illegal base64 data at input byte 4 - /var/lib/cure/nonexistent: - illegal base64 data at input byte 8 -`, []error{ - &pkg.ChecksumMismatchError{Got: pkg.MustDecode( - "vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", - ), Want: pkg.Checksum{0xff, 0}}, - - pkg.InvalidFileModeError(fs.ModeSymlink), - base64.CorruptInputError(4), - - &os.PathError{ - Op: "readlink", - Path: fhs.AbsVarLib.Append("cure").Append( - "identifier", - pkg.Encode(pkg.ID{0xfe, 0xff}), - ).String(), - Err: syscall.EINVAL, - }, - - base64.CorruptInputError(4), - base64.CorruptInputError(8), - }}, - - {"full", pkg.ScrubError{ - ChecksumMismatches: []pkg.ChecksumMismatchError{ - {Want: pkg.MustDecode("CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN")}, - }, - DanglingIdentifiers: []pkg.ID{ - (pkg.ID)(bytes.Repeat([]byte{0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f}, 8)), - (pkg.ID)(bytes.Repeat([]byte{0x71, 0xa7, 0xde, 0x6d, 0xa6, 0xde}, 8)), - }, - Errs: map[unique.Handle[string]][]error{ - unique.Make("/proc/nonexistent"): { - stub.UniqueError(0xcafe), - stub.UniqueError(0xbad), - stub.UniqueError(0xff), - }, - }, - }, `checksum mismatches: -got AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA instead of CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN - -dangling identifiers: -deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef -cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe - -errors during scrub: - /proc/nonexistent: - unique error 51966 injected by the test suite - unique error 2989 injected by the test suite - unique error 255 injected by the test suite -`, []error{ - &pkg.ChecksumMismatchError{Want: pkg.MustDecode("CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN")}, - stub.UniqueError(0xcafe), - stub.UniqueError(0xbad), - stub.UniqueError(0xff), - }}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - if got := tc.err.Error(); got != tc.want { - t.Errorf("Error:\n\n%s\n\nwant\n\n%s", got, tc.want) - } - - if unwrap := tc.err.Unwrap(); !reflect.DeepEqual(unwrap, tc.unwrap) { - t.Errorf("Unwrap: %#v, want %#v", unwrap, tc.unwrap) - } - }) - } -} - -func TestInputError(t *testing.T) { - t.Parallel() - - makeIdent := func(ident ...byte) pkg.Artifact { - var a overrideIdent - copy(a.id[:], ident) - // does not compare equal - a.TrivialArtifact = new(stubArtifact) - return a - } - - testCases := []struct { - name string - err pkg.InputError - want string - unwrap []error - }{ - {"simple", pkg.InputError{ - makeIdent(0xff, 9): stub.UniqueError(0xbad09), - makeIdent(0xff, 0): stub.UniqueError(0xbad00), - makeIdent(0xff, 0xf): stub.UniqueError(0xbad0f), - makeIdent(0xff, 1): stub.UniqueError(0xbad01), - }, `errors curing inputs: - _wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765184 injected by the test suite - _wEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765185 injected by the test suite - _wkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765193 injected by the test suite - _w8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765199 injected by the test suite`, []error{ - stub.UniqueError(0xbad00), - stub.UniqueError(0xbad01), - stub.UniqueError(0xbad09), - stub.UniqueError(0xbad0f), - }}, - - {"dedup", pkg.InputError{ - makeIdent(0xff, 9): stub.UniqueError(0xbad09), - makeIdent(0xff, 9): stub.UniqueError(0xbad09), - makeIdent(0xff, 9): stub.UniqueError(0xbad09), - makeIdent(0xff, 0): stub.UniqueError(0xbad00), - makeIdent(0xff, 0): stub.UniqueError(0xbad00), - makeIdent(0xff, 1): stub.UniqueError(0xbad01), - }, `errors curing inputs: - _wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765184 injected by the test suite - _wEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765185 injected by the test suite - _wkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: unique error 765193 injected by the test suite`, []error{ - stub.UniqueError(0xbad00), - stub.UniqueError(0xbad01), - stub.UniqueError(0xbad09), - }}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - if got := tc.err.Error(); got != tc.want { - t.Errorf("Error:\n%s\nwant\n%s", got, tc.want) - } - - if unwrap := tc.err.Unwrap(); !reflect.DeepEqual(unwrap, tc.unwrap) { - t.Errorf("Unwrap: %#v, want %#v", unwrap, tc.unwrap) - } - }) - } -} - -// earlyFailureF is a [FloodArtifact] with a large dependency graph resulting in -// a large [DependencyCureError]. -type earlyFailureF int - -func (earlyFailureF) Kind() pkg.Kind { return pkg.KindExec } -func (earlyFailureF) Params(*pkg.IContext) {} -func (earlyFailureF) IsExclusive() bool { return false } - -func (a earlyFailureF) Inputs() []pkg.Artifact { - deps := make([]pkg.Artifact, a) - for i := range deps { - deps[i] = a - 1 - } - return deps -} - -func (a earlyFailureF) Cure(*pkg.FContext) error { - if a != 0 { - panic("unexpected cure on " + strconv.Itoa(int(a))) - } - return stub.UniqueError(0xcafe) -} - -func BenchmarkEarlyDCE(b *testing.B) { - msg := message.New(log.New(os.Stderr, "dce: ", 0)) - msg.SwapVerbose(testing.Verbose()) - - for b.Loop() { - b.StopTimer() - c, err := pkg.Open(b.Context(), msg, check.MustAbs(b.TempDir()), nil) - if err != nil { - b.Fatal(err) - } - b.StartTimer() - _, _, err = c.Cure(earlyFailureF(8)) - b.StopTimer() - if !errors.Is(err, stub.UniqueError(0xcafe)) { - b.Fatalf("Cure: error = %v", err) - } - c.Close() - b.StartTimer() - } -} - -func TestDependencyCureErrorEarly(t *testing.T) { - t.Parallel() - - checkWithCache(t, []cacheTestCase{ - {"early", 0, nil, func(t *testing.T, _ *check.Absolute, c *pkg.Cache) { - _, _, err := c.Cure(earlyFailureF(8)) - if !errors.Is(err, stub.UniqueError(0xcafe)) { - t.Fatalf("Cure: error = %v", err) - } - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - "checksum": {Mode: fs.ModeDir | 0700}, - "identifier": {Mode: fs.ModeDir | 0700}, - "substitute": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} - -func TestOpen(t *testing.T) { - t.Parallel() - - t.Run("nonexistent", func(t *testing.T) { - t.Parallel() - - wantErr := &os.PathError{ - Op: "mkdir", - Path: container.Nonexistent, - Err: syscall.ENOENT, - } - if _, err := pkg.Open( - t.Context(), - message.New(nil), - check.MustAbs(container.Nonexistent), - nil, - ); !reflect.DeepEqual(err, wantErr) { - t.Errorf("Open: error = %#v, want %#v", err, wantErr) - } - }) - - t.Run("permission", func(t *testing.T) { - t.Parallel() - - tempDir := check.MustAbs(t.TempDir()) - if err := os.Chmod(tempDir.String(), 0); err != nil { - t.Fatal(err) - } else { - t.Cleanup(func() { - if err = os.Chmod(tempDir.String(), 0700); err != nil { - t.Fatal(err) - } - }) - } - - wantErr := &os.PathError{ - Op: "mkdir", - Path: tempDir.Append("cache").String(), - Err: syscall.EACCES, - } - if _, err := pkg.Open( - t.Context(), - message.New(nil), - tempDir.Append("cache"), - nil, - ); !reflect.DeepEqual(err, wantErr) { - t.Errorf("Open: error = %#v, want %#v", err, wantErr) - } - }) - - t.Run("dirty", func(t *testing.T) { - t.Parallel() - - tempDir := check.MustAbs(t.TempDir()) - if err := os.MkdirAll(tempDir.Append( - "cache", - "work", - "dirty", - ).String(), 0755); err != nil { - t.Fatal(err) - } - - wantErr := errors.New("work is not empty, scrub likely required") - if _, err := pkg.Open( - t.Context(), - message.New(nil), - tempDir.Append("cache"), - nil, - ); !reflect.DeepEqual(err, wantErr) { - t.Errorf("Open: error = %#v, want %#v", err, wantErr) - } - }) - - t.Run("scratch", func(t *testing.T) { - t.Parallel() - - tempDir := check.MustAbs(t.TempDir()) - if err := os.MkdirAll(tempDir.Append( - "cache", - "scratch", - ).String(), 0755); err != nil { - t.Fatal(err) - } - - wantErr := errors.New("scratch is present, scrub likely required") - if _, err := pkg.Open( - t.Context(), - message.New(nil), - tempDir.Append("cache"), nil, - ); !reflect.DeepEqual(err, wantErr) { - t.Errorf("Open: error = %#v, want %#v", err, wantErr) - } - }) -} - -func TestExtensionRegister(t *testing.T) { - extensionOld := extension - openedOld := opened - t.Cleanup(func() { extension = extensionOld; opened = openedOld }) - extension = "" - opened = false - - t.Run("set", func(t *testing.T) { - t.Cleanup(func() { extension = "" }) - - const want = "rosa" - pkg.SetExtension(want) - if got := pkg.Extension(); got != want { - t.Fatalf("Extension: %q, want %q", got, want) - } - }) - - t.Run("twice", func(t *testing.T) { - t.Cleanup(func() { extension = "" }) - - defer func() { - const wantPanic = "attempting to set extension twice" - if r := recover(); r != wantPanic { - t.Errorf("panic: %#v, want %q", r, wantPanic) - } - }() - pkg.SetExtension("rosa") - pkg.SetExtension("rosa") - }) - - t.Run("invalid", func(t *testing.T) { - defer func() { - var wantPanic = pkg.ErrInvalidExtension - if r := recover(); r != wantPanic { - t.Errorf("panic: %#v, want %#v", r, wantPanic) - } - }() - pkg.SetExtension(" ") - }) - - t.Run("opened", func(t *testing.T) { - t.Cleanup(func() { opened = false }) - - if _, err := pkg.Open( - t.Context(), - message.New(log.Default()), - check.MustAbs(container.Nonexistent), - nil, - ); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("Open: error = %v", err) - } - - t.Run("variant", func(t *testing.T) { - defer func() { - const wantPanic = "attempting to set extension after open" - if r := recover(); r != wantPanic { - t.Errorf("panic: %#v, want %q", r, wantPanic) - } - }() - pkg.SetExtension("rosa") - }) - - t.Run("register", func(t *testing.T) { - defer func() { - const wantPanic = "attempting to register after open" - if r := recover(); r != wantPanic { - t.Errorf("panic: %#v, want %q", r, wantPanic) - } - }() - pkg.Register(pkg.KindCustomOffset, nil) - }) - }) - - t.Run("incomplete", func(t *testing.T) { - t.Cleanup(func() { delete(irArtifact, pkg.KindCustomOffset) }) - - defer func() { - const wantPanic = "attempting to open cache with incomplete variant setup" - if r := recover(); r != wantPanic { - t.Errorf("panic: %#v, want %q", r, wantPanic) - } - }() - pkg.Register(pkg.KindCustomOffset, nil) - - t.Cleanup(func() { opened = false }) - _, _ = pkg.Open(nil, nil, nil, nil) - panic("unreachable") - }) - - t.Run("create", func(t *testing.T) { - t.Cleanup(func() { extension = "" }) - const want = "rosa" - pkg.SetExtension(want) - - base := check.MustAbs(t.TempDir()) - t.Cleanup(func() { opened = false }) - if c, err := pkg.Open(t.Context(), nil, base, nil); err != nil { - t.Fatal(err) - } else { - c.Close() - } - - if got, err := os.ReadFile(base.Append("variant").String()); err != nil { - t.Fatal(err) - } else if string(got) != want { - t.Fatalf("variant: %q", string(got)) - } - }) - - t.Run("access", func(t *testing.T) { - base := check.MustAbs(t.TempDir()) - t.Cleanup(func() { opened = false }) - - if err := os.WriteFile(base.Append("variant").String(), nil, 0); err != nil { - t.Fatal(err) - } - - wantErr := &os.PathError{ - Op: "open", - Path: base.Append("variant").String(), - Err: syscall.EACCES, - } - if _, err := pkg.Open( - t.Context(), nil, - base, nil, - ); !reflect.DeepEqual(err, wantErr) { - t.Fatalf("Open: error = %v, want %v", err, wantErr) - } - }) - - t.Run("promote", func(t *testing.T) { - t.Cleanup(func() { extension = "" }) - const want = "rosa" - pkg.SetExtension(want) - - base := check.MustAbs(t.TempDir()) - t.Cleanup(func() { opened = false }) - - variantPath := base.Append("variant") - if err := os.WriteFile(variantPath.String(), nil, 0600); err != nil { - t.Fatal(err) - } - - if _, err := pkg.Open( - t.Context(), nil, - base, - nil, - ); !reflect.DeepEqual(err, pkg.ErrWouldPromote) { - t.Fatalf("Open: error = %v", err) - } - - if p, err := os.ReadFile(variantPath.String()); err != nil { - t.Fatal(err) - } else if len(p) != 0 { - t.Fatalf("variant: %q", string(p)) - } - - if c, err := pkg.Open( - t.Context(), nil, - base, &pkg.CacheAttr{Flags: pkg.CPromoteVariant}, - ); err != nil { - t.Fatalf("Open: error = %v", err) - } else { - c.Close() - } - - if p, err := os.ReadFile(variantPath.String()); err != nil { - t.Fatal(err) - } else if string(p) != want { - t.Fatalf("variant: %q, want %q", string(p), want) - } - }) - - t.Run("open invalid", func(t *testing.T) { - base := check.MustAbs(t.TempDir()) - t.Cleanup(func() { opened = false }) - - variantPath := base.Append("variant") - if err := os.WriteFile(variantPath.String(), make([]byte, 129), 0400); err != nil { - t.Fatal(err) - } - - if _, err := pkg.Open( - t.Context(), nil, - base, nil, - ); !reflect.DeepEqual(err, pkg.ErrInvalidExtension) { - t.Fatalf("Open: error = %v", err) - } - }) - - t.Run("unsupported", func(t *testing.T) { - base := check.MustAbs(t.TempDir()) - t.Cleanup(func() { opened = false }) - - variantPath := base.Append("variant") - if err := os.WriteFile(variantPath.String(), []byte("rosa"), 0400); err != nil { - t.Fatal(err) - } - - if _, err := pkg.Open( - t.Context(), nil, - base, nil, - ); !reflect.DeepEqual(err, pkg.UnsupportedVariantError("rosa")) { - t.Fatalf("Open: error = %v", err) - } - }) -} diff --git a/internal/pkg/tar.go b/internal/pkg/tar.go deleted file mode 100644 index 9a8c4b76..00000000 --- a/internal/pkg/tar.go +++ /dev/null @@ -1,226 +0,0 @@ -package pkg - -import ( - "archive/tar" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" -) - -// A tarArtifact is an [Artifact] unpacking a tarball backed by a [FileArtifact]. -type tarArtifact struct { - // Caller-supplied backing tarball. - f Artifact -} - -var _ CuresExempt = new(tarArtifact) - -// tarArtifactNamed embeds tarArtifact for a [fmt.Stringer] tarball. -type tarArtifactNamed struct { - tarArtifact - // Copied from tarArtifact.f. - name string -} - -var _ fmt.Stringer = new(tarArtifactNamed) - -// String returns the name of the underlying [Artifact] prefixed with unpack. -func (a *tarArtifactNamed) String() string { return "unpack-" + a.name } - -// NewTar returns a new [Artifact] unpacking the tar stream produced by the -// backing [Artifact]. The source [Artifact] must be a [FileArtifact]. -func NewTar(a Artifact) Artifact { - ta := tarArtifact{a} - if s, ok := a.(fmt.Stringer); ok { - if name := s.String(); name != "" { - return &tarArtifactNamed{ta, name} - } - } - return &ta -} - -// Kind returns the hardcoded [Kind] constant. -func (a *tarArtifact) Kind() Kind { return KindTar } - -// Params is a noop. -func (a *tarArtifact) Params(*IContext) {} - -func init() { - register(KindTar, func(r *IRReader) Artifact { - a := NewTar(r.Next()) - if _, ok := r.Finalise(); ok { - panic(ErrUnexpectedChecksum) - } - return a - }) -} - -// Inputs returns a slice containing the backing file. -func (a *tarArtifact) Inputs() []Artifact { - return []Artifact{a.f} -} - -// IsExclusive returns false: decompressor and tar reader are fully sequential. -func (a *tarArtifact) IsExclusive() bool { return false } - -// A DisallowedTypeflagError describes a disallowed typeflag encountered while -// unpacking a tarball. -type DisallowedTypeflagError byte - -func (e DisallowedTypeflagError) Error() string { - return "disallowed typeflag '" + string(e) + "'" -} - -// Cure cures the [Artifact], producing a directory located at work. -func (a *tarArtifact) Cure(t *TContext) (err error) { - var r io.ReadCloser - if r, err = t.Open(a.f); err != nil { - return - } - - defer func() { - closeErr := r.Close() - if err == nil { - err = closeErr - } - }() - - type dirTargetPerm struct { - path string - mode fs.FileMode - } - var madeDirectories []dirTargetPerm - - if err = os.MkdirAll(t.GetTempDir().String(), 0700); err != nil { - return - } - var root *os.Root - if root, err = os.OpenRoot(t.GetTempDir().String()); err != nil { - return - } - defer func() { - closeErr := root.Close() - if err == nil { - err = closeErr - } - }() - - var header *tar.Header - tr := tar.NewReader(r) - for header, err = tr.Next(); err == nil; header, err = tr.Next() { - typeflag := header.Typeflag - if typeflag == 0 { - if len(header.Name) > 0 && header.Name[len(header.Name)-1] == '/' { - typeflag = tar.TypeDir - } else { - typeflag = tar.TypeReg - } - } - - if typeflag >= '0' && typeflag <= '9' && typeflag != tar.TypeDir { - if err = root.MkdirAll(filepath.Dir(header.Name), 0700); err != nil { - return - } - } - - switch typeflag { - case tar.TypeReg: - var f *os.File - if f, err = root.OpenFile( - header.Name, - os.O_CREATE|os.O_EXCL|os.O_WRONLY, - header.FileInfo().Mode()&0500, - ); err != nil { - return - } - if _, err = io.Copy(f, tr); err != nil { - _ = f.Close() - return - } else if err = f.Close(); err != nil { - return - } - break - - case tar.TypeLink: - if err = root.Link( - header.Linkname, - header.Name, - ); err != nil { - return - } - break - - case tar.TypeSymlink: - if err = root.Symlink( - header.Linkname, - header.Name, - ); err != nil { - return - } - break - - case tar.TypeDir: - madeDirectories = append(madeDirectories, dirTargetPerm{ - path: header.Name, - mode: header.FileInfo().Mode(), - }) - if err = root.MkdirAll(header.Name, 0700); err != nil { - return - } - break - - case tar.TypeChar: - t.GetMessage().Verbosef( - "%sskipping character device %d, %d %q%s", - t.cache.sgrWarn, header.Devmajor, header.Devminor, header.Name, t.cache.sgrRes, - ) - continue - - case tar.TypeXGlobalHeader: - continue // ignore - - default: - return DisallowedTypeflagError(typeflag) - } - } - if errors.Is(err, io.EOF) { - err = nil - } - if err == nil { - for _, e := range madeDirectories { - if err = root.Chmod(e.path, e.mode&0500); err != nil { - return - } - } - } else { - return - } - - temp := t.GetTempDir() - if err = os.Chmod(temp.String(), 0700); err != nil { - return - } - - var entries []os.DirEntry - if entries, err = os.ReadDir(temp.String()); err != nil { - return - } - - if len(entries) == 1 && entries[0].IsDir() { - p := temp.Append(entries[0].Name()) - if err = os.Chmod(p.String(), 0700); err != nil { - return - } - err = os.Rename(p.String(), t.GetWorkDir().String()) - } else { - err = os.Rename(temp.String(), t.GetWorkDir().String()) - } - return -} - -// CuresExempt exempts the cheap [KindTar] implementation often at the end of a -// [FileArtifact] pipeline. -func (*tarArtifact) CuresExempt() {} diff --git a/internal/pkg/tar_test.go b/internal/pkg/tar_test.go deleted file mode 100644 index 1f280e9c..00000000 --- a/internal/pkg/tar_test.go +++ /dev/null @@ -1,225 +0,0 @@ -package pkg_test - -import ( - "archive/tar" - "bytes" - "crypto/sha512" - "errors" - "io/fs" - "net/http" - "os" - "testing" - "testing/fstest" - - "hakurei.app/check" - "hakurei.app/internal/pkg" - "hakurei.app/internal/stub" -) - -func TestTar(t *testing.T) { - t.Parallel() - - want := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "checksum": {Mode: fs.ModeDir | 0500}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP": {Mode: fs.ModeDir | 0500}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check": {Mode: 0400, Data: []byte{0, 0}}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib": {Mode: fs.ModeDir | 0500}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig": {Mode: fs.ModeDir | 0500}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - - "identifier": {Mode: fs.ModeDir | 0500}, - "identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, - "identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, - - "work": {Mode: fs.ModeDir | 0500}, - } - wantEncode := pkg.Encode(want.hash()) - - wantExpand := expectsFS{ - ".": {Mode: fs.ModeDir | 0500}, - - "libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - } - wantExpandEncode := pkg.Encode(wantExpand.hash()) - - checkWithCache(t, []cacheTestCase{ - {"http", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - checkTarHTTP(t, base, c, fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP": {Mode: fs.ModeDir | 0700}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check": {Mode: 0400, Data: []byte{0, 0}}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib": {Mode: fs.ModeDir | 0700}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig": {Mode: fs.ModeDir | 0700}, - "checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, - "identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, - - "work": {Mode: fs.ModeDir | 0700}, - }, want) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantEncode + "/checksum": {Mode: fs.ModeDir | 0500}, - "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP": {Mode: fs.ModeDir | 0500}, - "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/check": {Mode: 0400, Data: []byte{0, 0}}, - "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib": {Mode: fs.ModeDir | 0500}, - "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - "checksum/" + wantEncode + "/checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP/lib/pkgconfig": {Mode: fs.ModeDir | 0500}, - "checksum/" + wantEncode + "/identifier": {Mode: fs.ModeDir | 0500}, - "checksum/" + wantEncode + "/identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, - "checksum/" + wantEncode + "/identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/1TL00Qb8dcqayX7wTO8WNaraHvY6b-KCsctLDTrb64QBCmxj_-byK1HdIUwMaFEP")}, - "checksum/" + wantEncode + "/work": {Mode: fs.ModeDir | 0500}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/snWp53xxNdx6cV5KSJqzyVWzLlTsFd8udOisIu8Rgxly09mGcxufYCoQ5CQrHDyZ": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantEncode)}, - "identifier/v9DUj2R4YK_3ae0m9VuUYI-HJysDDCENu1k10thxioycRzFH1Qejw5bBGukm4IdL": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantEncode)}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - - {"http expand", 0, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) { - checkTarHTTP(t, base, c, fstest.MapFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "lib": {Mode: fs.ModeDir | 0700}, - "lib/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - }, wantExpand) - }, expectsFS{ - ".": {Mode: fs.ModeDir | 0700}, - - "checksum": {Mode: fs.ModeDir | 0700}, - "checksum/" + wantExpandEncode: {Mode: fs.ModeDir | 0500}, - "checksum/" + wantExpandEncode + "/libedac.so": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent/libedac.so")}, - - "identifier": {Mode: fs.ModeDir | 0700}, - "identifier/1SuvgGltcsgqps9tWqjULG8RCdpx_6KVWXdsMd5xPKvdtysOxqpE-3bLu4j0WpGk": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantExpandEncode)}, - "identifier/snWp53xxNdx6cV5KSJqzyVWzLlTsFd8udOisIu8Rgxly09mGcxufYCoQ5CQrHDyZ": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/" + wantExpandEncode)}, - - "substitute": {Mode: fs.ModeDir | 0700}, - - "temp": {Mode: fs.ModeDir | 0700}, - "work": {Mode: fs.ModeDir | 0700}, - }}, - }) -} - -func checkTarHTTP( - t *testing.T, - base *check.Absolute, - c *pkg.Cache, - testdataFsys fs.FS, - want expectsKnown, -) { - var testdata string - { - var buf bytes.Buffer - w := tar.NewWriter(&buf) - if err := w.AddFS(testdataFsys); err != nil { - t.Fatalf("AddFS: error = %v", err) - } - if err := w.Close(); err != nil { - t.Fatalf("Close: error = %v", err) - } - testdata = buf.String() - } - - testdataChecksum := func() pkg.Checksum { - h := sha512.New384() - h.Write([]byte(testdata)) - return (pkg.Checksum)(h.Sum(nil)) - }() - - var transport http.Transport - client := http.Client{Transport: &transport} - transport.RegisterProtocol("file", http.NewFileTransportFS(fstest.MapFS{ - "testdata": {Data: []byte(testdata), Mode: 0400}, - })) - - tarDir := stubArtifact{ - kind: pkg.KindExec, - params: []byte("directory containing a single regular file"), - cure: func(t *pkg.TContext) error { - work := t.GetWorkDir() - if err := os.MkdirAll(work.String(), 0700); err != nil { - return err - } - return os.WriteFile( - work.Append("sample.tar.gz").String(), - []byte(testdata), - 0400, - ) - }, - } - tarDirMulti := stubArtifact{ - kind: pkg.KindExec, - params: []byte("directory containing a multiple entries"), - cure: func(t *pkg.TContext) error { - work := t.GetWorkDir() - if err := os.MkdirAll(work.Append( - "garbage", - ).String(), 0700); err != nil { - return err - } - return os.WriteFile( - work.Append("sample.tar.gz").String(), - []byte(testdata), - 0400, - ) - }, - } - tarDirType := stubArtifact{ - kind: pkg.KindExec, - params: []byte("directory containing a symbolic link"), - cure: func(t *pkg.TContext) error { - work := t.GetWorkDir() - if err := os.MkdirAll(work.String(), 0700); err != nil { - return err - } - return os.Symlink( - work.String(), - work.Append("sample.tar.gz").String(), - ) - }, - } - // destroy these to avoid including it in flatten test case - defer newDestroyArtifactFunc(&tarDir)(t, base, c) - defer newDestroyArtifactFunc(&tarDirMulti)(t, base, c) - defer newDestroyArtifactFunc(&tarDirType)(t, base, c) - - cureMany(t, c, []cureStep{ - {"file", pkg.NewTar(pkg.NewHTTPGet( - &client, - "file:///testdata", - testdataChecksum, - )), ignorePathname, want, pkg.WNew, nil}, - - {"directory", pkg.NewTar(&tarDir), ignorePathname, want, pkg.WNew, nil}, - - {"multiple entries", pkg.NewTar(&tarDirMulti), nil, nil, pkg.WNew, errors.New( - "input directory does not contain a single regular file", - )}, - - {"bad type", pkg.NewTar(&tarDirType), nil, nil, pkg.WNew, errors.New( - "input directory does not contain a single regular file", - )}, - - {"error passthrough", pkg.NewTar(&stubArtifact{ - kind: pkg.KindExec, - params: []byte("doomed artifact"), - cure: func(t *pkg.TContext) error { - return stub.UniqueError(0xcafe) - }, - }), nil, nil, pkg.WNew, stub.UniqueError(0xcafe)}, - }) -} diff --git a/internal/rosa/azalea/evaluate.go b/internal/rosa/azalea/evaluate.go index b35a056a..cb5e8d12 100644 --- a/internal/rosa/azalea/evaluate.go +++ b/internal/rosa/azalea/evaluate.go @@ -200,12 +200,15 @@ func (e EvaluationError) Error() string { } var ( - // IdentInputs is a special array argument in a package declaration whose + + // IdentTools is a special array argument in a package declaration whose // values of [Ident] are kept as is when passed to a [PF]. + IdentTools = unique.Make(Ident("tools")) + // IdentInputs has the same semantics as [IdentTools]. IdentInputs = unique.Make(Ident("inputs")) - // IdentRuntime has the same semantics as [IdentInputs]. + // IdentRuntime has the same semantics as [IdentTools]. IdentRuntime = unique.Make(Ident("runtime")) - // IdentExtra has the same semantics as [IdentInputs]. + // IdentExtra has the same semantics as [IdentTools]. IdentExtra = unique.Make(Ident("extra")) // IdentSource is a special argument in a package declaration where an // assignment of a [Val] with a single [Ident] passes through the evaluator. @@ -356,6 +359,7 @@ func evaluateAny(d PF, s []Frame, expr, rp any) bool { farg := FArg{R: arg.R} if e.Package { for _, special := range [...]unique.Handle[Ident]{ + IdentTools, IdentInputs, IdentRuntime, IdentExtra, @@ -365,7 +369,13 @@ func evaluateAny(d PF, s []Frame, expr, rp any) bool { panic(ErrInvalidSpecial) } farg.K = names[0] - if err := storeE(&farg.V, arg.V[0]); err != nil { + + v := arg.V[0] + if p, _ok := v.(Ident); _ok && p == "nil" { + v = Nil{} + } + + if err := storeE(&farg.V, v); err != nil { panic(err) } fargs = append(fargs, farg) diff --git a/internal/rosa/builtins.go b/internal/rosa/builtins.go index dbafa397..edca2333 100644 --- a/internal/rosa/builtins.go +++ b/internal/rosa/builtins.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) const ( diff --git a/internal/rosa/cmake.go b/internal/rosa/cmake.go index 8a67f4bc..f9e5e38c 100644 --- a/internal/rosa/cmake.go +++ b/internal/rosa/cmake.go @@ -34,14 +34,17 @@ type CMakeHelper struct { var _ Helper = new(CMakeHelper) -// extra returns the cmake handle alongside either ninja or make. -func (attr *CMakeHelper) extra() P { +// tools returns the cmake handle alongside either ninja or make. +func (attr *CMakeHelper) tools() P { if attr != nil && attr.Make { return P{_cmake, _make} } return P{_cmake, _ninja} } +// inputs is a noop. +func (*CMakeHelper) inputs() P { return nil } + // wantsChmod returns false. func (*CMakeHelper) wantsChmod() bool { return false } diff --git a/internal/rosa/etc.go b/internal/rosa/etc.go index 18c9750b..0af51830 100644 --- a/internal/rosa/etc.go +++ b/internal/rosa/etc.go @@ -8,7 +8,7 @@ import ( "syscall" "hakurei.app/fhs" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) // cureEtc contains deterministic elements of /etc, made available as part of diff --git a/internal/rosa/gentoo.go b/internal/rosa/gentoo.go index a0a8e525..a08fd705 100644 --- a/internal/rosa/gentoo.go +++ b/internal/rosa/gentoo.go @@ -6,7 +6,7 @@ import ( "os" "path/filepath" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) // gentooOverlay are symlinks on top of a Gentoo LLVM stage3 tarball for diff --git a/internal/rosa/git.go b/internal/rosa/git.go index bda380ce..70d5b6f8 100644 --- a/internal/rosa/git.go +++ b/internal/rosa/git.go @@ -4,7 +4,7 @@ import ( "path" "strings" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) var ( diff --git a/internal/rosa/go.go b/internal/rosa/go.go index 1b42447a..1127cf3d 100644 --- a/internal/rosa/go.go +++ b/internal/rosa/go.go @@ -3,7 +3,7 @@ package rosa import ( "slices" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) // newGo returns a specific version of the Go toolchain. diff --git a/internal/rosa/llvm.go b/internal/rosa/llvm.go index dddd6388..6d1dcf0b 100644 --- a/internal/rosa/llvm.go +++ b/internal/rosa/llvm.go @@ -5,7 +5,7 @@ import ( "slices" "strings" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) var ( @@ -54,7 +54,7 @@ func init() { Description: "a collection of modular and reusable compiler and toolchain technologies", Website: "https://llvm.org", - Dependencies: P{ + Runtimes: P{ _zlib, _zstd, _musl, diff --git a/internal/rosa/llvm_test.go b/internal/rosa/llvm_test.go index 52562579..3fa8fc9d 100644 --- a/internal/rosa/llvm_test.go +++ b/internal/rosa/llvm_test.go @@ -3,12 +3,12 @@ package rosa_test import ( "testing" - "hakurei.app/internal/pkg" "hakurei.app/internal/rosa" + "hakurei.app/pkg" ) func TestLLVMInputs(t *testing.T) { - const wantInputCount = 466 + const wantInputCount = 472 _, llvm := rosa.MustLoad(rosa.H("llvm")) var n int diff --git a/internal/rosa/make.go b/internal/rosa/make.go index 710225a2..1c0e52d6 100644 --- a/internal/rosa/make.go +++ b/internal/rosa/make.go @@ -51,8 +51,8 @@ type MakeHelper struct { var _ Helper = new(MakeHelper) -// extra returns make and other optional dependencies. -func (attr *MakeHelper) extra() P { +// tools returns make and other optional dependencies. +func (attr *MakeHelper) tools() P { if attr == nil || !attr.OmitDefaults { return P{ _make, @@ -64,6 +64,9 @@ func (attr *MakeHelper) extra() P { return P{_make} } +// inputs is a noop. +func (*MakeHelper) inputs() P { return nil } + // wantsChmod returns whether the build system needs to be generated. func (attr *MakeHelper) wantsChmod() bool { return attr != nil && attr.Generate != "" diff --git a/internal/rosa/meson.go b/internal/rosa/meson.go index 65e9fde9..97262fbb 100644 --- a/internal/rosa/meson.go +++ b/internal/rosa/meson.go @@ -34,14 +34,17 @@ type MesonHelper struct { var _ Helper = new(MesonHelper) -// extra returns hardcoded meson runtime dependencies. -func (attr *MesonHelper) extra() P { +// tools returns hardcoded meson runtime dependencies. +func (attr *MesonHelper) tools() P { if !attr.isPython() { return P{_muon} } return P{_meson} } +// inputs is a noop. +func (*MesonHelper) inputs() P { return nil } + // wantsChmod returns false. func (*MesonHelper) wantsChmod() bool { return false } diff --git a/internal/rosa/mirror.go b/internal/rosa/mirror.go index 80d4509c..20dd3231 100644 --- a/internal/rosa/mirror.go +++ b/internal/rosa/mirror.go @@ -17,8 +17,8 @@ import ( "syscall" "unique" - "hakurei.app/internal/pkg" "hakurei.app/message" + "hakurei.app/pkg" ) // Remote is an authenticated cache mirror. @@ -451,7 +451,6 @@ func writeF[B any]( return } m.msg.GetLogger().Println(err) - w.WriteHeader(http.StatusInternalServerError) } // Register configures an [http.ServeMux] for servicing mirror requests. diff --git a/internal/rosa/mirror_test.go b/internal/rosa/mirror_test.go index 6c3ec92e..d45f9ae8 100644 --- a/internal/rosa/mirror_test.go +++ b/internal/rosa/mirror_test.go @@ -13,9 +13,9 @@ import ( "hakurei.app/check" "hakurei.app/fhs" - "hakurei.app/internal/pkg" "hakurei.app/internal/rosa" "hakurei.app/message" + "hakurei.app/pkg" ) func TestMirror(t *testing.T) { diff --git a/internal/rosa/package/acl/package.az b/internal/rosa/package/acl/package.az index 6908ff88..06c9abe2 100644 --- a/internal/rosa/package/acl/package.az +++ b/internal/rosa/package/acl/package.az @@ -5,7 +5,7 @@ package attr { version# = "2.6.0"; source = remoteTar { - url = "https://download.savannah.nongnu.org/releases/attr/"+ + url = "https://download-mirror.savannah.nongnu.org/releases/attr/"+ "attr-"+version+".tar.gz"; checksum = "pp-NvD1cMIwZycNwZGW2ez-PbTEpHxrRnVH27csj9QdN4oEBkEbJOZX1IIvKnTWS"; compress = gzip; @@ -17,11 +17,8 @@ package attr { populateUsrBin = true; exec = make {}; - inputs = [ - perl, - - kernel-headers, - ]; + tools = [ perl ]; + inputs = [ kernel-headers ]; } package acl { @@ -31,7 +28,7 @@ package acl { version# = "2.4.0"; source = remoteTar { - url = "https://download.savannah.nongnu.org/releases/acl/"+ + url = "https://download-mirror.savannah.nongnu.org/releases/acl/"+ "acl-"+version+".tar.gz"; checksum = "U2eaAsWrhhzEfhyprwHQQG55bFPZxtsHk1Usnd9Jb_g-H__pJod4H3RTrL4ipXDw"; compress = gzip; @@ -43,5 +40,4 @@ package acl { }; inputs = [ attr, kernel-headers ]; - runtime = [ attr ]; } diff --git a/internal/rosa/package/argp-standalone.az b/internal/rosa/package/argp-standalone.az index 13433ac8..c8a2943e 100644 --- a/internal/rosa/package/argp-standalone.az +++ b/internal/rosa/package/argp-standalone.az @@ -21,5 +21,6 @@ install -D -m755 libargp.a /work/system/lib/libargp.a `; }; - inputs = [ diffutils ]; + tools = [ diffutils ]; + unlikely = true; } diff --git a/internal/rosa/package/awk.az b/internal/rosa/package/awk.az index 2b2dc4a4..b572f7a4 100644 --- a/internal/rosa/package/awk.az +++ b/internal/rosa/package/awk.az @@ -23,7 +23,7 @@ package awk { install = "install -vD a.out /work/system/bin/awk"; }; - inputs = [ + tools = [ m4, byacc, diffutils, diff --git a/internal/rosa/package/bison.az b/internal/rosa/package/bison.az index 67aed088..9a75032e 100644 --- a/internal/rosa/package/bison.az +++ b/internal/rosa/package/bison.az @@ -26,5 +26,5 @@ package bison { ]; }; - inputs = [ m4 ]; + tools = [ m4 ]; } diff --git a/internal/rosa/package/buildcatrust.az b/internal/rosa/package/buildcatrust.az index 362ec13c..4c9f94ff 100644 --- a/internal/rosa/package/buildcatrust.az +++ b/internal/rosa/package/buildcatrust.az @@ -15,5 +15,4 @@ package buildcatrust { exec = pip {}; inputs = [ python-flit-core ]; - runtime = [ python ]; } diff --git a/internal/rosa/package/cairo.az b/internal/rosa/package/cairo.az index 46102c19..7ffbb304 100644 --- a/internal/rosa/package/cairo.az +++ b/internal/rosa/package/cairo.az @@ -30,18 +30,9 @@ package cairo { test = false; }; - inputs = [ - awk, - - zlib, - libpng, - fontconfig, - freetype, - pixman, - xcb, - ]; + tools = [ awk ]; - runtime = [ + inputs = [ zlib, libpng, fontconfig, diff --git a/internal/rosa/package/check.az b/internal/rosa/package/check.az index 8b4d6268..2598483f 100644 --- a/internal/rosa/package/check.az +++ b/internal/rosa/package/check.az @@ -16,5 +16,5 @@ package check { }; }; - inputs = [ coreutils ]; + tools = [ coreutils ]; } diff --git a/internal/rosa/package/connman/package.az b/internal/rosa/package/connman/package.az index e4b09aa5..0d8eb5af 100644 --- a/internal/rosa/package/connman/package.az +++ b/internal/rosa/package/connman/package.az @@ -16,22 +16,17 @@ package connman { generate = "./bootstrap"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, - - dbus, - iptables, - gnutls, - readline, - kernel-headers, ]; - runtime = [ + inputs = [ dbus, iptables, gnutls, readline, + kernel-headers, ]; } diff --git a/internal/rosa/package/cryptsetup.az b/internal/rosa/package/cryptsetup.az index 94a9309e..2c3b679c 100644 --- a/internal/rosa/package/cryptsetup.az +++ b/internal/rosa/package/cryptsetup.az @@ -3,12 +3,12 @@ package cryptsetup { website = "https://gitlab.com/cryptsetup/cryptsetup"; anitya = 13709; - version# = "2.8.7"; + version# = "2.8.8"; source = remoteGitLab { domain = "gitlab.com"; suffix = "cryptsetup/cryptsetup"; ref = "v"+version; - checksum = "eDi_UaMtwhIzqpsdcA91aVb56zCYDrsDzeMWfWXAnPbuvn2bEgeH5dFs7mouHz8V"; + checksum = "Q88h-7N-QFJcBe-MbtkFu0fSjrhRUF8WYCH4vhznbtFjniXLXRKMku-JoZAtHQMS"; }; bin = [ "bash" ]; @@ -20,24 +20,19 @@ package cryptsetup { }; }; - inputs = [ + tools = [ bash, gettext, coreutils, - - popt, - lvm2, - json-c, - e2fsprogs, - argp-standalone, - kernel-headers, ]; - runtime = [ + inputs = [ popt, lvm2, json-c, e2fsprogs, openssl, + argp-standalone, + kernel-headers, ]; } diff --git a/internal/rosa/package/curl.az b/internal/rosa/package/curl.az index eee21bdd..a54deccc 100644 --- a/internal/rosa/package/curl.az +++ b/internal/rosa/package/curl.az @@ -3,10 +3,10 @@ package curl { website = "https://curl.se"; anitya = 381; - version# = "8.21.0"; + version# = "8.22.0"; source = remoteTar { url = "https://curl.se/download/curl-"+version+".tar.bz2"; - checksum = "lJSm8bVjS0OmsarEdbvejdQdvXsb7yGarlr6oMtA9FW1EXOga8zZxa1LPtfaq_qX"; + checksum = "zeIqlmHElDa3IPLZq86ICRxn1YoiXlNCdx30BuahcUToYqjSuyvu8AtRqbzKH_06"; compress = bzip2; }; @@ -28,17 +28,14 @@ package curl { ]; }; - inputs = [ + tools = [ perl, python, pkg-config, diffutils, - - libpsl, - openssl, ]; - runtime = [ + inputs = [ zlib, zstd, libpsl, diff --git a/internal/rosa/package/dbus.az b/internal/rosa/package/dbus.az index aacf2f81..402cc6fb 100644 --- a/internal/rosa/package/dbus.az +++ b/internal/rosa/package/dbus.az @@ -29,11 +29,6 @@ package dbus { glib, libexpat, ]; - - runtime = [ - glib, - libexpat, - ]; } package xdg-dbus-proxy { @@ -41,11 +36,11 @@ package xdg-dbus-proxy { website = "https://github.com/flatpak/xdg-dbus-proxy"; anitya = 58434; - version# = "0.1.8"; + version# = "0.1.9"; source = remoteGitHub { suffix = "flatpak/xdg-dbus-proxy"; tag = version; - checksum = "ygdW0EEvZ-_PJfnFTNoobITmRzqJecTinBsJd_iJQrRvzidwqESCjeNkB55smZ2y"; + checksum = "HfAi2M60v3wE_TbESYZ4eMB4G2-spBt0x0UrKaRp7FNVywDbg9d_L3_kr2QXzlfp"; }; exec = meson { diff --git a/internal/rosa/package/dtc.az b/internal/rosa/package/dtc.az index e894ad48..62f58560 100644 --- a/internal/rosa/package/dtc.az +++ b/internal/rosa/package/dtc.az @@ -22,7 +22,7 @@ package dtc { }; }; - inputs = [ + tools = [ flex, bison, m4, diff --git a/internal/rosa/package/e2fsprogs.az b/internal/rosa/package/e2fsprogs.az index 5e799de3..833d6e14 100644 --- a/internal/rosa/package/e2fsprogs.az +++ b/internal/rosa/package/e2fsprogs.az @@ -17,12 +17,12 @@ package e2fsprogs { }; }; - inputs = [ + tools = [ perl, gzip, bzip2, diffutils, - - kernel-headers, ]; + + inputs = [ kernel-headers ]; } diff --git a/internal/rosa/package/elfutils/package.az b/internal/rosa/package/elfutils/package.az index 89db25ff..5f35670e 100644 --- a/internal/rosa/package/elfutils/package.az +++ b/internal/rosa/package/elfutils/package.az @@ -27,10 +27,12 @@ package elfutils { check = nil; }; - inputs = [ + tools = [ m4, pkgconf, + ]; + inputs = [ zlib, bzip2, zstd, @@ -39,12 +41,4 @@ package elfutils { musl-obstack, kernel-headers, ]; - - runtime = [ - zlib, - bzip2, - zstd, - musl-fts, - musl-obstack, - ]; } diff --git a/internal/rosa/package/fcft.az b/internal/rosa/package/fcft.az index 51e742fe..55b4b80c 100644 --- a/internal/rosa/package/fcft.az +++ b/internal/rosa/package/fcft.az @@ -19,10 +19,4 @@ package fcft { utf8proc, tllist, ]; - - runtime = [ - pixman, - fontconfig, - utf8proc, - ]; } diff --git a/internal/rosa/package/firmware.az b/internal/rosa/package/firmware.az index a1cc64b3..e87a34c1 100644 --- a/internal/rosa/package/firmware.az +++ b/internal/rosa/package/firmware.az @@ -3,12 +3,12 @@ package firmware { website = "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git"; anitya = 141464; - version# = "20260810"; + version# = "20260916"; source = remoteGitLab { domain = "gitlab.com"; suffix = "kernel-firmware/linux-firmware"; ref = version; - checksum = "XNUhz9QstaDKNegM3Bl0-TtZzpsvX7-E8WMnDCO7KBhBC6t_TxsdG05-ETdXynry"; + checksum = "Yv3yaffN6-ARSFSnDm-0gHM3T7m3ZwMqBLIP8UA3Jpe23Jk8MyNLUE44wJMy_DH6"; }; // dedup creates temporary file @@ -37,7 +37,7 @@ package firmware { install = "make " + jobsFlagE + " " + loadFlagE + " DESTDIR=/work/system dedup"; }; - inputs = [ + tools = [ parallel, rdfind, zstd, diff --git a/internal/rosa/package/flex.az b/internal/rosa/package/flex.az index 2358b1d3..78bba323 100644 --- a/internal/rosa/package/flex.az +++ b/internal/rosa/package/flex.az @@ -14,5 +14,5 @@ package flex { exec = make {}; - inputs = [ m4 ]; + tools = [ m4 ]; } diff --git a/internal/rosa/package/fontconfig.az b/internal/rosa/package/fontconfig.az index 07c4499e..ad46b158 100644 --- a/internal/rosa/package/fontconfig.az +++ b/internal/rosa/package/fontconfig.az @@ -15,7 +15,7 @@ package fontconfig { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, @@ -23,13 +23,9 @@ package fontconfig { gperf, gzip, python, - - zlib, - libexpat, - freetype, ]; - runtime = [ + inputs = [ zlib, libexpat, freetype, diff --git a/internal/rosa/package/foot.az b/internal/rosa/package/foot.az index e3166db7..90d3cbe3 100644 --- a/internal/rosa/package/foot.az +++ b/internal/rosa/package/foot.az @@ -3,27 +3,23 @@ package foot { website = "https://codeberg.org/dnkl/foot"; anitya = 141611; - version# = "1.27.0"; + version# = "1.28.0"; source = remoteTar { url = "https://codeberg.org/dnkl/foot/releases/download/"+version+ "/foot-"+version+".tar.gz"; - checksum = "22rG3rMuQI8Ae2XTcZgOJ-6AbF7Taex8XpgvD6Fyg1Mn5Va67FfUcK3tRaq61TFg"; + checksum = "w16syUlOAYlsJZ-a4QY9dfu4ph3yFypw05RXocNmTrefghzmT-HhV1O89QPAj5mV"; compress = gzip; }; exec = meson {}; - inputs = [ - ncurses, + tools = [ ncurses ]; + inputs = [ xkbcommon, tllist, fcft, + wayland-protocols, kernel-headers, ]; - - runtime = [ - xkbcommon, - fcft, - ]; } diff --git a/internal/rosa/package/freetype.az b/internal/rosa/package/freetype.az index cfa5933a..19af2dd4 100644 --- a/internal/rosa/package/freetype.az +++ b/internal/rosa/package/freetype.az @@ -5,7 +5,7 @@ package freetype { version# = "2.14.3"; source = remoteTar { - url = "https://download.savannah.gnu.org/releases/freetype/"+ + url = "https://download-mirror.savannah.gnu.org/releases/freetype/"+ "freetype-"+version+".tar.gz"; checksum = "-WfLv8fVJNyCHpP_lriiDzOcVbBL9ajdQ3tl8AzIIUa9-8sVpU9irxOmSMgRHWYz"; compress = gzip; diff --git a/internal/rosa/package/fribidi.az b/internal/rosa/package/fribidi.az index 0bd45693..28df4375 100644 --- a/internal/rosa/package/fribidi.az +++ b/internal/rosa/package/fribidi.az @@ -3,11 +3,11 @@ package fribidi { website = "https://github.com/fribidi/fribidi"; anitya = 857; - version# = "1.0.16"; + version# = "1.0.17"; source = remoteGitHub { suffix = "fribidi/fribidi"; tag = "v"+version; - checksum = "Sn9hwdgViXW9_MOi8ovAcBgQy6s6o4cc0o260k76nqCRzxmxxD1-B37GVkJ2EoMy"; + checksum = "yJWcJqYDmzdyFB-kc9J1JDPLiqWHTw2n2bheXvqzP66bk56HYWW9sz_w-TMUqiEV"; }; exec = meson { diff --git a/internal/rosa/package/fuse/package.az b/internal/rosa/package/fuse/package.az index 398fe0df..3e43c3c6 100644 --- a/internal/rosa/package/fuse/package.az +++ b/internal/rosa/package/fuse/package.az @@ -3,12 +3,12 @@ package fuse { website = "https://github.com/libfuse/libfuse"; anitya = 861; - version# = "3.18.2"; + version# = "3.18.3"; source = remoteGitHubRelease { suffix = "libfuse/libfuse"; tag = "fuse-"+version; name = "fuse-"+version+".tar.gz"; - checksum = "iL-7b7eUtmlVSf5cSq0dzow3UiqSjBmzV3cI_ENPs1tXcHdktkG45j1V12h-4jZe"; + checksum = "zQuo1ZLCePjlgxbOO8BuiLPPHwjbU0IItSo3K9_BtcxIid027b7XQWfm07c1T5D4"; compress = gzip; }; patches = [ "example-include.patch" ]; @@ -20,15 +20,8 @@ package fuse { "useroot": "false"; "initscriptdir": "/system/etc"; }; - - postCompile = "python3 -m pytest test/"; - // this project uses pytest - test = false; }; - inputs = [ - python-pytest, - - kernel-headers, - ]; + tools = [ bash ]; + inputs = [ kernel-headers ]; } diff --git a/internal/rosa/package/gawk/package.az b/internal/rosa/package/gawk/package.az index adb6761b..18215337 100644 --- a/internal/rosa/package/gawk/package.az +++ b/internal/rosa/package/gawk/package.az @@ -19,7 +19,7 @@ package gawk { exec = make {}; - inputs = [ + tools = [ diffutils, util-linux, ]; diff --git a/internal/rosa/package/gcc/package.az b/internal/rosa/package/gcc/package.az index e9a908bc..b6dc9692 100644 --- a/internal/rosa/package/gcc/package.az +++ b/internal/rosa/package/gcc/package.az @@ -12,8 +12,7 @@ package binutils { patches = [ "binutils-remove-unused.patch" ]; exec = make {}; - - inputs = [ bash ]; + tools = [ bash ]; } package gcc { @@ -69,21 +68,15 @@ ln -s system/lib /work/ check = nil; }; - inputs = [ - binutils, + tools = [ binutils ]; - mpc, - zlib, - libucontext, - kernel-headers, - ]; - - runtime = [ + inputs = [ binutils, mpc, zlib, zstd, libucontext, + kernel-headers, ]; } diff --git a/internal/rosa/package/git.az b/internal/rosa/package/git.az index 6eee0d43..7ebb8113 100644 --- a/internal/rosa/package/git.az +++ b/internal/rosa/package/git.az @@ -63,6 +63,7 @@ disable_test t5515-fetch-merge-logic disable_test t4124-apply-ws-rule disable_test t3404-rebase-interactive disable_test t7527-builtin-fsmonitor +disable_test t5702-protocol-v2 `; check = [ @@ -79,23 +80,19 @@ disable_test t7527-builtin-fsmonitor install`; }; - inputs = [ + tools = [ // test suite hangs on mksh bash, diffutils, autoconf, gettext, - - zlib, - curl, - libexpat, - kernel-headers, ]; - runtime = [ + inputs = [ zlib, curl, libexpat, + kernel-headers, ]; } diff --git a/internal/rosa/package/glib/package.az b/internal/rosa/package/glib/package.az index fc0ad64b..5c82be35 100644 --- a/internal/rosa/package/glib/package.az +++ b/internal/rosa/package/glib/package.az @@ -3,11 +3,11 @@ package glib { website = "https://developer.gnome.org/glib"; anitya = 10024; - version# = "2.89.4"; + version# = "2.90.0"; source = remoteGit { url = "https://gitlab.gnome.org/GNOME/glib.git"; tag = version; - checksum = "I_nTEVp4rpOQ4VCmJpesjvq3VbFJ3oLRAUVvsdJEfFiY4XXsvi0ITTqCEWe9Lr47"; + checksum = "7TWtYVOWNTkqg1SLAtNyhkUnvXB9jshJHCModCZl9JAzogKKoBebfMDmq2edOpXm"; }; files = { @@ -23,16 +23,12 @@ package glib { skipTests = [ "gio" ]; }; - inputs = [ + tools = [ python-packaging, bash, - - pcre2, - libffi, - zlib, ]; - runtime = [ + inputs = [ pcre2, libffi, zlib, diff --git a/internal/rosa/package/glslang.az b/internal/rosa/package/glslang.az index 39674f98..3cbd4fa0 100644 --- a/internal/rosa/package/glslang.az +++ b/internal/rosa/package/glslang.az @@ -21,7 +21,7 @@ package glslang { }; }; - inputs = [ + tools = [ python, bash, diffutils, diff --git a/internal/rosa/package/gnu.az b/internal/rosa/package/gnu.az index b3ed798c..2c408206 100644 --- a/internal/rosa/package/gnu.az +++ b/internal/rosa/package/gnu.az @@ -22,11 +22,8 @@ chmod +w tests/test-c32ispunct.sh && echo '#!/bin/sh' > tests/test-c32ispunct.sh defaults = false; }; - inputs = [ - diffutils, - - kernel-headers, - ]; + tools = [ diffutils ]; + inputs = [ kernel-headers ]; } package autoconf { @@ -51,7 +48,7 @@ package autoconf { ]; }; - inputs = [ + tools = [ m4, perl, bash, @@ -69,10 +66,10 @@ package automake { website = "https://www.gnu.org/software/automake"; anitya = 144; - version# = "1.18.1"; + version# = "1.19"; source = remoteTar { url = "https://mirrors.kernel.org/gnu/automake/automake-"+version+".tar.gz"; - checksum = "FjvLG_GdQP7cThTZJLDMxYpRcKdpAVG-YDs1Fj1yaHlSdh_Kx6nRGN14E0r_BjcG"; + checksum = "p-EeV6uh3lSbLtsnxI-l0HxC_QAnANoEdrQTI2l_VCRtYQpvmbwXIglo8S6DAI7G"; compress = gzip; }; @@ -90,7 +87,7 @@ test_disable '#!/bin/sh' t/pr9.sh exec = make {}; - inputs = [ + tools = [ grep, gzip, autoconf, @@ -122,30 +119,12 @@ package libtool { ]; }; - inputs = [ + tools = [ m4, diffutils, ]; } -package gzip { - description = "a popular data compression program"; - website = "https://www.gnu.org/software/gzip"; - anitya = 1290; - - version# = "1.14"; - source = remoteTar { - url = "https://mirrors.kernel.org/gnu/gzip/gzip-"+version+".tar.gz"; - checksum = "NWhjUavnNfTDFkZJyAUonL9aCOak8GVajWX2OMlzpFnuI0ErpBFyj88mz2xSjz0q"; - compress = gzip; - }; - - exec = make { - // dependency loop - check = nil; - }; -} - package sed { description = "a non-interactive command-line text editor"; website = "https://www.gnu.org/software/sed"; @@ -166,12 +145,8 @@ test_disable '#!/bin/sh' testsuite/read-error-stale-errno.sh `; exec = make {}; - - inputs = [ - diffutils, - - kernel-headers, - ]; + tools = [ diffutils ]; + inputs = [ kernel-headers ]; } package diffutils { @@ -248,8 +223,7 @@ test_disable 'int main(){return 0;}' gnulib-tests/test-c32ispunct.c `; exec = make {}; - - inputs = [ diffutils ]; + tools = [ diffutils ]; } package gettext { @@ -283,13 +257,13 @@ touch gettext-tools/autotools/archive.dir.tar exec = make {}; - inputs = [ + tools = [ diffutils, gzip, sed, - - kernel-headers, ]; + + inputs = [ kernel-headers ]; } package findutils { @@ -314,7 +288,7 @@ echo 'int main(){return 0;}' > tests/xargs/test-sigusr.c exec = make {}; - inputs = [ + tools = [ diffutils, sed, ]; @@ -347,10 +321,10 @@ package coreutils { website = "https://www.gnu.org/software/coreutils"; anitya = 343; - version# = "9.11"; + version# = "9.12"; source = remoteTar { url = "https://mirrors.kernel.org/gnu/coreutils/coreutils-"+version+".tar.gz"; - checksum = "t8UMed5wpFEoC56aa42_yidfOAaRGzOfj7MRtQkkqgGbpXiskNA8bd-EmVSQkZie"; + checksum = "MYYday_Pwp2YFSgVHeCQWJBhJCBhqkIkfML4_a_uHvM_FsKbYQ4vbXy60FprcUoo"; compress = gzip; }; @@ -377,12 +351,12 @@ test_disable 'int main(){return 0;}' gnulib-tests/test-lchown.c }; }; - inputs = [ + tools = [ perl, bash, - - kernel-headers, ]; + + inputs = [ kernel-headers ]; } package tar { @@ -413,7 +387,7 @@ package tar { ]; }; - inputs = [ + tools = [ diffutils, gzip, @@ -439,7 +413,7 @@ package texinfo { check = nil; }; - inputs = [ perl ]; + tools = [ perl ]; runtime = [ perl, @@ -461,7 +435,7 @@ package gperf { exec = make {}; - inputs = [ diffutils ]; + tools = [ diffutils ]; } package bc { @@ -482,9 +456,7 @@ package bc { exec = make {}; - inputs = [ - texinfo, - ]; + tools = [ texinfo ]; } package libiconv { @@ -507,17 +479,17 @@ package parallel { website = "https://www.gnu.org/software/parallel"; anitya = 5448; - version# = "20260722"; + version# = "20260822"; source = remoteTar { url = "https://mirrors.kernel.org/gnu/parallel/parallel-"+version+".tar.bz2"; - checksum = "liG9pBSRqUPqDuRXmnJPoVh-G53lDKkkO8GGabSxQnSho4b9BD0EaJDdScgVhnwj"; + checksum = "zf-5ZEpAZ073coMSyOY7NRmXwCRPWvpjbjOMOvirU9fHlU0WWovwss-bAMrh0q_G"; compress = bzip2; }; bin = [ "bash" ]; exec = make {}; - inputs = [ + tools = [ perl, bash, ]; @@ -546,10 +518,7 @@ test_disable 'int main(){return 0;}' tests/test-c32ispunct.c `; exec = make {}; - - inputs = [ - diffutils, - ]; + tools = [ diffutils ]; } package libtasn1 { @@ -587,7 +556,6 @@ package readline { }; inputs = [ ncurses ]; - runtime = [ ncurses ]; } package gmp { @@ -609,7 +577,7 @@ package gmp { exec = make {}; - inputs = [ m4 ]; + tools = [ m4 ]; } package mpfr { @@ -628,7 +596,6 @@ package mpfr { exec = make {}; inputs = [ gmp ]; - runtime = [ gmp ]; } package mpc { @@ -652,13 +619,11 @@ package mpc { generate = "autoreconf -vfi"; }; - inputs = [ + tools = [ automake, libtool, texinfo, - - mpfr, ]; - runtime = [ mpfr ]; + inputs = [ mpfr ]; } diff --git a/internal/rosa/package/gnutls/package.az b/internal/rosa/package/gnutls/package.az index 833b614f..9ab17176 100644 --- a/internal/rosa/package/gnutls/package.az +++ b/internal/rosa/package/gnutls/package.az @@ -40,7 +40,7 @@ echo 'int main(){return 0;}' > tests/mini-dtls-fragments.c }; }; - inputs = [ + tools = [ gzip, automake, libtool, @@ -54,20 +54,15 @@ echo 'int main(){return 0;}' > tests/mini-dtls-fragments.c diffutils, nss-cacert, - libev, - zlib, - zstd, - p11-kit, - nettle, - libunistring, - kernel-headers, ]; - runtime = [ + inputs = [ zlib, zstd, p11-kit, nettle, + libev, libunistring, + kernel-headers, ]; } diff --git a/internal/rosa/package/go/package.az b/internal/rosa/package/go.az index d06720b5..89d3732d 100644 --- a/internal/rosa/package/go/package.az +++ b/internal/rosa/package/go.az @@ -3,13 +3,12 @@ package go { website = "https://go.dev"; anitya = 1227; - version# = "1.27.0"; + version# = "1.27.1"; source = remoteTar { url = "https://go.dev/dl/go"+version+".src.tar.gz"; - checksum = "Ookl2vP6KMfJuxaxX6nXZ1gnL43wZPl5GjpVsrI7IJiB4b3xr1PbnbYIOJ8v5jzG"; + checksum = "Jw7OCTZbixLPF0E2WafFBh6DnxbsElvEg_RsMGkrzAmi0pyJTI3eQqLJsvXlVtbK"; compress = gzip; }; - patches = [ "820480.patch" ]; env = [ "CC=cc", @@ -54,5 +53,5 @@ ln -s \ `; }; - inputs = [ bash, go-bootstrap ]; + tools = [ bash, go-bootstrap ]; } diff --git a/internal/rosa/package/go/820480.patch b/internal/rosa/package/go/820480.patch deleted file mode 100644 index 05459a80..00000000 --- a/internal/rosa/package/go/820480.patch +++ /dev/null @@ -1,24 +0,0 @@ -From fa10bbcce5635dd533ba933252e9dabd8a7f5e2a Mon Sep 17 00:00:00 2001 -From: David Chase <drchase@google.com> -Date: Mon, 24 Aug 2026 15:20:40 -0400 -Subject: [PATCH] simd/archsimd: added PMULL gate on carryless multiply test - -Fixes #80991. - -Change-Id: I637de72acbb338c4c3b0a47f4a5520c33fe81fd2 ---- - -diff --git a/src/simd/archsimd/internal/simd_test/simd_arm64_test.go b/src/simd/archsimd/internal/simd_test/simd_arm64_test.go -index f009488..7836fc8 100644 ---- a/src/simd/archsimd/internal/simd_test/simd_arm64_test.go -+++ b/src/simd/archsimd/internal/simd_test/simd_arm64_test.go -@@ -22,6 +22,9 @@ - } - - func TestClMul(t *testing.T) { -+ if !archsimd.ARM64.PMULL() { -+ t.Skip("no carryless multiply") -+ } - var x = archsimd.LoadUint64x2([]uint64{1, 5}) - var y = archsimd.LoadUint64x2([]uint64{3, 9}) - diff --git a/internal/rosa/package/gzip/late-include.patch b/internal/rosa/package/gzip/late-include.patch new file mode 100644 index 00000000..4638d578 --- /dev/null +++ b/internal/rosa/package/gzip/late-include.patch @@ -0,0 +1,21 @@ +diff --git a/gzip.c b/gzip.c +index 220f6fc..ef79a89 100644 +--- a/gzip.c ++++ b/gzip.c +@@ -56,6 +56,8 @@ static char const license_msg[] = + + #include <config.h> + ++#include <signal.h> ++ + #include "tailor.h" + + #include "gzip.h" +@@ -79,7 +81,6 @@ static char const license_msg[] = + #include <inttypes.h> + #include <limits.h> + #include <locale.h> +-#include <signal.h> + #include <stdcountof.h> + #include <stddef.h> + #include <stdlib.h> diff --git a/internal/rosa/package/gzip/package.az b/internal/rosa/package/gzip/package.az new file mode 100644 index 00000000..6c6b3b1f --- /dev/null +++ b/internal/rosa/package/gzip/package.az @@ -0,0 +1,18 @@ +package gzip { + description = "a popular data compression program"; + website = "https://www.gnu.org/software/gzip"; + anitya = 1290; + + version# = "1.15"; + source = remoteTar { + url = "https://mirrors.kernel.org/gnu/gzip/gzip-"+version+".tar.gz"; + checksum = "N8v7cH-Vhmxvk1PYaF-B1vH-r-rMT3Z3VyMdCFXhBd74a8v62wT3dKes-u1JCvUG"; + compress = gzip; + }; + patches = [ "late-include.patch" ]; + + exec = make { + // dependency loop + check = nil; + }; +} diff --git a/internal/rosa/package/hakurei.az b/internal/rosa/package/hakurei.az index 24a2a7c9..f8b03b74 100644 --- a/internal/rosa/package/hakurei.az +++ b/internal/rosa/package/hakurei.az @@ -2,11 +2,11 @@ package hakurei-source { description = "hakurei source tree"; exclude = true; - version# = "0.4.7"; + version# = "0.4.8"; output = remoteGit { - url = "https://git.gensokyo.uk/rosa/hakurei.git"; + url = "https://src.rosa.moe/hakurei"; tag = "v"+version; - checksum = "gQs-fBuPsYjGEBgxe3pSiqa7aMrqUkimi6iyjrhR6YKrOiZLfd4bFwiB9V8k3cKO"; + checksum = "B6aoojyXblZ7eYwdc3Ak3wDbBpu63j8S79Gpiul9Obb8diqzBIJ2qNCgWVH6qLsc"; }; } @@ -61,11 +61,14 @@ mkdir -p /work/system/bin/ `; }; - inputs = [ + tools = [ go, pkgconf, net-tools, + acl, + ]; + inputs = [ libseccomp, acl, fuse, @@ -74,6 +77,8 @@ mkdir -p /work/system/bin/ wayland-protocols, kernel-headers, ]; + + runtime = nil; } package hakurei-dist { @@ -102,11 +107,14 @@ DESTDIR=/work "./${NAME}.sh" `; }; - inputs = [ + tools = [ go, pkgconf, net-tools, + acl, + ]; + inputs = [ libseccomp, acl, fuse, @@ -115,6 +123,8 @@ DESTDIR=/work "./${NAME}.sh" wayland-protocols, kernel-headers, ]; + + runtime = nil; } package earlyinit { @@ -145,5 +155,5 @@ go build -trimpath -v -o /work/ -ldflags="-s -w echo`; }; - inputs = [ go ]; + tools = [ go ]; } diff --git a/internal/rosa/package/harfbuzz.az b/internal/rosa/package/harfbuzz.az index 20b000b7..89de6df7 100644 --- a/internal/rosa/package/harfbuzz.az +++ b/internal/rosa/package/harfbuzz.az @@ -3,11 +3,11 @@ package harfbuzz { website = "https://harfbuzz.github.io"; anitya = 1299; - version# = "14.4.0"; + version# = "14.5.0"; source = remoteGitHub { suffix = "harfbuzz/harfbuzz"; tag = version; - checksum = "Avi4h70Src2_E7l8ag1NP64KH2moVvAWb_cCay4b9FB_5zAJzcTTJLCTf_-nnCHw"; + checksum = "d5NdKP_w04g9mMLc8bW7s7DqC9uKz3vi5TczEHasJ2Wmm6vV9dvj0thuebR-7vz7"; }; exec = meson { @@ -23,9 +23,4 @@ package harfbuzz { zlib, cairo, ]; - - runtime = [ - zlib, - cairo, - ]; } diff --git a/internal/rosa/package/hwdata.az b/internal/rosa/package/hwdata.az index b826953c..57bde7e8 100644 --- a/internal/rosa/package/hwdata.az +++ b/internal/rosa/package/hwdata.az @@ -3,11 +3,11 @@ package hwdata { website = "https://github.com/vcrhonek/hwdata"; anitya = 5387; - version# = "0.410"; + version# = "0.411"; source = remoteGitHub { suffix = "vcrhonek/hwdata"; tag = "v"+version; - checksum = "w1fbTz8NR7C6SYOupqTyNZGJHa_cDRAsGs4guZ3LddqlmawgEP_CWFw9-OYFKJ1q"; + checksum = "5YWsZC8NgZShWWHmC66xqEkTJLAUYJoDt8kmstJyLWfsNqAWAhJZGkVUfgqbTv-6"; }; writable = true; @@ -20,4 +20,6 @@ package hwdata { // lspci: Unknown option 'A' (see "lspci --help") check = nil; }; + + unlikely = true; } diff --git a/internal/rosa/package/images/package.az b/internal/rosa/package/images/package.az index 5c67fbaa..631f5622 100644 --- a/internal/rosa/package/images/package.az +++ b/internal/rosa/package/images/package.az @@ -14,7 +14,7 @@ package initramfs-image { "zstd > /work/initramfs.zst"; }; - inputs = [ + tools = [ zstd, gen_init_cpio, ]; @@ -40,5 +40,5 @@ package system-image { build = "mksquashfs /mnt/system /work/system.img"; }; - inputs = [ squashfs-tools ]; + tools = [ squashfs-tools ]; } diff --git a/internal/rosa/package/ipmitool.az b/internal/rosa/package/ipmitool.az index 4a6dd38b..90e6c3c2 100644 --- a/internal/rosa/package/ipmitool.az +++ b/internal/rosa/package/ipmitool.az @@ -30,11 +30,10 @@ package ipmitool { generate = "./bootstrap"; }; - inputs = [ + tools = [ automake, libtool, - - readline, ]; - runtime = [ readline ]; + + inputs = [ readline ]; } diff --git a/internal/rosa/package/iproute2.az b/internal/rosa/package/iproute2.az index 3a5829df..0730e9b3 100644 --- a/internal/rosa/package/iproute2.az +++ b/internal/rosa/package/iproute2.az @@ -3,11 +3,11 @@ package iproute2 { website = "https://wiki.linuxfoundation.org/networking/iproute2"; anitya = 1392; - version# = "7.1.0"; + version# = "7.2.0"; source = remoteTar { url = "https://git.kernel.org/pub/scm/network/iproute2/iproute2.git/"+ "snapshot/iproute2-"+version+".tar.gz"; - checksum = "yc6NdF6GtFUpKtKg6xwF_yxYGyj0eqFQW8RGsgBzKrOiVlE2bpVQlfeZf_ca5wJL"; + checksum = "VECMJeSm-lHWpyJqnI-1CeHFejAgi_mb8LJjMGIx5rqww6AlZii3g_tUGPLu4npo"; compress = gzip; }; @@ -33,23 +33,18 @@ package iproute2 { check = nil; }; - inputs = [ + tools = [ pkgconf, m4, byacc, flex, - - libbpf, - libmnl, - db, - libcap, - kernel-headers, ]; - runtime = [ + inputs = [ libbpf, libmnl, db, libcap, + kernel-headers, ]; } diff --git a/internal/rosa/package/kernel/amd64.config b/internal/rosa/package/kernel/amd64.config index d560bdb8..81601250 100644 --- a/internal/rosa/package/kernel/amd64.config +++ b/internal/rosa/package/kernel/amd64.config @@ -1,16 +1,16 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/x86 6.18.47 Kernel Configuration +# Linux/x86 6.18.52 Kernel Configuration # -CONFIG_CC_VERSION_TEXT="clang version 23.1.0" +CONFIG_CC_VERSION_TEXT="clang version 23.1.1" CONFIG_GCC_VERSION=0 CONFIG_CC_IS_CLANG=y -CONFIG_CLANG_VERSION=230100 +CONFIG_CLANG_VERSION=230101 CONFIG_AS_IS_LLVM=y -CONFIG_AS_VERSION=230100 +CONFIG_AS_VERSION=230101 CONFIG_LD_VERSION=0 CONFIG_LD_IS_LLD=y -CONFIG_LLD_VERSION=230100 +CONFIG_LLD_VERSION=230101 CONFIG_RUSTC_VERSION=0 CONFIG_RUSTC_LLVM_VERSION=0 CONFIG_CC_HAS_ASM_GOTO_OUTPUT=y @@ -658,7 +658,7 @@ CONFIG_ACPI_IPMI=m CONFIG_ACPI_HOTPLUG_CPU=y CONFIG_ACPI_PROCESSOR_AGGREGATOR=m CONFIG_ACPI_THERMAL=m -CONFIG_ACPI_PLATFORM_PROFILE=m +CONFIG_ACPI_PLATFORM_PROFILE=y CONFIG_ACPI_CUSTOM_DSDT_FILE="" CONFIG_ARCH_HAS_ACPI_TABLE_UPGRADE=y CONFIG_ACPI_TABLE_UPGRADE=y @@ -730,6 +730,7 @@ CONFIG_X86_INTEL_PSTATE=y CONFIG_X86_PCC_CPUFREQ=m CONFIG_X86_AMD_PSTATE=y CONFIG_X86_AMD_PSTATE_DEFAULT_MODE=3 +CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP=y CONFIG_X86_AMD_PSTATE_UT=m CONFIG_X86_ACPI_CPUFREQ=m CONFIG_X86_ACPI_CPUFREQ_CPB=y @@ -1044,6 +1045,7 @@ CONFIG_FUNCTION_ALIGNMENT_16B=y CONFIG_FUNCTION_ALIGNMENT=16 CONFIG_CC_HAS_SANE_FUNCTION_ALIGNMENT=y CONFIG_ARCH_HAS_CPU_ATTACK_VECTORS=y +CONFIG_HAVE_ARCH_GET_SECUREBOOT=y # end of General architecture-dependent options CONFIG_RT_MUTEXES=y diff --git a/internal/rosa/package/kernel/arm64.config b/internal/rosa/package/kernel/arm64.config index c8658775..78fc713b 100644 --- a/internal/rosa/package/kernel/arm64.config +++ b/internal/rosa/package/kernel/arm64.config @@ -1,16 +1,16 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/arm64 6.18.47 Kernel Configuration +# Linux/arm64 6.18.52 Kernel Configuration # -CONFIG_CC_VERSION_TEXT="clang version 23.1.0" +CONFIG_CC_VERSION_TEXT="clang version 23.1.1" CONFIG_GCC_VERSION=0 CONFIG_CC_IS_CLANG=y -CONFIG_CLANG_VERSION=230100 +CONFIG_CLANG_VERSION=230101 CONFIG_AS_IS_LLVM=y -CONFIG_AS_VERSION=230100 +CONFIG_AS_VERSION=230101 CONFIG_LD_VERSION=0 CONFIG_LD_IS_LLD=y -CONFIG_LLD_VERSION=230100 +CONFIG_LLD_VERSION=230101 CONFIG_RUSTC_VERSION=0 CONFIG_RUSTC_LLVM_VERSION=0 CONFIG_CC_HAS_ASM_GOTO_OUTPUT=y @@ -575,7 +575,6 @@ CONFIG_ARM64_TLB_RANGE=y # CONFIG_AS_HAS_ARMV8_5=y CONFIG_ARM64_BTI=y -CONFIG_ARM64_BTI_KERNEL=y CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI=y CONFIG_ARM64_E0PD=y CONFIG_ARM64_AS_HAS_MTE=y @@ -975,6 +974,7 @@ CONFIG_HAVE_GCC_PLUGINS=y CONFIG_FUNCTION_ALIGNMENT_4B=y CONFIG_FUNCTION_ALIGNMENT=4 CONFIG_CC_HAS_SANE_FUNCTION_ALIGNMENT=y +CONFIG_HAVE_ARCH_GET_SECUREBOOT=y # end of General architecture-dependent options CONFIG_RT_MUTEXES=y @@ -12217,9 +12217,10 @@ CONFIG_PXA1908_PM_DOMAINS=m # CONFIG_MTK_SCPSYS=y CONFIG_MTK_SCPSYS_PM_DOMAINS=y -CONFIG_AIROHA_CPU_PM_DOMAIN=m # end of MediaTek PM Domains +CONFIG_AIROHA_CPU_PM_DOMAIN=m + # # Qualcomm PM Domains # @@ -14220,11 +14221,9 @@ CONFIG_CRYPTO_DEV_SUN4I_SS=m CONFIG_CRYPTO_DEV_SUN8I_CE=m # CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG is not set # CONFIG_CRYPTO_DEV_SUN8I_CE_HASH is not set -# CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG is not set # CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG is not set CONFIG_CRYPTO_DEV_SUN8I_SS=m # CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG is not set -# CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG is not set # CONFIG_CRYPTO_DEV_SUN8I_SS_HASH is not set CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON=m CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC=m diff --git a/internal/rosa/package/kernel/package.az b/internal/rosa/package/kernel/package.az index e2f6339e..5a0ffae6 100644 --- a/internal/rosa/package/kernel/package.az +++ b/internal/rosa/package/kernel/package.az @@ -3,11 +3,11 @@ package kernel-source { website = "https://kernel.org"; exclude = true; - version# = "6.18.48"; + version# = "6.18.52"; output = remoteTar { url = "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/"+ "snapshot/linux-"+version+".tar.gz"; - checksum = "XzcqKk0ODA4tTjl5tVQdckuLHVnITGalDH_EIYUC8WHimc7O6bbews-vFRHUK01m"; + checksum = "3OysuyxqFKX0FvKqkwH-Y9t0gB0PpE-ErMb2lITWAJtN0OSDqlHTbeifhgn6VgkV"; compress = gzip; }; } @@ -46,11 +46,13 @@ cat \ checksum = arch { default = ""; - amd64 = "uwTZX3iLp6iPsR0-F0Jz2aUviLMMqv4xGQHoKeemo2HrNLD7fDlIDqqYT--i_5HJ"; - arm64 = "A9EWzTGEChVLJVm5BP1GzpibdUddXUN2hIjAGkD-KQr5sc6RuEeIQle8VAiAYUqp"; - riscv64 = "IB1oG3tEAN15wjYl8S5ekYXVFYN7-rb3m2f1Kdi58dGqnrIBTG2r1qb1utb5D1lb"; + amd64 = "yLzBQuYIpzbjEglADpiAjDsI5OhsOZejLUG-3w1DlVfmU4LaH44W5KercoxrEuGQ"; + arm64 = "aah3h8pHdYNrevRBBIkt8DFi6wzRWZe9uxWcrTwEf1l3X-UWxRfnGxCwJdn0VySM"; + riscv64 = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; }; - inputs = [ rsync ]; + + tools = [ rsync ]; + unlikely = true; } package kernel { @@ -127,7 +129,7 @@ rm -v "/work/system/lib/modules/$(make -f /usr/src/kernel/Makefile kernelversion `; }; - inputs = [ + tools = [ flex, bison, m4, @@ -141,13 +143,19 @@ rm -v "/work/system/lib/modules/$(make -f /usr/src/kernel/Makefile kernelversion bash, python, - xz, - gzip, kmod, elfutils, util-linux, + ]; + + inputs = [ + xz, + gzip, kernel-headers, ]; + + runtime = nil; + unlikely = true; } package gen_init_cpio { diff --git a/internal/rosa/package/kernel/version.h b/internal/rosa/package/kernel/version.h index 4fc45286..e8d4316f 100644 --- a/internal/rosa/package/kernel/version.h +++ b/internal/rosa/package/kernel/version.h @@ -1,5 +1,5 @@ -#define LINUX_VERSION_CODE 397870 +#define LINUX_VERSION_CODE 397876 #define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c))) #define LINUX_VERSION_MAJOR 6 #define LINUX_VERSION_PATCHLEVEL 18 -#define LINUX_VERSION_SUBLEVEL 46 +#define LINUX_VERSION_SUBLEVEL 52 diff --git a/internal/rosa/package/kmod.az b/internal/rosa/package/kmod.az index c56ba846..732d3340 100644 --- a/internal/rosa/package/kmod.az +++ b/internal/rosa/package/kmod.az @@ -31,10 +31,4 @@ package kmod { openssl, kernel-headers, ]; - - runtime = [ - zlib, - zstd, - openssl, - ]; } diff --git a/internal/rosa/package/libaio.az b/internal/rosa/package/libaio.az index 5b7072eb..ee408872 100644 --- a/internal/rosa/package/libaio.az +++ b/internal/rosa/package/libaio.az @@ -29,9 +29,10 @@ package libaio { install = "make prefix=/work/system install"; }; - inputs = [ + tools = [ bash, e2fsprogs, - kernel-headers, ]; + + inputs = [ kernel-headers ]; } diff --git a/internal/rosa/package/libarchive/package.az b/internal/rosa/package/libarchive/package.az index cb6c080f..b06daaf3 100644 --- a/internal/rosa/package/libarchive/package.az +++ b/internal/rosa/package/libarchive/package.az @@ -17,7 +17,7 @@ package libarchive { exec = cmake {}; - runtime = [ + inputs = [ zlib, zstd, ]; diff --git a/internal/rosa/package/libbpf.az b/internal/rosa/package/libbpf.az index 9a10bc45..33df95e4 100644 --- a/internal/rosa/package/libbpf.az +++ b/internal/rosa/package/libbpf.az @@ -26,6 +26,4 @@ package libbpf { elfutils, kernel-headers, ]; - - runtime = [ elfutils ]; } diff --git a/internal/rosa/package/libbsd.az b/internal/rosa/package/libbsd.az index 4acc520b..27d2fc77 100644 --- a/internal/rosa/package/libbsd.az +++ b/internal/rosa/package/libbsd.az @@ -14,10 +14,11 @@ package libbsd { generate = "echo '" + version + "' > .dist-version && ./autogen"; }; - inputs = [ + tools = [ automake, libtool, - - libmd, ]; + + inputs = [ libmd ]; + runtime = nil; } diff --git a/internal/rosa/package/libcap.az b/internal/rosa/package/libcap.az index be48ed01..86bdc63a 100644 --- a/internal/rosa/package/libcap.az +++ b/internal/rosa/package/libcap.az @@ -29,7 +29,7 @@ package libcap { check = [ "CC=cc", "test" ]; }; - inputs = [ + tools = [ bash, diffutils, ]; diff --git a/internal/rosa/package/libdisplay-info.az b/internal/rosa/package/libdisplay-info.az index da563d01..8780bc10 100644 --- a/internal/rosa/package/libdisplay-info.az +++ b/internal/rosa/package/libdisplay-info.az @@ -12,10 +12,6 @@ package libdisplay-info { }; exec = meson {}; - - inputs = [ - diffutils, - - hwdata, - ]; + tools = [ diffutils ]; + inputs = [ hwdata ]; } diff --git a/internal/rosa/package/libdrm.az b/internal/rosa/package/libdrm.az index efa6f166..6491da28 100644 --- a/internal/rosa/package/libdrm.az +++ b/internal/rosa/package/libdrm.az @@ -17,13 +17,13 @@ package libdrm { }; }; - inputs = [ + tools = [ // symbols check fail with llvm nm binutils, + ]; + inputs = [ libpciaccess, kernel-headers, ]; - - runtime = [ libpciaccess ]; } diff --git a/internal/rosa/package/libepoxy.az b/internal/rosa/package/libepoxy.az index f8762a00..4c0efab0 100644 --- a/internal/rosa/package/libepoxy.az +++ b/internal/rosa/package/libepoxy.az @@ -21,4 +21,6 @@ package libepoxy { libglvnd, libX11, ]; + + runtime = nil; } diff --git a/internal/rosa/package/libevdev.az b/internal/rosa/package/libevdev.az index bb6ffc16..85c9b674 100644 --- a/internal/rosa/package/libevdev.az +++ b/internal/rosa/package/libevdev.az @@ -17,10 +17,10 @@ package libevdev { }; }; - inputs = [ + tools = [ check, bash, - - kernel-headers, ]; + + inputs = [ kernel-headers ]; } diff --git a/internal/rosa/package/libexpat.az b/internal/rosa/package/libexpat.az index 20e4f510..acb8445b 100644 --- a/internal/rosa/package/libexpat.az +++ b/internal/rosa/package/libexpat.az @@ -3,7 +3,7 @@ package libexpat { website = "https://libexpat.github.io"; anitya = 770; - version# = "2.8.3"; + version# = "2.8.4"; source = remoteGitHubRelease { suffix = "libexpat/libexpat"; tag = "R_"+replace { @@ -12,11 +12,11 @@ package libexpat { new = "_"; }; name = "expat-"+version+".tar.bz2"; - checksum = "prHQc14YY3Gp6BfflneNcVyRXM-AxFCqiigxrGL9TT1sGhVODvI_9JNR2xnZlZSj"; + checksum = "Nbi2wl8U9KHf3LVLczfAb5Kqr7HFv_myReVjxKXUkfhToWfL-pP2o44fDpWHJ4ig"; compress = bzip2; }; exec = make {}; - inputs = [ bash ]; + tools = [ bash ]; } diff --git a/internal/rosa/package/libgd.az b/internal/rosa/package/libgd.az index 8ce41894..b1b7b327 100644 --- a/internal/rosa/package/libgd.az +++ b/internal/rosa/package/libgd.az @@ -21,5 +21,4 @@ package libgd { exec = make {}; inputs = [ zlib ]; - runtime = [ zlib ]; } diff --git a/internal/rosa/package/libglvnd.az b/internal/rosa/package/libglvnd.az index 02ce0bd7..9be9e2a9 100644 --- a/internal/rosa/package/libglvnd.az +++ b/internal/rosa/package/libglvnd.az @@ -22,17 +22,16 @@ export DISPLAY=':0' Xvfb & XVFB_PID="$!" trap 'kill $XVFB_PID && wait $XVFB_PID' EXIT +while [ ! -e /tmp/.X11-unix/X0 ]; do sleep 0.1; done `; }; - inputs = [ + tools = [ // symbols check fail with llvm nm binutils, // test suite wants X server xserver, - - libXext, ]; - runtime = [ libXext ]; + inputs = [ libXext ]; } diff --git a/internal/rosa/package/libinput.az b/internal/rosa/package/libinput.az index 4cb1b764..3769f680 100644 --- a/internal/rosa/package/libinput.az +++ b/internal/rosa/package/libinput.az @@ -3,12 +3,12 @@ package libinput { website = "https://www.freedesktop.org/wiki/Software/libinput"; anitya = 5781; - version# = "1.31.3"; + version# = "1.32.0"; source = remoteGitLab { domain = "gitlab.freedesktop.org"; suffix = "libinput/libinput"; ref = version; - checksum = "GxBGPN6YybQxrD2MDsIL8gdDYImXn4NAJi6EvTx_Hb_1jcbjwCrjeyjY2upUyTMi"; + checksum = "hjuI9afDPrCORLCCM5koBjyRNmTr_YCUmmF4Tl0bqK_7fz0beC8125wt3lmimtgL"; }; bin = [ "bash" ]; @@ -20,18 +20,15 @@ package libinput { }; }; - inputs = [ + tools = [ bash, diffutils, python-pytest, - - libudev-zero, - libevdev, - kernel-headers, ]; - runtime = [ + inputs = [ libudev-zero, libevdev, + kernel-headers, ]; } diff --git a/internal/rosa/package/libliftoff/0001-test-Disable-alloc-deadline-on-tests.patch b/internal/rosa/package/libliftoff/0001-test-Disable-alloc-deadline-on-tests.patch new file mode 100644 index 00000000..6747ac01 --- /dev/null +++ b/internal/rosa/package/libliftoff/0001-test-Disable-alloc-deadline-on-tests.patch @@ -0,0 +1,139 @@ +From 343a7d02cf5417e0732d8ba73089f86dd6173253 Mon Sep 17 00:00:00 2001 +From: David Turner <david.turner@raspberrypi.com> +Date: Wed, 5 Mar 2025 16:02:25 +0000 +Subject: [PATCH] test: Disable alloc deadline on tests + +I've been seeing occasional spurious CI failures due to +liftoff_output_apply() deadline timeouts, presumably when the runner is +busy. Disable the allocation deadline in all tests to stop this +happening. +--- + test/test_alloc.c | 16 ++++++++++++---- + test/test_candidate.c | 4 +++- + test/test_dynamic.c | 8 ++++++-- + test/test_priority.c | 4 +++- + test/test_prop.c | 4 +++- + 5 files changed, 27 insertions(+), 9 deletions(-) + +diff --git a/test/test_alloc.c b/test/test_alloc.c +index 8d0caf7..a002c46 100644 +--- a/test/test_alloc.c ++++ b/test/test_alloc.c +@@ -803,7 +803,9 @@ run_test(const struct test_case *test) + } + + req = drmModeAtomicAlloc(); +- ret = liftoff_output_apply(output, req, 0, NULL); ++ ret = liftoff_output_apply(output, req, 0, &(struct liftoff_output_apply_options){ ++ .timeout_ns = INT64_MAX, ++ }); + assert(ret == 0); + ret = drmModeAtomicCommit(drm_fd, req, 0, NULL); + assert(ret == 0); +@@ -875,7 +877,9 @@ test_basic(void) + liftoff_mock_plane_add_compatible_layer(mock_plane, layer); + + req = drmModeAtomicAlloc(); +- ret = liftoff_output_apply(output, req, 0, NULL); ++ ret = liftoff_output_apply(output, req, 0, &(struct liftoff_output_apply_options){ ++ .timeout_ns = INT64_MAX, ++ }); + assert(ret == 0); + ret = drmModeAtomicCommit(drm_fd, req, 0, NULL); + assert(ret == 0); +@@ -913,7 +917,9 @@ test_no_props_fail(void) + liftoff_mock_plane_add_compatible_layer(mock_plane, layer); + + req = drmModeAtomicAlloc(); +- ret = liftoff_output_apply(output, req, 0, NULL); ++ ret = liftoff_output_apply(output, req, 0, &(struct liftoff_output_apply_options){ ++ .timeout_ns = INT64_MAX, ++ }); + assert(ret == 0); + ret = drmModeAtomicCommit(drm_fd, req, 0, NULL); + assert(ret == 0); +@@ -959,7 +965,9 @@ test_composition_no_props(void) + liftoff_mock_plane_add_compatible_layer(mock_plane, layer_with_fb); + + req = drmModeAtomicAlloc(); +- ret = liftoff_output_apply(output, req, 0, NULL); ++ ret = liftoff_output_apply(output, req, 0, &(struct liftoff_output_apply_options){ ++ .timeout_ns = INT64_MAX, ++ }); + assert(ret == 0); + ret = drmModeAtomicCommit(drm_fd, req, 0, NULL); + assert(ret == 0); +diff --git a/test/test_candidate.c b/test/test_candidate.c +index 5e7f698..74a0bcb 100644 +--- a/test/test_candidate.c ++++ b/test/test_candidate.c +@@ -60,7 +60,9 @@ test_basic(void) + liftoff_layer_set_property(layer, "COLOR_RANGE", 0); + + req = drmModeAtomicAlloc(); +- ret = liftoff_output_apply(output, req, 0, NULL); ++ ret = liftoff_output_apply(output, req, 0, &(struct liftoff_output_apply_options){ ++ .timeout_ns = INT64_MAX, ++ }); + assert(ret == 0); + ret = drmModeAtomicCommit(drm_fd, req, 0, NULL); + assert(ret == 0); +diff --git a/test/test_dynamic.c b/test/test_dynamic.c +index 3a95a7f..8c1049d 100644 +--- a/test/test_dynamic.c ++++ b/test/test_dynamic.c +@@ -50,7 +50,9 @@ first_commit(struct context *ctx) + assert(ctx->commit_count == 0); + + req = drmModeAtomicAlloc(); +- ret = liftoff_output_apply(ctx->output, req, 0, NULL); ++ ret = liftoff_output_apply(ctx->output, req, 0, &(struct liftoff_output_apply_options){ ++ .timeout_ns = INT64_MAX, ++ }); + assert(ret == 0); + ret = drmModeAtomicCommit(ctx->drm_fd, req, 0, NULL); + assert(ret == 0); +@@ -72,7 +74,9 @@ second_commit(struct context *ctx, bool want_reuse_prev_alloc) + int ret; + + req = drmModeAtomicAlloc(); +- ret = liftoff_output_apply(ctx->output, req, 0, NULL); ++ ret = liftoff_output_apply(ctx->output, req, 0, &(struct liftoff_output_apply_options){ ++ .timeout_ns = INT64_MAX, ++ }); + assert(ret == 0); + if (want_reuse_prev_alloc) { + /* The library should perform only one TEST_ONLY commit with the +diff --git a/test/test_priority.c b/test/test_priority.c +index cf2e4c5..41ede18 100644 +--- a/test/test_priority.c ++++ b/test/test_priority.c +@@ -76,7 +76,9 @@ main(int argc, char *argv[]) + + liftoff_layer_set_property(layer, "FB_ID", fbs[j % 2]); + +- ret = liftoff_output_apply(output, req, 0, NULL); ++ ret = liftoff_output_apply(output, req, 0, &(struct liftoff_output_apply_options){ ++ .timeout_ns = INT64_MAX, ++ }); + assert(ret == 0); + ret = drmModeAtomicCommit(drm_fd, req, 0, NULL); + assert(ret == 0); +diff --git a/test/test_prop.c b/test/test_prop.c +index 05193c7..093b581 100644 +--- a/test/test_prop.c ++++ b/test/test_prop.c +@@ -34,7 +34,9 @@ commit(int drm_fd, struct liftoff_output *output) + int ret; + + req = drmModeAtomicAlloc(); +- ret = liftoff_output_apply(output, req, 0, NULL); ++ ret = liftoff_output_apply(output, req, 0, &(struct liftoff_output_apply_options){ ++ .timeout_ns = INT64_MAX, ++ }); + assert(ret == 0); + ret = drmModeAtomicCommit(drm_fd, req, 0, NULL); + assert(ret == 0); +-- +2.55.0 + diff --git a/internal/rosa/package/libliftoff.az b/internal/rosa/package/libliftoff/package.az index 16e34aed..7d8900bc 100644 --- a/internal/rosa/package/libliftoff.az +++ b/internal/rosa/package/libliftoff/package.az @@ -10,6 +10,7 @@ package libliftoff { ref = "v"+version; checksum = "anxKfrW56iK7l6-bm6RGcaC_uk-Kc8x3_VqKaywmSjv5HLCs6A2sHyjGUbOC9xZv"; }; + patches = [ "0001-test-Disable-alloc-deadline-on-tests.patch" ]; exec = meson {}; @@ -17,4 +18,6 @@ package libliftoff { libdrm, kernel-headers, ]; + + runtime = nil; } diff --git a/internal/rosa/package/libmd.az b/internal/rosa/package/libmd.az index 1cdc4b65..e618d11e 100644 --- a/internal/rosa/package/libmd.az +++ b/internal/rosa/package/libmd.az @@ -15,7 +15,7 @@ package libmd { preMake = "\ninstall -D /usr/src/libmd/src/helper.c src/helper.c\n"; }; - inputs = [ + tools = [ automake, libtool, ]; diff --git a/internal/rosa/package/libpng.az b/internal/rosa/package/libpng.az index 9fdea1d6..4711cbce 100644 --- a/internal/rosa/package/libpng.az +++ b/internal/rosa/package/libpng.az @@ -23,5 +23,4 @@ package libpng { exec = make {}; inputs = [ zlib ]; - runtime = [ zlib ]; } diff --git a/internal/rosa/package/libpsl.az b/internal/rosa/package/libpsl.az index 2a45cf20..343a2bf6 100644 --- a/internal/rosa/package/libpsl.az +++ b/internal/rosa/package/libpsl.az @@ -21,7 +21,7 @@ test_disable 'int main(){return 0;}' tests/test-is-public-builtin.c exec = make {}; - inputs = [ + tools = [ pkgconf, python, ]; diff --git a/internal/rosa/package/libseccomp.az b/internal/rosa/package/libseccomp.az index 37231185..9089db83 100644 --- a/internal/rosa/package/libseccomp.az +++ b/internal/rosa/package/libseccomp.az @@ -15,11 +15,11 @@ package libseccomp { bin = [ "bash" ]; exec = make {}; - inputs = [ + tools = [ bash, diffutils, gperf, - - kernel-headers, ]; + + inputs = [ kernel-headers ]; } diff --git a/internal/rosa/package/libtirpc.az b/internal/rosa/package/libtirpc.az index a05b75df..b1a49c79 100644 --- a/internal/rosa/package/libtirpc.az +++ b/internal/rosa/package/libtirpc.az @@ -3,7 +3,7 @@ package libtirpc { website = "https://sourceforge.net/projects/libtirpc"; anitya = 1740; - version# = "1.3.7"; + version# = "1.3.8"; source = remoteGit { url = "git://linux-nfs.org/~steved/libtirpc"; tag = "libtirpc-"+join { @@ -14,7 +14,7 @@ package libtirpc { }; sep = "-"; }; - checksum = "nzFfu7LNvnSNiNAryD1vtnNWnU-Xqee8KqfXUKoBf5yjb5-dkeRkYuRijdCoYLof"; + checksum = "OYM7CljjvlHv3Xzc9bXC2WRfBQY0QcBEZ9ubE8qVDweq9ZvQPrAWxSOFVhFBoToz"; }; exec = make { @@ -25,12 +25,20 @@ package libtirpc { }; }; - inputs = [ + tools = [ automake, libtool, pkgconf, + // version script assignment of 'TIRPC_1.3.7' to symbol 'authdes_getucred' failed: symbol not defined + // version script assignment of 'TIRPC_1.3.7' to symbol 'rpc_gss_getcred' failed: symbol not defined + binutils, + ]; + + inputs = [ libbsd, kernel-headers, ]; + + runtime = nil; } diff --git a/internal/rosa/package/libva.az b/internal/rosa/package/libva.az index 5cb3a9b4..463220d6 100644 --- a/internal/rosa/package/libva.az +++ b/internal/rosa/package/libva.az @@ -25,11 +25,4 @@ package libva { wayland, kernel-headers, ]; - - runtime = [ - libdrm, - libXfixes, - libglvnd, - wayland, - ]; } diff --git a/internal/rosa/package/libxml2.az b/internal/rosa/package/libxml2.az index 39146e8b..a169c0e4 100644 --- a/internal/rosa/package/libxml2.az +++ b/internal/rosa/package/libxml2.az @@ -3,12 +3,12 @@ package libxml2 { website = "https://gitlab.gnome.org/GNOME/libxml2"; anitya = 1783; - version# = "2.15.3"; + version# = "2.15.4"; source = remoteGitLab { domain = "gitlab.gnome.org"; suffix = "GNOME/libxml2"; ref = "v"+version; - checksum = "oJy74htGlEpf70KPvpW18fYJo0RQQkCXZRwqUz6NoXborS3HCq3Nm4gsyaSeNmUH"; + checksum = "zDpmFcO_u6FU91sjvOiySGIx6tFVomhxC3GXfBYR_sDXBCs4WFIEY_9NyQ24-zfO"; }; // can't create shell.out: Read-only file system @@ -16,7 +16,7 @@ package libxml2 { exec = meson {}; - inputs = [ + tools = [ git, diffutils, ]; diff --git a/internal/rosa/package/libxslt.az b/internal/rosa/package/libxslt.az index 34ba7a71..8518c325 100644 --- a/internal/rosa/package/libxslt.az +++ b/internal/rosa/package/libxslt.az @@ -18,14 +18,12 @@ package libxslt { check = nil; }; - inputs = [ + tools = [ automake, libtool, python, pkgconf, - - libxml2, ]; - runtime = [ libxml2 ]; + inputs = [ libxml2 ]; } diff --git a/internal/rosa/package/lit.az b/internal/rosa/package/lit.az index 846b5fe6..dc02bdbb 100644 --- a/internal/rosa/package/lit.az +++ b/internal/rosa/package/lit.az @@ -11,5 +11,4 @@ package lit { }; inputs = [ python-setuptools ]; - runtime = [ python ]; } diff --git a/internal/rosa/package/llvm/package.az b/internal/rosa/package/llvm/package.az index 11210563..73b0ed8e 100644 --- a/internal/rosa/package/llvm/package.az +++ b/internal/rosa/package/llvm/package.az @@ -2,11 +2,11 @@ package llvm-project { description = "LLVM monorepo with Rosa OS patches"; anitya = 1830; - version# = "23.1.0"; + version# = "23.1.1"; output = remoteGitHub { suffix = "llvm/llvm-project"; tag = "llvmorg-"+version; - checksum = "wgirRNAhU9451qsZn4FXcaIButgRNZWb4kb8pQwERyF1PGc-iJQVewr6xGk7ctNj"; + checksum = "3PRmlMQcmYffm_uj3DxEeDaukPHBS5fpCsWYLHX-9HZXwlgZQG43li2lJLJ33_SN"; }; patches = [ "increase-stack-size-unconditional.patch", diff --git a/internal/rosa/package/lm_sensors.az b/internal/rosa/package/lm_sensors.az index 8edc2447..2b6b58dc 100644 --- a/internal/rosa/package/lm_sensors.az +++ b/internal/rosa/package/lm_sensors.az @@ -24,8 +24,7 @@ package lm_sensors { install = "make DESTDIR=/work PREFIX=/system install"; }; - inputs = [ - perl, + tools = [ perl-Test-Cmd, m4, diff --git a/internal/rosa/package/lua/package.az b/internal/rosa/package/lua/package.az index ab74dfdd..54d5abe6 100644 --- a/internal/rosa/package/lua/package.az +++ b/internal/rosa/package/lua/package.az @@ -33,4 +33,6 @@ sed \ ../lua.pc > /work/system/lib/pkgconfig/lua.pc `; }; + + unlikely = true; } diff --git a/internal/rosa/package/lvm2/package.az b/internal/rosa/package/lvm2/package.az index bb683509..3a363b54 100644 --- a/internal/rosa/package/lvm2/package.az +++ b/internal/rosa/package/lvm2/package.az @@ -19,10 +19,12 @@ package lvm2 { check = nil; }; - inputs = [ - pkgconf, + tools = [ pkgconf ]; + inputs = [ libaio, kernel-headers, ]; + + runtime = nil; } diff --git a/internal/rosa/package/mesa/include.patch b/internal/rosa/package/mesa/include.patch index b1942625..4a8a7fe9 100644 --- a/internal/rosa/package/mesa/include.patch +++ b/internal/rosa/package/mesa/include.patch @@ -1,15 +1,3 @@ -diff --git a/src/amd/compiler/aco_util.h b/src/amd/compiler/aco_util.h -index 82d754f155a..9f20e80a554 100644 ---- a/src/amd/compiler/aco_util.h -+++ b/src/amd/compiler/aco_util.h -@@ -12,6 +12,7 @@ - #include "util/macros.h" - #include "util/u_math.h" - -+#include <algorithm> - #include <array> - #include <cassert> - #include <cstddef> diff --git a/src/freedreno/common/freedreno_common.h b/src/freedreno/common/freedreno_common.h index 36dea4e12b5..cfa243c270f 100644 --- a/src/freedreno/common/freedreno_common.h diff --git a/internal/rosa/package/mesa/package.az b/internal/rosa/package/mesa/package.az index 7967e3d7..ec7956fb 100644 --- a/internal/rosa/package/mesa/package.az +++ b/internal/rosa/package/mesa/package.az @@ -4,12 +4,12 @@ package mesa { anitya = 1970; latest = anityaFallback; - version# = "26.2.1"; + version# = "26.2.3"; source = remoteGitLab { domain = "gitlab.freedesktop.org"; suffix = "mesa/mesa"; ref = "mesa-"+version; - checksum = "WZYAysQFZt8KMFAxQheKFZHarR1isgOeSw4LyfqAcmaHwXMCLiVao6eGuEuhJTJv"; + checksum = "VAGZWGIjAnBqWi7N_Tye3ihtoc66Gu8AlKGtpmQOfwcwQqhZ6Sxk2Y_IH49jWOrJ"; }; patches = [ "include.patch" ]; diff --git a/internal/rosa/package/meson/package.az b/internal/rosa/package/meson/package.az index 6fd06923..78fa2035 100644 --- a/internal/rosa/package/meson/package.az +++ b/internal/rosa/package/meson/package.az @@ -43,7 +43,7 @@ python3 ./run_project_tests.py \ `; }; - inputs = [ + tools = [ python-setuptools, pkg-config, cmake, diff --git a/internal/rosa/package/mksh.az b/internal/rosa/package/mksh.az index 6e728285..878bba83 100644 --- a/internal/rosa/package/mksh.az +++ b/internal/rosa/package/mksh.az @@ -31,7 +31,7 @@ ln -vs ../system/bin/sh /work/bin/ `; }; - inputs = [ + tools = [ perl, coreutils, ]; diff --git a/internal/rosa/package/muon.az b/internal/rosa/package/muon.az index 512844a9..dc183591 100644 --- a/internal/rosa/package/muon.az +++ b/internal/rosa/package/muon.az @@ -53,10 +53,7 @@ sed -i '/common.220 fs module/d' subprojects/meson-tests/meson.build install = "./muon install -d /work\n"; }; - inputs = [ - python-early, - kernel-headers, - ]; - + tools = [ python-early ]; + inputs = [ kernel-headers ]; runtime = [ pkg-config ]; } diff --git a/internal/rosa/package/musl-fts.az b/internal/rosa/package/musl-fts.az index cce1ea1f..195928f5 100644 --- a/internal/rosa/package/musl-fts.az +++ b/internal/rosa/package/musl-fts.az @@ -18,7 +18,7 @@ package musl-fts { generate = "./bootstrap.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, diff --git a/internal/rosa/package/musl-obstack.az b/internal/rosa/package/musl-obstack.az index 218a718d..95388e33 100644 --- a/internal/rosa/package/musl-obstack.az +++ b/internal/rosa/package/musl-obstack.az @@ -18,7 +18,7 @@ package musl-obstack { generate = "./bootstrap.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, diff --git a/internal/rosa/package/musl/package.az b/internal/rosa/package/musl/package.az index 3207ecde..c264c782 100644 --- a/internal/rosa/package/musl/package.az +++ b/internal/rosa/package/musl/package.az @@ -29,7 +29,7 @@ package musl-headers { install = "# headers installed during make"; }; - inputs = [ coreutils ]; + tools = [ coreutils ]; } package musl { @@ -61,5 +61,5 @@ rmdir -v /work/lib `; }; - inputs = [ coreutils ]; + tools = [ coreutils ]; } diff --git a/internal/rosa/package/ncurses.az b/internal/rosa/package/ncurses.az index 189d8fb5..b4d8e26e 100644 --- a/internal/rosa/package/ncurses.az +++ b/internal/rosa/package/ncurses.az @@ -29,5 +29,5 @@ find /work/system/lib \ `; }; - inputs = [ pkg-config ]; + tools = [ pkg-config ]; } diff --git a/internal/rosa/package/net-tools/package.az b/internal/rosa/package/net-tools/package.az index 62a50289..0f2ac828 100644 --- a/internal/rosa/package/net-tools/package.az +++ b/internal/rosa/package/net-tools/package.az @@ -25,9 +25,6 @@ package net-tools { check = nil; }; - inputs = [ - bash, - - kernel-headers, - ]; + tools = [ bash ]; + inputs = [ kernel-headers ]; } diff --git a/internal/rosa/package/netfilter/package.az b/internal/rosa/package/netfilter/package.az index 3dea2b87..5e5e1624 100644 --- a/internal/rosa/package/netfilter/package.az +++ b/internal/rosa/package/netfilter/package.az @@ -22,6 +22,8 @@ package libmnl { libbsd, kernel-headers, ]; + + runtime = nil; } package libnftnl { @@ -29,11 +31,11 @@ package libnftnl { website = "https://www.netfilter.org/projects/libnftnl"; anitya = 1681; - version# = "1.3.1"; + version# = "1.3.2"; source = remoteGit { url = "https://git.netfilter.org/libnftnl.git"; tag = "libnftnl-"+version; - checksum = "91ou66K-I17iX6DB6hiQkhhC_v4DFW5iDGzwjVRNbJNEmKqowLZBlh3FY-ZDO0r9"; + checksum = "bW_GokelPejp2SKOo_OgFFYjf-7rn3GL6x1Z4kZb9NkfZe20WrRLNjQWVxxS9oII"; }; env = [ @@ -47,16 +49,16 @@ package libnftnl { }; }; - inputs = [ + tools = [ automake, libtool, pkgconf, + ]; + inputs = [ libmnl, kernel-headers, ]; - - runtime = [ libmnl ]; } package iptables { @@ -92,16 +94,16 @@ chmod +w /etc/ && ln -s ../usr/src/iptables/etc/ethertypes /etc/ `; }; - inputs = [ + tools = [ automake, libtool, pkgconf, bash, python, + ]; + inputs = [ libnftnl, kernel-headers, ]; - - runtime = [ libnftnl ]; } diff --git a/internal/rosa/package/nettle.az b/internal/rosa/package/nettle.az index 1ce1f9ec..e30696a2 100644 --- a/internal/rosa/package/nettle.az +++ b/internal/rosa/package/nettle.az @@ -12,12 +12,10 @@ package nettle { exec = make {}; - inputs = [ + tools = [ m4, diffutils, - - gmp, ]; - runtime = [ gmp ]; + inputs = [ gmp ]; } diff --git a/internal/rosa/package/ninja.az b/internal/rosa/package/ninja.az index 2c1bd8b3..da5fa715 100644 --- a/internal/rosa/package/ninja.az +++ b/internal/rosa/package/ninja.az @@ -33,7 +33,7 @@ mkdir -p /work/system/bin/ cp ninja /work/system/bin/`; }; - inputs = [ + tools = [ python-early, bash, ]; diff --git a/internal/rosa/package/noto.az b/internal/rosa/package/noto.az index 64d27a58..7481b5e7 100644 --- a/internal/rosa/package/noto.az +++ b/internal/rosa/package/noto.az @@ -3,11 +3,11 @@ package noto { website = "https://fonts.google.com/noto"; anitya = 10671; - version# = "2026.08.01"; + version# = "2026.09.01"; source = remoteGitHub { suffix = "notofonts/notofonts.github.io"; tag = "noto-monthly-release-"+version; - checksum = "xTuvGQBtw9FWrbMQ0OHC-EVpUWYwLxNHPtaind_t6nP37En0r8orIY9N0X_YrPH7"; + checksum = "ZITq5IyZNZovca3t7snPndDgjdzbCeBIoGYpY7bbnR-w9oAiz1vkN9zwRWz5JJw5"; }; enterSource = true; @@ -28,5 +28,5 @@ rename -v 's/\[.*\]//' $DEST/* `; }; - inputs = [ rename ]; + tools = [ rename ]; } diff --git a/internal/rosa/package/nss.az b/internal/rosa/package/nss.az index 32110d04..e76be8f2 100644 --- a/internal/rosa/package/nss.az +++ b/internal/rosa/package/nss.az @@ -3,7 +3,7 @@ package nss-cacert { website = "https://firefox-source-docs.mozilla.org/security/nss/index.html"; anitya = 2503; - version# = "3.128"; + version# = "3.129"; source = remoteGitHub { suffix = "nss-dev/nss"; tag = "NSS_"+join { @@ -14,7 +14,7 @@ package nss-cacert { }; sep = "_"; }+"_RTM"; - checksum = "uVP9Tbv17Vo7tUAm1LK16vu-9PA4Mecjc1Kr6y4ptXby9e0zT7I15rzsD2fMApqA"; + checksum = "nrmKcrMZ_Bq5FiWBmJWOPrJvP0IB9jfj4zEm1gzufN4hiuHFlNFjJExBpQVk_l8i"; }; enterSource = true; @@ -32,7 +32,7 @@ buildcatrust \ `; }; - inputs = [ + tools = [ bash, buildcatrust, diff --git a/internal/rosa/package/openssl.az b/internal/rosa/package/openssl.az index 907d394a..cb76d0fa 100644 --- a/internal/rosa/package/openssl.az +++ b/internal/rosa/package/openssl.az @@ -38,11 +38,15 @@ package openssl { ]; }; - inputs = [ + tools = [ perl, coreutils, + ]; + inputs = [ zlib, kernel-headers, ]; + + runtime = nil; } diff --git a/internal/rosa/package/p11-kit.az b/internal/rosa/package/p11-kit.az index bb96c4dd..3fa9ceb7 100644 --- a/internal/rosa/package/p11-kit.az +++ b/internal/rosa/package/p11-kit.az @@ -17,14 +17,12 @@ package p11-kit { }; }; - inputs = [ + tools = [ coreutils, diffutils, - - libtasn1, ]; - runtime = [ + inputs = [ libffi, libtasn1, ]; diff --git a/internal/rosa/package/pango/package.az b/internal/rosa/package/pango/package.az index e1d91a7d..d1b78387 100644 --- a/internal/rosa/package/pango/package.az +++ b/internal/rosa/package/pango/package.az @@ -28,11 +28,4 @@ package pango { fribidi, harfbuzz, ]; - - runtime = [ - glib, - cairo, - fribidi, - harfbuzz, - ]; } diff --git a/internal/rosa/package/pcre2.az b/internal/rosa/package/pcre2.az index 27f21bf7..85f603a9 100644 --- a/internal/rosa/package/pcre2.az +++ b/internal/rosa/package/pcre2.az @@ -3,12 +3,12 @@ package pcre2 { website = "https://pcre2project.github.io/pcre2"; anitya = 5832; - version# = "10.47"; + version# = "10.48"; source = remoteGitHubRelease { suffix = "PCRE2Project/pcre2"; tag = "pcre2-"+version; name = "pcre2-"+version+".tar.bz2"; - checksum = "IbC24vVayju6nB9EhrBPSDexk22wDecdpyrjgC3nCZXkwTnUjq4CD2q5sopqu6CW"; + checksum = "3HorftFDryXBh8JLSbEVWpU3OtSF_wUhtNZNCRvCDiGlBsMU150CxrDFKE8lVo6x"; compress = bzip2; }; @@ -23,5 +23,5 @@ package pcre2 { }; }; - inputs = [ diffutils ]; + tools = [ diffutils ]; } diff --git a/internal/rosa/package/perl.az b/internal/rosa/package/perl.az index c2539ee3..8d21c8ca 100644 --- a/internal/rosa/package/perl.az +++ b/internal/rosa/package/perl.az @@ -81,8 +81,8 @@ perl Build.PL --prefix=/system `; }; + tools = [ perl ]; inputs = [ perl ]; - runtime = [ perl ]; } package perl-Locale-gettext { @@ -99,8 +99,6 @@ package perl-Locale-gettext { }; exec = makeMaker {}; - - runtime = [ perl ]; } package perl-Pod-Parser { @@ -117,8 +115,6 @@ package perl-Pod-Parser { }; exec = makeMaker {}; - - runtime = [ perl ]; } package perl-SGMLS { @@ -136,8 +132,6 @@ package perl-SGMLS { }; exec = makeMaker {}; - - runtime = [ perl ]; } package perl-Term-ReadKey { @@ -154,8 +148,6 @@ package perl-Term-ReadKey { }; exec = makeMaker {}; - - runtime = [ perl ]; } package perl-Text-CharWidth { @@ -172,8 +164,6 @@ package perl-Text-CharWidth { }; exec = makeMaker {}; - - runtime = [ perl ]; } package perl-Text-WrapI18N { @@ -192,7 +182,6 @@ package perl-Text-WrapI18N { exec = makeMaker {}; inputs = [ perl-Text-CharWidth ]; - runtime = [ perl-Text-CharWidth ]; } package perl-MIME-Charset { @@ -209,8 +198,6 @@ package perl-MIME-Charset { }; exec = makeMaker {}; - - runtime = [ perl ]; } package perl-Unicode-LineBreak { @@ -229,7 +216,6 @@ package perl-Unicode-LineBreak { exec = makeMaker {}; inputs = [ perl-MIME-Charset ]; - runtime = [ perl-MIME-Charset ]; } package perl-YAML-Tiny { @@ -246,8 +232,6 @@ package perl-YAML-Tiny { }; exec = makeMaker {}; - - runtime = [ perl ]; } package perl-Test-Cmd { @@ -264,6 +248,4 @@ package perl-Test-Cmd { }; exec = makeMaker {}; - - runtime = [ perl ]; } diff --git a/internal/rosa/package/pipewire.az b/internal/rosa/package/pipewire.az index 1cbdeda9..30d95de2 100644 --- a/internal/rosa/package/pipewire.az +++ b/internal/rosa/package/pipewire.az @@ -3,12 +3,12 @@ package wireplumber { website = "https://gitlab.freedesktop.org/pipewire/wireplumber"; anitya = 235056; - version# = "0.5.16"; + version# = "0.5.17"; source = remoteGitLab { domain = "gitlab.freedesktop.org"; suffix = "pipewire/wireplumber"; ref = version; - checksum = "Ftt6s3tt5eDlJPizOWsvjHRfOQezVHOp3sDATJr2_B6MORbFW9XgWSHUNdy-dIp0"; + checksum = "2vWxpOqnQmOlgMlNLYMu9ebPbF2y3W_eqDFM_bwAtCwnHhYE0S4kM8MkpFuXyVlu"; }; // OSError: [Errno 30] Read-only file system: '../../usr/src/wireplumber/po/conf.pot' @@ -30,11 +30,6 @@ package wireplumber { glib, pipewire, ]; - - runtime = [ - glib, - pipewire, - ]; } package pipewire { @@ -42,12 +37,12 @@ package pipewire { website = "https://pipewire.org"; anitya = 57357; - version# = "1.6.8"; + version# = "1.6.9"; source = remoteGitLab { domain = "gitlab.freedesktop.org"; suffix = "pipewire/pipewire"; ref = version; - checksum = "KPPJDODFi6RpNpoGztQGNTiINRubQ_7WVIngsboHQi0XzEhMAKVp7iaUG74QX3aE"; + checksum = "EtqI-7Gwgp_6rHRqVlhuGAt39vOlX_SAoUWzaeeCaUqX4X6feCQvqmSwQvzR88Dr"; }; exec = meson { @@ -66,6 +61,4 @@ package pipewire { kernel-headers, ]; - - runtime = [ dbus ]; } diff --git a/internal/rosa/package/pkg-config.az b/internal/rosa/package/pkg-config.az index 6bde89b7..b8e91958 100644 --- a/internal/rosa/package/pkg-config.az +++ b/internal/rosa/package/pkg-config.az @@ -19,7 +19,7 @@ package pkg-config { }; }; - inputs = [ + tools = [ automake, libtool, ]; diff --git a/internal/rosa/package/pkgconf.az b/internal/rosa/package/pkgconf.az index 7ce1d72c..1ff20388 100644 --- a/internal/rosa/package/pkgconf.az +++ b/internal/rosa/package/pkgconf.az @@ -3,11 +3,11 @@ package pkgconf { website = "https://github.com/pkgconf/pkgconf"; anitya = 12753; - version# = "3.0.6"; + version# = "3.0.7"; source = remoteGitHub { suffix = "pkgconf/pkgconf"; tag = "pkgconf-"+version; - checksum = "2BY20A8o2LTp6qa5MNsH_UPwkN7OGI6DBXGLqpF0g5mOKazYaCVkslFjnI81B2NI"; + checksum = "_IpcXySlbiPfFIS95oAg9C65_wV3BAoLD_QVP2OHx4MNTgdndsgT_WjtJkCbhlEd"; }; exec = meson { diff --git a/internal/rosa/package/popt.az b/internal/rosa/package/popt.az index 964533a8..c063bcb0 100644 --- a/internal/rosa/package/popt.az +++ b/internal/rosa/package/popt.az @@ -14,7 +14,7 @@ package popt { generate = "./autogen.sh"; }; - inputs = [ + tools = [ gzip, automake, libtool, diff --git a/internal/rosa/package/procps.az b/internal/rosa/package/procps.az index 23ba5c80..5152076b 100644 --- a/internal/rosa/package/procps.az +++ b/internal/rosa/package/procps.az @@ -18,7 +18,7 @@ package procps { }; }; - inputs = [ + tools = [ automake, gettext, libtool, diff --git a/internal/rosa/package/python/package.az b/internal/rosa/package/python/package.az index 525cb8c2..dfcb6940 100644 --- a/internal/rosa/package/python/package.az +++ b/internal/rosa/package/python/package.az @@ -56,24 +56,18 @@ package python { postInstall = "ln -s python3 /work/system/bin/python"; }; + tools = [ pkg-config ]; + inputs = [ zlib, bzip2, libffi, openssl, - pkg-config, xz, kernel-headers, ]; - - runtime = [ - zlib, - bzip2, - libffi, - openssl, - ]; } package python-early { @@ -138,12 +132,6 @@ package python-early { kernel-headers, ]; - - runtime = [ - zlib, - bzip2, - libffi, - ]; } package python-setuptools { @@ -165,7 +153,7 @@ package python-setuptools { check = nil; }; - runtime = [ python ]; + unlikely = true; } package python-wheel { @@ -188,20 +176,19 @@ package python-wheel { python-flit-core, python-setuptools, ]; - - runtime = [ python ]; } package python-vcs-versioning { description = "core VCS versioning functionality extracted as a standalone library"; website = "https://setuptools-scm.readthedocs.io/en/latest"; anitya = 389421; + block = "frequent ineffectual updates; questionable maintainer decisions"; - version# = "2.3.1"; + version# = "2.4.0"; source = remoteGitHub { suffix = "pypa/setuptools-scm"; tag = "vcs-versioning-v"+version; - checksum = "rcbu8LlWlY6jmhC95pfnEKgGQuRxHY8_Pvo2jjmlJWSlyHSs1tFIRQ_FoMfi8YXo"; + checksum = "Ziau4BBAwjQVEVe-MlKpA-27nqOm30LlGfHG5SqIrqKbCABWBndCwzpXLxdnZuGn"; }; env = [ @@ -219,23 +206,19 @@ package python-vcs-versioning { python-setuptools, python-packaging, ]; - - runtime = [ - python-setuptools, - python-packaging, - ]; } package python-setuptools-scm { description = "extracts Python package versions from Git or Mercurial metadata"; website = "https://setuptools-scm.readthedocs.io/en/latest"; anitya = 7874; + block = "see package python-vcs-versioning"; - version# = "10.2.1"; + version# = "10.3.2"; source = remoteGitHub { suffix = "pypa/setuptools-scm"; tag = "setuptools-scm-v"+version; - checksum = "XnfdKDHzgtY3imPPz1lA1H0QHnjXlN5ZI-er0I-kfUq4w_a6RzUq8FCOJNBOkpK1"; + checksum = "wnOU9_VeAiNp881loQiJD3KuWgbufNXhQKqHOGBdt9oEjIMXosHjUseE67fW4ips"; }; env = [ @@ -258,6 +241,8 @@ package python-setuptools-scm { python-setuptools, python-vcs-versioning, ]; + + unlikely = true; } package python-flit-core { @@ -265,11 +250,11 @@ package python-flit-core { website = "https://flit.pypa.io"; anitya = 44841; - version# = "4.0.2"; + version# = "4.1.0"; source = remoteGitHub { suffix = "pypa/flit"; tag = version; - checksum = "LAYWMeQ2wf2fUavUctRHlEVCUUfsVu0so4d0DZh-DAkAvd8kf9ZHQd-o7dXr3tq6"; + checksum = "c-iUN-XyUB-1jF4CYaQaMNa5W0G4c3UcpjQcwSI9KOZFQ_my3ebpPjBvfVchjywC"; }; exec = pip { @@ -279,7 +264,7 @@ package python-flit-core { check = nil; }; - runtime = [ python ]; + unlikely = true; } package python-packaging { @@ -300,7 +285,6 @@ package python-packaging { }; inputs = [ python-flit-core ]; - runtime = [ python ]; } package python-pathspec { @@ -321,7 +305,6 @@ package python-pathspec { }; inputs = [ python-flit-core ]; - runtime = [ python ]; } package python-trove-classifiers { @@ -342,7 +325,6 @@ package python-trove-classifiers { }; inputs = [ python-setuptools ]; - runtime = [ python ]; } package python-pluggy { @@ -367,7 +349,6 @@ package python-pluggy { }; inputs = [ python-setuptools-scm ]; - runtime = [ python ]; } package python-poetry-core { @@ -375,17 +356,19 @@ package python-poetry-core { website = "https://github.com/python-poetry/poetry-core"; anitya = 71155; - version# = "2.4.1"; + version# = "2.5.0"; source = remoteGitHub { suffix = "python-poetry/poetry-core"; tag = version; - checksum = "tf9OeSUv9TJwKd48fQIU2uGtZ36GN-PDmPDZzjClozzLjv-rQ5nIbOg9gOXpegiH"; + checksum = "FZCJeScQLZBm2W1JiqgPozV2QKxkU0Dr0tQDsFf-H3PNswQwxSLPEGgEl94va7r3"; }; exec = pip { // pytest circular dependency check = nil; }; + + unlikely = true; } package python-tomlkit { @@ -413,11 +396,11 @@ package python-hatchling { website = "https://hatch.pypa.io"; anitya = 16137; - version# = "1.18.0"; + version# = "1.18.1"; source = remoteGitHub { suffix = "pypa/hatch"; tag = "hatch-v"+version; - checksum = "a_ntndxBlMdqSVW3PkiAY42m-NDgyWJcKLWFIVTB_5PciVeJdYYdbcdF6jsd0U96"; + checksum = "MQHbzLg0Vo9Wp8oNwFqzyr6Z8xIqI8uxp1r_sue5r_QMI6Ryfx4wXJWpQQgdmUdJ"; }; exec = pip { @@ -435,13 +418,7 @@ package python-hatchling { python-tomlkit, ]; - runtime = [ - python-packaging, - python-pathspec, - python-trove-classifiers, - python-pluggy, - python-tomlkit, - ]; + unlikely = true; } package python-pygments { @@ -462,7 +439,6 @@ package python-pygments { }; inputs = [ python-hatchling ]; - runtime = [ python ]; } package python-iniconfig { @@ -487,7 +463,6 @@ package python-iniconfig { }; inputs = [ python-setuptools-scm ]; - runtime = [ python ]; } package python-pytest { @@ -519,13 +494,6 @@ package python-pytest { python-setuptools-scm, ]; - - runtime = [ - python-iniconfig, - python-packaging, - python-pluggy, - python-pygments, - ]; } package python-markupsafe { @@ -546,7 +514,6 @@ package python-markupsafe { }; inputs = [ python-setuptools ]; - runtime = [ python ]; } package python-mako { @@ -576,8 +543,6 @@ package python-mako { python-setuptools, ]; - - runtime = [ python-markupsafe ]; } package python-pyyaml { @@ -598,7 +563,6 @@ package python-pyyaml { }; inputs = [ python-setuptools ]; - runtime = [ python ]; } package python-pycparser { @@ -621,5 +585,4 @@ ln -s clang /system/bin/gcc exec = pip {}; inputs = [ python-setuptools ]; - runtime = [ python ]; } diff --git a/internal/rosa/package/qemu/package.az b/internal/rosa/package/qemu/package.az index e18794ca..3e7b8fcf 100644 --- a/internal/rosa/package/qemu/package.az +++ b/internal/rosa/package/qemu/package.az @@ -62,7 +62,7 @@ sed -i \ ]; }; - inputs = [ + tools = [ bash, python-setuptools, python-wheel, @@ -70,22 +70,17 @@ sed -i \ ninja, pkgconf, diffutils, - openssl, - xz, - flex, bison, m4, - - glib, - zstd, dtc, - kernel-headers, ]; - runtime = [ + inputs = [ + xz, glib, zstd, + kernel-headers, ]; } diff --git a/internal/rosa/package/rdfind.az b/internal/rosa/package/rdfind.az index 5c16548d..7d737eab 100644 --- a/internal/rosa/package/rdfind.az +++ b/internal/rosa/package/rdfind.az @@ -15,5 +15,4 @@ package rdfind { exec = make {}; inputs = [ nettle ]; - runtime = [ nettle ]; } diff --git a/internal/rosa/package/rename.az b/internal/rosa/package/rename.az index 9e98ae0f..f6c36204 100644 --- a/internal/rosa/package/rename.az +++ b/internal/rosa/package/rename.az @@ -12,6 +12,4 @@ package rename { }; exec = makeMaker {}; - - runtime = [ perl ]; } diff --git a/internal/rosa/package/spirv.az b/internal/rosa/package/spirv.az index cefa4d0a..79ac5a8d 100644 --- a/internal/rosa/package/spirv.az +++ b/internal/rosa/package/spirv.az @@ -36,13 +36,8 @@ package spirv-tools { }; }; - inputs = [ - python, - - spirv-headers, - ]; - - runtime = [ spirv-headers ]; + tools = [ python ]; + inputs = [ spirv-headers ]; } package spirv-llvm-translator { @@ -50,11 +45,11 @@ package spirv-llvm-translator { website = "https://github.com/KhronosGroup/SPIRV-LLVM-Translator"; anitya = 227273; - version# = "23.1.0"; + version# = "23.1.1"; source = remoteGitHub { suffix = "KhronosGroup/SPIRV-LLVM-Translator"; tag = "v"+version; - checksum = "bKflf64sXw0NDhtqE_gT5cHrQV9JNZsAhvpTkDImp01xcyTXWPYoNB1CsbzGgFxh"; + checksum = "8Sr087sQcaGw1yOodbRQAOv5bBj_QpxbOWGN75xxKqhRcuAUvdPZ9a009nlXZRwK"; }; // litArgs emits shell syntax @@ -157,12 +152,10 @@ package spirv-llvm-translator { }; }; - inputs = [ + tools = [ bash, lit, - - spirv-tools, ]; - runtime = [ spirv-tools ]; + inputs = [ spirv-tools ]; } diff --git a/internal/rosa/package/squashfs-tools.az b/internal/rosa/package/squashfs-tools.az index b266e3fe..145c1cb9 100644 --- a/internal/rosa/package/squashfs-tools.az +++ b/internal/rosa/package/squashfs-tools.az @@ -34,15 +34,9 @@ package squashfs-tools { install = "make INSTALL_PREFIX=/work/system install"; }; - inputs = [ - sed, + tools = [ sed ]; - zstd, - gzip, - zlib, - ]; - - runtime = [ + inputs = [ zstd, gzip, zlib, diff --git a/internal/rosa/package/stage0.az b/internal/rosa/package/stage0.az index e415bbbe..d79851ef 100644 --- a/internal/rosa/package/stage0.az +++ b/internal/rosa/package/stage0.az @@ -19,7 +19,7 @@ tar -vc -C /stage0 . | \ `; }; - inputs = [ zstd ]; + tools = [ zstd ]; } package stage0-dist { diff --git a/internal/rosa/package/strace/package.az b/internal/rosa/package/strace/package.az index 7e46147d..6173ae38 100644 --- a/internal/rosa/package/strace/package.az +++ b/internal/rosa/package/strace/package.az @@ -25,11 +25,11 @@ package strace { }; }; - inputs = [ + tools = [ bash, gawk, automake, - - kernel-headers, ]; + + inputs = [ kernel-headers ]; } diff --git a/internal/rosa/package/sway/package.az b/internal/rosa/package/sway/package.az index 6ff1b365..20834397 100644 --- a/internal/rosa/package/sway/package.az +++ b/internal/rosa/package/sway/package.az @@ -11,6 +11,7 @@ package sway { }; patches = [ "remove-unused.patch" ]; + env = [ "CFLAGS=-Wno-switch" ]; exec = meson {}; inputs = [ @@ -21,12 +22,4 @@ package sway { wlroots, kernel-headers, ]; - - runtime = [ - json-c, - pcre2, - pango, - libinput, - wlroots, - ]; } diff --git a/internal/rosa/package/tamago.az b/internal/rosa/package/tamago.az index e0aca055..d784c85a 100644 --- a/internal/rosa/package/tamago.az +++ b/internal/rosa/package/tamago.az @@ -3,11 +3,11 @@ package tamago { website = "https://github.com/usbarmory/tamago-go"; anitya = 388872; - version# = "1.27.0"; + version# = "1.27.1"; source = remoteGitHub { suffix = "usbarmory/tamago-go"; tag = "tamago-go"+version; - checksum = "PV8a5fccqZifVUG7HJ6jTFk_0x6YIu1BSgYByCdJoTllm4NuQc9-Ngk_nuS7Nb3i"; + checksum = "TaWPzqqBMuhGslUm9CQvShJRjfONbFEr51mcM5zFlopYhYA6Tb58g8wRPc5a5lQ0"; }; env = [ @@ -43,5 +43,5 @@ set -u install = "../bin/go tool dist banner # print build info"; }; - inputs = [ bash, go ]; + tools = [ bash, go ]; } diff --git a/internal/rosa/package/tllist.az b/internal/rosa/package/tllist.az index a02196b8..acce550e 100644 --- a/internal/rosa/package/tllist.az +++ b/internal/rosa/package/tllist.az @@ -12,4 +12,6 @@ package tllist { }; exec = meson {}; + + unlikely = true; } diff --git a/internal/rosa/package/toybox.az b/internal/rosa/package/toybox.az index f54db3d2..bd3dc966 100644 --- a/internal/rosa/package/toybox.az +++ b/internal/rosa/package/toybox.az @@ -60,12 +60,12 @@ ln -s ../../system/bin/env /work/usr/bin `; }; - inputs = [ + tools = [ bash, gzip, - - kernel-headers, ]; + + inputs = [ kernel-headers ]; } package toybox-early { @@ -125,10 +125,10 @@ ln -s ../../system/bin/env /work/usr/bin `; }; - inputs = [ + tools = [ bash, gzip, - - kernel-headers, ]; + + inputs = [ kernel-headers ]; } diff --git a/internal/rosa/package/unzip.az b/internal/rosa/package/unzip.az index 6fda9763..3bc8ae8b 100644 --- a/internal/rosa/package/unzip.az +++ b/internal/rosa/package/unzip.az @@ -29,7 +29,7 @@ mv unzip /work/system/bin/ `; }; - inputs = [ + tools = [ make, coreutils, ]; diff --git a/internal/rosa/package/util-linux/a323dddbcd1ed05a10e7e870b3e1a48b4ed44a43.patch b/internal/rosa/package/util-linux/a323dddbcd1ed05a10e7e870b3e1a48b4ed44a43.patch new file mode 100644 index 00000000..36d6f968 --- /dev/null +++ b/internal/rosa/package/util-linux/a323dddbcd1ed05a10e7e870b3e1a48b4ed44a43.patch @@ -0,0 +1,35 @@ +From a323dddbcd1ed05a10e7e870b3e1a48b4ed44a43 Mon Sep 17 00:00:00 2001 +From: Karel Zak <kzak@redhat.com> +Date: Wed, 2 Sep 2026 13:32:27 +0200 +Subject: [PATCH] libmount: add missing fileutils.h include to hook_idmap.c + +The hook_idmap.c uses RESOLVE_NO_SYMLINKS (added by commit fb8e26535) +but does not include fileutils.h, which provides the fallback #define +for this constant. + +On Fedora (glibc 2.40+), this is masked because glibc's +<bits/fcntl-linux.h> transitively includes <linux/openat2.h>, which +defines RESOLVE_NO_SYMLINKS. On Ubuntu (and other distros with older +glibc), <fcntl.h> does not pull in openat2.h, so the build fails: + + hook_idmap.c:335:33: error: 'RESOLVE_NO_SYMLINKS' undeclared + +Fixes: fb8e26535 ("libmount: pin source path with openat2() for restricted users") +Signed-off-by: Karel Zak <kzak@redhat.com> +(cherry picked from commit 7e2e010874b10b3aabdc3c4c844c9ffc46a4a374) +--- + libmount/src/hook_idmap.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/libmount/src/hook_idmap.c b/libmount/src/hook_idmap.c +index 77494e29810..2c697b17154 100644 +--- a/libmount/src/hook_idmap.c ++++ b/libmount/src/hook_idmap.c +@@ -23,6 +23,7 @@ + + #include "strutils.h" + #include "all-io.h" ++#include "fileutils.h" + #include "namespace.h" + + #include "mountP.h" diff --git a/internal/rosa/package/util-linux.az b/internal/rosa/package/util-linux/package.az index 405f2ab7..1317e50f 100644 --- a/internal/rosa/package/util-linux.az +++ b/internal/rosa/package/util-linux/package.az @@ -5,7 +5,7 @@ package util-linux { // release candidates confuse Anitya latest = anityaFallback; - version# = "2.42.2"; + version# = "2.42.3"; source = remoteTar { url = "https://www.kernel.org/pub/linux/utils/util-linux/"+ "v"+join { @@ -19,9 +19,10 @@ package util-linux { }; sep = "."; }+"/util-linux-"+version+".tar.gz"; - checksum = "Z4IZapPCKQP37aeWMDhxcvuOCy-IR_aHZvfUEMZ0T7trwC1znZACKH_3ddqXwBCg"; + checksum = "VQ3NCM4kt0-TTrdJElBfckXNCGBv6UfXGTRm1poQ0XP0WBND4eG-O7n5Ly_a-Iw_"; compress = gzip; }; + patches = [ "a323dddbcd1ed05a10e7e870b3e1a48b4ed44a43.patch" ]; bin = [ "bash" ]; exec = make { @@ -50,12 +51,10 @@ package util-linux { check = nil; }; - inputs = [ - bash, + tools = [ bash ]; + inputs = [ ncurses, kernel-headers, ]; - - runtime = [ ncurses ]; } diff --git a/internal/rosa/package/vim.az b/internal/rosa/package/vim.az index 1ea7883c..81e1cab3 100644 --- a/internal/rosa/package/vim.az +++ b/internal/rosa/package/vim.az @@ -28,5 +28,4 @@ package vim { }; inputs = [ ncurses ]; - runtime = [ ncurses ]; } diff --git a/internal/rosa/package/wayland.az b/internal/rosa/package/wayland.az index b4d755a1..3b240994 100644 --- a/internal/rosa/package/wayland.az +++ b/internal/rosa/package/wayland.az @@ -25,16 +25,12 @@ echo 'int main(){}' > tests/sanity-test.c }; }; - inputs = [ + tools = [ awk, diffutils, - - libffi, - libexpat, - libxml2, ]; - runtime = [ + inputs = [ libffi, libexpat, libxml2, @@ -57,4 +53,6 @@ package wayland-protocols { exec = meson {}; inputs = [ wayland ]; + runtime = nil; + unlikely = true; } diff --git a/internal/rosa/package/wlroots.az b/internal/rosa/package/wlroots.az index 62593779..7cfeb261 100644 --- a/internal/rosa/package/wlroots.az +++ b/internal/rosa/package/wlroots.az @@ -24,6 +24,7 @@ package wlroots { libglvnd, hwdata, wayland, + wayland-protocols, xwayland, libxcb-wm, xkbcommon, @@ -34,15 +35,6 @@ package wlroots { kernel-headers, ]; - runtime = [ - mesa, - libglvnd, - wayland, - libxcb-wm, - xkbcommon, - pixman, - seatd, - libinput, - libliftoff, - ]; + join = true; + runtime = [ wayland-protocols ]; } diff --git a/internal/rosa/package/x.az b/internal/rosa/package/x.az index a36febd0..94c48ac8 100644 --- a/internal/rosa/package/x.az +++ b/internal/rosa/package/x.az @@ -12,6 +12,7 @@ package util-macros { }; exec = make {}; + unlikely = true; } package libxtrans { @@ -31,13 +32,15 @@ package libxtrans { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, - - util-macros, ]; + + inputs = [ util-macros ]; + + unlikely = true; } package xorgproto { @@ -57,12 +60,14 @@ package xorgproto { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, pkgconf, - - util-macros, ]; + + inputs = [ util-macros ]; + + unlikely = true; } package libXau { @@ -102,7 +107,7 @@ package xcb-proto { exec = make {}; - inputs = [ python ]; + tools = [ python ]; } package xcb { @@ -119,15 +124,12 @@ package xcb { exec = make {}; - inputs = [ + tools = [ python, pkgconf, - - xcb-proto, - libXau, ]; - runtime = [ + inputs = [ xcb-proto, libXau, ]; @@ -146,14 +148,8 @@ package libxcb-keysyms { }; exec = make {}; - - inputs = [ - pkgconf, - - xcb, - ]; - - runtime = [ xcb ]; + tools = [ pkgconf ]; + inputs = [ xcb ]; } package libxcb-wm { @@ -170,14 +166,12 @@ package libxcb-wm { exec = make {}; - inputs = [ + tools = [ m4, pkgconf, - - xcb, ]; - runtime = [ xcb ]; + inputs = [ xcb ]; } package libxcb-util { @@ -195,14 +189,12 @@ package libxcb-util { exec = make {}; - inputs = [ - pkgconf, + tools = [ pkgconf ]; + inputs = [ util-macros, xcb, ]; - - runtime = [ xcb ]; } package libxcb-image { @@ -218,14 +210,8 @@ package libxcb-image { }; exec = make {}; - - inputs = [ - pkgconf, - - libxcb-util, - ]; - - runtime = [ libxcb-util ]; + tools = [ pkgconf ]; + inputs = [ libxcb-util ]; } package libxcb-render-util { @@ -243,14 +229,12 @@ package libxcb-render-util { exec = make {}; - inputs = [ - pkgconf, + tools = [ pkgconf ]; + inputs = [ util-macros, xcb, ]; - - runtime = [ xcb ]; } package libX11 { @@ -275,18 +259,18 @@ package libX11 { }; }; - inputs = [ + tools = [ automake, libtool, pkgconf, + ]; + inputs = [ util-macros, libxtrans, xorgproto, xcb, ]; - - runtime = [ xcb ]; } package libXext { @@ -306,16 +290,16 @@ package libXext { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, + ]; + inputs = [ util-macros, libX11, ]; - - runtime = [ libX11 ]; } package libXfixes { @@ -335,17 +319,17 @@ package libXfixes { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, + ]; + inputs = [ util-macros, xorgproto, libX11, ]; - - runtime = [ libX11 ]; } package libXrender { @@ -365,16 +349,16 @@ package libXrender { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, + ]; + inputs = [ util-macros, libX11, ]; - - runtime = [ libX11 ]; } package libxshmfence { @@ -394,11 +378,13 @@ package libxshmfence { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, + ]; + inputs = [ util-macros, xorgproto, ]; @@ -421,17 +407,17 @@ package libXxf86vm { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, + ]; + inputs = [ util-macros, libXext, xorgproto, ]; - - runtime = [ libXext ]; } package libXrandr { @@ -451,21 +437,18 @@ package libXrandr { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, + ]; + inputs = [ util-macros, libXext, libXrender, xorgproto, ]; - - runtime = [ - libXext, - libXrender, - ]; } package font-util { @@ -485,13 +468,13 @@ package font-util { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, - - util-macros, ]; + + inputs = [ util-macros ]; } package libfontenc { @@ -511,13 +494,15 @@ package libfontenc { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, + font-util, + ]; + inputs = [ util-macros, - font-util, xorgproto, ]; } @@ -538,7 +523,6 @@ package libxkbfile { exec = meson {}; inputs = [ libX11 ]; - runtime = [ libX11 ]; } package xkbcomp { @@ -558,17 +542,17 @@ package xkbcomp { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, byacc, + ]; + inputs = [ util-macros, libxkbfile, ]; - - runtime = [ libxkbfile ]; } package libXfont2 { @@ -588,22 +572,19 @@ package libXfont2 { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, + ]; + inputs = [ util-macros, freetype, xorgproto, libxtrans, libfontenc, ]; - - runtime = [ - freetype, - libfontenc, - ]; } package libxcvt { @@ -639,11 +620,13 @@ package libXdmcp { generate = "NOCONFIGURE=1 ./autogen.sh"; }; - inputs = [ + tools = [ automake, libtool, pkgconf, + ]; + inputs = [ util-macros, xorgproto, ]; @@ -663,8 +646,7 @@ package xkeyboard-config { }; exec = meson {}; - - inputs = [ perl ]; + tools = [ perl ]; } package libpciaccess { @@ -687,7 +669,6 @@ package libpciaccess { }; inputs = [ zlib ]; - runtime = [ zlib ]; } package xserver { @@ -695,12 +676,12 @@ package xserver { website = "https://gitlab.freedesktop.org/xorg/xserver"; anitya = 5250; - version# = "26.0.99.901"; + version# = "26.0.99.902"; source = remoteGitLab { domain = "gitlab.freedesktop.org"; suffix = "xorg/xserver"; ref = "xorg-server-"+version; - checksum = "2Liwqj-tu_NuD8BuWZR0PyYVJ_A0nj_sgx-LZWtdVpKAKQHtN-dSWCl-IPVOlcHG"; + checksum = "lg1TlhQHnQowDeYzJQVoMHZsUSfTLbZVGwMi8afUHCVlVTbCuRRXLtQWoZjkt4QT"; }; exec = meson { @@ -719,9 +700,9 @@ package xserver { }; }; - inputs = [ - awk, + tools = [ awk ]; + inputs = [ xorgproto, libxtrans, libxshmfence, @@ -730,7 +711,6 @@ package xserver { xkbcomp, xkeyboard-config, libXfont2, - dbus, font-util, libxcvt, libXext, @@ -773,12 +753,12 @@ package xwayland { website = "https://gitlab.freedesktop.org/xorg/xserver"; anitya = 180949; - version# = "26.0.99.901"; + version# = "26.0.99.902"; source = remoteGitLab { domain = "gitlab.freedesktop.org"; suffix = "xorg/xserver"; ref = "xwayland-"+version; - checksum = "Vgtnwme2HU3CCeLE_xNScV_uJ55kPMremaOyVWRUFaRpwpaOQxsNB2fde_9Rx5zm"; + checksum = "D60hCizAFqVKBH49_FbHRigeEgzoOUXRgvEncuE6kzdoIUr5iC8RIBncVuTFgY0D"; }; bin = [ "bash" ]; @@ -789,10 +769,12 @@ package xwayland { }; }; - inputs = [ + tools = [ bash, awk, + ]; + inputs = [ mesa, libglvnd, libepoxy, diff --git a/internal/rosa/package/xkbcommon.az b/internal/rosa/package/xkbcommon.az index a07d4d94..d6eb1512 100644 --- a/internal/rosa/package/xkbcommon.az +++ b/internal/rosa/package/xkbcommon.az @@ -21,17 +21,12 @@ package xkbcommon { }; }; - inputs = [ + tools = [ m4, bison, - - libxml2, - wayland, - wayland-protocols, - xkeyboard-config, ]; - runtime = [ + inputs = [ libxml2, wayland, wayland-protocols, diff --git a/internal/rosa/package/xz.az b/internal/rosa/package/xz.az index a02916b4..3bb91b5f 100644 --- a/internal/rosa/package/xz.az +++ b/internal/rosa/package/xz.az @@ -3,16 +3,18 @@ package xz { website = "https://tukaani.org/xz"; anitya = 5277; - version# = "5.8.3"; + version# = "5.8.4"; source = remoteGitHubRelease { suffix = "tukaani-project/xz"; tag = "v"+version; name = "xz-"+version+".tar.bz2"; - checksum = "nCdayphPGdIdVoAZ2hR4vYlhDG9LeVKho_i7ealTud4Vxy5o5dWe0VwFlN7utuUL"; + checksum = "sYLUjO5X0QREmCBYceLU3CSj-HUCd3g0kJAHq8F2GO0Y6-LcyKYbESyRq4sr7Bih"; compress = bzip2; }; exec = make {}; - inputs = [ diffutils ]; + tools = [ diffutils ]; + + unlikely = true; } diff --git a/internal/rosa/package/zstd.az b/internal/rosa/package/zstd.az index ec1e21b1..29e2c51f 100644 --- a/internal/rosa/package/zstd.az +++ b/internal/rosa/package/zstd.az @@ -24,7 +24,7 @@ ZSTD_BIN=/cure/programs/zstd /usr/src/zstd/tests/playTests.sh `; }; - inputs = [ + tools = [ make, diffutils, ]; diff --git a/internal/rosa/perl.go b/internal/rosa/perl.go index a828609b..705b1b8d 100644 --- a/internal/rosa/perl.go +++ b/internal/rosa/perl.go @@ -2,14 +2,19 @@ package rosa var _perl = H("perl") +var _ Helper = new(MakeMakerHelper) + // MakeMakerHelper builds projects using the included MakeMaker script. type MakeMakerHelper struct { // Whether to skip the check target. SkipCheck bool } -// extra returns perl. -func (*MakeMakerHelper) extra() P { return P{_perl, _make} } +// tools returns perl and make. +func (*MakeMakerHelper) tools() P { return P{_perl, _make} } + +// inputs returns perl. +func (*MakeMakerHelper) inputs() P { return P{_perl} } // wantsChmod returns true. func (*MakeMakerHelper) wantsChmod() bool { return true } diff --git a/internal/rosa/python.go b/internal/rosa/python.go index 96f7714c..54022bd7 100644 --- a/internal/rosa/python.go +++ b/internal/rosa/python.go @@ -25,14 +25,17 @@ type PipHelper struct { var _ Helper = new(PipHelper) -// extra returns python, or pytest if defaults are assumed. -func (attr *PipHelper) extra() P { +// tools returns python and pytest if defaults are assumed. +func (attr *PipHelper) tools() P { if attr == nil || (!attr.SkipCheck && attr.Check == "") { - return P{_pythonPyTest} + return P{_python, _pythonPyTest} } return P{_python} } +// inputs returns python. +func (*PipHelper) inputs() P { return P{_python} } + // wantsChmod returns true. func (*PipHelper) wantsChmod() bool { return true } diff --git a/internal/rosa/report.go b/internal/rosa/report.go index 7792d9e9..7a1f4521 100644 --- a/internal/rosa/report.go +++ b/internal/rosa/report.go @@ -14,8 +14,8 @@ import ( "unique" "unsafe" - "hakurei.app/internal/pkg" "hakurei.app/message" + "hakurei.app/pkg" ) // wordSize is the boundary which binary segments are always aligned to. diff --git a/internal/rosa/report_test.go b/internal/rosa/report_test.go index 2ca8ac57..8154bc5a 100644 --- a/internal/rosa/report_test.go +++ b/internal/rosa/report_test.go @@ -8,8 +8,8 @@ import ( "testing" "unique" - "hakurei.app/internal/pkg" "hakurei.app/internal/rosa" + "hakurei.app/pkg" ) func TestReportZeroLength(t *testing.T) { diff --git a/internal/rosa/rosa.go b/internal/rosa/rosa.go index 7b06700c..beffb3ff 100644 --- a/internal/rosa/rosa.go +++ b/internal/rosa/rosa.go @@ -10,7 +10,7 @@ import ( "unsafe" "hakurei.app/fhs" - "hakurei.app/internal/pkg" + "hakurei.app/pkg" ) // Extension is the variant identification string of custom artifact @@ -277,10 +277,12 @@ function apply { // helperInPlace is a special directory value for omitting the cd statement. const helperInPlace = "\x00" -// Helper is a build system helper for [Toolchain.NewPackage]. +// Helper is a build system helper for [Toolchain.New]. type Helper interface { - // extra returns helper-specific dependencies. - extra() P + // tools returns helper-specific tool handles. + tools() P + // inputs returns helper-specific input handles. + inputs() P // wantsChmod returns whether the source directory should be made writable. wantsChmod() bool @@ -335,6 +337,8 @@ type PackageAttr struct { // Whether to create [pkg.KindExecNet] instead of [pkg.KindExec]. HostNet bool + // Native utilities used by the build process. + Tools P // Unregistered extras. Extra []pkg.Artifact // Passed through to [Toolchain.New], before source. @@ -362,7 +366,7 @@ func (t Toolchain) appendHandle(a []pkg.Artifact, pv pa, p ArtifactH) []pkg.Arti pv[p] = struct{}{} meta, u := t.MustLoad(p) - for _, d := range meta.Dependencies { + for _, d := range meta.Runtimes { a = t.appendHandle(a, pv, d) } return append(a, u) @@ -393,7 +397,7 @@ func (t Toolchain) New( source pkg.Artifact, attr *PackageAttr, helper Helper, - extra ...ArtifactH, + inputs ...ArtifactH, ) pkg.Artifact { if attr == nil { attr = new(PackageAttr) @@ -411,7 +415,10 @@ func (t Toolchain) New( rn = name + "-" + version } - root := make([]pkg.Artifact, 0, 1<<3+len(attr.Extra)+len(extra)) + root := make([]pkg.Artifact, 0, 1<<3+ + len(attr.Extra)+ + len(attr.Tools)+ + len(inputs)) root = append(root, attr.Extra...) const lcMessages = "LC_MESSAGES=C.UTF-8" @@ -485,10 +492,16 @@ func (t Toolchain) New( wantsChmod, wantsWrite := helper.wantsChmod(), helper.wantsWrite() { pv := paGet() - for _, p := range helper.extra() { + for _, p := range helper.tools() { root = t.appendHandle(root, pv, p) } - for _, p := range extra { + for _, p := range helper.inputs() { + root = t.appendHandle(root, pv, p) + } + for _, p := range attr.Tools { + root = t.appendHandle(root, pv, p) + } + for _, p := range inputs { root = t.appendHandle(root, pv, p) } boot := t.S.New(t.stage - 1) @@ -592,8 +605,11 @@ type GenericHelper struct { var _ Helper = new(GenericHelper) -// extra is a noop. -func (*GenericHelper) extra() P { return nil } +// tools is a noop. +func (*GenericHelper) tools() P { return nil } + +// inputs is a noop. +func (*GenericHelper) inputs() P { return nil } // wantsChmod returns false. func (*GenericHelper) wantsChmod() bool { return false } diff --git a/internal/rosa/rosa_test.go b/internal/rosa/rosa_test.go index 76e8a53e..ce3dad42 100644 --- a/internal/rosa/rosa_test.go +++ b/internal/rosa/rosa_test.go @@ -10,9 +10,9 @@ import ( "testing" "hakurei.app/check" - "hakurei.app/internal/pkg" "hakurei.app/internal/rosa" "hakurei.app/message" + "hakurei.app/pkg" ) var ( diff --git a/internal/rosa/state.go b/internal/rosa/state.go index ba17ea23..01abf736 100644 --- a/internal/rosa/state.go +++ b/internal/rosa/state.go @@ -25,8 +25,8 @@ import ( "hakurei.app/check" "hakurei.app/container" - "hakurei.app/internal/pkg" "hakurei.app/internal/rosa/azalea" + "hakurei.app/pkg" ) // ArtifactH is a handle of the unique name of a prepared [pkg.Artifact]. @@ -91,7 +91,9 @@ type Metadata struct { Website string `json:"website,omitempty"` // Runtime dependencies. - Dependencies P `json:"dependencies"` + Runtimes P `json:"runtime"` + // Whether this package is unlike to be a runtime dependency. + Unlikely bool `json:"unlikely,omitempty"` // Package version. Version string `json:"version"` @@ -1028,6 +1030,12 @@ func (t Toolchain) evalArtifact(v any, meta *Metadata) pkg.Artifact { } } +// isUnlikely returns whether p is unlikely to be a runtime dependency. +func (t Toolchain) isUnlikely(handle ArtifactH) bool { + meta, _ := t.MustLoad(handle) + return meta.Unlikely +} + // pf implements [azalea.PF]. func (ctx *evalContext) pf( name azalea.Ident, @@ -1049,9 +1057,11 @@ func (ctx *evalContext) pf( source any helper Helper overlay string + join bool - inputs, runtimes, extra azalea.Array + tools, inputs, extra azalea.Array + runtimes = azalea.Array{} bootstrap = true ) if err = args.Apply(map[unique.Handle[azalea.Ident]]any{ @@ -1082,7 +1092,10 @@ func (ctx *evalContext) pf( k("source"): &source, k("exec"): &helper, k("inputs"): &inputs, + k("tools"): &tools, k("runtime"): &runtimes, + k("join"): &join, + k("unlikely"): &meta.Unlikely, k("extra"): &extra, k("overlay"): &overlay, }); err != nil { @@ -1092,12 +1105,39 @@ func (ctx *evalContext) pf( var inputsH, extraH P if inputsH, err = toHandles(inputs); err != nil { return - } else if meta.Dependencies, err = toHandles(runtimes); err != nil { + } else if attr.Tools, err = toHandles(tools); err != nil { return } else if extraH, err = toHandles(extra); err != nil { return } + if join || (runtimes != nil && len(runtimes) == 0 && helper != nil) { + hi := helper.inputs() + meta.Runtimes = make(P, 0, len(hi)+len(inputsH)+len(runtimes)) + for _, h := range hi { + if ctx.t.isUnlikely(h) { + continue + } + meta.Runtimes = append(meta.Runtimes, h) + } + for _, h := range inputsH { + if ctx.t.isUnlikely(h) { + continue + } + meta.Runtimes = append(meta.Runtimes, h) + } + + if join { + var extraRuntimes P + if extraRuntimes, err = toHandles(runtimes); err != nil { + return + } + meta.Runtimes = append(meta.Runtimes, extraRuntimes...) + } + } else if meta.Runtimes, err = toHandles(runtimes); err != nil { + return + } + meta.ID = int(anitya) for _, pathname := range patches { var p []byte @@ -1299,6 +1339,30 @@ func (s *S) RegisterFS(fsys fs.FS) error { return nil } +// setSource overrides the hakurei-source package with the specified tarball and +// version string. The resulting IR is curable on the daemon. Must not be used +// concurrently with any other method. +func (s *S) setSource(p []byte, version string) { + const name = "hakurei-source" + s.p.Store( + H(name), + Artifact(func(t Toolchain) (*Metadata, pkg.Artifact) { + return &Metadata{ + Name: name, + Description: "hakurei source tree (current)", + Version: version, + Exclude: true, + }, pkg.NewTar( + pkg.NewDecompress( + pkg.NewFile("hakurei-src-current.tar.gz", p), + pkg.Gzip, + ), + ) + }), + ) + s.DropCaches(s.Arch(), s.Flags()) +} + // SetSource overrides the hakurei-source package with a cached tarball of fsys. // The resulting IR is curable on the daemon. Must not be used concurrently with // any other method. @@ -1376,20 +1440,7 @@ func (s *S) SetSource(fsys fs.FS) error { return err } - const name = "hakurei-source" - a := pkg.NewFile("hakurei-src-current.tar.gz", buf.Bytes()) - s.p.Store( - H(name), - Artifact(func(t Toolchain) (*Metadata, pkg.Artifact) { - return &Metadata{ - Name: name, - Description: "hakurei source tree (current)", - Version: version, - Exclude: true, - }, pkg.NewTar(pkg.NewDecompress(a, pkg.Gzip)) - }), - ) - s.DropCaches(s.Arch(), s.Flags()) + s.setSource(buf.Bytes(), version) return nil } diff --git a/internal/store/data.go b/internal/store/data.go index 34a0f261..380481a0 100644 --- a/internal/store/data.go +++ b/internal/store/data.go @@ -23,7 +23,8 @@ func entryEncode(w io.Writer, s *hst.State) error { } } -// entryDecodeHeader calls entryReadHeader, returning [hst.AppError] for a non-nil error. +// entryDecodeHeader calls entryReadHeader, returning [hst.AppError] for a +// non-nil error. func entryDecodeHeader(r io.Reader) (hst.Enablements, error) { if et, err := entryReadHeader(r); err != nil { return 0, &hst.AppError{Step: "decode state header", Err: err} @@ -44,11 +45,14 @@ func entryDecode(r io.Reader, p *hst.State) (hst.Enablements, error) { } else if err = p.Config.Validate(hst.VAllowInsecure); err != nil { return et, err } else if p.Enablements.Unwrap() != et { - return et, &hst.AppError{Step: "validate state enablement", Err: os.ErrInvalid, + return et, &hst.AppError{ + Step: "validate state enablement", + Err: os.ErrInvalid, Msg: fmt.Sprintf( "state entry %s has unexpected enablement byte %#x, %#x", p.ID.String(), byte(p.Enablements.Unwrap()), byte(et), - )} + ), + } } else { return et, nil } diff --git a/internal/store/header.go b/internal/store/header.go index 5b5d73be..0e5a92a2 100644 --- a/internal/store/header.go +++ b/internal/store/header.go @@ -60,7 +60,8 @@ func (e *EntrySizeError) Error() string { return "state entry file " + strconv.Quote(e.Name) + " is too short" } -// entryCheckFile checks whether [os.FileInfo] refers to a file that might hold [hst.State]. +// entryCheckFile checks whether [os.FileInfo] refers to a file that might hold +// [hst.State]. func entryCheckFile(fi os.FileInfo) error { if fi.IsDir() { return syscall.EISDIR diff --git a/internal/store/segment.go b/internal/store/segment.go index 06849749..8b7dd0d6 100644 --- a/internal/store/segment.go +++ b/internal/store/segment.go @@ -7,6 +7,7 @@ import ( "os" "strconv" "sync" + "syscall" "hakurei.app/check" "hakurei.app/hst" @@ -76,13 +77,20 @@ func (eh *EntryHandle) save(state *hst.State) error { return err } +// KillFunc is the function signature of syscall.Kill. +type KillFunc func(pid int, sig syscall.Signal) (err error) + // Load loads and validates the state entry header, and returns the // [hst.Enablements] byte. For a non-nil v, the full state payload is decoded -// and stored in the value pointed to by v. +// and stored in the value pointed to by v, and if kill is non-nil, the presence +// of the monitoring process is checked, and a stale entry is destroyed. // // Load validates the embedded [hst.Config] value. A non-nil error returned by // Load is of type [hst.AppError]. -func (eh *EntryHandle) Load(v *hst.State) (hst.Enablements, error) { +func (eh *EntryHandle) Load( + v *hst.State, + kill KillFunc, +) (hst.Enablements, error) { f, err := eh.open(os.O_RDONLY, 0) if err != nil { return 0, err @@ -92,8 +100,41 @@ func (eh *EntryHandle) Load(v *hst.State) (hst.Enablements, error) { if v != nil { et, err = entryDecode(f, v) if err == nil && v.ID != eh.ID { - err = &hst.AppError{Step: "validate state identifier", Err: os.ErrInvalid, - Msg: fmt.Sprintf("state entry %s has unexpected id %s", eh.ID.String(), v.ID.String())} + err = &hst.AppError{ + Step: "validate state identifier", + Err: os.ErrInvalid, + Msg: fmt.Sprintf( + "state entry %s has unexpected id %s", + eh.ID.String(), v.ID.String(), + ), + } + } + if kill != nil { + errno := kill(v.PID, 0) + if errno != nil { + if !errors.Is(errno, syscall.ESRCH) { + err = &hst.AppError{ + Step: "check monitor process", + Err: errno, + } + } else { + if err = eh.Destroy(); err != nil { + err = &hst.AppError{ + Step: "destroy stale entry", + Err: err, + } + } else { + err = &hst.AppError{ + Step: "load stale entry", + Err: errno, + Msg: fmt.Sprintf( + "stale entry %s", + eh.ID.String(), + ), + } + } + } + } } } else { et, err = entryDecodeHeader(f) @@ -127,7 +168,10 @@ type Handle struct { // A non-nil error returned by Lock is of type [hst.AppError]. func (h *Handle) Lock() (unlock func(), err error) { if unlock, err = h.fileMu.Lock(); err != nil { - return nil, &hst.AppError{Step: "acquire lock on store segment " + strconv.Itoa(h.Identity), Err: err} + return nil, &hst.AppError{ + Step: "acquire lock on store segment " + strconv.Itoa(h.Identity), + Err: err, + } } return } @@ -174,8 +218,11 @@ func (h *Handle) Entries() (iter.Seq[*EntryHandle], int, error) { // this should never happen if ent.IsDir() { - eh.DecodeErr = &hst.AppError{Step: step, - Err: errors.New("unexpected directory " + strconv.Quote(ent.Name()) + " in store")} + eh.DecodeErr = &hst.AppError{ + Step: step, + Err: errors.New("unexpected directory " + + strconv.Quote(ent.Name()) + " in store"), + } goto out } @@ -186,7 +233,10 @@ func (h *Handle) Entries() (iter.Seq[*EntryHandle], int, error) { // this either indicates a serious bug or external interference if err := eh.ID.UnmarshalText([]byte(ent.Name())); err != nil { - eh.DecodeErr = &hst.AppError{Step: "decode store segment entry", Err: err} + eh.DecodeErr = &hst.AppError{ + Step: "decode store segment entry", + Err: err, + } goto out } diff --git a/internal/store/segment_test.go b/internal/store/segment_test.go index a244fcdc..6316a278 100644 --- a/internal/store/segment_test.go +++ b/internal/store/segment_test.go @@ -2,6 +2,7 @@ package store_test import ( "errors" + "fmt" "io" "iter" "os" @@ -55,7 +56,7 @@ func TestStateEntryHandle(t *testing.T) { if err := save(&eh, nil); !reflect.DeepEqual(err, wantErr()) { t.Errorf("save: error = %v, want %v", err, wantErr()) } - if _, err := eh.Load(nil); !reflect.DeepEqual(err, wantErr()) { + if _, err := eh.Load(nil, nil); !reflect.DeepEqual(err, wantErr()) { t.Errorf("load: error = %v, want %v", err, wantErr()) } }) @@ -95,8 +96,10 @@ func TestStateEntryHandle(t *testing.T) { t.Run("saveload", func(t *testing.T) { t.Parallel() - eh := store.EntryHandle{Pathname: check.MustAbs(t.TempDir()).Append("entry"), - ID: store.NewTemplateState().ID} + eh := store.EntryHandle{ + Pathname: check.MustAbs(t.TempDir()).Append("entry"), + ID: store.NewTemplateState().ID, + } if err := save(&eh, store.NewTemplateState()); err != nil { t.Fatalf("save: error = %v", err) @@ -125,7 +128,7 @@ func TestStateEntryHandle(t *testing.T) { t.Run("load header only", func(t *testing.T) { t.Parallel() - if et, err := eh.Load(nil); err != nil { + if et, err := eh.Load(nil, nil); err != nil { t.Fatalf("load: error = %v", err) } else if want := store.NewTemplateState().Enablements.Unwrap(); et != want { t.Errorf("load: et = %x, want %x", et, want) @@ -136,7 +139,7 @@ func TestStateEntryHandle(t *testing.T) { t.Parallel() var got hst.State - if _, err := eh.Load(&got); err != nil { + if _, err := eh.Load(&got, nil); err != nil { t.Fatalf("load: error = %v", err) } else if want := store.NewTemplateState(); !reflect.DeepEqual(&got, want) { t.Errorf("load: %#v, want %#v", &got, want) @@ -145,11 +148,64 @@ func TestStateEntryHandle(t *testing.T) { t.Run("load inconsistent", func(t *testing.T) { t.Parallel() - wantErr := &hst.AppError{Step: "validate state identifier", Err: os.ErrInvalid, - Msg: "state entry 00000000000000000000000000000000 has unexpected id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + wantErr := &hst.AppError{ + Step: "validate state identifier", + Err: os.ErrInvalid, + Msg: "state entry 00000000000000000000000000000000 has unexpected id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + } ehi := store.EntryHandle{Pathname: eh.Pathname} - if _, err := ehi.Load(new(hst.State)); !reflect.DeepEqual(err, wantErr) { + if _, err := ehi.Load(new(hst.State), nil); !reflect.DeepEqual(err, wantErr) { + t.Errorf("load: error = %#v, want %#v", err, wantErr) + } + }) + + t.Run("stale fault", func(t *testing.T) { + t.Parallel() + wantErr := &hst.AppError{ + Step: "check monitor process", + Err: syscall.EFAULT, + } + + if _, err := eh.Load(new(hst.State), func(pid int, sig syscall.Signal) (err error) { + if pid != store.NewTemplateState().PID { + return fmt.Errorf("bad pid %d", pid) + } + if sig != 0 { + return fmt.Errorf("bad signal %d", sig) + } + return syscall.EFAULT + }); !reflect.DeepEqual(err, wantErr) { + t.Errorf("load: error = %#v, want %#v", err, wantErr) + } + }) + + t.Run("stale", func(t *testing.T) { + t.Parallel() + wantErr := &hst.AppError{ + Step: "load stale entry", + Err: syscall.ESRCH, + Msg: "stale entry aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + } + + ehi := store.EntryHandle{ + Pathname: check.MustAbs(t.TempDir()).Append("entry"), + ID: eh.ID, + } + + if err := save(&ehi, store.NewTemplateState()); err != nil { + t.Fatalf("save: error = %v", err) + } + + if _, err := ehi.Load(new(hst.State), func(pid int, sig syscall.Signal) (err error) { + if pid != store.NewTemplateState().PID { + return fmt.Errorf("bad pid %d", pid) + } + if sig != 0 { + return fmt.Errorf("bad signal %d", sig) + } + return syscall.ESRCH + }); !reflect.DeepEqual(err, wantErr) { t.Errorf("load: error = %#v, want %#v", err, wantErr) } }) @@ -229,7 +285,9 @@ func TestSegmentHandle(t *testing.T) { } } - slices.SortFunc(got, func(a, b *store.EntryHandle) int { return strings.Compare(a.Pathname.String(), b.Pathname.String()) }) + slices.SortFunc(got, func(a, b *store.EntryHandle) int { + return strings.Compare(a.Pathname.String(), b.Pathname.String()) + }) want := tc.want(func(err error, name string) *store.EntryHandle { eh := store.EntryHandle{DecodeErr: err, Pathname: segment.Append(name)} if err == nil { @@ -247,6 +305,8 @@ func TestSegmentHandle(t *testing.T) { } t.Run("nonexistent", func(t *testing.T) { + t.Parallel() + var wantErr = &hst.AppError{Step: "read store segment entries", Err: &os.PathError{ Op: "open", Path: "/proc/nonexistent", diff --git a/internal/store/store.go b/internal/store/store.go index 3604521c..eb9cec41 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1,4 +1,5 @@ -// Package store implements cross-process state tracking for hakurei container instances. +// Package store provides storage for hakurei instance states, safe for +// concurrent and cross-process use. package store import ( @@ -15,11 +16,15 @@ import ( "hakurei.app/internal/lockedfile" ) -// MutexName is the pathname of the file backing [lockedfile.Mutex] of a [Store] and [Handle]. +// MutexName is the pathname of the file backing [lockedfile.Mutex] of a [Store] +// and [Handle]. const MutexName = "lock" -// A Store keeps track of [hst.State] via a well-known filesystem accessible to all hakurei priv-side processes. -// Access to store data and related resources are synchronised on a per-segment basis via [Handle]. +// A Store keeps track of [hst.State] via a well-known filesystem accessible to +// all hakurei priv-side processes. +// +// Access to store data and related resources are synchronised on a per-segment +// basis via [Handle]. type Store struct { // Pathname of directory that the store is rooted in. base *check.Absolute @@ -28,8 +33,10 @@ type Store struct { handles sync.Map // Inter-process mutex to synchronise operations against the entire store. - // Held during List and when initialising previously unknown identities during Do. - // Must not be accessed directly. Callers should use the bigLock method instead. + // + // Held during List and when initialising previously unknown identities + // during Do. Must not be accessed directly. Callers should use the bigLock + // method instead. fileMu *lockedfile.Mutex // For creating the base directory. @@ -43,11 +50,17 @@ type Store struct { func (s *Store) bigLock() (unlock func(), err error) { s.mkdirOnce.Do(func() { s.mkdirErr = os.MkdirAll(s.base.String(), 0700) }) if s.mkdirErr != nil { - return nil, &hst.AppError{Step: "create state store directory", Err: s.mkdirErr} + return nil, &hst.AppError{ + Step: "create state store directory", + Err: s.mkdirErr, + } } if unlock, err = s.fileMu.Lock(); err != nil { - return nil, &hst.AppError{Step: "acquire lock on the state store", Err: err} + return nil, &hst.AppError{ + Step: "acquire lock on the state store", + Err: err, + } } return } @@ -73,7 +86,10 @@ func (s *Store) Handle(identity int) (*Handle, error) { if err != nil && !errors.Is(err, fs.ErrExist) { // handle methods will likely return ENOENT s.handles.CompareAndDelete(identity, h) - return nil, &hst.AppError{Step: "create store segment directory", Err: err} + return nil, &hst.AppError{ + Step: "create store segment directory", + Err: err, + } } } return h, nil @@ -88,8 +104,9 @@ type SegmentIdentity struct { } // Segments returns an iterator over all [SegmentIdentity] known to the [Store]. -// To obtain a [Handle] on a segment, caller must then call [Store.Handle]. -// A non-nil error returned by segments is of type [hst.AppError]. +// +// To obtain a [Handle] on a segment, caller must then call [Store.Handle]. A +// non-nil error returned by segments is of type [hst.AppError]. func (s *Store) Segments() (iter.Seq[SegmentIdentity], int, error) { // read directory contents, should only contain storeMutexName and identity var entries []os.DirEntry @@ -102,7 +119,10 @@ func (s *Store) Segments() (iter.Seq[SegmentIdentity], int, error) { unlock() if err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, -1, &hst.AppError{Step: "read store segments", Err: err} + return nil, -1, &hst.AppError{ + Step: "read store segments", + Err: err, + } } } @@ -126,19 +146,25 @@ func (s *Store) Segments() (iter.Seq[SegmentIdentity], int, error) { } // this should never happen - si.Err = &hst.AppError{Step: step, Err: syscall.ENOTDIR, - Msg: "skipped non-directory entry " + strconv.Quote(ent.Name())} + si.Err = &hst.AppError{ + Step: step, Err: syscall.ENOTDIR, + Msg: "skipped non-directory entry " + strconv.Quote(ent.Name()), + } goto out } // failure paths either indicates a serious bug or external interference if v, err := strconv.Atoi(ent.Name()); err != nil { - si.Err = &hst.AppError{Step: step, Err: err, - Msg: "skipped non-identity entry " + strconv.Quote(ent.Name())} + si.Err = &hst.AppError{ + Step: step, Err: err, + Msg: "skipped non-identity entry " + strconv.Quote(ent.Name()), + } goto out } else if v < hst.IdentityStart || v > hst.IdentityEnd { - si.Err = &hst.AppError{Step: step, Err: syscall.ERANGE, - Msg: "skipped out of bounds entry " + strconv.Itoa(v)} + si.Err = &hst.AppError{ + Step: step, Err: syscall.ERANGE, + Msg: "skipped out of bounds entry " + strconv.Itoa(v), + } goto out } else { si.Identity = v @@ -152,9 +178,12 @@ func (s *Store) Segments() (iter.Seq[SegmentIdentity], int, error) { }, l, nil } -// All returns a non-reusable iterator over all [EntryHandle] known to this [Store]. -// Callers must call copyError after completing iteration and handle the error accordingly. -// A non-nil error returned by copyError is of type [hst.AppError]. +// All returns a non-reusable iterator over all [EntryHandle] known to this +// [Store]. The resulting handles may be retained, but are only safe to use +// during the iteration producing them. +// +// Callers must call copyError after completing iteration and handle the error +// accordingly. A non-nil error returned by copyError is of type [hst.AppError]. func (s *Store) All() (entries iter.Seq[*EntryHandle], copyError func() error) { var savedErr error return func(yield func(*EntryHandle) bool) { @@ -197,7 +226,12 @@ func (s *Store) All() (entries iter.Seq[*EntryHandle], copyError func() error) { } // New returns the address of a new instance of [Store]. -// Multiple instances of [Store] rooted in the same directory is possible, but unsupported. +// +// Multiple instances of [Store] rooted in the same directory is possible, but +// unsupported. func New(base *check.Absolute) *Store { - return &Store{base: base, fileMu: lockedfile.MutexAt(base.Append(MutexName).String())} + return &Store{ + base: base, + fileMu: lockedfile.MutexAt(base.Append(MutexName).String()), + } } diff --git a/internal/uevent/coldboot_test.go b/internal/uevent/coldboot_test.go index 0a50ead4..aa6428a1 100644 --- a/internal/uevent/coldboot_test.go +++ b/internal/uevent/coldboot_test.go @@ -11,8 +11,8 @@ import ( "testing" "hakurei.app/check" - "hakurei.app/internal/pkg" "hakurei.app/internal/uevent" + "hakurei.app/pkg" ) func TestColdboot(t *testing.T) { diff --git a/internal/workflows/doc.go b/internal/workflows/doc.go new file mode 100644 index 00000000..0c7e94b9 --- /dev/null +++ b/internal/workflows/doc.go @@ -0,0 +1,133 @@ +//go:build !workflows + +/* +The workflows program manages the .gitea directory to avoid having to write +yaml. It is never invoked directly or built for the distribution tarball. + +This program is an internal detail of the hakurei project and is not usable +on its own. It is not covered by the compatibility promise. + +# Configuring act_runner + +It is unreasonable and impractical to bootstrap Rosa OS each time a job runs. +This section documents an ideal configuration for exposing the cmd/mbf CI +service to all job containers, while avoiding some issues with the container +setup documented upstream. + +Microsoft Github workflows expect a docker socket in the job container. This is +problematic because docker has a hard time running inside a docker container. +The Gitea act_runner simply bind mounts whatever socket it sees into the +container. With a regular docker daemon, this allows not only a simple container +escape, but also privilege escalation as unconstrained root in the init +namespace. To mitigate this, set up an unprivileged podman daemon and expose its +socket to the container instead. + +On Alpine Linux, this is achieved by: + + # Configuration for /etc/init.d/podman + + # See podman-system-service(1) for service description + # and available options. + #podman_opts="--time 0" + + # API endpoint in URI form. Leave empty to use defaults. + podman_uri="unix:///srv/act_runner/podman" + + # Setting root user will start rootful service. + # Use any other user for rootless mode. + podman_user="act_runner" + +A symlink must then be created at /var/run/docker.sock pointing to the socket +owned by the unprivileged daemon. This should be done in a /etc/local.d script. + +The container can be specified via docker-compose: + + version: "3.8" + services: + runner: + restart: always + image: docker.io/gitea/act_runner:nightly + environment: + CONFIG_FILE: /config.yaml + GITEA_INSTANCE_URL: "https://git.gensokyo.uk" + GITEA_RUNNER_LABELS: "rosa:docker://docker.gitea.com/runner-images:ubuntu-latest" + + GITEA_RUNNER_REGISTRATION_TOKEN: "${REGISTRATION_TOKEN}" + GITEA_RUNNER_NAME: "${RUNNER_NAME}" + volumes: + - ./config.yaml:/config.yaml + - ./data:/data + - ./podman:/var/run/docker.sock + +Before starting the container, configure act_runner via config.yaml: + + container: + network: host + options: -e PAGER=cat -e MBF_CACHE_DIR=/rosa -e MBF_POISON_OPEN=1 + -v /var/lib/rosa:/rosa + --device=/dev/kvm + valid_volumes: + - /var/lib/rosa + +where /var/lib/rosa is the absolute pathname of the cache directory in the init +namespace. Setting MBF_POISON_OPEN enables cmd/mbf to run as root. It is also +a good idea here to set runner.capacity to reflect the capacity of the guest, so +jobs can be consumed quicker. + +Build a statically-linked cmd/mbf: + + go build -trimpath \ + -ldflags='-s -w + -buildid= + -linkmode external + -extldflags=-static' \ + ./cmd/mbf + +Create a directory named "bin" in the cache directory and install the resulting +binary. + +The cmd/mbf CI service should generally run as a separate user. To achieve this, +change the socket permission to 0777 after the service starts. On Alpine Linux, +this can be achieved by the init script: + + #!/sbin/openrc-run + supervisor=supervise-daemon + + name="Rosa OS cmd/mbf CI backend" + description="Listening service that services CI workloads from act_runner" + + command=/var/lib/rosa/bin/mbf + command_args="-d /var/lib/rosa ci daemon" + command_user="${mbf_user:=mbf}" + + output_logger="logger" + error_logger="logger" + + depend() { + need sysfs + } + + start_post() { + while [ ! -e /var/lib/rosa/ci ]; do sleep 0.1; done + chmod 0777 /var/lib/rosa/ci + } + +It is often a good idea to populate the cache from a mirror service before the +first workflow job is started and re-populate it after every cmd/mbf update. + +# Security + +The design of Microsoft Github workflows is inherently insecure: it requires +internet access just to fetch actions, and these actions all expect internet +access as well. Gitea, being a drop-in replacement, retains the same problems. + +The setup documented above already avoids the obvious container escape via the +bind-mounted docker socket, but it is still easily possible to escape as the +context of the unconstrained runner user. For this reason, the runner must +reside in a hardened guest. Since internet access cannot be disabled, extra care +must be taken when setting up the host firewall to prevent access to internal +systems from the untrusted guest. +*/ +package workflows + +//go:generate go run -tags=workflows . diff --git a/internal/workflows/main.go b/internal/workflows/main.go new file mode 100644 index 00000000..dca480a5 --- /dev/null +++ b/internal/workflows/main.go @@ -0,0 +1,149 @@ +//go:build workflows + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" +) + +// KV represents a string to value pair. +type KV[T any] struct { + K string + V T +} + +// A Map represents an ordered set of key-value pairs. +type Map[T any] []KV[T] + +// MarshalJSON encodes m as a key-value pair in JSON. +func (m Map[T]) MarshalJSON() ([]byte, error) { + if m == nil || len(m) == 0 { + return []byte("{}"), nil + } + + buf := bytes.NewBuffer(make([]byte, 0, len(m)<<5)) + buf.WriteByte('{') + + last := len(m) - 1 + for i, pair := range m { + p, err := json.Marshal(pair.K) + if err != nil { + return nil, fmt.Errorf("key %q: %w", pair.K, err) + } + buf.Write(p) + buf.WriteByte(':') + + p, err = json.Marshal(pair.V) + if err != nil { + return nil, fmt.Errorf("value of key %q: %w", pair.K, err) + } + buf.Write(p) + + if i < last { + buf.WriteByte(',') + } + } + + buf.WriteByte('}') + return buf.Bytes(), nil +} + +// A Step is a task ran as part of a [Job]. +type Step struct { + // A name for the step. + Name string `json:"name,omitempty"` + // Selects an action to run as part of a step in the job. + Uses string `json:"uses,omitempty"` + // A unique identifier for the step. + ID string `json:"id,omitempty"` + // Prevent a step from running unless a condition is met. + If string `json:"if,omitempty"` + // Runs command-line programs that do not exceed 21,000 characters using the + // operating system's shell. + Run string `json:"run,omitempty"` + // Key-value pairs of input parameters defined by the action. + With Map[any] `json:"with,omitempty"` +} + +// A Job is an entry in a [Workflow] document. +type Job struct { + // Set a name for the job. + Name string `json:"name,omitempty"` + // Identifies any jobs that must complete successfully before this job will + // run. + Needs []string `json:"needs,omitempty"` + // Define the type of machine to run the job on. + On string `json:"runs-on,omitempty"` + // A sequence of tasks. + Steps []Step `json:"steps,omitempty"` +} + +// Workflow describes a workflow document. +type Workflow struct { + // The name of the workflow. + Name string `json:"name,omitempty"` + // The name for workflow runs generated from the workflow. + RunName string `json:"run-name,omitempty"` + // Define which events can cause the workflow to run. + On any `json:"on"` + + // Named jobs part of the workflow run. + Jobs Map[Job] `json:"jobs,omitempty"` +} + +// workflowsDir is the relative pathname of the workflows directory. +const workflowsDir = "../../.gitea/workflows" + +// writeFile writes the action to the .gitea directory. +func (w *Workflow) writeFile(name string) error { + f, err := os.OpenFile( + filepath.Join(workflowsDir, name+".yml"), + os.O_CREATE|os.O_EXCL|os.O_WRONLY, + 0444, + ) + if err != nil { + return err + } + + err = json.NewEncoder(f).Encode(w) + if closeErr := f.Close(); err == nil { + err = closeErr + } + return err +} + +// removeOnce synchronises removal of stale workflow files. +var removeOnce sync.Once + +// removeWorkflows removes existing workflow files once. +func removeWorkflows() { + removeOnce.Do(func() { + if err := os.RemoveAll(workflowsDir); err != nil { + fmt.Println(err) + os.Exit(1) + } + if err := os.Mkdir(workflowsDir, 0755); err != nil { + fmt.Println(err) + os.Exit(1) + } + }) +} + +// mustWriteFile is like writeFile, but terminates the program if a non-nil +// error is returned. +func (w *Workflow) mustWriteFile(name string) (_ struct{}) { + removeWorkflows() + + if err := w.writeFile(name); err != nil { + fmt.Println(err) + os.Exit(1) + } + return +} + +func main() {} diff --git a/internal/workflows/release.go b/internal/workflows/release.go new file mode 100644 index 00000000..f220c0f7 --- /dev/null +++ b/internal/workflows/release.go @@ -0,0 +1,52 @@ +//go:build workflows + +package main + +var _ = (&Workflow{ + Name: "Release", + On: Map[any]{{"push", Map[[]string]{{"tags", []string{"v*"}}}}}, + + Jobs: []KV[Job]{ + {"release", Job{ + Name: "Create release", + On: "rosa", + + Steps: []Step{ + fixup, + + { + Name: "Checkout", + Uses: "actions/checkout@v4", + + With: []KV[any]{ + {"fetch-depth", 0}, + {"fetch-tags", true}, + }, + }, + + { + Name: "Request distribution", + ID: "dist", + Run: "/rosa/bin/mbf ci dist -o result " + + ". \"$(cat cmd/dist/VERSION)-act\" && " + + "git tag " + + "--list " + + "--format='%(contents:body)' " + + "\"$(cat cmd/dist/VERSION)\" >> " + + "\"/tmp/release-body\"", + }, + + { + Name: "Create release", + Uses: "https://gitea.com/actions/gitea-release-action@v1.3.7", + + With: []KV[any]{ + {"files", "result/hakurei-**"}, + {"body_path", "/tmp/release-body"}, + {"draft", true}, + }, + }, + }, + }}, + }, +}).mustWriteFile("release") diff --git a/internal/workflows/step.go b/internal/workflows/step.go new file mode 100644 index 00000000..423a48ff --- /dev/null +++ b/internal/workflows/step.go @@ -0,0 +1,53 @@ +//go:build workflows + +package main + +// fixup is a step to work around a Podman regression introduced by a bad +// container escape mitigation. +var fixup = Step{ + Name: "Fix container filesystem", + Run: "rm /var/run && ln -sf ../run /var", +} + +// checkout is the standard actions/checkout step. +var checkout = Step{Name: "Checkout", Uses: "actions/checkout@v4"} + +// newUploadArtifact returns an actions/upload-artifact step uploading +// everything in the result directory as the specified name. +func newUploadArtifact(display, name string) Step { + return Step{ + Name: "Upload " + display, + Uses: "actions/upload-artifact@v3", + + With: Map[any]{ + {"name", name}, + {"path", "result/*"}, + + {"retention-days", 1}, + }, + } +} + +// newCIRequest returns a step for requesting a mbf CI task. +func newCIRequest(display, name, id string) Step { + return Step{ + Name: "Request " + display, + ID: id, + Run: "export HAKUREI_REV=\"$(git rev-parse --short HEAD)\" && " + + "/rosa/bin/mbf ci " + name + " " + + ". \"$(cat cmd/dist/VERSION)-$HAKUREI_REV\" && " + + "echo \"rev=$HAKUREI_REV\" >> \"$GITHUB_OUTPUT\"", + } +} + +// newNixOSTest returns a step for running the named NixOS test. +func newNixOSTest(name string) Step { + return Step{ + Name: "Run NixOS test", + Run: "nix build " + + "--out-link result " + + "--print-out-paths " + + "--print-build-logs " + + "./test#checks.x86_64-linux." + name, + } +} diff --git a/internal/workflows/test.go b/internal/workflows/test.go new file mode 100644 index 00000000..88e91a22 --- /dev/null +++ b/internal/workflows/test.go @@ -0,0 +1,105 @@ +//go:build workflows + +package main + +var _ = (&Workflow{ + Name: "Test", + On: []any{"push"}, + + Jobs: Map[Job]{ + {"hakurei", Job{ + Name: "Hakurei", + On: "nix", + + Steps: []Step{ + checkout, + newNixOSTest("hakurei"), + newUploadArtifact("test output", "hakurei-vm-output"), + }, + }}, + + {"race", Job{ + Name: "Hakurei (race detector)", + On: "nix", + + Steps: []Step{ + checkout, + newNixOSTest("race"), + newUploadArtifact("test output", "hakurei-race-vm-output"), + }, + }}, + + {"sandbox", Job{ + Name: "Sandbox", + On: "nix", + + Steps: []Step{ + checkout, + newNixOSTest("sandbox"), + newUploadArtifact("test output", "sandbox-vm-output"), + }, + }}, + + {"sandbox-race", Job{ + Name: "Sandbox (race detector)", + On: "nix", + + Steps: []Step{ + checkout, + newNixOSTest("sandbox-race"), + newUploadArtifact("test output", "sandbox-race-vm-output"), + }, + }}, + + {"sharefs", Job{ + Name: "ShareFS", + On: "nix", + + Steps: []Step{ + checkout, + newNixOSTest("sharefs"), + newUploadArtifact("test output", "sharefs-vm-output"), + }, + }}, + + {"check", Job{ + Name: "Flake checks", + On: "nix", + + Needs: []string{ + "hakurei", + "race", + "sandbox", + "sandbox-race", + "sharefs", + }, + + Steps: []Step{ + checkout, + + { + Name: "Run checks", + Run: "nix " + + "--print-build-logs " + + "--experimental-features 'nix-command flakes' " + + "flake check ./test", + }, + }, + }}, + + {"dist", Job{ + Name: "Create distribution", + On: "rosa", + + Steps: []Step{ + fixup, + checkout, + newCIRequest("distribution", "dist -o result", "dist-test"), + newUploadArtifact( + "distribution", + "hakurei-${{ steps.dist-test.outputs.rev }}", + ), + }, + }}, + }, +}).mustWriteFile("test") |