aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
Diffstat
-rw-r--r--internal/pkg/exec.go46
-rw-r--r--internal/pkg/exec_test.go20
-rw-r--r--internal/pkg/file.go25
-rw-r--r--internal/pkg/net.go6
-rw-r--r--internal/pkg/net_test.go18
-rw-r--r--internal/pkg/pkg.go236
-rw-r--r--internal/pkg/pkg_test.go93
-rw-r--r--internal/pkg/tar.go6
-rw-r--r--internal/pkg/tar_test.go25
9 files changed, 276 insertions, 199 deletions
diff --git a/internal/pkg/exec.go b/internal/pkg/exec.go
index 8d27d07d..f8131855 100644
--- a/internal/pkg/exec.go
+++ b/internal/pkg/exec.go
@@ -2,7 +2,6 @@ package pkg
import (
"bufio"
- "bytes"
"context"
"errors"
"fmt"
@@ -101,8 +100,9 @@ func (a *execNetArtifact) Checksum() Checksum { return a.checksum }
func (a *execNetArtifact) Kind() Kind { return KindExecNet }
// Params is [Checksum] concatenated with [KindExec] params.
-func (a *execNetArtifact) Params() []byte {
- return slices.Concat(a.checksum[:], a.execArtifact.Params())
+func (a *execNetArtifact) Params(ctx *IContext) {
+ ctx.GetHash().Write(a.checksum[:])
+ a.execArtifact.Params(ctx)
}
// Cure cures the [Artifact] in the container described by the caller. The
@@ -165,40 +165,40 @@ func NewExec(
// Kind returns the hardcoded [Kind] constant.
func (a *execArtifact) Kind() Kind { return KindExec }
-// Params returns paths, executable pathname and args concatenated together.
-func (a *execArtifact) Params() []byte {
- var buf bytes.Buffer
+// Params writes paths, executable pathname and args.
+func (a *execArtifact) Params(ctx *IContext) {
+ h := ctx.GetHash()
+
+ _0, _1 := []byte{0}, []byte{1}
for _, p := range a.paths {
if p.W {
- buf.WriteByte(1)
+ h.Write(_1)
} else {
- buf.WriteByte(0)
+ h.Write(_0)
}
if p.P != nil {
- buf.WriteString(p.P.String())
+ h.Write([]byte(p.P.String()))
} else {
- buf.WriteString("invalid P\x00")
+ h.Write([]byte("invalid P\x00"))
}
- buf.WriteByte(0)
+ h.Write(_0)
for _, d := range p.A {
- id := Ident(d)
- buf.Write(id[:])
+ ctx.WriteIdent(d)
}
- buf.WriteByte(0)
+ h.Write(_0)
}
- buf.WriteByte(0)
- buf.WriteString(a.dir.String())
- buf.WriteByte(0)
+ h.Write(_0)
+ h.Write([]byte(a.dir.String()))
+ h.Write(_0)
for _, e := range a.env {
- buf.WriteString(e)
+ h.Write([]byte(e))
}
- buf.WriteByte(0)
- buf.WriteString(a.path.String())
- buf.WriteByte(0)
+ h.Write(_0)
+ h.Write([]byte(a.path.String()))
+ h.Write(_0)
for _, arg := range a.args {
- buf.WriteString(arg)
+ h.Write([]byte(arg))
}
- return buf.Bytes()
}
// Dependencies returns a slice of all artifacts collected from caller-supplied
diff --git a/internal/pkg/exec_test.go b/internal/pkg/exec_test.go
index 97f72a38..558ab39b 100644
--- a/internal/pkg/exec_test.go
+++ b/internal/pkg/exec_test.go
@@ -50,7 +50,7 @@ func TestExec(t *testing.T) {
nil,
nil, nil,
)),
- pkg.MustPath("/.hakurei", false, stubArtifact{
+ pkg.MustPath("/.hakurei", false, &stubArtifact{
kind: pkg.KindTar,
params: []byte("empty directory"),
cure: func(t *pkg.TContext) error {
@@ -67,7 +67,7 @@ func TestExec(t *testing.T) {
check.MustAbs("/opt/bin/testtool"),
[]string{"testtool"},
- pkg.MustPath("/proc/nonexistent", false, stubArtifact{
+ pkg.MustPath("/proc/nonexistent", false, &stubArtifact{
kind: pkg.KindTar,
params: []byte("doomed artifact"),
cure: func(t *pkg.TContext) error {
@@ -124,7 +124,7 @@ func TestExec(t *testing.T) {
nil,
nil, nil,
)),
- pkg.MustPath("/.hakurei", false, stubArtifact{
+ pkg.MustPath("/.hakurei", false, &stubArtifact{
kind: pkg.KindTar,
params: []byte("empty directory"),
cure: func(t *pkg.TContext) error {
@@ -150,7 +150,7 @@ func TestExec(t *testing.T) {
check.MustAbs("/opt/bin/testtool"),
[]string{"testtool"},
- pkg.MustPath("/", true, stubArtifact{
+ pkg.MustPath("/", true, &stubArtifact{
kind: pkg.KindTar,
params: []byte("empty directory"),
cure: func(t *pkg.TContext) error {
@@ -176,13 +176,13 @@ func TestExec(t *testing.T) {
check.MustAbs("/work/bin/testtool"),
[]string{"testtool"},
- pkg.MustPath("/", true, stubArtifact{
+ pkg.MustPath("/", true, &stubArtifact{
kind: pkg.KindTar,
params: []byte("empty directory"),
cure: func(t *pkg.TContext) error {
return os.MkdirAll(t.GetWorkDir().String(), 0700)
},
- }), pkg.MustPath("/work/", false, stubArtifact{
+ }), pkg.MustPath("/work/", false, &stubArtifact{
kind: pkg.KindTar,
params: []byte("empty directory"),
cure: func(t *pkg.TContext) error {
@@ -207,13 +207,13 @@ func TestExec(t *testing.T) {
check.MustAbs("/opt/bin/testtool"),
[]string{"testtool", "layers"},
- pkg.MustPath("/", true, stubArtifact{
+ pkg.MustPath("/", true, &stubArtifact{
kind: pkg.KindTar,
params: []byte("empty directory"),
cure: func(t *pkg.TContext) error {
return os.MkdirAll(t.GetWorkDir().String(), 0700)
},
- }, stubArtifactF{
+ }, &stubArtifactF{
kind: pkg.KindExec,
params: []byte("test sample with dependencies"),
@@ -222,7 +222,7 @@ func TestExec(t *testing.T) {
pkg.ID{0xfe, 0},
nil,
nil, nil,
- ), stubArtifact{
+ ), &stubArtifact{
kind: pkg.KindTar,
params: []byte("empty directory"),
@@ -255,7 +255,7 @@ func newTesttool() (
testtoolDestroy func(t *testing.T, base *check.Absolute, c *pkg.Cache),
) {
// testtoolBin is built during go:generate and is not deterministic
- testtool = overrideIdent{pkg.ID{0xfe, 0xff}, stubArtifact{
+ testtool = overrideIdent{pkg.ID{0xfe, 0xff}, &stubArtifact{
kind: pkg.KindTar,
cure: func(t *pkg.TContext) error {
work := t.GetWorkDir()
diff --git a/internal/pkg/file.go b/internal/pkg/file.go
index 98b8eee6..4558eb40 100644
--- a/internal/pkg/file.go
+++ b/internal/pkg/file.go
@@ -9,7 +9,7 @@ import (
// A fileArtifact is an [Artifact] that cures into data known ahead of time.
type fileArtifact []byte
-var _ KnownChecksum = fileArtifact{}
+var _ KnownChecksum = new(fileArtifact)
// fileArtifactNamed embeds fileArtifact alongside a caller-supplied name.
type fileArtifactNamed struct {
@@ -18,10 +18,11 @@ type fileArtifactNamed struct {
name string
}
-var _ fmt.Stringer = fileArtifactNamed{}
+var _ fmt.Stringer = new(fileArtifactNamed)
+var _ KnownChecksum = new(fileArtifactNamed)
// String returns the caller-supplied reporting name.
-func (a fileArtifactNamed) String() string { return a.name }
+func (a *fileArtifactNamed) String() string { return a.name }
// NewFile returns a [File] that cures into a caller-supplied byte slice.
//
@@ -29,26 +30,26 @@ func (a fileArtifactNamed) String() string { return a.name }
func NewFile(name string, data []byte) File {
f := fileArtifact(data)
if name != "" {
- return fileArtifactNamed{f, name}
+ return &fileArtifactNamed{f, name}
}
- return f
+ return &f
}
// Kind returns the hardcoded [Kind] constant.
-func (a fileArtifact) Kind() Kind { return KindFile }
+func (a *fileArtifact) Kind() Kind { return KindFile }
-// Params returns the result of Data.
-func (a fileArtifact) Params() []byte { return a }
+// Params writes the result of Cure.
+func (a *fileArtifact) Params(ctx *IContext) { ctx.GetHash().Write(*a) }
// Dependencies returns a nil slice.
-func (a fileArtifact) Dependencies() []Artifact { return nil }
+func (a *fileArtifact) Dependencies() []Artifact { return nil }
// Checksum computes and returns the checksum of caller-supplied data.
-func (a fileArtifact) Checksum() Checksum {
+func (a *fileArtifact) Checksum() Checksum {
h := sha512.New384()
- h.Write(a)
+ h.Write(*a)
return Checksum(h.Sum(nil))
}
// Cure returns the caller-supplied data.
-func (a fileArtifact) Cure(context.Context) ([]byte, error) { return a, nil }
+func (a *fileArtifact) Cure(context.Context) ([]byte, error) { return *a, nil }
diff --git a/internal/pkg/net.go b/internal/pkg/net.go
index b8d85b91..c481b52e 100644
--- a/internal/pkg/net.go
+++ b/internal/pkg/net.go
@@ -50,9 +50,11 @@ func NewHTTPGet(
// Kind returns the hardcoded [Kind] constant.
func (a *httpArtifact) Kind() Kind { return KindHTTPGet }
-// Params returns the backing url string. Context is not represented as it does
+// Params writes the backing url string. Client is not represented as it does
// not affect [Cache.Cure] outcome.
-func (a *httpArtifact) Params() []byte { return []byte(a.url) }
+func (a *httpArtifact) Params(ctx *IContext) {
+ ctx.GetHash().Write([]byte(a.url))
+}
// Dependencies returns a nil slice.
func (a *httpArtifact) Dependencies() []Artifact { return nil }
diff --git a/internal/pkg/net_test.go b/internal/pkg/net_test.go
index 72460ee4..b7f70569 100644
--- a/internal/pkg/net_test.go
+++ b/internal/pkg/net_test.go
@@ -35,13 +35,10 @@ func TestHTTPGet(t *testing.T) {
"file:///testdata",
testdataChecksum,
)
- wantIdent := pkg.KindHTTPGet.Ident([]byte("file:///testdata"))
if got, err := f.Cure(t.Context()); err != nil {
t.Fatalf("Cure: error = %v", err)
} else if string(got) != testdata {
t.Fatalf("Cure: %x, want %x", got, testdata)
- } else if gotIdent := pkg.Ident(f); gotIdent != wantIdent {
- t.Fatalf("Ident: %s, want %s", pkg.Encode(gotIdent), pkg.Encode(wantIdent))
}
// check direct validation
@@ -55,8 +52,6 @@ func TestHTTPGet(t *testing.T) {
}
if _, err := f.Cure(t.Context()); !reflect.DeepEqual(err, wantErrMismatch) {
t.Fatalf("Cure: error = %#v, want %#v", err, wantErrMismatch)
- } else if gotIdent := pkg.Ident(f); gotIdent != wantIdent {
- t.Fatalf("Ident: %s, want %s", pkg.Encode(gotIdent), pkg.Encode(wantIdent))
}
// check direct response error
@@ -65,12 +60,9 @@ func TestHTTPGet(t *testing.T) {
"file:///nonexistent",
pkg.Checksum{},
)
- wantIdentNonexistent := pkg.KindHTTPGet.Ident([]byte("file:///nonexistent"))
wantErrNotFound := pkg.ResponseStatusError(http.StatusNotFound)
if _, err := f.Cure(t.Context()); !reflect.DeepEqual(err, wantErrNotFound) {
t.Fatalf("Cure: error = %#v, want %#v", err, wantErrNotFound)
- } else if gotIdent := pkg.Ident(f); gotIdent != wantIdentNonexistent {
- t.Fatalf("Ident: %s, want %s", pkg.Encode(gotIdent), pkg.Encode(wantIdentNonexistent))
}
}, pkg.MustDecode("E4vEZKhCcL2gPZ2Tt59FS3lDng-d_2SKa2i5G_RbDfwGn6EemptFaGLPUDiOa94C")},
@@ -80,10 +72,9 @@ func TestHTTPGet(t *testing.T) {
"file:///testdata",
testdataChecksum,
)
- wantIdent := pkg.KindHTTPGet.Ident([]byte("file:///testdata"))
wantPathname := base.Append(
"identifier",
- pkg.Encode(wantIdent),
+ "NqVORkT6L9HX6Za7kT2zcibY10qFqBaxEjPiYFrBQX-ZFr3yxCzJxbKOP0zVjeWb",
)
if pathname, checksum, err := c.Cure(f); err != nil {
t.Fatalf("Cure: error = %v", err)
@@ -97,8 +88,6 @@ func TestHTTPGet(t *testing.T) {
t.Fatalf("Cure: error = %v", err)
} else if string(got) != testdata {
t.Fatalf("Cure: %x, want %x", got, testdata)
- } else if gotIdent := pkg.Ident(f); gotIdent != wantIdent {
- t.Fatalf("Ident: %s, want %s", pkg.Encode(gotIdent), pkg.Encode(wantIdent))
}
// check load from cache
@@ -111,8 +100,6 @@ func TestHTTPGet(t *testing.T) {
t.Fatalf("Cure: error = %v", err)
} else if string(got) != testdata {
t.Fatalf("Cure: %x, want %x", got, testdata)
- } else if gotIdent := pkg.Ident(f); gotIdent != wantIdent {
- t.Fatalf("Ident: %s, want %s", pkg.Encode(gotIdent), pkg.Encode(wantIdent))
}
// check error passthrough
@@ -121,12 +108,9 @@ func TestHTTPGet(t *testing.T) {
"file:///nonexistent",
pkg.Checksum{},
)
- wantIdentNonexistent := pkg.KindHTTPGet.Ident([]byte("file:///nonexistent"))
wantErrNotFound := pkg.ResponseStatusError(http.StatusNotFound)
if _, _, err := c.Cure(f); !reflect.DeepEqual(err, wantErrNotFound) {
t.Fatalf("Pathname: error = %#v, want %#v", err, wantErrNotFound)
- } else if gotIdent := pkg.Ident(f); gotIdent != wantIdentNonexistent {
- t.Fatalf("Ident: %s, want %s", pkg.Encode(gotIdent), pkg.Encode(wantIdentNonexistent))
}
}, pkg.MustDecode("bqtn69RkV5E7V7GhhgCFjcvbxmaqrO8DywamM4Tyjf10F6EJBHjXiIa_tFRtF4iN")},
})
diff --git a/internal/pkg/pkg.go b/internal/pkg/pkg.go
index a9e9d18a..d156e757 100644
--- a/internal/pkg/pkg.go
+++ b/internal/pkg/pkg.go
@@ -9,6 +9,7 @@ import (
"encoding/binary"
"errors"
"fmt"
+ "hash"
"io"
"io/fs"
"iter"
@@ -20,6 +21,7 @@ import (
"strings"
"sync"
"syscall"
+ "unique"
"unsafe"
"hakurei.app/container/check"
@@ -60,6 +62,35 @@ func MustDecode(s string) Checksum {
}
}
+// IContext is passed to [Artifact.Params] and provides identifier information
+// and the target [hash.Hash] for writing params into.
+//
+// Methods of IContext are safe for concurrent use. IContext is valid
+// until [Artifact.Params] returns.
+type IContext struct {
+ // Address of underlying [Cache], should be zeroed or made unusable after
+ // [Artifact.Params] returns and must not be exposed directly.
+ cache *Cache
+ // Made available for writing, should be zeroed after [Artifact.Params]
+ // returns. Internal state must not be inspected.
+ h hash.Hash
+}
+
+// Unwrap returns the underlying [context.Context].
+func (i *IContext) Unwrap() context.Context { return i.cache.ctx }
+
+// GetHash returns the underlying [hash.Hash] for writing. Callers must not
+// attempt to inspect its internal state.
+func (i *IContext) GetHash() hash.Hash { return i.h }
+
+// WriteIdent writes the identifier of [Artifact] to the underlying [hash.Hash].
+func (i *IContext) WriteIdent(a Artifact) {
+ buf := i.cache.getIdentBuf()
+ *(*ID)(buf[wordSize:]) = i.cache.Ident(a).Value()
+ i.h.Write(buf[wordSize:])
+ i.cache.putIdentBuf(buf)
+}
+
// TContext is passed to [TrivialArtifact.Cure] and provides information and
// methods required for curing the [TrivialArtifact].
//
@@ -154,7 +185,7 @@ type FContext struct {
TContext
// Cured top-level dependencies looked up by Pathname.
- deps map[ID]*check.Absolute
+ deps map[Artifact]*check.Absolute
}
// InvalidLookupError is the identifier of non-dependency [Artifact] looked up
@@ -171,11 +202,10 @@ var _ error = InvalidLookupError{}
// with an [Artifact] not part of the slice returned by [Artifact.Dependencies]
// panics.
func (f *FContext) Pathname(a Artifact) *check.Absolute {
- id := Ident(a)
- if p, ok := f.deps[id]; ok {
+ if p, ok := f.deps[a]; ok {
return p
} else {
- panic(InvalidLookupError(id))
+ panic(InvalidLookupError(f.cache.Ident(a).Value()))
}
}
@@ -188,14 +218,13 @@ type Artifact interface {
// [Artifact] is allowed to return the same [Kind] value.
Kind() Kind
- // Params returns opaque bytes that describes [Artifact]. Implementations
+ // Params writes opaque bytes that describes [Artifact]. Implementations
// must guarantee that these values are unique among differing instances
- // of the same implementation with the same dependencies.
- //
- // Callers must not modify the retuned byte slice.
+ // of the same implementation with the same dependencies. Callers must not
+ // attempt to interpret these params.
//
// Result must remain identical across multiple invocations.
- Params() []byte
+ Params(ctx *IContext)
// Dependencies returns a slice of [Artifact] that the current instance
// depends on to produce its contents.
@@ -290,17 +319,9 @@ type File interface {
Artifact
}
-// Ident returns the identifier of an [Artifact].
-func Ident(a Artifact) ID {
- if ki, ok := a.(KnownIdent); ok {
- return ki.ID()
- }
- return a.Kind().Ident(a.Params(), a.Dependencies()...)
-}
-
-// reportNameIdent is like reportName but does not recompute [ID].
-func reportNameIdent(a Artifact, id ID) string {
- r := Encode(id)
+// reportName returns a string describing [Artifact] presented to the user.
+func reportName(a Artifact, id unique.Handle[ID]) string {
+ r := Encode(id.Value())
if s, ok := a.(fmt.Stringer); ok {
if name := s.String(); name != "" {
r += "-" + name
@@ -309,9 +330,6 @@ func reportNameIdent(a Artifact, id ID) string {
return r
}
-// reportName returns a string describing [Artifact] presented to the user.
-func reportName(a Artifact) string { return reportNameIdent(a, Ident(a)) }
-
// Kind corresponds to the concrete type of [Artifact] and is used to create
// identifier for an [Artifact] with dependencies.
type Kind uint64
@@ -334,31 +352,6 @@ const (
KindCustomOffset = 1 << 31
)
-// Ident returns a deterministic identifier for the supplied params and
-// dependencies. The caller is responsible for ensuring params uniquely and
-// deterministically describes the current [Artifact].
-func (k Kind) Ident(params []byte, deps ...Artifact) ID {
- type extIdent [len(ID{}) + wordSize]byte
- identifiers := make([]extIdent, len(deps))
- for i, a := range deps {
- id := Ident(a)
- copy(identifiers[i][wordSize:], id[:])
- binary.LittleEndian.PutUint64(identifiers[i][:], uint64(a.Kind()))
- }
- slices.SortFunc(identifiers, func(a, b extIdent) int {
- return bytes.Compare(a[:], b[:])
- })
- identifiers = slices.Compact(identifiers)
-
- h := sha512.New384()
- h.Write(binary.LittleEndian.AppendUint64(nil, uint64(k)))
- h.Write(params)
- for _, e := range identifiers {
- h.Write(e[:])
- }
- return ID(h.Sum(nil))
-}
-
const (
// dirIdentifier is the directory name appended to Cache.base for storing
// artifacts named after their [ID].
@@ -429,15 +422,20 @@ type Cache struct {
// Maximum size of a dependency graph.
threshold uintptr
+ // Artifact to [unique.Handle] of identifier cache.
+ artifact sync.Map
+ // Identifier free list, must not be accessed directly.
+ identPool sync.Pool
+
// Synchronises access to dirChecksum.
checksumMu sync.RWMutex
// Identifier to content pair cache.
- ident map[ID]Checksum
+ ident map[unique.Handle[ID]]Checksum
// Identifier to error pair for unrecoverably faulted [Artifact].
- identErr map[ID]error
+ identErr map[unique.Handle[ID]]error
// Pending identifiers, accessed through Cure for entries not in ident.
- identPending map[ID]<-chan struct{}
+ identPending map[unique.Handle[ID]]<-chan struct{}
// Synchronises access to ident and corresponding filesystem entries.
identMu sync.RWMutex
}
@@ -458,6 +456,89 @@ func (c *Cache) SetStrict(strict bool) { c.strict = strict }
// This method is not safe for concurrent use with any other method.
func (c *Cache) SetThreshold(threshold uintptr) { c.threshold = threshold }
+// extIdent is a [Kind] concatenated with [ID].
+type extIdent [wordSize + len(ID{})]byte
+
+// getIdentBuf returns the address of an extIdent for Ident.
+func (c *Cache) getIdentBuf() *extIdent { return c.identPool.Get().(*extIdent) }
+
+// putIdentBuf adds buf to identPool.
+func (c *Cache) putIdentBuf(buf *extIdent) { c.identPool.Put(buf) }
+
+// storeIdent adds an [Artifact] to the artifact cache.
+func (c *Cache) storeIdent(a Artifact, buf *extIdent) unique.Handle[ID] {
+ idu := unique.Make(ID(buf[wordSize:]))
+ c.artifact.Store(a, idu)
+ return idu
+}
+
+// Ident returns the identifier of an [Artifact].
+func (c *Cache) Ident(a Artifact) unique.Handle[ID] {
+ buf, idu := c.unsafeIdent(a, false)
+ if buf != nil {
+ idu = c.storeIdent(a, buf)
+ c.putIdentBuf(buf)
+ }
+ return idu
+}
+
+// unsafeIdent implements Ident but returns the underlying buffer for a newly
+// computed identifier. Callers must return this buffer to identPool. encodeKind
+// is only a hint, kind may still be encoded in the buffer.
+func (c *Cache) unsafeIdent(a Artifact, encodeKind bool) (
+ buf *extIdent,
+ idu unique.Handle[ID],
+) {
+ if id, ok := c.artifact.Load(a); ok {
+ idu = id.(unique.Handle[ID])
+ return
+ }
+
+ if ki, ok := a.(KnownIdent); ok {
+ buf = c.getIdentBuf()
+ if encodeKind {
+ binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind()))
+ }
+ *(*ID)(buf[wordSize:]) = ki.ID()
+ return
+ }
+
+ deps := a.Dependencies()
+ idents := make([]*extIdent, len(deps))
+ for i, d := range deps {
+ dbuf, did := c.unsafeIdent(d, true)
+ if dbuf == nil {
+ dbuf = c.getIdentBuf()
+ binary.LittleEndian.PutUint64(dbuf[:], uint64(d.Kind()))
+ *(*ID)(dbuf[wordSize:]) = did.Value()
+ } else {
+ c.storeIdent(d, dbuf)
+ }
+ defer c.putIdentBuf(dbuf)
+ idents[i] = dbuf
+ }
+ slices.SortFunc(idents, func(a, b *extIdent) int {
+ return bytes.Compare(a[:], b[:])
+ })
+ idents = slices.CompactFunc(idents, func(a, b *extIdent) bool {
+ return *a == *b
+ })
+
+ buf = c.getIdentBuf()
+ h := sha512.New384()
+ binary.LittleEndian.PutUint64(buf[:], uint64(a.Kind()))
+ h.Write(buf[:wordSize])
+ i := IContext{c, h}
+ a.Params(&i)
+ i.cache, i.h = nil, nil
+ for _, dn := range idents {
+ h.Write(dn[:])
+ }
+
+ h.Sum(buf[wordSize:wordSize])
+ return
+}
+
// A ChecksumMismatchError describes an [Artifact] with unexpected content.
type ChecksumMismatchError struct {
// Actual and expected checksums.
@@ -535,8 +616,8 @@ func (c *Cache) Scrub() error {
c.checksumMu.Lock()
defer c.checksumMu.Unlock()
- c.ident = make(map[ID]Checksum)
- c.identErr = make(map[ID]error)
+ c.ident = make(map[unique.Handle[ID]]Checksum)
+ c.identErr = make(map[unique.Handle[ID]]error)
var se ScrubError
@@ -687,7 +768,7 @@ func (c *Cache) Scrub() error {
// loadOrStoreIdent attempts to load a cached [Artifact] by its identifier or
// wait for a pending [Artifact] to cure. If neither is possible, the current
// identifier is stored in identPending and a non-nil channel is returned.
-func (c *Cache) loadOrStoreIdent(id *ID) (
+func (c *Cache) loadOrStoreIdent(id unique.Handle[ID]) (
done chan<- struct{},
checksum Checksum,
err error,
@@ -695,29 +776,29 @@ func (c *Cache) loadOrStoreIdent(id *ID) (
var ok bool
c.identMu.Lock()
- if checksum, ok = c.ident[*id]; ok {
+ if checksum, ok = c.ident[id]; ok {
c.identMu.Unlock()
return
}
- if err, ok = c.identErr[*id]; ok {
+ if err, ok = c.identErr[id]; ok {
c.identMu.Unlock()
return
}
var notify <-chan struct{}
- if notify, ok = c.identPending[*id]; ok {
+ if notify, ok = c.identPending[id]; ok {
c.identMu.Unlock()
<-notify
c.identMu.RLock()
- if checksum, ok = c.ident[*id]; !ok {
- err = c.identErr[*id]
+ if checksum, ok = c.ident[id]; !ok {
+ err = c.identErr[id]
}
c.identMu.RUnlock()
return
}
d := make(chan struct{})
- c.identPending[*id] = d
+ c.identPending[id] = d
c.identMu.Unlock()
done = d
return
@@ -727,17 +808,17 @@ func (c *Cache) loadOrStoreIdent(id *ID) (
// previously submitted to identPending.
func (c *Cache) finaliseIdent(
done chan<- struct{},
- id *ID,
+ id unique.Handle[ID],
checksum *Checksum,
err error,
) {
c.identMu.Lock()
if err != nil {
- c.identErr[*id] = err
+ c.identErr[id] = err
} else {
- c.ident[*id] = *checksum
+ c.ident[id] = *checksum
}
- delete(c.identPending, *id)
+ delete(c.identPending, id)
c.identMu.Unlock()
close(done)
@@ -758,7 +839,7 @@ func (c *Cache) openFile(f File) (r io.ReadCloser, err error) {
c.identMu.RLock()
r, err = os.Open(c.base.Append(
dirIdentifier,
- Encode(Ident(f)),
+ Encode(c.Ident(f).Value()),
).String())
c.identMu.RUnlock()
}
@@ -768,7 +849,7 @@ func (c *Cache) openFile(f File) (r io.ReadCloser, err error) {
return
}
if c.msg.IsVerbose() {
- rn := reportName(f)
+ rn := reportName(f, c.Ident(f))
c.msg.Verbosef("curing %s to memory...", rn)
defer func() {
if err == nil {
@@ -955,8 +1036,8 @@ func (c *Cache) cure(a Artifact) (
checksum Checksum,
err error,
) {
- id := Ident(a)
- ids := Encode(id)
+ id := c.Ident(a)
+ ids := Encode(id.Value())
pathname = c.base.Append(
dirIdentifier,
ids,
@@ -969,11 +1050,11 @@ func (c *Cache) cure(a Artifact) (
}()
var done chan<- struct{}
- done, checksum, err = c.loadOrStoreIdent(&id)
+ done, checksum, err = c.loadOrStoreIdent(id)
if done == nil {
return
} else {
- defer func() { c.finaliseIdent(done, &id, &checksum, err) }()
+ defer func() { c.finaliseIdent(done, id, &checksum, err) }()
}
_, err = os.Lstat(pathname.String())
@@ -1026,7 +1107,7 @@ func (c *Cache) cure(a Artifact) (
}
if c.msg.IsVerbose() {
- rn := reportNameIdent(a, id)
+ rn := reportName(a, id)
c.msg.Verbosef("curing %s...", rn)
defer func() {
if err != nil {
@@ -1126,7 +1207,7 @@ func (c *Cache) cure(a Artifact) (
case FloodArtifact:
deps := a.Dependencies()
- f := FContext{t, make(map[ID]*check.Absolute, len(deps))}
+ f := FContext{t, make(map[Artifact]*check.Absolute, len(deps))}
var wg sync.WaitGroup
wg.Add(len(deps))
@@ -1155,7 +1236,7 @@ func (c *Cache) cure(a Artifact) (
return
}
for i, p := range res {
- f.deps[Ident(deps[i])] = p
+ f.deps[deps[i]] = p
}
defer f.destroy(&err)
@@ -1165,7 +1246,7 @@ func (c *Cache) cure(a Artifact) (
break
default:
- err = InvalidArtifactError(id)
+ err = InvalidArtifactError(id.Value())
return
}
t.cache = nil
@@ -1285,13 +1366,14 @@ func New(
msg: msg,
base: base,
- ident: make(map[ID]Checksum),
- identErr: make(map[ID]error),
- identPending: make(map[ID]<-chan struct{}),
+ ident: make(map[unique.Handle[ID]]Checksum),
+ identErr: make(map[unique.Handle[ID]]error),
+ identPending: make(map[unique.Handle[ID]]<-chan struct{}),
}
c.ctx, c.cancel = context.WithCancel(ctx)
cureDep := make(chan *pendingArtifactDep, cures)
c.cureDep = cureDep
+ c.identPool.New = func() any { return new(extIdent) }
if cures < 1 {
cures = runtime.NumCPU()
diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go
index f5e6dc5c..bb48dfab 100644
--- a/internal/pkg/pkg_test.go
+++ b/internal/pkg/pkg_test.go
@@ -6,7 +6,6 @@ import (
"context"
"crypto/sha512"
"encoding/base64"
- "encoding/binary"
"errors"
"fmt"
"io"
@@ -18,6 +17,7 @@ import (
"reflect"
"syscall"
"testing"
+ "unique"
"unsafe"
"hakurei.app/container"
@@ -82,10 +82,10 @@ type stubArtifact struct {
cure func(t *pkg.TContext) error
}
-func (a stubArtifact) Kind() pkg.Kind { return a.kind }
-func (a stubArtifact) Params() []byte { return a.params }
-func (a stubArtifact) Dependencies() []pkg.Artifact { return a.deps }
-func (a stubArtifact) Cure(t *pkg.TContext) error { return a.cure(t) }
+func (a *stubArtifact) Kind() pkg.Kind { return a.kind }
+func (a *stubArtifact) Params(ctx *pkg.IContext) { ctx.GetHash().Write(a.params) }
+func (a *stubArtifact) Dependencies() []pkg.Artifact { return a.deps }
+func (a *stubArtifact) Cure(t *pkg.TContext) error { return a.cure(t) }
// A stubArtifactF implements [FloodArtifact] with hardcoded behaviour.
type stubArtifactF struct {
@@ -96,10 +96,10 @@ type stubArtifactF struct {
cure func(f *pkg.FContext) error
}
-func (a stubArtifactF) Kind() pkg.Kind { return a.kind }
-func (a stubArtifactF) Params() []byte { return a.params }
-func (a stubArtifactF) Dependencies() []pkg.Artifact { return a.deps }
-func (a stubArtifactF) Cure(f *pkg.FContext) error { return a.cure(f) }
+func (a *stubArtifactF) Kind() pkg.Kind { return a.kind }
+func (a *stubArtifactF) Params(ctx *pkg.IContext) { ctx.GetHash().Write(a.params) }
+func (a *stubArtifactF) Dependencies() []pkg.Artifact { return a.deps }
+func (a *stubArtifactF) Cure(f *pkg.FContext) error { return a.cure(f) }
// A stubFile implements [File] with hardcoded behaviour.
type stubFile struct {
@@ -109,7 +109,7 @@ type stubFile struct {
stubArtifact
}
-func (a stubFile) Cure(context.Context) ([]byte, error) { return a.data, a.err }
+func (a *stubFile) Cure(context.Context) ([]byte, error) { return a.data, a.err }
// newStubFile returns an implementation of [pkg.File] with hardcoded behaviour.
func newStubFile(
@@ -119,7 +119,7 @@ func newStubFile(
data []byte,
err error,
) pkg.File {
- f := overrideIdentFile{id, stubFile{data, err, stubArtifact{
+ f := overrideIdentFile{id, &stubFile{data, err, stubArtifact{
kind,
nil,
nil,
@@ -193,27 +193,38 @@ func TestIdent(t *testing.T) {
testCases := []struct {
name string
a pkg.Artifact
- want pkg.ID
+ want unique.Handle[pkg.ID]
}{
- {"tar", stubArtifact{
+ {"tar", &stubArtifact{
pkg.KindTar,
[]byte{pkg.TarGzip, 0, 0, 0, 0, 0, 0, 0},
[]pkg.Artifact{
- overrideIdent{pkg.ID{}, stubArtifact{}},
+ overrideIdent{pkg.ID{}, new(stubArtifact)},
},
nil,
- }, pkg.MustDecode(
+ }, unique.Make[pkg.ID](pkg.MustDecode(
"HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY",
- )},
+ ))},
}
+
+ msg := message.New(log.New(os.Stderr, "ident: ", 0))
+ msg.SwapVerbose(true)
+ var cache *pkg.Cache
+ if a, err := check.NewAbs(t.TempDir()); err != nil {
+ t.Fatal(err)
+ } else if cache, err = pkg.New(t.Context(), msg, 0, a); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(cache.Close)
+
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
- if got := pkg.Ident(tc.a); got != tc.want {
+ if got := cache.Ident(tc.a); got != tc.want {
t.Errorf("Ident: %s, want %s",
- pkg.Encode(got),
- pkg.Encode(tc.want),
+ pkg.Encode(got.Value()),
+ pkg.Encode(tc.want.Value()),
)
}
})
@@ -438,7 +449,7 @@ func TestCache(t *testing.T) {
0xa9, 0xc2, 0x08, 0xa1, 0x17, 0x17,
}, nil},
- {"incomplete implementation", struct{ pkg.Artifact }{stubArtifact{
+ {"incomplete implementation", struct{ pkg.Artifact }{&stubArtifact{
kind: pkg.KindExec,
params: []byte("artifact overridden to be incomplete"),
}}, nil, pkg.Checksum{}, pkg.InvalidArtifactError(pkg.MustDecode(
@@ -459,7 +470,7 @@ func TestCache(t *testing.T) {
nil, nil,
), nil, pkg.Checksum{}, stub.UniqueError(0xcafe)},
- {"cache hit bad type", overrideChecksum{testdataChecksum, overrideIdent{pkg.ID{0xff, 2}, stubArtifact{
+ {"cache hit bad type", overrideChecksum{testdataChecksum, overrideIdent{pkg.ID{0xff, 2}, &stubArtifact{
kind: pkg.KindTar,
}}}, nil, pkg.Checksum{}, pkg.InvalidFileModeError(
0400,
@@ -493,7 +504,7 @@ func TestCache(t *testing.T) {
// cure after close
c.Close()
- if _, _, err = c.Cure(stubArtifactF{
+ if _, _, err = c.Cure(&stubArtifactF{
kind: pkg.KindExec,
params: []byte("unreachable artifact cured after cancel"),
deps: []pkg.Artifact{pkg.NewFile("", []byte("unreachable dependency"))},
@@ -504,9 +515,8 @@ func TestCache(t *testing.T) {
}, pkg.MustDecode("St9rlE-mGZ5gXwiv_hzQ_B8bZP-UUvSNmf4nHUZzCMOumb6hKnheZSe0dmnuc4Q2")},
{"directory", nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
- id := pkg.KindTar.Ident(
- binary.LittleEndian.AppendUint64(nil, pkg.TarGzip),
- overrideIdent{testdataChecksum, stubArtifact{}},
+ id := pkg.MustDecode(
+ "HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY",
)
makeSample := func(t *pkg.TContext) error {
work := t.GetWorkDir()
@@ -545,9 +555,8 @@ func TestCache(t *testing.T) {
pkg.Encode(id),
)
- id0 := pkg.KindTar.Ident(
- binary.LittleEndian.AppendUint64(nil, pkg.TarGzip),
- overrideIdent{pkg.ID{}, stubArtifact{}},
+ id0 := pkg.MustDecode(
+ "Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa",
)
wantPathname0 := base.Append(
"identifier",
@@ -594,28 +603,28 @@ func TestCache(t *testing.T) {
}
cureMany(t, c, []cureStep{
- {"initial directory", overrideChecksum{wantChecksum, overrideIdent{id, stubArtifact{
+ {"initial directory", overrideChecksum{wantChecksum, overrideIdent{id, &stubArtifact{
kind: pkg.KindTar,
cure: makeSample,
}}}, wantPathname, wantChecksum, nil},
- {"identical identifier", overrideChecksum{wantChecksum, overrideIdent{id, stubArtifact{
+ {"identical identifier", overrideChecksum{wantChecksum, overrideIdent{id, &stubArtifact{
kind: pkg.KindTar,
}}}, wantPathname, wantChecksum, nil},
- {"identical checksum", overrideIdent{id0, stubArtifact{
+ {"identical checksum", overrideIdent{id0, &stubArtifact{
kind: pkg.KindTar,
cure: makeSample,
}}, wantPathname0, wantChecksum, nil},
- {"cure fault", overrideIdent{pkg.ID{0xff, 0}, stubArtifact{
+ {"cure fault", overrideIdent{pkg.ID{0xff, 0}, &stubArtifact{
kind: pkg.KindTar,
cure: func(t *pkg.TContext) error {
return makeGarbage(t.GetWorkDir(), stub.UniqueError(0xcafe))
},
}}, nil, pkg.Checksum{}, stub.UniqueError(0xcafe)},
- {"checksum mismatch", overrideChecksum{pkg.Checksum{}, overrideIdent{pkg.ID{0xff, 1}, stubArtifact{
+ {"checksum mismatch", overrideChecksum{pkg.Checksum{}, overrideIdent{pkg.ID{0xff, 1}, &stubArtifact{
kind: pkg.KindTar,
cure: func(t *pkg.TContext) error {
return makeGarbage(t.GetWorkDir(), nil)
@@ -635,7 +644,7 @@ func TestCache(t *testing.T) {
fs.ModeDir | 0500,
)},
- {"openFile directory", overrideIdent{pkg.ID{0xff, 3}, stubArtifact{
+ {"openFile directory", overrideIdent{pkg.ID{0xff, 3}, &stubArtifact{
kind: pkg.KindTar,
cure: func(t *pkg.TContext) error {
r, err := t.Open(overrideChecksumFile{checksum: wantChecksum})
@@ -654,21 +663,21 @@ func TestCache(t *testing.T) {
Err: syscall.EISDIR,
}},
- {"no output", overrideIdent{pkg.ID{0xff, 4}, stubArtifact{
+ {"no output", overrideIdent{pkg.ID{0xff, 4}, &stubArtifact{
kind: pkg.KindTar,
cure: func(t *pkg.TContext) error {
return nil
},
}}, nil, pkg.Checksum{}, pkg.NoOutputError{}},
- {"file output", overrideIdent{pkg.ID{0xff, 5}, stubArtifact{
+ {"file output", overrideIdent{pkg.ID{0xff, 5}, &stubArtifact{
kind: pkg.KindTar,
cure: func(t *pkg.TContext) error {
return os.WriteFile(t.GetWorkDir().String(), []byte{0}, 0400)
},
}}, nil, pkg.Checksum{}, errors.New("non-file artifact produced regular file")},
- {"symlink output", overrideIdent{pkg.ID{0xff, 6}, stubArtifact{
+ {"symlink output", overrideIdent{pkg.ID{0xff, 6}, &stubArtifact{
kind: pkg.KindTar,
cure: func(t *pkg.TContext) error {
return os.Symlink(
@@ -688,7 +697,7 @@ func TestCache(t *testing.T) {
wantErr := stub.UniqueError(0xcafe)
n, ready := make(chan struct{}), make(chan struct{})
go func() {
- if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, stubArtifact{
+ if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, &stubArtifact{
kind: pkg.KindTar,
cure: func(t *pkg.TContext) error {
close(ready)
@@ -703,7 +712,7 @@ func TestCache(t *testing.T) {
<-ready
wCureDone := make(chan struct{})
go func() {
- if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, stubArtifact{
+ if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, &stubArtifact{
kind: pkg.KindTar,
}}); !reflect.DeepEqual(err, wantErr) {
panic(fmt.Sprintf("Cure: error = %v, want %v", err, wantErr))
@@ -720,7 +729,7 @@ func TestCache(t *testing.T) {
nil, stub.UniqueError(0xbad),
), nil, pkg.Checksum{}, stub.UniqueError(0xbad)},
- {"file output", overrideIdent{pkg.ID{0xff, 2}, stubArtifact{
+ {"file output", overrideIdent{pkg.ID{0xff, 2}, &stubArtifact{
kind: pkg.KindTar,
cure: func(t *pkg.TContext) error {
return os.WriteFile(
@@ -745,8 +754,8 @@ func TestCache(t *testing.T) {
identPending := reflect.NewAt(
identPendingVal.Type(),
unsafe.Pointer(identPendingVal.UnsafeAddr()),
- ).Elem().Interface().(map[pkg.ID]<-chan struct{})
- notify := identPending[pkg.ID{0xff}]
+ ).Elem().Interface().(map[unique.Handle[pkg.ID]]<-chan struct{})
+ notify := identPending[unique.Make(pkg.ID{0xff})]
go close(n)
<-notify
<-wCureDone
diff --git a/internal/pkg/tar.go b/internal/pkg/tar.go
index eda41401..faad4854 100644
--- a/internal/pkg/tar.go
+++ b/internal/pkg/tar.go
@@ -70,9 +70,9 @@ func NewHTTPGetTar(
// Kind returns the hardcoded [Kind] constant.
func (a *tarArtifact) Kind() Kind { return KindTar }
-// Params returns compression encoded in little endian.
-func (a *tarArtifact) Params() []byte {
- return binary.LittleEndian.AppendUint64(nil, a.compression)
+// Params writes compression encoded in little endian.
+func (a *tarArtifact) Params(ctx *IContext) {
+ ctx.GetHash().Write(binary.LittleEndian.AppendUint64(nil, a.compression))
}
// Dependencies returns a slice containing the backing file.
diff --git a/internal/pkg/tar_test.go b/internal/pkg/tar_test.go
index f468b62a..d9635fb5 100644
--- a/internal/pkg/tar_test.go
+++ b/internal/pkg/tar_test.go
@@ -101,8 +101,11 @@ func checkTarHTTP(
h.Write([]byte{byte(pkg.KindTar), 0, 0, 0, 0, 0, 0, 0})
h.Write([]byte{pkg.TarGzip, 0, 0, 0, 0, 0, 0, 0})
h.Write([]byte{byte(pkg.KindHTTPGet), 0, 0, 0, 0, 0, 0, 0})
- httpIdent := pkg.KindHTTPGet.Ident([]byte("file:///testdata"))
- h.Write(httpIdent[:])
+
+ h0 := sha512.New384()
+ h0.Write([]byte{byte(pkg.KindHTTPGet), 0, 0, 0, 0, 0, 0, 0})
+ h0.Write([]byte("file:///testdata"))
+ h.Write(h0.Sum(nil))
return pkg.ID(h.Sum(nil))
}()
@@ -113,10 +116,6 @@ func checkTarHTTP(
pkg.TarGzip,
)
- if id := pkg.Ident(a); id != wantIdent {
- t.Fatalf("Ident: %s, want %s", pkg.Encode(id), pkg.Encode(wantIdent))
- }
-
tarDir := stubArtifact{
kind: pkg.KindExec,
params: []byte("directory containing a single regular file"),
@@ -164,9 +163,9 @@ func checkTarHTTP(
},
}
// destroy these to avoid including it in flatten test case
- defer newDestroyArtifactFunc(tarDir)(t, base, c)
- defer newDestroyArtifactFunc(tarDirMulti)(t, base, c)
- defer newDestroyArtifactFunc(tarDirType)(t, base, c)
+ defer newDestroyArtifactFunc(&tarDir)(t, base, c)
+ defer newDestroyArtifactFunc(&tarDirMulti)(t, base, c)
+ defer newDestroyArtifactFunc(&tarDirType)(t, base, c)
cureMany(t, c, []cureStep{
{"file", a, base.Append(
@@ -175,25 +174,25 @@ func checkTarHTTP(
), wantChecksum, nil},
{"directory", pkg.NewTar(
- tarDir,
+ &tarDir,
pkg.TarGzip,
), ignorePathname, wantChecksum, nil},
{"multiple entries", pkg.NewTar(
- tarDirMulti,
+ &tarDirMulti,
pkg.TarGzip,
), nil, pkg.Checksum{}, errors.New(
"input directory does not contain a single regular file",
)},
{"bad type", pkg.NewTar(
- tarDirType,
+ &tarDirType,
pkg.TarGzip,
), nil, pkg.Checksum{}, errors.New(
"input directory does not contain a single regular file",
)},
- {"error passthrough", pkg.NewTar(stubArtifact{
+ {"error passthrough", pkg.NewTar(&stubArtifact{
kind: pkg.KindExec,
params: []byte("doomed artifact"),
cure: func(t *pkg.TContext) error {
generated by cgit v1.3.1 (git 2.54.0) at 2026-08-29 23:49:59 +0000