aboutsummaryrefslogtreecommitdiffhomepage
path: root/internal/pkg
diff options
context:
space:
mode:
Diffstat (limited to 'internal/pkg')
-rw-r--r--internal/pkg/ir.go39
-rw-r--r--internal/pkg/pkg.go105
-rw-r--r--internal/pkg/pkg_test.go146
3 files changed, 258 insertions, 32 deletions
diff --git a/internal/pkg/ir.go b/internal/pkg/ir.go
index 35fe76e7..5a7e7f56 100644
--- a/internal/pkg/ir.go
+++ b/internal/pkg/ir.go
@@ -90,7 +90,7 @@ type IContext struct {
// returns and must not be exposed directly.
w io.Writer
// Optional [Artifact] to cureRes cache, replaces [IRKindIdent] with
- // checksum values if non-nil.
+ // checksum values if non-nil. The pathname field may not be populated.
inputs map[Artifact]cureRes
}
@@ -173,7 +173,7 @@ func (i *IContext) mustWrite(p []byte) {
// WriteIdent writes the identifier of [Artifact] to the IR. The behaviour of
// WriteIdent is not defined for an [Artifact] not part of the slice returned by
-// [Artifact.Dependencies].
+// [Artifact.Inputs].
func (i *IContext) WriteIdent(a Artifact) {
buf := i.ic.getIdentBuf()
defer i.ic.putIdentBuf(buf)
@@ -235,7 +235,8 @@ func (ic *irCache) Encode(w io.Writer, a Artifact) (err error) {
}
// encode implements Encode but replaces identifiers with their cured checksums
-// for a non-nil ident. Caller must acquire Cache.identMu.
+// for a non-nil ident. The pathname field is unused. Caller must acquire
+// Cache.identMu.
func (ic *irCache) encode(
w io.Writer,
a Artifact,
@@ -432,9 +433,9 @@ type (
// Common buffer for word-sized reads.
buf [wordSize]byte
- // Dependencies sent before params, sorted by identifier. Resliced on
+ // Inputs sent before params, sorted by identifier. Resliced on
// each call to Next and checked to be depleted during Finalise.
- deps []*extIdent
+ inputs []*extIdent
// Number of values already read, -1 denotes a finalised IRReader.
count int
@@ -518,7 +519,7 @@ const (
irMaxValues = 1 << 12
// irMaxDeps is the arbitrary maximum number of direct dependencies allowed
- // to be returned by [Artifact.Dependencies] and subsequently decoded by
+ // to be returned by [Artifact.Inputs] and subsequently decoded by
// [IRDecoder].
irMaxDeps = 1 << 10
)
@@ -603,17 +604,17 @@ func (ir *IRReader) mustReadHeader(k IRValueKind) {
// putAll returns all dependency buffers to the underlying [Cache].
func (ir *IRReader) putAll() {
- for _, buf := range ir.deps {
+ for _, buf := range ir.inputs {
ir.d.c.putIdentBuf(buf)
}
- ir.deps = nil
+ ir.inputs = nil
}
// DiscardAll discards all unstructured dependencies. This is useful to
// implementations that encode dependencies as [IRKindIdent] which are read back
// via ReadIdent.
func (ir *IRReader) DiscardAll() {
- if ir.deps == nil {
+ if ir.inputs == nil {
panic("attempting to discard dependencies twice")
}
ir.putAll()
@@ -625,13 +626,13 @@ var ErrDependencyDepleted = errors.New("reading past end of dependencies")
// Next returns the next unstructured dependency.
func (ir *IRReader) Next() Artifact {
- if len(ir.deps) == 0 {
+ if len(ir.inputs) == 0 {
panic(ErrDependencyDepleted)
}
- id := unique.Make(ID(ir.deps[0][wordSize:]))
- ir.d.c.putIdentBuf(ir.deps[0])
- ir.deps = ir.deps[1:]
+ id := unique.Make(ID(ir.inputs[0][wordSize:]))
+ ir.d.c.putIdentBuf(ir.inputs[0])
+ ir.inputs = ir.inputs[1:]
if a, ok := ir.d.ident[id]; !ok {
ir.putAll()
@@ -684,8 +685,8 @@ func (ir *IRReader) Finalise() (checksum unique.Handle[Checksum], ok bool) {
}
ir.r, ir.ibw = nil, nil
- if len(ir.deps) != 0 {
- panic(MissedDependencyError(len(ir.deps)))
+ if len(ir.inputs) != 0 {
+ panic(MissedDependencyError(len(ir.inputs)))
}
return
@@ -778,11 +779,11 @@ func (d *IRDecoder) decode() (a Artifact, err error) {
err = ErrIRDepend
return
}
- ir.deps = make([]*extIdent, sz)
- for i := range ir.deps {
- ir.deps[i] = d.c.getIdentBuf()
+ ir.inputs = make([]*extIdent, sz)
+ for i := range ir.inputs {
+ ir.inputs[i] = d.c.getIdentBuf()
}
- for _, buf := range ir.deps {
+ for _, buf := range ir.inputs {
ir.mustRead(buf[:])
}
diff --git a/internal/pkg/pkg.go b/internal/pkg/pkg.go
index 78e62e85..aadfa308 100644
--- a/internal/pkg/pkg.go
+++ b/internal/pkg/pkg.go
@@ -368,7 +368,7 @@ var _ error = InvalidLookupError{}
// GetArtifact returns the identifier pathname and checksum of an [Artifact].
// Calling Pathname with an [Artifact] not part of the slice returned by
-// [Artifact.Dependencies] panics.
+// [Artifact.Inputs] panics.
func (f *FContext) GetArtifact(a Artifact) (
pathname *check.Absolute,
checksum unique.Handle[Checksum],
@@ -678,6 +678,10 @@ const (
// CIgnoreSubstitutes disables content-based dependency substitution.
CIgnoreSubstitutes
+
+ // CExternShallow arranges for only non-flood inputs to be fetched when
+ // curing an [Artifact] available via the external cache.
+ CExternShallow
)
// toplevel holds [context.WithCancel] over caller-supplied context, where all
@@ -1914,25 +1918,57 @@ func (c *Cache) tryExtern(ctx context.Context, id unique.Handle[ID]) (
}
// cureMany concurrently collects outcome of multiple [Artifact].
-func (c *Cache) cureMany(inputs []Artifact, r map[Artifact]cureRes) error {
+func (c *Cache) cureMany(
+ inputs []Artifact,
+ r map[Artifact]cureRes,
+ shallow bool,
+) ([]bool, error) {
var wg sync.WaitGroup
wg.Add(len(inputs))
+ var mask []bool
res := make([]cureRes, len(inputs))
errs := make(DependencyCureError, 0, len(inputs))
var errsMu sync.Mutex
+ if shallow {
+ mask = make([]bool, len(inputs))
+ }
for i, d := range inputs {
+ if shallow {
+ if _, ok := d.(FloodArtifact); ok {
+ mask[i] = true
+ wg.Done()
+ continue
+ }
+
+ if kc, ok := d.(KnownChecksum); ok {
+ res[i].checksum = unique.Make(kc.Checksum())
+ wg.Done()
+ continue
+ }
+ }
pending := pendingArtifactDep{d, &res[i], &errs, &errsMu, &wg}
go pending.cure(c)
}
wg.Wait()
if len(errs) > 0 {
- return &errs
+ return mask, &errs
}
for i, p := range res {
+ if shallow && mask[i] {
+ continue
+ }
r[inputs[i]] = p
}
- return nil
+ return mask, nil
+}
+
+// HangingInputError describes an input of an [Artifact] on a sparse cache
+// without an outcome available locally or via [External].
+type HangingInputError unique.Handle[ID]
+
+func (e HangingInputError) Error() string {
+ return Encode(unique.Handle[ID](e).Value()) + " is unavailable"
}
// cure implements Cure without acquiring a read lock on abortMu. cure must not
@@ -2193,12 +2229,62 @@ func (c *Cache) cure(a Artifact, curesExempt bool) (
break
case FloodArtifact:
+ var externChecksum unique.Handle[Checksum]
+ if externChecksum, err = c.tryExtern(ctx, id); err != nil {
+ if c.msg.IsVerbose() {
+ c.msg.Verbosef("extern %s: %v", reportName(ca, id), err)
+ }
+ return
+ }
+ extern := externChecksum != zeroChecksum
+ shallow := extern && c.flags&CExternShallow != 0
+
inputs := a.Inputs()
f := FContext{t, make(map[Artifact]cureRes, len(inputs))}
- if err = c.cureMany(inputs, f.inputs); err != nil {
+ var mask []bool
+ if mask, err = c.cureMany(inputs, f.inputs, shallow); err != nil {
return
}
+ if shallow {
+ for i, d := range inputs {
+ if !mask[i] {
+ continue
+ }
+
+ if kc, ok := d.(KnownChecksum); ok {
+ f.inputs[d] = cureRes{checksum: unique.Make(kc.Checksum())}
+ continue
+ }
+
+ var sum unique.Handle[Checksum]
+ did := c.Ident(d)
+ sum, err = c.tryLocal(did)
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ return
+ }
+
+ sum, err = c.tryExtern(ctx, did)
+ if err != nil {
+ return
+ }
+ if sum == zeroChecksum {
+ if c.msg.IsVerbose() {
+ c.msg.Verbosef(
+ "input %s not available",
+ reportName(d, did),
+ )
+ }
+ err = HangingInputError(did)
+ return
+ }
+ }
+
+ f.inputs[d] = cureRes{checksum: sum}
+ }
+ }
+
sh := sha512.New384()
err = c.encode(sh, a, f.inputs)
if err != nil {
@@ -2245,14 +2331,7 @@ func (c *Cache) cure(a Artifact, curesExempt bool) (
defer f.destroy(&err)
- var externChecksum unique.Handle[Checksum]
- if externChecksum, err = c.tryExtern(ctx, id); err != nil {
- if c.msg.IsVerbose() {
- c.msg.Verbosef("extern %s: %v", reportName(ca, id), err)
- }
- return
- }
- if externChecksum != zeroChecksum {
+ if extern {
if checksum != zeroChecksum && externChecksum != checksum {
err = &ChecksumMismatchError{externChecksum.Value(), checksum.Value()}
if c.msg.IsVerbose() {
diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go
index 01cd2617..d03216f6 100644
--- a/internal/pkg/pkg_test.go
+++ b/internal/pkg/pkg_test.go
@@ -1621,6 +1621,152 @@ func TestCache(t *testing.T) {
"work": {Mode: fs.ModeDir | 0700},
}},
+
+ {"extern shallow", pkg.CExternShallow, nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+ a := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("extern"),
+ }
+ wantIdent := c.Ident(a)
+ wantOutput := expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+ "result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")},
+ }
+ var wantChecksum pkg.Checksum
+ if err := pkg.SumFS(
+ &wantChecksum,
+ fstest.MapFS(wantOutput),
+ ".",
+ ); err != nil {
+ t.Fatal(err)
+ }
+ wantChecksumH := unique.Make(wantChecksum)
+
+ _a := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("extern substitute"),
+ deps: []pkg.Artifact{pkg.NewFile("", nil)},
+ }
+ _wantIdent := c.Ident(_a)
+ _wantOutput := expectsFS{
+ ".": {Mode: fs.ModeDir | 0500},
+ }
+ var _wantChecksum pkg.Checksum
+ if err := pkg.SumFS(
+ &_wantChecksum,
+ fstest.MapFS(_wantOutput),
+ ".",
+ ); err != nil {
+ t.Fatal(err)
+ }
+ _wantChecksumH := unique.Make(_wantChecksum)
+
+ kca := pkg.NewExec(
+ "", "",
+ new(pkg.Checksum), 0, false, false,
+ fhs.AbsRoot, nil, fhs.AbsRoot, nil,
+ )
+ kcIdent := c.Ident(kca)
+
+ fia := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("flood input"),
+ deps: []pkg.Artifact{_a},
+ }
+ fiaIdent := c.Ident(fia)
+
+ flia := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("flood input local"),
+ deps: []pkg.Artifact{_a},
+ }
+ fliaIdent := c.Ident(flia)
+
+ h := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("hanging input"),
+ }
+
+ fhia := &stubArtifactF{
+ kind: pkg.KindExec,
+ params: []byte("flood input local"),
+ deps: []pkg.Artifact{h},
+ }
+ fhiaIdent := c.Ident(fhia)
+
+ c.SetExternal(stubExtern{
+ artifact: map[unique.Handle[pkg.ID]]pkg.Checksum{
+ wantIdent: wantChecksum,
+ _wantIdent: _wantChecksum,
+ kcIdent: wantChecksum,
+ fiaIdent: wantChecksum,
+ fliaIdent: wantChecksum,
+ fhiaIdent: wantChecksum,
+ },
+ checksum: map[unique.Handle[pkg.Checksum]]fstest.MapFS{
+ wantChecksumH: fstest.MapFS(wantOutput),
+ _wantChecksumH: fstest.MapFS(_wantOutput),
+ },
+ status: map[unique.Handle[pkg.ID]]string{
+ wantIdent: "\x00",
+ kcIdent: "unreachable",
+ },
+ })
+
+ cureMany(t, c, []cureStep{
+ {"extern", a, base.Append(
+ "identifier",
+ pkg.Encode(wantIdent.Value()),
+ ), wantOutput, nil},
+
+ {"flood shallow extern", fia, base.Append(
+ "identifier",
+ pkg.Encode(fiaIdent.Value()),
+ ), wantOutput, nil},
+
+ {"substitute", _a, base.Append(
+ "identifier",
+ pkg.Encode(_wantIdent.Value()),
+ ), _wantOutput, nil},
+
+ {"flood shallow local", flia, base.Append(
+ "identifier",
+ pkg.Encode(fliaIdent.Value()),
+ ), wantOutput, nil},
+
+ {"flood shallow hanging", fhia, nil, nil,
+ pkg.HangingInputError(c.Ident(h))},
+
+ {"mismatch", kca, nil, nil, &pkg.ChecksumMismatchError{
+ Got: wantChecksum,
+ }},
+ })
+ }, expectsFS{
+ ".": {Mode: fs.ModeDir | 0700},
+
+ "checksum": {Mode: fs.ModeDir | 0700},
+ "checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU": {Mode: fs.ModeDir | 0500},
+ "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl": {Mode: fs.ModeDir | 0500},
+ "checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl/result": {Mode: fs.ModeSymlink | 0777, Data: []byte("/proc/nonexistent")},
+
+ "identifier": {Mode: fs.ModeDir | 0700},
+ "identifier/4HqRo4uTwRQjfy3d2cujMoDC_pC4iv20h4a7NYlx0UdbVuky18o5iK78TFEfPX2U": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+ "identifier/7AZcJm58ghFyTVf_v2baSntgpsxkP5el7ti9dC77C29n8YTEqQW9jRW92KGNdYnz": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+ "identifier/RJbTOOIlqsEkAxGlTIsbp4mHPO0Q2IlwU7Gh_FIaTxnhgxUjAQPXXgPyP2GJWX2A": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+ "identifier/c4aCI00C-ZVyo_FQDQLl1OYK4U_kjzxwrLdFDiXMHnbMcZXCkXo_nxUWauScZ_4V": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+ "identifier/gvCqzexZVqXjF8B5lKMcP5onmq3jJ6AKqzOW_WN0Fl2yTr9NKhPt9l_ClD2EOSlS": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+ "identifier/lqjQtzpJ6asInpsdxqNYVjTkRHK-gW0eUCqNCCk1KhZTuKwKAc5Z0H7tS-GWfRlg": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+
+ "status": {Mode: fs.ModeDir | 0700},
+ "status/gvCqzexZVqXjF8B5lKMcP5onmq3jJ6AKqzOW_WN0Fl2yTr9NKhPt9l_ClD2EOSlS": {Mode: 0400, Data: []byte("\x00")},
+
+ "substitute": {Mode: fs.ModeDir | 0700},
+ "substitute/qOYrxy9ztKeOA96Os811_0Ox5sd8FBOxis6psJAnRJL5MLazFMaqmd4g7t7k1OHk": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/MGWmEfjut2QE2xPJwTsmUzpff4BN_FEnQ7T0j7gvUCCiugJQNwqt9m151fm9D1yU")},
+ "substitute/vLTXKbImP9XTTSi7P3PIuqy5L5UgX4dFQm2traX4X_pCDvg9GDraX1LmaUpgLu4w": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+ "substitute/yn1xEs5uSnu5EFLtkvjFSQ0C_fv50hU_w2wfDH_WBEzQhUiPJfyS3Eegy6YSclYN": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/fHkl_RuHOoc4rso__nV-qreikovd6Yhrq5mpBlkf5hmPGaxDlik2bYOQ4dhUQjtl")},
+
+ "work": {Mode: fs.ModeDir | 0700},
+ }},
}
checkWithCache(t, testCases)
}
generated by cgit v1.3.1 (git 2.54.0) at 2026-08-30 04:39:00 +0000