internal/pkg: replace outcomes from external cache

This is primarily useful for implementing a mirror service. This change also works around the zero-dependency FloodArtifact edge case.

Signed-off-by: Ophestra <cat@gensokyo.uk>

1 files changed, 136 insertions, 18 deletions

diff --git a/internal/pkg/pkg.go b/internal/pkg/pkg.go
index f5b586ee..6542741f 100644
--- a/internal/pkg/pkg.go
+++ b/internal/pkg/pkg.go
@@ -187,7 +187,7 @@ func makeStatusHeader(extension string) string {
 var statusHeader = makeStatusHeader("")
 
 // prepareStatus initialises the status file once.
-func (t *TContext) prepareStatus() error {
+func (t *TContext) prepareStatus(writeHeader bool) error {
 	if t.statusPath != nil || t.status != nil {
 		return t.statusErr
 	}
@@ -204,14 +204,16 @@ func (t *TContext) prepareStatus() error {
 		return t.statusErr
 	}
 
-	_, t.statusErr = t.status.WriteString(statusHeader)
+	if writeHeader {
+		_, t.statusErr = t.status.WriteString(statusHeader)
+	}
 	return t.statusErr
 }
 
 // GetStatusWriter returns a [io.Writer] for build logs. The caller must not
 // seek this writer before the position it was first returned in.
 func (t *TContext) GetStatusWriter() (io.Writer, error) {
-	err := t.prepareStatus()
+	err := t.prepareStatus(true)
 	return t.status, err
 }
 
@@ -332,6 +334,28 @@ type FContext struct {
 	deps map[Artifact]cureRes
 }
 
+// linkSubstitute links status for substitute if populated.
+func (f *FContext) linkSubstitute(ids, substitutes string) (err error) {
+	if f.status == nil || ids == substitutes {
+		return
+	}
+
+	statusS := f.cache.base.Append(
+		dirStatus,
+		substitutes,
+	)
+	f.cache.checksumMu.Lock()
+	err = os.Link(f.cache.base.Append(
+		dirStatus,
+		ids,
+	).String(), statusS.String())
+	f.cache.checksumMu.Unlock()
+	if err == nil {
+		f.statusSPath = statusS
+	}
+	return
+}
+
 // InvalidLookupError is the identifier of non-dependency [Artifact] looked up
 // via [FContext.GetArtifact] by a misbehaving [Artifact] implementation.
 type InvalidLookupError ID
@@ -670,6 +694,20 @@ type pendingCure struct {
 	cancel context.CancelFunc
 }
 
+// An External cache provides prepared [Artifact] cure outcomes.
+type External interface {
+	// Artifact returns the address of the [Checksum] of the cure outcome of
+	// an [Artifact] corresponding to id, or nil if this [Artifact] is not
+	// available in the external cache.
+	Artifact(id unique.Handle[ID]) (*Checksum, error)
+	// Checksum returns an [Artifact] producing the specified checksum.
+	Checksum(checksum unique.Handle[Checksum]) Artifact
+	// Status returns [io.ReadCloser] of the status file of an [Artifact]
+	// corresponding to id, or nil if this [Artifact] is not available or a
+	// status file is not present.
+	Status(id unique.Handle[ID]) (io.ReadCloser, error)
+}
+
 // Cache is a support layer that implementations of [Artifact] can use to store
 // cured [Artifact] data in a content addressed fashion.
 type Cache struct {
@@ -720,6 +758,11 @@ type Cache struct {
 	// Buffered I/O free list, must not be accessed directly.
 	brPool, bwPool sync.Pool
 
+	// Optional external cache implementation.
+	extern External
+	// Synchronises access to extern.
+	externMu sync.RWMutex
+
 	// Unlocks the on-filesystem cache. Must only be called from Close.
 	unlock func()
 	// Whether [Cache] is considered closed.
@@ -874,6 +917,13 @@ func readlinkChecksum(a *check.Absolute, buf *Checksum) error {
 	return Decode(buf, linkname[len(checksumLinknamePrefix):])
 }
 
+// SetExternal sets e as the [External] implementation of c.
+func (c *Cache) SetExternal(e External) {
+	c.externMu.Lock()
+	c.extern = e
+	c.externMu.Unlock()
+}
+
 // ScrubError describes the outcome of a [Cache.Scrub] call where errors were
 // found and removed from the underlying storage of [Cache].
 type ScrubError struct {
@@ -1782,6 +1832,40 @@ func (r *RContext) NewMeasuredReader(
 	return r.cache.newMeasuredReader(rc, checksum)
 }
 
+// tryExtern attempts to obtain an [Artifact] outcome from extern.
+func (c *Cache) tryExtern(id unique.Handle[ID]) (
+	unique.Handle[Checksum],
+	io.ReadCloser,
+	error,
+) {
+	c.externMu.RLock()
+	defer c.externMu.RUnlock()
+
+	if c.extern == nil {
+		return zeroChecksum, nil, nil
+	}
+
+	v, err := c.extern.Artifact(id)
+	if err != nil {
+		return zeroChecksum, nil, err
+	}
+	if v == nil {
+		return zeroChecksum, nil, nil
+	}
+	checksum := unique.Make(*v)
+
+	var got unique.Handle[Checksum]
+	if _, got, err = c.Cure(c.extern.Checksum(checksum)); err != nil {
+		return checksum, nil, err
+	} else if got != checksum {
+		return zeroChecksum, nil, &ChecksumMismatchError{got.Value(), checksum.Value()}
+	}
+
+	var status io.ReadCloser
+	status, err = c.extern.Status(id)
+	return checksum, status, err
+}
+
 // cure implements Cure without acquiring a read lock on abortMu. cure must not
 // be entered during Abort.
 func (c *Cache) cure(a Artifact, curesExempt bool) (
@@ -1848,7 +1932,7 @@ func (c *Cache) cure(a Artifact, curesExempt bool) (
 				err = zeroTimes(pathname.String())
 			}
 
-			if err == nil && alternative != nil {
+			if err == nil && alternative != nil && substitute != id {
 				c.substituteMu.Lock()
 				err = os.Symlink(
 					linkname,
@@ -2116,26 +2200,60 @@ func (c *Cache) cure(a Artifact, curesExempt bool) (
 		}
 
 		defer f.destroy(&err)
+
+		var (
+			externChecksum unique.Handle[Checksum]
+			externStatus   io.ReadCloser
+		)
+		if externChecksum, externStatus, err = c.tryExtern(id); err != nil {
+			if c.msg.IsVerbose() {
+				c.msg.Verbosef("extern %s: %v", reportName(ca, id), err)
+			}
+			return
+		}
+		if externChecksum != zeroChecksum {
+			if checksum != zeroChecksum && externChecksum != checksum {
+				if externStatus != nil {
+					_ = externStatus.Close()
+				}
+				err = &ChecksumMismatchError{externChecksum.Value(), checksum.Value()}
+				if c.msg.IsVerbose() {
+					c.msg.Verbosef("extern %s: %v", reportName(ca, id), err)
+				}
+				return
+			}
+
+			checksum = externChecksum
+			checksums = Encode(checksum.Value())
+			checksumPathname = c.base.Append(
+				dirChecksum,
+				checksums,
+			)
+
+			if externStatus != nil {
+				if err = f.prepareStatus(false); err != nil {
+					_ = externStatus.Close()
+					return
+				} else if _, err = io.Copy(f.status, externStatus); err != nil {
+					_ = externStatus.Close()
+					return
+				} else if err = externStatus.Close(); err != nil {
+					return
+				} else if err = f.linkSubstitute(ids, substitutes); err != nil {
+					return
+				}
+			}
+			return
+		}
+
 		if err = c.enterCure(a, curesExempt); err != nil {
 			return
 		}
 		err = ca.Cure(&f)
 		c.exitCure(a, curesExempt)
 
-		if err == nil && f.status != nil {
-			statusS := c.base.Append(
-				dirStatus,
-				substitutes,
-			)
-			c.checksumMu.Lock()
-			err = os.Link(c.base.Append(
-				dirStatus,
-				ids,
-			).String(), statusS.String())
-			c.checksumMu.Unlock()
-			if err == nil {
-				f.statusSPath = statusS
-			}
+		if err == nil {
+			err = f.linkSubstitute(ids, substitutes)
 		}
 		if err != nil {
 			if c.msg.IsVerbose() {