internal/pkg: move dependency flooding to cache

This imposes a hard upper limit to concurrency during dependency satisfaction and moves all dependency-related code out of individual implementations of Artifact. This change also includes ctx and msg as part of Cache.

Signed-off-by: Ophestra <cat@gensokyo.uk>

1 files changed, 92 insertions, 31 deletions

diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go
index bb7d760b..29291fcc 100644
--- a/internal/pkg/pkg_test.go
+++ b/internal/pkg/pkg_test.go
@@ -3,6 +3,7 @@ package pkg_test
 import (
 	"archive/tar"
 	"bytes"
+	"context"
 	"crypto/sha512"
 	"encoding/base64"
 	"encoding/binary"
@@ -10,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"log"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -22,6 +24,7 @@ import (
 	"hakurei.app/container/check"
 	"hakurei.app/container/stub"
 	"hakurei.app/internal/pkg"
+	"hakurei.app/message"
 )
 
 func TestMain(m *testing.M) { container.TryArgv0(nil); os.Exit(m.Run()) }
@@ -29,7 +32,7 @@ func TestMain(m *testing.M) { container.TryArgv0(nil); os.Exit(m.Run()) }
 // overrideIdent overrides the ID method of [Artifact].
 type overrideIdent struct {
 	id pkg.ID
-	pkg.Artifact
+	pkg.TrivialArtifact
 }
 
 func (a overrideIdent) ID() pkg.ID { return a.id }
@@ -45,7 +48,7 @@ func (a overrideIdentFile) ID() pkg.ID { return a.id }
 // A knownIdentArtifact implements [pkg.KnownIdent] and [Artifact]
 type knownIdentArtifact interface {
 	pkg.KnownIdent
-	pkg.Artifact
+	pkg.TrivialArtifact
 }
 
 // A knownIdentFile implements [pkg.KnownIdent] and [File]
@@ -70,19 +73,33 @@ type overrideChecksumFile struct {
 
 func (a overrideChecksumFile) Checksum() pkg.Checksum { return a.checksum }
 
-// A stubArtifact implements [Artifact] with hardcoded behaviour.
+// A stubArtifact implements [TrivialArtifact] with hardcoded behaviour.
 type stubArtifact struct {
 	kind   pkg.Kind
 	params []byte
 	deps   []pkg.Artifact
 
-	cure func(c *pkg.CureContext) error
+	cure func(t *pkg.TContext) error
 }
 
-func (a stubArtifact) Kind() pkg.Kind                { return a.kind }
-func (a stubArtifact) Params() []byte                { return a.params }
-func (a stubArtifact) Dependencies() []pkg.Artifact  { return a.deps }
-func (a stubArtifact) Cure(c *pkg.CureContext) error { return a.cure(c) }
+func (a stubArtifact) Kind() pkg.Kind               { return a.kind }
+func (a stubArtifact) Params() []byte               { return a.params }
+func (a stubArtifact) Dependencies() []pkg.Artifact { return a.deps }
+func (a stubArtifact) Cure(t *pkg.TContext) error   { return a.cure(t) }
+
+// A stubArtifactF implements [FloodArtifact] with hardcoded behaviour.
+type stubArtifactF struct {
+	kind   pkg.Kind
+	params []byte
+	deps   []pkg.Artifact
+
+	cure func(f *pkg.FContext) error
+}
+
+func (a stubArtifactF) Kind() pkg.Kind               { return a.kind }
+func (a stubArtifactF) Params() []byte               { return a.params }
+func (a stubArtifactF) Dependencies() []pkg.Artifact { return a.deps }
+func (a stubArtifactF) Cure(f *pkg.FContext) error   { return a.cure(f) }
 
 // A stubFile implements [File] with hardcoded behaviour.
 type stubFile struct {
@@ -92,7 +109,7 @@ type stubFile struct {
 	stubArtifact
 }
 
-func (a stubFile) Data() ([]byte, error) { return a.data, a.err }
+func (a stubFile) Cure() ([]byte, error) { return a.data, a.err }
 
 // newStubFile returns an implementation of [pkg.File] with hardcoded behaviour.
 func newStubFile(
@@ -106,7 +123,7 @@ func newStubFile(
 		kind,
 		nil,
 		nil,
-		func(*pkg.CureContext) error {
+		func(*pkg.TContext) error {
 			panic("unreachable")
 		},
 	}}}
@@ -241,10 +258,14 @@ func checkWithCache(t *testing.T, testCases []cacheTestCase) {
 				}
 			})
 
+			msg := message.New(log.New(os.Stderr, "cache: ", 0))
+			msg.SwapVerbose(testing.Verbose())
+
 			var scrubFunc func() error // scrub after hashing
-			if c, err := pkg.New(base); err != nil {
+			if c, err := pkg.New(t.Context(), msg, 0, base); err != nil {
 				t.Fatalf("New: error = %v", err)
 			} else {
+				t.Cleanup(c.Close)
 				if tc.early != nil {
 					tc.early(t, base)
 				}
@@ -312,6 +333,8 @@ var ignorePathname = check.MustAbs("/\x00")
 
 // cureMany cures many artifacts against a [Cache] and checks their outcomes.
 func cureMany(t *testing.T, c *pkg.Cache, steps []cureStep) {
+	t.Helper()
+
 	for _, step := range steps {
 		t.Log("cure step:", step.name)
 		if pathname, checksum, err := c.Cure(step.a); !reflect.DeepEqual(err, step.err) {
@@ -415,6 +438,13 @@ func TestCache(t *testing.T) {
 					0xa9, 0xc2, 0x08, 0xa1, 0x17, 0x17,
 				}, nil},
 
+				{"incomplete implementation", struct{ pkg.Artifact }{stubArtifact{
+					kind:   pkg.KindExec,
+					params: []byte("artifact overridden to be incomplete"),
+				}}, nil, pkg.Checksum{}, pkg.InvalidArtifactError(pkg.MustDecode(
+					"da4kLKa94g1wN2M0qcKflqgf2-Y2UL36iehhczqsIIW8G0LGvM7S8jjtnBc0ftB0",
+				))},
+
 				{"error passthrough", newStubFile(
 					pkg.KindHTTPGet,
 					pkg.ID{0xff, 1},
@@ -436,9 +466,14 @@ func TestCache(t *testing.T) {
 				)},
 			})
 
-			if c0, err := pkg.New(base); err != nil {
+			if c0, err := pkg.New(
+				t.Context(),
+				message.New(nil),
+				0, base,
+			); err != nil {
 				t.Fatalf("New: error = %v", err)
 			} else {
+				t.Cleanup(c.Close) // check doubled cancel
 				cureMany(t, c0, []cureStep{
 					{"cache hit ident", overrideIdent{
 						id: identifier,
@@ -455,6 +490,16 @@ func TestCache(t *testing.T) {
 						pkg.Encode(testdataChecksum),
 					), testdataChecksum, nil},
 				})
+
+				// cure after close
+				c.Close()
+				if _, _, err = c.Cure(stubArtifactF{
+					kind:   pkg.KindExec,
+					params: []byte("unreachable artifact cured after cancel"),
+					deps:   []pkg.Artifact{pkg.NewFile([]byte("unreachable dependency"))},
+				}); !reflect.DeepEqual(err, context.Canceled) {
+					t.Fatalf("(closed) Cure: error = %v", err)
+				}
 			}
 		}, pkg.MustDecode("St9rlE-mGZ5gXwiv_hzQ_B8bZP-UUvSNmf4nHUZzCMOumb6hKnheZSe0dmnuc4Q2")},
 
@@ -463,8 +508,8 @@ func TestCache(t *testing.T) {
 				binary.LittleEndian.AppendUint64(nil, pkg.TarGzip),
 				overrideIdent{testdataChecksum, stubArtifact{}},
 			)
-			makeSample := func(c *pkg.CureContext) error {
-				work := c.GetWorkDir()
+			makeSample := func(t *pkg.TContext) error {
+				work := t.GetWorkDir()
 				if err := os.Mkdir(work.String(), 0700); err != nil {
 					return err
 				}
@@ -565,15 +610,15 @@ func TestCache(t *testing.T) {
 
 				{"cure fault", overrideIdent{pkg.ID{0xff, 0}, stubArtifact{
 					kind: pkg.KindTar,
-					cure: func(c *pkg.CureContext) error {
-						return makeGarbage(c.GetWorkDir(), stub.UniqueError(0xcafe))
+					cure: func(t *pkg.TContext) error {
+						return makeGarbage(t.GetWorkDir(), stub.UniqueError(0xcafe))
 					},
 				}}, nil, pkg.Checksum{}, stub.UniqueError(0xcafe)},
 
 				{"checksum mismatch", overrideChecksum{pkg.Checksum{}, overrideIdent{pkg.ID{0xff, 1}, stubArtifact{
 					kind: pkg.KindTar,
-					cure: func(c *pkg.CureContext) error {
-						return makeGarbage(c.GetWorkDir(), nil)
+					cure: func(t *pkg.TContext) error {
+						return makeGarbage(t.GetWorkDir(), nil)
 					},
 				}}}, nil, pkg.Checksum{}, &pkg.ChecksumMismatchError{
 					Got: pkg.MustDecode(
@@ -592,8 +637,8 @@ func TestCache(t *testing.T) {
 
 				{"openFile directory", overrideIdent{pkg.ID{0xff, 3}, stubArtifact{
 					kind: pkg.KindTar,
-					cure: func(c *pkg.CureContext) error {
-						r, err := c.OpenFile(overrideChecksumFile{checksum: wantChecksum})
+					cure: func(t *pkg.TContext) error {
+						r, err := t.Open(overrideChecksumFile{checksum: wantChecksum})
 						if err != nil {
 							panic(err)
 						}
@@ -611,24 +656,24 @@ func TestCache(t *testing.T) {
 
 				{"no output", overrideIdent{pkg.ID{0xff, 4}, stubArtifact{
 					kind: pkg.KindTar,
-					cure: func(c *pkg.CureContext) error {
+					cure: func(t *pkg.TContext) error {
 						return nil
 					},
 				}}, nil, pkg.Checksum{}, pkg.NoOutputError{}},
 
 				{"file output", overrideIdent{pkg.ID{0xff, 5}, stubArtifact{
 					kind: pkg.KindTar,
-					cure: func(c *pkg.CureContext) error {
-						return os.WriteFile(c.GetWorkDir().String(), []byte{0}, 0400)
+					cure: func(t *pkg.TContext) error {
+						return os.WriteFile(t.GetWorkDir().String(), []byte{0}, 0400)
 					},
 				}}, nil, pkg.Checksum{}, errors.New("non-file artifact produced regular file")},
 
 				{"symlink output", overrideIdent{pkg.ID{0xff, 6}, stubArtifact{
 					kind: pkg.KindTar,
-					cure: func(c *pkg.CureContext) error {
+					cure: func(t *pkg.TContext) error {
 						return os.Symlink(
-							c.GetWorkDir().String(),
-							c.GetWorkDir().String(),
+							t.GetWorkDir().String(),
+							t.GetWorkDir().String(),
 						)
 					},
 				}}, nil, pkg.Checksum{}, pkg.InvalidFileModeError(
@@ -645,7 +690,7 @@ func TestCache(t *testing.T) {
 			go func() {
 				if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, stubArtifact{
 					kind: pkg.KindTar,
-					cure: func(c *pkg.CureContext) error {
+					cure: func(t *pkg.TContext) error {
 						close(ready)
 						<-n
 						return wantErr
@@ -677,9 +722,9 @@ func TestCache(t *testing.T) {
 
 				{"file output", overrideIdent{pkg.ID{0xff, 2}, stubArtifact{
 					kind: pkg.KindTar,
-					cure: func(c *pkg.CureContext) error {
+					cure: func(t *pkg.TContext) error {
 						return os.WriteFile(
-							c.GetWorkDir().String(),
+							t.GetWorkDir().String(),
 							[]byte{0},
 							0400,
 						)
@@ -785,6 +830,14 @@ func TestErrors(t *testing.T) {
 		err  error
 		want string
 	}{
+		{"InvalidLookupError", pkg.InvalidLookupError{
+			0xff, 0xf0,
+		}, "attempting to look up non-dependency artifact __AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"},
+
+		{"InvalidArtifactError", pkg.InvalidArtifactError{
+			0xff, 0xfd,
+		}, "artifact __0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA cannot be cured"},
+
 		{"ChecksumMismatchError", &pkg.ChecksumMismatchError{
 			Want: (pkg.Checksum)(bytes.Repeat([]byte{
 				0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f,
@@ -885,7 +938,11 @@ func TestNew(t *testing.T) {
 			Path: container.Nonexistent,
 			Err:  syscall.ENOENT,
 		}
-		if _, err := pkg.New(check.MustAbs(container.Nonexistent)); !reflect.DeepEqual(err, wantErr) {
+		if _, err := pkg.New(
+			t.Context(),
+			message.New(nil),
+			0, check.MustAbs(container.Nonexistent),
+		); !reflect.DeepEqual(err, wantErr) {
 			t.Errorf("New: error = %#v, want %#v", err, wantErr)
 		}
 	})
@@ -909,7 +966,11 @@ func TestNew(t *testing.T) {
 			Path: tempDir.Append("cache").String(),
 			Err:  syscall.EACCES,
 		}
-		if _, err := pkg.New(tempDir.Append("cache")); !reflect.DeepEqual(err, wantErr) {
+		if _, err := pkg.New(
+			t.Context(),
+			message.New(nil),
+			0, tempDir.Append("cache"),
+		); !reflect.DeepEqual(err, wantErr) {
 			t.Errorf("New: error = %#v, want %#v", err, wantErr)
 		}
 	})