internal/pkg: move dependency flooding to cache

This imposes a hard upper limit to concurrency during dependency satisfaction and moves all dependency-related code out of individual implementations of Artifact. This change also includes ctx and msg as part of Cache.

Signed-off-by: Ophestra <cat@gensokyo.uk>

1 files changed, 66 insertions, 49 deletions

diff --git a/internal/pkg/exec_test.go b/internal/pkg/exec_test.go
index d0a54576..e6928582 100644
--- a/internal/pkg/exec_test.go
+++ b/internal/pkg/exec_test.go
@@ -5,16 +5,15 @@ package pkg_test
 import (
 	_ "embed"
 	"errors"
-	"log"
 	"os"
 	"os/exec"
+	"slices"
 	"testing"
 
 	"hakurei.app/container/check"
 	"hakurei.app/container/stub"
 	"hakurei.app/hst"
 	"hakurei.app/internal/pkg"
-	"hakurei.app/message"
 )
 
 // testtoolBin is the container test tool binary made available to the
@@ -35,14 +34,8 @@ func TestExec(t *testing.T) {
 			c.SetStrict(true)
 			testtool, testtoolDestroy := newTesttool()
 
-			msg := message.New(log.New(os.Stderr, "container: ", 0))
-			msg.SwapVerbose(testing.Verbose())
-
 			cureMany(t, c, []cureStep{
 				{"container", pkg.NewExec(
-					t.Context(),
-					msg,
-					0,
 					nil,
 					pkg.AbsWork,
 					[]string{"HAKUREI_TEST=1"},
@@ -58,17 +51,14 @@ func TestExec(t *testing.T) {
 					pkg.MustPath("/.hakurei", false, stubArtifact{
 						kind:   pkg.KindTar,
 						params: []byte("empty directory"),
-						cure: func(c *pkg.CureContext) error {
-							return os.MkdirAll(c.GetWorkDir().String(), 0700)
+						cure: func(t *pkg.TContext) error {
+							return os.MkdirAll(t.GetWorkDir().String(), 0700)
 						},
 					}),
 					pkg.MustPath("/opt", false, testtool),
 				), ignorePathname, wantChecksumOffline, nil},
 
 				{"error passthrough", pkg.NewExec(
-					t.Context(),
-					msg,
-					0,
 					nil,
 					pkg.AbsWork,
 					[]string{"HAKUREI_TEST=1"},
@@ -78,16 +68,13 @@ func TestExec(t *testing.T) {
 					pkg.MustPath("/proc/nonexistent", false, stubArtifact{
 						kind:   pkg.KindTar,
 						params: []byte("doomed artifact"),
-						cure: func(c *pkg.CureContext) error {
+						cure: func(t *pkg.TContext) error {
 							return stub.UniqueError(0xcafe)
 						},
 					}),
 				), nil, pkg.Checksum{}, errors.Join(stub.UniqueError(0xcafe))},
 
 				{"invalid paths", pkg.NewExec(
-					t.Context(),
-					msg,
-					0,
 					nil,
 					pkg.AbsWork,
 					[]string{"HAKUREI_TEST=1"},
@@ -101,9 +88,6 @@ func TestExec(t *testing.T) {
 			// check init failure passthrough
 			var exitError *exec.ExitError
 			if _, _, err := c.Cure(pkg.NewExec(
-				t.Context(),
-				msg,
-				0,
 				nil,
 				pkg.AbsWork,
 				nil,
@@ -121,17 +105,11 @@ func TestExec(t *testing.T) {
 			c.SetStrict(true)
 			testtool, testtoolDestroy := newTesttool()
 
-			msg := message.New(log.New(os.Stderr, "container: ", 0))
-			msg.SwapVerbose(testing.Verbose())
-
 			wantChecksum := pkg.MustDecode(
 				"a1F_i9PVQI4qMcoHgTQkORuyWLkC1GLIxOhDt2JpU1NGAxWc5VJzdlfRK-PYBh3W",
 			)
 			cureMany(t, c, []cureStep{
 				{"container", pkg.NewExec(
-					t.Context(),
-					msg,
-					0,
 					&wantChecksum,
 					pkg.AbsWork,
 					[]string{"HAKUREI_TEST=1"},
@@ -147,8 +125,8 @@ func TestExec(t *testing.T) {
 					pkg.MustPath("/.hakurei", false, stubArtifact{
 						kind:   pkg.KindTar,
 						params: []byte("empty directory"),
-						cure: func(c *pkg.CureContext) error {
-							return os.MkdirAll(c.GetWorkDir().String(), 0700)
+						cure: func(t *pkg.TContext) error {
+							return os.MkdirAll(t.GetWorkDir().String(), 0700)
 						},
 					}),
 					pkg.MustPath("/opt", false, testtool),
@@ -162,14 +140,8 @@ func TestExec(t *testing.T) {
 			c.SetStrict(true)
 			testtool, testtoolDestroy := newTesttool()
 
-			msg := message.New(log.New(os.Stderr, "container: ", 0))
-			msg.SwapVerbose(testing.Verbose())
-
 			cureMany(t, c, []cureStep{
 				{"container", pkg.NewExec(
-					t.Context(),
-					msg,
-					0,
 					nil,
 					pkg.AbsWork,
 					[]string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"},
@@ -179,8 +151,8 @@ func TestExec(t *testing.T) {
 					pkg.MustPath("/", true, stubArtifact{
 						kind:   pkg.KindTar,
 						params: []byte("empty directory"),
-						cure: func(c *pkg.CureContext) error {
-							return os.MkdirAll(c.GetWorkDir().String(), 0700)
+						cure: func(t *pkg.TContext) error {
+							return os.MkdirAll(t.GetWorkDir().String(), 0700)
 						},
 					}),
 					pkg.MustPath("/opt", false, testtool),
@@ -194,14 +166,8 @@ func TestExec(t *testing.T) {
 			c.SetStrict(true)
 			testtool, testtoolDestroy := newTesttool()
 
-			msg := message.New(log.New(os.Stderr, "container: ", 0))
-			msg.SwapVerbose(testing.Verbose())
-
 			cureMany(t, c, []cureStep{
 				{"container", pkg.NewExec(
-					t.Context(),
-					msg,
-					0,
 					nil,
 					pkg.AbsWork,
 					[]string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"},
@@ -211,14 +177,14 @@ func TestExec(t *testing.T) {
 					pkg.MustPath("/", true, stubArtifact{
 						kind:   pkg.KindTar,
 						params: []byte("empty directory"),
-						cure: func(c *pkg.CureContext) error {
-							return os.MkdirAll(c.GetWorkDir().String(), 0700)
+						cure: func(t *pkg.TContext) error {
+							return os.MkdirAll(t.GetWorkDir().String(), 0700)
 						},
 					}), pkg.MustPath("/work/", false, stubArtifact{
 						kind:   pkg.KindTar,
 						params: []byte("empty directory"),
-						cure: func(c *pkg.CureContext) error {
-							return os.MkdirAll(c.GetWorkDir().String(), 0700)
+						cure: func(t *pkg.TContext) error {
+							return os.MkdirAll(t.GetWorkDir().String(), 0700)
 						},
 					}), pkg.Path(pkg.AbsWork, false /* ignored */, testtool),
 				), ignorePathname, wantChecksumOffline, nil},
@@ -226,6 +192,57 @@ func TestExec(t *testing.T) {
 
 			testtoolDestroy(t, base, c)
 		}, pkg.MustDecode("RibudsoY1X4_dtshfvL5LYfCPcxVnP0ikOn3yBHzOrt6BpevQiANLJF6Xua76-gM")},
+
+		{"multiple layers", nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
+			c.SetStrict(true)
+			testtool, testtoolDestroy := newTesttool()
+
+			cureMany(t, c, []cureStep{
+				{"container", pkg.NewExec(
+					nil,
+					pkg.AbsWork,
+					[]string{"HAKUREI_TEST=1", "HAKUREI_ROOT=1"},
+					check.MustAbs("/opt/bin/testtool"),
+					[]string{"testtool", "layers"},
+
+					pkg.MustPath("/", true, stubArtifact{
+						kind:   pkg.KindTar,
+						params: []byte("empty directory"),
+						cure: func(t *pkg.TContext) error {
+							return os.MkdirAll(t.GetWorkDir().String(), 0700)
+						},
+					}, stubArtifactF{
+						kind:   pkg.KindExec,
+						params: []byte("test sample with dependencies"),
+
+						deps: slices.Repeat([]pkg.Artifact{newStubFile(
+							pkg.KindHTTPGet,
+							pkg.ID{0xfe, 0},
+							nil,
+							nil, nil,
+						), stubArtifact{
+							kind:   pkg.KindTar,
+							params: []byte("empty directory"),
+
+							// this is queued and might run instead of the other
+							// one so do not leave it as nil
+							cure: func(t *pkg.TContext) error {
+								return os.MkdirAll(t.GetWorkDir().String(), 0700)
+							},
+						}}, 1<<5 /* concurrent cache hits */), cure: func(f *pkg.FContext) error {
+							work := f.GetWorkDir()
+							if err := os.MkdirAll(work.String(), 0700); err != nil {
+								return err
+							}
+							return os.WriteFile(work.Append("check").String(), []byte("layers"), 0400)
+						},
+					}),
+					pkg.MustPath("/opt", false, testtool),
+				), ignorePathname, wantChecksumOffline, nil},
+			})
+
+			testtoolDestroy(t, base, c)
+		}, pkg.MustDecode("a05V6iVGAfLO8aG0VSSFyB1QvnzJoMmGbMX6ud8CMMbatvyBv90_xGn1qWEIsjkQ")},
 	})
 }
 
@@ -238,15 +255,15 @@ func newTesttool() (
 	// testtoolBin is built during go:generate and is not deterministic
 	testtool = overrideIdent{pkg.ID{0xfe, 0xff}, stubArtifact{
 		kind: pkg.KindTar,
-		cure: func(c *pkg.CureContext) error {
-			work := c.GetWorkDir()
+		cure: func(t *pkg.TContext) error {
+			work := t.GetWorkDir()
 			if err := os.MkdirAll(
 				work.Append("bin").String(),
 				0700,
 			); err != nil {
 				return err
 			}
-			return os.WriteFile(c.GetWorkDir().Append(
+			return os.WriteFile(t.GetWorkDir().Append(
 				"bin",
 				"testtool",
 			).String(), testtoolBin, 0500)