internal/pkg: content-based dependency substitution

This change introduces a new fast path for FloodArtifact. It is taken when a curing artifact has identical-by-content controlled relevant inputs and are otherwise identical to an already-cured artifact.

Signed-off-by: Ophestra <cat@gensokyo.uk>

1 files changed, 103 insertions, 0 deletions

diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go
index 75f14a0e..060dea6d 100644
--- a/internal/pkg/pkg_test.go
+++ b/internal/pkg/pkg_test.go
@@ -392,6 +392,12 @@ type cacheTestCase struct {
 	want  expectsFS
 }
 
+const (
+	// checkDestroySubstitutes arranges for substitutes to be destroyed before
+	// measurement during checkWithCache.
+	checkDestroySubstitutes = 1 << (iota + 32)
+)
+
 // checkWithCache runs a slice of cacheTestCase.
 func checkWithCache(t *testing.T, testCases []cacheTestCase) {
 	t.Helper()
@@ -472,6 +478,16 @@ func checkWithCache(t *testing.T, testCases []cacheTestCase) {
 				}
 			}
 
+			// destroy non-deterministic substitutes
+			if tc.flags&checkDestroySubstitutes != 0 {
+				substitute := base.Append("substitute")
+				if err := os.RemoveAll(substitute.String()); err != nil {
+					t.Fatal(err)
+				} else if err = os.Mkdir(substitute.String(), 0700); err != nil {
+					t.Fatal(err)
+				}
+			}
+
 			// destroy non-deterministic status files
 			if err := os.RemoveAll(base.Append("status").String()); err != nil {
 				t.Fatal(err)
@@ -788,6 +804,8 @@ func TestCache(t *testing.T) {
 			"identifier/cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq")},
 			"identifier/deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/0bSFPu5Tnd-2Jj0Mv6co23PW2t3BmHc7eLFj9TgY3eIBg8zislo7xZYNBqovVLcq")},
 
+			"substitute": {Mode: fs.ModeDir | 0700},
+
 			"work": {Mode: fs.ModeDir | 0700},
 		}},
 
@@ -972,11 +990,81 @@ func TestCache(t *testing.T) {
 				}}, nil, nil, pkg.InvalidFileModeError(
 					fs.ModeSymlink | 0777,
 				)},
+
+				{"alternative", &stubArtifactF{
+					kind:   pkg.KindExec,
+					params: []byte("substitutable artifact"),
+					deps: []pkg.Artifact{newStubFile(
+						pkg.KindHTTPGet,
+						pkg.ID{0xff, 8},
+						nil,
+						[]byte("substitutable dependency"),
+						nil,
+					)},
+
+					cure: func(f *pkg.FContext) error {
+						return makeSample(&f.TContext)
+					},
+				}, base.Append(
+					"identifier",
+					"xMDWovje7OfyIaDy_2VnjpKxRqSOQ_LoeD946t-3WsS2V2SeMJ7nDGrNfpa4Pbc-",
+				), want, nil},
+
+				{"substitutable", &stubArtifactF{
+					kind:   pkg.KindExec,
+					params: []byte("substitutable artifact"),
+					deps: []pkg.Artifact{newStubFile(
+						pkg.KindHTTPGet,
+						pkg.ID{0xff, 10},
+						nil,
+						[]byte("substitutable dependency"),
+						nil,
+					)},
+
+					cure: func(f *pkg.FContext) error {
+						panic("substitution missed")
+					},
+				}, base.Append(
+					"identifier",
+					"k2ilgG5KQ9NXnMoT2oB6NdwOnSPRn_H24oXQc4l6qOYIxIG9XfuEczeyrR8UEv_f",
+				), want, nil},
 			})
+
+			if c0, err := unsafeOpen(
+				t.Context(),
+				message.New(nil),
+				0, 0, 0, base, false,
+			); err != nil {
+				t.Fatalf("open: error = %v", err)
+			} else {
+				t.Cleanup(c.Close)
+				cureMany(t, c0, []cureStep{
+					{"substitutable", &stubArtifactF{
+						kind:   pkg.KindExec,
+						params: []byte("substitutable artifact"),
+						deps: []pkg.Artifact{newStubFile(
+							pkg.KindHTTPGet,
+							pkg.ID{0xff, 0xff, 0xfd, 0xfd},
+							nil,
+							[]byte("substitutable dependency"),
+							nil,
+						)},
+
+						cure: func(f *pkg.FContext) error {
+							panic("substitution missed")
+						},
+					}, base.Append(
+						"identifier",
+						"_EmV5nsYZ2UWHgRmLDMU8i-rJWDx-kv5_1pFrzQI7vMMCM5mAXivO8UZtVfOqMR_",
+					), want, nil},
+				})
+
+			}
 		}, expectsFS{
 			".": {Mode: fs.ModeDir | 0700},
 
 			"checksum": {Mode: fs.ModeDir | 0700},
+			"checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO":                {Mode: 0400, Data: []byte("substitutable dependency")},
 			"checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b":                {Mode: fs.ModeDir | 0500},
 			"checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/check":          {Mode: 0400, Data: []byte{0, 0}},
 			"checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b/lib":            {Mode: fs.ModeDir | 0700},
@@ -986,6 +1074,15 @@ func TestCache(t *testing.T) {
 			"identifier": {Mode: fs.ModeDir | 0700},
 			"identifier/HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")},
 			"identifier/Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")},
+			"identifier/_EmV5nsYZ2UWHgRmLDMU8i-rJWDx-kv5_1pFrzQI7vMMCM5mAXivO8UZtVfOqMR_": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")},
+			"identifier/___9_QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO")},
+			"identifier/_wgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO")},
+			"identifier/_woAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/7lfQ4QwSpV8nw7IDh0JiQ_jqUPrPv3_Vfie034RxsSy-cy4vO8DVvxgpx2LW08oO")},
+			"identifier/k2ilgG5KQ9NXnMoT2oB6NdwOnSPRn_H24oXQc4l6qOYIxIG9XfuEczeyrR8UEv_f": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")},
+			"identifier/xMDWovje7OfyIaDy_2VnjpKxRqSOQ_LoeD946t-3WsS2V2SeMJ7nDGrNfpa4Pbc-": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")},
+
+			"substitute": {Mode: fs.ModeDir | 0700},
+			"substitute/OyBGorh72Z9kVw35JUa8FbqDbpR4DqT-MX1jic0uKN5PdYmUBiAF38BRsIRnBigf": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/qRN6in76LndiiOZJheHkwyW8UT1N5-f-bXvHfDvwrMw2fSkOoZdh8pWE1qhLk65b")},
 
 			"work": {Mode: fs.ModeDir | 0700},
 		}},
@@ -1061,6 +1158,7 @@ func TestCache(t *testing.T) {
 			".":          {Mode: fs.ModeDir | 0700},
 			"checksum":   {Mode: fs.ModeDir | 0700},
 			"identifier": {Mode: fs.ModeDir | 0700},
+			"substitute": {Mode: fs.ModeDir | 0700},
 			"work":       {Mode: fs.ModeDir | 0700},
 		}},
 
@@ -1119,6 +1217,7 @@ func TestCache(t *testing.T) {
 			".":          {Mode: fs.ModeDir | 0700},
 			"checksum":   {Mode: fs.ModeDir | 0700},
 			"identifier": {Mode: fs.ModeDir | 0700},
+			"substitute": {Mode: fs.ModeDir | 0700},
 			"work":       {Mode: fs.ModeDir | 0700},
 		}},
 
@@ -1180,6 +1279,8 @@ func TestCache(t *testing.T) {
 			"identifier/_wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M")},
 			"identifier/_wEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA": {Mode: fs.ModeSymlink | 0777, Data: []byte("../checksum/Aubi5EG4_Y8DhL9bQ3Q4HFBhLRF7X5gt9D3CNCQfT-TeBtlRXc7Zi_JYZEMoCC7M")},
 
+			"substitute": {Mode: fs.ModeDir | 0700},
+
 			"work": {Mode: fs.ModeDir | 0700},
 		}},
 
@@ -1230,6 +1331,7 @@ func TestCache(t *testing.T) {
 			".":          {Mode: fs.ModeDir | 0700},
 			"checksum":   {Mode: fs.ModeDir | 0700},
 			"identifier": {Mode: fs.ModeDir | 0700},
+			"substitute": {Mode: fs.ModeDir | 0700},
 			"work":       {Mode: fs.ModeDir | 0700},
 		}},
 	}
@@ -1514,6 +1616,7 @@ func TestDependencyCureErrorEarly(t *testing.T) {
 			".":          {Mode: fs.ModeDir | 0700},
 			"checksum":   {Mode: fs.ModeDir | 0700},
 			"identifier": {Mode: fs.ModeDir | 0700},
+			"substitute": {Mode: fs.ModeDir | 0700},
 			"work":       {Mode: fs.ModeDir | 0700},
 		}},
 	})