internal/pkg: cache computed identifiers

This eliminates duplicate identifier computations. The new implementation also significantly reduces allocations while computing identifier for a large dependency tree.

Signed-off-by: Ophestra <cat@gensokyo.uk>

1 files changed, 51 insertions, 42 deletions

diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go
index f5e6dc5c..bb48dfab 100644
--- a/internal/pkg/pkg_test.go
+++ b/internal/pkg/pkg_test.go
@@ -6,7 +6,6 @@ import (
 	"context"
 	"crypto/sha512"
 	"encoding/base64"
-	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
@@ -18,6 +17,7 @@ import (
 	"reflect"
 	"syscall"
 	"testing"
+	"unique"
 	"unsafe"
 
 	"hakurei.app/container"
@@ -82,10 +82,10 @@ type stubArtifact struct {
 	cure func(t *pkg.TContext) error
 }
 
-func (a stubArtifact) Kind() pkg.Kind               { return a.kind }
-func (a stubArtifact) Params() []byte               { return a.params }
-func (a stubArtifact) Dependencies() []pkg.Artifact { return a.deps }
-func (a stubArtifact) Cure(t *pkg.TContext) error   { return a.cure(t) }
+func (a *stubArtifact) Kind() pkg.Kind               { return a.kind }
+func (a *stubArtifact) Params(ctx *pkg.IContext)     { ctx.GetHash().Write(a.params) }
+func (a *stubArtifact) Dependencies() []pkg.Artifact { return a.deps }
+func (a *stubArtifact) Cure(t *pkg.TContext) error   { return a.cure(t) }
 
 // A stubArtifactF implements [FloodArtifact] with hardcoded behaviour.
 type stubArtifactF struct {
@@ -96,10 +96,10 @@ type stubArtifactF struct {
 	cure func(f *pkg.FContext) error
 }
 
-func (a stubArtifactF) Kind() pkg.Kind               { return a.kind }
-func (a stubArtifactF) Params() []byte               { return a.params }
-func (a stubArtifactF) Dependencies() []pkg.Artifact { return a.deps }
-func (a stubArtifactF) Cure(f *pkg.FContext) error   { return a.cure(f) }
+func (a *stubArtifactF) Kind() pkg.Kind               { return a.kind }
+func (a *stubArtifactF) Params(ctx *pkg.IContext)     { ctx.GetHash().Write(a.params) }
+func (a *stubArtifactF) Dependencies() []pkg.Artifact { return a.deps }
+func (a *stubArtifactF) Cure(f *pkg.FContext) error   { return a.cure(f) }
 
 // A stubFile implements [File] with hardcoded behaviour.
 type stubFile struct {
@@ -109,7 +109,7 @@ type stubFile struct {
 	stubArtifact
 }
 
-func (a stubFile) Cure(context.Context) ([]byte, error) { return a.data, a.err }
+func (a *stubFile) Cure(context.Context) ([]byte, error) { return a.data, a.err }
 
 // newStubFile returns an implementation of [pkg.File] with hardcoded behaviour.
 func newStubFile(
@@ -119,7 +119,7 @@ func newStubFile(
 	data []byte,
 	err error,
 ) pkg.File {
-	f := overrideIdentFile{id, stubFile{data, err, stubArtifact{
+	f := overrideIdentFile{id, &stubFile{data, err, stubArtifact{
 		kind,
 		nil,
 		nil,
@@ -193,27 +193,38 @@ func TestIdent(t *testing.T) {
 	testCases := []struct {
 		name string
 		a    pkg.Artifact
-		want pkg.ID
+		want unique.Handle[pkg.ID]
 	}{
-		{"tar", stubArtifact{
+		{"tar", &stubArtifact{
 			pkg.KindTar,
 			[]byte{pkg.TarGzip, 0, 0, 0, 0, 0, 0, 0},
 			[]pkg.Artifact{
-				overrideIdent{pkg.ID{}, stubArtifact{}},
+				overrideIdent{pkg.ID{}, new(stubArtifact)},
 			},
 			nil,
-		}, pkg.MustDecode(
+		}, unique.Make[pkg.ID](pkg.MustDecode(
 			"HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY",
-		)},
+		))},
 	}
+
+	msg := message.New(log.New(os.Stderr, "ident: ", 0))
+	msg.SwapVerbose(true)
+	var cache *pkg.Cache
+	if a, err := check.NewAbs(t.TempDir()); err != nil {
+		t.Fatal(err)
+	} else if cache, err = pkg.New(t.Context(), msg, 0, a); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(cache.Close)
+
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 
-			if got := pkg.Ident(tc.a); got != tc.want {
+			if got := cache.Ident(tc.a); got != tc.want {
 				t.Errorf("Ident: %s, want %s",
-					pkg.Encode(got),
-					pkg.Encode(tc.want),
+					pkg.Encode(got.Value()),
+					pkg.Encode(tc.want.Value()),
 				)
 			}
 		})
@@ -438,7 +449,7 @@ func TestCache(t *testing.T) {
 					0xa9, 0xc2, 0x08, 0xa1, 0x17, 0x17,
 				}, nil},
 
-				{"incomplete implementation", struct{ pkg.Artifact }{stubArtifact{
+				{"incomplete implementation", struct{ pkg.Artifact }{&stubArtifact{
 					kind:   pkg.KindExec,
 					params: []byte("artifact overridden to be incomplete"),
 				}}, nil, pkg.Checksum{}, pkg.InvalidArtifactError(pkg.MustDecode(
@@ -459,7 +470,7 @@ func TestCache(t *testing.T) {
 					nil, nil,
 				), nil, pkg.Checksum{}, stub.UniqueError(0xcafe)},
 
-				{"cache hit bad type", overrideChecksum{testdataChecksum, overrideIdent{pkg.ID{0xff, 2}, stubArtifact{
+				{"cache hit bad type", overrideChecksum{testdataChecksum, overrideIdent{pkg.ID{0xff, 2}, &stubArtifact{
 					kind: pkg.KindTar,
 				}}}, nil, pkg.Checksum{}, pkg.InvalidFileModeError(
 					0400,
@@ -493,7 +504,7 @@ func TestCache(t *testing.T) {
 
 				// cure after close
 				c.Close()
-				if _, _, err = c.Cure(stubArtifactF{
+				if _, _, err = c.Cure(&stubArtifactF{
 					kind:   pkg.KindExec,
 					params: []byte("unreachable artifact cured after cancel"),
 					deps:   []pkg.Artifact{pkg.NewFile("", []byte("unreachable dependency"))},
@@ -504,9 +515,8 @@ func TestCache(t *testing.T) {
 		}, pkg.MustDecode("St9rlE-mGZ5gXwiv_hzQ_B8bZP-UUvSNmf4nHUZzCMOumb6hKnheZSe0dmnuc4Q2")},
 
 		{"directory", nil, func(t *testing.T, base *check.Absolute, c *pkg.Cache) {
-			id := pkg.KindTar.Ident(
-				binary.LittleEndian.AppendUint64(nil, pkg.TarGzip),
-				overrideIdent{testdataChecksum, stubArtifact{}},
+			id := pkg.MustDecode(
+				"HnySzeLQvSBZuTUcvfmLEX_OmH4yJWWH788NxuLuv7kVn8_uPM6Ks4rqFWM2NZJY",
 			)
 			makeSample := func(t *pkg.TContext) error {
 				work := t.GetWorkDir()
@@ -545,9 +555,8 @@ func TestCache(t *testing.T) {
 				pkg.Encode(id),
 			)
 
-			id0 := pkg.KindTar.Ident(
-				binary.LittleEndian.AppendUint64(nil, pkg.TarGzip),
-				overrideIdent{pkg.ID{}, stubArtifact{}},
+			id0 := pkg.MustDecode(
+				"Zx5ZG9BAwegNT3zQwCySuI2ktCXxNgxirkGLFjW4FW06PtojYVaCdtEw8yuntPLa",
 			)
 			wantPathname0 := base.Append(
 				"identifier",
@@ -594,28 +603,28 @@ func TestCache(t *testing.T) {
 			}
 
 			cureMany(t, c, []cureStep{
-				{"initial directory", overrideChecksum{wantChecksum, overrideIdent{id, stubArtifact{
+				{"initial directory", overrideChecksum{wantChecksum, overrideIdent{id, &stubArtifact{
 					kind: pkg.KindTar,
 					cure: makeSample,
 				}}}, wantPathname, wantChecksum, nil},
 
-				{"identical identifier", overrideChecksum{wantChecksum, overrideIdent{id, stubArtifact{
+				{"identical identifier", overrideChecksum{wantChecksum, overrideIdent{id, &stubArtifact{
 					kind: pkg.KindTar,
 				}}}, wantPathname, wantChecksum, nil},
 
-				{"identical checksum", overrideIdent{id0, stubArtifact{
+				{"identical checksum", overrideIdent{id0, &stubArtifact{
 					kind: pkg.KindTar,
 					cure: makeSample,
 				}}, wantPathname0, wantChecksum, nil},
 
-				{"cure fault", overrideIdent{pkg.ID{0xff, 0}, stubArtifact{
+				{"cure fault", overrideIdent{pkg.ID{0xff, 0}, &stubArtifact{
 					kind: pkg.KindTar,
 					cure: func(t *pkg.TContext) error {
 						return makeGarbage(t.GetWorkDir(), stub.UniqueError(0xcafe))
 					},
 				}}, nil, pkg.Checksum{}, stub.UniqueError(0xcafe)},
 
-				{"checksum mismatch", overrideChecksum{pkg.Checksum{}, overrideIdent{pkg.ID{0xff, 1}, stubArtifact{
+				{"checksum mismatch", overrideChecksum{pkg.Checksum{}, overrideIdent{pkg.ID{0xff, 1}, &stubArtifact{
 					kind: pkg.KindTar,
 					cure: func(t *pkg.TContext) error {
 						return makeGarbage(t.GetWorkDir(), nil)
@@ -635,7 +644,7 @@ func TestCache(t *testing.T) {
 					fs.ModeDir | 0500,
 				)},
 
-				{"openFile directory", overrideIdent{pkg.ID{0xff, 3}, stubArtifact{
+				{"openFile directory", overrideIdent{pkg.ID{0xff, 3}, &stubArtifact{
 					kind: pkg.KindTar,
 					cure: func(t *pkg.TContext) error {
 						r, err := t.Open(overrideChecksumFile{checksum: wantChecksum})
@@ -654,21 +663,21 @@ func TestCache(t *testing.T) {
 					Err: syscall.EISDIR,
 				}},
 
-				{"no output", overrideIdent{pkg.ID{0xff, 4}, stubArtifact{
+				{"no output", overrideIdent{pkg.ID{0xff, 4}, &stubArtifact{
 					kind: pkg.KindTar,
 					cure: func(t *pkg.TContext) error {
 						return nil
 					},
 				}}, nil, pkg.Checksum{}, pkg.NoOutputError{}},
 
-				{"file output", overrideIdent{pkg.ID{0xff, 5}, stubArtifact{
+				{"file output", overrideIdent{pkg.ID{0xff, 5}, &stubArtifact{
 					kind: pkg.KindTar,
 					cure: func(t *pkg.TContext) error {
 						return os.WriteFile(t.GetWorkDir().String(), []byte{0}, 0400)
 					},
 				}}, nil, pkg.Checksum{}, errors.New("non-file artifact produced regular file")},
 
-				{"symlink output", overrideIdent{pkg.ID{0xff, 6}, stubArtifact{
+				{"symlink output", overrideIdent{pkg.ID{0xff, 6}, &stubArtifact{
 					kind: pkg.KindTar,
 					cure: func(t *pkg.TContext) error {
 						return os.Symlink(
@@ -688,7 +697,7 @@ func TestCache(t *testing.T) {
 			wantErr := stub.UniqueError(0xcafe)
 			n, ready := make(chan struct{}), make(chan struct{})
 			go func() {
-				if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, stubArtifact{
+				if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, &stubArtifact{
 					kind: pkg.KindTar,
 					cure: func(t *pkg.TContext) error {
 						close(ready)
@@ -703,7 +712,7 @@ func TestCache(t *testing.T) {
 			<-ready
 			wCureDone := make(chan struct{})
 			go func() {
-				if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, stubArtifact{
+				if _, _, err := c.Cure(overrideIdent{pkg.ID{0xff}, &stubArtifact{
 					kind: pkg.KindTar,
 				}}); !reflect.DeepEqual(err, wantErr) {
 					panic(fmt.Sprintf("Cure: error = %v, want %v", err, wantErr))
@@ -720,7 +729,7 @@ func TestCache(t *testing.T) {
 					nil, stub.UniqueError(0xbad),
 				), nil, pkg.Checksum{}, stub.UniqueError(0xbad)},
 
-				{"file output", overrideIdent{pkg.ID{0xff, 2}, stubArtifact{
+				{"file output", overrideIdent{pkg.ID{0xff, 2}, &stubArtifact{
 					kind: pkg.KindTar,
 					cure: func(t *pkg.TContext) error {
 						return os.WriteFile(
@@ -745,8 +754,8 @@ func TestCache(t *testing.T) {
 			identPending := reflect.NewAt(
 				identPendingVal.Type(),
 				unsafe.Pointer(identPendingVal.UnsafeAddr()),
-			).Elem().Interface().(map[pkg.ID]<-chan struct{})
-			notify := identPending[pkg.ID{0xff}]
+			).Elem().Interface().(map[unique.Handle[pkg.ID]]<-chan struct{})
+			notify := identPending[unique.Make(pkg.ID{0xff})]
 			go close(n)
 			<-notify
 			<-wCureDone