From 5936e6a4aab541a437fa9464468b51f91d019a2a Mon Sep 17 00:00:00 2001 From: Ophestra Date: Fri, 16 Jan 2026 02:09:12 +0900 Subject: internal/pkg: parallelise scrub This significantly improves scrubbing performance. Since the cache directory structure is friendly to simultaneous access, this is possible without synchronisation. Signed-off-by: Ophestra --- internal/pkg/pkg_test.go | 142 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 106 insertions(+), 36 deletions(-) (limited to 'internal/pkg/pkg_test.go') diff --git a/internal/pkg/pkg_test.go b/internal/pkg/pkg_test.go index bb48dfab..7e22aa6f 100644 --- a/internal/pkg/pkg_test.go +++ b/internal/pkg/pkg_test.go @@ -22,6 +22,7 @@ import ( "hakurei.app/container" "hakurei.app/container/check" + "hakurei.app/container/fhs" "hakurei.app/container/stub" "hakurei.app/internal/pkg" "hakurei.app/message" @@ -281,7 +282,7 @@ func checkWithCache(t *testing.T, testCases []cacheTestCase) { tc.early(t, base) } tc.f(t, base, c) - scrubFunc = c.Scrub + scrubFunc = func() error { return c.Scrub(1 << 7) } } var restoreTemp bool @@ -293,7 +294,8 @@ func checkWithCache(t *testing.T, testCases []cacheTestCase) { restoreTemp = true } - if checksum, err := pkg.HashDir(base); err != nil { + var checksum pkg.Checksum + if err := pkg.HashDir(&checksum, base); err != nil { t.Fatalf("HashDir: error = %v", err) } else if checksum != tc.want { t.Fatalf("HashDir: %v", &pkg.ChecksumMismatchError{ @@ -316,7 +318,7 @@ func checkWithCache(t *testing.T, testCases []cacheTestCase) { } // validate again to make sure scrub did not condemn anything - if checksum, err := pkg.HashDir(base); err != nil { + if err := pkg.HashDir(&checksum, base); err != nil { t.Fatalf("HashDir: error = %v", err) } else if checksum != tc.want { t.Fatalf("(scrubbed) HashDir: %v", &pkg.ChecksumMismatchError{ @@ -364,6 +366,49 @@ func cureMany(t *testing.T, c *pkg.Cache, steps []cureStep) { } } +// newWantScrubError returns the address to a new [ScrubError] for base. +func newWantScrubError(base *check.Absolute) *pkg.ScrubError { + return &pkg.ScrubError{ + ChecksumMismatches: []pkg.ChecksumMismatchError{ + {Got: pkg.MustDecode( + "vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", + ), Want: pkg.Checksum{0xff, 0}}, + }, + DanglingIdentifiers: []pkg.ID{ + {0xfe, 0}, + {0xfe, 0xfe}, + {0xfe, 0xff}, + }, + Errs: map[unique.Handle[string]][]error{ + base.Append("checksum", "__8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA").Handle(): { + pkg.InvalidFileModeError(fs.ModeSymlink), + }, + + base.Append("checksum", "invalid").Handle(): { + base64.CorruptInputError(4), + }, + + base.Append("nonexistent").Handle(): { + base64.CorruptInputError(8), + }, + + base.Append("identifier", pkg.Encode(pkg.ID{0xfe, 0xff})).Handle(): { + &os.PathError{ + Op: "readlink", + Path: base.Append( + "identifier", + pkg.Encode(pkg.ID{0xfe, 0xff}), + ).String(), + Err: syscall.EINVAL, + }, + }, + base.Append("identifier", "invalid").Handle(): { + base64.CorruptInputError(4), + }, + }, + } +} + func TestCache(t *testing.T) { t.Parallel() @@ -744,9 +789,11 @@ func TestCache(t *testing.T) { }) wantErrScrub := &pkg.ScrubError{ - Errs: []error{errors.New("scrub began with pending artifacts")}, + Errs: map[unique.Handle[string]][]error{ + base.Handle(): {errors.New("scrub began with pending artifacts")}, + }, } - if err := c.Scrub(); !reflect.DeepEqual(err, wantErrScrub) { + if err := c.Scrub(1 << 6); !reflect.DeepEqual(err, wantErrScrub) { t.Fatalf("Scrub: error = %#v, want %#v", err, wantErrScrub) } @@ -800,30 +847,8 @@ func TestCache(t *testing.T) { } } - wantErr := &pkg.ScrubError{ - ChecksumMismatches: []pkg.ChecksumMismatchError{ - {Got: pkg.MustDecode( - "vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", - ), Want: pkg.Checksum{0xff, 0}}, - }, - DanglingIdentifiers: []pkg.ID{ - {0xfe, 0}, - {0xfe, 0xfe}, - {0xfe, 0xff}, - }, - Errs: []error{ - pkg.InvalidFileModeError(fs.ModeSymlink), - base64.CorruptInputError(4), - base64.CorruptInputError(8), - &os.PathError{ - Op: "readlink", - Path: base.Append("identifier", pkg.Encode(pkg.ID{0xfe, 0xff})).String(), - Err: syscall.EINVAL, - }, - base64.CorruptInputError(4), - }, - } - if err := c.Scrub(); !reflect.DeepEqual(err, wantErr) { + wantErr := newWantScrubError(base) + if err := c.Scrub(1 << 6); !reflect.DeepEqual(err, wantErr) { t.Fatalf("Scrub: error =\n%s\nwant\n%s", err, wantErr) } }, pkg.MustDecode("E4vEZKhCcL2gPZ2Tt59FS3lDng-d_2SKa2i5G_RbDfwGn6EemptFaGLPUDiOa94C")}, @@ -890,6 +915,48 @@ func TestScrubError(t *testing.T) { want string unwrap []error }{ + {"sample", *newWantScrubError( + fhs.AbsVarLib.Append("cure"), + ), `checksum mismatches: +got vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX instead of _wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + +dangling identifiers: +_gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +_v4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + +errors during scrub: + /var/lib/cure/checksum/__8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: + artifact did not produce a regular file or directory + /var/lib/cure/checksum/invalid: + illegal base64 data at input byte 4 + /var/lib/cure/identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: + readlink /var/lib/cure/identifier/_v8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: invalid argument + /var/lib/cure/identifier/invalid: + illegal base64 data at input byte 4 + /var/lib/cure/nonexistent: + illegal base64 data at input byte 8 +`, []error{ + &pkg.ChecksumMismatchError{Got: pkg.MustDecode( + "vsAhtPNo4waRNOASwrQwcIPTqb3SBuJOXw2G4T1mNmVZM-wrQTRllmgXqcIIoRcX", + ), Want: pkg.Checksum{0xff, 0}}, + + pkg.InvalidFileModeError(fs.ModeSymlink), + base64.CorruptInputError(4), + + &os.PathError{ + Op: "readlink", + Path: fhs.AbsVarLib.Append("cure").Append( + "identifier", + pkg.Encode(pkg.ID{0xfe, 0xff}), + ).String(), + Err: syscall.EINVAL, + }, + + base64.CorruptInputError(4), + base64.CorruptInputError(8), + }}, + {"full", pkg.ScrubError{ ChecksumMismatches: []pkg.ChecksumMismatchError{ {Want: pkg.MustDecode("CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN")}, @@ -898,10 +965,12 @@ func TestScrubError(t *testing.T) { (pkg.ID)(bytes.Repeat([]byte{0x75, 0xe6, 0x9d, 0x6d, 0xe7, 0x9f}, 8)), (pkg.ID)(bytes.Repeat([]byte{0x71, 0xa7, 0xde, 0x6d, 0xa6, 0xde}, 8)), }, - Errs: []error{ - stub.UniqueError(0xcafe), - stub.UniqueError(0xbad), - stub.UniqueError(0xff), + Errs: map[unique.Handle[string]][]error{ + unique.Make("/proc/nonexistent"): { + stub.UniqueError(0xcafe), + stub.UniqueError(0xbad), + stub.UniqueError(0xff), + }, }, }, `checksum mismatches: got AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA instead of CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN @@ -911,9 +980,10 @@ deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe errors during scrub: -unique error 51966 injected by the test suite -unique error 2989 injected by the test suite -unique error 255 injected by the test suite + /proc/nonexistent: + unique error 51966 injected by the test suite + unique error 2989 injected by the test suite + unique error 255 injected by the test suite `, []error{ &pkg.ChecksumMismatchError{Want: pkg.MustDecode("CH3AiUrCCcVOjOYLaMKKK1Da78989JtfHeIsxMzWOQFiN4mrCLDYpoDxLWqJWCUN")}, stub.UniqueError(0xcafe), -- cgit v1.3.1