internal/pkg: streaming archive reader/writer

This is much more robust and efficient than the simple buffering implementation for larger files. Allocations happen almost exclusively in WalkDir.

Signed-off-by: Ophestra <cat@gensokyo.uk>

1 files changed, 198 insertions, 0 deletions

diff --git a/internal/pkg/archive_test.go b/internal/pkg/archive_test.go
new file mode 100644
index 00000000..a51cfee5
--- /dev/null
+++ b/internal/pkg/archive_test.go
@@ -0,0 +1,198 @@
+package pkg_test
+
+import (
+	"bytes"
+	"io"
+	"io/fs"
+	"reflect"
+	"testing"
+	"testing/fstest"
+	"unsafe"
+
+	"hakurei.app/internal/pkg"
+)
+
+func TestArchive(t *testing.T) {
+	t.Parallel()
+
+	type entry struct {
+		path string
+		mode fs.FileMode
+		data string
+	}
+	testCases := []struct {
+		name    string
+		fsys    fs.FS
+		entries []entry
+		sum     pkg.Checksum
+		err     error
+	}{
+		{"bad type", fstest.MapFS{
+			".":       {Mode: fs.ModeDir | 0700},
+			"invalid": {Mode: fs.ModeCharDevice | 0400},
+		}, nil, pkg.Checksum{}, pkg.InvalidFileModeError(
+			fs.ModeCharDevice | 0400,
+		)},
+
+		{"coldboot", fstest.MapFS{
+			".": {Mode: fs.ModeDir | 0700},
+
+			"devices":        {Mode: fs.ModeDir | 0700},
+			"devices/uevent": {Mode: 0600, Data: []byte("add")},
+			"devices/empty":  {Mode: fs.ModeDir | 0700},
+
+			"devices/sub":        {Mode: fs.ModeDir | 0700},
+			"devices/sub/uevent": {Mode: 0600, Data: []byte("add")},
+
+			"block":        {Mode: fs.ModeDir | 0700},
+			"block/uevent": {Mode: 0600},
+		}, []entry{
+			{".", fs.ModeDir | 0700, ""},
+
+			{"block", fs.ModeDir | 0700, ""},
+			{"block/uevent", 0600, ""},
+
+			{"devices", fs.ModeDir | 0700, ""},
+			{"devices/empty", fs.ModeDir | 0700, ""},
+			{"devices/sub", fs.ModeDir | 0700, ""},
+			{"devices/sub/uevent", 0600, "add"},
+			{"devices/uevent", 0600, "add"},
+		}, pkg.MustDecode("mEy_Lf5KotThm7OwMx7yTKZh5HCCyaB41pVAvI9uDMgVQFM91iosBLYsRm8bDsX8"), nil},
+
+		{"empty", fstest.MapFS{
+			".":          {Mode: fs.ModeDir | 0700},
+			"checksum":   {Mode: fs.ModeDir | 0700},
+			"identifier": {Mode: fs.ModeDir | 0700},
+			"work":       {Mode: fs.ModeDir | 0700},
+		}, []entry{
+			{".", fs.ModeDir | 0700, ""},
+			{"checksum", fs.ModeDir | 0700, ""},
+			{"identifier", fs.ModeDir | 0700, ""},
+			{"work", fs.ModeDir | 0700, ""},
+		}, pkg.MustDecode("E4vEZKhCcL2gPZ2Tt59FS3lDng-d_2SKa2i5G_RbDfwGn6EemptFaGLPUDiOa94C"), nil},
+
+		{"sample directory step garbage", fstest.MapFS{
+			".": {Mode: fs.ModeDir | 0500},
+
+			"lib":       {Mode: fs.ModeDir | 0500},
+			"lib/check": {Mode: 0400},
+
+			"lib/pkgconfig": {Mode: fs.ModeDir | 0500},
+		}, []entry{
+			{".", fs.ModeDir | 0500, ""},
+
+			{"lib", fs.ModeDir | 0500, ""},
+			{"lib/check", 0400, ""},
+
+			{"lib/pkgconfig", fs.ModeDir | 0500, ""},
+		}, pkg.MustDecode("CUx-3hSbTWPsbMfDhgalG4Ni_GmR9TnVX8F99tY_P5GtkYvczg9RrF5zO0jX9XYT"), nil},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			t.Run("roundtrip", func(t *testing.T) {
+				t.Parallel()
+
+				var buf bytes.Buffer
+				if err := pkg.Write(
+					tc.fsys,
+					".",
+					&buf,
+				); !reflect.DeepEqual(err, tc.err) {
+					t.Fatalf("Flatten: error = %v, want %v", err, tc.err)
+				} else if tc.err != nil {
+					return
+				}
+
+				r := pkg.NewReader(bytes.NewReader(buf.Bytes()))
+				var got []entry
+				for {
+					h, err := r.Next()
+					if err != nil {
+						if err == io.EOF {
+							break
+						}
+						t.Fatalf("Next: error = %v", err)
+					}
+
+					var data []byte
+					if data, err = io.ReadAll(r); err != nil {
+						t.Fatalf("Read: error = %v", err)
+					}
+
+					got = append(got, entry{
+						path: h.Path,
+						mode: h.Mode,
+						data: unsafe.String(unsafe.SliceData(data), len(data)),
+					})
+				}
+
+				if !reflect.DeepEqual(got, tc.entries) {
+					t.Fatalf("Reader: %#v, want %#v", got, tc.entries)
+				}
+			})
+
+			if tc.err != nil {
+				return
+			}
+
+			t.Run("hash", func(t *testing.T) {
+				t.Parallel()
+
+				var got pkg.Checksum
+				if err := pkg.HashFS(&got, tc.fsys, "."); err != nil {
+					t.Fatalf("HashFS: error = %v", err)
+				} else if got != tc.sum {
+					t.Fatalf("HashFS: %v", &pkg.ChecksumMismatchError{
+						Got:  got,
+						Want: tc.sum,
+					})
+				}
+			})
+		})
+	}
+}
+
+var archiveTestdata = fstest.MapFS{
+	".": {Mode: fs.ModeDir | 0700},
+
+	"devices":        {Mode: fs.ModeDir | 0700},
+	"devices/uevent": {Mode: 0600, Data: []byte("add")},
+	"devices/empty":  {Mode: fs.ModeDir | 0700},
+
+	"devices/sub":        {Mode: fs.ModeDir | 0700},
+	"devices/sub/uevent": {Mode: 0600, Data: []byte("add")},
+
+	"block":        {Mode: fs.ModeDir | 0700},
+	"block/uevent": {Mode: 0600},
+}
+
+func BenchmarkArchiveRead(b *testing.B) {
+	var buf bytes.Buffer
+	if err := pkg.Write(archiveTestdata, ".", &buf); err != nil {
+		b.Fatal(err)
+	}
+	testdata := buf.Bytes()
+
+	for b.Loop() {
+		r := pkg.NewReader(bytes.NewReader(testdata))
+		for {
+			_, err := r.Next()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+func BenchmarkArchiveWrite(b *testing.B) {
+	for b.Loop() {
+		if err := pkg.Write(archiveTestdata, ".", io.Discard); err != nil {
+			b.Fatal(err)
+		}
+	}
+}