internal/store: destroy stale instances

This recovers from inconsistent state on power loss and /tmp/ is not wiped on startup, or if hakurei somehow crashes.

Signed-off-by: Ophestra <cat@gensokyo.uk>

4 files changed, 109 insertions, 13 deletions

diff --git a/internal/outcome/process.go b/internal/outcome/process.go
index 614b22fc..599af8f2 100644
--- a/internal/outcome/process.go
+++ b/internal/outcome/process.go
@@ -303,7 +303,7 @@ func (k *outcome) main(msg message.Msg, identifierFd int) {
 					)
 					for eh := range entries {
 						var et hst.Enablements
-						if et, err = eh.Load(nil); err != nil {
+						if et, err = eh.Load(nil, syscall.Kill); err != nil {
 							perror(err, "read state header of instance "+eh.ID.String())
 						} else {
 							rt |= et
diff --git a/internal/store/segment.go b/internal/store/segment.go
index 32c8a141..8b7dd0d6 100644
--- a/internal/store/segment.go
+++ b/internal/store/segment.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"strconv"
 	"sync"
+	"syscall"
 
 	"hakurei.app/check"
 	"hakurei.app/hst"
@@ -76,13 +77,20 @@ func (eh *EntryHandle) save(state *hst.State) error {
 	return err
 }
 
+// KillFunc is the function signature of syscall.Kill.
+type KillFunc func(pid int, sig syscall.Signal) (err error)
+
 // Load loads and validates the state entry header, and returns the
 // [hst.Enablements] byte. For a non-nil v, the full state payload is decoded
-// and stored in the value pointed to by v.
+// and stored in the value pointed to by v, and if kill is non-nil, the presence
+// of the monitoring process is checked, and a stale entry is destroyed.
 //
 // Load validates the embedded [hst.Config] value. A non-nil error returned by
 // Load is of type [hst.AppError].
-func (eh *EntryHandle) Load(v *hst.State) (hst.Enablements, error) {
+func (eh *EntryHandle) Load(
+	v *hst.State,
+	kill KillFunc,
+) (hst.Enablements, error) {
 	f, err := eh.open(os.O_RDONLY, 0)
 	if err != nil {
 		return 0, err
@@ -101,6 +109,33 @@ func (eh *EntryHandle) Load(v *hst.State) (hst.Enablements, error) {
 				),
 			}
 		}
+		if kill != nil {
+			errno := kill(v.PID, 0)
+			if errno != nil {
+				if !errors.Is(errno, syscall.ESRCH) {
+					err = &hst.AppError{
+						Step: "check monitor process",
+						Err:  errno,
+					}
+				} else {
+					if err = eh.Destroy(); err != nil {
+						err = &hst.AppError{
+							Step: "destroy stale entry",
+							Err:  err,
+						}
+					} else {
+						err = &hst.AppError{
+							Step: "load stale entry",
+							Err:  errno,
+							Msg: fmt.Sprintf(
+								"stale entry %s",
+								eh.ID.String(),
+							),
+						}
+					}
+				}
+			}
+		}
 	} else {
 		et, err = entryDecodeHeader(f)
 	}
diff --git a/internal/store/segment_test.go b/internal/store/segment_test.go
index a244fcdc..6316a278 100644
--- a/internal/store/segment_test.go
+++ b/internal/store/segment_test.go
@@ -2,6 +2,7 @@ package store_test
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"iter"
 	"os"
@@ -55,7 +56,7 @@ func TestStateEntryHandle(t *testing.T) {
 		if err := save(&eh, nil); !reflect.DeepEqual(err, wantErr()) {
 			t.Errorf("save: error = %v, want %v", err, wantErr())
 		}
-		if _, err := eh.Load(nil); !reflect.DeepEqual(err, wantErr()) {
+		if _, err := eh.Load(nil, nil); !reflect.DeepEqual(err, wantErr()) {
 			t.Errorf("load: error = %v, want %v", err, wantErr())
 		}
 	})
@@ -95,8 +96,10 @@ func TestStateEntryHandle(t *testing.T) {
 
 	t.Run("saveload", func(t *testing.T) {
 		t.Parallel()
-		eh := store.EntryHandle{Pathname: check.MustAbs(t.TempDir()).Append("entry"),
-			ID: store.NewTemplateState().ID}
+		eh := store.EntryHandle{
+			Pathname: check.MustAbs(t.TempDir()).Append("entry"),
+			ID:       store.NewTemplateState().ID,
+		}
 
 		if err := save(&eh, store.NewTemplateState()); err != nil {
 			t.Fatalf("save: error = %v", err)
@@ -125,7 +128,7 @@ func TestStateEntryHandle(t *testing.T) {
 			t.Run("load header only", func(t *testing.T) {
 				t.Parallel()
 
-				if et, err := eh.Load(nil); err != nil {
+				if et, err := eh.Load(nil, nil); err != nil {
 					t.Fatalf("load: error = %v", err)
 				} else if want := store.NewTemplateState().Enablements.Unwrap(); et != want {
 					t.Errorf("load: et = %x, want %x", et, want)
@@ -136,7 +139,7 @@ func TestStateEntryHandle(t *testing.T) {
 				t.Parallel()
 
 				var got hst.State
-				if _, err := eh.Load(&got); err != nil {
+				if _, err := eh.Load(&got, nil); err != nil {
 					t.Fatalf("load: error = %v", err)
 				} else if want := store.NewTemplateState(); !reflect.DeepEqual(&got, want) {
 					t.Errorf("load: %#v, want %#v", &got, want)
@@ -145,11 +148,64 @@ func TestStateEntryHandle(t *testing.T) {
 
 			t.Run("load inconsistent", func(t *testing.T) {
 				t.Parallel()
-				wantErr := &hst.AppError{Step: "validate state identifier", Err: os.ErrInvalid,
-					Msg: "state entry 00000000000000000000000000000000 has unexpected id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
+				wantErr := &hst.AppError{
+					Step: "validate state identifier",
+					Err:  os.ErrInvalid,
+					Msg:  "state entry 00000000000000000000000000000000 has unexpected id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+				}
 
 				ehi := store.EntryHandle{Pathname: eh.Pathname}
-				if _, err := ehi.Load(new(hst.State)); !reflect.DeepEqual(err, wantErr) {
+				if _, err := ehi.Load(new(hst.State), nil); !reflect.DeepEqual(err, wantErr) {
+					t.Errorf("load: error = %#v, want %#v", err, wantErr)
+				}
+			})
+
+			t.Run("stale fault", func(t *testing.T) {
+				t.Parallel()
+				wantErr := &hst.AppError{
+					Step: "check monitor process",
+					Err:  syscall.EFAULT,
+				}
+
+				if _, err := eh.Load(new(hst.State), func(pid int, sig syscall.Signal) (err error) {
+					if pid != store.NewTemplateState().PID {
+						return fmt.Errorf("bad pid %d", pid)
+					}
+					if sig != 0 {
+						return fmt.Errorf("bad signal %d", sig)
+					}
+					return syscall.EFAULT
+				}); !reflect.DeepEqual(err, wantErr) {
+					t.Errorf("load: error = %#v, want %#v", err, wantErr)
+				}
+			})
+
+			t.Run("stale", func(t *testing.T) {
+				t.Parallel()
+				wantErr := &hst.AppError{
+					Step: "load stale entry",
+					Err:  syscall.ESRCH,
+					Msg:  "stale entry aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+				}
+
+				ehi := store.EntryHandle{
+					Pathname: check.MustAbs(t.TempDir()).Append("entry"),
+					ID:       eh.ID,
+				}
+
+				if err := save(&ehi, store.NewTemplateState()); err != nil {
+					t.Fatalf("save: error = %v", err)
+				}
+
+				if _, err := ehi.Load(new(hst.State), func(pid int, sig syscall.Signal) (err error) {
+					if pid != store.NewTemplateState().PID {
+						return fmt.Errorf("bad pid %d", pid)
+					}
+					if sig != 0 {
+						return fmt.Errorf("bad signal %d", sig)
+					}
+					return syscall.ESRCH
+				}); !reflect.DeepEqual(err, wantErr) {
 					t.Errorf("load: error = %#v, want %#v", err, wantErr)
 				}
 			})
@@ -229,7 +285,9 @@ func TestSegmentHandle(t *testing.T) {
 				}
 			}
 
-			slices.SortFunc(got, func(a, b *store.EntryHandle) int { return strings.Compare(a.Pathname.String(), b.Pathname.String()) })
+			slices.SortFunc(got, func(a, b *store.EntryHandle) int {
+				return strings.Compare(a.Pathname.String(), b.Pathname.String())
+			})
 			want := tc.want(func(err error, name string) *store.EntryHandle {
 				eh := store.EntryHandle{DecodeErr: err, Pathname: segment.Append(name)}
 				if err == nil {
@@ -247,6 +305,8 @@ func TestSegmentHandle(t *testing.T) {
 	}
 
 	t.Run("nonexistent", func(t *testing.T) {
+		t.Parallel()
+
 		var wantErr = &hst.AppError{Step: "read store segment entries", Err: &os.PathError{
 			Op:   "open",
 			Path: "/proc/nonexistent",
diff --git a/internal/store/store.go b/internal/store/store.go
index 4383d55f..eb9cec41 100644
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -179,7 +179,8 @@ func (s *Store) Segments() (iter.Seq[SegmentIdentity], int, error) {
 }
 
 // All returns a non-reusable iterator over all [EntryHandle] known to this
-// [Store].
+// [Store]. The resulting handles may be retained, but are only safe to use
+// during the iteration producing them.
 //
 // Callers must call copyError after completing iteration and handle the error
 // accordingly. A non-nil error returned by copyError is of type [hst.AppError].