internal/store: destroy stale instances

This recovers from inconsistent state on power loss and /tmp/ is not wiped on startup, or if hakurei somehow crashes.

Signed-off-by: Ophestra <cat@gensokyo.uk>

10 files changed, 134 insertions, 25 deletions

diff --git a/cmd/app/lock.go b/cmd/app/lock.go
index 5a22efba..88068381 100644
--- a/cmd/app/lock.go
+++ b/cmd/app/lock.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"syscall"
 
 	"hakurei.app/check"
 	"hakurei.app/fhs"
@@ -57,7 +58,7 @@ func informTemplate(base *check.Absolute, name string, mutable bool) (func() err
 	var s hst.State
 	for eh := range entries {
 		s = hst.State{}
-		if _, err := eh.Load(&s); err != nil {
+		if _, err := eh.Load(&s, syscall.Kill); err != nil {
 			return nil, err
 		}
 
diff --git a/cmd/hakurei/command.go b/cmd/hakurei/command.go
index 374d521c..76004d4f 100644
--- a/cmd/hakurei/command.go
+++ b/cmd/hakurei/command.go
@@ -351,7 +351,7 @@ func buildCommand(ctx context.Context, msg message.Msg, early *earlyHardeningErr
 				if !flagNoStore {
 					var sc hst.Paths
 					env.CopyPaths().Copy(&sc, new(outcome.Hsu).MustID(nil))
-					entry = tryIdentifier(msg, name, outcome.NewStore(&sc))
+					entry = tryIdentifier(msg, syscall.Kill, name, outcome.NewStore(&sc))
 				}
 
 				if entry == nil {
@@ -380,7 +380,7 @@ func buildCommand(ctx context.Context, msg message.Msg, early *earlyHardeningErr
 
 		var sc hst.Paths
 		env.CopyPaths().Copy(&sc, new(outcome.Hsu).MustID(nil))
-		entry := tryIdentifier(msg, args[0], outcome.NewStore(&sc))
+		entry := tryIdentifier(msg, syscall.Kill, args[0], outcome.NewStore(&sc))
 		if entry == nil {
 			log.Fatalf("%q does not match any active instance", args[0])
 		}
@@ -395,7 +395,7 @@ func buildCommand(ctx context.Context, msg message.Msg, early *earlyHardeningErr
 		c.NewCommand("ps", "List active instances", func(args []string) error {
 			var sc hst.Paths
 			env.CopyPaths().Copy(&sc, new(outcome.Hsu).MustID(nil))
-			printPs(msg, os.Stdout, time.Now().UTC(), outcome.NewStore(&sc), flagShort, flagJSON)
+			printPs(msg, syscall.Kill, os.Stdout, time.Now().UTC(), outcome.NewStore(&sc), flagShort, flagJSON)
 			return errSuccess
 		}).Flag(&flagShort, "short", command.BoolFlag(false), "Print instance id")
 	}
diff --git a/cmd/hakurei/parse.go b/cmd/hakurei/parse.go
index 31d0dfa6..09721fad 100644
--- a/cmd/hakurei/parse.go
+++ b/cmd/hakurei/parse.go
@@ -99,7 +99,12 @@ func shortIdentifierString(s string) string {
 
 // tryIdentifier attempts to match [hst.State] from a [hex] representation of
 // [hst.ID] or a prefix of its lower half.
-func tryIdentifier(msg message.Msg, name string, s *store.Store) *hst.State {
+func tryIdentifier(
+	msg message.Msg,
+	kill store.KillFunc,
+	name string,
+	s *store.Store,
+) *hst.State {
 	const (
 		likeShort = 1 << iota
 		likeFull
@@ -145,7 +150,7 @@ func tryIdentifier(msg message.Msg, name string, s *store.Store) *hst.State {
 
 			if strings.HasPrefix(eh.ID.String()[len(hst.ID{}):], name) {
 				var entry hst.State
-				if _, err := eh.Load(&entry); err != nil {
+				if _, err := eh.Load(&entry, kill); err != nil {
 					msg.GetLogger().Println(getMessage("cannot load state entry:", err))
 					continue
 				}
@@ -168,7 +173,7 @@ func tryIdentifier(msg message.Msg, name string, s *store.Store) *hst.State {
 
 			if eh.ID == likelyID {
 				var entry hst.State
-				if _, err := eh.Load(&entry); err != nil {
+				if _, err := eh.Load(&entry, kill); err != nil {
 					msg.GetLogger().Println(getMessage("cannot load state entry:", err))
 					continue
 				}
diff --git a/cmd/hakurei/parse_test.go b/cmd/hakurei/parse_test.go
index 1a574966..6f8a2299 100644
--- a/cmd/hakurei/parse_test.go
+++ b/cmd/hakurei/parse_test.go
@@ -108,7 +108,7 @@ func TestTryIdentifier(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 
-			got := tryIdentifier(msg, tc.s, store.New(base))
+			got := tryIdentifier(msg, nil, tc.s, store.New(base))
 			if !reflect.DeepEqual(got, tc.want) {
 				t.Errorf("tryIdentifier: %#v, want %#v", got, tc.want)
 			}
diff --git a/cmd/hakurei/print.go b/cmd/hakurei/print.go
index 780dc55b..2ac626e7 100644
--- a/cmd/hakurei/print.go
+++ b/cmd/hakurei/print.go
@@ -170,7 +170,14 @@ func printShowInstance(
 }
 
 // printPs writes a representation of active instances to output.
-func printPs(msg message.Msg, output io.Writer, now time.Time, s *store.Store, short, flagJSON bool) {
+func printPs(
+	msg message.Msg,
+	kill store.KillFunc,
+	output io.Writer,
+	now time.Time,
+	s *store.Store,
+	short, flagJSON bool,
+) {
 	f := func(a func(eh *store.EntryHandle)) {
 		entries, copyError := s.All()
 		for eh := range entries {
@@ -184,7 +191,7 @@ func printPs(msg message.Msg, output io.Writer, now time.Time, s *store.Store, s
 	if short { // short output requires identifier only
 		var identifiers []*hst.ID
 		f(func(eh *store.EntryHandle) {
-			if _, err := eh.Load(nil); err != nil { // passes through decode error
+			if _, err := eh.Load(nil, kill); err != nil { // passes through decode error
 				msg.GetLogger().Println(getMessage("cannot validate state entry header:", err))
 				return
 			}
@@ -206,7 +213,7 @@ func printPs(msg message.Msg, output io.Writer, now time.Time, s *store.Store, s
 	var instances []*hst.State
 	f(func(eh *store.EntryHandle) {
 		var state hst.State
-		if _, err := eh.Load(&state); err != nil { // passes through decode error
+		if _, err := eh.Load(&state, kill); err != nil { // passes through decode error
 			msg.GetLogger().Println(getMessage("cannot load state entry:", err))
 			return
 		}
diff --git a/cmd/hakurei/print_test.go b/cmd/hakurei/print_test.go
index a6e213c2..310fc885 100644
--- a/cmd/hakurei/print_test.go
+++ b/cmd/hakurei/print_test.go
@@ -770,7 +770,7 @@ func TestPrintPs(t *testing.T) {
 			var printBuf, logBuf bytes.Buffer
 			msg := message.New(log.New(&logBuf, "check: ", 0))
 			msg.SwapVerbose(true)
-			printPs(msg, &printBuf, testTime, s, tc.short, tc.json)
+			printPs(msg, nil, &printBuf, testTime, s, tc.short, tc.json)
 			if got := printBuf.String(); got != tc.want {
 				t.Errorf("printPs:\n%s\nwant\n%s", got, tc.want)
 				return
diff --git a/internal/outcome/process.go b/internal/outcome/process.go
index 614b22fc..599af8f2 100644
--- a/internal/outcome/process.go
+++ b/internal/outcome/process.go
@@ -303,7 +303,7 @@ func (k *outcome) main(msg message.Msg, identifierFd int) {
 					)
 					for eh := range entries {
 						var et hst.Enablements
-						if et, err = eh.Load(nil); err != nil {
+						if et, err = eh.Load(nil, syscall.Kill); err != nil {
 							perror(err, "read state header of instance "+eh.ID.String())
 						} else {
 							rt |= et
diff --git a/internal/store/segment.go b/internal/store/segment.go
index 32c8a141..8b7dd0d6 100644
--- a/internal/store/segment.go
+++ b/internal/store/segment.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"strconv"
 	"sync"
+	"syscall"
 
 	"hakurei.app/check"
 	"hakurei.app/hst"
@@ -76,13 +77,20 @@ func (eh *EntryHandle) save(state *hst.State) error {
 	return err
 }
 
+// KillFunc is the function signature of syscall.Kill.
+type KillFunc func(pid int, sig syscall.Signal) (err error)
+
 // Load loads and validates the state entry header, and returns the
 // [hst.Enablements] byte. For a non-nil v, the full state payload is decoded
-// and stored in the value pointed to by v.
+// and stored in the value pointed to by v, and if kill is non-nil, the presence
+// of the monitoring process is checked, and a stale entry is destroyed.
 //
 // Load validates the embedded [hst.Config] value. A non-nil error returned by
 // Load is of type [hst.AppError].
-func (eh *EntryHandle) Load(v *hst.State) (hst.Enablements, error) {
+func (eh *EntryHandle) Load(
+	v *hst.State,
+	kill KillFunc,
+) (hst.Enablements, error) {
 	f, err := eh.open(os.O_RDONLY, 0)
 	if err != nil {
 		return 0, err
@@ -101,6 +109,33 @@ func (eh *EntryHandle) Load(v *hst.State) (hst.Enablements, error) {
 				),
 			}
 		}
+		if kill != nil {
+			errno := kill(v.PID, 0)
+			if errno != nil {
+				if !errors.Is(errno, syscall.ESRCH) {
+					err = &hst.AppError{
+						Step: "check monitor process",
+						Err:  errno,
+					}
+				} else {
+					if err = eh.Destroy(); err != nil {
+						err = &hst.AppError{
+							Step: "destroy stale entry",
+							Err:  err,
+						}
+					} else {
+						err = &hst.AppError{
+							Step: "load stale entry",
+							Err:  errno,
+							Msg: fmt.Sprintf(
+								"stale entry %s",
+								eh.ID.String(),
+							),
+						}
+					}
+				}
+			}
+		}
 	} else {
 		et, err = entryDecodeHeader(f)
 	}
diff --git a/internal/store/segment_test.go b/internal/store/segment_test.go
index a244fcdc..6316a278 100644
--- a/internal/store/segment_test.go
+++ b/internal/store/segment_test.go
@@ -2,6 +2,7 @@ package store_test
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"iter"
 	"os"
@@ -55,7 +56,7 @@ func TestStateEntryHandle(t *testing.T) {
 		if err := save(&eh, nil); !reflect.DeepEqual(err, wantErr()) {
 			t.Errorf("save: error = %v, want %v", err, wantErr())
 		}
-		if _, err := eh.Load(nil); !reflect.DeepEqual(err, wantErr()) {
+		if _, err := eh.Load(nil, nil); !reflect.DeepEqual(err, wantErr()) {
 			t.Errorf("load: error = %v, want %v", err, wantErr())
 		}
 	})
@@ -95,8 +96,10 @@ func TestStateEntryHandle(t *testing.T) {
 
 	t.Run("saveload", func(t *testing.T) {
 		t.Parallel()
-		eh := store.EntryHandle{Pathname: check.MustAbs(t.TempDir()).Append("entry"),
-			ID: store.NewTemplateState().ID}
+		eh := store.EntryHandle{
+			Pathname: check.MustAbs(t.TempDir()).Append("entry"),
+			ID:       store.NewTemplateState().ID,
+		}
 
 		if err := save(&eh, store.NewTemplateState()); err != nil {
 			t.Fatalf("save: error = %v", err)
@@ -125,7 +128,7 @@ func TestStateEntryHandle(t *testing.T) {
 			t.Run("load header only", func(t *testing.T) {
 				t.Parallel()
 
-				if et, err := eh.Load(nil); err != nil {
+				if et, err := eh.Load(nil, nil); err != nil {
 					t.Fatalf("load: error = %v", err)
 				} else if want := store.NewTemplateState().Enablements.Unwrap(); et != want {
 					t.Errorf("load: et = %x, want %x", et, want)
@@ -136,7 +139,7 @@ func TestStateEntryHandle(t *testing.T) {
 				t.Parallel()
 
 				var got hst.State
-				if _, err := eh.Load(&got); err != nil {
+				if _, err := eh.Load(&got, nil); err != nil {
 					t.Fatalf("load: error = %v", err)
 				} else if want := store.NewTemplateState(); !reflect.DeepEqual(&got, want) {
 					t.Errorf("load: %#v, want %#v", &got, want)
@@ -145,11 +148,64 @@ func TestStateEntryHandle(t *testing.T) {
 
 			t.Run("load inconsistent", func(t *testing.T) {
 				t.Parallel()
-				wantErr := &hst.AppError{Step: "validate state identifier", Err: os.ErrInvalid,
-					Msg: "state entry 00000000000000000000000000000000 has unexpected id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
+				wantErr := &hst.AppError{
+					Step: "validate state identifier",
+					Err:  os.ErrInvalid,
+					Msg:  "state entry 00000000000000000000000000000000 has unexpected id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+				}
 
 				ehi := store.EntryHandle{Pathname: eh.Pathname}
-				if _, err := ehi.Load(new(hst.State)); !reflect.DeepEqual(err, wantErr) {
+				if _, err := ehi.Load(new(hst.State), nil); !reflect.DeepEqual(err, wantErr) {
+					t.Errorf("load: error = %#v, want %#v", err, wantErr)
+				}
+			})
+
+			t.Run("stale fault", func(t *testing.T) {
+				t.Parallel()
+				wantErr := &hst.AppError{
+					Step: "check monitor process",
+					Err:  syscall.EFAULT,
+				}
+
+				if _, err := eh.Load(new(hst.State), func(pid int, sig syscall.Signal) (err error) {
+					if pid != store.NewTemplateState().PID {
+						return fmt.Errorf("bad pid %d", pid)
+					}
+					if sig != 0 {
+						return fmt.Errorf("bad signal %d", sig)
+					}
+					return syscall.EFAULT
+				}); !reflect.DeepEqual(err, wantErr) {
+					t.Errorf("load: error = %#v, want %#v", err, wantErr)
+				}
+			})
+
+			t.Run("stale", func(t *testing.T) {
+				t.Parallel()
+				wantErr := &hst.AppError{
+					Step: "load stale entry",
+					Err:  syscall.ESRCH,
+					Msg:  "stale entry aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+				}
+
+				ehi := store.EntryHandle{
+					Pathname: check.MustAbs(t.TempDir()).Append("entry"),
+					ID:       eh.ID,
+				}
+
+				if err := save(&ehi, store.NewTemplateState()); err != nil {
+					t.Fatalf("save: error = %v", err)
+				}
+
+				if _, err := ehi.Load(new(hst.State), func(pid int, sig syscall.Signal) (err error) {
+					if pid != store.NewTemplateState().PID {
+						return fmt.Errorf("bad pid %d", pid)
+					}
+					if sig != 0 {
+						return fmt.Errorf("bad signal %d", sig)
+					}
+					return syscall.ESRCH
+				}); !reflect.DeepEqual(err, wantErr) {
 					t.Errorf("load: error = %#v, want %#v", err, wantErr)
 				}
 			})
@@ -229,7 +285,9 @@ func TestSegmentHandle(t *testing.T) {
 				}
 			}
 
-			slices.SortFunc(got, func(a, b *store.EntryHandle) int { return strings.Compare(a.Pathname.String(), b.Pathname.String()) })
+			slices.SortFunc(got, func(a, b *store.EntryHandle) int {
+				return strings.Compare(a.Pathname.String(), b.Pathname.String())
+			})
 			want := tc.want(func(err error, name string) *store.EntryHandle {
 				eh := store.EntryHandle{DecodeErr: err, Pathname: segment.Append(name)}
 				if err == nil {
@@ -247,6 +305,8 @@ func TestSegmentHandle(t *testing.T) {
 	}
 
 	t.Run("nonexistent", func(t *testing.T) {
+		t.Parallel()
+
 		var wantErr = &hst.AppError{Step: "read store segment entries", Err: &os.PathError{
 			Op:   "open",
 			Path: "/proc/nonexistent",
diff --git a/internal/store/store.go b/internal/store/store.go
index 4383d55f..eb9cec41 100644
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -179,7 +179,8 @@ func (s *Store) Segments() (iter.Seq[SegmentIdentity], int, error) {
 }
 
 // All returns a non-reusable iterator over all [EntryHandle] known to this
-// [Store].
+// [Store]. The resulting handles may be retained, but are only safe to use
+// during the iteration producing them.
 //
 // Callers must call copyError after completing iteration and handle the error
 // accordingly. A non-nil error returned by copyError is of type [hst.AppError].