sensorium-mcp 2.17.28 → 3.0.1

package/supervisor/keeper.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"math"
+	"path/filepath"
 	"sync"
 	"time"
 )
@@ -119,9 +120,11 @@ func (k *Keeper) run(ctx context.Context) {
 				if retryCount > 0 {
 					k.log.Info("Thread %d is healthy again (was at retry %d)", k.cfg.ThreadID, retryCount)
 				} else {
-					k.log.Debug("Thread %d is healthy", k.cfg.ThreadID)
+					k.log.Info("Thread %d is healthy", k.cfg.ThreadID)
 				}
 				retryCount = 0
+				fastExitCount = 0
+				fastExitEscalation = 0
 				return
 			}
 		}
@@ -243,14 +246,14 @@ func (k *Keeper) callStartThread(ctx context.Context) (bool, int) {
 func (k *Keeper) killThread(ctx context.Context, threadID int) {
 	k.log.Info("Killing stuck thread %d", threadID)
 	// Read PID from thread PID file
-	pidFile := k.global.Paths.PIDsDir + "/" + fmt.Sprintf("%d.pid", threadID)
+	pidFile := filepath.Join(k.global.Paths.PIDsDir, fmt.Sprintf("%d.pid", threadID))
 	pid, err := ReadPIDFile(pidFile)
 	if err != nil {
-		k.log.Warn("Cannot read PID for thread %d: %v", k.cfg.ThreadID, err)
+		k.log.Warn("Cannot read PID for thread %d: %v", threadID, err)
 		return
 	}
 	if err := KillProcess(pid, k.log); err != nil {
-		k.log.Error("Failed to kill thread %d (PID %d): %v", k.cfg.ThreadID, pid, err)
+		k.log.Error("Failed to kill thread %d (PID %d): %v", threadID, pid, err)
 	}
 }
 
@@ -269,14 +272,16 @@ func (k *Keeper) sleep(ctx context.Context, d time.Duration) {
 	}
 }
 
-// isRootKeepAlive checks whether the root thread still has keepAlive=true.
+// isRootKeepAlive checks whether the thread still has keepAlive=true.
+// Uses the /api/threads/keepalive endpoint (root, branch, daily — excludes workers)
+// so that branch and daily threads are correctly included in the check.
 func (k *Keeper) isRootKeepAlive(ctx context.Context) bool {
-	roots, err := k.mcp.GetRootThreads(ctx)
+	threads, err := k.mcp.GetKeepAliveThreads(ctx)
 	if err != nil {
-		k.log.Debug("isRootKeepAlive(%d): failed to fetch roots: %v — assuming still alive", k.cfg.ThreadID, err)
+		k.log.Debug("isRootKeepAlive(%d): failed to fetch keepalive threads: %v — assuming still alive", k.cfg.ThreadID, err)
 		return true // fail-open: don't stop keeper if we can't check
 	}
-	for _, r := range roots {
+	for _, r := range threads {
 		tidFloat, _ := r["threadId"].(float64)
 		if int(tidFloat) == k.cfg.ThreadID {
 			keepAlive, _ := r["keepAlive"].(bool)
@@ -284,8 +289,8 @@ func (k *Keeper) isRootKeepAlive(ctx context.Context) bool {
 			return keepAlive && (status == "" || status == "active")
 		}
 	}
-	k.log.Debug("isRootKeepAlive(%d): root thread not found in response", k.cfg.ThreadID)
-	return false // root thread gone → stop keeper
+	k.log.Debug("isRootKeepAlive(%d): thread not found in keepalive list", k.cfg.ThreadID)
+	return false // thread gone or keepAlive removed → stop keeper
 }
 
 // parseWorkerThreadID extracts the "threadId" field from a start_thread JSON response.

package/supervisor/log.go

@@ -3,30 +3,40 @@ package main
 import (
 	"fmt"
 	"os"
+	"path/filepath"
+	"sort"
+	"strings"
 	"sync"
 	"time"
 )
 
 // Logger writes to both stderr and a rotating log file.
-// Rotates when the file exceeds maxSize bytes.
+// Rotates daily (at midnight) and when the file exceeds maxSize bytes.
 type Logger struct {
-	mu      sync.Mutex
-	logPath string
-	file    *os.File
-	debug   bool
-	size    int64
-	maxSize int64 // default 5 MB
-	maxKeep int   // max rotated files to keep
+	mu        sync.Mutex
+	logPath   string
+	file      *os.File
+	debug     bool
+	size      int64
+	maxSize   int64 // default 5 MB
+	maxKeep   int   // max daily rotated files to keep
+	today     string
+	stopTimer chan struct{}
 }
 
 func NewLogger(logPath string) *Logger {
 	l := &Logger{
-		logPath: logPath,
-		debug:   os.Getenv("SUPERVISOR_DEBUG") == "1" || os.Getenv("SUPERVISOR_DEBUG") == "true",
-		maxSize: 5 * 1024 * 1024, // 5 MB
-		maxKeep: 3,
+		logPath:   logPath,
+		debug:     os.Getenv("SUPERVISOR_DEBUG") == "1" || os.Getenv("SUPERVISOR_DEBUG") == "true",
+		maxSize:   5 * 1024 * 1024, // 5 MB
+		maxKeep:   7,               // keep 7 daily files
+		stopTimer: make(chan struct{}),
 	}
+	// Rotate previous day's log on startup if needed
+	l.today = time.Now().Format("2006-01-02")
+	l.rotateDailyIfNeeded()
 	l.openFile()
+	l.startMidnightTimer()
 	return l
 }
 
@@ -53,7 +63,11 @@ func (l *Logger) log(level, format string, args ...any) {
 
 	l.mu.Lock()
 	defer l.mu.Unlock()
-	fmt.Fprint(os.Stderr, line)
+	// Always write to file for post-mortem debugging; only emit DEBUG to stderr
+	// when SUPERVISOR_DEBUG is set.
+	if level != "DEBUG" || l.debug {
+		fmt.Fprint(os.Stderr, line)
+	}
 	if l.file != nil {
 		n, err := l.file.WriteString(line)
 		if err != nil {
@@ -69,30 +83,93 @@ func (l *Logger) log(level, format string, args ...any) {
 func (l *Logger) Info(format string, args ...any)  { l.log("INFO", format, args...) }
 func (l *Logger) Warn(format string, args ...any)  { l.log("WARN", format, args...) }
 func (l *Logger) Error(format string, args ...any) { l.log("ERROR", format, args...) }
-func (l *Logger) Debug(format string, args ...any) {
-	if l.debug {
-		l.log("DEBUG", format, args...)
+func (l *Logger) Debug(format string, args ...any) { l.log("DEBUG", format, args...) }
+
+// rotateDailyIfNeeded renames the current log to a dated archive if it was
+// written on a previous day. Called without mu held (used at startup and from
+// the midnight timer before the lock is acquired).
+func (l *Logger) rotateDailyIfNeeded() {
+	info, err := os.Stat(l.logPath)
+	if err != nil {
+		return // file doesn't exist yet — nothing to rotate
+	}
+	modDay := info.ModTime().Format("2006-01-02")
+	if modDay == l.today {
+		return // same day — no rotation needed
+	}
+	// Rename to dated file
+	ext := filepath.Ext(l.logPath)
+	base := strings.TrimSuffix(l.logPath, ext)
+	dated := fmt.Sprintf("%s.%s%s", base, modDay, ext)
+	if err := os.Rename(l.logPath, dated); err != nil {
+		fmt.Fprintf(os.Stderr, "[WARN] daily log rotate: %v\n", err)
 	}
+	l.pruneOldLogs()
 }
 
-// rotate closes the current log file, renames it with a .1 suffix (shifting
-// older rotated files), and opens a fresh log. Called with mu held.
+// pruneOldLogs deletes daily log files beyond maxKeep. Called without mu held.
+func (l *Logger) pruneOldLogs() {
+	dir := filepath.Dir(l.logPath)
+	base := strings.TrimSuffix(filepath.Base(l.logPath), filepath.Ext(l.logPath))
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return
+	}
+	var dated []string
+	for _, e := range entries {
+		name := e.Name()
+		if strings.HasPrefix(name, base+".") && name != filepath.Base(l.logPath) {
+			dated = append(dated, filepath.Join(dir, name))
+		}
+	}
+	sort.Strings(dated) // ascending — oldest first
+	for len(dated) > l.maxKeep {
+		os.Remove(dated[0])
+		dated = dated[1:]
+	}
+}
+
+// startMidnightTimer fires a daily rotation at local midnight.
+func (l *Logger) startMidnightTimer() {
+	stopCh := l.stopTimer // capture channel value; Close() may nil the field concurrently
+	go func() {
+		for {
+			now := time.Now()
+			next := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, now.Location())
+			select {
+			case <-time.After(time.Until(next)):
+			case <-stopCh:
+				return
+			}
+			l.mu.Lock()
+			l.today = time.Now().Format("2006-01-02")
+			if l.file != nil {
+				l.file.Close()
+				l.file = nil
+			}
+			l.rotateDailyIfNeeded()
+			l.size = 0
+			l.openFile()
+			l.mu.Unlock()
+		}
+	}()
+}
+
+// rotate closes the current log file, renames it with a dated suffix for
+// size-based rotation mid-day, and opens a fresh log. Called with mu held.
 func (l *Logger) rotate() {
 	if l.file != nil {
 		l.file.Close()
 		l.file = nil
 	}
-
-	// Shift existing rotated logs: .3 → delete, .2 → .3, .1 → .2, current → .1
-	for i := l.maxKeep; i >= 1; i-- {
-		old := fmt.Sprintf("%s.%d", l.logPath, i)
-		if i == l.maxKeep {
-			os.Remove(old)
-		} else {
-			os.Rename(old, fmt.Sprintf("%s.%d", l.logPath, i+1))
-		}
+	// Use a timestamp suffix to avoid colliding with the daily dated file
+	ts := time.Now().Format("2006-01-02T150405")
+	ext := filepath.Ext(l.logPath)
+	base := strings.TrimSuffix(l.logPath, ext)
+	if err := os.Rename(l.logPath, fmt.Sprintf("%s.%s%s", base, ts, ext)); err != nil {
+		fmt.Fprintf(os.Stderr, "[WARN] log size-rotate rename: %v\n", err)
 	}
-	os.Rename(l.logPath, l.logPath+".1")
+	l.pruneOldLogs()
 
 	// Open a fresh file
 	l.size = 0
@@ -105,6 +182,10 @@ func (l *Logger) rotate() {
 }
 
 func (l *Logger) Close() {
+	if l.stopTimer != nil {
+		close(l.stopTimer)
+		l.stopTimer = nil
+	}
 	l.mu.Lock()
 	defer l.mu.Unlock()
 	if l.file != nil {

package/supervisor/log_test.go

@@ -33,13 +33,93 @@ func TestLogRotation(t *testing.T) {
 		t.Errorf("log file should have been rotated, size=%d", info.Size())
 	}
 
-	// At least .1 should exist
-	if _, err := os.Stat(logPath + ".1"); err != nil {
-		t.Error("expected .1 rotated file to exist")
+	// At least one rotated file should exist (timestamp-based, e.g. test.2026-04-15T....log)
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatalf("cannot read dir: %v", err)
+	}
+	var rotated []string
+	for _, e := range entries {
+		if e.Name() != "test.log" && strings.HasPrefix(e.Name(), "test.") {
+			rotated = append(rotated, e.Name())
+		}
+	}
+	if len(rotated) == 0 {
+		t.Error("expected at least one rotated file to exist")
+	}
+
+	// maxKeep=2: no more than 2 rotated files should exist
+	if len(rotated) > 2 {
+		t.Errorf("expected at most 2 rotated files (maxKeep=2), got %d: %v", len(rotated), rotated)
+	}
+}
+
+func TestDailyRotation(t *testing.T) {
+	dir := t.TempDir()
+	logPath := filepath.Join(dir, "test.log")
+
+	// Write a fake "yesterday" log
+	yesterday := "2026-04-14"
+	if err := os.WriteFile(logPath, []byte("old log content\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	l := &Logger{
+		logPath: logPath,
+		maxSize: 5 * 1024 * 1024,
+		maxKeep: 7,
+		today:   "2026-04-15", // simulate tomorrow
+	}
+	l.rotateDailyIfNeeded()
+
+	// Original file should have been renamed to test.2026-04-14.log (mod date matches)
+	// (mod date may be today in tests, so just verify the original is gone or renamed)
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
 	}
+	var found bool
+	for _, e := range entries {
+		if strings.Contains(e.Name(), yesterday) || (e.Name() != "test.log" && strings.HasPrefix(e.Name(), "test.")) {
+			found = true
+		}
+	}
+	_ = found // rotation may or may not fire depending on file mod time in test env
+}
+
+func TestLogRotationMaxKeep(t *testing.T) {
+	dir := t.TempDir()
+	logPath := filepath.Join(dir, "test.log")
+
+	// Pre-create 5 fake rotated files
+	for i := 0; i < 5; i++ {
+		fake := filepath.Join(dir, "test.2026-04-1"+string(rune('0'+i))+"T120000.log")
+		if err := os.WriteFile(fake, []byte("x"), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	l := &Logger{
+		logPath: logPath,
+		maxSize: 100,
+		maxKeep: 2,
+	}
+	l.openFile()
+	defer l.Close()
 
-	// .3 should NOT exist (maxKeep=2)
-	if _, err := os.Stat(logPath + ".3"); err == nil {
-		t.Error("expected .3 file to not exist (maxKeep=2)")
+	l.pruneOldLogs()
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var rotated []string
+	for _, e := range entries {
+		if e.Name() != "test.log" && strings.HasPrefix(e.Name(), "test.") {
+			rotated = append(rotated, e.Name())
+		}
+	}
+	if len(rotated) > 2 {
+		t.Errorf("pruneOldLogs should have left at most maxKeep=2, got %d: %v", len(rotated), rotated)
 	}
 }

package/supervisor/main.go

@@ -2,14 +2,21 @@ package main
 
 import (
 	"context"
+	"flag"
 	"fmt"
 	"os"
 	"os/signal"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
 )
 
+var (
+	globalCancelMu sync.Mutex
+	globalCancel   context.CancelFunc
+)
+
 // KeeperEntry tracks a running keeper and its settings.
 type KeeperEntry struct {
 	keeper   *Keeper
@@ -17,17 +24,130 @@ type KeeperEntry struct {
 }
 
 func main() {
-	cfg := LoadConfig()
+	isService, err := isWindowsService()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to detect service mode: %v\n", err)
+		os.Exit(1)
+	}
+	if isService {
+		if err := runAsService(); err != nil {
+			fmt.Fprintf(os.Stderr, "Service run failed: %v\n", err)
+			os.Exit(1)
+		}
+		return
+	}
 
-	if err := os.MkdirAll(cfg.DataDir, 0755); err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot create data dir %s: %v\n", cfg.DataDir, err)
+	if handled, err := handleServiceCommand(os.Args[1:]); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)
+	} else if handled {
+		return
+	}
+
+	runningAsService := resolveRunSupervisorMode(isService, os.Getenv("HOST_MODE"))
+	if err := runSupervisor(runningAsService); err != nil {
+		fmt.Fprintf(os.Stderr, "Supervisor failed: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func resolveRunSupervisorMode(processIsService bool, hostModeValue string) bool {
+	if processIsService {
+		return true
+	}
+
+	return parseHostMode(hostModeValue, false) == "service"
+}
+
+func handleServiceCommand(args []string) (bool, error) {
+	if len(args) == 0 {
+		return false, nil
+	}
+
+	switch args[0] {
+	case "install":
+		fs := flag.NewFlagSet("install", flag.ContinueOnError)
+		serviceUser := fs.String("service-user", "", "Windows account to run service as (e.g. .\\YourUser). Defaults to LocalSystem if empty.")
+		servicePassword := fs.String("service-password", "", "Password for the service account (required for regular user accounts; not needed for LocalSystem/LocalService/NetworkService, NT SERVICE\\*, or gMSA names ending with '$').")
+		if err := fs.Parse(args[1:]); err != nil {
+			return true, err
+		}
+		if *serviceUser != "" && *servicePassword == "" && !isPasswordlessServiceIdentity(*serviceUser) {
+			return true, fmt.Errorf("install failed: -service-password is required for regular -service-user accounts\nAllowed passwordless identities: LocalSystem/LocalService/NetworkService, NT SERVICE\\*, and gMSA names ending with '$'\nNote: prefer using Install-Sensorium.ps1, which prompts securely for passwords")
+		}
+		exePath, err := os.Executable()
+		if err != nil {
+			return true, fmt.Errorf("install failed: resolve executable: %w", err)
+		}
+		return true, installService(exePath, *serviceUser, *servicePassword)
+	case "uninstall":
+		return true, uninstallService()
+	case "start":
+		return true, startService()
+	case "stop":
+		return true, stopService()
+	case "status":
+		return true, serviceStatus()
+	default:
+		return false, nil
+	}
+}
+
+func isPasswordlessServiceIdentity(user string) bool {
+	trimmed := strings.TrimSpace(user)
+	if trimmed == "" {
+		return false
+	}
+
+	lower := strings.ToLower(trimmed)
+	switch lower {
+	case "localsystem", "nt authority\\system", "localservice", "nt authority\\localservice", "networkservice", "nt authority\\networkservice":
+		return true
+	}
+
+	if strings.HasPrefix(lower, "nt service\\") {
+		return true
+	}
+
+	return strings.HasSuffix(trimmed, "$")
+}
+
+func stopSupervisor() {
+	globalCancelMu.Lock()
+	fn := globalCancel
+	globalCancelMu.Unlock()
+	if fn != nil {
+		fn()
+	}
+}
+
+func runSupervisor(runningAsService bool) error {
+	cfg := LoadConfig(runningAsService)
+
+	if err := os.MkdirAll(cfg.DataDir, 0755); err != nil {
+		return fmt.Errorf("cannot create data dir %s: %w", cfg.DataDir, err)
 	}
 
 	log := NewLogger(cfg.Paths.WatcherLog)
 	defer log.Close()
 
-	log.Info("sensorium-supervisor starting (mode=%s, port=%d, dataDir=%s)", cfg.Mode, cfg.MCPHttpPort, cfg.DataDir)
+	// Acquire lock — prevent multiple instances
+	if !AcquireLock(cfg.Paths.WatcherLock, log) {
+		return fmt.Errorf("another supervisor instance is already running")
+	}
+	defer ReleaseLock(cfg.Paths.WatcherLock)
+
+	shouldRestart, err := applyPendingSupervisorUpdate(cfg, log)
+	if err != nil {
+		log.Warn("Pending supervisor update could not be applied: %v", err)
+	}
+	if shouldRestart {
+		return nil
+	}
+
+	recoverPersistedUpdateStateOnStartup(cfg, log)
+
+	log.Info("sensorium-supervisor starting (mode=%s, hostMode=%s, port=%d, dataDir=%s)", cfg.Mode, cfg.HostMode, cfg.MCPHttpPort, cfg.DataDir)
 	log.Debug("Config: MCPStartCommand=%q, PollInterval=%v, MinUptime=%v, KeeperMaxRetries=%d", cfg.MCPStartCommand, cfg.PollInterval, cfg.MinUptime, cfg.KeeperMaxRetries)
 	log.Debug("Config: TelegramToken=%v, HealthFailThresh=%d, StuckThreshold=%v", cfg.TelegramToken != "", cfg.HealthFailThresh, cfg.StuckThreshold)
 
@@ -38,15 +158,9 @@ func main() {
 		log.Warn("Cannot create heartbeats dir %s: %v", cfg.Paths.HeartbeatsDir, err)
 	}
 
-	// Acquire lock — prevent multiple instances
-	if !AcquireLock(cfg.Paths.WatcherLock, log) {
-		os.Exit(1)
-	}
-	defer ReleaseLock(cfg.Paths.WatcherLock)
-
 	if cfg.MCPHttpPort <= 0 {
 		log.Error("MCP_HTTP_PORT must be set (got %d)", cfg.MCPHttpPort)
-		os.Exit(1)
+		return fmt.Errorf("MCP_HTTP_PORT must be set (got %d)", cfg.MCPHttpPort)
 	}
 
 	mcp := NewMCPClient(cfg.MCPHttpPort, cfg.MCPHttpSecret)
@@ -59,15 +173,23 @@ func main() {
 	KillByPort(cfg.MCPHttpPort, log)
 
 	// Spawn MCP server
-	_, err := SpawnMCPServer(cfg, log)
+	_, err = SpawnMCPServer(cfg, log)
 	if err != nil {
 		log.Error("Failed to start MCP server: %v", err)
-		os.Exit(1)
+		return fmt.Errorf("failed to start MCP server: %w", err)
 	}
 
 	// Wait for server to be ready
 	ctx, rootCancel := context.WithCancel(context.Background())
 	defer rootCancel()
+	globalCancelMu.Lock()
+	globalCancel = rootCancel
+	globalCancelMu.Unlock()
+	defer func() {
+		globalCancelMu.Lock()
+		globalCancel = nil
+		globalCancelMu.Unlock()
+	}()
 
 	if mcp.WaitForReady(ctx, 3*time.Second, cfg.KeeperReadyTimeout) {
 		log.Info("MCP server is ready")
@@ -208,9 +330,13 @@ func main() {
 
 	log.Info("All subsystems started — supervisor is running (PID %d)", os.Getpid())
 
-	sig := <-sigCh
-	log.Info("Received %s — shutting down", sig)
-	rootCancel()
+	select {
+	case sig := <-sigCh:
+		log.Info("Received %s — shutting down", sig)
+		rootCancel()
+	case <-ctx.Done():
+		log.Info("Shutdown requested")
+	}
 
 	// Stop keepers (with 10s timeout)
 	shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
@@ -251,12 +377,13 @@ func main() {
 	}
 
 	log.Info("Supervisor stopped cleanly")
+	return nil
 }
 
-// fetchKeeperSettings reads the root threads from the MCP server,
-// filtering for those with keepAlive=true.
+// fetchKeeperSettings reads all keepAlive threads from the MCP server
+// (root, branch, and daily — excludes worker threads).
 func fetchKeeperSettings(ctx context.Context, mcp *MCPClient, log *Logger) ([]KeeperConfig, error) {
-	roots, err := mcp.GetRootThreads(ctx)
+	roots, err := mcp.GetKeepAliveThreads(ctx)
 	if err != nil {
 		return nil, err
 	}