sensorium-mcp 2.17.26 → 2.17.28

package/supervisor/keeper.go

@@ -0,0 +1,303 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"sync"
+	"time"
+)
+
+// KeeperConfig describes a thread that must be kept alive.
+type KeeperConfig struct {
+	ThreadID         int
+	SessionName      string
+	Client           string // e.g. "claude-code", "codex"
+	WorkingDirectory string
+	MaxRetries       int
+	CooldownMs       int
+}
+
+// Keeper supervises a single thread, restarting it via the MCP server's
+// start_thread tool when it stops running. One goroutine per keeper.
+type Keeper struct {
+	cfg     KeeperConfig
+	global  Config
+	mcp     *MCPClient
+	log     *Logger
+	onDeath func(threadID int, sessionName string)
+
+	mu      sync.Mutex
+	stopped bool
+	cancel  context.CancelFunc
+	done    chan struct{}
+}
+
+func NewKeeper(cfg KeeperConfig, global Config, mcp *MCPClient, log *Logger, onDeath func(int, string)) *Keeper {
+	maxRetries := cfg.MaxRetries
+	if maxRetries <= 0 {
+		maxRetries = global.KeeperMaxRetries
+	}
+	cfg.MaxRetries = maxRetries
+	return &Keeper{
+		cfg:     cfg,
+		global:  global,
+		mcp:     mcp,
+		log:     log,
+		onDeath: onDeath,
+		done:    make(chan struct{}),
+	}
+}
+
+// Start begins the keeper loop in a separate goroutine.
+func (k *Keeper) Start() {
+	ctx, cancel := context.WithCancel(context.Background())
+	k.mu.Lock()
+	k.cancel = cancel
+	k.mu.Unlock()
+
+	go k.run(ctx)
+}
+
+// Stop signals the keeper to shut down and waits for it to finish.
+func (k *Keeper) Stop() {
+	k.mu.Lock()
+	k.stopped = true
+	if k.cancel != nil {
+		k.cancel()
+	}
+	k.mu.Unlock()
+	<-k.done
+}
+
+func (k *Keeper) isStopped() bool {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	return k.stopped
+}
+
+func (k *Keeper) run(ctx context.Context) {
+	defer close(k.done)
+	defer func() {
+		if r := recover(); r != nil {
+			k.log.Error("Keeper panicked for thread %d: %v", k.cfg.ThreadID, r)
+		}
+	}()
+	defer k.log.Info("Keeper stopped for thread %d", k.cfg.ThreadID)
+
+	k.log.Info("Keeper started for thread %d ('%s') [client=%s]", k.cfg.ThreadID, k.cfg.SessionName, k.cfg.Client)
+
+	// Wait for MCP server to be ready
+	ready := k.mcp.WaitForReady(ctx, k.global.KeeperReadyPollInterval, k.global.KeeperReadyTimeout)
+	if !ready && !k.isStopped() {
+		k.log.Warn("MCP server not ready after %v — attempting start_thread anyway", k.global.KeeperReadyTimeout)
+	}
+
+	retryCount := 0
+	fastExitCount := 0
+	fastExitEscalation := 0
+	var lastStartTime time.Time
+	activeThreadID := k.cfg.ThreadID // may differ from root after start_thread
+
+	checkAndStart := func() {
+		if k.isStopped() {
+			return
+		}
+
+		// Check if thread is running — use activeThreadID (worker) not root
+		running := k.mcp.IsThreadRunning(ctx, activeThreadID)
+		if running {
+			// Check if stuck
+			if k.mcp.IsThreadStuck(ctx, activeThreadID, k.global.StuckThreshold) {
+				k.log.Warn("Thread %d (worker %d) is stuck (no heartbeat for %v) — restarting", k.cfg.ThreadID, activeThreadID, k.global.StuckThreshold)
+				// Kill via MCP API, then fall through to restart
+				k.killThread(ctx, activeThreadID)
+				activeThreadID = k.cfg.ThreadID // reset — will get new worker ID on restart
+			} else {
+				// Healthy — reset counters
+				if retryCount > 0 {
+					k.log.Info("Thread %d is healthy again (was at retry %d)", k.cfg.ThreadID, retryCount)
+				} else {
+					k.log.Debug("Thread %d is healthy", k.cfg.ThreadID)
+				}
+				retryCount = 0
+				return
+			}
+		}
+
+		// Thread is not running (or was stuck and killed)
+		if retryCount >= k.cfg.MaxRetries {
+			cooldown := k.global.KeeperCooldown
+			if k.cfg.CooldownMs > 0 {
+				cooldown = time.Duration(k.cfg.CooldownMs) * time.Millisecond
+			}
+			k.log.Warn("Max retries (%d) exceeded — cooling down for %v", k.cfg.MaxRetries, cooldown)
+			if k.onDeath != nil {
+				k.onDeath(k.cfg.ThreadID, k.cfg.SessionName)
+			}
+			k.sleep(ctx, cooldown)
+			retryCount = 0
+			fastExitCount = 0
+			return
+		}
+
+		k.log.Info("Thread %d not running — calling start_thread (attempt %d/%d)", k.cfg.ThreadID, retryCount+1, k.cfg.MaxRetries)
+
+		// Re-verify root still has keepAlive before restarting
+		// (user may have archived the worker, which disables keepAlive on the root)
+		if !k.isRootKeepAlive(ctx) {
+			k.log.Info("Thread %d root has keepAlive=false — stopping keeper", k.cfg.ThreadID)
+			k.mu.Lock()
+			k.stopped = true
+			k.mu.Unlock()
+			return
+		}
+
+		lastStartTime = time.Now()
+		ok, workerID := k.callStartThread(ctx)
+
+		if ok {
+			if workerID > 0 {
+				activeThreadID = workerID
+				k.log.Info("Thread %d start_thread succeeded (worker %d)", k.cfg.ThreadID, workerID)
+			} else {
+				k.log.Info("Thread %d start_thread succeeded", k.cfg.ThreadID)
+			}
+			retryCount = 0
+			// Check for fast exit on next check
+		} else {
+			// Check for fast exit pattern
+			if !lastStartTime.IsZero() && time.Since(lastStartTime) < k.global.FastExitThreshold {
+				fastExitCount++
+				if fastExitCount >= k.global.FastExitMaxCount {
+					cooldown := time.Duration(float64(k.global.FastExitBaseCooldown) * math.Pow(2, float64(fastExitEscalation)))
+					if cooldown > k.global.FastExitMaxCooldown {
+						cooldown = k.global.FastExitMaxCooldown
+					}
+					k.log.Warn("Thread %d: %d consecutive fast exits — backing off %v", k.cfg.ThreadID, fastExitCount, cooldown)
+					if k.onDeath != nil {
+						k.onDeath(k.cfg.ThreadID, k.cfg.SessionName+" (repeated fast exits — check credits/API key)")
+					}
+					fastExitEscalation++
+					k.sleep(ctx, cooldown)
+					fastExitCount = 0
+					retryCount = 0
+					return
+				}
+			} else {
+				fastExitCount = 0
+				fastExitEscalation = 0
+			}
+
+			retryCount++
+			delay := k.backoff(retryCount)
+			k.log.Info("Backing off %v before next attempt", delay)
+			k.sleep(ctx, delay)
+		}
+	}
+
+	// Initial check
+	checkAndStart()
+
+	// Health check loop
+	for {
+		if k.isStopped() {
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(k.global.KeeperHealthCheckInterval):
+			checkAndStart()
+		}
+	}
+}
+
+// callStartThread starts the thread via MCP. Returns (success, workerThreadID).
+// workerThreadID is extracted from the response JSON when available.
+func (k *Keeper) callStartThread(ctx context.Context) (bool, int) {
+	sessionID, err := k.mcp.OpenMCPSession(ctx)
+	if err != nil {
+		k.log.Error("Failed to open MCP session: %v", err)
+		return false, 0
+	}
+	defer k.mcp.CloseMCPSession(ctx, sessionID)
+
+	text, err := k.mcp.CallStartThread(ctx, sessionID, k.cfg.ThreadID, k.cfg.SessionName, k.cfg.Client, k.cfg.WorkingDirectory)
+	if err != nil {
+		k.log.Error("start_thread failed: %v", err)
+		return false, 0
+	}
+
+	if text != "" {
+		k.log.Info("start_thread response: %.200s", text)
+	}
+
+	// Parse the worker threadId from the response JSON
+	workerID := parseWorkerThreadID(text)
+	return true, workerID
+}
+
+func (k *Keeper) killThread(ctx context.Context, threadID int) {
+	k.log.Info("Killing stuck thread %d", threadID)
+	// Read PID from thread PID file
+	pidFile := k.global.Paths.PIDsDir + "/" + fmt.Sprintf("%d.pid", threadID)
+	pid, err := ReadPIDFile(pidFile)
+	if err != nil {
+		k.log.Warn("Cannot read PID for thread %d: %v", k.cfg.ThreadID, err)
+		return
+	}
+	if err := KillProcess(pid, k.log); err != nil {
+		k.log.Error("Failed to kill thread %d (PID %d): %v", k.cfg.ThreadID, pid, err)
+	}
+}
+
+func (k *Keeper) backoff(retry int) time.Duration {
+	delay := time.Duration(float64(k.global.KeeperBaseBackoff) * math.Pow(2, float64(retry)))
+	if delay > k.global.KeeperMaxBackoff {
+		delay = k.global.KeeperMaxBackoff
+	}
+	return delay
+}
+
+func (k *Keeper) sleep(ctx context.Context, d time.Duration) {
+	select {
+	case <-ctx.Done():
+	case <-time.After(d):
+	}
+}
+
+// isRootKeepAlive checks whether the root thread still has keepAlive=true.
+func (k *Keeper) isRootKeepAlive(ctx context.Context) bool {
+	roots, err := k.mcp.GetRootThreads(ctx)
+	if err != nil {
+		k.log.Debug("isRootKeepAlive(%d): failed to fetch roots: %v — assuming still alive", k.cfg.ThreadID, err)
+		return true // fail-open: don't stop keeper if we can't check
+	}
+	for _, r := range roots {
+		tidFloat, _ := r["threadId"].(float64)
+		if int(tidFloat) == k.cfg.ThreadID {
+			keepAlive, _ := r["keepAlive"].(bool)
+			status, _ := r["status"].(string)
+			return keepAlive && (status == "" || status == "active")
+		}
+	}
+	k.log.Debug("isRootKeepAlive(%d): root thread not found in response", k.cfg.ThreadID)
+	return false // root thread gone → stop keeper
+}
+
+// parseWorkerThreadID extracts the "threadId" field from a start_thread JSON response.
+func parseWorkerThreadID(text string) int {
+	if text == "" {
+		return 0
+	}
+	var resp struct {
+		ThreadID int `json:"threadId"`
+	}
+	if json.Unmarshal([]byte(text), &resp) == nil && resp.ThreadID > 0 {
+		return resp.ThreadID
+	}
+	return 0
+}

package/supervisor/keeper_test.go

@@ -0,0 +1,27 @@
+package main
+
+import "testing"
+
+func TestParseWorkerThreadID(t *testing.T) {
+	tests := []struct {
+		name string
+		text string
+		want int
+	}{
+		{"normal response", `{"threadId":11226,"status":"restarted","name":"Sensorium 2","pid":87108}`, 11226},
+		{"already_running", `{"threadId":11226,"status":"already_running","name":"Sensorium 2","pid":40568}`, 11226},
+		{"empty string", "", 0},
+		{"no threadId", `{"status":"error"}`, 0},
+		{"threadId zero", `{"threadId":0}`, 0},
+		{"invalid JSON", `not json`, 0},
+		{"negative threadId", `{"threadId":-5}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := parseWorkerThreadID(tt.text)
+			if got != tt.want {
+				t.Errorf("parseWorkerThreadID(%q) = %d, want %d", tt.text, got, tt.want)
+			}
+		})
+	}
+}

package/supervisor/lock.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+)
+
+// AcquireLock creates a lock file to prevent multiple supervisor instances.
+// Uses O_CREATE|O_EXCL for atomic creation. If a stale lock exists (PID not
+// running), it reclaims the lock.
+func AcquireLock(lockPath string, log *Logger) bool {
+	// Try atomic create first
+	f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+	if err == nil {
+		// Lock acquired — write our PID
+		fmt.Fprintf(f, "%d", os.Getpid())
+		f.Close()
+		log.Info("Lock acquired: %s (PID %d)", lockPath, os.Getpid())
+		return true
+	}
+
+	// Lock file exists — check if the PID is still alive
+	data, err := os.ReadFile(lockPath)
+	if err != nil {
+		log.Error("Failed to read lockfile %s: %v", lockPath, err)
+		return false
+	}
+
+	pidStr := strings.TrimSpace(string(data))
+	pid, err := strconv.Atoi(pidStr)
+	if err == nil && pid > 0 && IsProcessAlive(pid) {
+		log.Error("Another supervisor is running (PID %d). Lockfile: %s", pid, lockPath)
+		return false
+	}
+
+	// Stale lock — reclaim
+	log.Warn("Reclaimed stale supervisor lockfile (old PID %s)", pidStr)
+	_ = os.Remove(lockPath)
+
+	// Re-acquire atomically
+	f, err = os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+	if err != nil {
+		log.Error("Failed to acquire lockfile after reclaim: %v", err)
+		return false
+	}
+	fmt.Fprintf(f, "%d", os.Getpid())
+	f.Close()
+	return true
+}
+
+// ReleaseLock removes the lock file.
+func ReleaseLock(lockPath string) {
+	_ = os.Remove(lockPath)
+}

package/supervisor/lock_test.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestAcquireLock_Fresh(t *testing.T) {
+	dir := t.TempDir()
+	lockPath := filepath.Join(dir, "test.lock")
+	log := NewLogger("")
+
+	if !AcquireLock(lockPath, log) {
+		t.Fatal("expected lock acquisition to succeed")
+	}
+	defer ReleaseLock(lockPath)
+
+	// Verify lock file was created with our PID
+	data, err := os.ReadFile(lockPath)
+	if err != nil {
+		t.Fatalf("lock file not created: %v", err)
+	}
+	if len(data) == 0 {
+		t.Fatal("lock file is empty")
+	}
+}
+
+func TestAcquireLock_StalePID(t *testing.T) {
+	dir := t.TempDir()
+	lockPath := filepath.Join(dir, "test.lock")
+	log := NewLogger("")
+
+	// Write a stale lock with PID 1 (guaranteed to not be a supervisor)
+	// Use PID 99999999 which is almost certainly not running
+	os.WriteFile(lockPath, []byte("99999999"), 0644)
+
+	if !AcquireLock(lockPath, log) {
+		t.Fatal("expected stale lock to be reclaimed")
+	}
+	defer ReleaseLock(lockPath)
+}
+
+func TestReleaseLock(t *testing.T) {
+	dir := t.TempDir()
+	lockPath := filepath.Join(dir, "test.lock")
+	os.WriteFile(lockPath, []byte("12345"), 0644)
+
+	ReleaseLock(lockPath)
+
+	if _, err := os.Stat(lockPath); !os.IsNotExist(err) {
+		t.Error("expected lock file to be removed")
+	}
+}

package/supervisor/log.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"sync"
+	"time"
+)
+
+// Logger writes to both stderr and a rotating log file.
+// Rotates when the file exceeds maxSize bytes.
+type Logger struct {
+	mu      sync.Mutex
+	logPath string
+	file    *os.File
+	debug   bool
+	size    int64
+	maxSize int64 // default 5 MB
+	maxKeep int   // max rotated files to keep
+}
+
+func NewLogger(logPath string) *Logger {
+	l := &Logger{
+		logPath: logPath,
+		debug:   os.Getenv("SUPERVISOR_DEBUG") == "1" || os.Getenv("SUPERVISOR_DEBUG") == "true",
+		maxSize: 5 * 1024 * 1024, // 5 MB
+		maxKeep: 3,
+	}
+	l.openFile()
+	return l
+}
+
+func (l *Logger) openFile() {
+	if l.logPath == "" {
+		return
+	}
+	f, err := os.OpenFile(l.logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "[WARN] cannot open log file %s: %v\n", l.logPath, err)
+		return
+	}
+	l.file = f
+	// Seed current size for rotation checks
+	if info, err := f.Stat(); err == nil {
+		l.size = info.Size()
+	}
+}
+
+func (l *Logger) log(level, format string, args ...any) {
+	ts := time.Now().Format("2006-01-02 15:04:05")
+	msg := fmt.Sprintf(format, args...)
+	line := fmt.Sprintf("[%s] [%s] %s\n", ts, level, msg)
+
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	fmt.Fprint(os.Stderr, line)
+	if l.file != nil {
+		n, err := l.file.WriteString(line)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "[ERR] log write failed: %v\n", err)
+		}
+		l.size += int64(n)
+		if l.maxSize > 0 && l.size >= l.maxSize {
+			l.rotate()
+		}
+	}
+}
+
+func (l *Logger) Info(format string, args ...any)  { l.log("INFO", format, args...) }
+func (l *Logger) Warn(format string, args ...any)  { l.log("WARN", format, args...) }
+func (l *Logger) Error(format string, args ...any) { l.log("ERROR", format, args...) }
+func (l *Logger) Debug(format string, args ...any) {
+	if l.debug {
+		l.log("DEBUG", format, args...)
+	}
+}
+
+// rotate closes the current log file, renames it with a .1 suffix (shifting
+// older rotated files), and opens a fresh log. Called with mu held.
+func (l *Logger) rotate() {
+	if l.file != nil {
+		l.file.Close()
+		l.file = nil
+	}
+
+	// Shift existing rotated logs: .3 → delete, .2 → .3, .1 → .2, current → .1
+	for i := l.maxKeep; i >= 1; i-- {
+		old := fmt.Sprintf("%s.%d", l.logPath, i)
+		if i == l.maxKeep {
+			os.Remove(old)
+		} else {
+			os.Rename(old, fmt.Sprintf("%s.%d", l.logPath, i+1))
+		}
+	}
+	os.Rename(l.logPath, l.logPath+".1")
+
+	// Open a fresh file
+	l.size = 0
+	f, err := os.OpenFile(l.logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "[WARN] log rotate: cannot create fresh log: %v\n", err)
+		return
+	}
+	l.file = f
+}
+
+func (l *Logger) Close() {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.file != nil {
+		l.file.Close()
+		l.file = nil
+	}
+}

package/supervisor/log_test.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestLogRotation(t *testing.T) {
+	dir := t.TempDir()
+	logPath := filepath.Join(dir, "test.log")
+
+	l := &Logger{
+		logPath: logPath,
+		maxSize: 100, // tiny threshold for test
+		maxKeep: 2,
+	}
+	l.openFile()
+	defer l.Close()
+
+	// Write enough lines to trigger multiple rotations
+	for i := 0; i < 50; i++ {
+		l.Info("line %d: %s", i, strings.Repeat("x", 20))
+	}
+
+	// Current log should exist and be under maxSize
+	info, err := os.Stat(logPath)
+	if err != nil {
+		t.Fatalf("log file missing: %v", err)
+	}
+	if info.Size() >= 100 {
+		t.Errorf("log file should have been rotated, size=%d", info.Size())
+	}
+
+	// At least .1 should exist
+	if _, err := os.Stat(logPath + ".1"); err != nil {
+		t.Error("expected .1 rotated file to exist")
+	}
+
+	// .3 should NOT exist (maxKeep=2)
+	if _, err := os.Stat(logPath + ".3"); err == nil {
+		t.Error("expected .3 file to not exist (maxKeep=2)")
+	}
+}