sensorium-mcp 3.0.4 → 3.0.6

package/supervisor/keeper.go

@@ -1,309 +0,0 @@
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"math"
-	"path/filepath"
-	"sync"
-	"time"
-)
-
-// KeeperConfig describes a thread that must be kept alive.
-type KeeperConfig struct {
-	ThreadID         int
-	SessionName      string
-	Client           string // e.g. "claude-code", "codex"
-	WorkingDirectory string
-	MaxRetries       int
-	CooldownMs       int
-}
-
-// Keeper supervises a single thread, restarting it via the MCP server's
-// start_thread tool when it stops running. One goroutine per keeper.
-type Keeper struct {
-	cfg     KeeperConfig
-	global  Config
-	mcp     *MCPClient
-	log     *Logger
-	onDeath func(threadID int, sessionName string)
-
-	mu      sync.Mutex
-	stopped bool
-	cancel  context.CancelFunc
-	done    chan struct{}
-}
-
-func NewKeeper(cfg KeeperConfig, global Config, mcp *MCPClient, log *Logger, onDeath func(int, string)) *Keeper {
-	maxRetries := cfg.MaxRetries
-	if maxRetries <= 0 {
-		maxRetries = global.KeeperMaxRetries
-	}
-	cfg.MaxRetries = maxRetries
-	return &Keeper{
-		cfg:     cfg,
-		global:  global,
-		mcp:     mcp,
-		log:     log,
-		onDeath: onDeath,
-		done:    make(chan struct{}),
-	}
-}
-
-// Start begins the keeper loop in a separate goroutine.
-func (k *Keeper) Start() {
-	ctx, cancel := context.WithCancel(context.Background())
-	k.mu.Lock()
-	k.cancel = cancel
-	k.mu.Unlock()
-
-	go k.run(ctx)
-}
-
-// Stop signals the keeper to shut down and waits for it to finish.
-func (k *Keeper) Stop() {
-	k.mu.Lock()
-	k.stopped = true
-	if k.cancel != nil {
-		k.cancel()
-	}
-	k.mu.Unlock()
-	<-k.done
-}
-
-func (k *Keeper) isStopped() bool {
-	k.mu.Lock()
-	defer k.mu.Unlock()
-	return k.stopped
-}
-
-func (k *Keeper) run(ctx context.Context) {
-	defer close(k.done)
-	defer func() {
-		if r := recover(); r != nil {
-			k.log.Error("Keeper panicked for thread %d: %v", k.cfg.ThreadID, r)
-		}
-	}()
-	defer k.log.Info("Keeper stopped for thread %d", k.cfg.ThreadID)
-
-	k.log.Info("Keeper started for thread %d ('%s') [client=%s]", k.cfg.ThreadID, k.cfg.SessionName, k.cfg.Client)
-
-	// Wait for MCP server to be ready
-	ready := k.mcp.WaitForReady(ctx, k.global.KeeperReadyPollInterval, k.global.KeeperReadyTimeout)
-	if !ready && !k.isStopped() {
-		k.log.Warn("MCP server not ready after %v — attempting start_thread anyway", k.global.KeeperReadyTimeout)
-	}
-
-	retryCount := 0
-	fastExitCount := 0
-	fastExitEscalation := 0
-	var lastStartTime time.Time
-	activeThreadID := k.cfg.ThreadID // may differ from root after start_thread
-
-	checkAndStart := func() {
-		if k.isStopped() {
-			return
-		}
-
-		// Check if thread is running — use activeThreadID (worker) not root
-		running := k.mcp.IsThreadRunning(ctx, activeThreadID)
-		if running {
-			// Check if stuck
-			if k.mcp.IsThreadStuck(ctx, activeThreadID, k.global.StuckThreshold) {
-				k.log.Warn("Thread %d (worker %d) is stuck (no heartbeat for %v) — restarting", k.cfg.ThreadID, activeThreadID, k.global.StuckThreshold)
-				// Kill via MCP API, then fall through to restart
-				k.killThread(ctx, activeThreadID)
-				retryCount = 0
-				activeThreadID = k.cfg.ThreadID // reset — will get new worker ID on restart
-			} else {
-				// Healthy — reset counters
-				if retryCount > 0 {
-					k.log.Info("Thread %d is healthy again (was at retry %d)", k.cfg.ThreadID, retryCount)
-				} else {
-					k.log.Info("Thread %d is healthy", k.cfg.ThreadID)
-				}
-				retryCount = 0
-				fastExitCount = 0
-				fastExitEscalation = 0
-				return
-			}
-		}
-
-		// Thread is not running (or was stuck and killed).
-		// Detect fast-exit: if the previous successful start was recent, the thread exited too quickly.
-		if !lastStartTime.IsZero() && time.Since(lastStartTime) < k.global.FastExitThreshold {
-			fastExitCount++
-			if fastExitCount >= k.global.FastExitMaxCount {
-				cooldown := time.Duration(float64(k.global.FastExitBaseCooldown) * math.Pow(2, float64(fastExitEscalation)))
-				if cooldown > k.global.FastExitMaxCooldown {
-					cooldown = k.global.FastExitMaxCooldown
-				}
-				k.log.Warn("Thread %d: %d consecutive fast exits — backing off %v", k.cfg.ThreadID, fastExitCount, cooldown)
-				if k.onDeath != nil {
-					k.onDeath(k.cfg.ThreadID, k.cfg.SessionName+" (repeated fast exits — check credits/API key)")
-				}
-				fastExitEscalation++
-				k.sleep(ctx, cooldown)
-				fastExitCount = 0
-				retryCount = 0
-				return
-			}
-		} else if !lastStartTime.IsZero() {
-			// Previous start was long ago — not a fast exit pattern, reset counters
-			fastExitCount = 0
-			fastExitEscalation = 0
-		}
-
-		if retryCount >= k.cfg.MaxRetries {
-			cooldown := k.global.KeeperCooldown
-			if k.cfg.CooldownMs > 0 {
-				cooldown = time.Duration(k.cfg.CooldownMs) * time.Millisecond
-			}
-			k.log.Warn("Max retries (%d) exceeded — cooling down for %v", k.cfg.MaxRetries, cooldown)
-			if k.onDeath != nil {
-				k.onDeath(k.cfg.ThreadID, k.cfg.SessionName)
-			}
-			k.sleep(ctx, cooldown)
-			retryCount = 0
-			fastExitCount = 0
-			return
-		}
-
-		k.log.Info("Thread %d not running — calling start_thread (attempt %d/%d)", k.cfg.ThreadID, retryCount+1, k.cfg.MaxRetries)
-
-		// Re-verify root still has keepAlive before restarting
-		// (user may have archived the worker, which disables keepAlive on the root)
-		if !k.isRootKeepAlive(ctx) {
-			k.log.Info("Thread %d root has keepAlive=false — stopping keeper", k.cfg.ThreadID)
-			k.mu.Lock()
-			k.stopped = true
-			k.mu.Unlock()
-			return
-		}
-
-		ok, workerID := k.callStartThread(ctx)
-
-		if ok {
-			lastStartTime = time.Now()
-			if workerID > 0 {
-				activeThreadID = workerID
-				k.log.Info("Thread %d start_thread succeeded (worker %d)", k.cfg.ThreadID, workerID)
-			} else {
-				k.log.Info("Thread %d start_thread succeeded", k.cfg.ThreadID)
-			}
-			retryCount = 0
-		} else {
-			retryCount++
-			delay := k.backoff(retryCount)
-			k.log.Info("Backing off %v before next attempt", delay)
-			k.sleep(ctx, delay)
-		}
-	}
-
-	// Initial check
-	checkAndStart()
-
-	// Health check loop
-	for {
-		if k.isStopped() {
-			return
-		}
-
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(k.global.KeeperHealthCheckInterval):
-			checkAndStart()
-		}
-	}
-}
-
-// callStartThread starts the thread via MCP. Returns (success, workerThreadID).
-// workerThreadID is extracted from the response JSON when available.
-func (k *Keeper) callStartThread(ctx context.Context) (bool, int) {
-	sessionID, err := k.mcp.OpenMCPSession(ctx)
-	if err != nil {
-		k.log.Error("Failed to open MCP session: %v", err)
-		return false, 0
-	}
-	defer k.mcp.CloseMCPSession(ctx, sessionID)
-
-	text, err := k.mcp.CallStartThread(ctx, sessionID, k.cfg.ThreadID, k.cfg.SessionName, k.cfg.Client, k.cfg.WorkingDirectory)
-	if err != nil {
-		k.log.Error("start_thread failed: %v", err)
-		return false, 0
-	}
-
-	if text != "" {
-		k.log.Info("start_thread response: %.200s", text)
-	}
-
-	// Parse the worker threadId from the response JSON
-	workerID := parseWorkerThreadID(text)
-	return true, workerID
-}
-
-func (k *Keeper) killThread(ctx context.Context, threadID int) {
-	k.log.Info("Killing stuck thread %d", threadID)
-	// Read PID from thread PID file
-	pidFile := filepath.Join(k.global.Paths.PIDsDir, fmt.Sprintf("%d.pid", threadID))
-	pid, err := ReadPIDFile(pidFile)
-	if err != nil {
-		k.log.Warn("Cannot read PID for thread %d: %v", threadID, err)
-		return
-	}
-	if err := KillProcess(pid, k.log); err != nil {
-		k.log.Error("Failed to kill thread %d (PID %d): %v", threadID, pid, err)
-	}
-}
-
-func (k *Keeper) backoff(retry int) time.Duration {
-	delay := time.Duration(float64(k.global.KeeperBaseBackoff) * math.Pow(2, float64(retry)))
-	if delay > k.global.KeeperMaxBackoff {
-		delay = k.global.KeeperMaxBackoff
-	}
-	return delay
-}
-
-func (k *Keeper) sleep(ctx context.Context, d time.Duration) {
-	select {
-	case <-ctx.Done():
-	case <-time.After(d):
-	}
-}
-
-// isRootKeepAlive checks whether the thread still has keepAlive=true.
-// Uses the /api/threads/keepalive endpoint (root, branch, daily — excludes workers)
-// so that branch and daily threads are correctly included in the check.
-func (k *Keeper) isRootKeepAlive(ctx context.Context) bool {
-	threads, err := k.mcp.GetKeepAliveThreads(ctx)
-	if err != nil {
-		k.log.Debug("isRootKeepAlive(%d): failed to fetch keepalive threads: %v — assuming still alive", k.cfg.ThreadID, err)
-		return true // fail-open: don't stop keeper if we can't check
-	}
-	for _, r := range threads {
-		tidFloat, _ := r["threadId"].(float64)
-		if int(tidFloat) == k.cfg.ThreadID {
-			keepAlive, _ := r["keepAlive"].(bool)
-			status, _ := r["status"].(string)
-			return keepAlive && (status == "" || status == "active")
-		}
-	}
-	k.log.Debug("isRootKeepAlive(%d): thread not found in keepalive list", k.cfg.ThreadID)
-	return false // thread gone or keepAlive removed → stop keeper
-}
-
-// parseWorkerThreadID extracts the "threadId" field from a start_thread JSON response.
-func parseWorkerThreadID(text string) int {
-	if text == "" {
-		return 0
-	}
-	var resp struct {
-		ThreadID int `json:"threadId"`
-	}
-	if json.Unmarshal([]byte(text), &resp) == nil && resp.ThreadID > 0 {
-		return resp.ThreadID
-	}
-	return 0
-}

package/supervisor/keeper_test.go

@@ -1,27 +0,0 @@
-package main
-
-import "testing"
-
-func TestParseWorkerThreadID(t *testing.T) {
-	tests := []struct {
-		name string
-		text string
-		want int
-	}{
-		{"normal response", `{"threadId":11226,"status":"restarted","name":"Sensorium 2","pid":87108}`, 11226},
-		{"already_running", `{"threadId":11226,"status":"already_running","name":"Sensorium 2","pid":40568}`, 11226},
-		{"empty string", "", 0},
-		{"no threadId", `{"status":"error"}`, 0},
-		{"threadId zero", `{"threadId":0}`, 0},
-		{"invalid JSON", `not json`, 0},
-		{"negative threadId", `{"threadId":-5}`, 0},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := parseWorkerThreadID(tt.text)
-			if got != tt.want {
-				t.Errorf("parseWorkerThreadID(%q) = %d, want %d", tt.text, got, tt.want)
-			}
-		})
-	}
-}

package/supervisor/lock.go

@@ -1,57 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-	"strconv"
-	"strings"
-)
-
-// AcquireLock creates a lock file to prevent multiple supervisor instances.
-// Uses O_CREATE|O_EXCL for atomic creation. If a stale lock exists (PID not
-// running), it reclaims the lock.
-func AcquireLock(lockPath string, log *Logger) bool {
-	// Try atomic create first
-	f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
-	if err == nil {
-		// Lock acquired — write our PID
-		fmt.Fprintf(f, "%d", os.Getpid())
-		f.Close()
-		log.Info("Lock acquired: %s (PID %d)", lockPath, os.Getpid())
-		return true
-	}
-
-	// Lock file exists — check if the PID is still alive
-	data, err := os.ReadFile(lockPath)
-	if err != nil {
-		log.Error("Failed to read lockfile %s: %v", lockPath, err)
-		return false
-	}
-
-	pidStr := strings.TrimSpace(string(data))
-	pid, err := strconv.Atoi(pidStr)
-	if err == nil && pid > 0 && IsProcessAlive(pid) {
-		log.Error("Another supervisor is running (PID %d). Lockfile: %s", pid, lockPath)
-		return false
-	}
-
-	// Stale lock — reclaim
-	log.Warn("Reclaimed stale supervisor lockfile (old PID %s)", pidStr)
-	_ = os.Remove(lockPath)
-
-	// Re-acquire atomically
-	f, err = os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
-	if err != nil {
-		log.Error("Failed to acquire lockfile after reclaim: %v", err)
-		return false
-	}
-	fmt.Fprintf(f, "%d", os.Getpid())
-	f.Close()
-	log.Info("Lock acquired: %s (PID %d)", lockPath, os.Getpid())
-	return true
-}
-
-// ReleaseLock removes the lock file.
-func ReleaseLock(lockPath string) {
-	_ = os.Remove(lockPath)
-}

package/supervisor/lock_test.go

@@ -1,54 +0,0 @@
-package main
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-func TestAcquireLock_Fresh(t *testing.T) {
-	dir := t.TempDir()
-	lockPath := filepath.Join(dir, "test.lock")
-	log := NewLogger("")
-
-	if !AcquireLock(lockPath, log) {
-		t.Fatal("expected lock acquisition to succeed")
-	}
-	defer ReleaseLock(lockPath)
-
-	// Verify lock file was created with our PID
-	data, err := os.ReadFile(lockPath)
-	if err != nil {
-		t.Fatalf("lock file not created: %v", err)
-	}
-	if len(data) == 0 {
-		t.Fatal("lock file is empty")
-	}
-}
-
-func TestAcquireLock_StalePID(t *testing.T) {
-	dir := t.TempDir()
-	lockPath := filepath.Join(dir, "test.lock")
-	log := NewLogger("")
-
-	// Write a stale lock with PID 1 (guaranteed to not be a supervisor)
-	// Use PID 99999999 which is almost certainly not running
-	os.WriteFile(lockPath, []byte("99999999"), 0644)
-
-	if !AcquireLock(lockPath, log) {
-		t.Fatal("expected stale lock to be reclaimed")
-	}
-	defer ReleaseLock(lockPath)
-}
-
-func TestReleaseLock(t *testing.T) {
-	dir := t.TempDir()
-	lockPath := filepath.Join(dir, "test.lock")
-	os.WriteFile(lockPath, []byte("12345"), 0644)
-
-	ReleaseLock(lockPath)
-
-	if _, err := os.Stat(lockPath); !os.IsNotExist(err) {
-		t.Error("expected lock file to be removed")
-	}
-}

package/supervisor/log.go

@@ -1,195 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-)
-
-// Logger writes to both stderr and a rotating log file.
-// Rotates daily (at midnight) and when the file exceeds maxSize bytes.
-type Logger struct {
-	mu        sync.Mutex
-	logPath   string
-	file      *os.File
-	debug     bool
-	size      int64
-	maxSize   int64 // default 5 MB
-	maxKeep   int   // max daily rotated files to keep
-	today     string
-	stopTimer chan struct{}
-}
-
-func NewLogger(logPath string) *Logger {
-	l := &Logger{
-		logPath:   logPath,
-		debug:     os.Getenv("SUPERVISOR_DEBUG") == "1" || os.Getenv("SUPERVISOR_DEBUG") == "true",
-		maxSize:   5 * 1024 * 1024, // 5 MB
-		maxKeep:   7,               // keep 7 daily files
-		stopTimer: make(chan struct{}),
-	}
-	// Rotate previous day's log on startup if needed
-	l.today = time.Now().Format("2006-01-02")
-	l.rotateDailyIfNeeded()
-	l.openFile()
-	l.startMidnightTimer()
-	return l
-}
-
-func (l *Logger) openFile() {
-	if l.logPath == "" {
-		return
-	}
-	f, err := os.OpenFile(l.logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "[WARN] cannot open log file %s: %v\n", l.logPath, err)
-		return
-	}
-	l.file = f
-	// Seed current size for rotation checks
-	if info, err := f.Stat(); err == nil {
-		l.size = info.Size()
-	}
-}
-
-func (l *Logger) log(level, format string, args ...any) {
-	ts := time.Now().Format("2006-01-02 15:04:05")
-	msg := fmt.Sprintf(format, args...)
-	line := fmt.Sprintf("[%s] [%s] %s\n", ts, level, msg)
-
-	l.mu.Lock()
-	defer l.mu.Unlock()
-	// Always write to file for post-mortem debugging; only emit DEBUG to stderr
-	// when SUPERVISOR_DEBUG is set.
-	if level != "DEBUG" || l.debug {
-		fmt.Fprint(os.Stderr, line)
-	}
-	if l.file != nil {
-		n, err := l.file.WriteString(line)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "[ERR] log write failed: %v\n", err)
-		}
-		l.size += int64(n)
-		if l.maxSize > 0 && l.size >= l.maxSize {
-			l.rotate()
-		}
-	}
-}
-
-func (l *Logger) Info(format string, args ...any)  { l.log("INFO", format, args...) }
-func (l *Logger) Warn(format string, args ...any)  { l.log("WARN", format, args...) }
-func (l *Logger) Error(format string, args ...any) { l.log("ERROR", format, args...) }
-func (l *Logger) Debug(format string, args ...any) { l.log("DEBUG", format, args...) }
-
-// rotateDailyIfNeeded renames the current log to a dated archive if it was
-// written on a previous day. Called without mu held (used at startup and from
-// the midnight timer before the lock is acquired).
-func (l *Logger) rotateDailyIfNeeded() {
-	info, err := os.Stat(l.logPath)
-	if err != nil {
-		return // file doesn't exist yet — nothing to rotate
-	}
-	modDay := info.ModTime().Format("2006-01-02")
-	if modDay == l.today {
-		return // same day — no rotation needed
-	}
-	// Rename to dated file
-	ext := filepath.Ext(l.logPath)
-	base := strings.TrimSuffix(l.logPath, ext)
-	dated := fmt.Sprintf("%s.%s%s", base, modDay, ext)
-	if err := os.Rename(l.logPath, dated); err != nil {
-		fmt.Fprintf(os.Stderr, "[WARN] daily log rotate: %v\n", err)
-	}
-	l.pruneOldLogs()
-}
-
-// pruneOldLogs deletes daily log files beyond maxKeep. Called without mu held.
-func (l *Logger) pruneOldLogs() {
-	dir := filepath.Dir(l.logPath)
-	base := strings.TrimSuffix(filepath.Base(l.logPath), filepath.Ext(l.logPath))
-	entries, err := os.ReadDir(dir)
-	if err != nil {
-		return
-	}
-	var dated []string
-	for _, e := range entries {
-		name := e.Name()
-		if strings.HasPrefix(name, base+".") && name != filepath.Base(l.logPath) {
-			dated = append(dated, filepath.Join(dir, name))
-		}
-	}
-	sort.Strings(dated) // ascending — oldest first
-	for len(dated) > l.maxKeep {
-		os.Remove(dated[0])
-		dated = dated[1:]
-	}
-}
-
-// startMidnightTimer fires a daily rotation at local midnight.
-func (l *Logger) startMidnightTimer() {
-	stopCh := l.stopTimer // capture channel value; Close() may nil the field concurrently
-	go func() {
-		for {
-			now := time.Now()
-			next := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, now.Location())
-			select {
-			case <-time.After(time.Until(next)):
-			case <-stopCh:
-				return
-			}
-			l.mu.Lock()
-			l.today = time.Now().Format("2006-01-02")
-			if l.file != nil {
-				l.file.Close()
-				l.file = nil
-			}
-			l.rotateDailyIfNeeded()
-			l.size = 0
-			l.openFile()
-			l.mu.Unlock()
-		}
-	}()
-}
-
-// rotate closes the current log file, renames it with a dated suffix for
-// size-based rotation mid-day, and opens a fresh log. Called with mu held.
-func (l *Logger) rotate() {
-	if l.file != nil {
-		l.file.Close()
-		l.file = nil
-	}
-	// Use a timestamp suffix to avoid colliding with the daily dated file
-	ts := time.Now().Format("2006-01-02T150405")
-	ext := filepath.Ext(l.logPath)
-	base := strings.TrimSuffix(l.logPath, ext)
-	if err := os.Rename(l.logPath, fmt.Sprintf("%s.%s%s", base, ts, ext)); err != nil {
-		fmt.Fprintf(os.Stderr, "[WARN] log size-rotate rename: %v\n", err)
-	}
-	l.pruneOldLogs()
-
-	// Open a fresh file
-	l.size = 0
-	f, err := os.OpenFile(l.logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "[WARN] log rotate: cannot create fresh log: %v\n", err)
-		return
-	}
-	l.file = f
-}
-
-func (l *Logger) Close() {
-	if l.stopTimer != nil {
-		close(l.stopTimer)
-		l.stopTimer = nil
-	}
-	l.mu.Lock()
-	defer l.mu.Unlock()
-	if l.file != nil {
-		l.file.Close()
-		l.file = nil
-	}
-}