sensorium-mcp 2.17.25 → 2.17.27

package/supervisor/main.go

@@ -0,0 +1,325 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+)
+
+// KeeperEntry tracks a running keeper and its settings.
+type KeeperEntry struct {
+	keeper   *Keeper
+	settings KeeperConfig
+}
+
+func main() {
+	cfg := LoadConfig()
+
+	if err := os.MkdirAll(cfg.DataDir, 0755); err != nil {
+		fmt.Fprintf(os.Stderr, "Cannot create data dir %s: %v\n", cfg.DataDir, err)
+		os.Exit(1)
+	}
+
+	log := NewLogger(cfg.Paths.WatcherLog)
+	defer log.Close()
+
+	log.Info("sensorium-supervisor starting (mode=%s, port=%d, dataDir=%s)", cfg.Mode, cfg.MCPHttpPort, cfg.DataDir)
+	log.Debug("Config: MCPStartCommand=%q, PollInterval=%v, MinUptime=%v, KeeperMaxRetries=%d", cfg.MCPStartCommand, cfg.PollInterval, cfg.MinUptime, cfg.KeeperMaxRetries)
+	log.Debug("Config: TelegramToken=%v, HealthFailThresh=%d, StuckThreshold=%v", cfg.TelegramToken != "", cfg.HealthFailThresh, cfg.StuckThreshold)
+
+	if err := os.MkdirAll(cfg.Paths.PIDsDir, 0755); err != nil {
+		log.Warn("Cannot create PIDs dir %s: %v", cfg.Paths.PIDsDir, err)
+	}
+	if err := os.MkdirAll(cfg.Paths.HeartbeatsDir, 0755); err != nil {
+		log.Warn("Cannot create heartbeats dir %s: %v", cfg.Paths.HeartbeatsDir, err)
+	}
+
+	// Acquire lock — prevent multiple instances
+	if !AcquireLock(cfg.Paths.WatcherLock, log) {
+		os.Exit(1)
+	}
+	defer ReleaseLock(cfg.Paths.WatcherLock)
+
+	if cfg.MCPHttpPort <= 0 {
+		log.Error("MCP_HTTP_PORT must be set (got %d)", cfg.MCPHttpPort)
+		os.Exit(1)
+	}
+
+	mcp := NewMCPClient(cfg.MCPHttpPort, cfg.MCPHttpSecret)
+	mcp.Log = log
+
+	// Clean stale PID files from previous runs
+	CleanStalePIDs(cfg.Paths.PIDsDir, log)
+
+	// Kill orphan process on our port
+	KillByPort(cfg.MCPHttpPort, log)
+
+	// Spawn MCP server
+	_, err := SpawnMCPServer(cfg, log)
+	if err != nil {
+		log.Error("Failed to start MCP server: %v", err)
+		os.Exit(1)
+	}
+
+	// Wait for server to be ready
+	ctx, rootCancel := context.WithCancel(context.Background())
+	defer rootCancel()
+
+	if mcp.WaitForReady(ctx, 3*time.Second, cfg.KeeperReadyTimeout) {
+		log.Info("MCP server is ready")
+	} else {
+		log.Warn("MCP server did not become ready in %v — proceeding anyway", cfg.KeeperReadyTimeout)
+	}
+
+	// Start keeper management
+	var mu sync.Mutex
+	keepers := make(map[int]*KeeperEntry)
+
+	onDeath := func(threadID int, sessionName string) {
+		log.Warn("Thread %d ('%s') died", threadID, sessionName)
+		NotifyOperator(cfg, log, fmt.Sprintf("💀 <b>%s</b> session died — restarting…", sessionName), threadID)
+	}
+
+	syncKeepers := func() {
+		if cfg.MCPHttpPort <= 0 {
+			log.Debug("syncKeepers: skipped (no port configured)")
+			return
+		}
+
+		log.Debug("syncKeepers: fetching keeper settings...")
+		settings, err := fetchKeeperSettings(ctx, mcp, log)
+		if err != nil {
+			log.Warn("Failed to fetch keeper settings: %v", err)
+			return
+		}
+		log.Debug("syncKeepers: got %d keeper configs", len(settings))
+
+		mu.Lock()
+		defer mu.Unlock()
+
+		// Find keepers to remove (no longer in settings)
+		wanted := make(map[int]bool)
+		for _, s := range settings {
+			wanted[s.ThreadID] = true
+		}
+		for tid, entry := range keepers {
+			if !wanted[tid] {
+				log.Info("Stopping keeper for removed thread %d", tid)
+				entry.keeper.Stop()
+				delete(keepers, tid)
+			}
+		}
+
+		// Start or update keepers
+		for _, s := range settings {
+			existing, exists := keepers[s.ThreadID]
+			if exists && settingsChanged(existing.settings, s) {
+				log.Info("Settings changed for thread %d — restarting keeper", s.ThreadID)
+				existing.keeper.Stop()
+				delete(keepers, s.ThreadID)
+				exists = false
+			}
+			if !exists {
+				k := NewKeeper(s, cfg, mcp, log, onDeath)
+				k.Start()
+				keepers[s.ThreadID] = &KeeperEntry{keeper: k, settings: s}
+				log.Info("Started keeper for thread %d ('%s')", s.ThreadID, s.SessionName)
+			}
+		}
+	}
+
+	// Initial sync
+	log.Info("Running initial keeper sync")
+	syncKeepers()
+
+	// Keeper settings poller (every 2 min)
+	keeperPollerDone := make(chan struct{})
+	go func() {
+		defer close(keeperPollerDone)
+		ticker := time.NewTicker(2 * time.Minute)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				log.Debug("Keeper settings poll triggered")
+				syncKeepers()
+			}
+		}
+	}()
+
+	// Start updater
+	log.Info("Starting auto-updater")
+	updater := NewUpdater(cfg, mcp, log)
+	updater.Start()
+
+	// Health check loop for the server process itself
+	healthDone := make(chan struct{})
+	go func() {
+		defer close(healthDone)
+		consecutiveFails := 0
+		ticker := time.NewTicker(60 * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				if mcp.IsServerReady(ctx) {
+					if consecutiveFails > 0 {
+						log.Info("Server health check recovered (was at %d fails)", consecutiveFails)
+					}
+					consecutiveFails = 0
+				} else {
+					consecutiveFails++
+					log.Warn("Server health check failed (%d/%d)", consecutiveFails, cfg.HealthFailThresh)
+					if consecutiveFails >= cfg.HealthFailThresh {
+						log.Error("Server unresponsive after %d consecutive failures — restarting", consecutiveFails)
+						NotifyOperator(cfg, log, "⚠️ Supervisor: server process not running — restarting...", 0)
+
+						// Kill and respawn
+						pid, pidErr := ReadPIDFile(cfg.Paths.ServerPID)
+						if pidErr != nil {
+							log.Warn("Could not read server PID file: %v", pidErr)
+						}
+						if pid > 0 {
+							_ = KillProcess(pid, log)
+						}
+						KillByPort(cfg.MCPHttpPort, log)
+
+						if _, err := SpawnMCPServer(cfg, log); err != nil {
+							log.Error("Failed to respawn server: %v", err)
+						}
+						consecutiveFails = 0
+					}
+				}
+			}
+		}
+	}()
+
+	// Wait for shutdown signal
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+
+	log.Info("All subsystems started — supervisor is running (PID %d)", os.Getpid())
+
+	sig := <-sigCh
+	log.Info("Received %s — shutting down", sig)
+	rootCancel()
+
+	// Stop keepers (with 10s timeout)
+	shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer shutdownCancel()
+
+	mu.Lock()
+	var wg sync.WaitGroup
+	for _, entry := range keepers {
+		wg.Add(1)
+		go func(k *Keeper) {
+			defer wg.Done()
+			k.Stop()
+		}(entry.keeper)
+	}
+	mu.Unlock()
+
+	doneCh := make(chan struct{})
+	go func() { wg.Wait(); close(doneCh) }()
+	select {
+	case <-doneCh:
+		log.Info("All keepers stopped")
+	case <-shutdownCtx.Done():
+		log.Warn("Keeper shutdown timed out after 10s")
+	}
+
+	// Stop updater
+	updater.Stop()
+
+	// Wait for background goroutines
+	<-keeperPollerDone
+	<-healthDone
+
+	// Kill server process
+	pid, err := ReadPIDFile(cfg.Paths.ServerPID)
+	if err == nil && pid > 0 {
+		log.Info("Stopping MCP server (PID %d)", pid)
+		_ = KillProcess(pid, log)
+	}
+
+	log.Info("Supervisor stopped cleanly")
+}
+
+// fetchKeeperSettings reads the root threads from the MCP server,
+// filtering for those with keepAlive=true.
+func fetchKeeperSettings(ctx context.Context, mcp *MCPClient, log *Logger) ([]KeeperConfig, error) {
+	roots, err := mcp.GetRootThreads(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	var result []KeeperConfig
+	for _, r := range roots {
+		keepAlive, _ := r["keepAlive"].(bool)
+		if !keepAlive {
+			continue
+		}
+
+		// Skip non-active roots (archived, expired, exited)
+		if status, _ := r["status"].(string); status != "" && status != "active" {
+			continue
+		}
+
+		tidFloat, _ := r["threadId"].(float64) // JSON numbers decode as float64
+		tid := int(tidFloat)
+		if tid <= 0 {
+			continue
+		}
+
+		client := "claude"
+		if c, ok := r["client"].(string); ok && c != "" {
+			client = c
+		}
+
+		sessionName := ""
+		if n, ok := r["name"].(string); ok {
+			sessionName = n
+		}
+
+		maxRetries := 5
+		if mr, ok := r["maxRetries"].(float64); ok {
+			maxRetries = int(mr)
+		}
+
+		cooldownMs := 300_000
+		if cd, ok := r["cooldownMs"].(float64); ok {
+			cooldownMs = int(cd)
+		}
+
+		workDir := ""
+		if wd, ok := r["workingDirectory"].(string); ok {
+			workDir = wd
+		}
+
+		result = append(result, KeeperConfig{
+			ThreadID:         tid,
+			SessionName:      sessionName,
+			Client:           client,
+			WorkingDirectory: workDir,
+			MaxRetries:       maxRetries,
+			CooldownMs:       cooldownMs,
+		})
+	}
+	return result, nil
+}
+
+func settingsChanged(a, b KeeperConfig) bool {
+	return a.MaxRetries != b.MaxRetries ||
+		a.CooldownMs != b.CooldownMs ||
+		a.Client != b.Client ||
+		a.SessionName != b.SessionName ||
+		a.WorkingDirectory != b.WorkingDirectory
+}

package/supervisor/notify.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// NotifyOperator sends a message via Telegram to the operator.
+// Silently fails if credentials are not configured.
+func NotifyOperator(cfg Config, log *Logger, text string, threadID int) {
+	if cfg.TelegramToken == "" || cfg.TelegramChatID == "" {
+		log.Debug("NotifyOperator: skipped (no Telegram credentials)")
+		return
+	}
+
+	log.Debug("NotifyOperator: sending to chat %s (threadID=%d)", cfg.TelegramChatID, threadID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	apiURL := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", cfg.TelegramToken)
+
+	form := url.Values{}
+	form.Set("chat_id", cfg.TelegramChatID)
+	form.Set("text", text)
+	form.Set("parse_mode", "HTML")
+	if threadID > 0 {
+		form.Set("message_thread_id", fmt.Sprintf("%d", threadID))
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, strings.NewReader(form.Encode()))
+	if err != nil {
+		log.Warn("Telegram notify: %v", err)
+		return
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Warn("Telegram notify: %v", err)
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode >= 400 {
+		log.Warn("Telegram notify: HTTP %d", resp.StatusCode)
+	} else {
+		log.Debug("Telegram notify: sent OK (HTTP %d)", resp.StatusCode)
+	}
+}

package/supervisor/process.go

@@ -0,0 +1,222 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// SpawnMCPServer starts the MCP server as a detached child process.
+// Returns the PID of the spawned process.
+func SpawnMCPServer(cfg Config, log *Logger) (int, error) {
+	parts := strings.Fields(cfg.MCPStartCommand)
+	if len(parts) == 0 {
+		return 0, errors.New("empty MCP_START_COMMAND")
+	}
+
+	cmd := exec.Command(parts[0], parts[1:]...)
+	cmd.Env = os.Environ()
+	cmd.Stdin = nil
+	cmd.Stdout = nil
+	cmd.Stderr = nil
+	setSysProcAttr(cmd)
+
+	log.Info("Starting MCP server: %s", cfg.MCPStartCommand)
+
+	if err := cmd.Start(); err != nil {
+		return 0, fmt.Errorf("spawn MCP server: %w", err)
+	}
+
+	pid := cmd.Process.Pid
+	log.Info("MCP server started with PID %d", pid)
+
+	// Don't wait — detached process
+	go func() { _ = cmd.Wait() }()
+
+	if err := writePIDFile(cfg.Paths.ServerPID, pid); err != nil {
+		log.Warn("Failed to write server PID file: %v", err)
+	}
+
+	return pid, nil
+}
+
+// KillProcess kills a process by PID. On Windows, uses taskkill /F /T for tree kill.
+func KillProcess(pid int, log *Logger) error {
+	if !IsProcessAlive(pid) {
+		log.Debug("KillProcess: PID %d already dead", pid)
+		return nil
+	}
+
+	log.Debug("KillProcess: killing PID %d", pid)
+
+	if runtime.GOOS == "windows" {
+		// taskkill /F /T kills the tree
+		out, err := exec.Command("taskkill", "/F", "/T", "/PID", strconv.Itoa(pid)).CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("taskkill PID %d: %w (%s)", pid, err, strings.TrimSpace(string(out)))
+		}
+		log.Info("Killed process tree PID %d", pid)
+		return nil
+	}
+
+	// Unix: SIGTERM, wait 2s, then SIGKILL
+	proc, err := os.FindProcess(pid)
+	if err != nil {
+		log.Debug("KillProcess: FindProcess(%d) failed: %v", pid, err)
+		return err
+	}
+	if err := proc.Signal(os.Interrupt); err != nil {
+		// Already dead
+		return nil
+	}
+	time.Sleep(2 * time.Second)
+	if IsProcessAlive(pid) {
+		_ = proc.Kill()
+		log.Info("Force-killed PID %d", pid)
+	} else {
+		log.Info("Process PID %d terminated gracefully", pid)
+	}
+	return nil
+}
+
+// KillByPort finds a process listening on the given port and kills it (Windows-only orphan cleanup).
+func KillByPort(port int, log *Logger) {
+	if runtime.GOOS != "windows" || port <= 0 || port > 65535 {
+		return
+	}
+	log.Debug("KillByPort: checking for processes on port %d", port)
+	out, err := exec.Command("cmd", "/c", fmt.Sprintf("netstat -aon | findstr \":%d.*LISTENING\"", port)).CombinedOutput()
+	if err != nil {
+		log.Debug("KillByPort: no listeners on port %d", port)
+		return
+	}
+	for _, line := range strings.Split(string(out), "\n") {
+		fields := strings.Fields(strings.TrimSpace(line))
+		if len(fields) >= 5 {
+			pid, err := strconv.Atoi(fields[len(fields)-1])
+			if err == nil && pid > 0 {
+				log.Info("Found orphan PID %d on port %d — killing", pid, port)
+				_ = KillProcess(pid, log)
+			}
+		}
+	}
+}
+
+// IsProcessAlive checks whether a process with the given PID exists.
+func IsProcessAlive(pid int) bool {
+	if pid <= 0 {
+		return false
+	}
+	if runtime.GOOS == "windows" {
+		out, err := exec.Command("tasklist", "/FI", fmt.Sprintf("PID eq %d", pid), "/NH").CombinedOutput()
+		if err != nil {
+			return false
+		}
+		return strings.Contains(string(out), strconv.Itoa(pid))
+	}
+	proc, err := os.FindProcess(pid)
+	if err != nil {
+		return false
+	}
+	return proc.Signal(nil) == nil // signal 0 = existence check on Unix
+}
+
+// --- PID File Helpers ---
+
+type pidJSON struct {
+	PID int `json:"pid"`
+}
+
+// ReadPIDFile reads a PID from a file. Supports both JSON {"pid":123} and raw integer formats.
+func ReadPIDFile(path string) (int, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return 0, err
+	}
+	raw := strings.TrimSpace(string(data))
+
+	// Try JSON first
+	var pj pidJSON
+	if json.Unmarshal([]byte(raw), &pj) == nil && pj.PID > 0 {
+		return pj.PID, nil
+	}
+
+	// Fallback: raw integer
+	pid, err := strconv.Atoi(raw)
+	if err != nil {
+		return 0, fmt.Errorf("invalid PID file content: %q", raw)
+	}
+	if pid <= 0 {
+		return 0, fmt.Errorf("invalid PID: %d", pid)
+	}
+	return pid, nil
+}
+
+func writePIDFile(path string, pid int) error {
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return err
+	}
+	data, _ := json.Marshal(pidJSON{PID: pid})
+	return atomicWrite(path, data)
+}
+
+// atomicWrite writes data to a temp file then renames — prevents partial reads.
+func atomicWrite(path string, data []byte) error {
+	tmp := fmt.Sprintf("%s.tmp.%d", path, os.Getpid())
+	if err := os.WriteFile(tmp, data, 0644); err != nil {
+		return err
+	}
+	return os.Rename(tmp, path)
+}
+
+// ListThreadPIDs returns a map of threadId → PID from the pids directory.
+func ListThreadPIDs(pidsDir string) map[string]int {
+	result := make(map[string]int)
+	entries, err := os.ReadDir(pidsDir)
+	if err != nil {
+		// directory may not exist yet
+		return result
+	}
+	for _, e := range entries {
+		if e.IsDir() || !strings.HasSuffix(e.Name(), ".pid") {
+			continue
+		}
+		threadID := strings.TrimSuffix(e.Name(), ".pid")
+		pid, err := ReadPIDFile(filepath.Join(pidsDir, e.Name()))
+		if err != nil {
+			continue
+		}
+		result[threadID] = pid
+	}
+	return result
+}
+
+// CleanStalePIDs removes PID files for processes that are no longer running.
+func CleanStalePIDs(pidsDir string, log *Logger) {
+	pids := ListThreadPIDs(pidsDir)
+	if len(pids) == 0 {
+		log.Debug("CleanStalePIDs: no PID files found in %s", pidsDir)
+		return
+	}
+	log.Debug("CleanStalePIDs: checking %d PID files", len(pids))
+	cleaned := 0
+	for threadID, pid := range pids {
+		if !IsProcessAlive(pid) {
+			path := filepath.Join(pidsDir, threadID+".pid")
+			log.Info("Removing stale PID file for thread %s (PID %d)", threadID, pid)
+			_ = os.Remove(path)
+			cleaned++
+		}
+	}
+	if cleaned > 0 {
+		log.Info("CleanStalePIDs: removed %d stale PID files", cleaned)
+	}
+}

package/supervisor/process_test.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestReadPIDFile_JSON(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.pid")
+	os.WriteFile(path, []byte(`{"pid":12345}`), 0644)
+
+	pid, err := ReadPIDFile(path)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if pid != 12345 {
+		t.Errorf("got %d, want 12345", pid)
+	}
+}
+
+func TestReadPIDFile_RawInt(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.pid")
+	os.WriteFile(path, []byte("54321\n"), 0644)
+
+	pid, err := ReadPIDFile(path)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if pid != 54321 {
+		t.Errorf("got %d, want 54321", pid)
+	}
+}
+
+func TestReadPIDFile_Invalid(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.pid")
+	os.WriteFile(path, []byte("not-a-pid"), 0644)
+
+	_, err := ReadPIDFile(path)
+	if err == nil {
+		t.Fatal("expected error for invalid PID content")
+	}
+}
+
+func TestReadPIDFile_Missing(t *testing.T) {
+	_, err := ReadPIDFile(filepath.Join(t.TempDir(), "missing.pid"))
+	if err == nil {
+		t.Fatal("expected error for missing file")
+	}
+}
+
+func TestAtomicWrite(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "data.txt")
+
+	if err := atomicWrite(path, []byte("hello")); err != nil {
+		t.Fatalf("atomicWrite failed: %v", err)
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("ReadFile failed: %v", err)
+	}
+	if string(data) != "hello" {
+		t.Errorf("got %q, want %q", string(data), "hello")
+	}
+}
+
+func TestListThreadPIDs(t *testing.T) {
+	dir := t.TempDir()
+	os.WriteFile(filepath.Join(dir, "1234.pid"), []byte(`{"pid":100}`), 0644)
+	os.WriteFile(filepath.Join(dir, "5678.pid"), []byte("200"), 0644)
+	os.WriteFile(filepath.Join(dir, "not-a-pid.txt"), []byte("300"), 0644)
+
+	result := ListThreadPIDs(dir)
+	if len(result) != 2 {
+		t.Fatalf("got %d entries, want 2", len(result))
+	}
+	if result["1234"] != 100 {
+		t.Errorf("result[1234] = %d, want 100", result["1234"])
+	}
+	if result["5678"] != 200 {
+		t.Errorf("result[5678] = %d, want 200", result["5678"])
+	}
+}
+
+func TestListThreadPIDs_MissingDir(t *testing.T) {
+	result := ListThreadPIDs(filepath.Join(t.TempDir(), "no-such-dir"))
+	if len(result) != 0 {
+		t.Errorf("expected empty map for missing directory, got %d entries", len(result))
+	}
+}

package/supervisor/process_unix.go

@@ -0,0 +1,14 @@
+//go:build !windows
+
+package main
+
+import (
+	"os/exec"
+	"syscall"
+)
+
+func setSysProcAttr(cmd *exec.Cmd) {
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setsid: true, // detach from parent session
+	}
+}