pi-deadman 1.0.0

package/extensions/signals.ts

@@ -0,0 +1,172 @@
+// signals.ts — macOS kernel memory metrics: sysctl, vm_stat, memory_pressure
+
+import { execSync } from "child_process";
+
+// Module-level state for rate computation
+let previousVmStat: Record<string, number> | null = null;
+let previousTimestamp: number | null = null;
+
+export interface SystemSignals {
+  swapout_rate: number;     // swapouts/sec delta from previous call, 0 on first call
+  swapin_rate: number;      // swapins/sec delta
+  decomp_rate: number;      // decompressions/sec delta
+  pressure_level: number;   // 1, 2, or 4 from memory_pressure command
+  memorystatus_level: number; // 0-100 from sysctl kern.memorystatus_level
+  swap_used_mb: number;     // from sysctl vm.swapusage
+  swap_free_mb: number;     // from sysctl vm.swapusage
+  compression_ratio: number; // pages_stored / compressor_page_count, min 1.0
+}
+
+export async function collectSignals(): Promise<SystemSignals> {
+  const timestamp = Date.now() / 1000; // Convert to seconds
+  
+  // Initialize with safe defaults
+  let swapout_rate = 0;
+  let swapin_rate = 0;
+  let decomp_rate = 0;
+  let pressure_level = 1;
+  let memorystatus_level = 0;
+  let swap_used_mb = 0;
+  let swap_free_mb = 0;
+  let compression_ratio = 1.0;
+
+  // Get current vm_stat data
+  const currentVmStat = getVmStat();
+
+  // Compute delta-based rates if we have previous data
+  if (previousVmStat !== null && previousTimestamp !== null) {
+    const timeDelta = timestamp - previousTimestamp;
+    if (timeDelta > 0) {
+      // Swapout rate: pages written to swap per second
+      const prevPageouts = previousVmStat.swapouts || 0;
+      const currPageouts = currentVmStat.swapouts || 0;
+      swapout_rate = Math.max(0, (currPageouts - prevPageouts) / timeDelta);
+
+      // Swapin rate: pages read from swap per second (swapins)
+      const prevPageins = previousVmStat.swapins || 0;
+      const currPageins = currentVmStat.swapins || 0;
+      swapin_rate = Math.max(0, (currPageins - prevPageins) / timeDelta);
+
+      // Decompression rate: pages decompressed per second
+      const prevDecomp = previousVmStat.decompressions || 0;
+      const currDecomp = currentVmStat.decompressions || 0;
+      decomp_rate = Math.max(0, (currDecomp - prevDecomp) / timeDelta);
+    }
+  }
+
+  // Store current state for next call
+  previousVmStat = currentVmStat;
+  previousTimestamp = timestamp;
+
+  // Get point-in-time signals
+  pressure_level = getPressureLevel();
+  memorystatus_level = getMemorystatusLevel();
+  [swap_used_mb, swap_free_mb] = getSwapUsage();
+  compression_ratio = getCompressionRatio(currentVmStat);
+
+  return {
+    swapout_rate,
+    swapin_rate,
+    decomp_rate,
+    pressure_level,
+    memorystatus_level,
+    swap_used_mb,
+    swap_free_mb,
+    compression_ratio
+  };
+}
+
+function getVmStat(): Record<string, number> {
+  try {
+    const output = execSync("vm_stat", { encoding: "utf8", timeout: 5000 });
+    const data: Record<string, number> = {};
+    
+    for (const line of output.trim().split("\n")) {
+      if (line.includes(":")) {
+        const [key, valueStr] = line.split(":");
+        const value = valueStr.trim().replace(/\.$/, ""); // Remove trailing period
+        const numValue = parseInt(value, 10);
+        if (!isNaN(numValue)) {
+          data[key.trim().toLowerCase()] = numValue;
+        }
+      }
+    }
+    
+    return data;
+  } catch {
+    return {};
+  }
+}
+
+function getPressureLevel(): number {
+  try {
+    const output = execSync("sysctl -n kern.memorystatus_vm_pressure_level", { 
+      encoding: "utf8", 
+      timeout: 5000 
+    });
+    const level = parseInt(output.trim(), 10);
+    if ([1, 2, 4].includes(level)) {
+      return level;
+    }
+    return 1; // Default to normal
+  } catch {
+    return 1;
+  }
+}
+
+function getMemorystatusLevel(): number {
+  try {
+    const output = execSync("sysctl -n kern.memorystatus_level", { 
+      encoding: "utf8", 
+      timeout: 5000 
+    });
+    const level = parseInt(output.trim(), 10);
+    return isNaN(level) ? 0 : Math.max(0, Math.min(100, level));
+  } catch {
+    return 0;
+  }
+}
+
+function getSwapUsage(): [number, number] {
+  try {
+    const output = execSync("sysctl -n vm.swapusage", { 
+      encoding: "utf8", 
+      timeout: 5000 
+    });
+    
+    let usedMb = 0;
+    let freeMb = 0;
+    
+    // Parse "used = X.XXM  free = Y.YYM"
+    const parts = output.trim().split(/\s+/);
+    for (let i = 0; i < parts.length - 2; i++) {
+      if (parts[i] === "used" && parts[i + 1] === "=") {
+        const value = parseFloat(parts[i + 2].replace("M", ""));
+        if (!isNaN(value)) usedMb = value;
+      } else if (parts[i] === "free" && parts[i + 1] === "=") {
+        const value = parseFloat(parts[i + 2].replace("M", ""));
+        if (!isNaN(value)) freeMb = value;
+      }
+    }
+    
+    return [usedMb, freeMb];
+  } catch {
+    return [0, 0];
+  }
+}
+
+function getCompressionRatio(vmData: Record<string, number>): number {
+  try {
+    const compressorPageCount = vmData["pages occupied by compressor"] || 0;
+    const pagesStoredInCompressor = vmData["pages stored in compressor"] || 0;
+    
+    if (compressorPageCount > 0 && pagesStoredInCompressor > 0) {
+      const ratio = pagesStoredInCompressor / compressorPageCount;
+      return Math.max(1.0, ratio);
+    }
+    
+    return 1.0;
+  } catch {
+    return 1.0;
+  }
+}

package/extensions/tree.ts

@@ -0,0 +1,218 @@
+export interface SnapshotProcess {
+  pid: number;
+  name: string;
+  footprint_mb: number;
+  age_seconds: number | null;
+}
+
+export interface GrowingProcess extends SnapshotProcess {
+  delta_mb: number;
+}
+
+// Format age in seconds to human-readable string
+export function formatAge(seconds: number | null): string {
+  if (seconds === null) {
+    return "unknown";
+  }
+  
+  if (seconds < 60) {
+    return `${seconds}s`;
+  }
+  
+  if (seconds < 3600) {
+    const minutes = Math.floor(seconds / 60);
+    const remainingSeconds = seconds % 60;
+    return `${minutes}m ${remainingSeconds}s`;
+  }
+  
+  // >= 3600 (1 hour or more)
+  const hours = Math.floor(seconds / 3600);
+  const remainingMinutes = Math.floor((seconds % 3600) / 60);
+  return `${hours}h ${remainingMinutes}m`;
+}
+
+/**
+ * Parse ps etime format to seconds.
+ * Formats: SS, MM:SS, HH:MM:SS, DD-HH:MM:SS
+ * Returns null on parse failure.
+ */
+export function parseEtime(etime: string): number | null {
+  const trimmed = etime.trim();
+  if (!trimmed) return null;
+
+  try {
+    let days = 0;
+    let timePart = trimmed;
+
+    if (timePart.includes("-")) {
+      const [dayStr, rest] = timePart.split("-", 2);
+      days = parseInt(dayStr, 10);
+      if (isNaN(days)) return null;
+      timePart = rest;
+    }
+
+    const parts = timePart.split(":").map(p => parseInt(p, 10));
+    if (parts.some(isNaN)) return null;
+
+    let seconds = days * 86400;
+    if (parts.length === 3) {
+      seconds += parts[0] * 3600 + parts[1] * 60 + parts[2];
+    } else if (parts.length === 2) {
+      seconds += parts[0] * 60 + parts[1];
+    } else if (parts.length === 1) {
+      seconds += parts[0];
+    } else {
+      return null;
+    }
+
+    return seconds;
+  } catch {
+    return null;
+  }
+}
+
+// Natural language description of a process
+export function describeProcess(proc: SnapshotProcess): string {
+  const age = formatAge(proc.age_seconds);
+  return `${proc.name} [PID ${proc.pid}] (${proc.footprint_mb} MB, running ${age})`;
+}
+
+// Find processes in current that don't exist in previous (by PID)
+export function findNewProcesses(
+  previous: SnapshotProcess[],
+  current: SnapshotProcess[],
+): SnapshotProcess[] {
+  const previousPids = new Set(previous.map(p => p.pid));
+  return current.filter(proc => !previousPids.has(proc.pid));
+}
+
+/**
+ * Find processes with sustained growth across a snapshot history.
+ *
+ * "Growing" = present in at least 3 snapshots, footprint increased between
+ * consecutive pairs in at least 3 of those transitions, AND total delta
+ * (newest - oldest observed footprint) >= thresholdMb.
+ *
+ * This is resilient to momentary plateaus — a process that grows, holds,
+ * then grows again still qualifies as long as 3+ transitions show increases.
+ *
+ * @param snapshotHistory - ring buffer of snapshots, oldest first (max ~10).
+ * @param currentChildren - current process list (used to filter to eligible PIDs only)
+ * @param thresholdMb - minimum total delta from oldest to newest (default 200)
+ * @param minGrowthIntervals - minimum number of intervals showing growth (default 3)
+ */
+export function findGrowingProcesses(
+  snapshotHistory: SnapshotProcess[][],
+  currentChildren: SnapshotProcess[],
+  thresholdMb: number = 200,
+  minGrowthIntervals: number = 3,
+): GrowingProcess[] {
+  // Need at least 3 snapshots to have 2 intervals
+  if (snapshotHistory.length < 3) {
+    return [];
+  }
+
+  const eligiblePids = new Set(currentChildren.map(p => p.pid));
+  const growing: GrowingProcess[] = [];
+
+  for (const pid of eligiblePids) {
+    // Collect footprint values for this PID across all snapshots (skip gaps)
+    const footprints: number[] = [];
+    for (const snapshot of snapshotHistory) {
+      const proc = snapshot.find(p => p.pid === pid);
+      if (proc) {
+        footprints.push(proc.footprint_mb);
+      }
+      // Missing from a snapshot is OK — we just skip it
+    }
+
+    // Need the PID present in at least 3 snapshots
+    if (footprints.length < 3) continue;
+
+    // Count intervals where footprint increased
+    let growthIntervals = 0;
+    for (let i = 1; i < footprints.length; i++) {
+      if (footprints[i] > footprints[i - 1]) {
+        growthIntervals++;
+      }
+    }
+
+    if (growthIntervals < minGrowthIntervals) continue;
+
+    // Check total delta meets threshold (newest - oldest observed)
+    const totalDelta = footprints[footprints.length - 1] - footprints[0];
+    if (totalDelta < thresholdMb) continue;
+
+    const currentProc = currentChildren.find(p => p.pid === pid)!;
+    growing.push({
+      ...currentProc,
+      delta_mb: totalDelta,
+    });
+  }
+
+  // Sort by total delta descending
+  return growing.sort((a, b) => b.delta_mb - a.delta_mb);
+}
+
+// Find the largest group of processes with the same name (>= 2 members)
+// Returns the group, or empty array if no group has >= 2 members
+export function findSimilarGroup(processes: SnapshotProcess[]): SnapshotProcess[] {
+  // Group processes by name
+  const groups = new Map<string, SnapshotProcess[]>();
+  
+  for (const proc of processes) {
+    if (!groups.has(proc.name)) {
+      groups.set(proc.name, []);
+    }
+    groups.get(proc.name)!.push(proc);
+  }
+  
+  // Find the largest group with >= 2 members
+  let largestGroup: SnapshotProcess[] = [];
+  
+  for (const group of groups.values()) {
+    if (group.length >= 2 && group.length > largestGroup.length) {
+      largestGroup = group;
+    }
+  }
+  
+  return largestGroup;
+}
+
+/**
+ * Find the heaviest swarm of same-name processes.
+ *
+ * A "swarm" is >= minCount processes with the same name whose combined
+ * footprint >= minCombinedMb. Returns the group with the highest combined
+ * footprint, sorted by individual footprint descending.
+ *
+ * Use case: 7 vitest workers each at 500 MB = 3.5 GB swarm.
+ */
+export function findSwarm(
+  processes: SnapshotProcess[],
+  minCount: number = 3,
+  minCombinedMb: number = 500,
+): SnapshotProcess[] {
+  const groups = new Map<string, SnapshotProcess[]>();
+
+  for (const proc of processes) {
+    if (!groups.has(proc.name)) {
+      groups.set(proc.name, []);
+    }
+    groups.get(proc.name)!.push(proc);
+  }
+
+  let bestGroup: SnapshotProcess[] = [];
+  let bestCombined = 0;
+
+  for (const group of groups.values()) {
+    if (group.length < minCount) continue;
+    const combined = group.reduce((sum, p) => sum + p.footprint_mb, 0);
+    if (combined >= minCombinedMb && combined > bestCombined) {
+      bestGroup = group;
+      bestCombined = combined;
+    }
+  }
+
+  return bestGroup.sort((a, b) => b.footprint_mb - a.footprint_mb);
+}

package/extensions/watchdog.ts

@@ -0,0 +1,138 @@
+import type { SnapshotProcess } from "./tree";
+import { findGrowingProcesses, findSwarm } from "./tree";
+
+export interface WatchdogState {
+  /** Ring buffer of recent snapshots (oldest first, max 10). Each entry is pi's child process list. */
+  snapshotHistory: SnapshotProcess[][];
+  /** Epoch seconds of the last snapshot taken while system was stably non-RED. */
+  lastNonRedTimestamp: number;
+  /** Count of consecutive non-RED polls. Used for hysteresis on lastNonRedTimestamp. */
+  consecutiveNonRedPolls: number;
+  lastKillTime: number;  // Date.now() of last kill, 0 if never
+  cooldownMs: number;    // minimum ms between kills
+}
+
+export interface KillDecision {
+  targets: SnapshotProcess[];  // processes to kill
+  reason: string;              // human-readable reason
+}
+
+// Should we auto-kill? Only on confirmed RED.
+export function shouldAutoKill(zone: string, confirmed: boolean): boolean {
+  return zone === "RED" && confirmed === true;
+}
+
+/** Minimum footprint to be considered a kill candidate. Filters infrastructure noise. */
+export const MIN_FOOTPRINT_MB = 50;
+
+/** Thresholds for growth detection */
+export const GROWTH_THRESHOLD_MB = 100;
+
+/** Thresholds for swarm detection */
+export const SWARM_MIN_COUNT = 3;
+export const SWARM_MIN_COMBINED_MB = 500;
+
+/** Thresholds for "heavy & young" detection */
+export const HEAVY_YOUNG_MAX_AGE_SECONDS = 600;  // 10 minutes
+export const HEAVY_YOUNG_MIN_FOOTPRINT_MB = 200;  // 200 MB (was 1 GB)
+
+/**
+ * Select what to kill from process list.
+ *
+ * Priority chain:
+ *   Floor: processes with footprint < 50 MB are never kill candidates
+ *   1. Growing — sustained delta ≥100 MB across 3+ of last 10 snapshots → kill ALL
+ *   2. Swarm — ≥3 same-name processes, combined footprint ≥500 MB → kill ALL in swarm
+ *   3. Heavy & young — age < 10 min AND footprint ≥ 200 MB → kill ALL, largest first
+ *   4. Newest — appeared after last stable non-RED state → kill LARGEST ONLY (not batch)
+ *   5. No match → don't kill, block commands and wait
+ *
+ * Returns null if nothing should be killed (no evidence, cooldown, empty).
+ */
+export function selectKillTarget(
+  currentChildren: SnapshotProcess[],
+  state: WatchdogState,
+  maxAgeSeconds: number,
+  protectedPid?: number,
+): KillDecision | null {
+  // 1. Check cooldown
+  if (state.lastKillTime > 0 && Date.now() - state.lastKillTime < state.cooldownMs) {
+    return null;
+  }
+
+  // 2. Apply footprint floor + age filter + protected PID
+  let eligible = currentChildren.filter(proc =>
+    proc.footprint_mb >= MIN_FOOTPRINT_MB &&
+    (proc.age_seconds === null || proc.age_seconds <= maxAgeSeconds)
+  );
+
+  if (protectedPid !== undefined) {
+    eligible = eligible.filter(proc => proc.pid !== protectedPid);
+  }
+
+  if (eligible.length === 0) {
+    return null;
+  }
+
+  // Priority 1: Growing — kill ALL processes with sustained growth
+  if (state.snapshotHistory.length >= 3) {
+    const growing = findGrowingProcesses(state.snapshotHistory, eligible, GROWTH_THRESHOLD_MB);
+    if (growing.length > 0) {
+      return {
+        targets: growing,
+        reason: `Killing ${growing.length} growing process(es): ${growing.map(p => `${p.name}(+${p.delta_mb}MB)`).join(", ")}`,
+      };
+    }
+  }
+
+  // Priority 2: Swarm — kill ALL in the heaviest same-name cluster
+  const swarm = findSwarm(eligible, SWARM_MIN_COUNT, SWARM_MIN_COMBINED_MB);
+  if (swarm.length > 0) {
+    const combined = swarm.reduce((sum, p) => sum + p.footprint_mb, 0);
+    return {
+      targets: swarm,
+      reason: `Killing swarm of ${swarm.length} ${swarm[0].name} processes (combined ${combined}MB): ${swarm.map(p => `${p.name}(${p.footprint_mb}MB)`).join(", ")}`,
+    };
+  }
+
+  // Priority 3: Heavy & young — kill ALL recently started heavy processes
+  const heavyYoung = eligible
+    .filter(proc =>
+      proc.age_seconds !== null &&
+      proc.age_seconds <= HEAVY_YOUNG_MAX_AGE_SECONDS &&
+      proc.footprint_mb >= HEAVY_YOUNG_MIN_FOOTPRINT_MB
+    )
+    .sort((a, b) => b.footprint_mb - a.footprint_mb);
+
+  if (heavyYoung.length > 0) {
+    return {
+      targets: heavyYoung,
+      reason: `Killing ${heavyYoung.length} heavy & young process(es): ${heavyYoung.map(p => `${p.name}(${p.footprint_mb}MB, age ${p.age_seconds}s)`).join(", ")}`,
+    };
+  }
+
+  // Priority 4: Newest — kill ONLY the single largest process that appeared
+  // after the system was last stably non-RED. Not batch — weakest signal.
+  if (state.lastNonRedTimestamp > 0) {
+    const secondsSinceHealthy = (Date.now() / 1000) - state.lastNonRedTimestamp;
+
+    const newest = eligible
+      .filter(proc =>
+        proc.age_seconds !== null &&
+        proc.age_seconds <= secondsSinceHealthy
+      )
+      .sort((a, b) => b.footprint_mb - a.footprint_mb);
+
+    if (newest.length > 0) {
+      // Only kill the single largest — temporal correlation alone is weak evidence
+      const target = newest[0];
+      return {
+        targets: [target],
+        reason: `Killing newest heavy process (appeared after last stable state ${Math.round(secondsSinceHealthy)}s ago): ${target.name}(${target.footprint_mb}MB, age ${target.age_seconds}s)`,
+      };
+    }
+  }
+
+  // No evidence — don't kill. Caller should block commands and wait.
+  return null;
+}