pi-harness-runtime 1.1.16 → 1.1.26

package/harness/ssh-hang-recovery.ts

@@ -0,0 +1,280 @@
+/**
+ * SSH Hang Recovery — Smart SSH wrapper that auto-recovers from hangs
+ *
+ * Problem: SSH commands can hang indefinitely (PM2 restart, sudo prompts, etc.)
+ * Solution: Timeout + process killing + retry + escalation
+ *
+ * Flow:
+ *   Execute SSH command
+ *     → Timeout reached? → Kill stuck processes → Retry
+ *     → Retry failed? → Kill PM2 daemon processes → Retry
+ *     → Still failing? → Escalate to human with diagnostic
+ */
+
+import { execSync, exec } from "child_process";
+import { setTimeout as setAsyncTimeout, clearTimeout } from "timers";
+
+export interface SSHConfig {
+	host: string;
+	user?: string;
+	timeout?: number; // ms, default 120000 (2 min)
+	maxRetries?: number;
+	strictHostKeyChecking?: boolean;
+}
+
+export interface SSHResult {
+	success: boolean;
+	stdout: string;
+	stderr: string;
+	elapsedMs: number;
+	retries: number;
+	hungProcessesKilled: boolean;
+	error?: string;
+}
+
+const DEFAULT_TIMEOUT = 120_000; // 2 minutes
+const HANG_THRESHOLD_MS = 60_000; // 1 minute = likely hang
+
+/**
+ * Sanitize input for shell commands - prevent command injection
+ */
+
+/**
+ * Execute SSH command with automatic hang recovery
+ */
+export async function sshWithRecovery(
+	config: SSHConfig,
+	command: string,
+): Promise<SSHResult> {
+	const {
+		host,
+		user = "frappe",
+		timeout = DEFAULT_TIMEOUT,
+		maxRetries = 3,
+		strictHostKeyChecking = false,
+	} = config;
+
+	const sshBase = `ssh -o BatchMode=yes -o ConnectTimeout=30 -o StrictHostKeyChecking=${strictHostKeyChecking ? "yes" : "no"} ${user}@${host}`;
+
+	let retries = 0;
+	let hungProcessesKilled = false;
+
+	while (retries <= maxRetries) {
+		const startTime = Date.now();
+
+		try {
+			// Execute with timeout
+			const result = await execAsync(
+				`${sshBase} '${command.replace(/'/g, "'\\''")}'`,
+				timeout,
+			);
+
+			return {
+				success: result.exitCode === 0,
+				stdout: result.stdout,
+				stderr: result.stderr,
+				elapsedMs: Date.now() - startTime,
+				retries,
+				hungProcessesKilled,
+				error: result.exitCode === 0 ? undefined : `Exit code: ${result.exitCode}`,
+			};
+		} catch (error: any) {
+			const elapsed = Date.now() - startTime;
+			const isTimeout = elapsed >= timeout;
+
+			console.error(
+				`[ssh-recovery] Attempt ${retries + 1} failed: ${error.message} (${elapsed}ms)`,
+			);
+
+			if (isTimeout) {
+				hungProcessesKilled = true;
+
+				// Kill stuck processes on remote server
+				await killStuckProcesses(host, user);
+
+				// Also kill any local SSH processes stuck on this host
+				killLocalSSHProcesses(host);
+
+				retries++;
+			} else {
+				// Non-timeout error (permission denied, etc.)
+				return {
+					success: false,
+					stdout: "",
+					stderr: "",
+					elapsedMs: elapsed,
+					retries,
+					hungProcessesKilled: false,
+					error: error.message,
+				};
+			}
+		}
+	}
+
+	// All retries exhausted
+	return {
+		success: false,
+		stdout: "",
+		stderr: "",
+		elapsedMs: 0,
+		retries,
+		hungProcessesKilled,
+		error: `Command failed after ${maxRetries + 1} attempts`,
+	};
+}
+
+/**
+ * Kill stuck processes on remote server
+ */
+async function killStuckProcesses(host: string, user: string): Promise<void> {
+	const killCommands = [
+		// Kill stuck SSH sessions
+		`pkill -f "ssh.*${host}" || true`,
+		// Kill stuck PM2 commands
+		`pkill -f "pm2.*amos-saas" || true`,
+		// Kill stuck sudo commands
+		`pkill -f "sudo.*pm2" || true`,
+	];
+
+	for (const cmd of killCommands) {
+		try {
+			execSync(
+				`ssh -o BatchMode=yes -o ConnectTimeout=10 ${user}@${host} '${cmd}'`,
+				{ timeout: 10_000, encoding: "utf-8" },
+			);
+		} catch {
+			// Ignore errors from kill commands
+		}
+	}
+}
+
+/**
+ * Kill local SSH processes stuck on this host
+ */
+function killLocalSSHProcesses(host: string): void {
+	try {
+		// Kill any local SSH processes connecting to this host
+		execSync(`pkill -f "ssh.*${host}" || true`, { timeout: 5000 });
+	} catch {
+		// Ignore
+	}
+}
+
+/**
+ * Check if server is responsive
+ */
+export async function pingServer(
+	host: string,
+	user: string = "frappe",
+): Promise<boolean> {
+	try {
+		execSync(
+			`ssh -o BatchMode=yes -o ConnectTimeout=5 ${user}@${host} 'echo ok'`,
+			{ timeout: 10_000 },
+		);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Get PM2 status with hang protection
+ */
+export async function getPM2Status(
+	host: string,
+	pm2Path?: string,
+): Promise<SSHResult> {
+	const pm2Cmd = pm2Path || "pm2";
+	const command = `PM2_HOME=/home/frappe/.pm2 ${pm2Cmd} status`;
+
+	return sshWithRecovery(
+		{
+			host,
+			timeout: 30_000, // Status should be fast
+			maxRetries: 1,
+		},
+		command,
+	);
+}
+
+/**
+ * Restart PM2 process with hang protection
+ */
+export async function pm2Restart(
+	host: string,
+	app: string,
+): Promise<SSHResult> {
+	const command = `PM2_HOME=/home/frappe/.pm2 pm2 restart ${app}`;
+
+	return sshWithRecovery(
+		{
+			host,
+			timeout: 180_000, // 3 min for restart
+			maxRetries: 2,
+		},
+		command,
+	);
+}
+
+/**
+ * Execute arbitrary command with hang protection
+ */
+export async function safeSSHCommand(
+	host: string,
+	command: string,
+	options: { timeout?: number; maxRetries?: number } = {},
+): Promise<SSHResult> {
+	return sshWithRecovery(
+		{
+			host,
+			timeout: options.timeout ?? 120_000,
+			maxRetries: options.maxRetries ?? 2,
+		},
+		command,
+	);
+}
+
+// --- Helper for promise-based exec with timeout ---
+
+interface ExecResult {
+	exitCode: number;
+	stdout: string;
+	stderr: string;
+}
+
+function execAsync(command: string, timeoutMs: number): Promise<ExecResult> {
+	return new Promise((resolve, reject) => {
+		const proc = exec(command, { encoding: "utf-8" }, (error, stdout, stderr) => {
+			if (error) {
+				reject(error);
+			} else {
+				resolve({ exitCode: 0, stdout, stderr });
+			}
+		});
+
+		const timer = setAsyncTimeout(() => {
+			// Kill the process tree
+			try {
+				proc.kill("SIGKILL");
+			} catch {
+				// Process already dead
+			}
+			reject(new Error(`Command timed out after ${timeoutMs}ms`));
+		}, timeoutMs);
+
+		proc.on("exit", (code) => {
+			clearTimeout(timer);
+			resolve({
+				exitCode: code ?? 0,
+				stdout: (proc.stdout || "").toString(),
+				stderr: (proc.stderr || "").toString(),
+			});
+		});
+
+		proc.on("error", (err) => {
+			clearTimeout(timer);
+			reject(err);
+		});
+	});
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-harness-runtime",
-  "version": "1.1.16",
+  "version": "1.1.26",
   "description": "[BETA] Codex-style /usage status + autonomous coding harness for pi. Not production ready — expect breaking changes.",
   "type": "module",
   "scripts": {

package/packages/a2a-adapter/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/a2a-adapter",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "description": "Google Agent-to-Agent (A2A) protocol adapter for pi-harness",
   "main": "dist/index.js",

package/packages/architecture-generator/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/architecture-generator",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Generate ADR and Mermaid diagrams from project analysis",
   "type": "module",
   "main": "./dist/index.js",

package/packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/auth",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "pi-harness-runtime — auth",
   "type": "module",
   "main": "./dist/index.js",

package/packages/autonomous-refactor/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/autonomous-refactor",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "description": "Autonomous refactoring engine",
   "main": "dist/index.js",

package/packages/autonomous-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/autonomous-runtime",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Autonomous Operations Runtime — task inbox, lease management, and worker lifecycle (RFC-0101)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/cache-strategy/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/cache-strategy",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "description": "Smart caching with TTL and invalidation strategies (RFC-0022)",
   "main": "dist/index.js",

package/packages/capability-registry/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/capability-registry",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Capability Registry for pi-harness-runtime — tracks model capabilities and enables capability-based routing",
   "type": "module",
   "main": "./dist/index.js",

package/packages/checkpoint/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/checkpoint",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Enhanced checkpoint system with incremental snapshots, diff-based storage, and recovery strategies",
   "type": "module",
   "main": "./dist/index.js",

package/packages/cli-plugin-sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/cli-plugin-sdk",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "description": "CLI Plugin SDK for pi-harness — install, list, remove, update, invoke plugins",
   "main": "dist/index.js",

package/packages/clipboard/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/clipboard",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/clipboard-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/clipboard-plugin",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/code-generation/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/code-generation",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Code generation pipeline for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/code-review/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/code-review",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Automated code review engine for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/codex-adapter/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/codex-adapter",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "description": "OpenAI Codex provider adapter for pi-harness",
   "main": "dist/index.js",

package/packages/config-capture/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moocoding/config-capture",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Auto-capture API-fetched configuration values to AGENTS.md",
   "type": "module",
   "main": "src/index.ts",

package/packages/context-compiler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/context-compiler",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Bounded context selection engine for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/context-discovery/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/context-discovery",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/context-manager/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/context-manager",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Context Window Manager — prevents model context overflow via compaction and resume prompts (RFC-0010)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/cookie-sanitizer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/cookie-sanitizer",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "pi-harness-runtime — cookie sanitizer (provider-quota cookie file normaliser)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/cost-optimizer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/cost-optimizer",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Cost Optimizer for pi-harness-runtime — budget tracking, forecasting, and model selection optimization",
   "type": "module",
   "main": "./dist/index.js",

package/packages/dependency-analyzer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/dependency-analyzer",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/django-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/django-plugin",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Django framework analysis — models, management commands, settings",
   "type": "module",
   "main": "./dist/index.js",

package/packages/doc-generator/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/doc-generator",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Auto-Doc Generator — generate documentation from source code (RFC-0014)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/evaluation-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/evaluation-engine",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Evaluation engine for pi-harness runtime (RFC-0057)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/evaluation-runner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/evaluation-runner",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "description": "Framework-agnostic evaluation framework for AI outputs (RFC-0020)",
   "main": "dist/index.js",

package/packages/event-bus/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/event-bus",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/event-store/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/event-store",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/experience-replay/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/experience-replay",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Experience replay for debugging and reproducibility (RFC-0059)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/feedback-collector/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/feedback-collector",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "description": "Collect and aggregate human/automated feedback (RFC-0021)",
   "main": "dist/index.js",

package/packages/file-copy-helper/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moocoding/file-copy-helper",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Injects cp/sudo cp rule when user asks to mimic/copy files",
   "type": "module",
   "main": "dist/index.js",

package/packages/framework-detector/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/framework-detector",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Framework detection system for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/framework-plugin-sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/framework-plugin-sdk",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Framework plugin SDK for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/frappe-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/frappe-plugin",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Frappe/ERPNext framework analysis — DocTypes, hooks, bench sites, site config",
   "type": "module",
   "main": "./dist/index.js",

package/packages/generic-web-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/generic-web-plugin",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Generic web framework analysis — REST/GraphQL endpoints, OpenAPI, auth",
   "type": "module",
   "main": "./dist/index.js",

package/packages/health-monitor/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/health-monitor",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "description": "Health monitoring and auto-healing",
   "main": "dist/index.js",

package/packages/intent-analyzer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/intent-analyzer",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/knowledge-graph/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/knowledge-graph",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Knowledge Graph extraction and provenance linking for SKILL.md files",
   "type": "module",
   "main": "./dist/index.js",

package/packages/knowledge-retrieval/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/knowledge-retrieval",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Knowledge retrieval via TencentDB-Agent-Memory MCP (RFC-0105)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/laravel-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/laravel-plugin",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Laravel framework analysis — artisan commands, Eloquent models, Blade",
   "type": "module",
   "main": "./dist/index.js",

package/packages/learning-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/learning-engine",
-  "version": "1.1.15",
+  "version": "1.1.26",
   "description": "Learning engine for extracting lessons from runtime execution (RFC-0058)",
   "type": "module",
   "main": "./dist/index.js",