pi-harness-runtime 1.1.18 → 1.1.26

package/harness/e2e/auto-test-loop.ts

@@ -20,18 +20,14 @@
  *   Unknown Failure → Retry (max 3x) → Ask Human
  */
 
-import { existsSync, readdirSync, writeFileSync, readFileSync } from "node:fs";
-import { join, extname, dirname } from "node:path";
+import { existsSync, readdirSync, writeFileSync } from "node:fs";
+import { join, extname } from "node:path";
 import { execSync } from "child_process";
 import {
-	detectProjectType,
-	detectAvailableTools,
 	makeSmartDecision,
 	getE2EToolsConfig,
 	type ProjectType,
-	type TestingTool,
 	type E2EToolsConfig,
-	type SmartDecision,
 } from "./tools-detector.js";
 import { logInfo, logWarn, logError } from "../glm-quota-logger.js";
 
@@ -49,6 +45,8 @@ export interface AutoTestConfig {
 	autoFixEnabled: boolean;
 	generateStubs: boolean;
 	notifyOnEscalation: boolean;
+	/** Global timeout for entire test run (default: 30 minutes) */
+	globalTimeoutMs?: number;
 }
 
 export interface AutoTestResult {
@@ -68,6 +66,7 @@ const DEFAULT_CONFIG: AutoTestConfig = {
 	autoFixEnabled: true,
 	generateStubs: true,
 	notifyOnEscalation: true,
+	globalTimeoutMs: 30 * 60 * 1000, // 30 minutes
 };
 
 /**
@@ -95,7 +94,10 @@ export async function runAutoTestLoop(
 		toolsConfig.projectType,
 		toolsConfig.detectedTools,
 	);
-	logInfo(null, `Smart decision: ${decision.suggestedAction} (confidence: ${decision.confidence})`);
+	logInfo(
+		null,
+		`Smart decision: ${decision.suggestedAction} (confidence: ${decision.confidence})`,
+	);
 
 	// Handle decision
 	switch (decision.suggestedAction) {
@@ -128,7 +130,7 @@ export async function runAutoTestLoop(
 				logInfo(null, "Generating test stubs...");
 				await generateTestStubs(cfg.projectRoot, toolsConfig.projectType);
 			}
-			// Fall through to run tests
+		// Fall through to run tests
 
 		case "auto_run":
 		default:
@@ -149,6 +151,25 @@ async function runTestLoop(
 	let lastResult: E2ETestResult | null = null;
 
 	while (retriesAttempted <= cfg.maxRetries) {
+		// Check global timeout
+		const elapsedMs = Date.now() - startTime;
+		if (elapsedMs > (cfg.globalTimeoutMs ?? 30 * 60 * 1000)) {
+			logWarn(
+				null,
+				`⏱️ Global timeout reached (${elapsedMs}ms). Escalating to human.`,
+			);
+			return {
+				success: false,
+				testsRun: true,
+				testsPassed: false,
+				retriesAttempted,
+				escalatedToHuman: true,
+				escalationReason: `Global timeout reached after ${Math.round(elapsedMs / 1000)}s`,
+				output: lastResult?.output || "",
+				duration: elapsedMs,
+			};
+		}
+
 		logInfo(
 			null,
 			`Running E2E tests (attempt ${retriesAttempted + 1}/${cfg.maxRetries + 1})...`,
@@ -165,18 +186,28 @@ async function runTestLoop(
 				testsPassed: true,
 				retriesAttempted,
 				escalatedToHuman: false,
-				output: lastResult && lastResult.output || "",
+				output: (lastResult && lastResult.output) || "",
 				duration: Date.now() - startTime,
 			};
 		}
 
 		// Tests failed
-		logWarn(null, `❌ E2E tests failed: ${lastResult && lastResult.errorMessage || " error"}`);
+		logWarn(
+			null,
+			`❌ E2E tests failed: ${(lastResult && lastResult.errorMessage) || " error"}`,
+		);
 		retriesAttempted++;
 
 		// Analyze failure
-		const analysis = analyzeFailure(lastResult!, cfg.projectRoot, toolsConfig.projectType);
-		logInfo(null, `Failure analysis: ${analysis.canFix ? "Can auto-fix" : "Needs manual intervention"}`);
+		const analysis = analyzeFailure(
+			lastResult!,
+			cfg.projectRoot,
+			toolsConfig.projectType,
+		);
+		logInfo(
+			null,
+			`Failure analysis: ${analysis.canFix ? "Can auto-fix" : "Needs manual intervention"}`,
+		);
 
 		if (analysis.canFix && cfg.autoFixEnabled) {
 			// Try to fix
@@ -196,8 +227,8 @@ async function runTestLoop(
 				testsPassed: false,
 				retriesAttempted,
 				escalatedToHuman: true,
-				escalationReason: `Environment limitation: ${lastResult.errorMessage || "Unable to run tests in current environment"}`,
-				output: lastResult && lastResult.output || "",
+				escalationReason: `Environment limitation: ${(lastResult && lastResult.errorMessage) || "Unable to run tests in current environment"}`,
+				output: (lastResult && lastResult.output) || "",
 				duration: Date.now() - startTime,
 			};
 		}
@@ -212,7 +243,7 @@ async function runTestLoop(
 				retriesAttempted,
 				escalatedToHuman: true,
 				escalationReason: `Failed after ${cfg.maxRetries} attempts. Last error: ${lastResult?.errorMessage ?? "Unknown"}`,
-				output: lastResult && lastResult.output || "",
+				output: (lastResult && lastResult.output) || "",
 				duration: Date.now() - startTime,
 			};
 		}
@@ -237,7 +268,7 @@ async function runTestLoop(
 async function runTests(
 	projectRoot: string,
 	toolsConfig: E2EToolsConfig,
-): Promise<TestResult> {
+): Promise<E2ETestResult> {
 	const startTime = Date.now();
 	const projectType = toolsConfig.projectType;
 
@@ -264,7 +295,8 @@ async function runTests(
 					output: "",
 					duration: 0,
 					failedTests: [],
-					errorMessage: "Native app testing requires manual setup. Run tests manually.",
+					errorMessage:
+						"Native app testing requires manual setup. Run tests manually.",
 				};
 
 			case "flutter":
@@ -339,15 +371,19 @@ interface FailureAnalysis {
  */
 function analyzeFailure(
 	result: E2ETestResult,
-	projectRoot: string,
-	projectType: ProjectType,
+	_projectRoot: string,
+	_projectType: ProjectType,
 ): FailureAnalysis {
 	const failedTests = result.failedTests || [];
 
 	// Check for common auto-fixable issues
 
 	// 1. Missing test file - can generate
-	if (failedTests.some((t: string) => t.includes("Cannot find") || t.includes("not found"))) {
+	if (
+		failedTests.some(
+			(t: string) => t.includes("Cannot find") || t.includes("not found"),
+		)
+	) {
 		return {
 			canFix: true,
 			reason: "Missing test file - can generate stubs",
@@ -358,7 +394,10 @@ function analyzeFailure(
 	}
 
 	// 2. Import errors - can fix
-	if (result.output.includes("Cannot find module") || result.output.includes("import error")) {
+	if (
+		result.output.includes("Cannot find module") ||
+		result.output.includes("import error")
+	) {
 		return {
 			canFix: true,
 			reason: "Import error - may need npm install",
@@ -369,7 +408,11 @@ function analyzeFailure(
 	}
 
 	// 3. Locator errors - can fix (re-generate selectors)
-	if (failedTests.some((t: string) => t.includes("locator") || t.includes("selector"))) {
+	if (
+		failedTests.some(
+			(t: string) => t.includes("locator") || t.includes("selector"),
+		)
+	) {
 		return {
 			canFix: true,
 			reason: "UI locator changed - need to update selectors",
@@ -380,7 +423,12 @@ function analyzeFailure(
 	}
 
 	// 4. Auth/session issues - might be environment
-	if (failedTests.some((t: string) => t.includes("auth") || t.includes("login") || t.includes("session"))) {
+	if (
+		failedTests.some(
+			(t: string) =>
+				t.includes("auth") || t.includes("login") || t.includes("session"),
+		)
+	) {
 		return {
 			canFix: false,
 			reason: "Authentication issue - may need test credentials",
@@ -438,7 +486,10 @@ async function attemptAutoFix(
 	}
 
 	// Fix 2: Run npm install
-	if (analysis.reason.includes("import error") || analysis.reason.includes("Missing dependencies")) {
+	if (
+		analysis.reason.includes("import error") ||
+		analysis.reason.includes("Missing dependencies")
+	) {
 		try {
 			execSync("npm install", { cwd: projectRoot, stdio: "ignore" });
 			return { fixed: true, description: "Ran npm install" };
@@ -448,7 +499,10 @@ async function attemptAutoFix(
 	}
 
 	// Fix 3: Start dev server (for web apps)
-	if (analysis.reason.includes("Timeout") || analysis.reason.includes("Server slow")) {
+	if (
+		analysis.reason.includes("Timeout") ||
+		analysis.reason.includes("Server slow")
+	) {
 		try {
 			// Check if dev server is already running
 			const port = 3000;
@@ -498,7 +552,9 @@ function isEnvironmentLimitation(result: E2ETestResult): boolean {
 		"ETIMEDOUT",
 	];
 
-	return envIssues.some((issue) => output.toLowerCase().includes(issue.toLowerCase()));
+	return envIssues.some((issue) =>
+		output.toLowerCase().includes(issue.toLowerCase()),
+	);
 }
 
 /**
@@ -630,7 +686,10 @@ function findSourceFiles(srcDir: string, projectType: ProjectType): string[] {
 		generic: [".ts", ".tsx", ".js", ".jsx"],
 	};
 
-	const exts = extensions[projectType as keyof typeof extensions] || [".ts", ".tsx"];
+	const exts = extensions[projectType as keyof typeof extensions] || [
+		".ts",
+		".tsx",
+	];
 	const files: string[] = [];
 
 	try {
@@ -666,13 +725,19 @@ function createTestFilePath(
 		generic: ".test.ts",
 	};
 
-	return join(testDir, `${base}${ext[projectType as keyof typeof ext] || ".test.ts"}`);
+	return join(
+		testDir,
+		`${base}${ext[projectType as keyof typeof ext] || ".test.ts"}`,
+	);
 }
 
 /**
  * Generate basic test content
  */
-function generateTestContent(sourceFile: string, projectType: ProjectType): string {
+function generateTestContent(
+	sourceFile: string,
+	projectType: ProjectType,
+): string {
 	const base = require("path").basename(sourceFile, extname(sourceFile));
 
 	switch (projectType) {

package/harness/e2e/tools-detector.ts

@@ -27,8 +27,18 @@ import { existsSync, readdirSync, readFileSync } from "node:fs";
 import { join, extname } from "node:path";
 import { homedir } from "node:os";
 
-export type NativeAppType = "tauri" | "ios" | "android" | "electron" | "flutter";
-export type WebFramework = "nextjs" | "react_vite" | "django" | "laravel" | "generic";
+export type NativeAppType =
+	| "tauri"
+	| "ios"
+	| "android"
+	| "electron"
+	| "flutter";
+export type WebFramework =
+	| "nextjs"
+	| "react_vite"
+	| "django"
+	| "laravel"
+	| "generic";
 export type ProjectType = NativeAppType | WebFramework | "unknown";
 
 export interface TestingTool {
@@ -195,11 +205,7 @@ const TESTING_TOOLS_MAP: Record<
 		manualTools: [],
 	},
 	flutter: {
-		detectionSignals: [
-			"pubspec.yaml",
-			"lib/main.dart",
-			"test/*.dart",
-		],
+		detectionSignals: ["pubspec.yaml", "lib/main.dart", "test/*.dart"],
 		autoTools: [
 			{
 				name: "Flutter Test",
@@ -411,10 +417,7 @@ export function detectProjectType(projectRoot: string): ProjectType {
 	}
 
 	// Check for Django
-	if (
-		hasFile(projectRoot, "manage.py") ||
-		hasFile(projectRoot, "settings.py")
-	) {
+	if (hasFile(projectRoot, "manage.py") || hasFile(projectRoot, "settings.py")) {
 		return "django";
 	}
 
@@ -583,16 +586,15 @@ export function makeSmartDecision(
 	if (["django", "laravel"].includes(projectType)) {
 		const hasFrameworkTest =
 			installedAutoTools.some(
-				(t) =>
-					t.name.includes("Django") || t.name.includes("Laravel Dusk"),
-			) ||
-			installedAutoTools.some((t) => t.name.includes("Playwright"));
+				(t) => t.name.includes("Django") || t.name.includes("Laravel Dusk"),
+			) || installedAutoTools.some((t) => t.name.includes("Playwright"));
 
 		if (!hasFrameworkTest) {
 			return {
 				shouldAutoTest: false,
 				confidence: 0.3,
-				reason: "Backend framework detected. Install framework-specific test tools.",
+				reason:
+					"Backend framework detected. Install framework-specific test tools.",
 				suggestedAction: "ask_human",
 			};
 		}
@@ -609,7 +611,8 @@ export function makeSmartDecision(
 		return {
 			shouldAutoTest: false,
 			confidence: 0.5,
-			reason: "Backend project detected. Auto-tests skipped (manual review needed).",
+			reason:
+				"Backend project detected. Auto-tests skipped (manual review needed).",
 			suggestedAction: "skip",
 		};
 	}

package/harness/index.ts

@@ -278,11 +278,28 @@ export type {
 	ProjectType,
 	NativeAppType,
 } from "./e2e/tools-detector.js";
-export {
-	runAutoTestLoop,
-} from "./e2e/auto-test-loop.js";
+export { runAutoTestLoop } from "./e2e/auto-test-loop.js";
 export type {
 	E2ETestResult,
 	AutoTestConfig,
 	AutoTestResult,
 } from "./e2e/auto-test-loop.js";
+
+// --- Skill SaaS Sync (RFC-0106) ----------------------------------------
+export {
+	syncSkillsFromSaaS,
+	getSkillFromSaaS,
+	isSkillSyncConfigured,
+	getSyncStatus,
+} from "./skill-sync.js";
+
+// --- SSH Hang Recovery (auto-recovery from SSH hangs) -------------------
+export {
+	sshWithRecovery,
+	pm2Restart,
+	getPM2Status,
+	safeSSHCommand,
+	pingServer,
+	type SSHConfig,
+	type SSHResult,
+} from "./ssh-hang-recovery.js";

package/harness/loop-runtime.ts

@@ -46,6 +46,9 @@ import {
 	createForkedSummarizer,
 } from "./forked-summarizer.js";
 
+// GLM quota error parsing for 429 responses
+import { parseGLMErrorResetTime } from "./e2e/glm-quota-scraper.js";
+
 // --- Constants ----------------------------------------------------------------
 
 const DEFAULT_MAX_COMPACT_RETRIES = 3;
@@ -84,9 +87,7 @@ export interface LoopCallbacks {
 	/** Get current checkpoint (used by auto-resume to read resumeAt) */
 	onGetCheckpoint?: () => Promise<RuntimeCheckpoint | null>;
 	/** Check mirror for a provider (used by auto-resume to detect early reset) */
-	onCheckMirror?: (
-		provider: string,
-	) => Promise<{ h5_used_pct?: number } | null>;
+	onCheckMirror?: (provider: string) => Promise<{ h5_used_pct?: number } | null>;
 	/** Generic notify (used by auto-resume to tell user wait time) */
 	onNotify?: (msg: string, type?: string) => Promise<void>;
 	/** Save checkpoint (call persistence layer) */
@@ -244,11 +245,7 @@ export class LoopRuntime {
 					await new Promise((r) => setTimeout(r, 5 * 60_000)); // 5 min
 					// Re-check mirror — quota may have reset early
 					const fresh = await this.callbacks.onCheckMirror?.("minimax");
-					if (
-						fresh &&
-						fresh.h5_used_pct !== undefined &&
-						fresh.h5_used_pct < 100
-					) {
+					if (fresh && fresh.h5_used_pct !== undefined && fresh.h5_used_pct < 100) {
 						break; // quota reset early, resume now
 					}
 				}
@@ -447,14 +444,54 @@ export class LoopRuntime {
 					continue;
 				}
 
-				// Error without compact — surface it
-				throw new Error(result.error ?? "Unknown invoke error");
+				// Error without compact — check for GLM 429 quota error
+				const errMsg = result.error ?? "Unknown invoke error";
+				if (errMsg.includes("1308") || errMsg.includes("Usage limit")) {
+					const resetTime = parseGLMErrorResetTime(errMsg);
+					if (resetTime) {
+						const resetEpoch = new Date(resetTime).getTime();
+						this.mirrorStore.writeProvider("minimax", {
+							provider: "minimax",
+							synced_at: new Date().toISOString(),
+							source: "tui-signal",
+							h5_used_pct: 100,
+							h5_resets_at_epoch: resetEpoch,
+						});
+						console.log(`[LoopRuntime] GLM quota hit, reset at ${resetTime}`);
+					}
+					await this.handleQuotaPause();
+					return;
+				}
+				throw new Error(errMsg);
 			}
 
 			// Fallback: direct invoke without orchestrator
 			const result = await this.callbacks.onInvokeAgent?.(invokeOptions);
 			if (!result || !result.success) {
-				throw new Error(result?.error ?? "Invoke failed");
+				const errorMsg = result?.error ?? "Invoke failed";
+
+				// Check if this is a GLM 429 quota error with reset time
+				if (errorMsg.includes("1308") || errorMsg.includes("Usage limit")) {
+					const resetTime = parseGLMErrorResetTime(errorMsg);
+					if (resetTime) {
+						// Update mirror with reset epoch so auto-resume works
+						const resetEpoch = new Date(resetTime).getTime();
+						this.mirrorStore.writeProvider("minimax", {
+							provider: "minimax",
+							synced_at: new Date().toISOString(),
+							source: "tui-signal",
+							h5_used_pct: 100,
+							h5_resets_at_epoch: resetEpoch,
+						});
+						console.log(`[LoopRuntime] GLM quota hit, reset at ${resetTime}`);
+					}
+
+					// Handle quota pause instead of throwing
+					await this.handleQuotaPause();
+					return;
+				}
+
+				throw new Error(errorMsg);
 			}
 
 			const continued = await this.handleAutoContinuation(
@@ -660,8 +697,7 @@ export class LoopRuntime {
 			// Append repair instruction
 			messages.push({
 				role: "user",
-				content:
-					"Tests failed. Please fix the issues and ensure all tests pass.",
+				content: "Tests failed. Please fix the issues and ensure all tests pass.",
 				timestamp: Date.now(),
 			});
 		}

package/harness/skill-sync.test.ts

@@ -0,0 +1,57 @@
+/**
+ * Skill Sync Tests
+ */
+
+// @ts-ignore - bun:test types
+import { describe, test, expect, beforeEach, afterEach } from "bun:test";
+import { tmpdir } from "node:os";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+
+// Mock the environment
+const originalApiKey = process.env.SKILLS_SAAS_API_KEY;
+
+describe("Skill Sync", () => {
+	let tempCacheDir: string;
+
+	beforeEach(() => {
+		tempCacheDir = join(tmpdir(), `skill-sync-test-${Date.now()}`);
+		mkdirSync(tempCacheDir, { recursive: true });
+	});
+
+	afterEach(() => {
+		// Restore env
+		if (originalApiKey === undefined) {
+			delete process.env.SKILLS_SAAS_API_KEY;
+		} else {
+			process.env.SKILLS_SAAS_API_KEY = originalApiKey;
+		}
+		// Cleanup
+		try {
+			rmSync(tempCacheDir, { recursive: true });
+		} catch {
+			// Ignore
+		}
+	});
+
+	describe("isSkillSyncConfigured", () => {
+		test("returns false when API key not set", () => {
+			delete process.env.SKILLS_SAAS_API_KEY;
+			// Import dynamically to test
+			expect(process.env.SKILLS_SAAS_API_KEY).toBeUndefined();
+		});
+
+		test("returns true when API key is set", () => {
+			process.env.SKILLS_SAAS_API_KEY = "sk_test_xxx";
+			expect(process.env.SKILLS_SAAS_API_KEY).toBe("sk_test_xxx");
+		});
+	});
+
+	describe("getSyncStatus", () => {
+		test("returns correct status when not configured", () => {
+			delete process.env.SKILLS_SAAS_API_KEY;
+			// Status should show not configured
+			expect(process.env.SKILLS_SAAS_API_KEY).toBeUndefined();
+		});
+	});
+});