pi-harness-runtime 1.1.16 → 1.1.26

package/harness/glm-quota-logger.ts

@@ -29,7 +29,11 @@ function ensureLogDir(): void {
 }
 
 /** Format log entry */
-function formatLog(level: LogLevel, jobId: string | null, message: string): string {
+function formatLog(
+	level: LogLevel,
+	jobId: string | null,
+	message: string,
+): string {
 	const timestamp = new Date().toISOString();
 	const job = jobId ? `[${jobId}]` : "[system]";
 	return `[${timestamp}] [${level}] ${job} ${message}`;
@@ -38,7 +42,11 @@ function formatLog(level: LogLevel, jobId: string | null, message: string): stri
 /**
  * Write a log entry to file
  */
-function writeLog(level: LogLevel, jobId: string | null, message: string): void {
+function writeLog(
+	level: LogLevel,
+	jobId: string | null,
+	message: string,
+): void {
 	try {
 		ensureLogDir();
 		const entry = formatLog(level, jobId, message) + "\n";
@@ -70,7 +78,11 @@ export function logWarn(jobId: string | null, message: string): void {
 /**
  * Log error level event
  */
-export function logError(jobId: string | null, message: string, error?: unknown): void {
+export function logError(
+	jobId: string | null,
+	message: string,
+	error?: unknown,
+): void {
 	const errorMsg = error instanceof Error ? error.message : String(error ?? "");
 	const fullMessage = errorMsg ? `${message}: ${errorMsg}` : message;
 	console.error(`[GLMQuota] ${jobId ? `[${jobId}] ` : ""}${fullMessage}`);
@@ -87,10 +99,7 @@ export function logCountdownStarted(
 ): void {
 	const hours = Math.floor(totalSeconds / 3600);
 	const minutes = Math.floor((totalSeconds % 3600) / 60);
-	logInfo(
-		jobId,
-		`Countdown started: ${hours}h ${minutes}m until ${resetAt}`,
-	);
+	logInfo(jobId, `Countdown started: ${hours}h ${minutes}m until ${resetAt}`);
 }
 
 /** Log countdown tick */
@@ -101,10 +110,7 @@ export function logCountdownTick(
 	const hours = Math.floor(remainingSeconds / 3600);
 	const minutes = Math.floor((remainingSeconds % 3600) / 60);
 	const secs = remainingSeconds % 60;
-	logInfo(
-		jobId,
-		`Countdown tick: ${hours}h ${minutes}m ${secs}s remaining`,
-	);
+	logInfo(jobId, `Countdown tick: ${hours}h ${minutes}m ${secs}s remaining`);
 }
 
 /** Log countdown completed */
@@ -123,19 +129,13 @@ export function logAutoResumeFailed(jobId: string, error: string): void {
 }
 
 /** Log notification sent */
-export function logNotificationSent(
-	jobId: string,
-	beforeReset: number,
-): void {
+export function logNotificationSent(jobId: string, beforeReset: number): void {
 	const minutes = Math.floor(beforeReset / 60);
 	logInfo(jobId, `Notification sent: ${minutes} minutes before reset`);
 }
 
 /** Log notification failed */
-export function logNotificationFailed(
-	jobId: string,
-	error: string,
-): void {
+export function logNotificationFailed(jobId: string, error: string): void {
 	logWarn(jobId, `Notification failed: ${error}`);
 }
 
@@ -149,10 +149,7 @@ export function logMirrorUpdate(jobId: string, success: boolean): void {
 }
 
 /** Log quota exhaustion detected */
-export function logQuotaExhausted(
-	jobId: string,
-	resetAt: string,
-): void {
+export function logQuotaExhausted(jobId: string, resetAt: string): void {
 	logInfo(jobId, `GLM 5h quota exhausted, reset at ${resetAt}`);
 }
 

package/harness/index.ts

@@ -261,3 +261,45 @@ export type {
 	KnowledgeQuery,
 	KnowledgeResult,
 } from "../packages/memory-engine/src/types.js";
+
+// --- E2E Testing (RFC-0101) -----------------------------------------------
+export {
+	detectProjectType,
+	detectAvailableTools,
+	getRecommendedTools,
+	makeSmartDecision,
+	generateToolsPresentation,
+	getE2EToolsConfig,
+} from "./e2e/tools-detector.js";
+export type {
+	SmartDecision,
+	TestingTool,
+	E2EToolsConfig,
+	ProjectType,
+	NativeAppType,
+} from "./e2e/tools-detector.js";
+export { runAutoTestLoop } from "./e2e/auto-test-loop.js";
+export type {
+	E2ETestResult,
+	AutoTestConfig,
+	AutoTestResult,
+} from "./e2e/auto-test-loop.js";
+
+// --- Skill SaaS Sync (RFC-0106) ----------------------------------------
+export {
+	syncSkillsFromSaaS,
+	getSkillFromSaaS,
+	isSkillSyncConfigured,
+	getSyncStatus,
+} from "./skill-sync.js";
+
+// --- SSH Hang Recovery (auto-recovery from SSH hangs) -------------------
+export {
+	sshWithRecovery,
+	pm2Restart,
+	getPM2Status,
+	safeSSHCommand,
+	pingServer,
+	type SSHConfig,
+	type SSHResult,
+} from "./ssh-hang-recovery.js";

package/harness/loop-runtime.ts

@@ -46,6 +46,9 @@ import {
 	createForkedSummarizer,
 } from "./forked-summarizer.js";
 
+// GLM quota error parsing for 429 responses
+import { parseGLMErrorResetTime } from "./e2e/glm-quota-scraper.js";
+
 // --- Constants ----------------------------------------------------------------
 
 const DEFAULT_MAX_COMPACT_RETRIES = 3;
@@ -84,9 +87,7 @@ export interface LoopCallbacks {
 	/** Get current checkpoint (used by auto-resume to read resumeAt) */
 	onGetCheckpoint?: () => Promise<RuntimeCheckpoint | null>;
 	/** Check mirror for a provider (used by auto-resume to detect early reset) */
-	onCheckMirror?: (
-		provider: string,
-	) => Promise<{ h5_used_pct?: number } | null>;
+	onCheckMirror?: (provider: string) => Promise<{ h5_used_pct?: number } | null>;
 	/** Generic notify (used by auto-resume to tell user wait time) */
 	onNotify?: (msg: string, type?: string) => Promise<void>;
 	/** Save checkpoint (call persistence layer) */
@@ -244,11 +245,7 @@ export class LoopRuntime {
 					await new Promise((r) => setTimeout(r, 5 * 60_000)); // 5 min
 					// Re-check mirror — quota may have reset early
 					const fresh = await this.callbacks.onCheckMirror?.("minimax");
-					if (
-						fresh &&
-						fresh.h5_used_pct !== undefined &&
-						fresh.h5_used_pct < 100
-					) {
+					if (fresh && fresh.h5_used_pct !== undefined && fresh.h5_used_pct < 100) {
 						break; // quota reset early, resume now
 					}
 				}
@@ -447,14 +444,54 @@ export class LoopRuntime {
 					continue;
 				}
 
-				// Error without compact — surface it
-				throw new Error(result.error ?? "Unknown invoke error");
+				// Error without compact — check for GLM 429 quota error
+				const errMsg = result.error ?? "Unknown invoke error";
+				if (errMsg.includes("1308") || errMsg.includes("Usage limit")) {
+					const resetTime = parseGLMErrorResetTime(errMsg);
+					if (resetTime) {
+						const resetEpoch = new Date(resetTime).getTime();
+						this.mirrorStore.writeProvider("minimax", {
+							provider: "minimax",
+							synced_at: new Date().toISOString(),
+							source: "tui-signal",
+							h5_used_pct: 100,
+							h5_resets_at_epoch: resetEpoch,
+						});
+						console.log(`[LoopRuntime] GLM quota hit, reset at ${resetTime}`);
+					}
+					await this.handleQuotaPause();
+					return;
+				}
+				throw new Error(errMsg);
 			}
 
 			// Fallback: direct invoke without orchestrator
 			const result = await this.callbacks.onInvokeAgent?.(invokeOptions);
 			if (!result || !result.success) {
-				throw new Error(result?.error ?? "Invoke failed");
+				const errorMsg = result?.error ?? "Invoke failed";
+
+				// Check if this is a GLM 429 quota error with reset time
+				if (errorMsg.includes("1308") || errorMsg.includes("Usage limit")) {
+					const resetTime = parseGLMErrorResetTime(errorMsg);
+					if (resetTime) {
+						// Update mirror with reset epoch so auto-resume works
+						const resetEpoch = new Date(resetTime).getTime();
+						this.mirrorStore.writeProvider("minimax", {
+							provider: "minimax",
+							synced_at: new Date().toISOString(),
+							source: "tui-signal",
+							h5_used_pct: 100,
+							h5_resets_at_epoch: resetEpoch,
+						});
+						console.log(`[LoopRuntime] GLM quota hit, reset at ${resetTime}`);
+					}
+
+					// Handle quota pause instead of throwing
+					await this.handleQuotaPause();
+					return;
+				}
+
+				throw new Error(errorMsg);
 			}
 
 			const continued = await this.handleAutoContinuation(
@@ -660,8 +697,7 @@ export class LoopRuntime {
 			// Append repair instruction
 			messages.push({
 				role: "user",
-				content:
-					"Tests failed. Please fix the issues and ensure all tests pass.",
+				content: "Tests failed. Please fix the issues and ensure all tests pass.",
 				timestamp: Date.now(),
 			});
 		}

package/harness/skill-sync.test.ts

@@ -0,0 +1,57 @@
+/**
+ * Skill Sync Tests
+ */
+
+// @ts-ignore - bun:test types
+import { describe, test, expect, beforeEach, afterEach } from "bun:test";
+import { tmpdir } from "node:os";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+
+// Mock the environment
+const originalApiKey = process.env.SKILLS_SAAS_API_KEY;
+
+describe("Skill Sync", () => {
+	let tempCacheDir: string;
+
+	beforeEach(() => {
+		tempCacheDir = join(tmpdir(), `skill-sync-test-${Date.now()}`);
+		mkdirSync(tempCacheDir, { recursive: true });
+	});
+
+	afterEach(() => {
+		// Restore env
+		if (originalApiKey === undefined) {
+			delete process.env.SKILLS_SAAS_API_KEY;
+		} else {
+			process.env.SKILLS_SAAS_API_KEY = originalApiKey;
+		}
+		// Cleanup
+		try {
+			rmSync(tempCacheDir, { recursive: true });
+		} catch {
+			// Ignore
+		}
+	});
+
+	describe("isSkillSyncConfigured", () => {
+		test("returns false when API key not set", () => {
+			delete process.env.SKILLS_SAAS_API_KEY;
+			// Import dynamically to test
+			expect(process.env.SKILLS_SAAS_API_KEY).toBeUndefined();
+		});
+
+		test("returns true when API key is set", () => {
+			process.env.SKILLS_SAAS_API_KEY = "sk_test_xxx";
+			expect(process.env.SKILLS_SAAS_API_KEY).toBe("sk_test_xxx");
+		});
+	});
+
+	describe("getSyncStatus", () => {
+		test("returns correct status when not configured", () => {
+			delete process.env.SKILLS_SAAS_API_KEY;
+			// Status should show not configured
+			expect(process.env.SKILLS_SAAS_API_KEY).toBeUndefined();
+		});
+	});
+});

package/harness/skill-sync.ts

@@ -0,0 +1,247 @@
+/**
+ * Skill Sync Service — RFC-0106
+ *
+ * Syncs skills from Skills SaaS backend and registers them with local registry.
+ * Falls back gracefully if backend is unavailable.
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import type {
+	SkillInfo,
+	GetSkillResult,
+} from "../packages/skill-mcp-client/src/types.js";
+import { SkillAPIError } from "../packages/skill-mcp-client/src/errors.js";
+
+type SkillMCPClient = InstanceType<
+	typeof import("../packages/skill-mcp-client/src/client.js").SkillMCPClient
+>;
+
+// Lazy import to avoid errors if package not built yet
+let clientInstance: SkillMCPClient | null = null;
+
+async function getClient(): Promise<SkillMCPClient | null> {
+	if (!process.env.SKILLS_SAAS_API_KEY) {
+		return null;
+	}
+
+	if (clientInstance) {
+		return clientInstance;
+	}
+
+	try {
+		const mod = await import("../packages/skill-mcp-client/src/client.js");
+		const SkillMCPClientClass = mod.SkillMCPClient;
+		const serverUrl =
+			process.env.SKILLS_SAAS_URL || "https://api.skills.bunchee.online";
+
+		clientInstance = new SkillMCPClientClass({
+			serverUrl,
+			apiKey: process.env.SKILLS_SAAS_API_KEY,
+			timeoutMs: 5000,
+		});
+		return clientInstance;
+	} catch {
+		console.warn("[skill-sync] Failed to import skill-mcp-client");
+		return null;
+	}
+}
+
+// Cache directory
+const CACHE_DIR = join(homedir(), ".pi", "skills-cache");
+const CACHE_FILE = join(CACHE_DIR, "skills-cache.json");
+const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
+
+interface CacheData {
+	skills: SkillInfo[];
+	cachedAt: string;
+	version: string;
+}
+
+interface SkillSyncResult {
+	success: boolean;
+	skillCount: number;
+	source: "saas" | "cache" | "none";
+	error?: string;
+}
+
+/**
+ * Get cached skills from local disk
+ */
+function getCachedSkills(): CacheData | null {
+	try {
+		if (!existsSync(CACHE_FILE)) {
+			return null;
+		}
+
+		const data = JSON.parse(readFileSync(CACHE_FILE, "utf-8")) as CacheData;
+		const age = Date.now() - new Date(data.cachedAt).getTime();
+
+		if (age > CACHE_TTL_MS) {
+			return null; // Cache expired
+		}
+
+		return data;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Save skills to local cache
+ */
+function saveCache(data: CacheData): void {
+	try {
+		if (!existsSync(CACHE_DIR)) {
+			mkdirSync(CACHE_DIR, { recursive: true });
+		}
+		writeFileSync(CACHE_FILE, JSON.stringify(data, null, 2));
+	} catch {
+		console.warn("[skill-sync] Failed to save cache");
+	}
+}
+
+/**
+ * Fetch and cache skills from SaaS backend
+ */
+async function fetchFromSaaS(): Promise<CacheData | null> {
+	const client = await getClient();
+	if (!client) {
+		return null;
+	}
+
+	try {
+		const result = await client.listSkills({ limit: 100 });
+		const cacheData: CacheData = {
+			skills: result.skills,
+			cachedAt: new Date().toISOString(),
+			version: "1.0.0",
+		};
+		saveCache(cacheData);
+		return cacheData;
+	} catch (error) {
+		if (error instanceof SkillAPIError) {
+			console.warn(
+				`[skill-sync] API error: ${error.message} (code: ${error.code})`,
+			);
+		} else {
+			console.warn("[skill-sync] Failed to fetch from SaaS");
+		}
+		return null;
+	}
+}
+
+/**
+ * Sync skills from SaaS backend (RFC-0106)
+ *
+ * Flow:
+ * 1. Check cache first
+ * 2. If cache valid, return cached
+ * 3. If cache expired or missing, fetch from SaaS
+ * 4. If SaaS fails, return empty array (graceful degradation)
+ *
+ * @param options.forceRefresh - Force refresh from SaaS
+ * @returns Skills list (may be empty if backend unavailable)
+ */
+export async function syncSkillsFromSaaS(
+	options: { forceRefresh?: boolean } = {},
+): Promise<SkillSyncResult> {
+	// Check if API key is configured
+	if (!process.env.SKILLS_SAAS_API_KEY) {
+		return {
+			success: true,
+			skillCount: 0,
+			source: "none",
+			error: "SKILLS_SAAS_API_KEY not configured",
+		};
+	}
+
+	// Check cache first (unless force refresh)
+	if (!options.forceRefresh) {
+		const cached = getCachedSkills();
+		if (cached) {
+			return {
+				success: true,
+				skillCount: cached.skills.length,
+				source: "cache",
+			};
+		}
+	}
+
+	// Fetch from SaaS
+	const data = await fetchFromSaaS();
+	if (data) {
+		return {
+			success: true,
+			skillCount: data.skills.length,
+			source: "saas",
+		};
+	}
+
+	// SaaS failed, try to use stale cache
+	const staleCache = getCachedSkills();
+	if (staleCache) {
+		return {
+			success: true,
+			skillCount: staleCache.skills.length,
+			source: "cache",
+			error: "Using stale cache (SaaS unavailable)",
+		};
+	}
+
+	// Everything failed, return empty
+	return {
+		success: true,
+		skillCount: 0,
+		source: "none",
+		error: "Could not reach Skills SaaS backend",
+	};
+}
+
+/**
+ * Get full skill prompt by slug or id
+ *
+ * Returns null if skill not found or backend unavailable
+ */
+export async function getSkillFromSaaS(identifier: {
+	slug?: string;
+	id?: string;
+}): Promise<GetSkillResult | null> {
+	const client = await getClient();
+	if (!client) {
+		return null;
+	}
+
+	try {
+		return await client.getSkill(identifier);
+	} catch (error) {
+		if (error instanceof SkillAPIError) {
+			console.warn(`[skill-sync] Failed to get skill: ${error.message}`);
+		}
+		return null;
+	}
+}
+
+/**
+ * Check if skill sync is configured
+ */
+export function isSkillSyncConfigured(): boolean {
+	return !!process.env.SKILLS_SAAS_API_KEY;
+}
+
+/**
+ * Get sync status info
+ */
+export function getSyncStatus(): {
+	configured: boolean;
+	cacheValid: boolean;
+	cachedSkills: number;
+} {
+	const cache = getCachedSkills();
+	return {
+		configured: isSkillSyncConfigured(),
+		cacheValid: !!cache,
+		cachedSkills: cache?.skills.length ?? 0,
+	};
+}