pi-crew 0.9.68 → 0.10.2

package/src/benchmark/benchmark-runner.ts

@@ -1,313 +0,0 @@
-/**
- * Benchmark runner - agent-eval inspired benchmarking system.
- * Provides tiered evaluation for workflow tasks.
- */
-
-import { execFileSync } from "node:child_process";
-
-export interface BenchmarkJudge {
-	type: "pytest" | "grep" | "command";
-	command?: string;
-	pattern?: string;
-	description: string;
-}
-
-export interface BenchmarkTask {
-	id: string;
-	name: string;
-	prompt: string;
-	judges: BenchmarkJudge[];
-	/** Optional task-type label used for aggregate metrics grouping. */
-	taskType?: string;
-}
-
-export interface BenchmarkResult {
-	taskId: string;
-	/** Task-type label for aggregation grouping. */
-	taskType?: string;
-	passed: boolean;
-	judgeResults: { description: string; passed: boolean; output?: string }[];
-	durationMs: number;
-	/** Estimated cost in dollars (0 if not tracked). */
-	cost: number;
-}
-
-/**
- * Validate command against allowlist to prevent shell injection.
- * Only allows specific safe commands with arguments.
- */
-/**
- * Validate command against allowlist to prevent shell injection.
- * Uses comprehensive shell metacharacter blocking similar to safe-bash.ts.
- */
-function validateCommand(command: string): void {
-	// Basic allowlist - must start with an allowed command.
-	// SECURITY (H-7): `npx`/`node` were removed because they enable arbitrary code
-	// execution without any shell metacharacter (e.g. `npx --yes evil-package`
-	// or `node -e "require('fs')…"`). Use `npm test`/`npm run …` instead of raw
-	// `node`/`npx` in benchmark task definitions.
-	// `echo` is allowed because the metachar blocker (validateGateCommand) rejects
-	// command substitution (`$(...)`, backticks), so `echo $(evil)` cannot run;
-	// bare `echo …` only prints. It's the canonical exit-0 command used in
-	// benchmark fixtures across Linux/macOS/Windows(sh).
-	const allowlist = /^(pytest|grep|npm test|cargo test|cargo clippy|echo) /;
-	if (!allowlist.test(command)) {
-		throw new Error(`Command not allowed: ${command}. Only pytest, grep, npm test, cargo test/clippy, echo allowed.`);
-	}
-
-	// Block shell metacharacters after command name
-	const afterCommand = command.substring(command.indexOf(" ") + 1);
-
-	// Block dangerous shell metacharacters
-	const dangerousPatterns = [
-		/[;&|`$(){}[\]<>\\]/, // Shell metacharacters
-		/\$\([^)]*\)/, // Command substitution $(...)
-		/`[^`]*`/, // Backtick command substitution
-		/\|/, // Pipe
-		/&&/, // And
-		/\|\|/, // Or
-		/>>/, // Append redirect
-		/2>&1/, // stderr redirect
-		/>/, // Output redirect
-		/</, // Input redirect
-	];
-
-	for (const pattern of dangerousPatterns) {
-		if (pattern.test(afterCommand)) {
-			throw new Error(`Shell metacharacters not allowed in command arguments`);
-		}
-	}
-}
-
-/**
- * Run a single benchmark task with tiered judges.
- * Tier 1: pytest (fast, deterministic)
- * Tier 2: grep pattern matching
- * Tier 3: command execution
- * Fails fast on first tier failure.
- */
-function splitCommand(command: string): { program: string; args: string[] } {
-	// Naive split on whitespace. validateCommand already rejects shell
-	// metacharacters, so a simple split is safe.
-	const parts = command.trim().split(/\s+/);
-	if (parts.length === 0) {
-		throw new Error("Empty command");
-	}
-	return { program: parts[0]!, args: parts.slice(1) };
-}
-
-export async function runBenchmark(task: BenchmarkTask): Promise<BenchmarkResult> {
-	const startTime = Date.now();
-	const judgeResults: BenchmarkResult["judgeResults"] = [];
-
-	for (const judge of task.judges) {
-		try {
-			let passed = false;
-			let output: string | undefined;
-
-			if (judge.type === "pytest" && judge.command) {
-				// Validate command before execution (defense-in-depth)
-				validateCommand(judge.command);
-				// Use execFileSync to avoid shell parsing. validateCommand
-				// already rejects metacharacters, so a simple split is safe.
-				const { program, args } = splitCommand(judge.command);
-				// Tier 1: pytest - fast deterministic check
-				output = execFileSync(program, args, {
-					timeout: 5000,
-					encoding: "utf-8",
-					cwd: process.cwd(),
-				});
-				// Look for pytest summary line with passed count
-				passed = output.includes("passed");
-			} else if (judge.type === "grep" && judge.pattern && judge.command) {
-				// Validate command before execution (defense-in-depth)
-				validateCommand(judge.command);
-				const { program, args } = splitCommand(judge.command);
-				// Tier 2: grep pattern matching
-				output = execFileSync(program, args, {
-					timeout: 5000,
-					encoding: "utf-8",
-					cwd: process.cwd(),
-				});
-				passed = output.includes(judge.pattern);
-			} else if (judge.type === "command" && judge.command) {
-				// Validate command before execution (defense-in-depth)
-				validateCommand(judge.command);
-				const { program, args } = splitCommand(judge.command);
-				// Tier 3: command execution
-				output = execFileSync(program, args, {
-					timeout: 10000,
-					encoding: "utf-8",
-					cwd: process.cwd(),
-				});
-				passed = true; // Command succeeded = pass
-			}
-
-			judgeResults.push({
-				description: judge.description,
-				passed: passed ?? false,
-				output,
-			});
-		} catch (e: unknown) {
-			const error = e as { message?: string };
-			judgeResults.push({
-				description: judge.description,
-				passed: false,
-				output: error.message ?? String(e),
-			});
-		}
-	}
-
-	return {
-		taskId: task.id,
-		passed: judgeResults.every((j) => j.passed),
-		judgeResults,
-		durationMs: Date.now() - startTime,
-		cost: 0,
-		taskType: task.taskType,
-	};
-}
-
-/**
- * Aggregate metrics computed over a group of benchmark results for a single task type.
- */
-export interface BenchmarkMetrics {
-	taskType: string;
-	totalTasks: number;
-	passedTasks: number;
-	/** Ratio of passed/total (0–1). */
-	passRate: number;
-	/** Mean execution duration in milliseconds. */
-	avgTimeMs: number;
-	/** Total estimated cost in dollars across all tasks. */
-	totalCost: number;
-	/** Mean cost in dollars per task. */
-	avgCost: number;
-}
-
-/**
- * Per-task-type aggregate metrics map.
- * Keys are task-type labels; "__default__" is used when a task has no label.
- */
-export type AggregateMetrics = Record<string, BenchmarkMetrics>;
-
-/**
- * Run multiple benchmark tasks and aggregate results.
- *
- * @param tasks - Benchmark tasks to execute. Each task may carry a `taskType` label.
- * @param taskTypes - Optional subset of task-type labels to run. If provided, only tasks
- *   whose `taskType` is in this set will be executed. If omitted, all tasks run.
- */
-export async function runBenchmarkSuite(
-	tasks: BenchmarkTask[],
-	taskTypes?: string[],
-): Promise<{
-	results: BenchmarkResult[];
-	totalPassed: number;
-	totalFailed: number;
-	totalDurationMs: number;
-	totalCost: number;
-}> {
-	const filtered = taskTypes ? tasks.filter((t) => t.taskType && taskTypes.includes(t.taskType)) : tasks;
-
-	const results: BenchmarkResult[] = [];
-
-	for (const task of filtered) {
-		const result = await runBenchmark(task);
-		results.push(result);
-	}
-
-	const totalPassed = results.filter((r) => r.passed).length;
-	const totalFailed = results.length - totalPassed;
-	const totalDurationMs = results.reduce((a, b) => a + b.durationMs, 0);
-	const totalCost = results.reduce((a, b) => a + b.cost, 0);
-
-	return { results, totalPassed, totalFailed, totalDurationMs, totalCost };
-}
-
-/**
- * Aggregate benchmark results into per-task-type metrics.
- *
- * @param results - Raw benchmark results (may include any task-type mix).
- * @returns A map from task-type label to `BenchmarkMetrics`. Tasks with no label
- *   are grouped under `"__default__"`.
- */
-export function aggregateBenchmarkMetrics(results: BenchmarkResult[]): AggregateMetrics {
-	const buckets: Record<string, BenchmarkResult[]> = {};
-
-	for (const result of results) {
-		const key = result.taskType ?? "__default__";
-		if (!buckets[key]) buckets[key] = [];
-		buckets[key].push(result);
-	}
-
-	const metrics: AggregateMetrics = {};
-
-	for (const [taskType, group] of Object.entries(buckets)) {
-		const totalTasks = group.length;
-		const passedTasks = group.filter((r) => r.passed).length;
-		const passRate = totalTasks > 0 ? passedTasks / totalTasks : 0;
-		const avgTimeMs = totalTasks > 0 ? group.reduce((s, r) => s + r.durationMs, 0) / totalTasks : 0;
-		const totalCost = group.reduce((s, r) => s + r.cost, 0);
-		const avgCost = totalTasks > 0 ? totalCost / totalTasks : 0;
-
-		metrics[taskType] = {
-			taskType,
-			totalTasks,
-			passedTasks,
-			passRate: Math.round(passRate * 1000) / 1000,
-			avgTimeMs: Math.round(avgTimeMs),
-			totalCost: Math.round(totalCost * 1e6) / 1e6,
-			avgCost: Math.round(avgCost * 1e6) / 1e6,
-		};
-	}
-
-	return metrics;
-}
-
-/**
- * Generate a markdown comparison table for benchmark results including per-type aggregates.
- *
- * @param results - Benchmark results to report.
- * @param includeTaskTypeComparison - When true (default), appends a per-task-type aggregate table.
- */
-export function generateBenchmarkReport(results: BenchmarkResult[], includeTaskTypeComparison = true): string {
-	const lines: string[] = ["# Benchmark Results", ""];
-
-	lines.push("| Task | Type | Status | Duration | Cost |");
-	lines.push("|------|------|--------|---------|------|");
-
-	for (const r of results) {
-		const status = r.passed ? "✅ PASS" : "❌ FAIL";
-		const type = r.taskType ?? "—";
-		const cost = r.cost > 0 ? `$${r.cost.toFixed(4)}` : "—";
-		lines.push(`| ${r.taskId} | ${type} | ${status} | ${r.durationMs}ms | ${cost} |`);
-	}
-
-	lines.push("");
-
-	// Per-type aggregate table.
-	if (includeTaskTypeComparison && results.length > 0) {
-		const metrics = aggregateBenchmarkMetrics(results);
-		const types = Object.keys(metrics).sort();
-
-		if (types.length > 0) {
-			lines.push("## Per-Task-Type Comparison", "");
-			lines.push("| Task Type | Total | Passed | Pass Rate | Avg Time | Avg Cost |");
-			lines.push("|-----------|-------|--------|-----------|----------|---------|");
-
-			for (const t of types) {
-				const m = metrics[t];
-				const passRatePct = `${(m.passRate * 100).toFixed(1)}%`;
-				const avgCostStr = m.avgCost > 0 ? `$${m.avgCost.toFixed(4)}` : "—";
-				lines.push(`| ${m.taskType} | ${m.totalTasks} | ${m.passedTasks} | ${passRatePct} | ${m.avgTimeMs}ms | ${avgCostStr} |`);
-			}
-		}
-	}
-
-	const passed = results.filter((r) => r.passed).length;
-	lines.push("");
-	lines.push(`**Total: ${passed}/${results.length} passed**`);
-
-	return lines.join("\n");
-}

package/src/benchmark/feedback-loop.ts

@@ -1,73 +0,0 @@
-/**
- * Feedback loop - continuous improvement cycle: evaluate → learn → apply → re-evaluate
- */
-
-import type { RunMetrics } from "../state/stores/run-metrics.ts";
-
-/** @internal */
-interface FeedbackLoopStats {
-	runsObserved: number;
-	avgSuccessRate: number;
-	recommendations: string[];
-}
-
-/** @internal */
-class FeedbackLoop {
-	private runs: RunMetrics[] = [];
-	private static readonly MAX_RUNS = 1000;
-
-	/**
-	 * Record a run's metrics for learning.
-	 */
-	recordRun(metrics: RunMetrics): void {
-		this.runs.push(metrics);
-		// Trim to MAX_RUNS to prevent unbounded memory growth
-		if (this.runs.length > FeedbackLoop.MAX_RUNS) {
-			this.runs = this.runs.slice(-FeedbackLoop.MAX_RUNS);
-		}
-	}
-
-	/**
-	 * Get current statistics and recommendations.
-	 */
-	getStats(): FeedbackLoopStats {
-		if (this.runs.length === 0) {
-			return {
-				runsObserved: 0,
-				avgSuccessRate: 0,
-				recommendations: ["No runs observed yet. Run some workflows to gather data."],
-			};
-		}
-
-		const successRates = this.runs.map((r) => (r.taskCount > 0 ? r.completedCount / r.taskCount : 0));
-		const avg = successRates.reduce((a, b) => a + b, 0) / successRates.length;
-
-		const recommendations: string[] = [];
-		if (avg >= 0.9) {
-			recommendations.push(`High success rate (${(avg * 100).toFixed(0)}%). Current configuration is working well.`);
-		} else if (avg >= 0.7) {
-			recommendations.push(`Moderate success rate (${(avg * 100).toFixed(0)}%). Consider reviewing failed tasks for patterns.`);
-		} else {
-			recommendations.push(`Low success rate (${(avg * 100).toFixed(0)}%). Investigate failure patterns.`);
-		}
-
-		// Cost awareness
-		const avgCost = this.runs.reduce((a, b) => a + b.totalCost, 0) / this.runs.length;
-		if (avgCost > 10) {
-			recommendations.push(`Average cost per run: $${avgCost.toFixed(2)}. Consider optimization.`);
-		}
-
-		return {
-			runsObserved: this.runs.length,
-			avgSuccessRate: avg,
-			recommendations,
-		};
-	}
-
-	/**
-	 * Clear recorded runs.
-	 */
-	clear(): void {
-		this.runs = [];
-	}
-}

package/src/config/resilient-parser.ts

@@ -1,117 +0,0 @@
-/**
- * Per-field resilient config parsing — continues on error, collects all field-level issues.
- */
-
-import type { TSchema } from "@sinclair/typebox";
-import { Value } from "@sinclair/typebox/value";
-import { PiTeamsConfigSchema } from "../schema/config-schema.ts";
-import { type PiTeamsConfig, parseConfig } from "./config.ts";
-import { suggestConfigKey } from "./suggestions.ts";
-
-// ---------------------------------------------------------------------------
-// Types
-// ---------------------------------------------------------------------------
-
-export interface FieldError {
-	field: string;
-	message: string;
-	value: unknown;
-	suggestion?: string;
-}
-
-export interface ResilientParseResult {
-	config: Record<string, unknown>;
-	errors: FieldError[];
-	warnings: string[];
-	valid: boolean;
-}
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
-const KNOWN_TOP_LEVEL_KEYS: readonly string[] = Object.keys(PiTeamsConfigSchema.properties ?? {});
-
-function isObject(value: unknown): value is Record<string, unknown> {
-	return value !== null && typeof value === "object" && !Array.isArray(value);
-}
-
-// ---------------------------------------------------------------------------
-// Implementation
-// ---------------------------------------------------------------------------
-
-/**
- * Parse a raw config value field-by-field.
- *
- * - Each top-level key is validated independently via TypeBox sub-schemas.
- * - Fields that fail are recorded as errors but do not block others.
- * - Unknown keys get a fuzzy suggestion via Levenshtein distance.
- * - The returned `config` contains only successfully-parsed fields (via `parseConfig`).
- */
-export function parseConfigResilient(raw: unknown): ResilientParseResult {
-	const errors: FieldError[] = [];
-	const warnings: string[] = [];
-
-	if (!isObject(raw)) {
-		return {
-			config: {},
-			errors: [
-				{
-					field: "config",
-					message: "config must be an object",
-					value: raw,
-				},
-			],
-			warnings,
-			valid: false,
-		};
-	}
-
-	// Collect unknown top-level keys with suggestions
-	for (const key of Object.keys(raw)) {
-		if (!KNOWN_TOP_LEVEL_KEYS.includes(key)) {
-			const suggestion = suggestConfigKey(key, KNOWN_TOP_LEVEL_KEYS);
-			errors.push({
-				field: key,
-				message: `unknown config key '${key}'`,
-				value: raw[key],
-				suggestion: suggestion ?? undefined,
-			});
-		}
-	}
-
-	// Use TypeBox per-field validation for known keys
-	for (const key of KNOWN_TOP_LEVEL_KEYS) {
-		if (!(key in raw)) continue;
-		const value = raw[key];
-		const subSchema = (PiTeamsConfigSchema.properties as Record<string, unknown>)[key];
-		if (subSchema && !Value.Check(subSchema as TSchema, value)) {
-			// Collect ALL errors for this field
-			const fieldErrors = [...Value.Errors(subSchema as TSchema, value)];
-			for (const err of fieldErrors) {
-				const errObj = err as unknown as Record<string, unknown>;
-				const msg = typeof errObj.message === "string" ? errObj.message : "invalid value";
-				errors.push({
-					field: key,
-					message: msg,
-					value,
-				});
-			}
-		}
-	}
-
-	// Build the valid subset using existing parseConfig (which silently drops invalid fields)
-	const parsed: PiTeamsConfig = parseConfig(raw);
-	const config: Record<string, unknown> = {};
-	for (const key of Object.keys(parsed)) {
-		const val = (parsed as Record<string, unknown>)[key];
-		if (val !== undefined) config[key] = val;
-	}
-
-	return {
-		config,
-		errors,
-		warnings,
-		valid: errors.length === 0,
-	};
-}

package/src/extension/crew-vibes/cat-frames.ts

@@ -1,18 +0,0 @@
-/**
- * ANSI art cat animation frames extracted from runner-spritesheet.png.
- * Used when PUA font glyphs are not available (web terminals like gotty).
- * Each frame is an array of strings (one per line).
- */
-export const CAT_FRAMES: readonly string[][] = [
-	// Frame 0 — standing
-	["▄▄ ███▄ ", "▀▀█████▄", "  ██▀██ ", "  ▀▀  ▀▀"],
-	// Frame 1 — step 1
-	["  ████  ", "  █████▄", "▄██████▀", "  ██ ▀█▄"],
-	// Frame 2 — step 2
-	[" ▄██████", " █████▀ ", "███████ ", "▀██████ "],
-	// Frame 3 — step 3
-	["  ████  ", "▄█████▄ ", "▀██████▄", "▄█▀  ██ "],
-] as const;
-
-/** Number of animation frames. */
-export const CAT_FRAME_COUNT = CAT_FRAMES.length;

package/src/extension/result-watcher.ts

@@ -1,139 +0,0 @@
-import * as fs from "node:fs";
-import * as path from "node:path";
-import { buildCompletionKey, getGlobalSeenMap, markSeenWithTtl } from "../utils/completion-dedupe.ts";
-import { createFileCoalescer } from "../utils/file-coalescer.ts";
-import { closeWatcher, watchWithErrorHandler } from "../utils/fs-watch.ts";
-import { logInternalError } from "../utils/internal-error.ts";
-
-export interface ResultWatcherEvents {
-	emit(event: string, data: unknown): void;
-}
-
-export interface ResultWatcherHandle {
-	start(): void;
-	prime(): void;
-	stop(): void;
-}
-
-interface ResultWatcherDependencies {
-	watch?: typeof watchWithErrorHandler;
-}
-
-export interface ResultWatcherOptions extends ResultWatcherDependencies {
-	eventName?: string;
-	completionTtlMs?: number;
-	isCurrent?: () => boolean;
-}
-
-const RESULT_WATCHER_RESTART_MS = 3000;
-const RESULT_WATCHER_POLL_MS = 1000;
-
-function shouldFallBackToPolling(error: unknown): boolean {
-	const code = error && typeof error === "object" ? (error as { code?: unknown }).code : undefined;
-	return code === "EMFILE" || code === "ENOSPC" || code === "EPERM";
-}
-
-function readJson(filePath: string): unknown | undefined {
-	try {
-		return JSON.parse(fs.readFileSync(filePath, "utf-8")) as unknown;
-	} catch (error) {
-		logInternalError("result-watcher.parse", error, `filePath=${filePath}`);
-		return undefined;
-	}
-}
-
-export function createResultWatcher(
-	events: ResultWatcherEvents,
-	resultsDir: string,
-	eventNameOrOptions: string | ResultWatcherOptions = "pi-crew:run-result",
-): ResultWatcherHandle {
-	const options: ResultWatcherOptions = typeof eventNameOrOptions === "string" ? { eventName: eventNameOrOptions } : eventNameOrOptions;
-	const eventName = options.eventName ?? "pi-crew:run-result";
-	const completionTtlMs = options.completionTtlMs ?? 5 * 60_000;
-	const watch = options.watch ?? watchWithErrorHandler;
-	const isCurrent = options.isCurrent ?? (() => true);
-	const seen = getGlobalSeenMap("pi-crew.result-watcher");
-	let watcher: fs.FSWatcher | null | undefined;
-	let restartTimer: ReturnType<typeof setTimeout> | undefined;
-	let pollTimer: ReturnType<typeof setInterval> | undefined;
-	const coalescer = createFileCoalescer((file) => {
-		if (!isCurrent()) return;
-		const filePath = path.join(resultsDir, file);
-		if (!file.endsWith(".json") || !fs.existsSync(filePath)) return;
-		const payload = readJson(filePath);
-		if (payload === undefined) {
-			coalescer.schedule(file, RESULT_WATCHER_POLL_MS);
-			return;
-		}
-		const key = buildCompletionKey(
-			payload && typeof payload === "object" && !Array.isArray(payload) ? (payload as Record<string, unknown>) : {},
-			`file:${file}`,
-		);
-		if (!markSeenWithTtl(seen, key, Date.now(), completionTtlMs)) {
-			events.emit(eventName, payload);
-		}
-		try {
-			fs.unlinkSync(filePath);
-		} catch (error) {
-			logInternalError("result-watcher.unlink", error, `filePath=${filePath}`);
-		}
-	}, 50);
-	const poll = () => {
-		if (!isCurrent() || !fs.existsSync(resultsDir)) return;
-		for (const file of fs.readdirSync(resultsDir).filter((entry) => entry.endsWith(".json"))) coalescer.schedule(file, 0);
-	};
-	const startPolling = () => {
-		if (pollTimer) return;
-		pollTimer = setInterval(poll, RESULT_WATCHER_POLL_MS);
-		pollTimer.unref();
-		poll();
-	};
-	const stopPolling = () => {
-		if (pollTimer) clearInterval(pollTimer);
-		pollTimer = undefined;
-	};
-	const scheduleRestart = (error?: unknown) => {
-		if (shouldFallBackToPolling(error)) startPolling();
-		if (restartTimer) clearTimeout(restartTimer);
-		restartTimer = setTimeout(() => {
-			restartTimer = undefined;
-			try {
-				if (!isCurrent()) return;
-				fs.mkdirSync(resultsDir, { recursive: true });
-				handle.start();
-			} catch (error) {
-				logInternalError("result-watcher.restart", error, `resultsDir=${resultsDir}`);
-			}
-		}, RESULT_WATCHER_RESTART_MS);
-		restartTimer.unref();
-	};
-	const handle: ResultWatcherHandle = {
-		start() {
-			if (!isCurrent()) return;
-			fs.mkdirSync(resultsDir, { recursive: true });
-			if (watcher) closeWatcher(watcher);
-			watcher = watch(
-				resultsDir,
-				(event, fileName) => {
-					if (event !== "rename" || !fileName) return;
-					coalescer.schedule(fileName.toString());
-				},
-				scheduleRestart,
-			);
-			if (watcher) stopPolling();
-			watcher?.unref();
-		},
-		prime() {
-			poll();
-		},
-		stop() {
-			if (restartTimer) clearTimeout(restartTimer);
-			restartTimer = undefined;
-			closeWatcher(watcher);
-			watcher = undefined;
-			stopPolling();
-			coalescer.clear();
-		},
-	};
-	return handle;
-}