pi-harness-runtime 1.1.49 → 1.1.71-3

package/packages/intent-analyzer/src/ping-pong.ts

@@ -0,0 +1,387 @@
+import {
+	existsSync,
+	readdirSync,
+	readFileSync,
+	statSync,
+} from "node:fs";
+import { join } from "node:path";
+import { IntentAnalyzer } from "./analyzer.js";
+import type { Intent, IntentKind } from "./types.js";
+
+export type PingPongDecision =
+	| "run_ping_pong"
+	| "suggest_ping_pong"
+	| "handle_inline";
+
+export interface CodebaseComplexityInput {
+	/** Approximate total source/config/documentation files in the current repo. */
+	fileCount?: number;
+	/** Number of top-level packages/apps/workspaces. */
+	packageCount?: number;
+	/** Number of framework/runtime signals found, such as package.json, Cargo.toml, or pyproject.toml. */
+	frameworkSignalCount?: number;
+	/** Number of changed files in the working tree, if known. */
+	changedFileCount?: number;
+	/** Paths the user explicitly mentioned. */
+	mentionedPaths?: string[];
+	/** True when tests already exist and changes should usually preserve them. */
+	hasTests?: boolean;
+	/** True when release/build automation exists. */
+	hasCiOrRelease?: boolean;
+}
+
+export interface PingPongDecisionOptions {
+	/** Project-level complexity signals. Pass scanCodebaseComplexity(cwd) when available. */
+	codebase?: CodebaseComplexityInput;
+	/** Lower thresholds for fully autonomous sessions. */
+	autonomous?: boolean;
+	/** Allow exact user override words. Default: true. */
+	allowExplicitOverride?: boolean;
+}
+
+export interface PingPongSignal {
+	id: string;
+	label: string;
+	weight: number;
+}
+
+export interface PingPongDecisionResult {
+	decision: PingPongDecision;
+	score: number;
+	thresholds: {
+		run: number;
+		suggest: number;
+	};
+	intent: Intent;
+	signals: PingPongSignal[];
+	reasons: string[];
+}
+
+const CODE_CHANGE_INTENTS = new Set<IntentKind>([
+	"bug_fix",
+	"feature",
+	"refactor",
+	"testing",
+	"migration",
+	"security",
+	"performance",
+	"deployment",
+]);
+
+const INLINE_INTENTS = new Set<IntentKind>([
+	"learning",
+	"research",
+	"code_review",
+	"documentation",
+	"general",
+]);
+
+const ACTION_PATTERNS: Array<[RegExp, string, number]> = [
+	[/\b(implement|build|create|add|introduce)\b/i, "implementation request", 2],
+	[/\b(fix|debug|repair|patch|resolve)\b/i, "bug-fix request", 2],
+	[/\b(refactor|rewrite|restructure|migrate|convert|port)\b/i, "structural change request", 3],
+	[/\b(test|spec|coverage|e2e|integration)\b/i, "test-impacting request", 1],
+	[/\b(release|deploy|publish|ship)\b/i, "release-impacting request", 2],
+];
+
+const COMPLEXITY_PATTERNS: Array<[RegExp, string, number]> = [
+	[/\b(system|architecture|daemon|runtime|orchestrator|scheduler|graph|loop|workflow|pipeline)\b/i, "system/architecture area", 3],
+	[/\b(concurrent|parallel|async|queue|event|bus|lease|worker|agent|multi[- ]?agent)\b/i, "coordination or concurrency", 3],
+	[/\b(cross[- ]?package|workspace|monorepo|package|plugin|extension)\b/i, "cross-package surface", 2],
+	[/\b(auth|token|secret|permission|security|quota|billing|cost)\b/i, "sensitive or quota-sensitive area", 2],
+	[/\b(database|migration|schema|cache|state|checkpoint|persistence)\b/i, "stateful change area", 2],
+];
+
+const INLINE_PATTERNS: Array<[RegExp, string, number]> = [
+	[/\b(explain|what is|how does|why|analyze|investigate|understand)\b/i, "analysis-only wording", -4],
+	[/\b(no code|do not edit|don't edit|only explain|just explain|read only|read-only)\b/i, "explicit no-edit wording", -8],
+	[/\b(show|list|status|inspect|summarize)\b/i, "inspection wording", -2],
+];
+
+const EXPLICIT_RUN = /\b(ping[- ]?pong|write[- ]?review|plan\s*->\s*cod(?:e|ing)|review\s*->\s*fix|write\s+(?:down\s+)?(?:the\s+)?plan\s+to\s+wiki|ask\s+minimax\s+to\s+(?:write|code|implement)|(?:then\s+)?you\s+review)\b/i;
+const EXPLICIT_SKIP = /\b(no ping[- ]?pong|skip ping[- ]?pong|do not run ping[- ]?pong|don't run ping[- ]?pong)\b/i;
+
+function pushSignal(
+	signals: PingPongSignal[],
+	id: string,
+	label: string,
+	weight: number,
+): void {
+	signals.push({ id, label, weight });
+}
+
+function countPathMentions(text: string): string[] {
+	const matches = text.match(
+		/(?:^|\s)((?:\.\/|\/)?(?:src|lib|packages|apps|harness|test|tests|docs|wiki|scripts|config|src-ui|src-tauri)[/\w.-]*\.[a-z0-9]+)/gi,
+	);
+	if (!matches) return [];
+	return matches.map((m) => m.trim());
+}
+
+function clampScore(score: number): number {
+	return Math.max(0, Math.min(20, score));
+}
+
+export class PingPongDecisionEngine {
+	private readonly intentAnalyzer: IntentAnalyzer;
+
+	constructor(intentAnalyzer = new IntentAnalyzer()) {
+		this.intentAnalyzer = intentAnalyzer;
+	}
+
+	analyze(
+		request: string,
+		options: PingPongDecisionOptions = {},
+	): PingPongDecisionResult {
+		const allowOverride = options.allowExplicitOverride ?? true;
+		const intent = this.intentAnalyzer.analyze(request);
+		const signals: PingPongSignal[] = [];
+		let score = 0;
+
+		if (allowOverride && EXPLICIT_SKIP.test(request)) {
+			pushSignal(signals, "override.skip", "explicit skip request", -20);
+		} else if (allowOverride && EXPLICIT_RUN.test(request)) {
+			pushSignal(signals, "override.run", "explicit PING-PONG request", 20);
+		}
+
+		if (CODE_CHANGE_INTENTS.has(intent.kind)) {
+			const weight = intent.confidence === "high" ? 4 : 3;
+			pushSignal(signals, `intent.${intent.kind}`, `${intent.kind} intent`, weight);
+		} else if (INLINE_INTENTS.has(intent.kind)) {
+			const weight = intent.kind === "general" ? -3 : -2;
+			pushSignal(signals, `intent.${intent.kind}`, `${intent.kind} intent`, weight);
+		}
+
+		for (const [pattern, label, weight] of ACTION_PATTERNS) {
+			if (pattern.test(request)) pushSignal(signals, `action.${label}`, label, weight);
+		}
+
+		for (const [pattern, label, weight] of COMPLEXITY_PATTERNS) {
+			if (pattern.test(request)) pushSignal(signals, `complexity.${label}`, label, weight);
+		}
+
+		for (const [pattern, label, weight] of INLINE_PATTERNS) {
+			if (pattern.test(request)) pushSignal(signals, `inline.${label}`, label, weight);
+		}
+
+		const mentionedPaths = [
+			...(options.codebase?.mentionedPaths ?? []),
+			...countPathMentions(request),
+		];
+		if (mentionedPaths.length >= 2) {
+			pushSignal(signals, "scope.paths.multi", "multiple files mentioned", 2);
+		} else if (mentionedPaths.length === 1) {
+			pushSignal(signals, "scope.paths.single", "specific file mentioned", 1);
+		}
+
+		this.addCodebaseSignals(signals, options.codebase);
+
+		score = clampScore(signals.reduce((sum, signal) => sum + signal.weight, 0));
+
+		const thresholds = {
+			run: options.autonomous ? 8 : 9,
+			suggest: options.autonomous ? 5 : 6,
+		};
+
+		let decision: PingPongDecision;
+		if (allowOverride && EXPLICIT_SKIP.test(request)) {
+			decision = "handle_inline";
+			score = 0;
+		} else if (allowOverride && EXPLICIT_RUN.test(request)) {
+			decision = "run_ping_pong";
+			score = Math.max(score, thresholds.run);
+		} else if (score >= thresholds.run) {
+			decision = "run_ping_pong";
+		} else if (score >= thresholds.suggest) {
+			decision = "suggest_ping_pong";
+		} else {
+			decision = "handle_inline";
+		}
+
+		return {
+			decision,
+			score,
+			thresholds,
+			intent,
+			signals,
+			reasons: signals
+				.filter((signal) => signal.weight !== 0)
+				.sort((a, b) => Math.abs(b.weight) - Math.abs(a.weight))
+				.map((signal) => `${signal.label} (${signal.weight > 0 ? "+" : ""}${signal.weight})`),
+		};
+	}
+
+	private addCodebaseSignals(
+		signals: PingPongSignal[],
+		codebase?: CodebaseComplexityInput,
+	): void {
+		if (!codebase) return;
+
+		if ((codebase.fileCount ?? 0) >= 1000) {
+			pushSignal(signals, "repo.files.large", "large repository", 3);
+		} else if ((codebase.fileCount ?? 0) >= 250) {
+			pushSignal(signals, "repo.files.medium", "medium repository", 2);
+		}
+
+		if ((codebase.packageCount ?? 0) >= 5) {
+			pushSignal(signals, "repo.packages.large", "many packages/apps", 3);
+		} else if ((codebase.packageCount ?? 0) >= 2) {
+			pushSignal(signals, "repo.packages.multi", "multi-package project", 2);
+		}
+
+		if ((codebase.frameworkSignalCount ?? 0) >= 4) {
+			pushSignal(signals, "repo.frameworks.many", "many framework/runtime signals", 2);
+		}
+
+		if ((codebase.changedFileCount ?? 0) >= 8) {
+			pushSignal(signals, "repo.changes.many", "many current changes", 2);
+		}
+
+		if (codebase.hasTests) {
+			pushSignal(signals, "repo.tests", "test suite present", 1);
+		}
+
+		if (codebase.hasCiOrRelease) {
+			pushSignal(signals, "repo.release", "release/build automation present", 1);
+		}
+	}
+}
+
+export interface ScanCodebaseOptions {
+	maxFiles?: number;
+	maxDepth?: number;
+}
+
+const SKIP_DIRS = new Set([
+	".git",
+	"node_modules",
+	"dist",
+	"build",
+	"coverage",
+	".next",
+	".turbo",
+	"target",
+]);
+
+const FRAMEWORK_FILES = new Set([
+	"package.json",
+	"tsconfig.json",
+	"Cargo.toml",
+	"pyproject.toml",
+	"composer.json",
+	"go.mod",
+	"bun.lock",
+	"pnpm-lock.yaml",
+	"vite.config.ts",
+	"next.config.js",
+	"tauri.conf.json",
+]);
+
+export function scanCodebaseComplexity(
+	root: string,
+	options: ScanCodebaseOptions = {},
+): CodebaseComplexityInput {
+	const maxFiles = options.maxFiles ?? 4000;
+	const maxDepth = options.maxDepth ?? 6;
+	let fileCount = 0;
+	let packageCount = 0;
+	let frameworkSignalCount = 0;
+	let hasTests = false;
+	let hasCiOrRelease = false;
+
+	function visit(dir: string, depth: number): void {
+		if (fileCount >= maxFiles || depth > maxDepth) return;
+		let entries: string[];
+		try {
+			entries = readdirSync(dir);
+		} catch {
+			return;
+		}
+
+		for (const entry of entries) {
+			if (fileCount >= maxFiles) return;
+			const path = join(dir, entry);
+			let stats;
+			try {
+				stats = statSync(path);
+			} catch {
+				continue;
+			}
+
+			if (stats.isDirectory()) {
+				if (SKIP_DIRS.has(entry)) continue;
+				if ((entry === "packages" || entry === "apps") && depth <= 1) {
+					packageCount += countChildDirs(path);
+				}
+				if (entry === "test" || entry === "tests" || entry === "__tests__") {
+					hasTests = true;
+				}
+				visit(path, depth + 1);
+				continue;
+			}
+
+			if (!stats.isFile()) continue;
+			fileCount += 1;
+
+			if (FRAMEWORK_FILES.has(entry)) frameworkSignalCount += 1;
+			if (/\.(test|spec)\.[cm]?[jt]sx?$/.test(entry)) hasTests = true;
+			if (
+				entry === "package.json" &&
+				readJsonHasPackageScripts(path, ["build", "release", "publish"])
+			) {
+				hasCiOrRelease = true;
+			}
+			if (
+				path.includes("/.github/workflows/") ||
+				entry === "release.yml" ||
+				entry === "release.yaml"
+			) {
+				hasCiOrRelease = true;
+			}
+		}
+	}
+
+	visit(root, 0);
+
+	return {
+		fileCount,
+		packageCount,
+		frameworkSignalCount,
+		hasTests,
+		hasCiOrRelease,
+	};
+}
+
+function countChildDirs(dir: string): number {
+	try {
+		return readdirSync(dir).filter((entry) => {
+			try {
+				return statSync(join(dir, entry)).isDirectory();
+			} catch {
+				return false;
+			}
+		}).length;
+	} catch {
+		return 0;
+	}
+}
+
+function readJsonHasPackageScripts(path: string, scriptNames: string[]): boolean {
+	if (!existsSync(path)) return false;
+	try {
+		const parsed = JSON.parse(readFileSync(path, "utf8")) as {
+			scripts?: Record<string, unknown>;
+		};
+		const scripts = parsed.scripts ?? {};
+		return scriptNames.some((name) => typeof scripts[name] === "string");
+	} catch {
+		return false;
+	}
+}
+
+export function analyzePingPongDecision(
+	request: string,
+	options: PingPongDecisionOptions = {},
+): PingPongDecisionResult {
+	return new PingPongDecisionEngine().analyze(request, options);
+}

package/packages/intent-analyzer/test/ping-pong.test.ts

@@ -0,0 +1,96 @@
+import { describe, it } from "node:test";
+import { equal, ok } from "node:assert";
+import {
+	PingPongDecisionEngine,
+	analyzePingPongDecision,
+	scanCodebaseComplexity,
+} from "../src/ping-pong.js";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+
+describe("PingPongDecisionEngine", () => {
+	const engine = new PingPongDecisionEngine();
+
+	it("runs PING-PONG for explicit plan-code-review phrasing", () => {
+		const result = engine.analyze(
+			"Good plan, then write down plan to wiki/ then I will ask Minimax to write code then you review",
+		);
+
+		equal(result.decision, "run_ping_pong");
+		ok(result.reasons.some((reason) => reason.includes("explicit PING-PONG")));
+	});
+
+	it("runs PING-PONG for complex runtime implementation work", () => {
+		const result = engine.analyze("implement daemon graph loop recovery across packages", {
+			codebase: {
+				fileCount: 1200,
+				packageCount: 12,
+				frameworkSignalCount: 8,
+				hasTests: true,
+				hasCiOrRelease: true,
+			},
+		});
+
+		equal(result.decision, "run_ping_pong");
+		ok(result.score >= result.thresholds.run);
+	});
+
+	it("keeps analysis-only questions inline", () => {
+		const result = engine.analyze("analyze why the loop is quiet, do not edit code");
+
+		equal(result.decision, "handle_inline");
+		equal(result.score, 0);
+	});
+
+	it("can suggest PING-PONG for borderline code work", () => {
+		const result = engine.analyze("add tests for parser", {
+			codebase: { hasTests: true },
+		});
+
+		equal(result.decision, "suggest_ping_pong");
+		ok(result.score >= result.thresholds.suggest);
+		ok(result.score < result.thresholds.run);
+	});
+
+	it("allows explicit skip to override code-change wording", () => {
+		const result = engine.analyze("fix the daemon bug but skip ping-pong for now");
+
+		equal(result.decision, "handle_inline");
+		equal(result.score, 0);
+	});
+});
+
+describe("scanCodebaseComplexity", () => {
+	it("extracts package, test, and release signals", () => {
+		const root = mkdtempSync(join(tmpdir(), "ping-pong-scan-"));
+		try {
+			mkdirSync(join(root, "packages", "alpha"), { recursive: true });
+			mkdirSync(join(root, "packages", "beta"), { recursive: true });
+			mkdirSync(join(root, "test"), { recursive: true });
+			writeFileSync(
+				join(root, "package.json"),
+				JSON.stringify({ scripts: { build: "tsc", release: "standard-version" } }),
+			);
+			writeFileSync(join(root, "tsconfig.json"), "{}", "utf8");
+			writeFileSync(join(root, "test", "demo.test.ts"), "", "utf8");
+
+			const result = scanCodebaseComplexity(root);
+
+			ok(result.fileCount >= 3);
+			equal(result.packageCount, 2);
+			ok(result.frameworkSignalCount >= 2);
+			equal(result.hasTests, true);
+			equal(result.hasCiOrRelease, true);
+		} finally {
+			rmSync(root, { recursive: true, force: true });
+		}
+	});
+});
+
+describe("analyzePingPongDecision", () => {
+	it("uses the default engine", () => {
+		const result = analyzePingPongDecision("ping-pong this implementation");
+		equal(result.decision, "run_ping_pong");
+	});
+});

package/packages/jev-judge/README.md

@@ -0,0 +1,234 @@
+# Jev Judge
+
+Structured decision model integration for pi-harness-runtime. Uses [TypeSafe Jev](https://typesafe.ai) via OpenRouter for fast, calibrated, hallucination-free decisions.
+
+## Overview
+
+Jev is a "System One" model - a new class of AI optimized for **structured decisions** rather than text generation:
+
+| Aspect | Traditional LLM | Jev (System One) |
+|--------|-----------------|------------------|
+| Output | Text (can hallucinate) | Type-safe values (0% hallucination) |
+| Speed | ~1-2s per call | **~250ms per call** |
+| Cost | Higher | ~5x cheaper |
+| Use case | Chat, reasoning | Classification, routing, decisions |
+
+## Features
+
+- **JevJudge** - Core API for structured decisions
+- **E2EJudge** - E2E testing decisions (flaky detection, build blocking, retry)
+- **FlakyDetector** - Specialized flaky test detection
+- **DecisionEngine** - Threshold-based routing and actions
+
+## Installation
+
+```bash
+# Via monorepo (already included)
+bun install
+
+# Or install separately
+npm install @pi-harness/jev-judge
+```
+
+## Configuration
+
+Requires OpenRouter API key with TypeSafe/Jev access:
+
+```bash
+export OPENROUTER_API_KEY="sk-or-..."
+```
+
+Or set in code:
+
+```typescript
+const judge = new JevJudge({
+  apiKey: process.env.OPENROUTER_API_KEY!,
+  baseUrl: "https://openrouter.ai/api/v1", // default
+  model: "typesafe/jev-1.13", // default
+});
+```
+
+## Usage
+
+### Basic Decisions
+
+```typescript
+import { JevJudge } from "@pi-harness/jev-judge";
+
+const judge = new JevJudge({ apiKey: process.env.OPENROUTER_API_KEY! });
+
+// Ask multiple questions at once
+const result = await judge.evaluate(
+  "The deploy failed with exit code 1. Error: ECONNREFUSED",
+  {
+    isUrgent: {
+      type: "noul",
+      instructions: "Does this require immediate attention?"
+    },
+    severity: {
+      type: "choice",
+      instructions: "What is the severity level?",
+      options: ["critical", "high", "medium", "low"]
+    }
+  }
+);
+
+console.log(result.decisions.isUrgent.probability); // 0.92
+console.log(result.decisions.severity.response.choice); // "critical"
+```
+
+### Task Tracker Routing (Todo vs BD)
+
+```typescript
+import { Router } from "@pi-harness/jev-judge/decision-engine";
+
+const result = await Router.routeTaskTracker(
+  "Fix the login bug on the checkout page",
+  judge
+);
+
+if (result.tracker === "bd") {
+  // Create GitHub issue
+} else {
+  // Use local todo
+}
+```
+
+### E2E Test Flaky Detection
+
+```typescript
+import { E2EJudge } from "@pi-harness/jev-judge/e2e-judge";
+
+const e2eJudge = new E2EJudge({ apiKey: process.env.OPENROUTER_API_KEY! });
+
+// Analyze a test failure
+const flakyAnalysis = await e2eJudge.analyzeFlakiness({
+  name: "checkout-flow.spec.ts",
+  status: "failed",
+  error: "Timeout: Element #checkout-button not visible after 5000ms",
+  retries: 1
+});
+
+if (flakyAnalysis.isFlaky) {
+  console.log(`Flaky detected: ${flakyAnalysis.pattern}`);
+  // Auto-skip or retry
+}
+```
+
+### Build Decision
+
+```typescript
+// Should this failure block the build?
+const blockDecision = await e2eJudge.shouldBlockBuild(testResult);
+
+if (blockDecision.shouldBlock) {
+  console.log(`Blocking build: ${blockDecision.reason}`);
+  process.exit(1);
+}
+```
+
+### Threshold-Based Actions
+
+```typescript
+import { DecisionEngine } from "@pi-harness/jev-judge/decision-engine";
+
+const engine = new DecisionEngine(
+  { apiKey: process.env.OPENROUTER_API_KEY! },
+  { approval: 0.9, review: 0.6, block: 0.3 }
+);
+
+const decision = engine.decide(0.85);
+
+switch (decision.action) {
+  case "proceed":
+    // Auto-approve
+    break;
+  case "review":
+    // Send to human
+    break;
+  case "block":
+    // Auto-reject
+    break;
+}
+```
+
+## API Reference
+
+### JevJudge
+
+Core class for making structured decisions.
+
+```typescript
+const judge = new JevJudge(config: JevJudgeConfig);
+```
+
+#### Methods
+
+- `evaluate(state, questions)` - Evaluate state against questions
+- `noul(state, question)` - Quick yes/no evaluation
+- `choice(state, question, options)` - Quick choice evaluation
+- `thresholdDecision(probability, threshold)` - Threshold-based decision
+- `batchEvaluate(states, questions)` - Batch evaluation
+
+### E2EJudge
+
+Specialized judge for E2E testing.
+
+```typescript
+const e2eJudge = new E2EJudge(config, options);
+```
+
+#### Methods
+
+- `analyzeFlakiness(testResult)` - Detect flaky tests
+- `shouldBlockBuild(testResult)` - Build blocking decision
+- `shouldRetry(testResult)` - Retry recommendation
+- `classifyError(errorMessage)` - Error classification
+- `batchAnalyze(testResults)` - Batch test analysis
+
+### FlakyDetector
+
+Specialized for flaky test detection with pattern matching.
+
+```typescript
+const detector = new FlakyDetector(config, threshold);
+```
+
+#### Methods
+
+- `detectPattern(errorMessage)` - Fast pattern-based detection
+- `detect(testResult)` - AI-powered detection
+- `batchDetect(testResults)` - Batch detection
+
+### DecisionEngine
+
+Threshold-based routing and actions.
+
+```typescript
+const engine = new DecisionEngine(config, thresholds);
+```
+
+#### Methods
+
+- `decide(probability)` - Decide action from probability
+- `decideWithThresholds(probabilities)` - Multi-question decisions
+- `route(decision, handlers)` - Route to handlers
+- `batchDecide(decisions)` - Batch decisions
+- `weightedVote(votes)` - Weighted voting
+
+## Pricing
+
+Jev via OpenRouter:
+- Input: $0.042 / 1M tokens
+- Output: Free
+- Latency: ~250ms P50
+
+See [OpenRouter pricing](https://openrouter.ai/typesafe/jev-1.13) for details.
+
+## Examples
+
+See `tests/` directory for example usage.
+
+## License
+
+MIT