ultimate-pi 0.18.0 → 0.19.0

package/.pi/lib/harness-lens/clients/path-utils.ts

@@ -0,0 +1,182 @@
+/**
+ * Path utilities for pi-lens
+ *
+ * Handles cross-platform path normalization, particularly
+ * Windows case-insensitivity issues when using paths as Map keys.
+ *
+ * Approach (inspired by OpenCode's Filesystem.normalizePath):
+ * - On Windows: try realpathSync.native() for canonical casing
+ * - Falls back to lowercase for files that don't exist yet
+ * - On non-Windows: return path as-is (case-sensitive filesystem)
+ * - Always convert backslashes to forward slashes for Map key consistency
+ */
+
+import { existsSync, realpathSync } from "node:fs";
+import { dirname, win32 } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+/**
+ * Detect if a path is a Windows path (has drive letter or UNC prefix).
+ */
+function isWindowsPath(filePath: string): boolean {
+	return /^[A-Za-z]:/.test(filePath) || filePath.startsWith("\\\\");
+}
+
+/**
+ * Normalize a file path for consistent Map key usage.
+ *
+ * On Windows:
+ * - If the file exists: uses realpathSync.native() to get the canonical
+ *   filesystem path (actual casing, resolved symlinks)
+ * - If the file doesn't exist: resolves the path and lowercases
+ *   (needed for new files where we haven't written yet)
+ *
+ * On non-Windows: returns path as-is (case-sensitive filesystem).
+ *
+ * Always converts backslashes to forward slashes for consistent Map keys.
+ */
+export function normalizeFilePath(filePath: string): string {
+	// Convert backslashes to forward slashes first
+	const normalized = filePath.replace(/\\/g, "/");
+
+	if (process.platform !== "win32" && !isWindowsPath(normalized)) {
+		return normalized;
+	}
+
+	// Windows: try realpathSync.native() for canonical casing
+	// This resolves symlinks and returns the actual filesystem casing
+	try {
+		const canonical = realpathSync.native(filePath);
+		return canonical.replace(/\\/g, "/");
+	} catch {
+		// File doesn't exist yet (new file) — resolve path and lowercase
+		// We need to walk up the directory tree to find the nearest existing
+		// parent, resolve its casing, then append the non-existent parts
+		try {
+			return resolveNonExisting(filePath);
+		} catch {
+			// Last resort: just lowercase the resolved path
+			const resolved = win32.normalize(win32.resolve(filePath));
+			return resolved.replace(/\\/g, "/").toLowerCase();
+		}
+	}
+}
+
+/**
+ * Resolve a non-existing path by finding the nearest existing parent,
+ * getting its canonical casing, then appending the non-existent parts lowercased.
+ *
+ * Example: C:\Users\Foo\newdir\file.ts
+ * - C:\Users\Foo exists → realpathSync gives C:\Users\Foo
+ * - newdir\file.ts doesn't exist → lowercased
+ * - Result: C:/Users/Foo/newdir/file.ts
+ */
+function resolveNonExisting(filePath: string): string {
+	const resolved = win32.resolve(filePath);
+	let current = resolved;
+	const nonExistentParts: string[] = [];
+
+	// Walk up until we find an existing directory
+	while (true) {
+		if (existsSync(current)) {
+			// Found existing ancestor — get its canonical casing
+			const canonical = realpathSync.native(current);
+			if (nonExistentParts.length === 0) {
+				return canonical.replace(/\\/g, "/");
+			}
+			// Append non-existent parts (lowercased for consistency)
+			const tail = nonExistentParts.reverse().join("/").toLowerCase();
+			const base = canonical.replace(/\\/g, "/");
+			return base.endsWith("/") ? base + tail : `${base}/${tail}`;
+		}
+
+		const parent = dirname(current);
+		if (parent === current) {
+			// Reached filesystem root without finding existing dir
+			// Fall back to full lowercase
+			throw new Error("No existing parent found");
+		}
+
+		nonExistentParts.push(win32.basename(current));
+		current = parent;
+	}
+}
+
+/**
+ * Convert a file:// URI to a normalized path.
+ * Handles URL decoding and Windows drive letter normalization.
+ */
+export function uriToPath(uri: string): string {
+	try {
+		const filePath = fileURLToPath(uri);
+		return normalizeFilePath(filePath);
+	} catch {
+		// Not a valid file:// URI, treat as plain path
+		return normalizeFilePath(uri);
+	}
+}
+
+/**
+ * Convert a path to a file:// URI.
+ * Does NOT normalize the path - URIs preserve original casing.
+ */
+export function pathToUri(filePath: string): string {
+	return pathToFileURL(filePath).href;
+}
+
+/**
+ * Normalize a Map key lookup for file paths.
+ * Use this when getting/setting values in Maps that use file paths as keys.
+ */
+export function normalizeMapKey(filePath: string): string {
+	return normalizeFilePath(filePath);
+}
+
+/**
+ * Compare two file paths for equality, handling Windows case-insensitivity
+ * and mixed separators (backslash vs forward slash).
+ */
+export function pathsEqual(a: string, b: string): boolean {
+	return normalizeFilePath(a) === normalizeFilePath(b);
+}
+
+/**
+ * Check if `child` is under `parent` directory.
+ * Separator-agnostic and case-insensitive on Windows.
+ */
+export function isUnderDir(child: string, parent: string): boolean {
+	const normChild = normalizeFilePath(child);
+	const normParent = normalizeFilePath(parent);
+	// Ensure parent ends with / for prefix matching
+	const parentPrefix = normParent.endsWith("/") ? normParent : `${normParent}/`;
+	return normChild === normParent || normChild.startsWith(parentPrefix);
+}
+
+const VENDOR_DIR_NAMES = new Set([
+	"node_modules",
+	"vendor",
+	"vendors",
+	"third_party",
+	"third-party",
+]);
+
+/**
+ * Returns true when a file should be treated as external/vendor and excluded
+ * from pipelines (LSP, diagnostics, complexity, etc.).
+ *
+ * Cases:
+ *   1. Outside the project root entirely (e.g. global npm packages, system files)
+ *   2. Inside the project but under a vendor directory (node_modules, vendor, third_party, etc.)
+ */
+export function isExternalOrVendorFile(
+	filePath: string,
+	projectRoot: string,
+): boolean {
+	if (!isUnderDir(filePath, projectRoot)) return true;
+	const normalized = normalizeFilePath(filePath);
+	const rootNorm = normalizeFilePath(projectRoot);
+	const rel = normalized.startsWith(rootNorm + "/")
+		? normalized.slice(rootNorm.length + 1)
+		: normalized;
+	return rel.split("/").some((seg) => VENDOR_DIR_NAMES.has(seg));
+}

package/.pi/lib/harness-lens/clients/pipeline.ts

@@ -0,0 +1,360 @@
+/**
+ * Post-write pipeline for harness lens.
+ *
+ * The ultimate-pi harness keeps lens focused on edit safety, formatting, LSP sync,
+ * and secret blocking.
+ */
+
+import * as nodeFs from "node:fs";
+import * as path from "node:path";
+import { detectFileKind, getFileKindLabel } from "./file-kinds.js";
+import type { FormatService } from "./format-service.js";
+import { logLatency } from "./latency-logger.js";
+import { emitLensAnalysisComplete } from "./lens-events.js";
+import { getLSPService } from "./lsp/index.js";
+import { RUNTIME_CONFIG } from "./runtime-config.js";
+import { formatSecrets, scanForSecrets } from "./secrets-scanner.js";
+
+const LSP_MAX_FILE_BYTES = RUNTIME_CONFIG.pipeline.lspMaxFileBytes;
+const LSP_MAX_FILE_LINES = RUNTIME_CONFIG.pipeline.lspMaxFileLines;
+const LSP_SPAWN_BUDGET_MS = RUNTIME_CONFIG.pipeline.lspSpawnBudgetMs;
+
+export interface PipelineContext {
+	filePath: string;
+	cwd: string;
+	toolName: string;
+	modifiedRanges?: { start: number; end: number }[];
+	telemetry?: {
+		model: string;
+		sessionId: string;
+		turnIndex: number;
+		writeIndex: number;
+	};
+	getFlag: (name: string) => boolean | string | undefined;
+	dbg: (msg: string) => void;
+}
+
+export interface PipelineDeps {
+	getFormatService: () => FormatService;
+}
+
+export interface PipelineResult {
+	output: string;
+	hasBlockers: boolean;
+	isError: boolean;
+	fileModified: boolean;
+	changedFiles?: string[];
+}
+
+interface PhaseTracker {
+	start(name: string): void;
+	end(name: string, metadata?: Record<string, unknown>): void;
+}
+
+type SecretDiagnostic = {
+	id: string;
+	message: string;
+	filePath: string;
+	line: number;
+	column: number;
+	severity: "error";
+	semantic: "blocking";
+	tool: "secrets-scanner";
+	rule: "secrets";
+	defectClass: "secrets";
+};
+
+function createPhaseTracker(toolName: string, filePath: string): PhaseTracker {
+	const phases: Array<{ name: string; startTime: number; ended: boolean }> = [];
+	return {
+		start(name: string) {
+			phases.push({ name, startTime: Date.now(), ended: false });
+		},
+		end(name: string, metadata?: Record<string, unknown>) {
+			const phase = phases.find((item) => item.name === name && !item.ended);
+			if (!phase) return;
+			phase.ended = true;
+			logLatency({
+				type: "phase",
+				toolName,
+				filePath,
+				phase: name,
+				durationMs: Date.now() - phase.startTime,
+				metadata,
+			});
+		},
+	};
+}
+
+function exceedsLspSyncLimits(content: string): {
+	tooLarge: boolean;
+	reason: string;
+} {
+	const sizeBytes = Buffer.byteLength(content, "utf-8");
+	if (sizeBytes > LSP_MAX_FILE_BYTES) {
+		return {
+			tooLarge: true,
+			reason: `${Math.round(sizeBytes / 1024)}KB exceeds ${Math.round(LSP_MAX_FILE_BYTES / 1024)}KB`,
+		};
+	}
+
+	const lineCount = content.split("\n").length;
+	if (lineCount > LSP_MAX_FILE_LINES) {
+		return {
+			tooLarge: true,
+			reason: `${lineCount} lines exceeds ${LSP_MAX_FILE_LINES}`,
+		};
+	}
+
+	return { tooLarge: false, reason: "" };
+}
+
+function displayPath(cwd: string, filePath: string): string {
+	const relative = path.relative(cwd, filePath);
+	return relative && !relative.startsWith("..") && !path.isAbsolute(relative)
+		? relative.replace(/\\/g, "/")
+		: filePath.replace(/\\/g, "/");
+}
+
+function buildAllClearOutput(elapsed: number, filePath: string): string {
+	const kind = detectFileKind(filePath);
+	const langLabel = kind ? getFileKindLabel(kind) : path.extname(filePath);
+	const parts = kind
+		? [`${langLabel} clean`, `${elapsed}ms`]
+		: [`${elapsed}ms`];
+	return `✓ ${parts.join(" · ")}`;
+}
+
+export interface FormatPhaseResult {
+	formatChanged: boolean;
+	formattersUsed: string[];
+	formatFailures: string[];
+	fileContent: string | undefined;
+}
+
+export async function runFormatPhase(
+	filePath: string,
+	getFormatService: () => FormatService,
+	dbg: PipelineContext["dbg"],
+): Promise<FormatPhaseResult> {
+	let formatChanged = false;
+	let formattersUsed: string[] = [];
+	const formatFailures: string[] = [];
+	let fileContent: string | undefined;
+
+	const formatService = getFormatService();
+	try {
+		formatService.recordRead(filePath);
+		const result = await formatService.formatFile(filePath);
+		formattersUsed = result.formatters.map((formatter) => formatter.name);
+		if (result.anyChanged) {
+			formatChanged = true;
+			dbg(
+				"autoformat: " +
+					result.formatters
+						.map(
+							(formatter) =>
+								`${formatter.name}(${formatter.changed ? "changed" : "unchanged"})`,
+						)
+						.join(", "),
+			);
+		}
+		if (!result.allSucceeded) {
+			const failures = result.formatters.filter(
+				(formatter) => !formatter.success,
+			);
+			formatFailures.push(
+				...failures.map(
+					(formatter) =>
+						`${formatter.name}: ${formatter.error ?? "unknown error"}`,
+				),
+			);
+			dbg(
+				"autoformat: " +
+					failures
+						.map(
+							(formatter) =>
+								`${formatter.name} failed: ${formatter.error ?? "unknown error"}`,
+						)
+						.join("; "),
+			);
+		}
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		formatFailures.push(message);
+		dbg(`autoformat error: ${err}`);
+	}
+
+	try {
+		fileContent = nodeFs.readFileSync(filePath, "utf-8");
+	} catch {
+		fileContent = undefined;
+	}
+
+	return { formatChanged, formattersUsed, formatFailures, fileContent };
+}
+
+export async function resyncLspFile(
+	filePath: string,
+	fileContent: string,
+	needsContentRefresh: boolean,
+	lspSyncCompleted: boolean,
+	getFlag: PipelineContext["getFlag"],
+	dbg: PipelineContext["dbg"],
+	formatChanged = false,
+): Promise<void> {
+	if (getFlag("no-lsp")) return;
+	if (!needsContentRefresh && lspSyncCompleted) return;
+	if (exceedsLspSyncLimits(fileContent).tooLarge) return;
+
+	try {
+		const lspService = getLSPService();
+		if (!lspService.supportsLSP(filePath)) return;
+		await lspService.openFile(filePath, fileContent, {
+			preserveDiagnostics: formatChanged,
+			spawnBudgetMs: LSP_SPAWN_BUDGET_MS,
+		});
+	} catch (err) {
+		dbg(`LSP resync error: ${err}`);
+	}
+}
+
+export async function runPipeline(
+	ctx: PipelineContext,
+	deps: PipelineDeps,
+): Promise<PipelineResult> {
+	const { filePath, cwd, toolName, getFlag, dbg } = ctx;
+	const { getFormatService } = deps;
+	const phase = createPhaseTracker(toolName, filePath);
+	const pipelineStart = Date.now();
+	phase.start("total");
+
+	phase.start("read_file");
+	let fileContent: string | undefined;
+	try {
+		fileContent = nodeFs.readFileSync(filePath, "utf-8");
+	} catch {
+		fileContent = undefined;
+	}
+	phase.end("read_file");
+
+	if (fileContent) {
+		const secretFindings = scanForSecrets(fileContent, filePath);
+		if (secretFindings.length > 0) {
+			const durationMs = Date.now() - pipelineStart;
+			logLatency({
+				type: "tool_result",
+				toolName,
+				filePath,
+				durationMs,
+				result: "blocked_secrets",
+				metadata: { secretsFound: secretFindings.length },
+			});
+			const secretDiagnostics: SecretDiagnostic[] = secretFindings.map(
+				(finding) => ({
+					id: `secrets:${finding.line}`,
+					message: finding.message,
+					filePath,
+					line: finding.line,
+					column: 1,
+					severity: "error",
+					semantic: "blocking",
+					tool: "secrets-scanner",
+					rule: "secrets",
+					defectClass: "secrets",
+				}),
+			);
+			emitLensAnalysisComplete({
+				cwd,
+				filePath,
+				toolName,
+				model: ctx.telemetry?.model ?? "unknown",
+				sessionId: ctx.telemetry?.sessionId ?? "unknown",
+				turnIndex: ctx.telemetry?.turnIndex ?? 0,
+				writeIndex: ctx.telemetry?.writeIndex ?? 0,
+				diagnostics: secretDiagnostics,
+				blockers: secretDiagnostics,
+				warnings: [],
+				fixed: [],
+				resolvedCount: 0,
+				hasBlockers: true,
+				fileModified: false,
+				changedFiles: [],
+				durationMs,
+			});
+			return {
+				output: `\n\n${formatSecrets(secretFindings, filePath)}`,
+				hasBlockers: true,
+				isError: true,
+				fileModified: false,
+				changedFiles: [],
+			};
+		}
+	}
+
+	phase.start("format");
+	let formatChanged = false;
+	let formatFailures: string[] = [];
+	const changedFiles = new Set<string>();
+	const autoformatDisabled = !!getFlag("no-autoformat");
+	const immediateFormat = !!getFlag("immediate-format");
+	const formatDeferred =
+		!autoformatDisabled && !immediateFormat && !!fileContent;
+	if (!autoformatDisabled && immediateFormat && fileContent) {
+		const formatResult = await runFormatPhase(filePath, getFormatService, dbg);
+		formatChanged = formatResult.formatChanged;
+		formatFailures = formatResult.formatFailures;
+		fileContent = formatResult.fileContent;
+		if (formatChanged) changedFiles.add(path.resolve(filePath));
+	} else if (formatDeferred) {
+		dbg(`autoformat: deferred until agent_end for ${filePath}`);
+	}
+	phase.end("format", { formatChanged, deferred: formatDeferred });
+
+	phase.start("lsp_sync");
+	let lspSyncCompleted = false;
+	if (fileContent) {
+		await resyncLspFile(
+			filePath,
+			fileContent,
+			true,
+			false,
+			getFlag,
+			dbg,
+			formatChanged,
+		);
+		lspSyncCompleted = true;
+	}
+	phase.end("lsp_sync", { completed: lspSyncCompleted, finalContent: true });
+
+	let output = "";
+	if (formatFailures.length > 0) {
+		const details = formatFailures.slice(0, 3).join("; ");
+		const suffix =
+			formatFailures.length > 3
+				? `; ... and ${formatFailures.length - 3} more`
+				: "";
+		output += `\n\n⚠️ Auto-format failed: ${details}${suffix}`;
+	}
+	if (formatChanged) {
+		const changedList = [...changedFiles].map((changedFile) =>
+			displayPath(cwd, changedFile),
+		);
+		const fileList = changedList.length
+			? `\nModified files:\n${changedList.map((file) => `  - ${file}`).join("\n")}`
+			: "";
+		output += `\n\n⚠️ **File was modified by auto-format. You MUST re-read modified file(s) before making any further edits — the content on disk has changed.**${fileList}`;
+	}
+
+	const elapsed = Date.now() - pipelineStart;
+	if (!output) output = buildAllClearOutput(elapsed, filePath);
+	phase.end("total", { hasOutput: !!output });
+
+	return {
+		output,
+		hasBlockers: false,
+		isError: false,
+		fileModified: formatChanged,
+		changedFiles: [...changedFiles],
+	};
+}

package/.pi/lib/harness-lens/clients/project-profile.ts

@@ -0,0 +1,117 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { detectFileKind, type FileKind } from "./file-kinds.js";
+import { isPathIgnoredByProject } from "./file-utils.js";
+
+export interface ProjectProfile {
+	present: Partial<Record<FileKind, boolean>>;
+	counts: Partial<Record<FileKind, number>>;
+	detectedKinds: FileKind[];
+}
+
+const MARKERS: Partial<Record<FileKind, string[]>> = {
+	go: ["go.mod"],
+	python: ["pyproject.toml", "requirements.txt", "setup.py"],
+	rust: ["Cargo.toml"],
+	jsts: ["package.json", "tsconfig.json"],
+	java: ["pom.xml", "build.gradle", "build.gradle.kts"],
+	ruby: ["Gemfile"],
+	php: ["composer.json"],
+};
+
+const LSP_DEFAULTS: Partial<Record<FileKind, string>> = {
+	jsts: "typescript-language-server",
+	python: "pyright",
+	rust: "rust-analyzer",
+	java: "jdtls",
+	ruby: "solargraph",
+	php: "intelephense",
+};
+
+const MAX_WALK_FILES = 4000;
+const MAX_LSP_PREINSTALL = 3;
+
+function countKind(
+	kind: FileKind,
+	counts: Partial<Record<FileKind, number>>,
+): void {
+	counts[kind] = (counts[kind] ?? 0) + 1;
+}
+
+function walkProject(root: string): Partial<Record<FileKind, number>> {
+	const counts: Partial<Record<FileKind, number>> = {};
+	let scanned = 0;
+	const queue = [root];
+
+	while (queue.length > 0 && scanned < MAX_WALK_FILES) {
+		const dir = queue.pop();
+		if (!dir) break;
+		let entries: fs.Dirent[];
+		try {
+			entries = fs.readdirSync(dir, { withFileTypes: true });
+		} catch {
+			continue;
+		}
+		for (const entry of entries) {
+			if (scanned >= MAX_WALK_FILES) break;
+			const full = path.join(dir, entry.name);
+			if (entry.isDirectory()) {
+				if (entry.name === "node_modules" || entry.name === ".git") continue;
+				if (isPathIgnoredByProject(full, root, true)) continue;
+				queue.push(full);
+				continue;
+			}
+			if (!entry.isFile()) continue;
+			if (isPathIgnoredByProject(full, root, false)) continue;
+			scanned += 1;
+			const kind = detectFileKind(full);
+			if (kind) countKind(kind, counts);
+		}
+	}
+
+	return counts;
+}
+
+function markerPresent(root: string, kind: FileKind): boolean {
+	for (const marker of MARKERS[kind] ?? []) {
+		if (fs.existsSync(path.join(root, marker))) return true;
+	}
+	return false;
+}
+
+export function detectProjectProfile(root: string): ProjectProfile {
+	const resolved = path.resolve(root);
+	const counts = walkProject(resolved);
+	const present: Partial<Record<FileKind, boolean>> = {};
+
+	for (const [kind, count] of Object.entries(counts)) {
+		if ((count ?? 0) > 0) present[kind as FileKind] = true;
+	}
+
+	for (const kind of Object.keys(MARKERS) as FileKind[]) {
+		if (markerPresent(resolved, kind)) present[kind] = true;
+	}
+
+	const detectedKinds = (Object.keys(present) as FileKind[])
+		.filter((kind) => present[kind])
+		.sort((a, b) => (counts[b] ?? 0) - (counts[a] ?? 0));
+
+	return { present, counts, detectedKinds };
+}
+
+export function lspPreinstallTools(profile: ProjectProfile): string[] {
+	const tools: string[] = [];
+	for (const kind of profile.detectedKinds) {
+		const tool = LSP_DEFAULTS[kind];
+		if (tool && !tools.includes(tool)) tools.push(tool);
+		if (tools.length >= MAX_LSP_PREINSTALL) break;
+	}
+	return tools;
+}
+
+export function resolveProjectRootForFile(
+	filePath: string,
+	fallbackRoot: string,
+): string {
+	return path.resolve(fallbackRoot);
+}