pi-lens 3.6.6 → 3.7.0

package/CHANGELOG.md

@@ -2,6 +2,116 @@
 
 All notable changes to pi-lens will be documented in this file.
 
+## [3.7.0] - 2026-04-05
+
+### Added
+- **Test runner in pipeline** — After every file write/edit, pi-lens now automatically detects and
+  runs the corresponding test file (vitest, jest, pytest). Results surface inline so the agent sees
+  failures immediately without a separate test step. Supports TypeScript/JS/Python; file-level
+  targeted — only the test for the edited file runs, not the full suite.
+
+- **Parallel dispatch groups** — Lint runners now execute in parallel across independent groups
+  (e.g. `lsp`, `tree-sitter`, `ast-grep-napi`, `type-safety`, `similarity` all fire at once).
+  Typical wall-clock savings: 500–1500ms per file write (`parallelGainMs` logged in latency log).
+
+### Fixed
+- **`semantic: "none"` when 0 diagnostics** — LSP, Pyright, and type-safety runners were returning
+  `semantic: "warning"` even when `diagnosticCount` was 0 (clean file). Now correctly returns
+  `"none"` when no diagnostics are present, `"warning"` when warnings exist, `"blocking"` on errors.
+
+- **`ast_grep_replace` with `apply=true` not writing files** — Replaced tool was silently
+  discarding the rewritten content instead of persisting it to disk.
+
+- **Pipeline event loop blocked during test execution** — `spawnSync` in the test runner was
+  blocking the Node.js event loop for the duration of the test run. Switched to async spawn.
+
+- **Formatters: venv/vendor/node_modules awareness** — Formatters now skip files inside virtual
+  environments, vendor directories, and `node_modules` instead of attempting to format them.
+  CSharpier detection also improved.
+
+- **Formatter nearest-wins resolution** — When multiple formatter configs exist at different
+  directory levels, the one closest to the edited file is now used (was previously using the
+  root-level config regardless of nesting).
+
+- **Prettier auto-install** — Prettier is now auto-installed when detected as the project
+  formatter but not present, consistent with the Biome/Ruff auto-install behaviour.
+
+- **6 missing formatters added** — `clang-format` (C/C++/ObjC), `ktlint` (Kotlin), `scalafmt`
+  (Scala), `mix format` (Elixir), `dart format` (Dart), `terraform fmt` (HCL) now detected
+  and invoked automatically.
+
+- **LSP tier-4 install prompts** — Corrected missing interactive-install prompts for tier-4
+  language servers (less common languages). Users now see the install suggestion instead of a
+  silent skip.
+
+### Changed
+- **`startedAt` added to latency log runner entries** — Every runner entry now records when it
+  started, making wall-clock vs. sequential comparisons accurate. `dispatch_complete` also logs
+  `parallelGainMs = sumMs - wallClockMs` to quantify parallelism benefit.
+
+- **Dynamic imports removed from hot path** — Dispatch module no longer uses `await import()`
+  for runner loading; all imports are static, eliminating ~50ms warm-up latency on first dispatch.
+
+### Tests
+- Added formatter venv/vendor resolution and interactive-install coverage
+- Added LSP lifecycle test suite with mock LSP server (process spawn, open/change/close, shutdown)
+
+---
+
+## [3.6.7] - 2026-04-04
+
+### Fixed
+- **LSP `ERR_STREAM_DESTROYED` crash** — When an LSP process (e.g. rust-analyzer) exits, Node.js emits
+  `'error'` events on the destroyed stdio streams. Without listeners these became uncaught exceptions
+  that crashed the extension. Added persistent `error` listeners to `stdin`, `stdout`, and `stderr`
+  before handing them to `vscode-jsonrpc`, covering the post-`connection.dispose()` window.
+  Same guard added to `NativeRustCoreClient` stdin writes.
+
+### Added
+- **Rust performance core (`pi-lens-core`)** — Optional Rust binary for CPU-intensive operations.
+  All features fall back to TypeScript automatically if the binary is not available (it is **not**
+  built automatically on `npm install` — run `npm run rust:build` once if you have Rust installed).
+  - **File scanning** — ripgrep’s `ignore` crate for `.gitignore`-aware project scanning
+  - **Similarity detection** — parallel 57×72 state-matrix index, persisted to
+    `.pi-lens/rust-index.json` between invocations (fixes in-memory cache that reset on every
+    process spawn)
+  - **Tree-sitter queries** — TypeScript and Rust AST queries via the binary
+  - **`NativeRustCoreClient`** — TypeScript wrapper with `isBinaryStale()` freshness detection,
+    JSON-IPC over stdin/stdout
+  - **Integration tests** — `npm run rust:test:integration` (37 assertions across all commands)
+
+- **Rust similarity fast-path in dispatch runner** — `similarity.ts` now tries the Rust binary
+  first (scan → build index → query), falls through to the TypeScript implementation on any
+  failure. Feature flag `USE_RUST = true` at top of file.
+
+### Changed
+- **Similarity threshold raised from 0.75 → 0.90** — Empirical evaluation showed that below 0.90
+  false positives (structurally similar but semantically unrelated functions) outnumber true
+  positives with the current 57×72 matrix resolution. Applies to both the dispatch runner and
+  `/lens-booboo`.
+
+- **Rust `kind_id` mapping improved** — Replaced `kind % dim` modulo (caused up to 4 unrelated
+  node types to share one matrix slot) with even-distribution across named slots plus a dedicated
+  last slot for anonymous punctuation tokens. Max named-slot collisions reduced from 4 to 3;
+  unnamed tokens no longer pollute named slots.
+
+### Fixed (Rust)
+- `tree_sitter_rust::language_rust()` → `language()` (correct API for tree-sitter-rust 0.21)
+- `FunctionInfo` missing `#[derive(Clone)]` — caused compile error in `find_similar_to`
+- `export function foo()` was missed by the index builder — TypeScript wraps exported functions
+  in `export_statement`; replaced flat top-level walk with recursive `collect_functions()`
+- `find_similar_to` returned only the first function in a file — changed `find` to `filter`
+- `tempfile` moved from `[dependencies]` to `[dev-dependencies]`
+- Deleted orphan `test_lsp.rs` (intentional type errors caused rust-analyzer to crash the LSP stream)
+
+### Repository
+- Rust source (`rust/src/`, `rust/Cargo.toml`) added to npm `files` whitelist so users can build
+  the binary from an npm-installed package
+- Removed stale `src/main.rs` rule from root `.gitignore` (no such file at repo root)
+- Untracked `docs/plans/2025-04-03-auto-install-logging.md` (committed before `*.md` exclusion rule)
+
+---
+
 ## [3.6.3] - 2026-04-03
 
 ### Removed (Dead Code Cleanup)

package/README.md

@@ -196,7 +196,7 @@ pi-lens uses a **dispatcher-runner architecture** for extensible multi-language
 | **shellcheck** | Shell | 20 | Warning | Bash/sh/zsh/fish linting |
 | **python-slop** | Python | 25 | Warning | AI slop detection (~40 patterns) |
 | **spellcheck** | Markdown | 30 | Warning | Typo detection in docs |
-| **similarity** | TS | 35 | Warning | Semantic duplicate detection (structural similarity) |
+| **similarity** | TS | 35 | Warning | Semantic duplicate detection (≥90% structural similarity, Rust-accelerated when available) |
 | **architect** | All | 40 | Warning | Architectural rule violations |
 | **go-vet** | Go | 50 | Warning | Go static analysis |
 | **rust-clippy** | Rust | 50 | Warning | Rust linting |
@@ -327,7 +327,7 @@ Full codebase analysis with **10 tracked runners** producing a comprehensive rep
 |---|--------|---------------|
 | 1 | **ast-grep (design smells)** | Structural issues (empty catch, no-debugger, etc.) |
 | 2 | **ast-grep (similar functions)** | Duplicate function patterns across files |
-| 3 | **semantic similarity (Amain)** | 57×72 matrix semantic clones (>75% similarity) |
+| 3 | **semantic similarity (Amain)** | 57×72 matrix semantic clones (≥90% similarity) |
 | 4 | **complexity metrics** | Low MI, high cognitive complexity, AI slop indicators |
 | 5 | **TODO scanner** | TODO/FIXME annotations and tech debt markers |
 | 6 | **dead code (Knip)** | Unused exports, files, dependencies |
@@ -501,6 +501,9 @@ pi-lens/
 ├── commands/         # /lens-booboo, /lens-format commands
 ├── docs/             # Documentation
 ├── rules/            # AST-grep rules
+├── rust/             # Optional Rust core for performance acceleration
+│   ├── src/          # Rust source (pi-lens-core binary)
+│   └── Cargo.toml
 ├── skills/           # Built-in pi skills
 ├── index.ts          # Main extension entry point
 └── package.json
@@ -510,6 +513,40 @@ See source for detailed structure.
 
 ---
 
+## Rust Core (Optional)
+
+pi-lens includes a **Rust performance core** (`pi-lens-core`) for CPU-intensive operations. It is entirely optional — all features fall back to the TypeScript implementation automatically if the binary is not available.
+
+**What it accelerates:**
+- **File scanning** — Uses ripgrep's `ignore` crate for fast, `.gitignore`-aware project scanning (~10× faster than glob)
+- **Similarity detection** — Parallel 57×72 state-matrix computation and index querying
+- **Tree-sitter queries** — Runs TypeScript and Rust AST queries directly from the binary
+
+**Status:** Does not work out of the box after `npm install`. The source is included in the package so you can build it yourself if you have Rust installed.
+
+**Build the binary (one-time):**
+```bash
+# Requires Rust toolchain — https://rustup.rs
+npm run rust:build          # release build (recommended)
+npm run rust:build:debug    # debug build
+```
+
+Once built, pi-lens will automatically use the Rust binary and fall back to TypeScript if it is absent, outdated, or fails.
+
+**Verify the binary is being used:**
+```bash
+node -e "import('./clients/native-rust-client.js').then(m => console.log('available:', m.getNativeRustCoreClient(true).isAvailable()))"
+```
+
+**Run integration tests** (requires debug binary):
+```bash
+npm run rust:build:debug
+npm run rust:test:integration   # 37 assertions
+npm run rust:test               # Rust unit tests
+```
+
+---
+
 ## Skills
 
 pi-lens includes two built-in skills that guide the LLM on when to use specific tools:

package/clients/ast-grep-client.ts

@@ -96,21 +96,43 @@ export class AstGrepClient {
 		paths: string[],
 		apply = false,
 	): Promise<{ matches: AstGrepMatch[]; applied: boolean; error?: string }> {
-		const args = [
+		const baseArgs = ["run", "-p", pattern, "-r", rewrite, "--lang", lang];
+
+		if (!apply) {
+			// Dry-run: --json=compact shows what would change without writing
+			const result = await this.runner.exec([
+				...baseArgs,
+				"--json=compact",
+				...paths,
+			]);
+			return { matches: result.matches, applied: false, error: result.error };
+		}
+
+		// Apply: --update-all and --json are MUTUALLY EXCLUSIVE in sg.
+		// Run twice:
+		//   1. --update-all to actually write the files
+		//   2. --json=compact (without rewrite) to collect matches for display
+		const applyResult = await this.runner.exec([
+			...baseArgs,
+			"--update-all",
+			...paths,
+		]);
+		if (applyResult.error) {
+			return { matches: [], applied: false, error: applyResult.error };
+		}
+
+		// Search for what was changed (pattern no longer matches after rewrite,
+		// so search for the rewrite pattern to show what was applied)
+		const searchResult = await this.runner.exec([
 			"run",
 			"-p",
-			pattern,
-			"-r",
 			rewrite,
 			"--lang",
 			lang,
 			"--json=compact",
-		];
-		if (apply) args.push("--update-all");
-		args.push(...paths);
-
-		const result = await this.runner.exec(args);
-		return { matches: result.matches, applied: apply, error: result.error };
+			...paths,
+		]);
+		return { matches: searchResult.matches, applied: true, error: undefined };
 	}
 
 	/**

package/clients/dispatch/dispatcher.ts

@@ -254,6 +254,135 @@ export function formatLatencyReport(report: DispatchLatencyReport): string {
 	return lines.join("\n");
 }
 
+// --- Group runner (used by dispatchForFile for parallel execution) ---
+
+interface GroupResult {
+	diagnostics: Diagnostic[];
+	latencies: RunnerLatency[];
+	hadBlocker: boolean;
+}
+
+/**
+ * Execute all runners in a single group.
+ *
+ * - mode "fallback": run runners sequentially and stop at the first
+ *   one that succeeds (returns status !== "skipped").
+ * - mode "all" (default): run all runners in the group sequentially
+ *   and collect every diagnostic.
+ *
+ * Groups themselves are run in parallel by dispatchForFile, so this
+ * function must NOT mutate shared state.
+ */
+async function runGroup(
+	ctx: DispatchContext,
+	group: RunnerGroup,
+): Promise<GroupResult> {
+	const diagnostics: Diagnostic[] = [];
+	const latencies: RunnerLatency[] = [];
+	let hadBlocker = false;
+
+	// Filter runners by kind if specified
+	const runnerIds = group.filterKinds
+		? group.runnerIds.filter((id) => {
+				const runner = getRunner(id);
+				return runner && ctx.kind && group.filterKinds?.includes(ctx.kind);
+			})
+		: group.runnerIds;
+
+	const semantic = group.semantic ?? "warning";
+
+	for (const runnerId of runnerIds) {
+		const runnerStart = Date.now();
+		const runner = getRunner(runnerId);
+
+		if (!runner) {
+			latencies.push({
+				runnerId,
+				startTime: runnerStart,
+				endTime: Date.now(),
+				durationMs: 0,
+				status: "skipped",
+				diagnosticCount: 0,
+				semantic: "unknown",
+			});
+			logLatency({
+				type: "runner",
+				filePath: ctx.filePath,
+				runnerId,
+				durationMs: 0,
+				status: "not_registered",
+				diagnosticCount: 0,
+				semantic: "unknown",
+			});
+			continue;
+		}
+
+		// Check preconditions
+		if (runner.when && !(await runner.when(ctx))) {
+			latencies.push({
+				runnerId,
+				startTime: runnerStart,
+				endTime: Date.now(),
+				durationMs: Date.now() - runnerStart,
+				status: "when_skipped",
+				diagnosticCount: 0,
+				semantic: runner.id,
+			});
+			logLatency({
+				type: "runner",
+				filePath: ctx.filePath,
+				runnerId,
+				durationMs: 0,
+				status: "when_skipped",
+				diagnosticCount: 0,
+				semantic: "when_condition",
+			});
+			continue;
+		}
+
+		const result = await runRunner(ctx, runner, semantic);
+		const runnerEnd = Date.now();
+		const duration = runnerEnd - runnerStart;
+
+		latencies.push({
+			runnerId,
+			startTime: runnerStart,
+			endTime: runnerEnd,
+			durationMs: duration,
+			status: result.status,
+			diagnosticCount: result.diagnostics.length,
+			semantic: result.semantic ?? semantic,
+		});
+		logLatency({
+			type: "runner",
+			filePath: ctx.filePath,
+			runnerId,
+			startedAt: new Date(runnerStart).toISOString(),
+			durationMs: duration,
+			status: result.status,
+			diagnosticCount: result.diagnostics.length,
+			semantic: result.semantic ?? semantic,
+		});
+
+		diagnostics.push(...result.diagnostics);
+
+		const resultSemantic = result.semantic ?? semantic;
+		if (
+			(resultSemantic === "blocking" && result.diagnostics.length > 0) ||
+			result.diagnostics.some((d) => d.semantic === "blocking")
+		) {
+			hadBlocker = true;
+		}
+
+		// mode:"fallback" — stop at first runner that produced results
+		if (group.mode === "fallback" && result.status !== "skipped") {
+			break;
+		}
+	}
+
+	return { diagnostics, latencies, hadBlocker };
+}
+
 // --- Main Dispatch Function ---
 
 export async function dispatchForFile(
@@ -280,124 +409,38 @@ export async function dispatchForFile(
 		},
 	});
 
-	for (const group of groups) {
-		if (stopped && ctx.pi.getFlag("stop-on-error")) {
-			break;
-		}
-
-		// Filter runners by kind if specified
-		const runnerIds = group.filterKinds
-			? group.runnerIds.filter((id) => {
-					const runner = getRunner(id);
-					return runner && ctx.kind && group.filterKinds?.includes(ctx.kind);
-				})
-			: group.runnerIds;
-
-		const semantic = group.semantic ?? "warning";
-
-		for (const runnerId of runnerIds) {
-			const runnerStart = Date.now();
-			const runner = getRunner(runnerId);
-			if (!runner) {
-				runnerLatencies.push({
-					runnerId,
-					startTime: runnerStart,
-					endTime: Date.now(),
-					durationMs: 0,
-					status: "skipped",
-					diagnosticCount: 0,
-					semantic: "unknown",
-				});
-				logLatency({
-					type: "runner",
-					filePath: ctx.filePath,
-					runnerId,
-					durationMs: 0,
-					status: "not_registered",
-					diagnosticCount: 0,
-					semantic: "unknown",
-				});
-				continue;
-			}
-
-			// Check preconditions
-			if (runner.when && !(await runner.when(ctx))) {
-				runnerLatencies.push({
-					runnerId,
-					startTime: runnerStart,
-					endTime: Date.now(),
-					durationMs: Date.now() - runnerStart,
-					status: "when_skipped",
-					diagnosticCount: 0,
-					semantic: runner.id,
-				});
-				logLatency({
-					type: "runner",
-					filePath: ctx.filePath,
-					runnerId,
-					durationMs: 0,
-					status: "when_skipped",
-					diagnosticCount: 0,
-					semantic: "when_condition",
-				});
-				continue;
-			}
-
-			const result = await runRunner(ctx, runner, semantic);
-			const runnerEnd = Date.now();
-			const duration = runnerEnd - runnerStart;
-
-			// Track latency for this runner
-			runnerLatencies.push({
-				runnerId,
-				startTime: runnerStart,
-				endTime: runnerEnd,
-				durationMs: duration,
-				status: result.status,
-				diagnosticCount: result.diagnostics.length,
-				semantic: result.semantic ?? semantic,
-			});
-
-			// IMMEDIATE LOG: Each runner result (for debugging)
-			logLatency({
-				type: "runner",
-				filePath: ctx.filePath,
-				runnerId,
-				durationMs: duration,
-				status: result.status,
-				diagnosticCount: result.diagnostics.length,
-				semantic: result.semantic ?? semantic,
-			});
-
-			// Apply delta mode filtering
-			let diagnostics = result.diagnostics;
-			if (ctx.deltaMode && result.semantic !== "silent") {
-				const before = ctx.baselines.get(ctx.filePath);
-				if (before) {
-					const filtered = filterDelta(
-						diagnostics,
-						before as Diagnostic[],
-						(d) => d.id,
-					);
-					diagnostics = filtered.new;
-					// TODO: Track fixed diagnostics
-				}
-				// Update baseline
-				ctx.baselines.set(ctx.filePath, [...allDiagnostics, ...diagnostics]);
-			}
-
-			allDiagnostics.push(...diagnostics);
+	// Run all groups in parallel — they are independent and don't depend on
+	// each other's results. Within each group, mode:"fallback" semantics are
+	// preserved (sequential first-success). Results are merged in original
+	// group order so output is deterministic.
+	const groupResults = await Promise.all(
+		groups.map((group) => runGroup(ctx, group)),
+	);
 
-			// Check for blockers - use result semantic (not group default) and check individual diagnostics
-			const resultSemantic = result.semantic ?? semantic;
-			if (resultSemantic === "blocking" && diagnostics.length > 0) {
-				stopped = true;
-			}
-			// Also check if any individual diagnostic is blocking
-			if (diagnostics.some((d) => d.semantic === "blocking")) {
-				stopped = true;
+	for (const {
+		diagnostics: groupDiags,
+		latencies,
+		hadBlocker,
+	} of groupResults) {
+		runnerLatencies.push(...latencies);
+
+		// Apply delta mode filtering across the accumulated set
+		let diagnostics = groupDiags;
+		if (ctx.deltaMode) {
+			const before = ctx.baselines.get(ctx.filePath);
+			if (before) {
+				const filtered = filterDelta(
+					diagnostics,
+					before as Diagnostic[],
+					(d) => d.id,
+				);
+				diagnostics = filtered.new;
 			}
+			ctx.baselines.set(ctx.filePath, [...allDiagnostics, ...diagnostics]);
 		}
+
+		allDiagnostics.push(...diagnostics);
+		if (hadBlocker) stopped = true;
 	}
 
 	// Categorize results
@@ -440,14 +483,20 @@ export async function dispatchForFile(
 	// No need to log again here - would create duplicates in the log
 
 	// Log summary to latency log only (not console - avoid noise)
+	const sumMs = runnerLatencies.reduce((s, r) => s + r.durationMs, 0);
+	const wallClockMs = latencyReport.totalDurationMs;
 	logLatency({
 		type: "tool_result",
 		filePath: ctx.filePath,
-		durationMs: latencyReport.totalDurationMs,
+		durationMs: wallClockMs,
+		wallClockMs,
+		sumMs,
+		parallelGainMs: Math.max(0, sumMs - wallClockMs),
 		result: "dispatch_complete",
 		metadata: {
 			runners: runnerLatencies.map((r) => ({
 				id: r.runnerId,
+				startedAt: new Date(r.startTime).toISOString(),
 				duration: r.durationMs,
 				status: r.status,
 			})),

package/clients/dispatch/integration.ts

@@ -11,10 +11,13 @@ import {
 	createBaselineStore,
 	createDispatchContext,
 	type DispatchLatencyReport,
+	dispatchForFile,
 	formatLatencyReport,
 	getLatencyReports,
+	getRunnersForKind,
 	type RunnerLatency,
 } from "./dispatcher.js";
+import { TOOL_PLANS } from "./plan.js";
 import type { BaselineStore, DispatchResult, PiAgentAPI } from "./types.js";
 
 export type { DispatchLatencyReport, RunnerLatency };
@@ -57,11 +60,6 @@ export async function dispatchLint(
 	// pre-existing issues after the first write.
 	const ctx = createDispatchContext(filePath, cwd, pi, sessionBaselines, true);
 
-	// Import dispatchForFile dynamically to avoid circular deps
-	const { dispatchForFile } = await import("./dispatcher.js");
-	const { getRunnersForKind } = await import("./dispatcher.js");
-	const { TOOL_PLANS } = await import("./plan.js");
-
 	const kind = ctx.kind;
 	if (!kind) return "";
 
@@ -82,10 +80,6 @@ export async function dispatchLintWithResult(
 ): Promise<DispatchResult> {
 	const ctx = createDispatchContext(filePath, cwd, pi, sessionBaselines, true);
 
-	const { dispatchForFile } = await import("./dispatcher.js");
-	const { getRunnersForKind } = await import("./dispatcher.js");
-	const { TOOL_PLANS } = await import("./plan.js");
-
 	const kind = ctx.kind;
 	if (!kind) {
 		return {
@@ -110,7 +104,7 @@ export async function dispatchLintWithResult(
 		};
 	}
 
-	return await dispatchForFile(ctx, plan.groups);
+	return dispatchForFile(ctx, plan.groups);
 }
 
 /**
@@ -136,7 +130,6 @@ export async function getAvailableRunners(filePath: string): Promise<string[]> {
 	const kind = detectFileKind(filePath);
 	if (!kind) return [];
 
-	const { getRunnersForKind } = await import("./dispatcher.js");
 	const runners = getRunnersForKind(kind);
 	return runners.map((r) => r.id);
 }

package/clients/dispatch/runners/lsp.ts

@@ -117,7 +117,11 @@ const lspRunner: RunnerDefinition = {
 		return {
 			status: hasErrors ? "failed" : "succeeded",
 			diagnostics,
-			semantic: hasErrors ? "blocking" : "warning",
+			semantic: hasErrors
+				? "blocking"
+				: diagnostics.length > 0
+					? "warning"
+					: "none",
 		};
 	},
 };

package/clients/dispatch/runners/pyright.ts

@@ -82,7 +82,11 @@ const pyrightRunner: RunnerDefinition = {
 			return {
 				status: hasErrors ? "failed" : "succeeded",
 				diagnostics,
-				semantic: hasErrors ? "blocking" : "warning",
+				semantic: hasErrors
+					? "blocking"
+					: diagnostics.length > 0
+						? "warning"
+						: "none",
 			};
 		} catch {
 			// JSON parse error