pi-lens 2.2.9 → 3.0.0

package/clients/dispatch/runners/similarity.ts

@@ -0,0 +1,339 @@
+/**
+ * Similarity Runner: Detect semantic code reuse opportunities
+ *
+ * Uses Amain's 57×72 state matrix algorithm to find similar functions.
+ * Integrated into dispatch flow as a warning (non-blocking) suggestion.
+ */
+
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import * as ts from "typescript";
+import { EXCLUDED_DIRS } from "../../file-utils.js";
+import {
+	buildProjectIndex,
+	findSimilarFunctions,
+	loadIndex,
+	type ProjectIndex,
+} from "../../project-index.js";
+import { buildStateMatrix, countTransitions } from "../../state-matrix.js";
+import type {
+	Diagnostic,
+	DispatchContext,
+	RunnerDefinition,
+	RunnerResult,
+} from "../types.js";
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+const CONFIG = {
+	SIMILARITY_THRESHOLD: 0.75, // 75% minimum similarity
+	MIN_TRANSITIONS: 20, // Skip functions with <20 AST transitions
+	MAX_SUGGESTIONS: 3, // Max 3 suggestions per file
+	USAGE_THRESHOLD: 2, // Only suggest utilities with 2+ uses (placeholder)
+};
+
+// ============================================================================
+// Runner Implementation
+// ============================================================================
+
+const similarityRunner: RunnerDefinition = {
+	id: "similarity",
+	appliesTo: ["jsts"], // TypeScript/JavaScript only for MVP
+	priority: 35, // After ts-lsp, before ast-grep
+	enabledByDefault: true,
+
+	async run(ctx: DispatchContext): Promise<RunnerResult> {
+		const { filePath } = ctx;
+
+		// Only check TypeScript files
+		if (!filePath.match(/\.tsx?$/)) {
+			return { status: "skipped", diagnostics: [], semantic: "none" };
+		}
+
+		// Load file content
+		const content = await fs.readFile(filePath, "utf-8").catch(() => null);
+		if (!content) {
+			return { status: "skipped", diagnostics: [], semantic: "none" };
+		}
+
+		// Find project root and load index
+		const projectRoot = await findProjectRoot(filePath);
+		if (!projectRoot) {
+			return { status: "skipped", diagnostics: [], semantic: "none" };
+		}
+
+		const index = await loadOrBuildIndex(projectRoot);
+		if (!index || index.entries.size === 0) {
+			return { status: "skipped", diagnostics: [], semantic: "none" };
+		}
+
+		// Parse the file
+		const sourceFile = ts.createSourceFile(
+			filePath,
+			content,
+			ts.ScriptTarget.Latest,
+			true,
+			ts.ScriptKind.TS,
+		);
+
+		// Extract functions and check for similarities
+		const newFunctions = extractFunctions(sourceFile, content);
+
+		const diagnostics: Diagnostic[] = [];
+
+		for (const func of newFunctions) {
+			// Guardrail: Skip tiny functions
+			if (func.transitionCount < CONFIG.MIN_TRANSITIONS) {
+				continue;
+			}
+
+			// Find similar functions in index
+			const matches = findSimilarFunctions(
+				func.matrix,
+				index,
+				CONFIG.SIMILARITY_THRESHOLD,
+				CONFIG.MAX_SUGGESTIONS,
+			);
+
+			// Create diagnostic for each match
+			for (const match of matches) {
+				// Skip if it's the same function (self-match by path/name)
+				if (
+					match.targetId ===
+					`${path.relative(projectRoot, filePath)}:${func.name}`
+				) {
+					continue;
+				}
+
+				diagnostics.push({
+					id: `similarity-${func.name}-${match.targetId}`,
+					tool: "similarity",
+					filePath,
+					line: func.line,
+					column: func.column,
+					message: buildSuggestionMessage(func, match),
+					severity: "warning", // 🟡 Not blocking
+					semantic: "warning",
+				});
+			}
+		}
+
+		// Return limited number of suggestions
+		const limitedResults = diagnostics.slice(0, CONFIG.MAX_SUGGESTIONS);
+
+		if (limitedResults.length === 0) {
+			return { status: "succeeded", diagnostics: [], semantic: "none" };
+		}
+
+		return {
+			status: "succeeded",
+			diagnostics: limitedResults,
+			semantic: "warning",
+		};
+	},
+};
+
+// ============================================================================
+// Function Extraction
+// ============================================================================
+
+interface ExtractedFunction {
+	name: string;
+	line: number;
+	column: number;
+	matrix: number[][];
+	transitionCount: number;
+	signature: string;
+}
+
+function extractFunctions(
+	sourceFile: ts.SourceFile,
+	_fullContent: string,
+): ExtractedFunction[] {
+	const functions: ExtractedFunction[] = [];
+
+	function visit(node: ts.Node) {
+		// Function declarations
+		if (ts.isFunctionDeclaration(node) && node.name) {
+			const { line, character } = sourceFile.getLineAndCharacterOfPosition(
+				node.getStart(sourceFile),
+			);
+			const funcCode = getNodeText(node, sourceFile);
+			const matrix = buildStateMatrix(funcCode);
+			const transitionCount = countTransitions(matrix);
+
+			functions.push({
+				name: node.name.text,
+				line: line + 1, // 1-indexed
+				column: character + 1, // 1-indexed
+				matrix,
+				transitionCount,
+				signature: getSignature(node),
+			});
+		}
+
+		// Arrow functions assigned to const
+		if (ts.isVariableStatement(node)) {
+			extractArrowFunctions(node, functions, sourceFile);
+		}
+
+		ts.forEachChild(node, visit);
+	}
+
+	visit(sourceFile);
+	return functions;
+}
+
+function extractArrowFunctions(
+	node: ts.VariableStatement,
+	functions: ExtractedFunction[],
+	sourceFile: ts.SourceFile,
+): void {
+	for (const decl of node.declarationList.declarations) {
+		if (!ts.isIdentifier(decl.name) || !decl.initializer) {
+			continue;
+		}
+
+		const func = decl.initializer;
+		if (!ts.isArrowFunction(func) && !ts.isFunctionExpression(func)) {
+			continue;
+		}
+
+		const { line, character } = sourceFile.getLineAndCharacterOfPosition(
+			node.getStart(sourceFile),
+		);
+		const funcCode = getNodeText(func, sourceFile);
+		const matrix = buildStateMatrix(funcCode);
+		const transitionCount = countTransitions(matrix);
+
+		functions.push({
+			name: decl.name.text,
+			line: line + 1,
+			column: character + 1,
+			matrix,
+			transitionCount,
+			signature: getArrowSignature(func),
+		});
+	}
+}
+
+function getNodeText(node: ts.Node, sourceFile: ts.SourceFile): string {
+	return sourceFile.text.substring(node.getStart(sourceFile), node.getEnd());
+}
+
+function getSignature(node: ts.FunctionDeclaration): string {
+	const params = node.parameters
+		.map((p) => (ts.isIdentifier(p.name) ? p.name.text : "param"))
+		.join(", ");
+	return `(${params})`;
+}
+
+function getArrowSignature(
+	node: ts.ArrowFunction | ts.FunctionExpression,
+): string {
+	const params = node.parameters
+		.map((p) => (ts.isIdentifier(p.name) ? p.name.text : "param"))
+		.join(", ");
+	return `(${params})`;
+}
+
+// ============================================================================
+// Message Building
+// ============================================================================
+
+function buildSuggestionMessage(
+	func: ExtractedFunction,
+	match: {
+		targetId: string;
+		targetName: string;
+		targetLocation: string;
+		similarity: number;
+	},
+): string {
+	const similarityPct = Math.round(match.similarity * 100);
+	const parts = match.targetId.split(":");
+	const file = parts[0];
+	const name = parts[1] || match.targetName;
+	const location = `${file}:1`; // TODO: get actual line
+
+	return `Function '${func.name}' has ${similarityPct}% similarity to existing utility '${name}()' in ${location}. Consider reusing the existing utility.`;
+}
+
+// ============================================================================
+// Index Management
+// ============================================================================
+
+const indexCache = new Map<string, ProjectIndex>();
+
+async function findProjectRoot(filePath: string): Promise<string | null> {
+	let dir = path.dirname(filePath);
+	while (dir !== path.dirname(dir)) {
+		try {
+			await fs.access(path.join(dir, "package.json"));
+			return dir;
+		} catch {
+			dir = path.dirname(dir);
+		}
+	}
+	return null;
+}
+
+async function loadOrBuildIndex(
+	projectRoot: string,
+): Promise<ProjectIndex | null> {
+	// Check cache
+	const cached = indexCache.get(projectRoot);
+	if (cached) {
+		return cached;
+	}
+
+	// Try to load existing index
+	const existing = await loadIndex(projectRoot);
+	if (existing) {
+		indexCache.set(projectRoot, existing);
+		return existing;
+	}
+
+	// Build new index
+	const { glob } = await import("glob");
+	// Build ignore patterns from centralized EXCLUDED_DIRS
+	const ignorePatterns = [
+		...EXCLUDED_DIRS.map((d) => `**/${d}/**`),
+		"**/*.test.ts",
+		"**/*.spec.ts",
+		"**/*.poc.test.ts",
+	];
+	const files = await glob("**/*.ts", {
+		cwd: projectRoot,
+		ignore: ignorePatterns,
+	});
+
+	if (files.length === 0) {
+		return null;
+	}
+
+	const absoluteFiles = files.map((f) => path.join(projectRoot, f));
+	const index = await buildProjectIndex(projectRoot, absoluteFiles);
+
+	indexCache.set(projectRoot, index);
+	return index;
+}
+
+// ============================================================================
+// Testing Helper
+// ============================================================================
+
+export async function buildIndexForTesting(
+	projectRoot: string,
+): Promise<ProjectIndex> {
+	const index = await loadOrBuildIndex(projectRoot);
+	if (!index) {
+		throw new Error("Failed to build index");
+	}
+	return index;
+}
+
+export { CONFIG };
+export default similarityRunner;

package/clients/dispatch/runners/spellcheck.js

@@ -0,0 +1,106 @@
+/**
+ * Spellcheck runner for dispatch system
+ *
+ * Uses typos-cli (Rust-based, fast, zero-config) to check spelling in:
+ * - Markdown files (.md, .mdx)
+ * - Code comments (optional, if typos is configured)
+ *
+ * Key features:
+ * - Fast (Rust-based, ~10x faster than cspell)
+ * - Low false positives (only checks known typos)
+ * - Zero-config by default
+ * - JSON output for easy parsing
+ *
+ * Alternative considered: cspell
+ * - cspell: More comprehensive, but higher false positives, needs config
+ * - typos-cli: Faster, less noise, works out of the box
+ *
+ * Install: cargo install typos-cli
+ * Or: npm install -g typos-cli (if wrapped)
+ */
+import { safeSpawn } from "../../safe-spawn.js";
+import { createAvailabilityChecker } from "./utils/runner-helpers.js";
+const typos = createAvailabilityChecker("typos", ".exe");
+/**
+ * Parse typos-cli JSON output (JSON Lines format)
+ *
+ * Each line is a JSON object:
+ * {
+ *   "path": "file.md",
+ *   "line_num": 42,
+ *   "byte_offset": 1234,
+ *   "typo": "recieve",
+ *   "corrections": ["receive"]
+ * }
+ */
+function parseTyposOutput(raw, filePath) {
+    const diagnostics = [];
+    if (!raw.trim()) {
+        return diagnostics;
+    }
+    const lines = raw.trim().split("\n").filter((l) => l.trim());
+    for (const line of lines) {
+        try {
+            const parsed = JSON.parse(line);
+            if (!parsed.typo || !parsed.line_num)
+                continue;
+            const corrections = parsed.corrections?.join(", ") || "no suggestions";
+            const message = `Typo: "${parsed.typo}" → ${corrections}`;
+            diagnostics.push({
+                id: `typos-${parsed.line_num}-${parsed.typo}`,
+                message,
+                filePath,
+                line: parsed.line_num,
+                column: 1, // typos-cli doesn't provide column, just byte offset
+                severity: "warning",
+                semantic: "warning",
+                tool: "typos",
+                rule: "typo",
+                fixable: !!parsed.corrections?.length,
+                fixSuggestion: parsed.corrections?.[0],
+            });
+        }
+        catch {
+            // Skip invalid JSON lines
+            continue;
+        }
+    }
+    return diagnostics;
+}
+const spellcheckRunner = {
+    id: "spellcheck",
+    appliesTo: ["markdown"],
+    priority: 30, // Run after code quality checks (biome=10, slop=25)
+    enabledByDefault: true,
+    skipTestFiles: false, // Check docs in test files too
+    async run(ctx) {
+        // Skip if typos-cli is not installed
+        if (!typos.isAvailable(ctx.cwd || process.cwd())) {
+            return { status: "skipped", diagnostics: [], semantic: "none" };
+        }
+        // Run typos-cli with JSON output
+        // --format json: Output JSON Lines
+        // --exclude <pattern>: Could be used to exclude code blocks if needed
+        const args = ["--format", "json", ctx.filePath];
+        const result = safeSpawn(typos.getCommand(), args, {
+            timeout: 15000,
+        });
+        // typos-cli exits with code 2 if typos found, 0 if clean
+        const hasTypos = result.status === 2 || result.stdout?.trim();
+        if (!hasTypos) {
+            return { status: "succeeded", diagnostics: [], semantic: "none" };
+        }
+        // Parse diagnostics
+        const raw = result.stdout + result.stderr;
+        const diagnostics = parseTyposOutput(raw, ctx.filePath);
+        if (diagnostics.length === 0) {
+            return { status: "succeeded", diagnostics: [], semantic: "none" };
+        }
+        return {
+            status: "failed",
+            diagnostics,
+            semantic: "warning",
+        };
+    },
+};
+export default spellcheckRunner;

package/clients/dispatch/runners/spellcheck.test.js

@@ -0,0 +1,158 @@
+/**
+ * Tests for spellcheck runner (typos-cli)
+ */
+import * as fs from "node:fs";
+import { createRequire } from "node:module";
+import * as path from "node:path";
+import { describe, expect, it } from "vitest";
+function createMockContext(filePath) {
+    return {
+        filePath,
+        cwd: process.cwd(),
+        kind: "markdown",
+        autofix: false,
+        deltaMode: false,
+        baselines: { get: () => [], add: () => { }, save: () => { } },
+        pi: {},
+        hasTool: async () => false,
+        log: () => { },
+    };
+}
+describe("spellcheck runner", () => {
+    const require = createRequire(import.meta.url);
+    it("should have correct runner definition", async () => {
+        const spellcheckModule = await import("./spellcheck.js");
+        const runner = spellcheckModule.default;
+        expect(runner.id).toBe("spellcheck");
+        expect(runner.appliesTo).toEqual(["markdown"]);
+        expect(runner.priority).toBe(30);
+        expect(runner.enabledByDefault).toBe(true);
+        expect(runner.skipTestFiles).toBe(false); // Check docs in test files too
+    });
+    it("should detect typos-cli availability", () => {
+        const { spawnSync } = require("node:child_process");
+        const result = spawnSync("typos", ["--version"], {
+            encoding: "utf-8",
+            timeout: 10000,
+            shell: true,
+        });
+        expect(result.error || result.status !== 0 ? "not available" : "available").toBeTruthy(); // May or may not be installed
+    });
+    it("should detect typos in markdown content", async () => {
+        const tmpFile = path.join(process.env.TEMP || "/tmp", `spellcheck_test_${Date.now()}.md`);
+        fs.writeFileSync(tmpFile, `# README
+
+This is a documnet about recieving data.
+The seperation of concerns is important.
+`);
+        try {
+            const spellcheckModule = await import("./spellcheck.js");
+            const runner = spellcheckModule.default;
+            const result = await runner.run(createMockContext(tmpFile));
+            // If typos-cli is installed, should detect typos
+            // If not installed, will be skipped
+            if (result.status !== "skipped") {
+                // Should detect at least "documnet" and "recieving"
+                expect(result.diagnostics.length).toBeGreaterThanOrEqual(1);
+                expect(result.diagnostics.some((d) => d.tool === "typos" &&
+                    (d.message.includes("documnet") ||
+                        d.message.includes("recieving") ||
+                        d.message.includes("seperation")))).toBe(true);
+            }
+        }
+        finally {
+            if (fs.existsSync(tmpFile)) {
+                fs.unlinkSync(tmpFile);
+            }
+        }
+    });
+    it("should suggest corrections for typos", async () => {
+        const tmpFile = path.join(process.env.TEMP || "/tmp", `spellcheck_fix_${Date.now()}.md`);
+        fs.writeFileSync(tmpFile, `# Test
+
+This is a recieving test.
+`);
+        try {
+            const spellcheckModule = await import("./spellcheck.js");
+            const runner = spellcheckModule.default;
+            const result = await runner.run(createMockContext(tmpFile));
+            if (result.status !== "skipped" && result.diagnostics.length > 0) {
+                // Should have fix suggestions
+                const fixableDiags = result.diagnostics.filter((d) => d.fixable);
+                expect(fixableDiags.length).toBeGreaterThanOrEqual(1);
+                expect(fixableDiags.some((d) => d.fixSuggestion?.toLowerCase().includes("receive"))).toBe(true);
+            }
+        }
+        finally {
+            if (fs.existsSync(tmpFile)) {
+                fs.unlinkSync(tmpFile);
+            }
+        }
+    });
+    it("should pass clean markdown files", async () => {
+        const tmpFile = path.join(process.env.TEMP || "/tmp", `spellcheck_ok_${Date.now()}.md`);
+        fs.writeFileSync(tmpFile, `# Clean README
+
+This is a correct document about receiving data.
+The separation of concerns is important.
+All spelling is proper in this file.
+`);
+        try {
+            const spellcheckModule = await import("./spellcheck.js");
+            const runner = spellcheckModule.default;
+            const result = await runner.run(createMockContext(tmpFile));
+            if (result.status !== "skipped") {
+                // Should have no typos
+                expect(result.diagnostics.length).toBe(0);
+                expect(result.status).toBe("succeeded");
+            }
+        }
+        finally {
+            if (fs.existsSync(tmpFile)) {
+                fs.unlinkSync(tmpFile);
+            }
+        }
+    });
+    it("should handle JSON parse errors gracefully", async () => {
+        const tmpFile = path.join(process.env.TEMP || "/tmp", `spellcheck_json_${Date.now()}.md`);
+        fs.writeFileSync(tmpFile, `# Test\n\nSimple file.`);
+        try {
+            const spellcheckModule = await import("./spellcheck.js");
+            const runner = spellcheckModule.default;
+            const result = await runner.run(createMockContext(tmpFile));
+            // Should not crash on JSON parse issues
+            expect(["succeeded", "failed", "skipped"]).toContain(result.status);
+        }
+        finally {
+            if (fs.existsSync(tmpFile)) {
+                fs.unlinkSync(tmpFile);
+            }
+        }
+    });
+    it("should skip when typos-cli is not available", async () => {
+        const tmpFile = path.join(process.env.TEMP || "/tmp", `spellcheck_skip_${Date.now()}.md`);
+        fs.writeFileSync(tmpFile, `# Test\n\nContent with typo: recieve.`);
+        try {
+            const spellcheckModule = await import("./spellcheck.js");
+            const runner = spellcheckModule.default;
+            // Check if typos is available
+            const { spawnSync } = require("node:child_process");
+            const checkResult = spawnSync("typos", ["--version"], {
+                encoding: "utf-8",
+                timeout: 5000,
+                shell: true,
+            });
+            const isAvailable = !checkResult.error && checkResult.status === 0;
+            const result = await runner.run(createMockContext(tmpFile));
+            if (!isAvailable) {
+                expect(result.status).toBe("skipped");
+                expect(result.diagnostics).toHaveLength(0);
+            }
+        }
+        finally {
+            if (fs.existsSync(tmpFile)) {
+                fs.unlinkSync(tmpFile);
+            }
+        }
+    });
+});