pi-lens 3.7.1 → 3.8.2

package/clients/source-filter.ts

@@ -0,0 +1,222 @@
+/**
+ * Source File Filter — Deduplicates source files by detecting build artifacts.
+ *
+ * Problem: When scanning a codebase, we encounter both source files and their
+ * compiled/transpiled outputs (TypeScript → JavaScript, Vue → JavaScript, etc.).
+ * Scanning both wastes time and produces duplicate findings.
+ *
+ * Solution: For each file, check if a "higher precedence" source sibling exists.
+ * If yes, skip the file as a build artifact. If no, keep it as hand-written source.
+ *
+ * Supported ecosystems:
+ * - TypeScript: .ts shadows .js, .tsx shadows .jsx
+ * - Vue/Svelte: .vue/.svelte shadows .js
+ * - CoffeeScript: .coffee shadows .js
+ *
+ * Files without higher-precedence siblings are always kept (hand-written JS, Python,
+ * Go, Rust, etc.).
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+/**
+ * Mapping of file extension to the extensions it shadows (build artifacts).
+ * Order matters: first entry has highest precedence.
+ */
+export const SOURCE_PRECEDENCE: Record<string, string[]> = {
+	".ts": [".js", ".mjs", ".cjs"],
+	".tsx": [".jsx", ".js", ".mjs", ".cjs"],
+	".vue": [".js", ".mjs"],
+	".svelte": [".js", ".mjs"],
+	".coffee": [".js"],
+};
+
+/**
+ * All extensions that could be source or artifacts, in precedence order.
+ */
+export const ALL_SCANNABLE_EXTENSIONS = [
+	".ts",
+	".tsx",
+	".js",
+	".jsx",
+	".mjs",
+	".cjs",
+	".vue",
+	".svelte",
+	".coffee",
+	".py",
+	".go",
+	".rs",
+	".rb",
+	".rake",
+	".gemspec",
+	".ru",
+];
+
+/**
+ * Extract the basename (filename without extension) from a path.
+ */
+function getBasename(filePath: string): string {
+	const ext = path.extname(filePath);
+	return path.basename(filePath, ext);
+}
+
+/**
+ * Get the directory of a file path.
+ */
+function getDir(filePath: string): string {
+	return path.dirname(filePath);
+}
+
+/**
+ * Check if a file has a higher-precedence source sibling.
+ * Returns the shadowing source file path if found, null otherwise.
+ */
+export function findSourceSibling(filePath: string): string | null {
+	const ext = path.extname(filePath).toLowerCase();
+	const dir = getDir(filePath);
+	const base = getBasename(filePath);
+
+	// Find which precedence group this extension belongs to
+	for (const [sourceExt, shadowedExts] of Object.entries(SOURCE_PRECEDENCE)) {
+		if (shadowedExts.includes(ext)) {
+			// This file could be shadowed by a source file with sourceExt
+			const siblingPath = path.join(dir, base + sourceExt);
+			if (fs.existsSync(siblingPath)) {
+				return siblingPath;
+			}
+		}
+	}
+
+	return null;
+}
+
+/**
+ * Check if a file is a build artifact (has a source sibling).
+ */
+export function isBuildArtifact(filePath: string): boolean {
+	return findSourceSibling(filePath) !== null;
+}
+
+/**
+ * Filter a list of files, removing build artifacts that have source siblings.
+ * Returns de-duplicated list keeping only highest-precedence sources.
+ */
+export function filterSourceFiles(filePaths: string[]): string[] {
+	// Track which files we're keeping and why we're skipping others
+	const keep: string[] = [];
+	const skipReasons = new Map<string, string>(); // skipped file -> kept source
+
+	for (const filePath of filePaths) {
+		const sourceSibling = findSourceSibling(filePath);
+		if (sourceSibling) {
+			// This is a build artifact, skip it
+			skipReasons.set(filePath, sourceSibling);
+		} else {
+			// No higher-precedence source, keep it
+			keep.push(filePath);
+		}
+	}
+
+	return keep;
+}
+
+/**
+ * Recursively collect all source files in a directory, excluding build artifacts.
+ *
+ * @param dir - Directory to scan
+ * @param options - Optional configuration
+ * @returns Array of absolute file paths that are source files (not build artifacts)
+ */
+export function collectSourceFiles(
+	dir: string,
+	options?: {
+		/** Additional directory names to exclude (merged with defaults) */
+		excludeDirs?: string[];
+		/** File extensions to consider (defaults to ALL_SCANNABLE_EXTENSIONS) */
+		extensions?: string[];
+		/** Whether to follow symlinks (default: false) */
+		followSymlinks?: boolean;
+	},
+): string[] {
+	const excludeDirs = new Set([
+		"node_modules",
+		".git",
+		"dist",
+		"build",
+		".next",
+		"coverage",
+		"__pycache__",
+		".cache",
+		"target", // Rust
+		"out",
+		"*.dSYM", // macOS debug symbols
+		...(options?.excludeDirs || []),
+	]);
+
+	const extensions = new Set(options?.extensions || ALL_SCANNABLE_EXTENSIONS);
+
+	const files: string[] = [];
+
+	function scan(currentDir: string) {
+		let entries: fs.Dirent[] = [];
+		try {
+			entries = fs.readdirSync(currentDir, { withFileTypes: true });
+		} catch {
+			return; // Permission denied or directory doesn't exist
+		}
+
+		for (const entry of entries) {
+			const fullPath = path.join(currentDir, entry.name);
+
+			if (entry.isDirectory()) {
+				if (excludeDirs.has(entry.name)) continue;
+				if (!options?.followSymlinks && entry.isSymbolicLink()) continue;
+				scan(fullPath);
+			} else if (entry.isFile()) {
+				const ext = path.extname(entry.name).toLowerCase();
+				if (!extensions.has(ext)) continue;
+
+				// Skip if this is a build artifact
+				if (isBuildArtifact(fullPath)) continue;
+
+				files.push(fullPath);
+			}
+		}
+	}
+
+	scan(path.resolve(dir));
+	return files;
+}
+
+/**
+ * Get statistics about source file filtering for debugging/monitoring.
+ */
+export function getFilterStats(
+	allFiles: string[],
+	filteredFiles: string[],
+): {
+	total: number;
+	kept: number;
+	skipped: number;
+	byType: Record<string, number>;
+} {
+	const skipped = allFiles.length - filteredFiles.length;
+	const byType: Record<string, number> = {};
+
+	// Count what we skipped
+	for (const file of allFiles) {
+		if (!filteredFiles.includes(file)) {
+			const ext = path.extname(file).toLowerCase();
+			byType[ext] = (byType[ext] || 0) + 1;
+		}
+	}
+
+	return {
+		total: allFiles.length,
+		kept: filteredFiles.length,
+		skipped,
+		byType,
+	};
+}

package/clients/startup-scan.ts

@@ -0,0 +1,142 @@
+/**
+ * Startup scan safety — gates eager cache warmups to real project roots.
+ *
+ * Prevents pi-lens from scanning $HOME or generic directories at session
+ * start, which would hang or produce meaningless results.
+ *
+ * Credit: alexx-ftw (PR #1)
+ */
+
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { EXCLUDED_DIRS } from "./file-utils.js";
+
+export const PROJECT_ROOT_MARKERS = [
+	".git",
+	"package.json",
+	"pyproject.toml",
+	"Cargo.toml",
+	"go.mod",
+	"composer.json",
+];
+
+export const MAX_STARTUP_SOURCE_FILES = 2000;
+
+const SOURCE_FILE_PATTERN = /\.(ts|tsx|js|jsx|py|go|rs|rb)$/;
+
+export interface StartupScanContext {
+	cwd: string;
+	scanRoot: string;
+	projectRoot: string | null;
+	canWarmCaches: boolean;
+	reason?: "home-dir" | "no-project-root" | "too-many-source-files";
+	sourceFileCount?: number;
+}
+
+export interface StartupScanOptions {
+	homeDir?: string;
+	maxSourceFiles?: number;
+}
+
+export function findNearestProjectRoot(startDir: string): string | null {
+	let current = path.resolve(startDir);
+	while (true) {
+		if (
+			PROJECT_ROOT_MARKERS.some((marker) =>
+				fs.existsSync(path.join(current, marker)),
+			)
+		) {
+			return current;
+		}
+		const parent = path.dirname(current);
+		if (parent === current) return null;
+		current = parent;
+	}
+}
+
+export function countSourceFilesWithinLimit(
+	dir: string,
+	limit: number,
+): number {
+	let count = 0;
+	const stack = [path.resolve(dir)];
+
+	while (stack.length > 0) {
+		const current = stack.pop();
+		if (!current) continue;
+
+		let entries: fs.Dirent[] = [];
+		try {
+			entries = fs.readdirSync(current, { withFileTypes: true });
+		} catch {
+			continue;
+		}
+
+		for (const entry of entries) {
+			if (entry.isDirectory()) {
+				if (EXCLUDED_DIRS.includes(entry.name)) continue;
+				stack.push(path.join(current, entry.name));
+				continue;
+			}
+			if (entry.isFile() && SOURCE_FILE_PATTERN.test(entry.name)) {
+				count += 1;
+				if (count > limit) return count;
+			}
+		}
+	}
+	return count;
+}
+
+export function resolveStartupScanContext(
+	cwd: string,
+	options: StartupScanOptions = {},
+): StartupScanContext {
+	const resolvedCwd = path.resolve(cwd);
+	const homeDir = path.resolve(options.homeDir ?? os.homedir());
+	const maxSourceFiles = options.maxSourceFiles ?? MAX_STARTUP_SOURCE_FILES;
+	const projectRoot = findNearestProjectRoot(resolvedCwd);
+
+	if (!projectRoot) {
+		return {
+			cwd: resolvedCwd,
+			scanRoot: resolvedCwd,
+			projectRoot: null,
+			canWarmCaches: false,
+			reason: resolvedCwd === homeDir ? "home-dir" : "no-project-root",
+		};
+	}
+
+	if (path.resolve(projectRoot) === homeDir) {
+		return {
+			cwd: resolvedCwd,
+			scanRoot: projectRoot,
+			projectRoot,
+			canWarmCaches: false,
+			reason: "home-dir",
+		};
+	}
+
+	const sourceFileCount = countSourceFilesWithinLimit(
+		projectRoot,
+		maxSourceFiles,
+	);
+	if (sourceFileCount > maxSourceFiles) {
+		return {
+			cwd: resolvedCwd,
+			scanRoot: projectRoot,
+			projectRoot,
+			canWarmCaches: false,
+			reason: "too-many-source-files",
+			sourceFileCount,
+		};
+	}
+
+	return {
+		cwd: resolvedCwd,
+		scanRoot: projectRoot,
+		projectRoot,
+		canWarmCaches: true,
+		sourceFileCount,
+	};
+}

package/clients/todo-scanner.ts

@@ -1,19 +1,20 @@
 /**
  * TODO Scanner for pi-local.
  *
- * Scans codebase for TODO, FIXME, HACK, XXX, and other annotations.
- * Helps me understand what's already flagged as problematic or incomplete.
+ * Scans codebase for TODO, FIXME, HACK, XXX, and BUG annotations.
+ * Helps understand what's already flagged as problematic or incomplete.
  *
  * No dependencies required — uses regex scanning.
  */
 
 import * as fs from "node:fs";
 import * as path from "node:path";
+import { collectSourceFiles } from "./source-filter.js";
 
 // --- Types ---
 
 export interface TodoItem {
-	type: "TODO" | "FIXME" | "HACK" | "XXX" | "NOTE" | "DEPRECATED" | "BUG";
+	type: "TODO" | "FIXME" | "HACK" | "XXX" | "BUG";
 	message: string;
 	file: string;
 	line: number;
@@ -29,8 +30,12 @@ export interface TodoScanResult {
 // --- Scanner ---
 
 export class TodoScanner {
-	private readonly pattern =
-		/\b(TODO|FIXME|HACK|XXX|NOTE|DEPRECATED|BUG)\b\s*[(:]?\s*(.+)/gi;
+	/**
+	 * Pattern matches actionable annotations only.
+	 * Excludes NOTE and DEPRECATED — these are documentation, not work items.
+	 * Case-sensitive to avoid matching "Note:" in prose.
+	 */
+	private readonly pattern = /\b(TODO|FIXME|HACK|XXX|BUG)\b\s*[(:]?\s*(.+)/g;
 
 	/**
 	 * Check if a match position is inside a comment context.
@@ -80,7 +85,7 @@ export class TodoScanner {
 	}
 
 	/**
-	 * Scan a single file for TODOs
+	 * Scan a single file for TODOs.
 	 */
 	scanFile(filePath: string): TodoItem[] {
 		const absolutePath = path.resolve(filePath);
@@ -98,7 +103,7 @@ export class TodoScanner {
 				// Skip matches that aren't inside comments
 				if (!this.isInComment(line, match.index ?? 0)) continue;
 
-				const type = match[1].toUpperCase() as TodoItem["type"];
+				const type = match[1] as TodoItem["type"];
 				const message = (match[2] || "").trim().replace(/\s*\*\/\s*$/, ""); // Strip closing comment
 
 				items.push({
@@ -115,52 +120,42 @@ export class TodoScanner {
 	}
 
 	/**
-	 * Scan a directory recursively
+	 * Scan a list of pre-filtered files (recommended — uses source-filter module).
+	 * Callers should use collectSourceFiles() to get deduplicated source files.
 	 */
-	scanDirectory(
-		dirPath: string,
-		extensions = [".ts", ".tsx", ".js", ".jsx", ".py"],
-	): TodoScanResult {
+	scanFiles(filePaths: string[]): TodoScanResult {
 		const items: TodoItem[] = [];
 
-		const scan = (dir: string) => {
-			if (!fs.existsSync(dir)) return;
-
-			const entries = fs.readdirSync(dir, { withFileTypes: true });
-
-			for (const entry of entries) {
-				const fullPath = path.join(dir, entry.name);
-
-				if (entry.isDirectory()) {
-					// Skip common non-source directories
-					if (
-						[
-							"node_modules",
-							".git",
-							"dist",
-							"build",
-							".next",
-							"coverage",
-						].includes(entry.name)
-					)
-						continue;
-					scan(fullPath);
-				} else if (extensions.some((ext) => entry.name.endsWith(ext))) {
-					// Skip this scanner file — its own type literals and regex cause false positives
-					if (
-						entry.name === "todo-scanner.ts" ||
-						entry.name === "todo-scanner.js"
-					)
-						continue;
-					// Skip test files — intentional annotations are test fixtures, not work items
-					if (/\.(test|spec)\.[jt]sx?$/.test(entry.name)) continue;
-					items.push(...this.scanFile(fullPath));
-				}
-			}
-		};
+		for (const filePath of filePaths) {
+			// Skip this scanner file — its own type literals and regex cause false positives
+			if (
+				filePath.endsWith("todo-scanner.ts") ||
+				filePath.endsWith("todo-scanner.js")
+			)
+				continue;
+			// Skip test files — intentional annotations are test fixtures, not work items
+			if (/\.(test|spec)\.[jt]sx?$/.test(filePath)) continue;
+
+			items.push(...this.scanFile(filePath));
+		}
+
+		return this.groupResults(items);
+	}
 
-		scan(path.resolve(dirPath));
+	/**
+	 * Scan a directory recursively using the source-filter module to exclude build artifacts.
+	 * This is the preferred entry point for new callers.
+	 */
+	scanDirectory(dirPath: string): TodoScanResult {
+		// Use source-filter to collect only source files (no build artifacts)
+		const sourceFiles = collectSourceFiles(dirPath);
+		return this.scanFiles(sourceFiles);
+	}
 
+	/**
+	 * Group scan results by type and file.
+	 */
+	private groupResults(items: TodoItem[]): TodoScanResult {
 		// Group by type
 		const byType = new Map<string, TodoItem[]>();
 		for (const item of items) {
@@ -181,7 +176,7 @@ export class TodoScanner {
 	}
 
 	/**
-	 * Format scan results for LLM consumption
+	 * Format scan results for LLM consumption.
 	 */
 	formatResult(result: TodoScanResult, maxItems = 30): string {
 		if (result.items.length === 0) return "";
@@ -203,10 +198,8 @@ export class TodoScanner {
 			"FIXME",
 			"HACK",
 			"BUG",
-			"DEPRECATED",
 			"TODO",
 			"XXX",
-			"NOTE",
 		];
 		const sorted = [...result.items].sort((a, b) => {
 			const aIdx = priorityOrder.indexOf(a.type);
@@ -234,14 +227,10 @@ export class TodoScanner {
 				return "🟠";
 			case "BUG":
 				return "🐛";
-			case "DEPRECATED":
-				return "⚠️";
 			case "TODO":
 				return "📝";
 			case "XXX":
 				return "❌";
-			case "NOTE":
-				return "ℹ️";
 			default:
 				return "•";
 		}