pi-lens 3.7.1 → 3.8.2

package/clients/tree-sitter-cache.ts

@@ -0,0 +1,315 @@
+/**
+ * Tree-sitter Tree Cache with Incremental Parsing Support
+ *
+ * Caches parsed ASTs and enables incremental updates for large files.
+ * This provides 10-100× speedup on edits to large files (>1000 lines).
+ */
+
+import * as crypto from "node:crypto";
+import * as fs from "node:fs";
+
+export interface CachedTree {
+	tree: any; // Tree-sitter Tree instance
+	contentHash: string;
+	languageId: string;
+	fileSize: number;
+	lineCount: number;
+	lastModified: number;
+}
+
+export class TreeCache {
+	private cache = new Map<string, CachedTree>();
+	private maxSize: number;
+	private debug: (msg: string) => void;
+
+	constructor(maxSize = 50, debug = false) {
+		this.maxSize = maxSize;
+		this.debug = debug
+			? (msg: string) => console.error(`[tree-cache] ${msg}`)
+			: () => {};
+	}
+
+	/**
+	 * Generate hash for file content
+	 */
+	private hashContent(content: string): string {
+		return crypto
+			.createHash("sha256")
+			.update(content)
+			.digest("hex")
+			.slice(0, 16);
+	}
+
+	/**
+	 * Get cache key for a file
+	 */
+	private getCacheKey(filePath: string, languageId: string): string {
+		return `${languageId}:${filePath}`;
+	}
+
+	/**
+	 * Check if tree is cached and valid
+	 */
+	get(filePath: string, content: string, languageId: string): any | null {
+		const key = this.getCacheKey(filePath, languageId);
+		const cached = this.cache.get(key);
+
+		if (!cached) {
+			this.debug(`Cache miss: ${filePath}`);
+			return null;
+		}
+
+		// Verify language matches
+		if (cached.languageId !== languageId) {
+			this.debug(`Language mismatch for ${filePath}`);
+			this.cache.delete(key);
+			return null;
+		}
+
+		// Check content hash
+		const contentHash = this.hashContent(content);
+		if (cached.contentHash !== contentHash) {
+			this.debug(
+				`Content changed: ${filePath} (${cached.lineCount} → ${content.split("\n").length} lines)`,
+			);
+			// Keep old tree for potential incremental update, but mark as stale
+			return null;
+		}
+
+		// Check if file was modified on disk (mtime changed)
+		try {
+			const stats = fs.statSync(filePath);
+			if (stats.mtimeMs !== cached.lastModified) {
+				this.debug(`File modified on disk: ${filePath}`);
+				this.cache.delete(key);
+				return null;
+			}
+		} catch {
+			// File might be deleted, invalidate cache
+			this.cache.delete(key);
+			return null;
+		}
+
+		this.debug(`Cache hit: ${filePath} (${cached.lineCount} lines)`);
+		return cached.tree;
+	}
+
+	/**
+	 * Store parsed tree in cache
+	 */
+	set(filePath: string, content: string, languageId: string, tree: any): void {
+		// Evict oldest entries if cache is full
+		if (this.cache.size >= this.maxSize) {
+			const firstKey = this.cache.keys().next().value;
+			if (firstKey) {
+				this.cache.delete(firstKey);
+				this.debug(`Evicted: ${firstKey}`);
+			}
+		}
+
+		const key = this.getCacheKey(filePath, languageId);
+		let mtime = 0;
+		try {
+			mtime = fs.statSync(filePath).mtimeMs;
+		} catch {
+			// File deleted between parse and cache — cache with mtime=0;
+			// next get() will miss on mtime check and re-parse
+		}
+
+		this.cache.set(key, {
+			tree,
+			contentHash: this.hashContent(content),
+			languageId,
+			fileSize: content.length,
+			lineCount: content.split("\n").length,
+			lastModified: mtime,
+		});
+
+		this.debug(`Cached: ${filePath} (${content.split("\n").length} lines)`);
+	}
+
+	/**
+	 * Calculate the diff between old and new content
+	 * Returns edit information for incremental parsing
+	 */
+	calculateEdit(
+		oldContent: string,
+		newContent: string,
+	): {
+		startIndex: number;
+		oldEndIndex: number;
+		newEndIndex: number;
+		startPosition: { row: number; column: number };
+		oldEndPosition: { row: number; column: number };
+		newEndPosition: { row: number; column: number };
+	} | null {
+		// Find the first difference
+		let startIndex = 0;
+		while (
+			startIndex < oldContent.length &&
+			startIndex < newContent.length &&
+			oldContent[startIndex] === newContent[startIndex]
+		) {
+			startIndex++;
+		}
+
+		// Find the last difference (working backwards)
+		let oldEndIndex = oldContent.length;
+		let newEndIndex = newContent.length;
+		while (
+			oldEndIndex > startIndex &&
+			newEndIndex > startIndex &&
+			oldContent[oldEndIndex - 1] === newContent[newEndIndex - 1]
+		) {
+			oldEndIndex--;
+			newEndIndex--;
+		}
+
+		// No change detected
+		if (startIndex === oldContent.length && startIndex === newContent.length) {
+			return null;
+		}
+
+		// Calculate positions
+		const startPosition = this.indexToPosition(oldContent, startIndex);
+		const oldEndPosition = this.indexToPosition(oldContent, oldEndIndex);
+		const newEndPosition = this.indexToPosition(newContent, newEndIndex);
+
+		return {
+			startIndex,
+			oldEndIndex,
+			newEndIndex,
+			startPosition,
+			oldEndPosition,
+			newEndPosition,
+		};
+	}
+
+	/**
+	 * Convert byte index to row/column position
+	 */
+	private indexToPosition(
+		content: string,
+		index: number,
+	): { row: number; column: number } {
+		const lines = content.slice(0, index).split("\n");
+		return {
+			row: lines.length - 1,
+			column: lines[lines.length - 1].length,
+		};
+	}
+
+	/**
+	 * Attempt incremental update using tree.edit()
+	 * Returns updated tree or null if incremental update failed
+	 */
+	async incrementalUpdate(
+		filePath: string,
+		oldContent: string,
+		newContent: string,
+		languageId: string,
+		parser: any,
+	): Promise<any | null> {
+		const key = this.getCacheKey(filePath, languageId);
+		const cached = this.cache.get(key);
+
+		if (!cached) {
+			this.debug(`No cached tree for incremental update: ${filePath}`);
+			return null;
+		}
+
+		// Only use incremental for large files (>100 lines)
+		const lineCount = oldContent.split("\n").length;
+		if (lineCount < 100) {
+			this.debug(
+				`File too small for incremental: ${filePath} (${lineCount} lines)`,
+			);
+			return null;
+		}
+
+		// Calculate edit
+		const edit = this.calculateEdit(oldContent, newContent);
+		if (!edit) {
+			this.debug(`No edit detected for: ${filePath}`);
+			return null;
+		}
+
+		this.debug(
+			`Incremental update: ${filePath} (lines ${edit.startPosition.row}-${edit.oldEndPosition.row})`,
+		);
+
+		try {
+			// Apply edit to tree
+			cached.tree.edit({
+				startIndex: edit.startIndex,
+				oldEndIndex: edit.oldEndIndex,
+				newEndIndex: edit.newEndIndex,
+				startPosition: edit.startPosition,
+				oldEndPosition: edit.oldEndPosition,
+				newEndPosition: edit.newEndPosition,
+			});
+
+			// Re-parse only changed region
+			const newTree = parser.parse(newContent, cached.tree);
+
+			// Update cache
+			this.set(filePath, newContent, languageId, newTree);
+
+			this.debug(`Incremental update successful: ${filePath}`);
+			return newTree;
+		} catch (err) {
+			this.debug(`Incremental update failed: ${err}`);
+			return null;
+		}
+	}
+
+	/**
+	 * Clear cache for a specific file
+	 */
+	invalidate(filePath: string, languageId?: string): void {
+		if (languageId) {
+			const key = this.getCacheKey(filePath, languageId);
+			this.cache.delete(key);
+			this.debug(`Invalidated: ${key}`);
+		} else {
+			// Invalidate all entries for this file
+			for (const [key, value] of this.cache.entries()) {
+				if (key.includes(filePath)) {
+					this.cache.delete(key);
+					this.debug(`Invalidated: ${key}`);
+				}
+			}
+		}
+	}
+
+	/**
+	 * Clear entire cache
+	 */
+	clear(): void {
+		this.cache.clear();
+		this.debug("Cache cleared");
+	}
+
+	/**
+	 * Get cache statistics
+	 */
+	getStats(): {
+		size: number;
+		maxSize: number;
+		totalLines: number;
+		totalBytes: number;
+	} {
+		let totalLines = 0;
+		let totalBytes = 0;
+		for (const entry of this.cache.values()) {
+			totalLines += entry.lineCount;
+			totalBytes += entry.fileSize;
+		}
+		return {
+			size: this.cache.size,
+			maxSize: this.maxSize,
+			totalLines,
+			totalBytes,
+		};
+	}
+}

package/clients/tree-sitter-client.ts

@@ -18,6 +18,9 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { EXCLUDED_DIRS } from "./file-utils.js";
+import { resolvePackagePath } from "./package-root.js";
+import { TreeCache } from "./tree-sitter-cache.js";
+import { TreeSitterNavigator } from "./tree-sitter-navigator.js";
 import {
 	type TreeSitterQuery,
 	TreeSitterQueryLoader,
@@ -88,6 +91,8 @@ export class TreeSitterClient {
 	private initialized = false;
 	private languages: Map<string, TreeSitterLanguage> = new Map();
 	private parsers: Map<string, TreeSitterParserInstance> = new Map();
+	private treeCache: TreeCache;
+	private navigator = new TreeSitterNavigator();
 	private grammarsDir: string;
 	// biome-ignore lint/suspicious/noExplicitAny: Optional dependency loaded dynamically
 	private ParserClass: any = null;
@@ -102,6 +107,7 @@ export class TreeSitterClient {
 	constructor(verbose = false) {
 		this.grammarsDir = this.findGrammarsDir();
 		this.verbose = verbose;
+		this.treeCache = new TreeCache(50, verbose);
 	}
 
 	/** Debug logging helper */
@@ -113,32 +119,17 @@ export class TreeSitterClient {
 
 	/** Find tree-sitter grammar directory */
 	private findGrammarsDir(): string {
-		// Check for downloaded grammars in web-tree-sitter/grammars first
+		const pkgGrammars = resolvePackagePath(
+			import.meta.url,
+			"node_modules",
+			"web-tree-sitter",
+			"grammars",
+		);
 		const downloadedGrammars = [
 			path.join(process.cwd(), "node_modules", "web-tree-sitter", "grammars"),
+			pkgGrammars,
 		];
 
-		// Add __dirname-based paths if __dirname is available (CommonJS)
-		if (typeof __dirname !== "undefined") {
-			downloadedGrammars.push(
-				path.join(
-					__dirname,
-					"..",
-					"..",
-					"node_modules",
-					"web-tree-sitter",
-					"grammars",
-				),
-				path.join(
-					__dirname,
-					"..",
-					"node_modules",
-					"web-tree-sitter",
-					"grammars",
-				),
-			);
-		}
-
 		for (const dir of downloadedGrammars) {
 			if (
 				fs.existsSync(dir) &&
@@ -148,31 +139,21 @@ export class TreeSitterClient {
 			}
 		}
 
-		// Fallback to legacy locations
 		const candidates: string[] = [
 			path.join(process.cwd(), "node_modules", "tree-sitter-wasms", "out"),
+			resolvePackagePath(
+				import.meta.url,
+				"node_modules",
+				"tree-sitter-wasms",
+				"out",
+			),
 		];
 
-		if (typeof __dirname !== "undefined") {
-			candidates.push(
-				path.join(
-					__dirname,
-					"..",
-					"..",
-					"node_modules",
-					"tree-sitter-wasms",
-					"out",
-				),
-				path.join(__dirname, "..", "node_modules", "tree-sitter-wasms", "out"),
-			);
-		}
-
 		for (const dir of candidates) {
 			if (fs.existsSync(dir)) return dir;
 		}
 
-		// Default to web-tree-sitter/grammars (may need manual download)
-		return downloadedGrammars[0];
+		return pkgGrammars;
 	}
 
 	/** Initialize tree-sitter WASM runtime */
@@ -194,8 +175,8 @@ export class TreeSitterClient {
 			this.LanguageLoader = mod.Language;
 
 			// Log what we're trying to load
-			const wasmPath = path.join(
-				process.cwd(),
+			const wasmPath = resolvePackagePath(
+				import.meta.url,
 				"node_modules",
 				"web-tree-sitter",
 				"tree-sitter.wasm",
@@ -206,9 +187,8 @@ export class TreeSitterClient {
 
 			await ParserClass.init({
 				locateFile: (scriptName: string) => {
-					// Always return the full path to the WASM file
-					const fullPath = path.join(
-						process.cwd(),
+					const fullPath = resolvePackagePath(
+						import.meta.url,
 						"node_modules",
 						"web-tree-sitter",
 						scriptName,
@@ -319,8 +299,21 @@ export class TreeSitterClient {
 		try {
 			const content = fs.readFileSync(filePath, "utf-8");
 			this.dbg(`File content length: ${content.length}`);
+
+			// Check cache first
+			const cachedTree = this.treeCache.get(filePath, content, languageId);
+			if (cachedTree) {
+				this.dbg(`Using cached tree for ${filePath}`);
+				return cachedTree;
+			}
+
+			// Parse and cache
 			const tree = parser.parse(content);
 			this.dbg(`Parsed, root node type: ${tree.rootNode.type}`);
+
+			// Cache the tree
+			this.treeCache.set(filePath, content, languageId, tree);
+
 			return tree;
 		} catch (err) {
 			this.dbg(`Parse error: ${err}`);
@@ -328,6 +321,67 @@ export class TreeSitterClient {
 		}
 	}
 
+	/**
+	 * Detect and extract injected content from template literals
+	 * Used for security analysis (SQL injection, unsafe regex, etc.)
+	 */
+	extractInjections(
+		filePath: string,
+		content: string,
+	): Array<{
+		type: "sql" | "css" | "html" | "gql" | "regex";
+		content: string;
+		line: number;
+		column: number;
+	}> {
+		const injections: Array<{
+			type: "sql" | "css" | "html" | "gql" | "regex";
+			content: string;
+			line: number;
+			column: number;
+		}> = [];
+
+		// Pattern: sql`SELECT * FROM users` or query`...`
+		const sqlPattern = /\b(sql|query|execute)\s*`([^`]+)`/gi;
+		let match: RegExpExecArray | null;
+		while ((match = sqlPattern.exec(content)) !== null) {
+			const lines = content.slice(0, match.index).split("\n");
+			injections.push({
+				type: "sql",
+				content: match[2],
+				line: lines.length,
+				column: lines[lines.length - 1].length,
+			});
+		}
+
+		// Pattern: styled.div`color: red;` or css`...`
+		const cssPattern = /\b(styled(?:\.\w+)?|css)\s*`([^`]+)`/gi;
+		while ((match = cssPattern.exec(content)) !== null) {
+			const lines = content.slice(0, match.index).split("\n");
+			injections.push({
+				type: "css",
+				content: match[2],
+				line: lines.length,
+				column: lines[lines.length - 1].length,
+			});
+		}
+
+		// Pattern: new RegExp(`pattern`)
+		const regexPattern = /new\s+RegExp\s*\(\s*`([^`]+)`/gi;
+		while ((match = regexPattern.exec(content)) !== null) {
+			const lines = content.slice(0, match.index).split("\n");
+			injections.push({
+				type: "regex",
+				content: match[1],
+				line: lines.length,
+				column: lines[lines.length - 1].length,
+			});
+		}
+
+		this.dbg(`Found ${injections.length} injections in ${filePath}`);
+		return injections;
+	}
+
 	/** Check if tree-sitter is available (grammars installed) */
 	isAvailable(): boolean {
 		return fs.existsSync(this.grammarsDir);
@@ -520,6 +574,10 @@ export class TreeSitterClient {
 		return { query: "", metavars: [] };
 	}
 
+	/**
+	 * Inject native tree-sitter predicates into S-expression query
+	 * This moves text filtering to WASM for better performance
+	 */
 	/** Generate cache key for compiled query */
 	private getQueryCacheKey(pattern: string, languageId: string): string {
 		// Simple hash for the query string
@@ -667,14 +725,6 @@ export class TreeSitterClient {
 					}
 				}
 
-				if (postFilter === "not_dbg_method") {
-					const methodNode = captures.METHOD;
-					if (methodNode) {
-						const methodName = methodNode.text;
-						if (methodName === "dbg") continue; // Skip console.dbg()
-					}
-				}
-
 				if (postFilter === "no_super_call") {
 					const bodyNode = captures.BODY;
 					if (bodyNode) {
@@ -687,6 +737,112 @@ export class TreeSitterClient {
 					}
 				}
 
+				// Scope-aware: keep only matches inside test blocks
+				if (postFilter === "in_test_block") {
+					const capturesArray = Object.values(captures);
+					if (capturesArray.length > 0 && capturesArray[0]) {
+						if (!this.navigator.isInTestBlock(capturesArray[0])) continue;
+					}
+				}
+
+				// Structural: keep only matches NOT inside a try/catch (or begin/rescue in Ruby)
+				if (postFilter === "not_in_try_catch") {
+					const capturesArray = Object.values(captures);
+					if (capturesArray.length > 0 && capturesArray[0]) {
+						if (this.navigator.isInTryCatch(capturesArray[0])) continue;
+					}
+				}
+
+				// Structural: keep only matches that ARE inside a try/catch (or begin/rescue)
+				if (postFilter === "in_try_catch") {
+					const capturesArray = Object.values(captures);
+					if (capturesArray.length > 0 && capturesArray[0]) {
+						if (!this.navigator.isInTryCatch(capturesArray[0])) continue;
+					}
+				}
+
+				// Scope-aware: keep only matches outside test blocks
+				if (postFilter === "not_in_test_block") {
+					const capturesArray = Object.values(captures);
+					if (capturesArray.length > 0 && capturesArray[0]) {
+						if (this.navigator.isInTestBlock(capturesArray[0])) continue;
+					}
+				}
+
+				// Structural: NAME capture must equal PARAM capture (actual shadowing check)
+				if (postFilter === "name_matches_param") {
+					const nameNode = captures.NAME;
+					const paramNode = captures.PARAM;
+					if (!nameNode || !paramNode) continue;
+					if (nameNode.text !== paramNode.text) continue;
+				}
+
+				// Scope: skip matches inside any function/method definition
+				if (postFilter === "not_in_function") {
+					const first = Object.values(captures)[0];
+					if (
+						first &&
+						this.navigator.isInside(first, [
+							"function_definition",
+							"function_declaration",
+							"method_definition",
+							"arrow_function",
+						])
+					)
+						continue;
+				}
+
+				// Security: variable name must match secret naming patterns
+				if (postFilter === "check_secret_pattern") {
+					const varName = (captures.VARNAME?.text ?? "").toLowerCase();
+					const secretPatterns = [
+						/api[_-]?key/,
+						/api[_-]?secret/,
+						/password/,
+						/passwd/,
+						/secret/,
+						/token/,
+						/auth/,
+						/private[_-]?key/,
+						/access[_-]?token/,
+						/credentials/,
+						/aws[_-]?secret/,
+						/github[_-]?token/,
+						/private[_-]?key/,
+						/client[_-]?secret/,
+					];
+					if (!secretPatterns.some((p) => p.test(varName))) continue;
+				}
+
+				// Python: except body that only contains pass (effectively empty)
+				if (postFilter === "python_empty_except") {
+					const bodyNode = captures.BODY;
+					if (bodyNode) {
+						// biome-ignore lint/suspicious/noExplicitAny: tree-sitter node
+						const realStmts = bodyNode.children.filter(
+							(c: any) =>
+								c.isNamed &&
+								c.type !== "pass_statement" &&
+								c.type !== "comment",
+						);
+						if (realStmts.length > 0) continue; // has real statements
+					}
+				}
+
+				// Ruby: rescue body with no meaningful statements
+				if (postFilter === "ruby_empty_rescue") {
+					const bodyNode = captures.BODY;
+					if (bodyNode) {
+						// biome-ignore lint/suspicious/noExplicitAny: tree-sitter node
+						const realStmts = bodyNode.children.filter(
+							(c: any) =>
+								c.isNamed &&
+								!["comment", "nil", "nil_literal"].includes(c.type),
+						);
+						if (realStmts.length > 0) continue;
+					}
+				}
+
 				// Use first capture for position info
 				if (match.captures.length > 0) {
 					const firstNode = match.captures[0].node;