pi-jev-find 0.1.0

package/src/cascade/lexical.ts

@@ -0,0 +1,143 @@
+/**
+ * Lexical prior: per-file keyword occurrence counts from one ripgrep pass,
+ * turned into IDF weights and a file score that ranks candidates before any
+ * judgment is spent.
+ *
+ * Ported from oh-my-pi `packages/coding-agent/src/tools/jfind/lexical.ts`
+ * (MIT); the native grep backend is replaced by `rg --json`. Deviation: rg
+ * skips files over 16 MB (`--max-filesize`), so `filesScanned` counts files rg
+ * actually searched rather than every file offered; IDF is clamped to
+ * [0.5, 6], which bounds the effect on ranking.
+ */
+import { RgError, runRg } from "../rg.ts";
+import { countOccurrences } from "./text.ts";
+
+export interface GrepIndex {
+	/** Lowercased, non-empty keywords; `perFileKw` vectors align with this. */
+	keywords: string[];
+	/** rel file path → per-keyword occurrence counts over matching lines. */
+	perFileKw: Map<string, number[]>;
+	/** Files the scan opened (rg `begin` events), including ones with no match. */
+	filesScanned: number;
+}
+
+/** Regex-escape a literal keyword for the rg alternation. */
+function escapeRegex(keyword: string): string {
+	return keyword.replace(/[\\.+*?()|[\]{}^$#&\-~]/g, "\\$&");
+}
+
+export interface GrepIndexOptions {
+	includeHidden: boolean;
+	signal?: AbortSignal;
+	timeoutMs?: number;
+}
+
+/** rg `--json` events this module consumes; everything else is ignored. */
+interface RgPathEvent {
+	type: string;
+	data?: { path?: { text?: string }; lines?: { text?: string } };
+}
+interface RgSummaryEvent {
+	type: "summary";
+	data?: { stats?: { searches?: number } };
+}
+
+/**
+ * Count keyword occurrences (case-insensitive, any keyword) in every file under
+ * `root`. Only lines containing a keyword are inspected, so counts are per
+ * matching line rather than per file byte.
+ */
+export async function grepIndex(
+	root: string,
+	rawKeywords: readonly string[],
+	options: GrepIndexOptions,
+): Promise<GrepIndex> {
+	const keywords = rawKeywords.map(keyword => keyword.toLowerCase()).filter(keyword => keyword.length > 0);
+	const index: GrepIndex = { keywords, perFileKw: new Map(), filesScanned: 0 };
+	if (keywords.length === 0) return index;
+	const run = await runRg(
+		root,
+		[
+			"--json",
+			"--ignore-case",
+			"--no-messages",
+			"--max-filesize",
+			"16M",
+			...(options.includeHidden ? ["--hidden"] : []),
+			"--",
+			keywords.map(escapeRegex).join("|"),
+		],
+		{ signal: options.signal, timeoutMs: options.timeoutMs },
+	);
+	for (const raw of run.stdout.toString("utf8").split("\n")) {
+		if (raw.length === 0) continue;
+		let event: RgPathEvent | RgSummaryEvent;
+		try {
+			event = JSON.parse(raw) as RgPathEvent | RgSummaryEvent;
+		} catch {
+			continue;
+		}
+		if (event.type === "summary") {
+			// The summary event is always emitted and reports the number of files
+			// rg actually opened (searches), with or without matches — the corpus
+			// size the IDF denominator needs. `begin` events would not do: rg only
+			// emits them for files that matched.
+			const searches = (event as RgSummaryEvent).data?.stats?.searches;
+			if (typeof searches === "number") index.filesScanned = Math.max(index.filesScanned, searches);
+		} else if (event.type === "match") {
+			const match = event as RgPathEvent;
+			const rel = match.data?.path?.text;
+			const line = match.data?.lines?.text;
+			if (rel === undefined || line === undefined) continue; // non-UTF-8 path payload
+			let counts = index.perFileKw.get(rel);
+			if (!counts) {
+				counts = Array.from({ length: keywords.length }, () => 0);
+				index.perFileKw.set(rel, counts);
+			}
+			const lower = line.toLowerCase();
+			for (let k = 0; k < keywords.length; k++) {
+				counts[k]! += countOccurrences(lower, keywords[k]!);
+			}
+		}
+	}
+	return index;
+}
+
+/**
+ * Inverse document frequency per keyword, clamped to `[0.5, 6]`: rarity is
+ * capped so a word occurring once in a test fixture cannot beat an
+ * implementation that contains several query concepts repeatedly.
+ */
+export function idf(index: GrepIndex): number[] {
+	return index.keywords.map((_, k) => {
+		let df = 0;
+		for (const counts of index.perFileKw.values()) {
+			if ((counts[k] ?? 0) > 0) df++;
+		}
+		const weight = Math.log((index.filesScanned + 1) / (df + 1));
+		return Math.min(6, Math.max(0.5, weight));
+	});
+}
+
+/**
+ * Lexical rank of a file: rare query terms count more, log-scaled frequency
+ * keeps common words in a giant file from overwhelming a compact
+ * implementation with several terms, and a keyword in the path is worth two
+ * extra log-units.
+ */
+export function fileScore(
+	counts: readonly number[],
+	weights: readonly number[],
+	rel: string,
+	keywords: readonly string[],
+): number {
+	const lower = rel.toLowerCase();
+	let score = 0;
+	for (let k = 0; k < keywords.length; k++) {
+		const inPath = lower.includes(keywords[k]!) ? 1 : 0;
+		score += weights[k]! * (2 * inPath + Math.log1p(counts[k] ?? 0));
+	}
+	return score;
+}
+
+export { RgError };

package/src/cascade/passages.ts

@@ -0,0 +1,171 @@
+/**
+ * Byte-bounded source passages. A file is cut into contiguous whole-line
+ * windows tagged with their original line numbers; the lexically strongest
+ * windows are selected, sketched for routing, and finally judged verbatim.
+ * Every reported range maps back to real line coordinates.
+ *
+ * Ported from oh-my-pi `packages/coding-agent/src/tools/jfind/passages.ts` (MIT);
+ * only the `FindRange` import is local.
+ */
+import type { FindRange } from "../types.ts";
+import { clipBytes, countOccurrences, lines } from "./text.ts";
+
+/** A contiguous run of tagged source lines (`L<n>| text`), 1-based inclusive. */
+export interface Passage {
+	start: number;
+	end: number;
+	text: string;
+	/** Lexical score used for window selection and tie-breaking. */
+	score: number;
+}
+
+/** A judged line range with its yes-probability and a one-line preview; the shape the renderer consumes. */
+export type HeatRange = FindRange;
+
+/**
+ * Contiguous whole-line windows bounded by bytes, including the final line. A
+ * single oversized line is clipped at a code-point boundary and keeps its real
+ * line id.
+ */
+export function windows(
+	text: string,
+	bytes: number,
+	keywords: readonly string[],
+	weights: readonly number[],
+): Passage[] {
+	const source = lines(text);
+	const passages: Passage[] = [];
+	let start = 0;
+	while (start < source.length) {
+		let end = start;
+		let content = "";
+		let used = 0;
+		while (end < source.length) {
+			const line = source[end]!;
+			const prefix = `L${end + 1}| `;
+			const overhead = prefix.length + 1;
+			if (end > start && used + Buffer.byteLength(line) + overhead > bytes) break;
+			const piece = `${prefix}${clipBytes(line, Math.max(0, bytes - (used + overhead)))}\n`;
+			content += piece;
+			used += Buffer.byteLength(piece);
+			end++;
+			if (used >= bytes) break;
+		}
+		const lower = content.toLowerCase();
+		let score = 0;
+		for (let k = 0; k < keywords.length; k++) {
+			score += weights[k]! * Math.log1p(countOccurrences(lower, keywords[k]!));
+		}
+		passages.push({ start: start + 1, end, text: content, score });
+		start = end;
+	}
+	return passages;
+}
+
+/**
+ * Keep the best lexical windows and, when no words match, distribute the
+ * budget evenly through the file instead of always falling back to its
+ * opening bytes. Returned in file order.
+ */
+export function selectWindows(passages: Passage[], limit: number): Passage[] {
+	let selected = passages;
+	if (selected.length > limit) {
+		if (selected.every(passage => passage.score === 0)) {
+			const len = selected.length;
+			const step = Math.max(limit - 1, 1);
+			const keep = new Set<number>();
+			for (let k = 0; k < limit; k++) keep.add(Math.floor((k * (len - 1)) / step));
+			selected = selected.filter((_, index) => keep.has(index));
+		} else {
+			selected = [...selected].sort((a, b) => b.score - a.score || a.start - b.start).slice(0, limit);
+		}
+	}
+	return [...selected].sort((a, b) => a.start - b.start);
+}
+
+/** Passage text with the generated `L<n>| ` tags stripped; the caller owns the line coordinates. */
+export function plainContent(passage: Passage): string {
+	let out = "";
+	const tagged = lines(passage.text);
+	for (let i = 0; i < tagged.length; i++) {
+		const line = tagged[i]!;
+		const prefix = `L${passage.start + i}| `;
+		out += `${line.startsWith(prefix) ? line.slice(prefix.length) : line}\n`;
+	}
+	return out;
+}
+
+/** Bytes reserved per selected sketch line for its `<line>: ` tag and separator. */
+const SKETCH_LINE_OVERHEAD = 12;
+/** Minimum bytes worth spending on one more sketch line. */
+const SKETCH_MIN_LINE = 24;
+/** Longest single sketch line. */
+const SKETCH_MAX_LINE = 180;
+
+/**
+ * A budgeted map of verbatim source lines, not an invented summary. Lines are
+ * ranked by keyword weight (plus a nudge for call-like lines) so deep
+ * implementation text can outrank headers, then emitted in file order.
+ */
+export function sketch(
+	passage: Passage,
+	keywords: readonly string[],
+	weights: readonly number[],
+	budget: number,
+): string {
+	const plain = lines(plainContent(passage));
+	const ranked: { index: number; score: number }[] = [];
+	for (let index = 0; index < plain.length; index++) {
+		const line = plain[index]!;
+		if (line.trim().length === 0) continue;
+		const lower = line.toLowerCase();
+		let score = line.includes("(") ? 0.1 : 0;
+		for (let k = 0; k < keywords.length; k++) {
+			if (lower.includes(keywords[k]!)) score += weights[k]!;
+		}
+		ranked.push({ index, score });
+	}
+	ranked.sort((a, b) => b.score - a.score || a.index - b.index);
+	const selected: { index: number; text: string }[] = [];
+	let used = 0;
+	for (const { index } of ranked) {
+		const available = Math.max(0, budget - (used + SKETCH_LINE_OVERHEAD));
+		if (available < SKETCH_MIN_LINE) break;
+		const line = plain[index]!.trim();
+		const text = `${passage.start + index}: ${clipBytes(line, Math.min(available, SKETCH_MAX_LINE))}`;
+		used += Buffer.byteLength(text) + 1;
+		selected.push({ index, text });
+	}
+	selected.sort((a, b) => a.index - b.index);
+	return selected.map(entry => entry.text).join("\n");
+}
+
+/**
+ * Union the judged-positive spans; never bridge an unjudged gap. A merged span
+ * keeps the max probability so repeated or overlapping asks are not rewarded.
+ * Strongest first, then earliest.
+ */
+export function mergeHeat(heat: readonly HeatRange[], threshold: number): HeatRange[] {
+	const kept = heat
+		.filter(range => range.p >= threshold && range.p > 0 && range.start <= range.end)
+		.sort((a, b) => a.start - b.start || a.end - b.end);
+	const merged: HeatRange[] = [];
+	for (const range of kept) {
+		const last = merged[merged.length - 1];
+		if (last && range.start <= last.end + 1) {
+			last.end = Math.max(last.end, range.end);
+			if (range.p > last.p) last.p = range.p;
+			continue;
+		}
+		merged.push({ ...range });
+	}
+	return merged.sort((a, b) => b.p - a.p || a.start - b.start);
+}
+
+/** The most relevant ranges, strongest first, then earliest. */
+export function rankedHeat(heat: readonly HeatRange[], limit: number): HeatRange[] {
+	return heat
+		.filter(range => range.p > 0)
+		.sort((a, b) => b.p - a.p || a.start - b.start)
+		.slice(0, limit);
+}

package/src/cascade/questions.ts

@@ -0,0 +1,143 @@
+/**
+ * The three request shapes the cascade sends to the judge, as `state` plus
+ * one noul question per entry. Nouls give absolute probabilities, so entries
+ * are thresholded independently and batches are comparable with each other.
+ *
+ * State objects are built with alphabetically ordered keys so the wire bytes
+ * match the reference implementation (which serializes through sorted maps);
+ * judgment quality was benchmarked against that exact layout.
+ *
+ * Ported from oh-my-pi `packages/coding-agent/src/tools/jfind/questions.ts`
+ * (MIT): templates are local modules (pi's extension loader is Node-style and
+ * does not support asset imports) and `prompt.render` is a local mustache-
+ * style substitute.
+ */
+import type { JsonValue, NoulQuestion, JudgeRequest } from "../judge/types.ts";
+import { nameQuestionTemplate } from "../prompts/find-name-question.ts";
+import { passageQuestionTemplate } from "../prompts/find-passage-question.ts";
+import { sketchQuestionTemplate } from "../prompts/find-sketch-question.ts";
+import { type Passage, plainContent } from "./passages.ts";
+import { type FileEntry, renderTree } from "./tree.ts";
+
+export type Request = JudgeRequest;
+export { type NoulQuestion };
+
+/** `{{key}}` placeholders replaced with the value (missing keys become empty). */
+function render(template: string, vars: Record<string, string>): string {
+	return template.replace(/\{\{\s*(\w+)\s*\}\}/g, (_, key: string) => vars[key] ?? "");
+}
+
+const TASK =
+	"Semantic grep over a source tree: locate files whose content matches the search description. Entries are judged by name, size, position in the tree, and (for folders) a sample of what they contain.";
+
+const TREE_FORMAT =
+	"`tree` is a directory listing. Lines starting with # are headers: `# dir/` is a folder; more #s means deeper nesting under the header above; a header may fold several levels (`# a/b/c/`). Every judgeable entry carries a tag like e017 right after the #s: files as `e017 name (size)`, folders as `e017 name/ — N entries: sample of names`. Untagged header lines are only structure.";
+
+const FILE_CRITERIA = {
+	// Generated implementation is still implementation. File provenance must
+	// not itself be negative evidence for source-code searches.
+	no: "The file is unrelated by name and location; generated executable implementation can still be relevant.",
+	yes: "A file at this path plausibly contains code, text, or data matching the search.",
+};
+
+const FOLDER_CRITERIA = {
+	no: "Nothing about the folder's name, location, or sampled contents suggests it holds a match.",
+	yes: "The folder plausibly contains, at any depth, at least one file matching the search.",
+};
+
+const SKETCH_CRITERIA = {
+	no: "Unrelated code; mere mentions, declarations, call sites, tests or configuration without implementation.",
+	yes: "Likely substantive implementation, definition or explanation of any part of the requested behavior. A matching helper for one step counts. Excerpts omit most source: favor recall.",
+};
+
+const PASSAGE_CRITERIA = {
+	no: "This passage only mentions, calls, imports, tests, or configures the subject, or contains unrelated code sharing keywords.",
+	yes: "This passage contains an implementation, definition, or substantive explanation of an important part of the search. A helper implementing one requested step counts even when other steps are elsewhere.",
+};
+
+/** Question key of the `i`th entry in a filename batch. */
+export function entryKey(i: number): string {
+	return `e${String(i).padStart(3, "0")}`;
+}
+
+/** Question key of the `k`th passage in a sketch or verification batch. */
+export function passageKey(k: number): string {
+	return `p${String(k).padStart(2, "0")}`;
+}
+
+/** Copy of `record` with keys in lexicographic order. */
+function sorted<T extends JsonValue>(record: Record<string, T>): Record<string, T> {
+	const out: Record<string, T> = {};
+	for (const key of Object.keys(record).sort()) out[key] = record[key]!;
+	return out;
+}
+
+/** One noul per file over a shared tree-rendered listing of the batch. */
+export function nameBatch(project: string, query: string, entries: readonly FileEntry[]): Request {
+	const questions: Record<string, NoulQuestion> = {};
+	entries.forEach((entry, i) => {
+		const key = entryKey(i);
+		const name = entry.rel.slice(entry.rel.lastIndexOf("/") + 1);
+		questions[key] = {
+			type: "noul",
+			instructions: render(nameQuestionTemplate, { key, name, query }).trim(),
+		};
+	});
+	return {
+		state: {
+			criteria: { file: FILE_CRITERIA, folder: FOLDER_CRITERIA },
+			format: TREE_FORMAT,
+			project,
+			search: query,
+			task: TASK,
+			tree: renderTree(entries, entryKey),
+		},
+		questions,
+	};
+}
+
+/** One sketch card: the file it came from and its budgeted verbatim lines. */
+export interface SketchCard {
+	fileKey: string;
+	rel: string;
+	sketch: string;
+}
+
+/**
+ * Mixed-file packing of sketch cards: one state ingestion pays for many
+ * independent small cards instead of rereading a whole file to discover its
+ * useful spans.
+ */
+export function sketchBatch(query: string, cards: readonly SketchCard[]): Request {
+	const files: Record<string, string> = {};
+	const passages: Record<string, [string, string]> = {};
+	const questions: Record<string, NoulQuestion> = {};
+	cards.forEach((card, k) => {
+		const key = passageKey(k);
+		files[card.fileKey] = card.rel;
+		passages[key] = [card.fileKey, card.sketch];
+		questions[key] = { type: "noul", instructions: render(sketchQuestionTemplate, { key }).trim() };
+	});
+	return {
+		state: { criteria: SKETCH_CRITERIA, files: sorted(files), passages, search: query },
+		questions,
+	};
+}
+
+/** Verification of complete passages from one file, judged independently. */
+export function passageBatch(query: string, rel: string, passages: readonly Passage[]): Request {
+	const entries: Record<string, string> = {};
+	const questions: Record<string, NoulQuestion> = {};
+	passages.forEach((passage, k) => {
+		const key = passageKey(k);
+		entries[key] = plainContent(passage);
+		questions[key] = {
+			type: "noul",
+			instructions: render(passageQuestionTemplate, { key, query }).trim(),
+		};
+	});
+	return {
+		state: { criteria: PASSAGE_CRITERIA, file: rel, passages: entries, search: query },
+		questions,
+	};
+}

package/src/cascade/text.ts

@@ -0,0 +1,116 @@
+/**
+ * Byte-budgeted text primitives shared by the lexical windows, sketches, and
+ * file reads. Budgets are UTF-8 bytes (what the judge is billed on), so
+ * clipping is always done at a code-point boundary.
+ *
+ * Ported from oh-my-pi `packages/coding-agent/src/tools/jfind/text.ts` (MIT);
+ * `readText` swapped from `Bun.file` to `node:fs/promises`.
+ */
+import * as fs from "node:fs/promises";
+
+/** Split like Rust `str::lines`: `\n`-separated, trailing `\r` stripped, no phantom last line after a final newline. */
+export function lines(text: string): string[] {
+	if (text.length === 0) return [];
+	const out = text.split("\n");
+	if (out[out.length - 1] === "") out.pop();
+	for (let i = 0; i < out.length; i++) {
+		const line = out[i]!;
+		if (line.endsWith("\r")) out[i] = line.slice(0, -1);
+	}
+	return out;
+}
+
+/** Longest prefix of `text` that fits in `bytes` UTF-8 bytes without splitting a code point. */
+export function clipBytes(text: string, bytes: number): string {
+	if (Buffer.byteLength(text) <= bytes) return text;
+	let used = 0;
+	let end = 0;
+	for (const char of text) {
+		const width = Buffer.byteLength(char);
+		if (used + width > bytes) break;
+		used += width;
+		end += char.length;
+	}
+	return text.slice(0, end);
+}
+
+/** First `count` code points of `text`. */
+export function takeChars(text: string, count: number): string {
+	let end = 0;
+	let taken = 0;
+	for (const char of text) {
+		if (taken === count) break;
+		end += char.length;
+		taken++;
+	}
+	return text.slice(0, end);
+}
+
+/** Non-overlapping occurrences of `needle` in `haystack`; 0 for an empty needle. */
+export function countOccurrences(haystack: string, needle: string): number {
+	if (needle.length === 0) return 0;
+	let count = 0;
+	let from = 0;
+	for (;;) {
+		const at = haystack.indexOf(needle, from);
+		if (at === -1) return count;
+		count++;
+		from = at + needle.length;
+	}
+}
+
+export interface ReadText {
+	text: string;
+	/** Bytes actually used (after trimming to a line boundary). */
+	bytes: number;
+	truncated: boolean;
+}
+
+/** Why a file was not read: not text, nothing in it, or the filesystem said no. */
+export type ReadTextFailure = "binary" | "empty" | "io";
+
+export class ReadTextError extends Error {
+	constructor(
+		readonly kind: ReadTextFailure,
+		message: string,
+	) {
+		super(message);
+		this.name = "ReadTextError";
+	}
+}
+
+const BINARY_PROBE_BYTES = 8192;
+
+/**
+ * Read up to `maxBytes` of a text file. Rejects binaries (NUL in the first 8 KB)
+ * and blank files; trims a truncated read back to the last full line.
+ * @throws {ReadTextError} `binary`, `empty`, or `io`.
+ */
+export async function readText(path: string, maxBytes: number): Promise<ReadText> {
+	let buf: Buffer;
+	try {
+		const handle = await fs.open(path, "r");
+		try {
+			// Read only maxBytes + 1 so a single bounded syscall detects truncation
+			// without loading a potentially huge file into memory.
+			const chunk = Buffer.alloc(maxBytes + 1);
+			const { bytesRead } = await handle.read(chunk, 0, chunk.length, 0);
+			buf = chunk.subarray(0, bytesRead);
+		} finally {
+			await handle.close();
+		}
+	} catch (error) {
+		throw new ReadTextError("io", error instanceof Error ? error.message : String(error));
+	}
+	const probe = buf.subarray(0, Math.min(buf.length, BINARY_PROBE_BYTES));
+	if (probe.includes(0)) throw new ReadTextError("binary", "binary");
+	const truncated = buf.length > maxBytes;
+	if (truncated) {
+		buf = buf.subarray(0, maxBytes);
+		const newline = buf.lastIndexOf(0x0a);
+		if (newline !== -1) buf = buf.subarray(0, newline + 1);
+	}
+	const text = new TextDecoder().decode(buf);
+	if (lines(text).every(line => line.trim().length === 0)) throw new ReadTextError("empty", "empty");
+	return { text, bytes: buf.length, truncated };
+}