pi-jev-find 0.1.0

package/src/cascade/tree.ts

@@ -0,0 +1,302 @@
+/**
+ * The searchable file set and its model-facing listing. Files are listed once
+ * up front (the cascade judges every candidate by name before reading any), so
+ * the tree is a flat list of eligible files rather than a lazily expanded
+ * directory graph. Eligibility deny-lists build noise, lockfiles, binaries, and
+ * obvious credential material.
+ *
+ * Ported from oh-my-pi `packages/coding-agent/src/tools/jfind/tree.ts` (MIT):
+ * the native glob backend is replaced by `rg --files`, and the prefix-folded
+ * tree renderer is local (omp uses `@oh-my-pi/pi-utils` walkPathTree).
+ */
+import * as fs from "node:fs/promises";
+import { runRg } from "../rg.ts";
+
+/** One eligible file under the search root. */
+export interface FileEntry {
+	/** Absolute path. */
+	path: string;
+	/** Root-relative display path with `/` separators. */
+	rel: string;
+	size: number;
+}
+
+const DENY_DIRS: Record<string, true> = {
+	".git": true,
+	node_modules: true,
+	target: true,
+	dist: true,
+	build: true,
+	out: true,
+	".next": true,
+	".nuxt": true,
+	".turbo": true,
+	".cache": true,
+	__pycache__: true,
+	".venv": true,
+	venv: true,
+	".tox": true,
+	coverage: true,
+	".idea": true,
+	".vscode": true,
+	".gradle": true,
+	".mypy_cache": true,
+	".pytest_cache": true,
+	".ruff_cache": true,
+	".parcel-cache": true,
+};
+
+const DENY_FILES: Record<string, true> = {
+	"Cargo.lock": true,
+	"package-lock.json": true,
+	"yarn.lock": true,
+	"pnpm-lock.yaml": true,
+	"bun.lock": true,
+	"bun.lockb": true,
+	"poetry.lock": true,
+	"Pipfile.lock": true,
+	"composer.lock": true,
+	"Gemfile.lock": true,
+	"go.sum": true,
+	"flake.lock": true,
+	".DS_Store": true,
+	"Thumbs.db": true,
+};
+
+/** Credential files by exact name. Never listed or read, even when hidden files are included. */
+const SECRET_FILES: Record<string, true> = {
+	".env": true,
+	".envrc": true,
+	".netrc": true,
+	".npmrc": true,
+	".pypirc": true,
+	".pgpass": true,
+	".boto": true,
+	".s3cfg": true,
+	".dockercfg": true,
+	".git-credentials": true,
+	".htpasswd": true,
+	htpasswd: true,
+	credentials: true,
+	"credentials.json": true,
+	"client_secret.json": true,
+	"service-account.json": true,
+	id_rsa: true,
+	id_dsa: true,
+	id_ecdsa: true,
+	id_ed25519: true,
+};
+
+/** Credential files by extension: keys, certificate stores, encrypted vaults, and infrastructure state that embeds secrets. */
+const SECRET_EXT = [
+	"pem",
+	"key",
+	"p12",
+	"pfx",
+	"jks",
+	"keystore",
+	"bks",
+	"ppk",
+	"kdbx",
+	"gpg",
+	"pgp",
+	"asc",
+	"der",
+	"crt",
+	"cer",
+	"tfvars",
+	"tfvars.json",
+	"tfstate",
+	"tfstate.backup",
+];
+
+const BINARY_EXT = [
+	"png",
+	"jpg",
+	"jpeg",
+	"gif",
+	"webp",
+	"avif",
+	"ico",
+	"bmp",
+	"tiff",
+	"psd",
+	"svg",
+	"woff",
+	"woff2",
+	"ttf",
+	"otf",
+	"eot",
+	"zip",
+	"gz",
+	"tgz",
+	"tar",
+	"bz2",
+	"xz",
+	"zst",
+	"7z",
+	"rar",
+	"pdf",
+	"mp3",
+	"mp4",
+	"mov",
+	"avi",
+	"mkv",
+	"wav",
+	"ogg",
+	"flac",
+	"wasm",
+	"so",
+	"dylib",
+	"dll",
+	"exe",
+	"o",
+	"a",
+	"class",
+	"jar",
+	"pyc",
+	"pyo",
+	"bin",
+	"dat",
+	"db",
+	"sqlite",
+	"sqlite3",
+	"lock",
+	"map",
+	"min.js",
+	"min.css",
+	"snap",
+	"pb",
+	"onnx",
+	"safetensors",
+	"parquet",
+	"arrow",
+	"ipynb",
+];
+
+const ENV_TEMPLATES: Record<string, true> = {
+	".env.example": true,
+	".env.sample": true,
+	".env.template": true,
+	".env.dist": true,
+};
+
+/** `lower` ends with `.<ext>` for some `ext` in `exts`. */
+function hasExt(lower: string, exts: readonly string[]): boolean {
+	return exts.some(ext => lower.length > ext.length && lower.endsWith(`.${ext}`));
+}
+
+/** Credential material: exact names, `.env.*` variants (except committed templates), and key/vault extensions. */
+function secret(name: string): boolean {
+	if (Object.hasOwn(SECRET_FILES, name)) return true;
+	if (name.startsWith(".env.")) return !Object.hasOwn(ENV_TEMPLATES, name);
+	return hasExt(name.toLowerCase(), SECRET_EXT);
+}
+
+/** Whether a root-relative regular file is searchable. */
+export function eligibleFile(rel: string, size: number, includeHidden: boolean): boolean {
+	if (size <= 0) return false;
+	const segments = rel.split("/");
+	const name = segments[segments.length - 1]!;
+	for (let i = 0; i < segments.length - 1; i++) {
+		const dir = segments[i]!;
+		if (Object.hasOwn(DENY_DIRS, dir) || (!includeHidden && dir.startsWith("."))) return false;
+	}
+	if (!includeHidden && name.startsWith(".")) return false;
+	return !Object.hasOwn(DENY_FILES, name) && !secret(name) && !hasExt(name.toLowerCase(), BINARY_EXT);
+}
+
+export interface ListFilesOptions {
+	includeHidden: boolean;
+	signal?: AbortSignal;
+}
+
+/** Stats are I/O only; batch them instead of serializing thousands of round trips. */
+const STAT_BATCH = 128;
+
+async function statSizes(root: string, rels: readonly string[]): Promise<Map<string, number>> {
+	const sizes = new Map<string, number>();
+	for (let offset = 0; offset < rels.length; offset += STAT_BATCH) {
+		const batch = rels.slice(offset, offset + STAT_BATCH);
+		const stats = await Promise.allSettled(
+			batch.map(rel => fs.stat(`${root}/${rel}`)),
+		);
+		for (let i = 0; i < stats.length; i++) {
+			const result = stats[i]!;
+			if (result.status === "fulfilled" && result.value.isFile()) {
+				sizes.set(batch[i]!, result.value.size);
+			}
+		}
+	}
+	return sizes;
+}
+
+/**
+ * Every eligible, non-gitignored regular file under `root`, in path order.
+ * Symlinks are never followed (rg does not list them without `-L`).
+ */
+export async function listFiles(root: string, options: ListFilesOptions): Promise<FileEntry[]> {
+	const run = await runRg(
+		root,
+		["--files", "--null", "--no-messages", ...(options.includeHidden ? ["--hidden"] : [])],
+		{ signal: options.signal },
+	);
+	const rels = run.stdout
+		.subarray(0, run.stdout.length > 0 && run.stdout[run.stdout.length - 1] === 0 ? run.stdout.length - 1 : run.stdout.length)
+		.toString("utf8")
+		.split("\0")
+		.filter(rel => rel.length > 0);
+	const sizes = await statSizes(root, rels);
+	const entries: FileEntry[] = [];
+	for (const rel of rels) {
+		const size = sizes.get(rel);
+		if (size === undefined || !eligibleFile(rel, size, options.includeHidden)) continue;
+		entries.push({ path: `${root}/${rel}`, rel, size });
+	}
+	entries.sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0));
+	return entries;
+}
+
+const SIZE_UNITS = ["B", "KB", "MB", "GB", "TB"];
+
+/** `812 B`, `3.4 KB`, … */
+export function humanSize(bytes: number): string {
+	let value = bytes;
+	let unit = 0;
+	while (value >= 1024 && unit < SIZE_UNITS.length - 1) {
+		value /= 1024;
+		unit++;
+	}
+	return unit === 0 ? `${bytes} B` : `${value.toFixed(1)} ${SIZE_UNITS[unit]}`;
+}
+
+/**
+ * Model-facing listing of `entries` as a prefix-folded directory tree: one `#`
+ * per depth, `# dir/` headers, and every file line tagged with its question key
+ * (`# e017 name (size)`), with a blank line before every directory header and
+ * every root-level file after the first line.
+ */
+export function renderTree(entries: readonly FileEntry[], tagOf: (index: number) => string): string {
+	let out = "";
+	let emitted = false;
+	/** Directory segments currently active in the emitted header stack. */
+	const stack: string[] = [];
+	for (let index = 0; index < entries.length; index++) {
+		const segments = entries[index]!.rel.split("/");
+		const dirCount = segments.length - 1;
+		let shared = 0;
+		while (shared < dirCount && shared < stack.length && stack[shared] === segments[shared]) shared++;
+		for (let depth = shared; depth < dirCount; depth++) {
+			if (emitted) out += "\n";
+			emitted = true;
+			out += `${"#".repeat(depth + 1)} ${segments[depth]}/\n`;
+		}
+		stack.length = dirCount;
+		for (let depth = shared; depth < dirCount; depth++) stack[depth] = segments[depth]!;
+		if (emitted && dirCount === 0) out += "\n";
+		emitted = true;
+		const hashes = "#".repeat(dirCount + 1);
+		out += `${hashes} ${tagOf(index)} ${segments[dirCount]!} (${humanSize(entries[index]!.size)})\n`;
+	}
+	return out;
+}

package/src/config.ts

@@ -0,0 +1,59 @@
+/**
+ * Environment-driven configuration. Every knob is optional; defaults mirror
+ * the jegrep / oh-my-pi jfind cascade budgets.
+ */
+
+export interface Budgets {
+	/** Requests in flight per dispatched phase. */
+	concurrency: number;
+	/** Files per filename-ranking request. */
+	nameBatch: number;
+	/** Lexically ranked files that receive a filename judgment. */
+	candidates: number;
+	/** Files whose content is read and sketched. */
+	files: number;
+	/** Windows kept per read file. */
+	windows: number;
+	/** Bytes per passage window, tags included. */
+	windowBytes: number;
+	/** Bytes per sketch card. */
+	sketchBytes: number;
+	/** Complete passages verified across all files. */
+	fullLimit: number;
+}
+
+export const DEFAULT_BUDGETS: Budgets = {
+	concurrency: 16,
+	nameBatch: 64,
+	candidates: 128,
+	files: 20,
+	windows: 24,
+	windowBytes: 8192,
+	sketchBytes: 384,
+	fullLimit: 40,
+};
+
+export interface JfConfig {
+	enabled: boolean;
+	budgets: Budgets;
+}
+
+function readInt(env: Record<string, string | undefined>, name: string, fallback: number, min: number, max: number): number {
+	const raw = env[name];
+	if (raw === undefined || raw.trim() === "") return fallback;
+	const value = Number.parseInt(raw, 10);
+	if (!Number.isFinite(value)) return fallback;
+	return Math.min(max, Math.max(min, value));
+}
+
+export function loadConfig(env: Record<string, string | undefined> = process.env): JfConfig {
+	const budgets = { ...DEFAULT_BUDGETS };
+	const b = budgets as { [K in keyof Budgets]: number };
+	for (const key of Object.keys(DEFAULT_BUDGETS) as (keyof Budgets)[]) {
+		b[key] = readInt(env, `JF_${key.toUpperCase()}`, DEFAULT_BUDGETS[key], 1, 10_000);
+	}
+	return {
+		enabled: env.JF_ENABLED !== "0",
+		budgets,
+	};
+}

package/src/index.ts

@@ -0,0 +1,214 @@
+/**
+ * pi-jev-find extension entry: registers the `find` tool (semantic grep with a
+ * judge-calibrated cascade) and a `/find` status command.
+ *
+ * The tool contract and model-facing digest are ported from oh-my-pi
+ * `packages/coding-agent/src/tools/jfind/index.ts` (MIT); the judge is the
+ * native Jev probability API (System One), configured purely through
+ * environment variables: JEV_API_KEY / JEV_BASE_URL / JEV_MODEL.
+ */
+import * as fs from "node:fs/promises";
+import type { Stats } from "node:fs";
+import * as path from "node:path";
+import { Type } from "typebox";
+import type { AgentToolResult, ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { runCascade } from "./cascade/cascade.ts";
+import { rankedHeat } from "./cascade/passages.ts";
+import { loadConfig } from "./config.ts";
+import { JevJudge, resolveJevConfig } from "./judge/jev-judge.ts";
+import { renderFindCall, renderFindResult } from "./render.ts";
+import type { FindDetails, FindToolParams } from "./types.ts";
+
+const parameters = Type.Object({
+	query: Type.String({
+		description: "what to find, in plain language (concept or behavior, not a regex)",
+	}),
+	grep_keywords: Type.Array(Type.String(), {
+		description:
+			"identifiers or terms likely to appear verbatim in matching source; steer lexical pre-ranking. [] when unsure",
+	}),
+	path: Type.Optional(Type.String({ description: "directory to search. Omitted -> the workspace root" })),
+});
+
+const DESCRIPTION = `Semantic grep: describe what you are looking for in plain language; returns the files and line ranges that implement it, each with a calibrated 0-1 relevance score. No index; searches the live workspace tree on every call.
+
+- \`query\`: a concept or behavior ("where do we verify JWT tokens?", "retry budget for failed requests"), not a regex. Quoted phrases in \`query\` are matched whole.
+- \`grep_keywords\`: identifiers, symbols, or terms likely to appear verbatim in matching source; they steer the lexical pre-ranking. Pass \`[]\` when nothing specific comes to mind.
+- \`path\`: one directory to search; omit for the workspace root. Narrow it when you already know the subsystem.
+- Results are strongest first as \`path:start-end score snippet\`; open ranges with \`read\`.
+- Scores are absolute yes/no probabilities: comparable across calls; below ~0.4 is weak evidence, so widen the query or fall back to \`grep\` before concluding absence.
+- \`grep\` is for exact strings, regexes, and known symbols; \`glob\` is for file names. Reach for them after \`find\` has narrowed the files, or when the target is literally a string.
+- Every call spends judge requests over the search scope; batch related questions into one descriptive \`query\` instead of many narrow calls.`;
+
+const PROMPT_SNIPPET =
+	"find: semantic search — describe a behavior in plain language, get files and calibrated line ranges that implement it";
+
+const PROMPT_GUIDELINES = [
+	"When you do not already know where a behavior lives, call `find` once with a descriptive query instead of chaining guessed `grep` patterns and `glob` sweeps followed by speculative reads.",
+	"`grep` and `glob` come after `find` has narrowed the files, or when the target is an exact string or file name.",
+];
+
+/** Line ranges shown per hit in the model-facing text, strongest first. */
+const RANGES_SHOWN = 3;
+
+function toDisplay(rel: string, root: string, cwd: string): string {
+	const relative = path.relative(cwd, path.join(root, rel));
+	if (relative.startsWith("..")) return rel;
+	return relative.split(path.sep).join("/");
+}
+
+async function resolveRoot(rawPath: string | undefined, cwd: string): Promise<string> {
+	const input = (rawPath ?? "").trim();
+	if (input.length === 0) return path.resolve(cwd);
+	const root = path.resolve(cwd, input);
+	let stat: Stats;
+	try {
+		stat = await fs.stat(root);
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === "ENOENT") throw new Error(`Path not found: ${input}`);
+		throw error;
+	}
+	if (!stat.isDirectory()) throw new Error(`Path is not a directory: ${input}`);
+	return root;
+}
+
+function formatBytes(bytes: number): string {
+	if (bytes < 1024) return `${bytes} B`;
+	if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+	return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+
+function formatDuration(ms: number): string {
+	return ms < 1000 ? `${Math.round(ms)}ms` : `${(ms / 1000).toFixed(1)}s`;
+}
+
+function formatNumber(value: number): string {
+	return value.toLocaleString("en-US");
+}
+
+function formatDigest(
+	query: string,
+	scopePath: string | undefined,
+	details: FindDetails,
+	stats: FindDetails["stats"],
+): string {
+	const where = scopePath === undefined ? "" : ` in ${scopePath}`;
+	const out: string[] = [];
+	if (details.hits.length === 0) {
+		out.push(`no hits for "${query}"${where} (τ ${details.threshold.toFixed(2)})`);
+	} else {
+		out.push(
+			`${details.hits.length} hit(s) for "${query}"${where} (τ ${details.threshold.toFixed(2)}), strongest first`,
+			"",
+		);
+		for (const hit of details.hits) {
+			const coverage = hit.truncated
+				? `${hit.linesSeen} lines judged, partial`
+				: `${hit.linesSeen} lines judged`;
+			out.push(`${hit.rel}  ${hit.contentScore.toFixed(2)}  ${coverage}`);
+			for (const range of rankedHeat(hit.ranges, RANGES_SHOWN)) {
+				const span = range.start === range.end ? String(range.start) : `${range.start}-${range.end}`;
+				out.push(`  ${hit.rel}:${span}  ${range.p.toFixed(2)}  ${range.snippet}`);
+			}
+		}
+	}
+	out.push(
+		"",
+		`listed ${stats.filesListed} · judged ${stats.judged} · read ${stats.filesRead} files (${formatBytes(stats.fileBytes)}) · ${stats.requests} requests · ${formatNumber(stats.inputTokens)} tokens · $${stats.cost.toFixed(4)} · ${formatDuration(details.elapsedMs)} wall / ${formatDuration(stats.apiMs)} api`,
+	);
+	if (stats.failures.length > 0) {
+		out.push(`${stats.errors} of ${stats.requests} requests failed:`, ...stats.failures.map(failure => `  ${failure}`));
+	}
+	return out.join("\n");
+}
+
+export default function jfindExtension(pi: ExtensionAPI): void {
+	const config = loadConfig();
+	if (!config.enabled) return;
+
+	pi.registerTool({
+		name: "find",
+		label: "Find",
+		description: DESCRIPTION,
+		promptSnippet: PROMPT_SNIPPET,
+		promptGuidelines: PROMPT_GUIDELINES,
+		parameters,
+		async execute(_toolCallId, params: FindToolParams, signal, onUpdate, ctx) {
+			const query = params.query.trim();
+			if (query.length === 0) throw new Error("`query` must be a non-empty description");
+			const root = await resolveRoot(params.path, ctx.cwd);
+			const scopePath =
+				root === path.resolve(ctx.cwd)
+					? undefined
+					: `${path.relative(ctx.cwd, root).split(path.sep).join("/")}/`;
+			const jev = resolveJevConfig();
+			const judge = new JevJudge(jev);
+			const started = performance.now();
+			const result = await runCascade({
+				root,
+				query,
+				extraKeywords: params.grep_keywords,
+				judge,
+				includeHidden: false,
+				budgets: config.budgets,
+				signal,
+				onProgress: message =>
+					onUpdate?.({ content: [{ type: "text", text: message }] } as AgentToolResult<FindDetails>),
+			});
+			const elapsedMs = performance.now() - started;
+			const { stats, threshold, keywords } = result;
+			const hits = result.hits.map(hit => ({ ...hit, rel: toDisplay(hit.rel, root, ctx.cwd) }));
+			const details: FindDetails = { query, keywords, threshold, hits, stats, elapsedMs, cwd: ctx.cwd, scopePath };
+			const digest = formatDigest(query, scopePath, details, stats);
+			if (stats.requests > 0 && stats.errors === stats.requests) {
+				throw new Error(
+					`find failed — all ${stats.requests} judge requests failed:\n${stats.failures.join("\n") || "unknown errors"}`,
+				);
+			}
+			return {
+				content: [{ type: "text", text: digest }],
+				details,
+				usage: {
+					input: stats.inputTokens,
+					output: stats.outputTokens,
+					cacheRead: 0,
+					cacheWrite: 0,
+					totalTokens: stats.inputTokens + stats.outputTokens,
+					cost: {
+						input: 0,
+						output: 0,
+						cacheRead: 0,
+						cacheWrite: 0,
+						total: stats.cost,
+					},
+				},
+			};
+		},
+		renderCall(args, theme) {
+			return renderFindCall(args, theme);
+		},
+		renderResult(result, options, theme) {
+			const details = result.details as FindDetails | undefined;
+			const isError = (result as { isError?: boolean }).isError === true;
+			return renderFindResult(details, isError, options.expanded, theme);
+		},
+	});
+
+	pi.registerCommand("find", {
+		description: "pi-jev-find: show the resolved Jev judge and cascade budgets",
+		handler: async (_args, ctx) => {
+			const current = loadConfig();
+			try {
+				const jev = resolveJevConfig();
+				ctx.ui.notify(`pi-jev-find judge: jev ${jev.model} @ ${jev.baseUrl} (key from ${jev.keySource})`, "info");
+			} catch (error) {
+				ctx.ui.notify(`pi-jev-find: ${error instanceof Error ? error.message : String(error)}`, "warning");
+				return;
+			}
+			ctx.ui.notify(
+				`budgets: candidates=${current.budgets.candidates} files=${current.budgets.files} windows=${current.budgets.windows} windowBytes=${current.budgets.windowBytes} sketchBytes=${current.budgets.sketchBytes} fullLimit=${current.budgets.fullLimit} concurrency=${current.budgets.concurrency}`,
+				"info",
+			);
+		},
+	});
+}