pi-crew 0.9.51 → 0.9.53

package/src/extension/import-index.ts

@@ -1,74 +0,0 @@
-import * as fs from "node:fs";
-import * as path from "node:path";
-import { DEFAULT_PATHS } from "../config/defaults.ts";
-import { projectCrewRoot, userCrewRoot } from "../utils/paths.ts";
-import { isSafePathId, resolveRealContainedPath } from "../utils/safe-paths.ts";
-
-export interface ImportedRunIndexEntry {
-	runId: string;
-	scope: "project" | "user";
-	bundlePath: string;
-	summaryPath: string;
-	importedAt?: string;
-	status?: string;
-	team?: string;
-	workflow?: string;
-	goal?: string;
-}
-
-function readEntry(root: string, scope: "project" | "user", runId: string): ImportedRunIndexEntry | undefined {
-	if (!isSafePathId(runId)) return undefined;
-	let bundlePath: string;
-	let summaryPath: string;
-	try {
-		const entryRoot = resolveRealContainedPath(root, runId);
-		bundlePath = resolveRealContainedPath(root, path.join(runId, "run-export.json"));
-		summaryPath = path.join(entryRoot, "README.md");
-	} catch {
-		return undefined;
-	}
-	if (!fs.existsSync(bundlePath)) return undefined;
-	try {
-		const raw = JSON.parse(fs.readFileSync(bundlePath, "utf-8")) as Record<string, unknown>;
-		const manifest =
-			raw.manifest && typeof raw.manifest === "object" && !Array.isArray(raw.manifest)
-				? (raw.manifest as Record<string, unknown>)
-				: {};
-		return {
-			runId,
-			scope,
-			bundlePath,
-			summaryPath,
-			importedAt: typeof raw.importedAt === "string" ? raw.importedAt : undefined,
-			status: typeof manifest.status === "string" ? manifest.status : undefined,
-			team: typeof manifest.team === "string" ? manifest.team : undefined,
-			workflow: typeof manifest.workflow === "string" ? manifest.workflow : undefined,
-			goal: typeof manifest.goal === "string" ? manifest.goal : undefined,
-		};
-	} catch {
-		return { runId, scope, bundlePath, summaryPath };
-	}
-}
-
-function collect(root: string, scope: "project" | "user"): ImportedRunIndexEntry[] {
-	if (!fs.existsSync(root)) return [];
-	try {
-		if (fs.lstatSync(root).isSymbolicLink()) return [];
-		resolveRealContainedPath(path.dirname(root), path.basename(root));
-	} catch {
-		return [];
-	}
-	return fs
-		.readdirSync(root)
-		.filter((entry) => isSafePathId(entry))
-		.map((entry) => readEntry(root, scope, entry))
-		.filter((entry): entry is ImportedRunIndexEntry => entry !== undefined);
-}
-
-export function listImportedRuns(cwd: string): ImportedRunIndexEntry[] {
-	const projectRoot = path.join(projectCrewRoot(cwd), DEFAULT_PATHS.state.importsSubdir);
-	const userRoot = path.join(userCrewRoot(), DEFAULT_PATHS.state.importsSubdir);
-	return [...collect(userRoot, "user"), ...collect(projectRoot, "project")].sort((a, b) =>
-		(b.importedAt ?? "").localeCompare(a.importedAt ?? ""),
-	);
-}

package/src/extension/knowledge-injection.ts

@@ -1,402 +0,0 @@
-/**
- * knowledge-injection.ts — Project knowledge that accumulates across runs (O4).
- *
- * ROADMAP Phase 1 / T1.3 ("downsized memory"): a deliberately minimal
- * replacement for the deleted 244-LOC MemoryStore. Crews and the main
- * session read `.crew/knowledge.md` and have it injected into the system
- * prompt, so pi-crew "remembers" project context across runs.
- *
- * Philosophy (Round 6 stress-test): radically downsized. Just a Markdown
- * file the user can edit, surfaced into every run. No vector DB, no
- * embeddings, no graph. Simple = trustworthy.
- *
- * B2 section-aware injection (2026-06-28): knowledge.md splits cleanly into
- * CONVENTIONS (always-relevant: Code Style, Environment, Architecture,
- * Testing, Release Process — ~2.5KB) and SESSION-LOG (per-version post-
- * mortems, incidents, fix detail — ~29KB, rarely relevant). Conventions
- * are ALWAYS injected; session-log sections are injected on-demand via
- * header-token IDF scoring against the task/goal, capped at
- * MAX_SESSION_LOG_BYTES. Non-matched session-log is summarized as a section
- * index with a `read` path-hint so the worker can recover any omitted topic.
- * Design doc: research-findings/b2-section-aware-design.md.
- */
-import * as fs from "node:fs";
-import * as path from "node:path";
-import { logInternalError } from "../utils/internal-error.ts";
-import { projectCrewRoot } from "../utils/paths.ts";
-import type { BeforeAgentStartEvent, ExtensionAPI } from "./pi-api.ts";
-
-/** The knowledge file, relative to the project crew root. */
-export const KNOWLEDGE_FILENAME = "knowledge.md";
-
-/**
- * Session-log injection cap (B2). Conventions are always injected in full;
- * matched session-log sections are capped here to bound total prompt size.
- * Sized so 2-3 typical sections (~1-2.5KB each) fit, while the largest single
- * outlier (v0.9.10 IN PROGRESS, ~7KB) is excluded from a full-match scenario.
- */
-const MAX_SESSION_LOG_BYTES = 5_000;
-
-/**
- * Relevance context for section-aware injection. When omitted (or when goal/
- * taskText are both absent), injection falls back to conventions-only — no
- * IDF computation, no session-log body. This keeps callers without query
- * context (main-session hook, legacy tests) stable.
- */
-export interface KnowledgeQuery {
-	/** The team run goal (broad topic signal). */
-	goal?: string;
-	/** The step/task instruction text (narrow topic signal). */
-	taskText?: string;
-	/** The worker role (reserved for future role-floor/boost; not scored yet). */
-	role?: string;
-}
-
-/**
- * Headers of sections always treated as CONVENTIONS (universal project rules).
- * Anything NOT matching these (after the convention run) is SESSION-LOG.
- * Header-matching is prefix + case-insensitive against the H2 title.
- */
-const CONVENTION_HEADERS = ["Code Style", "Environment", "Architecture", "Testing", "Release Process"];
-
-/** Resolve the knowledge file path for a cwd (may not exist yet). */
-export function knowledgePath(cwd: string): string {
-	return path.join(projectCrewRoot(cwd), KNOWLEDGE_FILENAME);
-}
-
-interface KnowledgeSection {
-	/** H2 header text (without leading `## `). */
-	header: string;
-	/** Full section body including the `## ` header line. */
-	body: string;
-	/** Header tokens (lowercased, stopword-filtered) for IDF scoring. */
-	headerTokens: Set<string>;
-}
-
-const STOPWORDS = new Set([
-	"the",
-	"a",
-	"an",
-	"is",
-	"are",
-	"of",
-	"to",
-	"in",
-	"for",
-	"and",
-	"or",
-	"not",
-	"with",
-	"this",
-	"that",
-	"it",
-	"be",
-	"on",
-	"at",
-	"by",
-	"do",
-	"does",
-	"how",
-	"what",
-	"when",
-	"from",
-	"fix",
-	"fixes",
-	"fixed",
-	"v0",
-	"v1",
-	"released",
-	"progress",
-	"uncommitted",
-	"not",
-	"pushed",
-	"published",
-	"session",
-	"post",
-]);
-
-/** Tokenize header text into a Set of lowercased non-stopword tokens. */
-function tokenizeHeader(header: string): Set<string> {
-	const tokens = new Set<string>();
-	for (const raw of header.toLowerCase().split(/[^a-z0-9.]+/)) {
-		// keep dot-paths (e.g. "run.lock", "config.ts") and alphanumerics >= 2 chars
-		const t = raw.replace(/^\.+|\.+$/g, "");
-		if (t.length >= 2 && !STOPWORDS.has(t) && !/^\d+$/.test(t)) tokens.add(t);
-	}
-	return tokens;
-}
-
-/** Parse knowledge.md into (conventions, sessionLog) sections by H2 header. */
-function parseKnowledgeSections(content: string): {
-	conventions: KnowledgeSection[];
-	sessionLog: KnowledgeSection[];
-} {
-	const lines = content.split(/\r?\n/);
-	const sections: KnowledgeSection[] = [];
-	let current: { header: string; lines: string[] } | null = null;
-	for (const line of lines) {
-		const m = /^##\s+(.+?)\s*$/.exec(line);
-		if (m) {
-			if (current)
-				sections.push({
-					header: current.header,
-					body: current.lines.join("\n"),
-					headerTokens: tokenizeHeader(current.header),
-				});
-			current = { header: m[1]!, lines: [line] };
-		} else if (current) {
-			current.lines.push(line);
-		}
-	}
-	if (current)
-		sections.push({
-			header: current.header,
-			body: current.lines.join("\n"),
-			headerTokens: tokenizeHeader(current.header),
-		});
-
-	// Classify: a section is a CONVENTION if its header starts-with one of the
-	// known convention titles. Once we leave the convention run (first non-
-	// matching H2 after conventions), everything else is SESSION-LOG. This is
-	// self-healing: if knowledge.md is restructured, the classifier adapts.
-	const conventions: KnowledgeSection[] = [];
-	const sessionLog: KnowledgeSection[] = [];
-	let leftConventionRun = false;
-	for (const sec of sections) {
-		const isConvention = !leftConventionRun && CONVENTION_HEADERS.some((c) => sec.header.toLowerCase().startsWith(c.toLowerCase()));
-		if (isConvention) conventions.push(sec);
-		else {
-			leftConventionRun = true;
-			sessionLog.push(sec);
-		}
-	}
-	return { conventions, sessionLog };
-}
-
-/** Compute IDF (inverse document frequency) over session-log header tokens. */
-function computeIdf(sections: KnowledgeSection[]): Map<string, number> {
-	const N = sections.length;
-	const df = new Map<string, number>();
-	for (const s of sections) for (const token of s.headerTokens) df.set(token, (df.get(token) ?? 0) + 1);
-	const idf = new Map<string, number>();
-	for (const [token, freq] of df) idf.set(token, Math.log(N / freq)); // standard IDF; freq >= 1
-	return idf;
-}
-
-/** Tokenize a free-form query (goal/taskText) the same way as headers. */
-function tokenizeQuery(query: string): Set<string> {
-	const tokens = new Set<string>();
-	for (const raw of query.toLowerCase().split(/[^a-z0-9.]+/)) {
-		const t = raw.replace(/^\.+|\.+$/g, "");
-		if (t.length >= 2 && !STOPWORDS.has(t) && !/^\d+$/.test(t)) tokens.add(t);
-	}
-	return tokens;
-}
-
-/** Score a section by summed IDF of query tokens present in its header. */
-function scoreSection(queryTokens: Set<string>, headerTokens: Set<string>, idf: Map<string, number>): number {
-	let score = 0;
-	for (const token of queryTokens) if (headerTokens.has(token)) score += idf.get(token) ?? 0;
-	return score;
-}
-
-/**
- * Select session-log sections relevant to the query, capped at budgetBytes.
- * Greedy by descending score (then original order), drop-whole policy with a
- * head-slice fallback for the single best match if nothing else fits (so a
- * matched query never returns zero bytes).
- */
-function selectSessionLog(query: string, sessionLog: KnowledgeSection[], budgetBytes: number): KnowledgeSection[] {
-	if (sessionLog.length === 0 || !query.trim()) return [];
-	const idf = computeIdf(sessionLog);
-	const queryTokens = tokenizeQuery(query);
-	// Match = ANY header-token overlap with the query. IDF still drives RANKING
-	// (rarer overlap ranks first), but a token that happens to appear in every
-	// section (IDF=0) must still count as a match — otherwise a query for a
-	// common-but-relevant keyword (e.g. "redaction") would return nothing.
-	const scored = sessionLog
-		.map((sec, idx) => {
-			const overlap = [...queryTokens].filter((t) => sec.headerTokens.has(t));
-			return {
-				sec,
-				score: overlap.reduce((sum, t) => sum + (idf.get(t) ?? 0), 0),
-				idx,
-				hasOverlap: overlap.length > 0,
-			};
-		})
-		.filter((x) => x.hasOverlap);
-	if (scored.length === 0) return [];
-	scored.sort((a, b) => b.score - a.score || a.idx - b.idx);
-
-	const selected: KnowledgeSection[] = [];
-	let used = 0;
-	let bestMatch: { sec: KnowledgeSection; score: number } | undefined;
-	for (const { sec, score } of scored) {
-		if (score > (bestMatch?.score ?? -1)) bestMatch = { sec, score };
-		if (used + sec.body.length <= budgetBytes) {
-			selected.push(sec);
-			used += sec.body.length;
-		}
-	}
-	// Head-slice fallback: if the best match didn't fit (or nothing fit), inject
-	// a head-sliced copy of the single best match so the query isn't empty.
-	if (selected.length === 0 && bestMatch) {
-		const sliced = bestMatch.sec.body.slice(0, Math.max(0, budgetBytes));
-		selected.push({
-			...bestMatch.sec,
-			body: `${sliced}\n\n<!-- section truncated (session-log budget ${budgetBytes} bytes). Full file: use \`read\`. -->`,
-		});
-	}
-	return selected;
-}
-
-/** Read knowledge content. "" if absent/empty. */
-export function readKnowledge(cwd: string, query?: KnowledgeQuery): string {
-	try {
-		const p = knowledgePath(cwd);
-		const stat = tryStat(p);
-		if (!stat) {
-			sectionCache.delete(p);
-			knowledgeCache.delete(p);
-			return "";
-		}
-		// Security: reject symlinks to prevent information disclosure.
-		// An attacker could replace knowledge.md with a symlink to /etc/shadow
-		// or other sensitive files.
-		if (stat.isSymbolicLink) {
-			logInternalError("knowledge-injection.symlink", new Error(`Refusing to read symlink: ${p}`));
-			return "";
-		}
-		// P5 (Round 15): mtime+size cache. readKnowledge fires on every agent
-		// start (main session + every worker), re-reading the file each time.
-		// For a run with N workers this is N redundant readFileSync of the same
-		// file. Cache by (mtimeMs, size) and only re-read when the file changes.
-		const cacheKey = `${stat.mtimeMs}:${stat.size}`;
-
-		// B2: when no query context is provided (main-session hook, legacy
-		// callers, tests), keep the simple head-only path. This preserves the
-		// 4 existing unit tests exactly (they call readKnowledge(cwd)).
-		if (!query || (!query.goal && !query.taskText)) {
-			const cached = knowledgeCache.get(p);
-			if (cached && cached.key === cacheKey) return cached.content;
-			let content = fs.readFileSync(p, "utf8").trim();
-			if (content.length > MAX_KNOWLEDGE_HEAD_BYTES) {
-				content = `${content.slice(0, MAX_KNOWLEDGE_HEAD_BYTES)}\n\n<!-- knowledge.md truncated at ${MAX_KNOWLEDGE_HEAD_BYTES} bytes (head shown). Full file: ${p} — use the \`read\` tool if you need sections beyond the head. -->`;
-			}
-			knowledgeCache.set(p, { key: cacheKey, content });
-			return content;
-		}
-
-		// Section-aware path: cache parsed sections, select per-query.
-		const cachedSections = sectionCache.get(p);
-		let parsed: {
-			conventions: KnowledgeSection[];
-			sessionLog: KnowledgeSection[];
-		};
-		if (cachedSections && cachedSections.key === cacheKey) {
-			parsed = {
-				conventions: cachedSections.conventions,
-				sessionLog: cachedSections.sessionLog,
-			};
-		} else {
-			const content = fs.readFileSync(p, "utf8").trim();
-			parsed = parseKnowledgeSections(content);
-			sectionCache.set(p, {
-				key: cacheKey,
-				conventions: parsed.conventions,
-				sessionLog: parsed.sessionLog,
-			});
-		}
-
-		const queryText = [query.goal, query.taskText].filter(Boolean).join(" \n ");
-		const matchedSessionLog = selectSessionLog(queryText, parsed.sessionLog, MAX_SESSION_LOG_BYTES);
-
-		const parts: string[] = [];
-		// Conventions: always full.
-		for (const sec of parsed.conventions) parts.push(sec.body);
-		// Matched session-log (drop-whole, budget-capped).
-		for (const sec of matchedSessionLog) parts.push(sec.body);
-		// Always: section-index of ALL session-log headers (recovery safety net).
-		if (parsed.sessionLog.length > 0) {
-			const indexLines = parsed.sessionLog.map((s) => `  - ${s.header}`);
-			parts.push(
-				`<!-- Session-log sections in knowledge.md (not injected unless matched above — use \`read\` for detail):\n${indexLines.join("\n")}\nFull file: ${p} -->`,
-			);
-		}
-		return parts.join("\n").trim();
-	} catch {
-		return "";
-	}
-}
-
-/** Stat helper returning undefined on error (file missing, perms, etc.). */
-function tryStat(p: string): { mtimeMs: number; size: number; isSymbolicLink: boolean } | undefined {
-	try {
-		// Use lstatSync to detect symlinks (statSync follows them).
-		const s = fs.lstatSync(p);
-		return { mtimeMs: s.mtimeMs, size: s.size, isSymbolicLink: s.isSymbolicLink() };
-	} catch {
-		return undefined;
-	}
-}
-
-interface CachedKnowledge {
-	key: string;
-	content: string;
-}
-const knowledgeCache = new Map<string, CachedKnowledge>();
-
-interface CachedSections {
-	key: string;
-	conventions: KnowledgeSection[];
-	sessionLog: KnowledgeSection[];
-}
-const sectionCache = new Map<string, CachedSections>();
-
-/** Head cap for the no-query (legacy / main-session) path. */
-const MAX_KNOWLEDGE_HEAD_BYTES = 2_000;
-
-/** Build the injected prompt fragment (empty if no knowledge). */
-export function buildKnowledgeFragment(cwd: string, query?: KnowledgeQuery): string {
-	const content = readKnowledge(cwd, query);
-	if (!content) return "";
-	return [
-		"",
-		"# Project knowledge (from .crew/knowledge.md)",
-		"The following project knowledge was captured by pi-crew from prior runs.",
-		"Use it to avoid repeating past mistakes and to respect project conventions.",
-		"You may update .crew/knowledge.md when you learn something durable.",
-		"",
-		content,
-	].join("\n");
-}
-
-/**
- * Register the knowledge-injection hook. Appends project knowledge to the
- * MAIN session's system prompt on `before_agent_start`. This hook does NOT
- * fire for crew workers: they are spawned with `--no-extensions`, so the
- * extension layer (and this hook) never loads in their process. Workers
- * instead receive knowledge via `buildKnowledgeFragment(task.cwd)` injected
- * into their prompt stablePrefix by `prompt-builder.ts`. Do NOT "fix" this
- * perceived gap by making the hook reach workers — it would cause
- * double-injection. (Verified by research workflow 2026-06-28.)
- *
- * The hook calls buildKnowledgeFragment(cwd) with NO query — so the main
- * session gets conventions-only (no session-log noise), which is the right
- * default for an interactive session that doesn't have a single task focus.
- * Workers (which DO have a task) use the section-aware path via prompt-builder.
- */
-export function registerKnowledgeInjection(pi: ExtensionAPI): void {
-	pi.on("before_agent_start", (event: BeforeAgentStartEvent) => {
-		const options =
-			(
-				event as BeforeAgentStartEvent & {
-					systemPromptOptions?: { cwd?: unknown };
-				}
-			).systemPromptOptions ?? {};
-		const cwd = typeof options.cwd === "string" ? options.cwd : process.cwd();
-		const fragment = buildKnowledgeFragment(cwd);
-		if (!fragment) return undefined;
-		return { systemPrompt: `${event.systemPrompt}${fragment}` };
-	});
-}