pi-harness-runtime 0.10.11 → 0.10.12

package/packages/context-compiler/src/compiler.ts

@@ -0,0 +1,314 @@
+/**
+ * Context Compiler - Compiler
+ *
+ * Main entry point for context compilation.
+ * Pipeline: collect → policy filter → score → deduplicate → fit budget → source graph.
+ */
+
+import type {
+	ContextCompileRequest,
+	ContextSourceEdge,
+	OmissionReason,
+	OmittedItem,
+} from "./types.js";
+import { ContextCompileError } from "./types.js";
+
+import { estimateTokens, fitToBudget, toCompiledItems } from "./budget.js";
+import { generateCacheKey } from "./cache.js";
+import { deduplicateCandidates, mergeFileSlices } from "./deduplicate.js";
+import { applyPolicyFilter, mergePolicy } from "./filter.js";
+import { mergeWeights, rankCandidates } from "./score.js";
+import {
+	loadOkfConcepts,
+	okfDirectoryExists,
+	getOkfPath,
+} from "./okf-loader.js";
+
+/**
+ * Default clock for generating timestamps.
+ */
+export function createClock() {
+	return { now: () => new Date() };
+}
+
+export interface Clock {
+	now: () => Date;
+}
+
+/**
+ * Compile context from candidates.
+ *
+ * Pipeline:
+ * 1. Apply policy filter (deny patterns, size limits)
+ * 2. Score optional candidates
+ * 3. Merge file slices
+ * 4. Deduplicate
+ * 5. Fit to token budget
+ * 6. Generate source graph
+ * 7. Return CompiledContext
+ */
+export async function compileContext(
+	request: ContextCompileRequest,
+	clock: Clock = createClock(),
+): Promise<CompiledContextOutput> {
+	const {
+		taskId,
+		taskObjective,
+		maximumTokens,
+		candidates,
+		scoringWeights,
+		policy,
+	} = request;
+
+	// 0. Validate required sources exist
+	const missingRequired = candidates
+		.filter((c) => c.required)
+		.filter((c) => !c.content.trim());
+
+	if (missingRequired.length > 0) {
+		const first = missingRequired[0];
+		if (!first)
+			throw new Error("Internal: missingRequired should not be empty");
+		throw new ContextCompileError(
+			"MISSING_REQUIRED_SOURCE",
+			`Required source "${first.source}" has no content`,
+			{
+				missingSources: missingRequired.map((c) => c.source),
+			},
+		);
+	}
+
+	// 1. Apply policy filter
+	const resolvedPolicy = mergePolicy(policy);
+	const { passed: policyPassed, denied: policyDenied } = applyPolicyFilter(
+		candidates,
+		resolvedPolicy,
+	);
+
+	const policyOmitted: OmittedItem[] = policyDenied.map(
+		({ candidate, reason }) => ({
+			id: candidate.id,
+			kind: candidate.kind,
+			reason: "policy_denied" as OmissionReason,
+			reasonDetail: reason,
+			estimatedTokens: estimateTokens(candidate.content),
+			deniedBy: reason,
+		}),
+	);
+
+	// 2. Score optional candidates
+	const weights = mergeWeights(scoringWeights);
+	const scoreContext = {
+		directFileReferences: extractDirectReferences(taskObjective),
+		taskDependencies: extractDependencies(taskObjective),
+	};
+
+	// 3. Merge file slices (before scoring)
+	const merged = mergeFileSlices(policyPassed);
+
+	// 4. Deduplicate
+	const { unique: deduped, duplicates: dupIds } = deduplicateCandidates(merged);
+
+	const dedupOmitted: OmittedItem[] = dupIds.map((id) => {
+		const c = candidates.find((x) => x.id === id);
+		return {
+			id,
+			kind: c?.kind ?? "source_file",
+			reason: "duplicate" as OmissionReason,
+			reasonDetail: `duplicate of ${c?.source ?? id}`,
+			estimatedTokens: estimateTokens(c?.content ?? ""),
+		};
+	});
+
+	// 5. Rank by score
+	const ranked = rankCandidates(deduped, weights, scoreContext);
+
+	// 6. Fit to budget
+	const { included, omitted: budgetOmitted } = fitToBudget(
+		ranked,
+		maximumTokens,
+	);
+
+	// 7. Convert to compiled items
+	const compiledItems = toCompiledItems(included);
+
+	// 8. Generate source graph
+	const sourceGraph = buildSourceGraph(compiledItems);
+
+	// 9. Generate cache key
+	const cacheKey = generateCacheKey(taskId, compiledItems, taskObjective);
+
+	// 10. Build mapping
+	const mapping: Record<string, string> = {};
+	for (const c of candidates) {
+		const compiled = compiledItems.find((i) => i.id === c.id);
+		if (compiled) {
+			mapping[c.id] = compiled.id;
+		}
+	}
+
+	const totalOmitted = [...policyOmitted, ...dedupOmitted, ...budgetOmitted];
+
+	return {
+		taskId,
+		items: compiledItems,
+		omitted: totalOmitted,
+		estimatedTokens: compiledItems.reduce(
+			(sum, item) => sum + item.estimatedTokens,
+			0,
+		),
+		sourceGraph,
+		generatedAt: clock.now().toISOString(),
+		mapping,
+		cacheKey,
+	};
+}
+
+/**
+ * Compiled context output (extended with cache key).
+ */
+export interface CompiledContextOutput {
+	taskId: string;
+	items: ReturnType<typeof toCompiledItems>[number][];
+	omitted: OmittedItem[];
+	estimatedTokens: number;
+	sourceGraph: ContextSourceEdge[];
+	generatedAt: string;
+	mapping: Record<string, string>;
+	cacheKey: string;
+}
+
+/**
+ * Extract direct file references from task objective.
+ * Looks for quoted paths, file patterns.
+ */
+function extractDirectReferences(objective: string): Set<string> {
+	const refs = new Set<string>();
+	// Match quoted paths
+	const quotedMatches = objective.matchAll(/(["'`])([^`1]+)\1/g);
+	for (const match of quotedMatches) {
+		const path = match[2];
+		if (path) refs.add(path);
+	}
+	// Match common file patterns
+	const filePattern =
+		/[a-zA-Z][a-zA-Z0-9_/-]*\.(ts|tsx|js|jsx|py|go|rs|java|cpp|h|c|md|json|yaml|yml|toml)\b/g;
+	for (const match of objective.matchAll(filePattern)) {
+		const path = match[0];
+		if (path) refs.add(path);
+	}
+	return refs;
+}
+
+/**
+ * Extract dependency references from task objective.
+ */
+function extractDependencies(objective: string): string[] {
+	const deps: string[] = [];
+	// Match "depends on X", "related to Y"
+	const depPatterns = [
+		/(?:depends on|related to|imports from)\s+([A-Za-z][A-Za-z0-9_.-]*)/gi,
+	];
+	for (const pattern of depPatterns) {
+		for (const match of objective.matchAll(pattern)) {
+			const name = match[1];
+			if (name) deps.push(name);
+		}
+	}
+	return [...new Set(deps)];
+}
+
+/**
+ * Build source graph edges between related items.
+ */
+function buildSourceGraph(
+	items: ReturnType<typeof toCompiledItems>[number][],
+): ContextSourceEdge[] {
+	const edges: ContextSourceEdge[] = [];
+
+	const okfItems = items.filter((i) => i.kind === "okf_concept");
+	const sourceItems = items.filter((i) => i.kind === "source_file");
+	const testItems = items.filter((i) => i.kind === "test_failure");
+
+	// OKF concepts → source files they reference
+	for (const okf of okfItems) {
+		for (const src of sourceItems) {
+			if (
+				okf.content.includes(src.source) ||
+				(src.filePath && okf.content.includes(src.filePath))
+			) {
+				edges.push({
+					from: okf.id,
+					to: src.id,
+					label: "references",
+					weight: 3,
+				});
+			}
+		}
+	}
+
+	// Source files → test failures they relate to
+	for (const src of sourceItems) {
+		for (const test of testItems) {
+			if (
+				test.content.includes(src.source) ||
+				(src.filePath && test.content.includes(src.filePath))
+			) {
+				edges.push({
+					from: src.id,
+					to: test.id,
+					label: "causes_failure",
+					weight: 5,
+				});
+			}
+		}
+	}
+
+	// Sort edges deterministically
+	edges.sort((a, b) => {
+		const keyA = `${a.from}:${a.to}:${a.label}`;
+		const keyB = `${b.from}:${b.to}:${b.label}`;
+		return keyA.localeCompare(keyB);
+	});
+
+	return edges;
+}
+
+// --- OKF Integration --------------------------------------------------------
+
+/**
+ * Enrich context candidates with OKF concepts from ~/.pi/okf/.
+ *
+ * This is optional - if the OKF directory doesn't exist,
+ * candidates remain unchanged.
+ *
+ * Usage:
+ * ```ts
+ * const enriched = enrichWithOkf(candidates);
+ * const result = await compileContext({ ...request, candidates: enriched });
+ * ```
+ */
+export function enrichWithOkf(
+	candidates: ContextCompileRequest["candidates"],
+	okfPath?: string,
+): ContextCompileRequest["candidates"] {
+	const okfConcepts = loadOkfConcepts(okfPath);
+
+	if (okfConcepts.length === 0) {
+		return candidates;
+	}
+
+	// Log for debugging (can be removed in production)
+	if (okfDirectoryExists(okfPath)) {
+		console.log(
+			`[OKF] Loaded ${okfConcepts.length} concepts from ${getOkfPath()}`,
+		);
+	}
+
+	// Merge OKF concepts with existing candidates
+	// Existing candidates take precedence (won't be overwritten)
+	const existingIds = new Set(candidates.map((c) => c.id));
+	const newConcepts = okfConcepts.filter((c) => !existingIds.has(c.id));
+
+	return [...candidates, ...newConcepts];
+}

package/packages/context-compiler/src/deduplicate.ts

@@ -0,0 +1,171 @@
+/**
+ * Context Compiler - Deduplication
+ *
+ * Removes duplicate candidates and merges overlapping source file slices.
+ * Uses content normalization for deduplication (same as RFC-0041).
+ */
+
+import type { ContextCandidate } from "./types.js";
+
+/**
+ * Deduplicate candidates by normalized content.
+ * Required candidates are never deduplicated (preserved).
+ *
+ * For source files with overlapping ranges, merge into a single slice.
+ *
+ * @returns [uniqueCandidates, duplicateIds]
+ */
+export function deduplicateCandidates(candidates: ContextCandidate[]): {
+	unique: ContextCandidate[];
+	duplicates: string[];
+} {
+	const seen = new Map<string, ContextCandidate>();
+	const duplicates: string[] = [];
+
+	for (const candidate of candidates) {
+		// Required items are never deduplicated
+		if (candidate.required) {
+			seen.set(candidate.id, candidate);
+			continue;
+		}
+
+		const normalized = normalizeContent(candidate.content);
+
+		if (seen.has(normalized)) {
+			duplicates.push(candidate.id);
+		} else {
+			seen.set(normalized, candidate);
+		}
+	}
+
+	return {
+		unique: [...seen.values()],
+		duplicates,
+	};
+}
+
+/**
+ * Merge overlapping file slices from the same file.
+ * Merges adjacent/overlapping line ranges and deduplicates symbols.
+ */
+export function mergeFileSlices(
+	candidates: ContextCandidate[],
+): ContextCandidate[] {
+	// Group by file path
+	const byFile = new Map<string, ContextCandidate[]>();
+	for (const c of candidates) {
+		if (c.kind === "source_file" && c.filePath) {
+			const existing = byFile.get(c.filePath);
+			if (existing) {
+				existing.push(c);
+			} else {
+				byFile.set(c.filePath, [c]);
+			}
+		} else {
+			// Non-source-file candidates pass through
+		}
+	}
+
+	const result: ContextCandidate[] = [];
+
+	for (const candidate of candidates) {
+		if (candidate.kind === "source_file" && candidate.filePath) {
+			// Only process once (first candidate for each file)
+			const slices = byFile.get(candidate.filePath) ?? [];
+			if (slices[0] !== candidate) continue;
+
+			if (slices.length === 1) {
+				result.push(candidate);
+			} else {
+				// Merge multiple slices
+				result.push(mergeSlicesForFile(candidate.filePath, slices));
+			}
+		} else {
+			result.push(candidate);
+		}
+	}
+
+	return result;
+}
+
+/**
+ * Merge multiple slices from the same file.
+ */
+function mergeSlicesForFile(
+	filePath: string,
+	slices: ContextCandidate[],
+): ContextCandidate {
+	// Sort by startLine
+	const sorted = [...slices].sort(
+		(a, b) => (a.startLine ?? 0) - (b.startLine ?? 0),
+	);
+
+	const first = sorted[0] ?? {
+		id: "",
+		kind: "source_file",
+		content: "",
+		source: "",
+		priority: 0 as const,
+		required: false,
+		trust: "unverified",
+	};
+	let endLine = first.endLine ?? 0;
+	const allSymbols = new Set<string>();
+	const allSources = new Set<string>();
+
+	for (const slice of sorted) {
+		if (slice.endLine && slice.endLine > endLine) {
+			endLine = slice.endLine;
+		}
+		for (const sym of slice.symbols ?? []) {
+			allSymbols.add(sym);
+		}
+		allSources.add(slice.source);
+	}
+
+	// Combine content from all slices
+	const combinedContent = sorted.map((s) => s.content).join("\n...\n");
+
+	return {
+		id: first.id,
+		kind: "source_file",
+		content: combinedContent,
+		source: [...allSources].join("; "),
+		priority: Math.max(
+			...sorted.map((s) => s.priority),
+		) as ContextCandidate["priority"],
+		required: sorted.some((s) => s.required),
+		trust: sorted.find((s) => s.trust === "authoritative")
+			? "authoritative"
+			: sorted.find((s) => s.trust === "generated")
+				? "generated"
+				: "unverified",
+		filePath,
+		startLine: first.startLine,
+		endLine,
+		symbols: [...allSymbols],
+		contentHash: sorted[0]?.contentHash,
+	};
+}
+
+/**
+ * Normalize content for deduplication comparison.
+ * Matches the RFC-0041 deduplication formula.
+ */
+function normalizeContent(text: string): string {
+	return collapseWhitespace(stripMarkdownDecoration(text.toLowerCase()));
+}
+
+function collapseWhitespace(text: string): string {
+	return text.replace(/\s+/g, " ").trim();
+}
+
+function stripMarkdownDecoration(text: string): string {
+	return text
+		.replace(/\[([^\]]+)\]\(([^)]*)\)/g, "$1")
+		.replace(/!\[([^\]]*)\]\(([^)]*)\)/g, "$1")
+		.replace(/[#*_`~>[\]]/g, "")
+		.replace(/```[\s\S]*?```/g, "")
+		.replace(/`([^`]+)`/g, "$1")
+		.trim();
+}

package/packages/context-compiler/src/filter.ts

@@ -0,0 +1,135 @@
+/**
+ * Context Compiler - Policy Filter
+ *
+ * Enforces deny patterns and size limits before any other processing.
+ * This runs first so denied content never reaches the cache.
+ */
+
+import type { ContextCandidate, ContextPolicy } from "./types.js";
+import { DEFAULT_POLICY } from "./types.js";
+
+/**
+ * Merge user policy with defaults.
+ */
+export function mergePolicy(user?: Partial<ContextPolicy>): ContextPolicy {
+	if (!user) return { ...DEFAULT_POLICY };
+	return {
+		deny: user.deny ?? DEFAULT_POLICY.deny,
+		allowLargeFiles: user.allowLargeFiles ?? DEFAULT_POLICY.allowLargeFiles,
+		maxFileBytes: user.maxFileBytes ?? DEFAULT_POLICY.maxFileBytes,
+		trustThreshold: user.trustThreshold,
+	};
+}
+
+/**
+ * Apply policy filters to candidates.
+ * Denied and oversized candidates are marked as omitted.
+ * This is the first stage — nothing else runs before this.
+ *
+ * @returns [passed, denied] - denied candidates with policy_denied reason
+ */
+export function applyPolicyFilter(
+	candidates: ContextCandidate[],
+	policy: ContextPolicy,
+): {
+	passed: ContextCandidate[];
+	denied: Array<{ candidate: ContextCandidate; reason: string }>;
+} {
+	const passed: ContextCandidate[] = [];
+	const denied: Array<{ candidate: ContextCandidate; reason: string }> = [];
+
+	for (const candidate of candidates) {
+		// Check deny patterns
+		const denyReason = matchesDenyPattern(candidate, policy.deny);
+		if (denyReason) {
+			denied.push({ candidate, reason: denyReason });
+			continue;
+		}
+
+		// Check file size limit
+		if (candidate.filePath && candidate.content.length > policy.maxFileBytes) {
+			if (!policy.allowLargeFiles) {
+				denied.push({
+					candidate,
+					reason: `file exceeds ${policy.maxFileBytes} bytes`,
+				});
+				continue;
+			}
+		}
+
+		// Check trust threshold
+		if (
+			policy.trustThreshold &&
+			!meetsTrustThreshold(candidate.trust, policy.trustThreshold)
+		) {
+			denied.push({
+				candidate,
+				reason: `trust level "${candidate.trust}" below threshold`,
+			});
+			continue;
+		}
+
+		passed.push(candidate);
+	}
+
+	return { passed, denied };
+}
+
+/**
+ * Check if a candidate matches any deny glob pattern.
+ */
+function matchesDenyPattern(
+	candidate: ContextCandidate,
+	denyPatterns: string[],
+): string | null {
+	const path = candidate.filePath ?? candidate.source;
+
+	for (const pattern of denyPatterns) {
+		if (globMatch(pattern, path)) {
+			return `denied by pattern: ${pattern}`;
+		}
+	}
+
+	return null;
+}
+
+// Simple glob matching for **/*.env style patterns.
+// Handles: *, **, and literal path separators.
+function globMatch(pattern: string, path: string): boolean {
+	// Normalize: ensure we match relative paths like .env or /project/.env
+	const normalizedPath = path.replace(/^\.\//, "");
+	const normalizedPattern = pattern.replace(/^\*\//, "**/");
+
+	if (normalizedPattern === "**") return true;
+
+	// Convert glob to regex
+	const regexStr = normalizedPattern
+		.replace(/\./g, "\\.") // escape dots
+		.replace(/\*\*/g, "{{DOUBLE_STAR}}")
+		.replace(/\*/g, "[^/]*")
+		.replace(/\{\{DOUBLE_STAR\}\}/g, ".*")
+		.replace(/\?/g, "."); // single char
+
+	try {
+		const regex = new RegExp(`^${regexStr}$`);
+		return regex.test(normalizedPath) || regex.test(`/${normalizedPath}`);
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Check if candidate trust meets the threshold.
+ * "authoritative" > "generated" > "unverified"
+ */
+function meetsTrustThreshold(
+	trust: ContextCandidate["trust"],
+	threshold: ContextCandidate["trust"],
+): boolean {
+	const levels: Record<ContextCandidate["trust"], number> = {
+		authoritative: 3,
+		generated: 2,
+		unverified: 1,
+	};
+	return levels[trust] >= levels[threshold];
+}

package/packages/context-compiler/src/index.ts

@@ -0,0 +1,61 @@
+/**
+ * Context Compiler - Public API
+ *
+ * Bounded context selection engine for pi-harness-runtime.
+ *
+ * @example
+ * import { compileContext } from "@pi/context-compiler";
+ *
+ * const ctx = await compileContext({
+ *   taskId: "task-001",
+ *   taskObjective: "Add login feature to src/auth/login.ts",
+ *   maximumTokens: 50_000,
+ *   candidates: [...],
+ * });
+ */
+
+export { compileContext } from "./compiler.js";
+export { createClock } from "./compiler.js";
+export type { Clock, CompiledContextOutput } from "./compiler.js";
+
+export type {
+	ContextKind,
+	ContextTrust,
+	ContextCandidate,
+	OmissionReason,
+	OmittedItem,
+	CompiledContextItem,
+	ContextSourceEdge,
+	CompiledContext,
+	ScoringWeights,
+	ContextPolicy,
+	ContextCompileRequest,
+	InvalidationReason,
+	ContextInvalidation,
+	ContextCompileErrorCode,
+} from "./types.js";
+
+export { DEFAULT_SCORING_WEIGHTS, DEFAULT_POLICY } from "./types.js";
+export { ContextCompileError } from "./types.js";
+
+export { mergePolicy, applyPolicyFilter } from "./filter.js";
+export { mergeWeights, rankCandidates } from "./score.js";
+export {
+	deduplicateCandidates,
+	mergeFileSlices,
+} from "./deduplicate.js";
+export { fitToBudget, estimateTokens } from "./budget.js";
+export {
+	generateCacheKey,
+	shouldInvalidate,
+	extractItemHashes,
+} from "./cache.js";
+
+// OKF integration
+export { enrichWithOkf } from "./compiler.js";
+export {
+	loadOkfConcepts,
+	okfDirectoryExists,
+	getOkfPath,
+} from "./okf-loader.js";
+export type { OkfConcept } from "./okf-loader.js";