pi-harness-runtime 1.1.445 → 1.1.447

package/packages/context-compiler/src/compiler.ts

@@ -1,314 +0,0 @@
-/**
- * Context Compiler - Compiler
- *
- * Main entry point for context compilation.
- * Pipeline: collect → policy filter → score → deduplicate → fit budget → source graph.
- */
-
-import type {
-	ContextCompileRequest,
-	ContextSourceEdge,
-	OmissionReason,
-	OmittedItem,
-} from "./types.js";
-import { ContextCompileError } from "./types.js";
-
-import { estimateTokens, fitToBudget, toCompiledItems } from "./budget.js";
-import { generateCacheKey } from "./cache.js";
-import { deduplicateCandidates, mergeFileSlices } from "./deduplicate.js";
-import { applyPolicyFilter, mergePolicy } from "./filter.js";
-import { mergeWeights, rankCandidates } from "./score.js";
-import {
-	loadOkfConcepts,
-	okfDirectoryExists,
-	getOkfPath,
-} from "./okf-loader.js";
-
-/**
- * Default clock for generating timestamps.
- */
-export function createClock() {
-	return { now: () => new Date() };
-}
-
-export interface Clock {
-	now: () => Date;
-}
-
-/**
- * Compile context from candidates.
- *
- * Pipeline:
- * 1. Apply policy filter (deny patterns, size limits)
- * 2. Score optional candidates
- * 3. Merge file slices
- * 4. Deduplicate
- * 5. Fit to token budget
- * 6. Generate source graph
- * 7. Return CompiledContext
- */
-export async function compileContext(
-	request: ContextCompileRequest,
-	clock: Clock = createClock(),
-): Promise<CompiledContextOutput> {
-	const {
-		taskId,
-		taskObjective,
-		maximumTokens,
-		candidates,
-		scoringWeights,
-		policy,
-	} = request;
-
-	// 0. Validate required sources exist
-	const missingRequired = candidates
-		.filter((c) => c.required)
-		.filter((c) => !c.content.trim());
-
-	if (missingRequired.length > 0) {
-		const first = missingRequired[0];
-		if (!first)
-			throw new Error("Internal: missingRequired should not be empty");
-		throw new ContextCompileError(
-			"MISSING_REQUIRED_SOURCE",
-			`Required source "${first.source}" has no content`,
-			{
-				missingSources: missingRequired.map((c) => c.source),
-			},
-		);
-	}
-
-	// 1. Apply policy filter
-	const resolvedPolicy = mergePolicy(policy);
-	const { passed: policyPassed, denied: policyDenied } = applyPolicyFilter(
-		candidates,
-		resolvedPolicy,
-	);
-
-	const policyOmitted: OmittedItem[] = policyDenied.map(
-		({ candidate, reason }) => ({
-			id: candidate.id,
-			kind: candidate.kind,
-			reason: "policy_denied" as OmissionReason,
-			reasonDetail: reason,
-			estimatedTokens: estimateTokens(candidate.content),
-			deniedBy: reason,
-		}),
-	);
-
-	// 2. Score optional candidates
-	const weights = mergeWeights(scoringWeights);
-	const scoreContext = {
-		directFileReferences: extractDirectReferences(taskObjective),
-		taskDependencies: extractDependencies(taskObjective),
-	};
-
-	// 3. Merge file slices (before scoring)
-	const merged = mergeFileSlices(policyPassed);
-
-	// 4. Deduplicate
-	const { unique: deduped, duplicates: dupIds } = deduplicateCandidates(merged);
-
-	const dedupOmitted: OmittedItem[] = dupIds.map((id) => {
-		const c = candidates.find((x) => x.id === id);
-		return {
-			id,
-			kind: c?.kind ?? "source_file",
-			reason: "duplicate" as OmissionReason,
-			reasonDetail: `duplicate of ${c?.source ?? id}`,
-			estimatedTokens: estimateTokens(c?.content ?? ""),
-		};
-	});
-
-	// 5. Rank by score
-	const ranked = rankCandidates(deduped, weights, scoreContext);
-
-	// 6. Fit to budget
-	const { included, omitted: budgetOmitted } = fitToBudget(
-		ranked,
-		maximumTokens,
-	);
-
-	// 7. Convert to compiled items
-	const compiledItems = toCompiledItems(included);
-
-	// 8. Generate source graph
-	const sourceGraph = buildSourceGraph(compiledItems);
-
-	// 9. Generate cache key
-	const cacheKey = generateCacheKey(taskId, compiledItems, taskObjective);
-
-	// 10. Build mapping
-	const mapping: Record<string, string> = {};
-	for (const c of candidates) {
-		const compiled = compiledItems.find((i) => i.id === c.id);
-		if (compiled) {
-			mapping[c.id] = compiled.id;
-		}
-	}
-
-	const totalOmitted = [...policyOmitted, ...dedupOmitted, ...budgetOmitted];
-
-	return {
-		taskId,
-		items: compiledItems,
-		omitted: totalOmitted,
-		estimatedTokens: compiledItems.reduce(
-			(sum, item) => sum + item.estimatedTokens,
-			0,
-		),
-		sourceGraph,
-		generatedAt: clock.now().toISOString(),
-		mapping,
-		cacheKey,
-	};
-}
-
-/**
- * Compiled context output (extended with cache key).
- */
-export interface CompiledContextOutput {
-	taskId: string;
-	items: ReturnType<typeof toCompiledItems>[number][];
-	omitted: OmittedItem[];
-	estimatedTokens: number;
-	sourceGraph: ContextSourceEdge[];
-	generatedAt: string;
-	mapping: Record<string, string>;
-	cacheKey: string;
-}
-
-/**
- * Extract direct file references from task objective.
- * Looks for quoted paths, file patterns.
- */
-function extractDirectReferences(objective: string): Set<string> {
-	const refs = new Set<string>();
-	// Match quoted paths
-	const quotedMatches = objective.matchAll(/(["'`])([^`1]+)\1/g);
-	for (const match of quotedMatches) {
-		const path = match[2];
-		if (path) refs.add(path);
-	}
-	// Match common file patterns
-	const filePattern =
-		/[a-zA-Z][a-zA-Z0-9_/-]*\.(ts|tsx|js|jsx|py|go|rs|java|cpp|h|c|md|json|yaml|yml|toml)\b/g;
-	for (const match of objective.matchAll(filePattern)) {
-		const path = match[0];
-		if (path) refs.add(path);
-	}
-	return refs;
-}
-
-/**
- * Extract dependency references from task objective.
- */
-function extractDependencies(objective: string): string[] {
-	const deps: string[] = [];
-	// Match "depends on X", "related to Y"
-	const depPatterns = [
-		/(?:depends on|related to|imports from)\s+([A-Za-z][A-Za-z0-9_.-]*)/gi,
-	];
-	for (const pattern of depPatterns) {
-		for (const match of objective.matchAll(pattern)) {
-			const name = match[1];
-			if (name) deps.push(name);
-		}
-	}
-	return [...new Set(deps)];
-}
-
-/**
- * Build source graph edges between related items.
- */
-function buildSourceGraph(
-	items: ReturnType<typeof toCompiledItems>[number][],
-): ContextSourceEdge[] {
-	const edges: ContextSourceEdge[] = [];
-
-	const okfItems = items.filter((i) => i.kind === "okf_concept");
-	const sourceItems = items.filter((i) => i.kind === "source_file");
-	const testItems = items.filter((i) => i.kind === "test_failure");
-
-	// OKF concepts → source files they reference
-	for (const okf of okfItems) {
-		for (const src of sourceItems) {
-			if (
-				okf.content.includes(src.source) ||
-				(src.filePath && okf.content.includes(src.filePath))
-			) {
-				edges.push({
-					from: okf.id,
-					to: src.id,
-					label: "references",
-					weight: 3,
-				});
-			}
-		}
-	}
-
-	// Source files → test failures they relate to
-	for (const src of sourceItems) {
-		for (const test of testItems) {
-			if (
-				test.content.includes(src.source) ||
-				(src.filePath && test.content.includes(src.filePath))
-			) {
-				edges.push({
-					from: src.id,
-					to: test.id,
-					label: "causes_failure",
-					weight: 5,
-				});
-			}
-		}
-	}
-
-	// Sort edges deterministically
-	edges.sort((a, b) => {
-		const keyA = `${a.from}:${a.to}:${a.label}`;
-		const keyB = `${b.from}:${b.to}:${b.label}`;
-		return keyA.localeCompare(keyB);
-	});
-
-	return edges;
-}
-
-// --- OKF Integration --------------------------------------------------------
-
-/**
- * Enrich context candidates with OKF concepts from ~/.pi/okf/.
- *
- * This is optional - if the OKF directory doesn't exist,
- * candidates remain unchanged.
- *
- * Usage:
- * ```ts
- * const enriched = enrichWithOkf(candidates);
- * const result = await compileContext({ ...request, candidates: enriched });
- * ```
- */
-export function enrichWithOkf(
-	candidates: ContextCompileRequest["candidates"],
-	okfPath?: string,
-): ContextCompileRequest["candidates"] {
-	const okfConcepts = loadOkfConcepts(okfPath);
-
-	if (okfConcepts.length === 0) {
-		return candidates;
-	}
-
-	// Log for debugging (can be removed in production)
-	if (okfDirectoryExists(okfPath)) {
-		console.log(
-			`[OKF] Loaded ${okfConcepts.length} concepts from ${getOkfPath()}`,
-		);
-	}
-
-	// Merge OKF concepts with existing candidates
-	// Existing candidates take precedence (won't be overwritten)
-	const existingIds = new Set(candidates.map((c) => c.id));
-	const newConcepts = okfConcepts.filter((c) => !existingIds.has(c.id));
-
-	return [...candidates, ...newConcepts];
-}

package/packages/context-compiler/src/deduplicate.ts

@@ -1,171 +0,0 @@
-/**
- * Context Compiler - Deduplication
- *
- * Removes duplicate candidates and merges overlapping source file slices.
- * Uses content normalization for deduplication (same as RFC-0041).
- */
-
-import type { ContextCandidate } from "./types.js";
-
-/**
- * Deduplicate candidates by normalized content.
- * Required candidates are never deduplicated (preserved).
- *
- * For source files with overlapping ranges, merge into a single slice.
- *
- * @returns [uniqueCandidates, duplicateIds]
- */
-export function deduplicateCandidates(candidates: ContextCandidate[]): {
-	unique: ContextCandidate[];
-	duplicates: string[];
-} {
-	const seen = new Map<string, ContextCandidate>();
-	const duplicates: string[] = [];
-
-	for (const candidate of candidates) {
-		// Required items are never deduplicated
-		if (candidate.required) {
-			seen.set(candidate.id, candidate);
-			continue;
-		}
-
-		const normalized = normalizeContent(candidate.content);
-
-		if (seen.has(normalized)) {
-			duplicates.push(candidate.id);
-		} else {
-			seen.set(normalized, candidate);
-		}
-	}
-
-	return {
-		unique: [...seen.values()],
-		duplicates,
-	};
-}
-
-/**
- * Merge overlapping file slices from the same file.
- * Merges adjacent/overlapping line ranges and deduplicates symbols.
- */
-export function mergeFileSlices(
-	candidates: ContextCandidate[],
-): ContextCandidate[] {
-	// Group by file path
-	const byFile = new Map<string, ContextCandidate[]>();
-	for (const c of candidates) {
-		if (c.kind === "source_file" && c.filePath) {
-			const existing = byFile.get(c.filePath);
-			if (existing) {
-				existing.push(c);
-			} else {
-				byFile.set(c.filePath, [c]);
-			}
-		} else {
-			// Non-source-file candidates pass through
-		}
-	}
-
-	const result: ContextCandidate[] = [];
-
-	for (const candidate of candidates) {
-		if (candidate.kind === "source_file" && candidate.filePath) {
-			// Only process once (first candidate for each file)
-			const slices = byFile.get(candidate.filePath) ?? [];
-			if (slices[0] !== candidate) continue;
-
-			if (slices.length === 1) {
-				result.push(candidate);
-			} else {
-				// Merge multiple slices
-				result.push(mergeSlicesForFile(candidate.filePath, slices));
-			}
-		} else {
-			result.push(candidate);
-		}
-	}
-
-	return result;
-}
-
-/**
- * Merge multiple slices from the same file.
- */
-function mergeSlicesForFile(
-	filePath: string,
-	slices: ContextCandidate[],
-): ContextCandidate {
-	// Sort by startLine
-	const sorted = [...slices].sort(
-		(a, b) => (a.startLine ?? 0) - (b.startLine ?? 0),
-	);
-
-	const first = sorted[0] ?? {
-		id: "",
-		kind: "source_file",
-		content: "",
-		source: "",
-		priority: 0 as const,
-		required: false,
-		trust: "unverified",
-	};
-	let endLine = first.endLine ?? 0;
-	const allSymbols = new Set<string>();
-	const allSources = new Set<string>();
-
-	for (const slice of sorted) {
-		if (slice.endLine && slice.endLine > endLine) {
-			endLine = slice.endLine;
-		}
-		for (const sym of slice.symbols ?? []) {
-			allSymbols.add(sym);
-		}
-		allSources.add(slice.source);
-	}
-
-	// Combine content from all slices
-	const combinedContent = sorted.map((s) => s.content).join("\n...\n");
-
-	return {
-		id: first.id,
-		kind: "source_file",
-		content: combinedContent,
-		source: [...allSources].join("; "),
-		priority: Math.max(
-			...sorted.map((s) => s.priority),
-		) as ContextCandidate["priority"],
-		required: sorted.some((s) => s.required),
-		trust: sorted.find((s) => s.trust === "authoritative")
-			? "authoritative"
-			: sorted.find((s) => s.trust === "generated")
-				? "generated"
-				: "unverified",
-		filePath,
-		startLine: first.startLine,
-		endLine,
-		symbols: [...allSymbols],
-		contentHash: sorted[0]?.contentHash,
-	};
-}
-
-/**
- * Normalize content for deduplication comparison.
- * Matches the RFC-0041 deduplication formula.
- */
-function normalizeContent(text: string): string {
-	return collapseWhitespace(stripMarkdownDecoration(text.toLowerCase()));
-}
-
-function collapseWhitespace(text: string): string {
-	return text.replace(/\s+/g, " ").trim();
-}
-
-function stripMarkdownDecoration(text: string): string {
-	return text
-		.replace(/\[([^\]]+)\]\(([^)]*)\)/g, "$1")
-		.replace(/!\[([^\]]*)\]\(([^)]*)\)/g, "$1")
-		.replace(/[#*_`~>[\]]/g, "")
-		.replace(/```[\s\S]*?```/g, "")
-		.replace(/`([^`]+)`/g, "$1")
-		.trim();
-}

package/packages/context-compiler/src/filter.ts

@@ -1,135 +0,0 @@
-/**
- * Context Compiler - Policy Filter
- *
- * Enforces deny patterns and size limits before any other processing.
- * This runs first so denied content never reaches the cache.
- */
-
-import type { ContextCandidate, ContextPolicy } from "./types.js";
-import { DEFAULT_POLICY } from "./types.js";
-
-/**
- * Merge user policy with defaults.
- */
-export function mergePolicy(user?: Partial<ContextPolicy>): ContextPolicy {
-	if (!user) return { ...DEFAULT_POLICY };
-	return {
-		deny: user.deny ?? DEFAULT_POLICY.deny,
-		allowLargeFiles: user.allowLargeFiles ?? DEFAULT_POLICY.allowLargeFiles,
-		maxFileBytes: user.maxFileBytes ?? DEFAULT_POLICY.maxFileBytes,
-		trustThreshold: user.trustThreshold,
-	};
-}
-
-/**
- * Apply policy filters to candidates.
- * Denied and oversized candidates are marked as omitted.
- * This is the first stage — nothing else runs before this.
- *
- * @returns [passed, denied] - denied candidates with policy_denied reason
- */
-export function applyPolicyFilter(
-	candidates: ContextCandidate[],
-	policy: ContextPolicy,
-): {
-	passed: ContextCandidate[];
-	denied: Array<{ candidate: ContextCandidate; reason: string }>;
-} {
-	const passed: ContextCandidate[] = [];
-	const denied: Array<{ candidate: ContextCandidate; reason: string }> = [];
-
-	for (const candidate of candidates) {
-		// Check deny patterns
-		const denyReason = matchesDenyPattern(candidate, policy.deny);
-		if (denyReason) {
-			denied.push({ candidate, reason: denyReason });
-			continue;
-		}
-
-		// Check file size limit
-		if (candidate.filePath && candidate.content.length > policy.maxFileBytes) {
-			if (!policy.allowLargeFiles) {
-				denied.push({
-					candidate,
-					reason: `file exceeds ${policy.maxFileBytes} bytes`,
-				});
-				continue;
-			}
-		}
-
-		// Check trust threshold
-		if (
-			policy.trustThreshold &&
-			!meetsTrustThreshold(candidate.trust, policy.trustThreshold)
-		) {
-			denied.push({
-				candidate,
-				reason: `trust level "${candidate.trust}" below threshold`,
-			});
-			continue;
-		}
-
-		passed.push(candidate);
-	}
-
-	return { passed, denied };
-}
-
-/**
- * Check if a candidate matches any deny glob pattern.
- */
-function matchesDenyPattern(
-	candidate: ContextCandidate,
-	denyPatterns: string[],
-): string | null {
-	const path = candidate.filePath ?? candidate.source;
-
-	for (const pattern of denyPatterns) {
-		if (globMatch(pattern, path)) {
-			return `denied by pattern: ${pattern}`;
-		}
-	}
-
-	return null;
-}
-
-// Simple glob matching for **/*.env style patterns.
-// Handles: *, **, and literal path separators.
-function globMatch(pattern: string, path: string): boolean {
-	// Normalize: ensure we match relative paths like .env or /project/.env
-	const normalizedPath = path.replace(/^\.\//, "");
-	const normalizedPattern = pattern.replace(/^\*\//, "**/");
-
-	if (normalizedPattern === "**") return true;
-
-	// Convert glob to regex
-	const regexStr = normalizedPattern
-		.replace(/\./g, "\\.") // escape dots
-		.replace(/\*\*/g, "{{DOUBLE_STAR}}")
-		.replace(/\*/g, "[^/]*")
-		.replace(/\{\{DOUBLE_STAR\}\}/g, ".*")
-		.replace(/\?/g, "."); // single char
-
-	try {
-		const regex = new RegExp(`^${regexStr}$`);
-		return regex.test(normalizedPath) || regex.test(`/${normalizedPath}`);
-	} catch {
-		return false;
-	}
-}
-
-/**
- * Check if candidate trust meets the threshold.
- * "authoritative" > "generated" > "unverified"
- */
-function meetsTrustThreshold(
-	trust: ContextCandidate["trust"],
-	threshold: ContextCandidate["trust"],
-): boolean {
-	const levels: Record<ContextCandidate["trust"], number> = {
-		authoritative: 3,
-		generated: 2,
-		unverified: 1,
-	};
-	return levels[trust] >= levels[threshold];
-}

package/packages/context-compiler/src/index.ts

@@ -1,61 +0,0 @@
-/**
- * Context Compiler - Public API
- *
- * Bounded context selection engine for pi-harness-runtime.
- *
- * @example
- * import { compileContext } from "@pi/context-compiler";
- *
- * const ctx = await compileContext({
- *   taskId: "task-001",
- *   taskObjective: "Add login feature to src/auth/login.ts",
- *   maximumTokens: 50_000,
- *   candidates: [...],
- * });
- */
-
-export { compileContext } from "./compiler.js";
-export { createClock } from "./compiler.js";
-export type { Clock, CompiledContextOutput } from "./compiler.js";
-
-export type {
-	ContextKind,
-	ContextTrust,
-	ContextCandidate,
-	OmissionReason,
-	OmittedItem,
-	CompiledContextItem,
-	ContextSourceEdge,
-	CompiledContext,
-	ScoringWeights,
-	ContextPolicy,
-	ContextCompileRequest,
-	InvalidationReason,
-	ContextInvalidation,
-	ContextCompileErrorCode,
-} from "./types.js";
-
-export { DEFAULT_SCORING_WEIGHTS, DEFAULT_POLICY } from "./types.js";
-export { ContextCompileError } from "./types.js";
-
-export { mergePolicy, applyPolicyFilter } from "./filter.js";
-export { mergeWeights, rankCandidates } from "./score.js";
-export {
-	deduplicateCandidates,
-	mergeFileSlices,
-} from "./deduplicate.js";
-export { fitToBudget, estimateTokens } from "./budget.js";
-export {
-	generateCacheKey,
-	shouldInvalidate,
-	extractItemHashes,
-} from "./cache.js";
-
-// OKF integration
-export { enrichWithOkf } from "./compiler.js";
-export {
-	loadOkfConcepts,
-	okfDirectoryExists,
-	getOkfPath,
-} from "./okf-loader.js";
-export type { OkfConcept } from "./okf-loader.js";