pi-harness-runtime 0.5.0 → 0.6.3-beta.13

package/packages/context-compiler/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@pi/context-compiler",
+  "version": "0.6.3-beta.13",
+  "description": "Bounded context selection engine for pi-harness-runtime",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "test": "bun test",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@pi-harness/requirement-compiler": "workspace:*",
+    "@pi-harness/task-compiler": "workspace:*",
+    "@pi-harness/types": "workspace:*"
+  },
+  "devDependencies": {
+    "bun-types": "^1.0.0",
+    "@biomejs/biome": "^1.9.0",
+    "@types/node": "^20.0.0"
+  }
+}

package/packages/context-compiler/src/budget.ts

@@ -0,0 +1,122 @@
+/**
+ * Context Compiler - Token Budget Fitting
+ *
+ * Fits optional candidates into the available token budget.
+ * Required candidates always fit first.
+ * Uses the RFC-0041 character-based estimation (1 token ≈ 4 characters).
+ */
+
+import type {
+	CompiledContextItem,
+	ContextCandidate,
+	OmittedItem,
+} from "./types.js";
+
+/**
+ * Result of budget fitting.
+ */
+export interface BudgetFitResult {
+	included: ContextCandidate[];
+	omitted: OmittedItem[];
+	estimatedTokens: number;
+}
+
+/**
+ * Estimate tokens from content using 4 chars/token approximation.
+ */
+export function estimateTokens(content: string): number {
+	if (!content || content.trim() === "") return 0;
+	return Math.ceil(content.length / 4);
+}
+
+/**
+ * Fit candidates into the token budget.
+ *
+ * Algorithm:
+ * 1. Always include required candidates
+ * 2. Sort optional by score (descending)
+ * 3. Greedily add optional until budget exhausted
+ * 4. Record all omitted with reason "budget"
+ *
+ * @param candidates - Ranked candidates (highest score first)
+ * @param maximumTokens - Available budget
+ * @returns Candidates to include and items to omit
+ */
+export function fitToBudget(
+	candidates: Array<{ candidate: ContextCandidate; score: number }>,
+	maximumTokens: number,
+): BudgetFitResult {
+	const included: ContextCandidate[] = [];
+	const omitted: OmittedItem[] = [];
+
+	let usedTokens = 0;
+
+	for (const { candidate } of candidates) {
+		const tokens = estimateTokens(candidate.content);
+
+		// Optional items with no content are omitted (broken/unavailable)
+		if (
+			!candidate.required &&
+			(!candidate.content || candidate.content.trim() === "")
+		) {
+			omitted.push({
+				id: candidate.id,
+				kind: candidate.kind,
+				reason: "source_unavailable",
+				reasonDetail: "no content",
+				estimatedTokens: 0,
+			});
+			continue;
+		}
+
+		if (candidate.required) {
+			// Required: always include
+			included.push(candidate);
+			usedTokens += tokens;
+			continue;
+		}
+
+		// Optional: check budget
+		if (usedTokens + tokens <= maximumTokens) {
+			included.push(candidate);
+			usedTokens += tokens;
+		} else {
+			omitted.push({
+				id: candidate.id,
+				kind: candidate.kind,
+				reason: "budget",
+				reasonDetail: `estimated ${tokens} tokens, only ${maximumTokens - usedTokens} remaining`,
+				estimatedTokens: tokens,
+			});
+		}
+	}
+
+	return {
+		included,
+		omitted,
+		estimatedTokens: usedTokens,
+	};
+}
+
+/**
+ * Convert included candidates to compiled items.
+ */
+export function toCompiledItems(
+	candidates: ContextCandidate[],
+): CompiledContextItem[] {
+	return candidates.map((c) => ({
+		id: c.id,
+		kind: c.kind,
+		content: c.content,
+		source: c.source,
+		priority: c.priority,
+		required: c.required,
+		trust: c.trust,
+		estimatedTokens: estimateTokens(c.content),
+		contentHash: c.contentHash,
+		filePath: c.filePath,
+		startLine: c.startLine,
+		endLine: c.endLine,
+		symbols: c.symbols,
+	}));
+}

package/packages/context-compiler/src/cache.ts

@@ -0,0 +1,134 @@
+/**
+ * Context Compiler - Cache & Invalidation
+ *
+ * Generates deterministic cache keys from selected source hashes.
+ * Tracks invalidation triggers.
+ */
+
+import type {
+	CompiledContext,
+	CompiledContextItem,
+	ContextInvalidation,
+} from "./types.js";
+
+/**
+ * Generate a deterministic cache key from compiled context items.
+ * Includes source file hashes for content invalidation.
+ */
+export function generateCacheKey(
+	taskId: string,
+	items: CompiledContextItem[],
+	taskObjective: string,
+): string {
+	// Sort by source path for determinism
+	const hashes = items
+		.filter((item) => item.contentHash)
+		.map((item) => `${item.source}:${item.contentHash}`)
+		.sort();
+
+	const objectiveHash = simpleHash(taskObjective);
+	const combined = `${taskId}:${objectiveHash}:${hashes.join(",")}`;
+
+	return `ctx:${simpleHash(combined)}`;
+}
+
+/**
+ * Check if a compiled context should be invalidated.
+ *
+ * Invalidation triggers:
+ * - A selected source file hash changed
+ * - A project rule changed
+ * - A required OKF concept changed
+ * - The task objective changed
+ * - A test failure superseded an old one
+ * - Worktree branch/HEAD changed
+ */
+export function shouldInvalidate(
+	previous: CompiledContext,
+	invalidation: ContextInvalidation,
+	currentItemHashes: Map<string, string>,
+): boolean {
+	switch (invalidation.reason) {
+		case "source_hash_changed": {
+			const changed = invalidation.details?.sourcePath as string | undefined;
+			if (!changed) return true;
+			const item = previous.items.find((i) => i.source === changed);
+			if (!item) return false;
+			const currentHash = currentItemHashes.get(changed);
+			return currentHash !== item.contentHash;
+		}
+
+		case "project_rule_changed": {
+			const rulePath = invalidation.details?.rulePath as string | undefined;
+			const ruleItem = previous.items.find(
+				(i) =>
+					i.kind === "project_rule" &&
+					(i.source === rulePath || i.filePath === rulePath),
+			);
+			return ruleItem !== undefined;
+		}
+
+		case "required_okf_changed": {
+			const okfId = invalidation.details?.okfId as string | undefined;
+			return (
+				previous.items.find(
+					(i) => i.kind === "okf_concept" && i.id === okfId && i.required,
+				) !== undefined
+			);
+		}
+
+		case "task_objective_changed": {
+			return true; // Always invalidate on objective change
+		}
+
+		case "test_failure_superseded": {
+			const oldFailure = invalidation.details?.oldFailureId as
+				| string
+				| undefined;
+			if (!oldFailure) return false;
+			const existingFailure = previous.items.find(
+				(i) => i.kind === "test_failure" && i.id === oldFailure,
+			);
+			return existingFailure !== undefined;
+		}
+
+		case "worktree_branch_changed": {
+			const oldBranch = invalidation.details?.oldBranch as string | undefined;
+			const newBranch = invalidation.details?.newBranch as string | undefined;
+			// Invalidate if any source file changed or if branch changed
+			return oldBranch !== newBranch;
+		}
+
+		default:
+			return false;
+	}
+}
+
+/**
+ * Generate current item hashes from a compiled context.
+ */
+export function extractItemHashes(
+	items: CompiledContextItem[],
+): Map<string, string> {
+	const map = new Map<string, string>();
+	for (const item of items) {
+		if (item.contentHash) {
+			map.set(item.source, item.contentHash);
+		}
+	}
+	return map;
+}
+
+/**
+ * Simple deterministic hash for strings.
+ * Used for cache keys when crypto is unavailable.
+ */
+export function simpleHash(text: string): string {
+	let hash = 0;
+	for (let i = 0; i < text.length; i++) {
+		const char = text.charCodeAt(i);
+		hash = (hash << 5) - hash + char;
+		hash |= 0;
+	}
+	return Math.abs(hash).toString(16).padStart(8, "0");
+}

package/packages/context-compiler/src/compiler.ts

@@ -0,0 +1,314 @@
+/**
+ * Context Compiler - Compiler
+ *
+ * Main entry point for context compilation.
+ * Pipeline: collect → policy filter → score → deduplicate → fit budget → source graph.
+ */
+
+import type {
+	ContextCompileRequest,
+	ContextSourceEdge,
+	OmissionReason,
+	OmittedItem,
+} from "./types.js";
+import { ContextCompileError } from "./types.js";
+
+import { estimateTokens, fitToBudget, toCompiledItems } from "./budget.js";
+import { generateCacheKey } from "./cache.js";
+import { deduplicateCandidates, mergeFileSlices } from "./deduplicate.js";
+import { applyPolicyFilter, mergePolicy } from "./filter.js";
+import { mergeWeights, rankCandidates } from "./score.js";
+import {
+	loadOkfConcepts,
+	okfDirectoryExists,
+	getOkfPath,
+} from "./okf-loader.js";
+
+/**
+ * Default clock for generating timestamps.
+ */
+export function createClock() {
+	return { now: () => new Date() };
+}
+
+export interface Clock {
+	now: () => Date;
+}
+
+/**
+ * Compile context from candidates.
+ *
+ * Pipeline:
+ * 1. Apply policy filter (deny patterns, size limits)
+ * 2. Score optional candidates
+ * 3. Merge file slices
+ * 4. Deduplicate
+ * 5. Fit to token budget
+ * 6. Generate source graph
+ * 7. Return CompiledContext
+ */
+export async function compileContext(
+	request: ContextCompileRequest,
+	clock: Clock = createClock(),
+): Promise<CompiledContextOutput> {
+	const {
+		taskId,
+		taskObjective,
+		maximumTokens,
+		candidates,
+		scoringWeights,
+		policy,
+	} = request;
+
+	// 0. Validate required sources exist
+	const missingRequired = candidates
+		.filter((c) => c.required)
+		.filter((c) => !c.content.trim());
+
+	if (missingRequired.length > 0) {
+		const first = missingRequired[0];
+		if (!first)
+			throw new Error("Internal: missingRequired should not be empty");
+		throw new ContextCompileError(
+			"MISSING_REQUIRED_SOURCE",
+			`Required source "${first.source}" has no content`,
+			{
+				missingSources: missingRequired.map((c) => c.source),
+			},
+		);
+	}
+
+	// 1. Apply policy filter
+	const resolvedPolicy = mergePolicy(policy);
+	const { passed: policyPassed, denied: policyDenied } = applyPolicyFilter(
+		candidates,
+		resolvedPolicy,
+	);
+
+	const policyOmitted: OmittedItem[] = policyDenied.map(
+		({ candidate, reason }) => ({
+			id: candidate.id,
+			kind: candidate.kind,
+			reason: "policy_denied" as OmissionReason,
+			reasonDetail: reason,
+			estimatedTokens: estimateTokens(candidate.content),
+			deniedBy: reason,
+		}),
+	);
+
+	// 2. Score optional candidates
+	const weights = mergeWeights(scoringWeights);
+	const scoreContext = {
+		directFileReferences: extractDirectReferences(taskObjective),
+		taskDependencies: extractDependencies(taskObjective),
+	};
+
+	// 3. Merge file slices (before scoring)
+	const merged = mergeFileSlices(policyPassed);
+
+	// 4. Deduplicate
+	const { unique: deduped, duplicates: dupIds } = deduplicateCandidates(merged);
+
+	const dedupOmitted: OmittedItem[] = dupIds.map((id) => {
+		const c = candidates.find((x) => x.id === id);
+		return {
+			id,
+			kind: c?.kind ?? "source_file",
+			reason: "duplicate" as OmissionReason,
+			reasonDetail: `duplicate of ${c?.source ?? id}`,
+			estimatedTokens: estimateTokens(c?.content ?? ""),
+		};
+	});
+
+	// 5. Rank by score
+	const ranked = rankCandidates(deduped, weights, scoreContext);
+
+	// 6. Fit to budget
+	const { included, omitted: budgetOmitted } = fitToBudget(
+		ranked,
+		maximumTokens,
+	);
+
+	// 7. Convert to compiled items
+	const compiledItems = toCompiledItems(included);
+
+	// 8. Generate source graph
+	const sourceGraph = buildSourceGraph(compiledItems);
+
+	// 9. Generate cache key
+	const cacheKey = generateCacheKey(taskId, compiledItems, taskObjective);
+
+	// 10. Build mapping
+	const mapping: Record<string, string> = {};
+	for (const c of candidates) {
+		const compiled = compiledItems.find((i) => i.id === c.id);
+		if (compiled) {
+			mapping[c.id] = compiled.id;
+		}
+	}
+
+	const totalOmitted = [...policyOmitted, ...dedupOmitted, ...budgetOmitted];
+
+	return {
+		taskId,
+		items: compiledItems,
+		omitted: totalOmitted,
+		estimatedTokens: compiledItems.reduce(
+			(sum, item) => sum + item.estimatedTokens,
+			0,
+		),
+		sourceGraph,
+		generatedAt: clock.now().toISOString(),
+		mapping,
+		cacheKey,
+	};
+}
+
+/**
+ * Compiled context output (extended with cache key).
+ */
+export interface CompiledContextOutput {
+	taskId: string;
+	items: ReturnType<typeof toCompiledItems>[number][];
+	omitted: OmittedItem[];
+	estimatedTokens: number;
+	sourceGraph: ContextSourceEdge[];
+	generatedAt: string;
+	mapping: Record<string, string>;
+	cacheKey: string;
+}
+
+/**
+ * Extract direct file references from task objective.
+ * Looks for quoted paths, file patterns.
+ */
+function extractDirectReferences(objective: string): Set<string> {
+	const refs = new Set<string>();
+	// Match quoted paths
+	const quotedMatches = objective.matchAll(/(["'`])([^`1]+)\1/g);
+	for (const match of quotedMatches) {
+		const path = match[2];
+		if (path) refs.add(path);
+	}
+	// Match common file patterns
+	const filePattern =
+		/[a-zA-Z][a-zA-Z0-9_/-]*\.(ts|tsx|js|jsx|py|go|rs|java|cpp|h|c|md|json|yaml|yml|toml)\b/g;
+	for (const match of objective.matchAll(filePattern)) {
+		const path = match[0];
+		if (path) refs.add(path);
+	}
+	return refs;
+}
+
+/**
+ * Extract dependency references from task objective.
+ */
+function extractDependencies(objective: string): string[] {
+	const deps: string[] = [];
+	// Match "depends on X", "related to Y"
+	const depPatterns = [
+		/(?:depends on|related to|imports from)\s+([A-Za-z][A-Za-z0-9_.-]*)/gi,
+	];
+	for (const pattern of depPatterns) {
+		for (const match of objective.matchAll(pattern)) {
+			const name = match[1];
+			if (name) deps.push(name);
+		}
+	}
+	return [...new Set(deps)];
+}
+
+/**
+ * Build source graph edges between related items.
+ */
+function buildSourceGraph(
+	items: ReturnType<typeof toCompiledItems>[number][],
+): ContextSourceEdge[] {
+	const edges: ContextSourceEdge[] = [];
+
+	const okfItems = items.filter((i) => i.kind === "okf_concept");
+	const sourceItems = items.filter((i) => i.kind === "source_file");
+	const testItems = items.filter((i) => i.kind === "test_failure");
+
+	// OKF concepts → source files they reference
+	for (const okf of okfItems) {
+		for (const src of sourceItems) {
+			if (
+				okf.content.includes(src.source) ||
+				(src.filePath && okf.content.includes(src.filePath))
+			) {
+				edges.push({
+					from: okf.id,
+					to: src.id,
+					label: "references",
+					weight: 3,
+				});
+			}
+		}
+	}
+
+	// Source files → test failures they relate to
+	for (const src of sourceItems) {
+		for (const test of testItems) {
+			if (
+				test.content.includes(src.source) ||
+				(src.filePath && test.content.includes(src.filePath))
+			) {
+				edges.push({
+					from: src.id,
+					to: test.id,
+					label: "causes_failure",
+					weight: 5,
+				});
+			}
+		}
+	}
+
+	// Sort edges deterministically
+	edges.sort((a, b) => {
+		const keyA = `${a.from}:${a.to}:${a.label}`;
+		const keyB = `${b.from}:${b.to}:${b.label}`;
+		return keyA.localeCompare(keyB);
+	});
+
+	return edges;
+}
+
+// ─── OKF Integration ────────────────────────────────────────────────────────
+
+/**
+ * Enrich context candidates with OKF concepts from ~/.pi/okf/.
+ *
+ * This is optional - if the OKF directory doesn't exist,
+ * candidates remain unchanged.
+ *
+ * Usage:
+ * ```ts
+ * const enriched = enrichWithOkf(candidates);
+ * const result = await compileContext({ ...request, candidates: enriched });
+ * ```
+ */
+export function enrichWithOkf(
+	candidates: ContextCompileRequest["candidates"],
+	okfPath?: string,
+): ContextCompileRequest["candidates"] {
+	const okfConcepts = loadOkfConcepts(okfPath);
+
+	if (okfConcepts.length === 0) {
+		return candidates;
+	}
+
+	// Log for debugging (can be removed in production)
+	if (okfDirectoryExists(okfPath)) {
+		console.log(
+			`[OKF] Loaded ${okfConcepts.length} concepts from ${getOkfPath()}`,
+		);
+	}
+
+	// Merge OKF concepts with existing candidates
+	// Existing candidates take precedence (won't be overwritten)
+	const existingIds = new Set(candidates.map((c) => c.id));
+	const newConcepts = okfConcepts.filter((c) => !existingIds.has(c.id));
+
+	return [...candidates, ...newConcepts];
+}