pi-harness-runtime 0.5.0 → 0.6.3-beta.13

package/packages/context-compiler/src/deduplicate.ts

@@ -0,0 +1,171 @@
+/**
+ * Context Compiler - Deduplication
+ *
+ * Removes duplicate candidates and merges overlapping source file slices.
+ * Uses content normalization for deduplication (same as RFC-0041).
+ */
+
+import type { ContextCandidate } from "./types.js";
+
+/**
+ * Deduplicate candidates by normalized content.
+ * Required candidates are never deduplicated (preserved).
+ *
+ * For source files with overlapping ranges, merge into a single slice.
+ *
+ * @returns [uniqueCandidates, duplicateIds]
+ */
+export function deduplicateCandidates(candidates: ContextCandidate[]): {
+	unique: ContextCandidate[];
+	duplicates: string[];
+} {
+	const seen = new Map<string, ContextCandidate>();
+	const duplicates: string[] = [];
+
+	for (const candidate of candidates) {
+		// Required items are never deduplicated
+		if (candidate.required) {
+			seen.set(candidate.id, candidate);
+			continue;
+		}
+
+		const normalized = normalizeContent(candidate.content);
+
+		if (seen.has(normalized)) {
+			duplicates.push(candidate.id);
+		} else {
+			seen.set(normalized, candidate);
+		}
+	}
+
+	return {
+		unique: [...seen.values()],
+		duplicates,
+	};
+}
+
+/**
+ * Merge overlapping file slices from the same file.
+ * Merges adjacent/overlapping line ranges and deduplicates symbols.
+ */
+export function mergeFileSlices(
+	candidates: ContextCandidate[],
+): ContextCandidate[] {
+	// Group by file path
+	const byFile = new Map<string, ContextCandidate[]>();
+	for (const c of candidates) {
+		if (c.kind === "source_file" && c.filePath) {
+			const existing = byFile.get(c.filePath);
+			if (existing) {
+				existing.push(c);
+			} else {
+				byFile.set(c.filePath, [c]);
+			}
+		} else {
+			// Non-source-file candidates pass through
+		}
+	}
+
+	const result: ContextCandidate[] = [];
+
+	for (const candidate of candidates) {
+		if (candidate.kind === "source_file" && candidate.filePath) {
+			// Only process once (first candidate for each file)
+			const slices = byFile.get(candidate.filePath) ?? [];
+			if (slices[0] !== candidate) continue;
+
+			if (slices.length === 1) {
+				result.push(candidate);
+			} else {
+				// Merge multiple slices
+				result.push(mergeSlicesForFile(candidate.filePath, slices));
+			}
+		} else {
+			result.push(candidate);
+		}
+	}
+
+	return result;
+}
+
+/**
+ * Merge multiple slices from the same file.
+ */
+function mergeSlicesForFile(
+	filePath: string,
+	slices: ContextCandidate[],
+): ContextCandidate {
+	// Sort by startLine
+	const sorted = [...slices].sort(
+		(a, b) => (a.startLine ?? 0) - (b.startLine ?? 0),
+	);
+
+	const first = sorted[0] ?? {
+		id: "",
+		kind: "source_file",
+		content: "",
+		source: "",
+		priority: 0 as const,
+		required: false,
+		trust: "unverified",
+	};
+	let endLine = first.endLine ?? 0;
+	const allSymbols = new Set<string>();
+	const allSources = new Set<string>();
+
+	for (const slice of sorted) {
+		if (slice.endLine && slice.endLine > endLine) {
+			endLine = slice.endLine;
+		}
+		for (const sym of slice.symbols ?? []) {
+			allSymbols.add(sym);
+		}
+		allSources.add(slice.source);
+	}
+
+	// Combine content from all slices
+	const combinedContent = sorted.map((s) => s.content).join("\n...\n");
+
+	return {
+		id: first.id,
+		kind: "source_file",
+		content: combinedContent,
+		source: [...allSources].join("; "),
+		priority: Math.max(
+			...sorted.map((s) => s.priority),
+		) as ContextCandidate["priority"],
+		required: sorted.some((s) => s.required),
+		trust: sorted.find((s) => s.trust === "authoritative")
+			? "authoritative"
+			: sorted.find((s) => s.trust === "generated")
+				? "generated"
+				: "unverified",
+		filePath,
+		startLine: first.startLine,
+		endLine,
+		symbols: [...allSymbols],
+		contentHash: sorted[0]?.contentHash,
+	};
+}
+
+/**
+ * Normalize content for deduplication comparison.
+ * Matches the RFC-0041 deduplication formula.
+ */
+function normalizeContent(text: string): string {
+	return collapseWhitespace(stripMarkdownDecoration(text.toLowerCase()));
+}
+
+function collapseWhitespace(text: string): string {
+	return text.replace(/\s+/g, " ").trim();
+}
+
+function stripMarkdownDecoration(text: string): string {
+	return text
+		.replace(/\[([^\]]+)\]\(([^)]*)\)/g, "$1")
+		.replace(/!\[([^\]]*)\]\(([^)]*)\)/g, "$1")
+		.replace(/[#*_`~>[\]]/g, "")
+		.replace(/```[\s\S]*?```/g, "")
+		.replace(/`([^`]+)`/g, "$1")
+		.trim();
+}

package/packages/context-compiler/src/filter.ts

@@ -0,0 +1,135 @@
+/**
+ * Context Compiler - Policy Filter
+ *
+ * Enforces deny patterns and size limits before any other processing.
+ * This runs first so denied content never reaches the cache.
+ */
+
+import type { ContextCandidate, ContextPolicy } from "./types.js";
+import { DEFAULT_POLICY } from "./types.js";
+
+/**
+ * Merge user policy with defaults.
+ */
+export function mergePolicy(user?: Partial<ContextPolicy>): ContextPolicy {
+	if (!user) return { ...DEFAULT_POLICY };
+	return {
+		deny: user.deny ?? DEFAULT_POLICY.deny,
+		allowLargeFiles: user.allowLargeFiles ?? DEFAULT_POLICY.allowLargeFiles,
+		maxFileBytes: user.maxFileBytes ?? DEFAULT_POLICY.maxFileBytes,
+		trustThreshold: user.trustThreshold,
+	};
+}
+
+/**
+ * Apply policy filters to candidates.
+ * Denied and oversized candidates are marked as omitted.
+ * This is the first stage — nothing else runs before this.
+ *
+ * @returns [passed, denied] - denied candidates with policy_denied reason
+ */
+export function applyPolicyFilter(
+	candidates: ContextCandidate[],
+	policy: ContextPolicy,
+): {
+	passed: ContextCandidate[];
+	denied: Array<{ candidate: ContextCandidate; reason: string }>;
+} {
+	const passed: ContextCandidate[] = [];
+	const denied: Array<{ candidate: ContextCandidate; reason: string }> = [];
+
+	for (const candidate of candidates) {
+		// Check deny patterns
+		const denyReason = matchesDenyPattern(candidate, policy.deny);
+		if (denyReason) {
+			denied.push({ candidate, reason: denyReason });
+			continue;
+		}
+
+		// Check file size limit
+		if (candidate.filePath && candidate.content.length > policy.maxFileBytes) {
+			if (!policy.allowLargeFiles) {
+				denied.push({
+					candidate,
+					reason: `file exceeds ${policy.maxFileBytes} bytes`,
+				});
+				continue;
+			}
+		}
+
+		// Check trust threshold
+		if (
+			policy.trustThreshold &&
+			!meetsTrustThreshold(candidate.trust, policy.trustThreshold)
+		) {
+			denied.push({
+				candidate,
+				reason: `trust level "${candidate.trust}" below threshold`,
+			});
+			continue;
+		}
+
+		passed.push(candidate);
+	}
+
+	return { passed, denied };
+}
+
+/**
+ * Check if a candidate matches any deny glob pattern.
+ */
+function matchesDenyPattern(
+	candidate: ContextCandidate,
+	denyPatterns: string[],
+): string | null {
+	const path = candidate.filePath ?? candidate.source;
+
+	for (const pattern of denyPatterns) {
+		if (globMatch(pattern, path)) {
+			return `denied by pattern: ${pattern}`;
+		}
+	}
+
+	return null;
+}
+
+// Simple glob matching for **/*.env style patterns.
+// Handles: *, **, and literal path separators.
+function globMatch(pattern: string, path: string): boolean {
+	// Normalize: ensure we match relative paths like .env or /project/.env
+	const normalizedPath = path.replace(/^\.\//, "");
+	const normalizedPattern = pattern.replace(/^\*\//, "**/");
+
+	if (normalizedPattern === "**") return true;
+
+	// Convert glob to regex
+	const regexStr = normalizedPattern
+		.replace(/\./g, "\\.") // escape dots
+		.replace(/\*\*/g, "{{DOUBLE_STAR}}")
+		.replace(/\*/g, "[^/]*")
+		.replace(/\{\{DOUBLE_STAR\}\}/g, ".*")
+		.replace(/\?/g, "."); // single char
+
+	try {
+		const regex = new RegExp(`^${regexStr}$`);
+		return regex.test(normalizedPath) || regex.test(`/${normalizedPath}`);
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Check if candidate trust meets the threshold.
+ * "authoritative" > "generated" > "unverified"
+ */
+function meetsTrustThreshold(
+	trust: ContextCandidate["trust"],
+	threshold: ContextCandidate["trust"],
+): boolean {
+	const levels: Record<ContextCandidate["trust"], number> = {
+		authoritative: 3,
+		generated: 2,
+		unverified: 1,
+	};
+	return levels[trust] >= levels[threshold];
+}

package/packages/context-compiler/src/index.ts

@@ -0,0 +1,61 @@
+/**
+ * Context Compiler - Public API
+ *
+ * Bounded context selection engine for pi-harness-runtime.
+ *
+ * @example
+ * import { compileContext } from "@pi/context-compiler";
+ *
+ * const ctx = await compileContext({
+ *   taskId: "task-001",
+ *   taskObjective: "Add login feature to src/auth/login.ts",
+ *   maximumTokens: 50_000,
+ *   candidates: [...],
+ * });
+ */
+
+export { compileContext } from "./compiler.js";
+export { createClock } from "./compiler.js";
+export type { Clock, CompiledContextOutput } from "./compiler.js";
+
+export type {
+	ContextKind,
+	ContextTrust,
+	ContextCandidate,
+	OmissionReason,
+	OmittedItem,
+	CompiledContextItem,
+	ContextSourceEdge,
+	CompiledContext,
+	ScoringWeights,
+	ContextPolicy,
+	ContextCompileRequest,
+	InvalidationReason,
+	ContextInvalidation,
+	ContextCompileErrorCode,
+} from "./types.js";
+
+export { DEFAULT_SCORING_WEIGHTS, DEFAULT_POLICY } from "./types.js";
+export { ContextCompileError } from "./types.js";
+
+export { mergePolicy, applyPolicyFilter } from "./filter.js";
+export { mergeWeights, rankCandidates } from "./score.js";
+export {
+	deduplicateCandidates,
+	mergeFileSlices,
+} from "./deduplicate.js";
+export { fitToBudget, estimateTokens } from "./budget.js";
+export {
+	generateCacheKey,
+	shouldInvalidate,
+	extractItemHashes,
+} from "./cache.js";
+
+// OKF integration
+export { enrichWithOkf } from "./compiler.js";
+export {
+	loadOkfConcepts,
+	okfDirectoryExists,
+	getOkfPath,
+} from "./okf-loader.js";
+export type { OkfConcept } from "./okf-loader.js";

package/packages/context-compiler/src/okf-loader.ts

@@ -0,0 +1,86 @@
+/**
+ * OKF Loader - Loads user knowledge from ~/.pi/okf/
+ *
+ * This module provides optional knowledge loading for context compilation.
+ * If the OKF directory doesn't exist, compilation proceeds without error.
+ */
+
+import { existsSync, readdirSync, readFileSync } from "fs";
+import { join, basename } from "path";
+import { homedir } from "os";
+
+const DEFAULT_OKF_PATH = join(homedir(), ".pi", "okf");
+
+export interface OkfConcept {
+	id: string;
+	kind: "okf_concept";
+	source: string;
+	content: string;
+	required: false;
+	priority: 1;
+	updatedAt: string;
+	trust: "authoritative" | "generated" | "unverified";
+}
+
+/**
+ * Load OKF concepts from ~/.pi/okf/ directory.
+ *
+ * Each .md file becomes one OKF concept.
+ * If directory doesn't exist, returns empty array (graceful degradation).
+ */
+export function loadOkfConcepts(
+	okfPath: string = DEFAULT_OKF_PATH,
+): OkfConcept[] {
+	// Graceful: if directory doesn't exist, return empty
+	if (!existsSync(okfPath)) {
+		return [];
+	}
+
+	try {
+		const files = readdirSync(okfPath);
+		const concepts: OkfConcept[] = [];
+
+		for (const file of files) {
+			if (!file.endsWith(".md") && !file.endsWith(".txt")) {
+				continue;
+			}
+
+			const filePath = join(okfPath, file);
+			const content = readFileSync(filePath, "utf-8");
+			const id = basename(file, ".md").replace(/[^a-zA-Z0-9_-]/g, "_");
+
+			concepts.push({
+				id,
+				kind: "okf_concept",
+				source: filePath,
+				content,
+				required: false, // OKF is always optional
+				priority: 1,
+				updatedAt: new Date().toISOString(),
+				trust: "authoritative", // User's own OKF is authoritative
+			});
+		}
+
+		return concepts;
+	} catch (error) {
+		// Graceful: if read fails, log and continue
+		console.warn(`[OKF] Failed to load from ${okfPath}:`, error);
+		return [];
+	}
+}
+
+/**
+ * Check if OKF directory exists (for informational purposes).
+ */
+export function okfDirectoryExists(
+	okfPath: string = DEFAULT_OKF_PATH,
+): boolean {
+	return existsSync(okfPath);
+}
+
+/**
+ * Get the default OKF path.
+ */
+export function getOkfPath(): string {
+	return DEFAULT_OKF_PATH;
+}

package/packages/context-compiler/src/score.ts

@@ -0,0 +1,130 @@
+/**
+ * Context Compiler - Scoring
+ *
+ * Scores optional candidates for inclusion priority.
+ * Required candidates bypass scoring.
+ * Deterministic: same inputs produce same scores.
+ */
+
+import type { ContextCandidate, ScoringWeights } from "./types.js";
+import { DEFAULT_SCORING_WEIGHTS } from "./types.js";
+
+/**
+ * Merge user weights with defaults.
+ */
+export function mergeWeights(user?: Partial<ScoringWeights>): ScoringWeights {
+	if (!user) return { ...DEFAULT_SCORING_WEIGHTS };
+	return {
+		priority: user.priority ?? DEFAULT_SCORING_WEIGHTS.priority,
+		directFileReference:
+			user.directFileReference ?? DEFAULT_SCORING_WEIGHTS.directFileReference,
+		recentFailureRelevance:
+			user.recentFailureRelevance ??
+			DEFAULT_SCORING_WEIGHTS.recentFailureRelevance,
+		dependencyRelevance:
+			user.dependencyRelevance ?? DEFAULT_SCORING_WEIGHTS.dependencyRelevance,
+		frameworkRelevance:
+			user.frameworkRelevance ?? DEFAULT_SCORING_WEIGHTS.frameworkRelevance,
+		stalePenalty: user.stalePenalty ?? DEFAULT_SCORING_WEIGHTS.stalePenalty,
+		duplicationPenalty:
+			user.duplicationPenalty ?? DEFAULT_SCORING_WEIGHTS.duplicationPenalty,
+	};
+}
+
+/**
+ * Score a candidate.
+ * Required candidates get MAX_SCORE (always included).
+ * Optional candidates get a weighted score.
+ *
+ * Score = priority * priority_weight
+ *       + direct_file_reference_bonus
+ *       + recent_failure_relevance
+ *       + dependency_relevance
+ *       + framework_relevance
+ *       - stale_penalty
+ *       - duplication_penalty
+ */
+export function scoreCandidate(
+	candidate: ContextCandidate,
+	weights: ScoringWeights,
+	context: ScoreContext,
+): number {
+	// Required items bypass scoring
+	if (candidate.required) return MAX_SCORE;
+
+	let score = 0;
+
+	// Base score from priority
+	score += candidate.priority * weights.priority;
+
+	// Direct file reference bonus
+	if (context.directFileReferences.has(candidate.source)) {
+		score += weights.directFileReference;
+	}
+
+	// Recent failure relevance
+	if (candidate.kind === "test_failure" && candidate.updatedAt) {
+		const age = Date.now() - new Date(candidate.updatedAt).getTime();
+		if (age < 24 * 60 * 60 * 1000) {
+			score += weights.recentFailureRelevance;
+		}
+	}
+
+	// Dependency relevance: candidates related to task dependencies score higher
+	if (context.taskDependencies.some((dep) => candidate.source.includes(dep))) {
+		score += weights.dependencyRelevance;
+	}
+
+	// Framework relevance: OKF concepts and framework files score higher
+	if (candidate.kind === "okf_concept" || candidate.kind === "project_rule") {
+		score += weights.frameworkRelevance;
+	}
+
+	// Stale penalty: items not updated recently
+	if (candidate.updatedAt) {
+		const age = Date.now() - new Date(candidate.updatedAt).getTime();
+		const staleDays = age / (24 * 60 * 60 * 1000);
+		score -= Math.min(
+			staleDays * weights.stalePenalty,
+			weights.stalePenalty * 3,
+		);
+	}
+
+	return score;
+}
+
+/**
+ * Score context for candidate evaluation.
+ */
+export interface ScoreContext {
+	directFileReferences: Set<string>;
+	taskDependencies: string[];
+}
+
+/**
+ * Maximum score — always included regardless of budget.
+ */
+export const MAX_SCORE = 9999;
+
+/**
+ * Sort candidates by score (descending), then by source (ascending) for determinism.
+ * Required candidates are always first.
+ */
+export function rankCandidates(
+	candidates: ContextCandidate[],
+	weights: ScoringWeights,
+	context: ScoreContext,
+): Array<{ candidate: ContextCandidate; score: number }> {
+	return [...candidates]
+		.map((c) => ({ candidate: c, score: scoreCandidate(c, weights, context) }))
+		.sort((a, b) => {
+			// Required candidates always first
+			if (a.candidate.required !== b.candidate.required) {
+				return a.candidate.required ? -1 : 1;
+			}
+			// Higher score first
+			if (a.score !== b.score) return b.score - a.score;
+			// Tiebreak: source alphabetically
+			return a.candidate.source.localeCompare(b.candidate.source);
+		});
+}