pi-harness-runtime 0.5.0 → 0.6.3-beta.13

package/packages/prompt-compiler/src/compiler.ts

@@ -0,0 +1,188 @@
+/**
+ * Prompt Compiler - Compiler
+ *
+ * Main entry point for prompt compilation.
+ * Orchestrates: normalize → build → redact → deduplicate → compact → validate → render → hash → persist.
+ */
+
+import type { PromptCompileRequest } from "./types.js";
+import type { PromptPackage } from "./types.js";
+import { PROVIDER_PROFILES, PromptCompileError } from "./types.js";
+
+import { compactToBudget, estimateTokens } from "./budget.js";
+import { deduplicateSections } from "./deduplicate.js";
+import { normalizeRequest } from "./normalize.js";
+import { renderForProvider } from "./render.js";
+import { buildSections } from "./section-builder.js";
+import { validateSections, validateTokenBudget } from "./validate.js";
+
+/**
+ * Dependencies required by the compiler.
+ * All I/O (filesystem, clock, hashing) is injected here.
+ */
+export interface PromptCompilerDependencies {
+	/** SHA-256 hash function */
+	hasher: {
+		sha256: (content: string) => string;
+	};
+	/** System clock */
+	clock: {
+		now: () => Date;
+	};
+	/** Redaction interface for security-sensitive content */
+	redactor: {
+		redact: (content: string) => string;
+	};
+	/** Persist the compiled prompt package */
+	persister?: {
+		persist: (pkg: PromptPackage) => Promise<void>;
+	};
+	/** Project rules (injected from project profile or policy engine) */
+	projectRules: string[];
+}
+
+/**
+ * Default in-memory redactor (identity function).
+ * Production should inject a real redaction implementation.
+ */
+export function createIdentityRedactor() {
+	return {
+		redact: (content: string) => content,
+	};
+}
+
+/**
+ * Default identity hasher (SHA-256 via Web Crypto or Node crypto).
+ */
+export function createHasher() {
+	return {
+		async sha256(content: string): Promise<string> {
+			// Use Web Crypto API (available in Node 18+ and browsers)
+			const encoder = new TextEncoder();
+			const data = encoder.encode(content);
+			const hashBuffer = await crypto.subtle.digest("SHA-256", data);
+			const hashArray = Array.from(new Uint8Array(hashBuffer));
+			return hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
+		},
+	};
+}
+
+/**
+ * Synchronous SHA-256 for testing (simple string-based).
+ */
+export function createSyncHasher() {
+	let sha256Impl: ((content: string) => string) | null = null;
+	return {
+		sha256(content: string): string {
+			if (!sha256Impl) {
+				// Lazy import for Node environment
+				try {
+					// eslint-disable-next-line @typescript-eslint/no-var-requires
+					const crypto = require("node:crypto");
+					sha256Impl = (text: string) =>
+						crypto.createHash("sha256").update(text).digest("hex");
+				} catch {
+					// Fallback: simple hash for testing
+					sha256Impl = (text: string) => {
+						let hash = 0;
+						for (let i = 0; i < text.length; i++) {
+							const char = text.charCodeAt(i);
+							hash = (hash << 5) - hash + char;
+							hash |= 0;
+						}
+						return Math.abs(hash).toString(16).padStart(8, "0");
+					};
+				}
+			}
+			return sha256Impl?.(content);
+		},
+	};
+}
+
+/**
+ * Compile a prompt from a normalized request.
+ *
+ * Reference algorithm from RFC-0041:
+ * 1. normalize input
+ * 2. select provider profile
+ * 3. assemble ordered sections
+ * 4. remove duplicate context
+ * 5. enforce policy constraints
+ * 6. estimate token size
+ * 7. compact optional sections
+ * 8. validate output contract
+ * 9. calculate content hash
+ * 10. persist PromptPackage
+ */
+export async function compilePrompt(
+	request: PromptCompileRequest,
+	deps: PromptCompilerDependencies,
+): Promise<PromptPackage> {
+	// 1. Normalize input
+	const normalized = normalizeRequest(request);
+
+	// 2. Select provider profile
+	const profile = PROVIDER_PROFILES[request.provider];
+	if (!profile) {
+		throw new PromptCompileError(
+			"INVALID_TASK",
+			`Unknown provider: ${request.provider}`,
+		);
+	}
+
+	// 3. Assemble sections
+	let sections = buildSections(normalized, deps.projectRules);
+
+	// 4. Redact secrets (before persistence)
+	sections = sections.map((section) => ({
+		...section,
+		content: deps.redactor.redact(section.content),
+	}));
+
+	// 5. Deduplicate
+	sections = deduplicateSections(sections);
+
+	// 6 & 7. Estimate tokens and compact to budget
+	const availableTokens =
+		profile.maximumInputTokens - profile.reservedOutputTokens;
+
+	const estimateSectionTokens = (content: string) => estimateTokens(content);
+
+	const budgeted = compactToBudget(
+		sections,
+		availableTokens,
+		estimateSectionTokens,
+	);
+	sections = budgeted.sections;
+
+	// 8. Validate
+	validateSections(sections, normalized, { provider: profile });
+	validateTokenBudget(budgeted.estimatedTokens, profile);
+
+	// 9. Render for provider
+	const rendered = renderForProvider(sections, profile);
+
+	// 10. Calculate content hash
+	const hashContent = `${rendered.system}\n${rendered.user}`;
+	const hash = deps.hasher.sha256(hashContent);
+
+	const pkg: PromptPackage = {
+		version: "1",
+		taskId: normalized.taskId,
+		provider: request.provider,
+		system: rendered.system,
+		user: rendered.user,
+		sections,
+		estimatedInputTokens: budgeted.estimatedTokens,
+		sourceRefs: sections.flatMap((s) => s.sourceRefs),
+		hash,
+		createdAt: deps.clock.now().toISOString(),
+	};
+
+	// 11. Persist (optional)
+	if (deps.persister) {
+		await deps.persister.persist(pkg);
+	}
+
+	return pkg;
+}

package/packages/prompt-compiler/src/deduplicate.ts

@@ -0,0 +1,84 @@
+/**
+ * Prompt Compiler - Deduplication
+ *
+ * Removes duplicate context entries using normalization.
+ * Near-duplicate removal is disabled by default (per RFC-0041).
+ */
+
+import type { PromptSection } from "./types.js";
+
+/**
+ * Deduplicate sections by removing entries with identical normalized text.
+ *
+ * Uses the RFC-0041 normalization:
+ *   normalize(text) = lowercase(collapseWhitespace(stripMarkdownDecoration(text)))
+ *
+ * Near-duplicate detection is disabled by default (configurable threshold).
+ */
+export function deduplicateSections(
+	sections: PromptSection[],
+	_nearDuplicateThreshold = 0,
+): PromptSection[] {
+	const seenNormalized = new Set<string>();
+
+	return sections.map((section) => {
+		if (!section.compactable) return section;
+
+		// Extract lines that are compactable
+		const lines = section.content.split("\n");
+		const keptLines: string[] = [];
+		const removed: string[] = [];
+
+		for (const line of lines) {
+			const normalized = normalizeForDeduplication(line);
+			if (normalized === "") {
+				// Empty/whitespace lines are always kept
+				keptLines.push(line);
+			} else if (!seenNormalized.has(normalized)) {
+				seenNormalized.add(normalized);
+				keptLines.push(line);
+			} else {
+				removed.push(line);
+			}
+		}
+
+		if (removed.length > 0) {
+			return {
+				...section,
+				content: keptLines.join("\n"),
+			};
+		}
+
+		return section;
+	});
+}
+
+/**
+ * Normalize text for deduplication per RFC-0041:
+ * - lowercase
+ * - collapse whitespace
+ * - strip markdown decoration
+ */
+export function normalizeForDeduplication(text: string): string {
+	return collapseWhitespace(stripMarkdownDecoration(text.toLowerCase()));
+}
+
+function collapseWhitespace(text: string): string {
+	return text.replace(/\s+/g, " ").trim();
+}
+
+function stripMarkdownDecoration(text: string): string {
+	return (
+		text
+			// Remove markdown links BEFORE stripping brackets so [text](url) → "text"
+			.replace(/\[([^\]]+)\]\(([^)]*)\)/g, "$1") // links → text
+			.replace(/!\[([^\]]*)\]\(([^)]*)\)/g, "$1") // images → alt text
+			// Strip remaining markdown symbols (brackets, bold, italic, etc.)
+			.replace(/[#*_`~>[\]]/g, "")
+			// Remove code blocks
+			.replace(/```[\s\S]*?```/g, "")
+			// Remove inline code backticks
+			.replace(/`([^`]+)`/g, "$1")
+			.trim()
+	);
+}

package/packages/prompt-compiler/src/index.ts

@@ -0,0 +1,50 @@
+/**
+ * Prompt Compiler - Public API
+ *
+ * Deterministic prompt compilation for pi-harness-runtime.
+ *
+ * @example
+ * import { compilePrompt, createHasher, createIdentityRedactor } from "@pi/prompt-compiler";
+ *
+ * const pkg = await compilePrompt(request, {
+ *   hasher: createHasher(),
+ *   redactor: createIdentityRedactor(),
+ *   clock: { now: () => new Date() },
+ *   projectRules: ["Use TypeScript", "No console.log in production"],
+ * });
+ */
+
+export { compilePrompt } from "./compiler.js";
+export {
+	createHasher,
+	createSyncHasher,
+	createIdentityRedactor,
+} from "./compiler.js";
+
+export type { PromptCompilerDependencies } from "./compiler.js";
+
+export { PromptCompileError } from "./types.js";
+export type {
+	ProviderTarget,
+	ProviderPromptProfile,
+	PromptSection,
+	PromptPackage,
+	PromptCompileRequest,
+	PromptCompileErrorCode,
+	ContinuationContext,
+	CompiledContext,
+	ContextEntry,
+	PromptCompileErrorCode as ErrorCode,
+} from "./types.js";
+
+export { PROVIDER_PROFILES, SECTION_ORDER } from "./types.js";
+
+export { normalizeRequest } from "./normalize.js";
+export { buildSections } from "./section-builder.js";
+export {
+	deduplicateSections,
+	normalizeForDeduplication,
+} from "./deduplicate.js";
+export { compactToBudget, estimateTokens } from "./budget.js";
+export { validateSections, validateTokenBudget } from "./validate.js";
+export { renderForProvider } from "./render.js";

package/packages/prompt-compiler/src/normalize.ts

@@ -0,0 +1,140 @@
+/**
+ * Prompt Compiler - Request Normalization
+ *
+ * Normalizes a PromptCompileRequest to ensure deterministic, stable output.
+ * Stable array ordering is critical for reproducible prompt hashes.
+ */
+
+import type { CompiledRequirement } from "@pi-harness/requirement-compiler";
+import type { CompiledTask } from "@pi-harness/task-compiler";
+import type { ContinuationContext, PromptCompileRequest } from "./types.js";
+
+/**
+ * Normalized compile request with stable ordering.
+ */
+export interface NormalizedRequest {
+	taskId: string;
+	requirementId: string;
+	provider: string;
+	attempt: number;
+	objective: string;
+	acceptanceCriteria: string[];
+	contextEntries: NormalizedContextEntry[];
+	constraints: string[];
+	filesInScope: string[];
+	expectedOutputs: string[];
+	toolPermissions: string[];
+	continuation: NormalizedContinuation | undefined;
+}
+
+/**
+ * Normalized context entry with stable string representation.
+ */
+export interface NormalizedContextEntry {
+	id: string;
+	content: string;
+	priority: number;
+	source: string;
+}
+
+/**
+ * Normalized continuation context.
+ */
+export interface NormalizedContinuation {
+	previousResponsePath: string;
+	completedItems: string[];
+	incompleteItems: string[];
+}
+
+/**
+ * Normalize a PromptCompileRequest for deterministic processing.
+ *
+ * - Arrays are sorted lexicographically by ID
+ * - Strings are trimmed
+ * - Empty values are removed
+ * - Duplicates are eliminated (stable unique)
+ */
+export function normalizeRequest(
+	request: PromptCompileRequest,
+): NormalizedRequest {
+	const { task, requirement, context, provider, attempt, continuation } =
+		request;
+
+	return {
+		taskId: task.id.trim(),
+		requirementId: requirement.id.trim(),
+		provider: provider.trim(),
+		attempt,
+		objective: task.objective.trim(),
+		acceptanceCriteria: stableUnique(
+			requirement.acceptanceCriteria
+				.map((ac: CompiledRequirement["acceptanceCriteria"][number]) =>
+					ac.outcome.join("; "),
+				)
+				.filter(Boolean),
+		),
+		contextEntries: normalizeContextEntries(context.entries),
+		constraints: stableUnique(
+			requirement.constraints
+				.map((c: CompiledRequirement["constraints"][number]) => c.description)
+				.filter(Boolean),
+		),
+		filesInScope: stableUnique(
+			task.filesInScope
+				.map((f: CompiledTask["filesInScope"][number]) =>
+					typeof f === "string"
+						? f.trim()
+						: ((f as unknown as { path?: string }).path ?? "").trim(),
+				)
+				.filter(Boolean),
+		),
+		expectedOutputs: stableUnique(
+			task.expectedOutputs
+				.map((o: CompiledTask["expectedOutputs"][number]) => {
+					if (o.kind === "file") return `file:${o.path}`;
+					if (o.kind === "test_result") return `test:${o.description}`;
+					return `${o.kind}:${o.description}`;
+				})
+				.filter(Boolean),
+		),
+		toolPermissions: stableUnique(task.permittedCommands.filter(Boolean)),
+		continuation: continuation
+			? normalizeContinuation(continuation)
+			: undefined,
+	};
+}
+
+function normalizeContextEntries(
+	entries: NormalizedRequest["contextEntries"],
+): NormalizedContextEntry[] {
+	return [...entries]
+		.sort((a, b) => a.priority - b.priority || a.id.localeCompare(b.id))
+		.map((e) => ({
+			id: e.id.trim(),
+			content: e.content.trim(),
+			priority: e.priority,
+			source: e.source.trim(),
+		}));
+}
+
+function normalizeContinuation(
+	cont: ContinuationContext,
+): NormalizedContinuation {
+	return {
+		previousResponsePath: cont.previousResponsePath.trim(),
+		completedItems: [...cont.completedItems].sort(),
+		incompleteItems: [...cont.incompleteItems].sort(),
+	};
+}
+
+/**
+ * Stable unique: preserves first occurrence order.
+ */
+function stableUnique(items: string[]): string[] {
+	const seen = new Set<string>();
+	return items.filter((item) => {
+		if (seen.has(item)) return false;
+		seen.add(item);
+		return true;
+	});
+}

package/packages/prompt-compiler/src/render.ts

@@ -0,0 +1,105 @@
+/**
+ * Prompt Compiler - Rendering
+ *
+ * Renders assembled sections into provider-specific prompt strings.
+ * Provider-specific formatting is applied here only.
+ * Core semantics must remain provider-independent.
+ */
+
+import type { PromptSection, ProviderPromptProfile } from "./types.js";
+import { SECTION_ORDER } from "./types.js";
+
+/**
+ * Rendered prompt strings.
+ */
+export interface RenderedPrompt {
+	system: string;
+	user: string;
+}
+
+/**
+ * Render sections into provider-specific prompt strings.
+ *
+ * System-prompt-supporting providers get rules in system, rest in user.
+ * Objective and acceptance criteria always go in user.
+ */
+export function renderForProvider(
+	sections: PromptSection[],
+	profile: ProviderPromptProfile,
+): RenderedPrompt {
+	// Sort sections deterministically
+	const sorted = [...sections].sort((a, b) => {
+		const aIdx = SECTION_ORDER.indexOf(a.kind);
+		const bIdx = SECTION_ORDER.indexOf(b.kind);
+		if (aIdx !== bIdx) return aIdx - bIdx;
+		return a.id.localeCompare(b.id);
+	});
+
+	// Separate into system vs user content
+	const systemSections = profile.supportsSystemPrompt
+		? sorted.filter(
+				(s) => s.kind === "project_rules" || s.kind === "tool_permissions",
+			)
+		: [];
+
+	const userSections = sorted.filter(
+		(s) => s.kind !== "tool_permissions" || !profile.supportsSystemPrompt,
+	);
+
+	const system = renderSections(systemSections, profile);
+	const user = renderSections(userSections, profile);
+
+	return { system, user };
+}
+
+function renderSections(
+	sections: PromptSection[],
+	profile: ProviderPromptProfile,
+): string {
+	const rendered = sections
+		.filter((s) => s.content.trim())
+		.map((s) => s.content)
+		.join("\n\n---\n\n");
+
+	return formatContent(rendered, profile.preferredInstructionStyle);
+}
+
+function formatContent(
+	content: string,
+	style: ProviderPromptProfile["preferredInstructionStyle"],
+): string {
+	if (!content.trim()) return "";
+
+	switch (style) {
+		case "xml":
+			return formatXml(content);
+		case "markdown":
+			return formatMarkdown(content);
+		default:
+			return stripMarkdown(content);
+	}
+}
+
+function formatXml(content: string): string {
+	return [
+		"<instructions>",
+		content
+			.split("\n")
+			.map((line) => `  ${line}`)
+			.join("\n"),
+		"</instructions>",
+	].join("\n");
+}
+
+function formatMarkdown(content: string): string {
+	return content;
+}
+
+function stripMarkdown(content: string): string {
+	return content
+		.replace(/^#{1,6}\s+/gm, "") // headers
+		.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") // links
+		.replace(/[*_`~]/g, "") // emphasis
+		.replace(/^\s*[-*+]\s+/gm, "- ") // list items
+		.trim();
+}