pi-taskflow 0.0.11 → 0.0.13

package/extensions/cache.ts

@@ -0,0 +1,263 @@
+/**
+ * Cross-run memoization: fingerprint resolver + persistent phase-result cache.
+ *
+ * See docs/rfc-cross-run-memoization.md. The cache lets a phase reuse the result
+ * of an identical-input phase from ANY prior run (scope: "cross-run"), for $0.00.
+ * Freshness is guarded by:
+ *   - the existing content-addressed inputHash (declared inputs)
+ *   - optional `fingerprint` entries folded into the key (git/glob/file/env)
+ *   - optional TTL
+ *   - default `run-only` scope (this module is only consulted for cross-run)
+ *
+ * Zero runtime dependencies: Node built-ins only (fs.globSync requires Node >=22,
+ * which the project already targets).
+ */
+
+import { execFileSync } from "node:child_process";
+import * as crypto from "node:crypto";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { cacheDir, withLock, writeFileAtomic } from "./store.ts";
+
+// ---------------------------------------------------------------------------
+// Fingerprint resolution
+// ---------------------------------------------------------------------------
+
+/** Per-file byte cap when content-hashing (mirrors store/context limits). */
+const FINGERPRINT_MAX_FILE_BYTES = 10 * 1024 * 1024; // 10 MB
+/** Cap on glob match count folded into a single fingerprint (defensive). */
+const FINGERPRINT_MAX_GLOB_MATCHES = 5000;
+
+/**
+ * Resolve a single fingerprint entry to a deterministic string. Never throws:
+ * missing files / non-git repos / unreadable paths resolve to a stable sentinel
+ * so the key stays deterministic (and a later appearance of the resource simply
+ * changes the key → cache miss, which is the safe direction).
+ */
+function resolveOne(entry: string, cwd: string): string {
+	try {
+		if (entry === "git:HEAD" || entry.startsWith("git:")) {
+			const ref = entry.slice("git:".length) || "HEAD";
+			// Reject refs that could be interpreted as git options (e.g. "--exec=...").
+			// The ref comes from a taskflow definition; refuse flag-like values so a
+			// crafted definition can't smuggle arguments into git.
+			if (ref.startsWith("-")) return `git:${ref}=<invalid-ref>`;
+			try {
+				const sha = execFileSync("git", ["rev-parse", ref], {
+					cwd,
+					encoding: "utf-8",
+					stdio: ["ignore", "pipe", "ignore"],
+				}).trim();
+				return `git:${ref}=${sha}`;
+			} catch {
+				return `git:${ref}=<no-git>`;
+			}
+		}
+
+		if (entry.startsWith("glob:") || entry.startsWith("glob!:")) {
+			const contentMode = entry.startsWith("glob!:");
+			const pattern = entry.slice(contentMode ? "glob!:".length : "glob:".length);
+			let matches: string[];
+			try {
+				// fs.globSync (Node >=22) — cwd-relative, returns posix-ish paths.
+				matches = (fs.globSync(pattern, { cwd }) as string[]).slice().sort();
+			} catch {
+				return `${entry}=<glob-error>`;
+			}
+			if (matches.length > FINGERPRINT_MAX_GLOB_MATCHES) {
+				matches = matches.slice(0, FINGERPRINT_MAX_GLOB_MATCHES);
+			}
+			const parts: string[] = [];
+			for (const rel of matches) {
+				const abs = path.resolve(cwd, rel);
+				try {
+					if (contentMode) {
+						const st = fs.statSync(abs);
+						if (st.isFile() && st.size <= FINGERPRINT_MAX_FILE_BYTES) {
+							const buf = fs.readFileSync(abs);
+							parts.push(`${rel}:${crypto.createHash("sha256").update(buf).digest("hex").slice(0, 16)}`);
+						} else {
+							parts.push(`${rel}:<skip>`);
+						}
+					} else {
+						const st = fs.statSync(abs);
+						parts.push(`${rel}:${st.size}:${Math.floor(st.mtimeMs)}`);
+					}
+				} catch {
+					parts.push(`${rel}:<stat-error>`);
+				}
+			}
+			const digest = crypto.createHash("sha256").update(parts.join("\u0000")).digest("hex").slice(0, 16);
+			return `${entry}=${digest}`;
+		}
+
+		if (entry.startsWith("file:")) {
+			const rel = entry.slice("file:".length);
+			const abs = path.resolve(cwd, rel);
+			try {
+				const st = fs.statSync(abs);
+				if (!st.isFile() || st.size > FINGERPRINT_MAX_FILE_BYTES) return `file:${rel}=<skip>`;
+				const buf = fs.readFileSync(abs);
+				return `file:${rel}=${crypto.createHash("sha256").update(buf).digest("hex").slice(0, 16)}`;
+			} catch {
+				return `file:${rel}=<missing>`;
+			}
+		}
+
+		if (entry.startsWith("env:")) {
+			const name = entry.slice("env:".length);
+			return `env:${name}=${process.env[name] ?? ""}`;
+		}
+	} catch {
+		// Fall through to sentinel below.
+	}
+	// Unknown prefixes are rejected at validation time; defensively encode.
+	return `${entry}=<unknown>`;
+}
+
+/**
+ * Resolve a phase's `fingerprint` list into a single deterministic string to be
+ * folded into the cache key. Returns "" when there are no entries (so the key is
+ * unchanged for phases that declare no fingerprint).
+ */
+export function resolveFingerprint(entries: string[] | undefined, cwd: string): string {
+	if (!entries || entries.length === 0) return "";
+	// Preserve author order (it's part of the declared key) but resolve each.
+	const resolved = entries.map((e) => resolveOne(e, cwd));
+	return crypto.createHash("sha256").update(resolved.join("\u0000")).digest("hex").slice(0, 16);
+}
+
+// ---------------------------------------------------------------------------
+// Cross-run cache store
+// ---------------------------------------------------------------------------
+
+export interface CacheEntry {
+	/** The full cache key (== phase inputHash incl. fingerprint). */
+	key: string;
+	createdAt: number;
+	/** Trimmed phase result surface that downstream phases consume. */
+	output?: string;
+	json?: unknown;
+	model?: string;
+	/** Provenance for audit / cleanup. */
+	flowName?: string;
+	phaseId?: string;
+	runId?: string;
+}
+
+/** Keep at most this many cache entries; LRU-ish eviction by createdAt. */
+const DEFAULT_MAX_ENTRIES = 1000;
+/** Drop entries older than this regardless of TTL (hard backstop). */
+const DEFAULT_MAX_AGE_MS = 90 * 24 * 60 * 60 * 1000; // 90 days
+
+/** A cache key is a 16-hex inputHash; constrain to that to prevent traversal. */
+function isValidKey(key: string): boolean {
+	return /^[0-9a-f]{8,64}$/.test(key);
+}
+
+function entryPath(dir: string, key: string): string {
+	return path.join(dir, `${key}.json`);
+}
+
+/**
+ * The cross-run cache, scoped to a working directory. Cheap to construct; all IO
+ * is lazy and failure-tolerant (a broken cache must never break a run).
+ */
+export class CacheStore {
+	private dir: string;
+
+	constructor(cwd: string) {
+		this.dir = cacheDir(cwd);
+	}
+
+	/** Look up a fresh entry. Returns null on miss, malformed key, or TTL expiry. */
+	get(key: string, ttlMs?: number): CacheEntry | null {
+		if (!isValidKey(key)) return null;
+		let entry: CacheEntry;
+		try {
+			const raw = fs.readFileSync(entryPath(this.dir, key), "utf-8");
+			entry = JSON.parse(raw) as CacheEntry;
+		} catch {
+			return null;
+		}
+		if (typeof entry?.createdAt !== "number") return null;
+		const age = Date.now() - entry.createdAt;
+		if (age > DEFAULT_MAX_AGE_MS) return null;
+		if (ttlMs !== undefined && age > ttlMs) return null;
+		return entry;
+	}
+
+	/** Store an entry (best-effort; never throws into the run). */
+	put(entry: CacheEntry): void {
+		if (!isValidKey(entry.key)) return;
+		try {
+			fs.mkdirSync(this.dir, { recursive: true });
+			const lock = path.join(this.dir, `${entry.key}.json.lock`);
+			withLock(lock, () => {
+				writeFileAtomic(entryPath(this.dir, entry.key), JSON.stringify(entry, null, 2));
+			});
+			this.cleanup();
+		} catch {
+			/* cache write failures are non-fatal */
+		}
+	}
+
+	/** Remove all cache entries. Returns the number removed. */
+	clear(): number {
+		let n = 0;
+		try {
+			for (const f of fs.readdirSync(this.dir)) {
+				if (f.endsWith(".json")) {
+					try {
+						fs.unlinkSync(path.join(this.dir, f));
+						n++;
+					} catch {
+						/* ignore */
+					}
+				}
+			}
+		} catch {
+			/* no dir → nothing to clear */
+		}
+		return n;
+	}
+
+	/** Opportunistic eviction: drop expired/oversized entries. Best-effort. */
+	private cleanup(): void {
+		let files: string[];
+		try {
+			files = fs.readdirSync(this.dir).filter((f) => f.endsWith(".json"));
+		} catch {
+			return;
+		}
+		const now = Date.now();
+		const live: Array<{ file: string; createdAt: number }> = [];
+		for (const f of files) {
+			const abs = path.join(this.dir, f);
+			try {
+				const e = JSON.parse(fs.readFileSync(abs, "utf-8")) as CacheEntry;
+				if (typeof e?.createdAt !== "number" || now - e.createdAt > DEFAULT_MAX_AGE_MS) {
+					fs.unlinkSync(abs);
+					continue;
+				}
+				live.push({ file: abs, createdAt: e.createdAt });
+			} catch {
+				try {
+					fs.unlinkSync(abs);
+				} catch {
+					/* ignore */
+				}
+			}
+		}
+		if (live.length > DEFAULT_MAX_ENTRIES) {
+			live.sort((a, b) => a.createdAt - b.createdAt); // oldest first
+			for (const victim of live.slice(0, live.length - DEFAULT_MAX_ENTRIES)) {
+				try {
+					fs.unlinkSync(victim.file);
+				} catch {
+					/* ignore */
+				}
+			}
+		}
+	}
+}

package/extensions/index.ts

@@ -12,8 +12,17 @@
 
 import type { AgentToolResult } from "@earendil-works/pi-agent-core";
 import { StringEnum } from "@earendil-works/pi-ai";
-import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { ExtensionAPI, ExtensionContext, ExtensionUIContext } from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
+import {
+	RECOMMENDED_DEFAULTS,
+	readSettings,
+	writeSettings,
+	formatRolesReport,
+	formatDiffReport,
+	formatFlowResult,
+	runInteractiveInit,
+} from "./init.ts";
 import { Type } from "typebox";
 import { type AgentScope, discoverAgents, readSubagentSettings } from "./agents.ts";
 import { renderRunResult, summarizeRun } from "./render.ts";
@@ -30,6 +39,7 @@ import {
 	saveFlow,
 	saveRun,
 } from "./store.ts";
+import { CacheStore } from "./cache.ts";
 
 interface TaskflowDetails {
 	state?: RunState;
@@ -50,8 +60,8 @@ const ShorthandStep = Type.Object(
 );
 
 const TaskflowParams = Type.Object({
-	action: StringEnum(["run", "save", "resume", "list", "agents"] as const, {
-		description: "What to do: run a flow, save a definition, resume a paused run, list saved flows, or list available agents you can use in phases",
+	action: StringEnum(["run", "save", "resume", "list", "agents", "init", "cache-clear"] as const, {
+		description: "What to do: run a flow, save a definition, resume a paused run, list saved flows, list available agents, init model role configuration, or clear the cross-run memoization cache",
 		default: "run",
 	}),
 	name: Type.Optional(Type.String({ description: "Name of a saved flow (for run/save without inline define)" })),
@@ -84,6 +94,19 @@ const TaskflowParams = Type.Object({
 	scope: Type.Optional(
 		StringEnum(["user", "project"] as const, { description: "Where to save (action=save)", default: "project" }),
 	),
+	mode: Type.Optional(
+		StringEnum(["show", "apply-defaults", "interactive"] as const, {
+			description:
+				"Init action mode. 'show' is read-only (default); 'apply-defaults' requires force:true; 'interactive' requires a UI session.",
+			default: "show",
+		}),
+	),
+	force: Type.Optional(
+		Type.Boolean({
+			description:
+				"Destructive: overwrites modelRoles in settings.json. Required for mode='apply-defaults'.",
+		}),
+	),
 });
 
 function makeRunState(def: Taskflow, args: Record<string, unknown>, cwd: string): RunState {
@@ -167,7 +190,20 @@ async function runFlow(
 		// the heartbeat timer is cleared by the finally block below.
 		const settings = readSubagentSettings();
 		const scope: AgentScope = def.agentScope ?? "user";
-		const { agents } = discoverAgents(ctx.cwd, scope, settings.agentOverrides);
+		const { agents } = discoverAgents(ctx.cwd, scope, settings.agentOverrides, settings.modelRoles);
+
+		// Hint: if any agent still has unresolved {{role}} references, suggest configuring modelRoles
+		const unresolvedRoles = agents
+			.filter(a => a.model && /^\{\{\w+\}\}$/.test(a.model))
+			.map(a => a.model!.match(/^\{\{(\w+)\}\}$/)![1]);
+		if (unresolvedRoles.length > 0) {
+			const unique = [...new Set(unresolvedRoles)];
+			console.warn(
+				`[taskflow] Hint: ${unique.length} model role(s) not configured: ${unique.join(", ")}. ` +
+				`Agents will use the default model (slower / less optimal). ` +
+				`Run /tf init to auto-generate modelRoles config.`
+			);
+		}
 
 		// Pre-flight: warn if any phase references an agent not in the registry
 		const agentNames = new Set(agents.map(a => a.name));
@@ -216,7 +252,20 @@ export default function (pi: ExtensionAPI) {
 		}
 	};
 
-	pi.on("session_start", async (_e, ctx) => registerSavedFlowCommands(ctx));
+	pi.on("session_start", async (_e, ctx) => {
+		registerSavedFlowCommands(ctx);
+
+		// Hint: prompt to configure model roles if not set
+		try {
+			const settings = readSubagentSettings();
+			if (!settings.modelRoles) {
+				console.warn(
+					`[taskflow] Model roles not configured — agents will use the default model. ` +
+					`Run /tf init to generate a recommended modelRoles config.`
+				);
+			}
+		} catch {}
+	});
 
 	// ---- The LLM-callable tool ----
 	pi.registerTool({
@@ -229,7 +278,7 @@ export default function (pi: ExtensionAPI) {
 			"For simple non-DAG delegations (like the subagent tool) skip the DSL: pass `task` (+optional `agent`) for one task, `tasks:[{task,agent?}]` to run in parallel, or `chain:[{task,agent?}]` to run sequentially (reference the prior step with {previous.output}).",
 			"Use action=save to persist a definition as a reusable /tf:<name> command. action=resume continues a paused run. action=list shows saved flows. Use action=agents to list available agents — do NOT invent agent names; either use an agent from that list or omit the 'agent' field to auto-select the default agent.",
 			"DSL: {name, args?, concurrency?, budget?:{maxUSD,maxTokens}, phases:[{id, type, agent, task, dependsOn?, join?:'all'|'any', when?, retry?:{max,backoffMs,factor}, over?(map), as?(map), branches?(parallel), from?(reduce), use?(flow), with?(flow), output?:'json', final?}]}.",
-			"Phase types: agent (one subagent), parallel (static branches), map (dynamic fan-out over an array), gate (VERDICT: PASS/BLOCK quality gate), reduce (aggregate from N phases), approval (human-in-the-loop pause), flow (run a saved sub-flow). join:'any' is an OR-join; when is a conditional guard; retry adds backoff; budget caps run cost.",
+			"Phase types: agent (one subagent), parallel (static branches), map (dynamic fan-out over an array), gate (VERDICT: PASS/BLOCK quality gate), reduce (aggregate from N phases), approval (human-in-the-loop pause), flow (run a saved sub-flow), loop (re-run a task until 'until' is truthy / converged / maxIterations; body reads {loop.iteration} and {loop.lastOutput}), tournament (spawn N variants of 'task' — or distinct 'branches' — then a judge picks the best / aggregates; mode:'best'|'aggregate'). join:'any' is an OR-join; when is a conditional guard; retry adds backoff; budget caps run cost.",
 			"Interpolation: {args.X}, {steps.ID.output}, {steps.ID.json}, {item} (map), {previous.output}.",
 		].join(" "),
 		parameters: TaskflowParams,
@@ -243,10 +292,88 @@ export default function (pi: ExtensionAPI) {
 		async execute(_id, params, signal, onUpdate, ctx) {
 			const action = params.action ?? "run";
 
-			// agents — list available agents the LLM can use in phase definitions
+			// init — configure model roles
+			if (action === "init") {
+				let settings: Record<string, unknown>;
+				try {
+					settings = readSettings();
+				} catch (e) {
+					return errorResult(
+						action,
+						`Failed to read settings.json: ${e instanceof Error ? e.message : String(e)}. ` +
+							`Fix the file or remove it.`,
+					);
+				}
+				const current = (settings.modelRoles ?? {}) as Record<string, string>;
+				const mode = params.mode;
+
+				// v0.0.13 deprecation bridge: mode omitted → old behavior
+				if (mode === undefined) {
+					if (Object.keys(current).length === 0) {
+						// v0.0.12 compat: auto-write recommended defaults when modelRoles is empty
+						console.warn(
+							"[taskflow] action=init with no mode is deprecated and will require explicit mode in v0.0.14. " +
+								"Use mode='apply-defaults' with force=true.",
+						);
+						writeSettings({ ...settings, modelRoles: { ...RECOMMENDED_DEFAULTS } });
+						const text = formatDiffReport({}, RECOMMENDED_DEFAULTS);
+						return { content: [{ type: "text", text }], details: { action } satisfies TaskflowDetails };
+					}
+					// mode omitted + modelRoles exist → show
+					const text = formatRolesReport(current);
+					return { content: [{ type: "text", text }], details: { action } satisfies TaskflowDetails };
+				}
+
+				// mode === "show" (read-only, never overwrites)
+				if (mode === "show") {
+					const text = formatRolesReport(current);
+					return { content: [{ type: "text", text }], details: { action } satisfies TaskflowDetails };
+				}
+
+				// mode === "apply-defaults" requires explicit force=true
+				if (mode === "apply-defaults") {
+					if (!params.force)
+						return errorResult(action, "mode=apply-defaults requires force=true to overwrite.");
+					const merged: Record<string, string> = { ...RECOMMENDED_DEFAULTS };
+					for (const key of Object.keys(current)) {
+						if (!(key in merged)) merged[key] = current[key]; // stale-preserved
+					}
+					writeSettings({ ...settings, modelRoles: merged });
+					const text = formatDiffReport(current, merged);
+					return { content: [{ type: "text", text }], details: { action } satisfies TaskflowDetails };
+				}
+
+				// mode === "interactive" — requires a UI session
+				if (mode === "interactive") {
+					if (!ctx.hasUI)
+						return errorResult(action, "mode=interactive requires an interactive session.");
+					const enabledModels = (settings.enabledModels as string[] | undefined) ?? [];
+					const modelList =
+						enabledModels.length > 0
+							? enabledModels
+									.map((id) => ctx.modelRegistry.find(id.split("/")[0], id.split("/").slice(1).join("/")))
+									.filter((m): m is NonNullable<typeof m> => m !== undefined)
+							: ctx.modelRegistry.getAvailable();
+					const result = await runInteractiveInit({
+						hasUI: ctx.hasUI,
+						signal: signal ?? new AbortController().signal,
+						ui: ctx.ui as ExtensionUIContext,
+						modelRegistry: ctx.modelRegistry,
+						modelList,
+						currentRoles: current,
+					});
+					const text = formatFlowResult(result);
+					return { content: [{ type: "text", text }], details: { action } satisfies TaskflowDetails };
+				}
+
+				return errorResult(action, `Unknown init mode: ${String(mode)}`);
+			}
+
+	// agents — list available agents the LLM can use in phase definitions
 			if (action === "agents") {
 				const scope = params.scope ?? "both";
-				const { agents } = discoverAgents(ctx.cwd, scope as AgentScope, undefined);
+				const settings2 = readSubagentSettings();
+				const { agents } = discoverAgents(ctx.cwd, scope as AgentScope, undefined, settings2.modelRoles);
 				const text = agents.length
 					? agents
 							.map(
@@ -267,6 +394,14 @@ export default function (pi: ExtensionAPI) {
 				return { content: [{ type: "text", text }], details: { action } satisfies TaskflowDetails };
 			}
 
+			if (action === "cache-clear") {
+				const removed = new CacheStore(ctx.cwd).clear();
+				return {
+					content: [{ type: "text", text: `Cleared ${removed} cross-run cache entr${removed === 1 ? "y" : "ies"}.` }],
+					details: { action } satisfies TaskflowDetails,
+				};
+			}
+
 			// resume
 			if (action === "resume") {
 				if (!params.runId)
@@ -386,9 +521,9 @@ export default function (pi: ExtensionAPI) {
 
 	// ---- The /tf user command ----
 	pi.registerCommand("tf", {
-		description: "Taskflow: list | run <name> | show <name> | runs",
+		description: "Taskflow: list | run <name> | show <name> | runs | init",
 		getArgumentCompletions: (prefix) => {
-			const subs = ["list", "run", "show", "runs", "resume"];
+			const subs = ["list", "run", "show", "runs", "resume", "init"];
 			const items = subs.map((s) => ({ value: s, label: s }));
 			const filtered = items.filter((i) => i.value.startsWith(prefix));
 			return filtered.length > 0 ? filtered : null;
@@ -480,6 +615,62 @@ export default function (pi: ExtensionAPI) {
 				return;
 			}
 
+			if (sub === "init") {
+				let settings: Record<string, unknown>;
+				try {
+					settings = readSettings();
+				} catch (e) {
+					ctx.ui.notify(
+						`Failed to read settings.json: ${e instanceof Error ? e.message : String(e)}`,
+						"error",
+					);
+					return;
+				}
+				const currentRoles = (settings.modelRoles ?? {}) as Record<string, string>;
+
+				if (!ctx.hasUI) {
+					if (Object.keys(currentRoles).length > 0) {
+						ctx.ui.notify(
+							formatRolesReport(currentRoles),
+							"info",
+						);
+					} else {
+						ctx.ui.notify(
+							"No modelRoles configured. Run /tf init in an interactive session to select models.",
+							"warning",
+						);
+					}
+					return;
+				}
+
+				const enabledModels = (settings.enabledModels as string[] | undefined) ?? [];
+				const modelList =
+					enabledModels.length > 0
+						? enabledModels
+								.map((id) => ctx.modelRegistry.find(id.split("/")[0], id.split("/").slice(1).join("/")))
+								.filter((m): m is NonNullable<typeof m> => m !== undefined)
+						: ctx.modelRegistry.getAvailable();
+				const result = await runInteractiveInit({
+					hasUI: ctx.hasUI,
+					signal: ctx.signal ?? new AbortController().signal,
+					ui: ctx.ui,
+					modelRegistry: ctx.modelRegistry,
+					modelList,
+					currentRoles,
+				});
+				ctx.ui.notify(
+					result.kind === "saved"
+						? `Saved model roles to ${result.savedPath}:\n${Object.entries(result.chosen)
+								.map(([k, v]) => `  ${k.padEnd(10)} → ${v}`)
+								.join("\n")}`
+						: result.kind === "no-change"
+							? "No changes made."
+							: "Init cancelled.",
+					result.kind === "saved" ? "info" : "info",
+				);
+				return;
+			}
+
 			ctx.ui.notify(`Unknown subcommand: ${sub}`, "warning");
 		},
 	});