substrate-ai 0.9.0 → 0.11.0

package/dist/experimenter-D_N_7ZF3.js

@@ -1,503 +0,0 @@
-import "./logger-D2fS2ccL.js";
-import { createDecision } from "./decisions-DhAA2HG2.js";
-import { EXPERIMENT_RESULT, getRunMetrics, getStoryMetricsForRun } from "./operational-BdcdmDqS.js";
-import { spawnGit } from "./git-utils-DxPx6erV.js";
-import { spawn } from "node:child_process";
-import { join } from "node:path";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
-
-//#region src/modules/supervisor/experimenter.ts
-/**
-* Default spawn implementation used when no `spawn` dep is injected.
-*/
-function spawnCommand(cmd, args, opts) {
-	return new Promise((resolve$1) => {
-		const proc = spawn(cmd, args, {
-			cwd: opts?.cwd,
-			env: opts?.env ?? process.env,
-			stdio: [
-				"ignore",
-				"pipe",
-				"pipe"
-			]
-		});
-		let stdout = "";
-		let stderr = "";
-		proc.stdout?.on("data", (chunk) => {
-			stdout += chunk.toString();
-		});
-		proc.stderr?.on("data", (chunk) => {
-			stderr += chunk.toString();
-		});
-		proc.on("close", (code) => {
-			resolve$1({
-				stdout: stdout.trim(),
-				stderr: stderr.trim(),
-				code: code ?? 1
-			});
-		});
-		proc.on("error", (err) => {
-			resolve$1({
-				stdout: "",
-				stderr: err.message,
-				code: 1
-			});
-		});
-	});
-}
-/**
-* Build a git branch name for an experiment.
-* Format: supervisor/experiment/<run-id-prefix>-<short-desc>
-*
-* The run-id is truncated to 8 characters. The short_desc is sanitized and
-* truncated to 30 characters.
-*/
-function buildBranchName(runId, shortDesc) {
-	const safe = shortDesc.toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").slice(0, 30).replace(/^-|-$/g, "");
-	const runIdShort = runId.slice(0, 8);
-	return `supervisor/experiment/${runIdShort}-${safe}`;
-}
-/**
-* Build a worktree directory path for an experiment.
-* Format: <projectRoot>/.claude/worktrees/experiment-<run-id-prefix>-<short-desc-truncated>
-*
-* The run-id is truncated to 8 characters. The short_desc is sanitized and
-* truncated to 20 characters for the directory name.
-*/
-function buildWorktreePath(projectRoot, baselineRunId, shortDesc) {
-	const safe = shortDesc.toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").slice(0, 20).replace(/^-|-$/g, "");
-	const idShort = baselineRunId.slice(0, 8);
-	return join(projectRoot, ".claude", "worktrees", `experiment-${idShort}-${safe}`);
-}
-/**
-* Maps phase names to their prompt template filenames.
-*/
-const PHASE_TO_PROMPT_FILE = {
-	"create-story": "create-story.md",
-	"dev-story": "dev-story.md",
-	"code-review": "code-review.md",
-	"fix": "fix-story.md"
-};
-/**
-* Build the modification directive to append to a prompt file.
-* The directive is a HTML comment that instructs the agent to apply
-* the recommended optimization strategy for this experiment.
-*
-* These are template-based strategies per recommendation type.
-* Future iterations could use an LLM to generate novel modifications.
-*/
-function buildModificationDirective(rec) {
-	switch (rec.type) {
-		case "token_regression": return `
-<!-- supervisor-experiment: token_regression fix — compress context injection, limit summaries to key points (story: ${rec.story_key}, phase: ${rec.phase}, delta: +${rec.delta_pct ?? "?"}%) -->`;
-		case "review_cycles": return `
-<!-- supervisor-experiment: review_cycles fix — accept passing implementations with minor style issues, reduce strictness for non-critical checks (story: ${rec.story_key}, cycles: ${rec.review_cycles ?? "?"}) -->`;
-		case "timing_bottleneck": return `
-<!-- supervisor-experiment: timing_bottleneck fix — reduce max_turns by 20% for this phase to enforce time-boxing (story: ${rec.story_key}, phase: ${rec.phase}) -->`;
-	}
-}
-/**
-* Resolve the absolute path to the prompt file for a recommendation's phase.
-*/
-function resolvePromptFile(rec, projectRoot, pack) {
-	const filename = PHASE_TO_PROMPT_FILE[rec.phase] ?? `${rec.phase}.md`;
-	return join(projectRoot, "packs", pack, "prompts", filename);
-}
-const REGRESSION_THRESHOLD_PCT = 20;
-/**
-* Determine the experiment verdict based on metric deltas and the recommendation type.
-*
-* Rules:
-*  - IMPROVED: target metric improved (negative delta) AND no other metric regressed by >20%
-*  - MIXED: target metric improved BUT at least one other metric regressed by >20%
-*  - REGRESSED: target metric did not improve (or could not be measured)
-*/
-function determineVerdict(rec, deltas) {
-	const targetImproved = isTargetMetricImproved(rec, deltas);
-	const hasRegression = hasNonTargetRegression(rec, deltas);
-	if (!targetImproved) return "REGRESSED";
-	if (hasRegression) return "MIXED";
-	return "IMPROVED";
-}
-function isTargetMetricImproved(rec, deltas) {
-	switch (rec.type) {
-		case "token_regression": return deltas.tokens_pct !== null && deltas.tokens_pct < 0;
-		case "review_cycles": return deltas.review_cycles_pct !== null && deltas.review_cycles_pct < 0;
-		case "timing_bottleneck": return deltas.wall_clock_pct !== null && deltas.wall_clock_pct < 0;
-	}
-}
-function hasNonTargetRegression(rec, deltas) {
-	if (rec.type !== "token_regression" && deltas.tokens_pct !== null && deltas.tokens_pct > REGRESSION_THRESHOLD_PCT) return true;
-	if (rec.type !== "review_cycles" && deltas.review_cycles_pct !== null && deltas.review_cycles_pct > REGRESSION_THRESHOLD_PCT) return true;
-	return false;
-}
-/**
-* Compute percentage deltas between baseline and experiment run metrics.
-* Negative values mean the experiment was better.
-*/
-function computeDeltas(baselineMetrics, experimentMetrics) {
-	const pct = (base, exp) => base === 0 ? null : Math.round((exp - base) / base * 100 * 10) / 10;
-	const baseTokens = (baselineMetrics.total_input_tokens ?? 0) + (baselineMetrics.total_output_tokens ?? 0);
-	const expTokens = (experimentMetrics.total_input_tokens ?? 0) + (experimentMetrics.total_output_tokens ?? 0);
-	return {
-		tokens_pct: pct(baseTokens, expTokens),
-		cost_pct: pct(baselineMetrics.total_cost_usd ?? 0, experimentMetrics.total_cost_usd ?? 0),
-		review_cycles_pct: pct(baselineMetrics.total_review_cycles ?? 0, experimentMetrics.total_review_cycles ?? 0),
-		wall_clock_pct: pct(baselineMetrics.wall_clock_seconds ?? 0, experimentMetrics.wall_clock_seconds ?? 0)
-	};
-}
-/**
-* Format a percentage delta for display (e.g., "+15%" or "-20%" or "N/A").
-*/
-function fmtPct(pct) {
-	if (pct === null) return "N/A";
-	return `${pct > 0 ? "+" : ""}${pct}%`;
-}
-/**
-* Build the GitHub PR body with a metrics comparison table and raw data.
-* Used by createPR() when the verdict is IMPROVED or MIXED (AC5).
-*/
-function buildPRBody(result) {
-	const { recommendation: rec, verdict, deltas, baselineRunId, experimentRunId, branchName } = result;
-	return [
-		`## Experiment Results`,
-		``,
-		`**Verdict**: \`${verdict}\``,
-		``,
-		`### Recommendation`,
-		`- **Type**: ${rec.type}`,
-		`- **Story**: ${rec.story_key}`,
-		`- **Phase**: ${rec.phase}`,
-		`- **Description**: ${rec.description}`,
-		``,
-		`### Metrics Comparison`,
-		``,
-		`| Metric | Delta |`,
-		`|--------|-------|`,
-		`| Tokens | ${fmtPct(deltas.tokens_pct)} |`,
-		`| Cost | ${fmtPct(deltas.cost_pct)} |`,
-		`| Review Cycles | ${fmtPct(deltas.review_cycles_pct)} |`,
-		`| Wall Clock | ${fmtPct(deltas.wall_clock_pct)} |`,
-		``,
-		`### Raw Data`,
-		`- Baseline Run: \`${baselineRunId}\``,
-		`- Experiment Run: \`${experimentRunId ?? "N/A"}\``,
-		`- Branch: \`${branchName}\``
-	].join("\n");
-}
-/**
-* Build an audit log entry for a single experiment result.
-* The entry is markdown formatted and suitable for appending to the audit log file.
-*/
-function buildAuditLogEntry(result, timestamp = new Date().toISOString()) {
-	const { recommendation: rec, verdict, deltas, error, prLink } = result;
-	const lines = [
-		`## Experiment: ${rec.short_desc} (${timestamp})`,
-		``,
-		`**Hypothesis**: ${rec.description}`,
-		``,
-		`**Modification**: Applied \`${rec.type}\` strategy to \`${rec.phase}\` prompt`,
-		``,
-		`**Results**:`,
-		`- Verdict: ${verdict}`,
-		`- Tokens delta: ${fmtPct(deltas.tokens_pct)}`,
-		`- Cost delta: ${fmtPct(deltas.cost_pct)}`,
-		`- Review cycles delta: ${fmtPct(deltas.review_cycles_pct)}`,
-		`- Wall clock delta: ${fmtPct(deltas.wall_clock_pct)}`
-	];
-	if (error) lines.push(`- Error: ${error}`);
-	lines.push(``);
-	if (prLink) lines.push(`**PR**: ${prLink}`, ``);
-	lines.push(`---`, ``);
-	return lines.join("\n");
-}
-/**
-* Create an Experimenter with the given configuration and injectable dependencies.
-*
-* The Experimenter implements the AC2/AC3/AC4 state machine:
-*   SELECTING → BRANCHING → MODIFYING → RUNNING → COMPARING → REPORTING
-*/
-function createExperimenter(config, deps) {
-	const resolvedDeps = {
-		git: deps?.git ?? spawnGit,
-		spawn: deps?.spawn ?? spawnCommand,
-		runStory: deps?.runStory ?? (async () => {
-			throw new Error("runStory dependency not provided");
-		}),
-		getRunMetrics: deps?.getRunMetrics ?? getRunMetrics,
-		getStoryMetrics: deps?.getStoryMetrics ?? getStoryMetricsForRun,
-		readFile: deps?.readFile ?? ((p) => readFile(p, "utf-8")),
-		writeFile: deps?.writeFile ?? ((p, c) => writeFile(p, c, "utf-8")),
-		mkdir: deps?.mkdir ?? ((p, o) => mkdir(p, o).then(() => void 0)),
-		log: deps?.log ?? ((msg) => process.stdout.write(msg + "\n"))
-	};
-	const { git, spawn: sp, runStory, getRunMetrics: getRun, getStoryMetrics: getStory, readFile: rf, writeFile: wf, mkdir: md, log } = resolvedDeps;
-	async function getCurrentBranch() {
-		const result = await git([
-			"rev-parse",
-			"--abbrev-ref",
-			"HEAD"
-		], { cwd: config.projectRoot });
-		return result.stdout.trim();
-	}
-	/**
-	* Create an isolated git worktree for an experiment.
-	* Uses `git worktree add <path> -b <branch>` so the main working tree is never affected.
-	*/
-	async function createWorktree(worktreePath, branchName) {
-		const result = await git([
-			"worktree",
-			"add",
-			worktreePath,
-			"-b",
-			branchName
-		], { cwd: config.projectRoot });
-		if (result.code !== 0) throw new Error(`Failed to create worktree ${worktreePath}: ${result.stderr}`);
-	}
-	/**
-	* Remove an experiment worktree after the experiment completes.
-	* Always called in the finally block (regardless of verdict).
-	* Uses --force to handle cases where the worktree directory may be dirty.
-	*/
-	async function removeWorktree(worktreePath) {
-		const result = await git([
-			"worktree",
-			"remove",
-			worktreePath,
-			"--force"
-		], { cwd: config.projectRoot });
-		if (result.code !== 0) log(`[experimenter] Warning: could not remove worktree ${worktreePath}: ${result.stderr}`);
-		else log(`[experimenter] Removed experiment worktree: ${worktreePath}`);
-	}
-	async function commitModification(rec, filePath, cwd) {
-		await git(["add", filePath], { cwd });
-		const message = `supervisor-experiment: ${rec.type} fix for ${rec.story_key}/${rec.phase}\n\nRecommendation: ${rec.description}`;
-		const result = await git([
-			"commit",
-			"-m",
-			message
-		], { cwd });
-		if (result.code !== 0) throw new Error(`Failed to commit modification: ${result.stderr}`);
-	}
-	async function deleteBranch(branchName) {
-		const result = await git([
-			"branch",
-			"-D",
-			branchName
-		], { cwd: config.projectRoot });
-		if (result.code !== 0) log(`[experimenter] Warning: could not delete branch ${branchName}: ${result.stderr}`);
-		else log(`[experimenter] Deleted REGRESSED experiment branch: ${branchName}`);
-	}
-	/**
-	* Open a GitHub PR for the experiment. Returns the PR URL, or null if gh is not available.
-	* Degrades gracefully if gh CLI is not installed.
-	*/
-	async function createPR(result) {
-		const { recommendation: rec, branchName, verdict } = result;
-		const title = `[supervisor] ${rec.description}`;
-		const body = buildPRBody(result);
-		const ghResult = await sp("gh", [
-			"pr",
-			"create",
-			"--title",
-			title,
-			"--body",
-			body,
-			"--label",
-			"supervisor",
-			"--label",
-			"automated-experiment",
-			"--head",
-			branchName
-		], { cwd: config.projectRoot });
-		if (ghResult.code !== 0) {
-			log(`[experimenter] Warning: gh pr create failed (verdict: ${verdict}): ${ghResult.stderr}`);
-			log(`[experimenter] Is the gh CLI installed and authenticated?`);
-			return null;
-		}
-		const prUrl = ghResult.stdout.trim();
-		log(`[experimenter] PR created: ${prUrl}`);
-		return prUrl;
-	}
-	/**
-	* Append an experiment result to the audit log file.
-	* Path: _bmad-output/supervisor-reports/<baselineRunId>-experiments.md
-	* Append-only: reads existing content and appends the new entry.
-	*/
-	async function appendExperimentLog(result) {
-		const { baselineRunId } = result;
-		const reportDir = join(config.projectRoot, "_bmad-output", "supervisor-reports");
-		const logPath = join(reportDir, `${baselineRunId}-experiments.md`);
-		try {
-			await md(reportDir, { recursive: true });
-			let existing = "";
-			try {
-				existing = await rf(logPath);
-			} catch {
-				existing = `# Supervisor Experiment Log\n\nRun ID: \`${baselineRunId}\`\n\n`;
-			}
-			const entry = buildAuditLogEntry(result);
-			await wf(logPath, existing + entry);
-			log(`[experimenter] Audit log updated: ${logPath}`);
-		} catch (err) {
-			log(`[experimenter] Warning: could not write audit log: ${err instanceof Error ? err.message : String(err)}`);
-		}
-	}
-	/**
-	* Check if the experiment run exceeded the token budget cap (AC6).
-	* Budget cap = baseline story tokens × tokenBudgetMultiplier.
-	*
-	* Returns false (budget exceeded) if the experiment used more than the cap.
-	* Returns true (within budget) if the cap is satisfied or metrics are unavailable.
-	*/
-	async function isWithinTokenBudget(db, storyKey, baselineRunId, experimentRunId) {
-		try {
-			const baselineStories = await getStory(db, baselineRunId);
-			const experimentStories = await getStory(db, experimentRunId);
-			const baselineStory = baselineStories.find((m) => m.story_key === storyKey);
-			const experimentStory = experimentStories.find((m) => m.story_key === storyKey);
-			if (!baselineStory || !experimentStory) return true;
-			const baselineTokens = (baselineStory.input_tokens ?? 0) + (baselineStory.output_tokens ?? 0);
-			const experimentTokens = (experimentStory.input_tokens ?? 0) + (experimentStory.output_tokens ?? 0);
-			if (baselineTokens === 0) return true;
-			const cap = baselineTokens * config.tokenBudgetMultiplier;
-			const withinBudget = experimentTokens <= cap;
-			if (!withinBudget) log(`[experimenter] Token budget exceeded: experiment used ${experimentTokens} tokens, cap is ${cap} (${config.tokenBudgetMultiplier}x baseline of ${baselineTokens})`);
-			return withinBudget;
-		} catch {
-			return true;
-		}
-	}
-	async function runOneExperiment(db, rec, baselineRunId) {
-		const branchName = buildBranchName(baselineRunId, rec.short_desc);
-		const worktreePath = buildWorktreePath(config.projectRoot, baselineRunId, rec.short_desc);
-		let experimentRunId = null;
-		let currentPhase = "SELECTING";
-		let verdict = "REGRESSED";
-		let deltas = {
-			tokens_pct: null,
-			cost_pct: null,
-			review_cycles_pct: null,
-			wall_clock_pct: null
-		};
-		let caughtError;
-		let worktreeCreated = false;
-		try {
-			currentPhase = "BRANCHING";
-			log(`[experimenter] Creating worktree: ${worktreePath} on branch ${branchName}`);
-			await createWorktree(worktreePath, branchName);
-			worktreeCreated = true;
-			currentPhase = "MODIFYING";
-			const promptFile = resolvePromptFile(rec, worktreePath, config.pack);
-			const directive = buildModificationDirective(rec);
-			const originalContent = await rf(promptFile);
-			await wf(promptFile, originalContent + directive);
-			await commitModification(rec, promptFile, worktreePath);
-			log(`[experimenter] Applied modification to ${promptFile}`);
-			currentPhase = "RUNNING";
-			log(`[experimenter] Running single-story experiment for ${rec.story_key}`);
-			const { runId, exitCode } = await runStory({
-				stories: rec.story_key,
-				projectRoot: worktreePath,
-				pack: config.pack
-			});
-			experimentRunId = runId;
-			log(`[experimenter] Experiment run completed: ${runId} (exit: ${exitCode})`);
-			currentPhase = "COMPARING";
-			const baselineMetrics = await getRun(db, baselineRunId);
-			const experimentMetrics = await getRun(db, runId);
-			if (!baselineMetrics || !experimentMetrics) {
-				log(`[experimenter] Warning: metrics unavailable for comparison`);
-				verdict = "REGRESSED";
-				caughtError = "Could not retrieve metrics for comparison";
-			} else {
-				deltas = computeDeltas(baselineMetrics, experimentMetrics);
-				const withinBudget = await isWithinTokenBudget(db, rec.story_key, baselineRunId, runId);
-				if (!withinBudget) {
-					verdict = "REGRESSED";
-					caughtError = `Token budget cap exceeded (${config.tokenBudgetMultiplier}x baseline)`;
-					log(`[experimenter] Aborting experiment: token budget exceeded`);
-				} else verdict = determineVerdict(rec, deltas);
-			}
-			currentPhase = "REPORTING";
-			log(`[experimenter] Verdict for ${rec.story_key}/${rec.type}: ${verdict}`);
-			log(`[experimenter] Deltas: tokens=${deltas.tokens_pct}% cost=${deltas.cost_pct}% cycles=${deltas.review_cycles_pct}% clock=${deltas.wall_clock_pct}%`);
-		} catch (err) {
-			caughtError = err instanceof Error ? err.message : String(err);
-			verdict = "REGRESSED";
-			log(`[experimenter] Error in phase ${currentPhase}: ${caughtError}`);
-		} finally {
-			if (worktreeCreated) try {
-				await removeWorktree(worktreePath);
-			} catch {
-				log(`[experimenter] Warning: could not remove worktree ${worktreePath}`);
-			}
-		}
-		let prLink = null;
-		if (verdict === "REGRESSED" && worktreeCreated) await deleteBranch(branchName);
-		else if (verdict === "IMPROVED" || verdict === "MIXED") {
-			const partialResult = {
-				recommendation: rec,
-				branchName,
-				baselineRunId,
-				experimentRunId,
-				verdict,
-				deltas,
-				currentPhase: "REPORTING",
-				...caughtError !== void 0 ? { error: caughtError } : {}
-			};
-			prLink = await createPR(partialResult);
-		}
-		const finalResult = {
-			recommendation: rec,
-			branchName,
-			baselineRunId,
-			experimentRunId,
-			verdict,
-			deltas,
-			currentPhase: "REPORTING",
-			...caughtError !== void 0 ? { error: caughtError } : {},
-			prLink
-		};
-		await appendExperimentLog(finalResult);
-		try {
-			const targetMetricValue = rec.type === "token_regression" ? rec.tokens_actual ?? 0 : rec.type === "review_cycles" ? rec.review_cycles ?? 0 : rec.timing_seconds ?? 0;
-			const afterValue = rec.type === "token_regression" ? deltas.tokens_pct !== null ? Math.round(targetMetricValue * (1 + deltas.tokens_pct / 100)) : targetMetricValue : rec.type === "review_cycles" ? deltas.review_cycles_pct !== null ? Math.round(targetMetricValue * (1 + deltas.review_cycles_pct / 100)) : targetMetricValue : deltas.wall_clock_pct !== null ? Math.round(targetMetricValue * (1 + deltas.wall_clock_pct / 100)) : targetMetricValue;
-			await createDecision(db, {
-				pipeline_run_id: baselineRunId,
-				phase: "supervisor",
-				category: EXPERIMENT_RESULT,
-				key: `experiment:${baselineRunId}:${Date.now()}`,
-				value: JSON.stringify({
-					target_metric: rec.type,
-					before: targetMetricValue,
-					after: afterValue,
-					verdict,
-					branch_name: verdict === "IMPROVED" || verdict === "MIXED" ? branchName : null
-				}),
-				rationale: `Experiment for ${rec.story_key}/${rec.phase}: ${rec.description}. Verdict: ${verdict}.`
-			});
-		} catch {}
-		return finalResult;
-	}
-	return { async runExperiments(db, recommendations, baselineRunId) {
-		if (recommendations.length === 0) return [];
-		const currentBranch = await getCurrentBranch();
-		const results = [];
-		const limit = Math.min(recommendations.length, config.maxExperiments);
-		log(`[experimenter] Starting experiment cycle: ${limit} of ${recommendations.length} recommendations`);
-		log(`[experimenter] Current branch: ${currentBranch} (worktrees will not affect this)`);
-		for (let i = 0; i < limit; i++) {
-			const rec = recommendations[i];
-			log(`[experimenter] Experiment ${i + 1}/${limit}: ${rec.type} for ${rec.story_key}/${rec.phase}`);
-			const result = await runOneExperiment(db, rec, baselineRunId);
-			results.push(result);
-		}
-		log(`[experimenter] Experiment cycle complete: ${results.filter((r) => r.verdict === "IMPROVED").length} improved, ${results.filter((r) => r.verdict === "MIXED").length} mixed, ${results.filter((r) => r.verdict === "REGRESSED").length} regressed`);
-		return results;
-	} };
-}
-
-//#endregion
-export { createExperimenter };
-//# sourceMappingURL=experimenter-D_N_7ZF3.js.map