projectinator 0.3.0 → 0.4.0

package/src/bakeoff.ts

@@ -1,214 +0,0 @@
-// Model bake-off — the founding idea: run ONE task across several models, then
-// compare cost, latency, and quality so you can pick the best model per role and
-// feed that back into the routing registry.
-//
-// v1 covers TEXT roles (plan, design, test-reasoning) where the deliverable is
-// text a judge can score. Code bake-off (per-candidate sandbox + real test
-// scoring) is a later step.
-
-import {
-  createAgentSession,
-  defineTool,
-  type AgentSession,
-} from "@earendil-works/pi-coding-agent";
-import { Type, type Static } from "typebox";
-import type { Capability, Difficulty, Provider, Task } from "./types.js";
-import { piRuntime, resolvePiModel } from "./executor.js"
-import { buildRolePrompt } from "./roles.js";
-import { estimateTokens } from "./estimate.js";
-import { addSessionCost } from "./session-cost.js";
-
-export interface Candidate {
-  provider: Provider;
-  model: string;
-}
-
-export interface BakeoffEntry {
-  provider: Provider;
-  model: string;
-  output: string;
-  cost: number;
-  ms: number;
-  outputTokens: number;
-  error?: string;
-}
-
-export interface JudgeScore {
-  model: string;
-  score: number; // 0-10
-  reason: string;
-}
-
-export interface BakeoffResult {
-  task: Task;
-  entries: BakeoffEntry[];
-  scores: JudgeScore[];
-  winner?: string; // "provider/model"
-  judge?: string; // judge model id
-}
-
-function lastAssistantText(session: AgentSession): string {
-  const msgs = session.messages as Array<{ role?: string; content?: unknown }>;
-  for (let i = msgs.length - 1; i >= 0; i--) {
-    const m = msgs[i];
-    if (m?.role !== "assistant") continue;
-    const c = m.content;
-    if (typeof c === "string") return c;
-    if (Array.isArray(c)) {
-      return c
-        .map((p: unknown) => (typeof p === "string" ? p : p && typeof p === "object" && "text" in p ? String((p as { text: unknown }).text) : ""))
-        .join("")
-        .trim();
-    }
-  }
-  return "";
-}
-
-const id = (c: Candidate) => `${c.provider}/${c.model}`;
-
-/** Run one candidate on the task, capturing output, cost, and latency. */
-async function runCandidate(task: Task, cand: Candidate): Promise<BakeoffEntry> {
-  const base: BakeoffEntry = { provider: cand.provider, model: cand.model, output: "", cost: 0, ms: 0, outputTokens: 0 };
-  try {
-    const runtime = await piRuntime();
-    const model = resolvePiModel(runtime, cand.provider, cand.model);
-    const { session } = await createAgentSession({
-      model,
-      modelRuntime: runtime,
-      thinkingLevel: "medium",
-      noTools: "all",
-    });
-    try {
-      const t0 = Date.now();
-      await session.prompt(buildRolePrompt(task, ""));
-      const ms = Date.now() - t0;
-      const stats = session.getSessionStats();
-      addSessionCost(stats.cost);
-      const out: BakeoffEntry = {
-        ...base,
-        output: lastAssistantText(session),
-        cost: Math.round(stats.cost * 10000) / 10000,
-        ms,
-        outputTokens: stats.tokens.output,
-      };
-      if (stats.tokens.total === 0) out.error = "returned 0 tokens (invalid key, no credit/balance, or no model access)";
-      return out;
-    } finally {
-      session.dispose(); // dispose even when prompt() throws (expected for inaccessible models)
-    }
-  } catch (e) {
-    return { ...base, error: e instanceof Error ? e.message : String(e) };
-  }
-}
-
-// ---- judge: score every output on one rubric, forced structured output ----
-
-const JudgeSchema = Type.Object(
-  {
-    scores: Type.Array(
-      Type.Object({
-        option: Type.String({ description: "the option letter, e.g. A" }),
-        score: Type.Number({ description: "0-10 quality for this deliverable" }),
-        reason: Type.String({ description: "one sentence" }),
-      }),
-    ),
-    winner: Type.String({ description: "the option letter of the best output" }),
-  },
-  { additionalProperties: true },
-);
-type JudgeRaw = Static<typeof JudgeSchema>;
-
-function buildJudgeTool() {
-  let captured: JudgeRaw | undefined;
-  const tool = defineTool({
-    name: "submit_scores",
-    label: "Submit Scores",
-    description: "Submit a 0-10 quality score and one-sentence reason for every option, plus the winning option letter.",
-    parameters: JudgeSchema,
-    execute: async (_id, params: JudgeRaw) => {
-      captured = params;
-      return { content: [{ type: "text", text: `Scored ${params.scores.length} options; winner ${params.winner}.` }], details: {} };
-    },
-  });
-  return { tool, get: () => captured };
-}
-
-/** Judge anonymised outputs (A, B, C…) on one rubric for the task's capability. */
-async function judge(task: Task, entries: BakeoffEntry[], judgeCand: Candidate): Promise<{ scores: JudgeScore[]; winner?: string; judgeId: string }> {
-  const scored = entries.filter((e) => !e.error && e.output);
-  if (scored.length < 2) return { scores: [], winner: undefined, judgeId: id(judgeCand) };
-
-  const letters = scored.map((_, i) => String.fromCharCode(65 + i)); // A, B, C…
-  const blocks = scored.map((e, i) => `### Option ${letters[i]}\n${e.output}`).join("\n\n");
-  const runtime = await piRuntime();
-  const model = resolvePiModel(runtime, judgeCand.provider, judgeCand.model);
-  const { tool, get } = buildJudgeTool();
-  const { session } = await createAgentSession({
-    model,
-    modelRuntime: runtime,
-    thinkingLevel: "medium",
-    noTools: "all",
-    customTools: [tool],
-    tools: ["submit_scores"],
-  });
-
-  const prompt = [
-    `You are judging ${scored.length} anonymous attempts at the same ${task.capability} task. Be a strict, fair critic.`,
-    `Task: ${task.title}`,
-    "",
-    `Score each option 0-10 on how well it delivers a high-quality ${task.capability} result (correctness, completeness, clarity, usefulness). Then pick the single best.`,
-    "Call submit_scores exactly once with a score+reason for EVERY option letter and the winner.",
-    "",
-    blocks,
-  ].join("\n");
-
-  try {
-    await session.prompt(prompt);
-    let raw = get();
-    for (let i = 0; i < 2 && !raw; i++) {
-      await session.prompt("Call submit_scores now with a score for every option letter and the winner.");
-      raw = get();
-    }
-    addSessionCost(session.getSessionStats().cost);
-    if (!raw) return { scores: [], winner: undefined, judgeId: id(judgeCand) };
-
-    const byLetter = new Map(letters.map((l, i) => [l, scored[i]!]));
-    const scores: JudgeScore[] = raw.scores
-      .map((s) => {
-        const e = byLetter.get(s.option.trim().toUpperCase().slice(0, 1));
-        return e ? { model: id(e), score: s.score, reason: s.reason } : undefined;
-      })
-      .filter((x): x is JudgeScore => !!x);
-    const winEntry = byLetter.get(String(raw.winner).trim().toUpperCase().slice(0, 1));
-    return { scores, winner: winEntry ? id(winEntry) : undefined, judgeId: id(judgeCand) };
-  } finally {
-    session.dispose();
-  }
-}
-
-export interface BakeoffOptions {
-  /** Model that scores the outputs. Defaults to the first candidate. */
-  judge?: Candidate;
-  onProgress?: (msg: string) => void;
-}
-
-/** Run the full bake-off: every candidate on the task, then judge. */
-export async function runBakeoff(task: Task, candidates: Candidate[], opts: BakeoffOptions = {}): Promise<BakeoffResult> {
-  const log = opts.onProgress ?? (() => {});
-  const entries: BakeoffEntry[] = [];
-  for (const c of candidates) {
-    log(`running ${id(c)}…`);
-    const e = await runCandidate(task, c);
-    log(e.error ? `  ${id(c)}: ERROR ${e.error}` : `  ${id(c)}: $${e.cost.toFixed(4)}  ${(e.ms / 1000).toFixed(1)}s  ${e.outputTokens} tok`);
-    entries.push(e);
-  }
-  const judgeCand = opts.judge ?? candidates[0]!;
-  log(`judging with ${id(judgeCand)}…`);
-  const { scores, winner, judgeId } = await judge(task, entries, judgeCand);
-  return { task, entries, scores, winner, judge: judgeId };
-}
-
-/** Convenience: build a one-off Task for a capability/difficulty from a prompt. */
-export function bakeoffTask(prompt: string, capability: Capability, difficulty: Difficulty = "medium"): Task {
-  return { id: "BAKE", title: prompt, capability, difficulty, dependsOn: [], estTokens: estimateTokens(capability, difficulty) };
-}

package/src/build-state.ts

@@ -1,47 +0,0 @@
-// Build persistence — checkpoint a run so a halt/crash/cancel can resume without
-// re-paying for finished tasks. The orchestrator itself stays fs-free; this module
-// (and run-build) own the disk I/O.
-
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import type { Task, TaskOutcome } from "./types.js";
-
-export interface BuildState {
-  id: string;
-  /** The original idea/request text, for display in the projects list. */
-  idea?: string;
-  /** Workflow used: auto-run, or approval-gated. */
-  mode?: "auto" | "approval";
-  tasks: Task[];
-  /** Full record of every task run, including feedback-loop retries (append-only). */
-  outcomes: TaskOutcome[];
-  totalCost: number;
-  status: "running" | "complete" | "halted";
-  haltReason?: string;
-  /** Per-project budget cap (USD). Overrides the global default when set. */
-  budgetCapUSD?: number;
-  /** Cached AI retro narrative (generated on demand). */
-  retroNarrative?: string;
-}
-
-export function newBuildState(id: string, tasks: Task[], idea?: string, mode?: "auto" | "approval"): BuildState {
-  return { id, idea, mode, tasks, outcomes: [], totalCost: 0, status: "running" };
-}
-
-export function saveState(state: BuildState, path: string): void {
-  writeFileSync(path, JSON.stringify(state, null, 2) + "\n");
-}
-
-export function loadState(path: string): BuildState | undefined {
-  if (!existsSync(path)) return undefined;
-  try {
-    return JSON.parse(readFileSync(path, "utf-8")) as BuildState;
-  } catch (e) {
-    throw new Error(`Bad build state at ${path}: ${e instanceof Error ? e.message : e}`);
-  }
-}
-
-/** Which task ids are already finished (last outcome wins; a failed attempt does not
- *  count). Used to skip on resume. */
-export function completedIds(state: BuildState): Set<string> {
-  return new Set(state.outcomes.filter((o) => !o.error).map((o) => o.taskId));
-}

package/src/burndown.ts

@@ -1,35 +0,0 @@
-// Burndown — tasks remaining and cumulative spend across the build. There are no
-// timestamps in build-state, so the X axis is task-completion order (step 1..N),
-// which is the natural timeline for a build. Retries add a step (and cost) without
-// burning down a task, so they show up as flat-remaining / rising-cost.
-
-import type { BuildState } from "./build-state.js";
-
-export interface BurndownStep {
-  taskId: string;
-  remaining: number; // distinct tasks still to do after this step
-  cumCost: number; // cumulative spend through this step
-  retry: boolean; // this step re-ran an already-done task
-}
-
-export interface Burndown {
-  taskCount: number;
-  totalCost: number;
-  steps: BurndownStep[];
-}
-
-const round2 = (n: number) => Math.round(n * 100) / 100;
-
-export function computeBurndown(state: BuildState): Burndown {
-  const taskCount = state.tasks.length;
-  const done = new Set<string>();
-  let cum = 0;
-  const steps: BurndownStep[] = [];
-  for (const o of state.outcomes) {
-    const retry = done.has(o.taskId);
-    done.add(o.taskId);
-    cum += o.cost;
-    steps.push({ taskId: o.taskId, remaining: taskCount - done.size, cumCost: round2(cum), retry });
-  }
-  return { taskCount, totalCost: round2(cum), steps };
-}

package/src/calibration.ts

@@ -1,88 +0,0 @@
-// Self-calibrating token estimates. After each real task, we record its measured
-// token usage per (capability, difficulty). estimateTokens uses the running average
-// once there are enough samples, so estimates sharpen with use. Persisted globally.
-
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
-import type { Capability, Difficulty } from "./types.js";
-
-interface Sample {
-  input: number; // total input tokens (fresh + cache-read)
-  output: number;
-  cachedFraction: number; // share of input served from cache
-  n: number; // sample count (capped so recent runs still move the average)
-}
-
-type Calibration = Record<string, Sample>;
-
-const MIN_SAMPLES = 2; // trust calibration only after a couple of runs
-const MAX_N = 20; // cap so old runs don't dominate
-
-function calPath(): string {
-  return join(homedir(), ".projectinator", "calibration.json");
-}
-const key = (c: Capability, d: Difficulty) => `${c}/${d}`;
-
-function load(): Calibration {
-  try {
-    if (!existsSync(calPath())) return {};
-    return JSON.parse(readFileSync(calPath(), "utf-8")) as Calibration;
-  } catch {
-    return {};
-  }
-}
-
-function save(cal: Calibration): void {
-  try {
-    mkdirSync(join(homedir(), ".projectinator"), { recursive: true });
-    writeFileSync(calPath(), JSON.stringify(cal, null, 2) + "\n");
-  } catch {
-    /* best effort — never break a build on a calibration write */
-  }
-}
-
-/** Fold a real measurement into the running average for its bucket. */
-export function recordActual(
-  capability: Capability,
-  difficulty: Difficulty,
-  inputTotal: number,
-  output: number,
-  cachedFraction: number,
-): void {
-  if (!(inputTotal > 0)) return;
-  const cal = load();
-  const k = key(capability, difficulty);
-  const prev = cal[k];
-  if (!prev) {
-    cal[k] = { input: inputTotal, output, cachedFraction, n: 1 };
-  } else {
-    const n = Math.min(prev.n, MAX_N);
-    cal[k] = {
-      input: (prev.input * n + inputTotal) / (n + 1),
-      output: (prev.output * n + output) / (n + 1),
-      cachedFraction: (prev.cachedFraction * n + cachedFraction) / (n + 1),
-      n: prev.n + 1,
-    };
-  }
-  save(cal);
-}
-
-/** All recorded samples keyed "capability/difficulty" (for the accuracy view). */
-export function allSamples(): Record<string, { input: number; output: number; cachedFraction: number; n: number }> {
-  return load();
-}
-
-/** Calibrated estimate for a bucket, once enough samples exist. */
-export function calibratedTokens(
-  capability: Capability,
-  difficulty: Difficulty,
-): { input: number; output: number; cachedInputFraction: number } | undefined {
-  const s = load()[key(capability, difficulty)];
-  if (!s || s.n < MIN_SAMPLES) return undefined;
-  return {
-    input: Math.round(s.input),
-    output: Math.round(s.output),
-    cachedInputFraction: Math.min(0.95, Math.max(0, s.cachedFraction)),
-  };
-}

package/src/cost.ts

@@ -1,43 +0,0 @@
-// Cost estimation. Pure math over a token estimate + a model's price block.
-
-import type { Model, TokenEstimate } from "./types.js";
-
-const PER_MILLION = 1_000_000;
-
-/** Resolve the applicable input/output/cache rates, honoring volume tiers.
- *  Highest matching `inputTokensAbove` threshold wins (Pi semantics). */
-function ratesFor(model: Model, inputTokens: number) {
-  const base = model.cost;
-  let rate = { input: base.input, output: base.output, cacheRead: base.cacheRead ?? base.input };
-  for (const tier of base.tiers ?? []) {
-    if (inputTokens > tier.inputTokensAbove) {
-      rate = {
-        input: tier.input,
-        output: tier.output,
-        cacheRead: tier.cacheRead ?? tier.input,
-      };
-    }
-  }
-  return rate;
-}
-
-/** Estimate USD cost for one task on a given model.
- *  A fraction of input can be served from cache at the cheaper cacheRead rate.
- *  Kept to 4 decimals, not cents: a backlog of sub-cent tasks would otherwise
- *  estimate to $0.00 and under-reserve budget in the parallel scheduler. */
-export function estimateCost(est: TokenEstimate, model: Model): number {
-  const cachedFraction = clamp01(est.cachedInputFraction ?? 0);
-  const rate = ratesFor(model, est.input);
-
-  const freshInput = est.input * (1 - cachedFraction);
-  const cachedInput = est.input * cachedFraction;
-
-  const inputCost = (freshInput / PER_MILLION) * rate.input + (cachedInput / PER_MILLION) * rate.cacheRead;
-  const outputCost = (est.output / PER_MILLION) * rate.output;
-
-  return Math.round((inputCost + outputCost) * 10_000) / 10_000;
-}
-
-function clamp01(n: number): number {
-  return Math.max(0, Math.min(1, n));
-}

package/src/council.ts

@@ -1,175 +0,0 @@
-// Council planning (opt-in "deep plan"). Three perspectives propose epics from
-// different lenses in parallel, then a synthesizer merges them into one ordered
-// epic list. The user approves the epics; expansion into tasks happens after,
-// via the normal decomposer seeded with these epics.
-
-import {
-  createAgentSession,
-  defineTool,
-  type AgentSession,
-} from "@earendil-works/pi-coding-agent";
-import { Type, type Static } from "typebox";
-import type { Backend, Provider } from "./types.js";
-import { findEntry } from "./registry.js";
-import { piRuntime, resolvePiModel } from "./executor.js"
-import { addSessionCost } from "./session-cost.js";
-
-export interface Epic {
-  name: string;
-  rationale: string;
-}
-
-const EpicsSchema = Type.Object(
-  {
-    epics: Type.Array(
-      Type.Object(
-        { name: Type.String(), rationale: Type.String({ description: "one line: why this epic exists" }) },
-        { additionalProperties: true },
-      ),
-    ),
-  },
-  { additionalProperties: true },
-);
-type EpicsRaw = Static<typeof EpicsSchema>;
-
-function buildEpicsTool(toolName: string) {
-  let captured: EpicsRaw | undefined;
-  const tool = defineTool({
-    name: toolName,
-    label: "Submit Epics",
-    description: "Submit the epic list. Call exactly once.",
-    parameters: EpicsSchema,
-    execute: async (_id, params) => {
-      captured = params as EpicsRaw;
-      return { content: [{ type: "text", text: `Got ${params.epics.length} epics.` }], details: {} };
-    },
-  });
-  return { tool, get: () => captured };
-}
-
-function lastAssistantText(session: AgentSession): string {
-  const msgs = session.messages as Array<{ role?: string; content?: unknown }>;
-  for (let i = msgs.length - 1; i >= 0; i--) {
-    const m = msgs[i];
-    if (m?.role !== "assistant") continue;
-    const c = m.content;
-    if (typeof c === "string") return c;
-    if (Array.isArray(c)) return c.map((p: unknown) => (p && typeof p === "object" && "text" in p ? String((p as { text: unknown }).text) : "")).join("");
-  }
-  return "";
-}
-
-interface Ctx {
-  backend: Backend;
-  modelOverride?: { provider: Provider; model: string };
-}
-
-async function runEpicAgent(idea: string, system: string, toolName: string, ctx: Ctx): Promise<Epic[]> {
-  const runtime = await piRuntime();
-  const { entry } = findEntry("plan", "mid");
-  const pick = ctx.modelOverride ?? entry.byBackend[ctx.backend];
-  try {
-    const model = resolvePiModel(runtime, pick.provider, pick.model);
-    const { tool, get } = buildEpicsTool(toolName);
-    const { session } = await createAgentSession({
-      model,
-      modelRuntime: runtime,
-      thinkingLevel: "low",
-      noTools: "all",
-      customTools: [tool],
-      tools: [toolName],
-    });
-    try {
-      await session.prompt(`${system}\n\n--- REQUEST ---\n${idea}`);
-      let raw = get();
-      if (!raw) { await session.prompt(`Call ${toolName} now.`); raw = get(); }
-      addSessionCost(session.getSessionStats().cost);
-      return (raw?.epics ?? [])
-        .map((e) => ({ name: String(e.name ?? "").trim(), rationale: String(e.rationale ?? "").trim() }))
-        .filter((e) => e.name);
-    } finally {
-      session.dispose();
-    }
-  } catch {
-    return [];
-  }
-}
-
-const LENSES: { key: string; system: string }[] = [
-  {
-    key: "architect",
-    system:
-      "You are the ARCHITECT on a planning council. Propose the EPICS (big areas of work) for this build " +
-      "from a TECHNICAL structure lens — components, data, integration, scaffolding. 3-6 epics, each a short " +
-      "name + one-line rationale. Call submit_epics_architect once.",
-  },
-  {
-    key: "product",
-    system:
-      "You are the PRODUCT lead on a planning council. Propose the EPICS from a USER-VALUE lens — the features " +
-      "and flows a user needs, in priority order. 3-6 epics, each a short name + one-line rationale. Call " +
-      "submit_epics_product once.",
-  },
-  {
-    key: "risk",
-    system:
-      "You are the RISK/QA lead on a planning council. Propose the EPICS from a WHAT-COULD-GO-WRONG lens — " +
-      "validation, edge cases, error/empty states, testing, and anything easy to forget. 3-6 epics, each a " +
-      "short name + one-line rationale. Call submit_epics_risk once.",
-  },
-];
-
-const SYNTH_SYSTEM = [
-  "You are the PROJECT MANAGER chairing a planning council. Three leads proposed epics from different lenses",
-  "(architect, product, risk). Merge them into ONE clean, ordered epic list for the build: dedupe overlaps,",
-  "keep what matters, drop noise, and order them the way the work should proceed. Aim for 3-7 epics. Each epic:",
-  "a short name + a one-line rationale that folds in the strongest point(s) from the leads. Call submit_epics once.",
-].join("\n");
-
-export interface CouncilResult {
-  epics: Epic[];
-  proposals: { lens: string; epics: Epic[] }[];
-}
-
-/** Run the council: 3 lenses in parallel, then synthesize. Falls back to a single
- *  lens's epics if synthesis fails; empty only if everything fails. */
-export async function councilEpics(idea: string, ctx: Ctx): Promise<CouncilResult> {
-  const proposalsRaw = await Promise.all(
-    LENSES.map((l) => runEpicAgent(idea, l.system, `submit_epics_${l.key}`, ctx).then((epics) => ({ lens: l.key, epics }))),
-  );
-  const proposals = proposalsRaw.filter((p) => p.epics.length);
-  if (!proposals.length) return { epics: [], proposals: [] };
-
-  // Synthesize.
-  const runtime = await piRuntime();
-  const { entry } = findEntry("plan", "mid");
-  const pick = ctx.modelOverride ?? entry.byBackend[ctx.backend];
-  try {
-    const model = resolvePiModel(runtime, pick.provider, pick.model);
-    const { tool, get } = buildEpicsTool("submit_epics");
-    const { session } = await createAgentSession({
-      model, modelRuntime: runtime, thinkingLevel: "low",
-      noTools: "all", customTools: [tool], tools: ["submit_epics"],
-    });
-    try {
-      const block = proposals
-        .map((p) => `### ${p.lens}\n${p.epics.map((e) => `- ${e.name}: ${e.rationale}`).join("\n")}`)
-        .join("\n\n");
-      await session.prompt(`${SYNTH_SYSTEM}\n\n--- REQUEST ---\n${idea}\n\n--- LEAD PROPOSALS ---\n${block}`);
-      let raw = get();
-      if (!raw) { await session.prompt("Call submit_epics now."); raw = get(); }
-      addSessionCost(session.getSessionStats().cost);
-      const epics = (raw?.epics ?? [])
-        .map((e) => ({ name: String(e.name ?? "").trim(), rationale: String(e.rationale ?? "").trim() }))
-        .filter((e) => e.name);
-      // Fall back to the largest single proposal if synthesis produced nothing.
-      const best = proposals.slice().sort((a, b) => b.epics.length - a.epics.length)[0]!.epics;
-      return { epics: epics.length ? epics : best, proposals };
-    } finally {
-      session.dispose();
-    }
-  } catch {
-    const best = proposals.slice().sort((a, b) => b.epics.length - a.epics.length)[0]!.epics;
-    return { epics: best, proposals };
-  }
-}