projectinator 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -11
- package/bin/projectinator.mjs +56 -11
- package/dist/bakeoff.js +163 -0
- package/dist/bakeoff.js.map +1 -0
- package/dist/build-state.js +26 -0
- package/dist/build-state.js.map +1 -0
- package/dist/burndown.js +19 -0
- package/dist/burndown.js.map +1 -0
- package/dist/calibration.js +86 -0
- package/dist/calibration.js.map +1 -0
- package/dist/cli.js +289 -0
- package/dist/cli.js.map +1 -0
- package/dist/cost.js +35 -0
- package/dist/cost.js.map +1 -0
- package/dist/council.js +147 -0
- package/dist/council.js.map +1 -0
- package/dist/demo.js +93 -0
- package/dist/demo.js.map +1 -0
- package/dist/estimate.js +98 -0
- package/dist/estimate.js.map +1 -0
- package/dist/executor.js +118 -0
- package/dist/executor.js.map +1 -0
- package/dist/git.js +82 -0
- package/dist/git.js.map +1 -0
- package/dist/intake.js +98 -0
- package/dist/intake.js.map +1 -0
- package/dist/models.js +171 -0
- package/dist/models.js.map +1 -0
- package/dist/narrate.js +76 -0
- package/dist/narrate.js.map +1 -0
- package/dist/openrouter.js +112 -0
- package/dist/openrouter.js.map +1 -0
- package/dist/orchestrator.js +273 -0
- package/dist/orchestrator.js.map +1 -0
- package/dist/pm.js +245 -0
- package/dist/pm.js.map +1 -0
- package/dist/preview.js +194 -0
- package/dist/preview.js.map +1 -0
- package/dist/registry-store.js +38 -0
- package/dist/registry-store.js.map +1 -0
- package/dist/registry.js +115 -0
- package/dist/registry.js.map +1 -0
- package/dist/research.js +98 -0
- package/dist/research.js.map +1 -0
- package/dist/retro.js +83 -0
- package/dist/retro.js.map +1 -0
- package/dist/roles.js +357 -0
- package/dist/roles.js.map +1 -0
- package/dist/router.js +95 -0
- package/dist/router.js.map +1 -0
- package/dist/run-bakeoff.js +68 -0
- package/dist/run-bakeoff.js.map +1 -0
- package/dist/run-build.js +193 -0
- package/dist/run-build.js.map +1 -0
- package/{src/run-dev.ts → dist/run-dev.js} +23 -39
- package/dist/run-dev.js.map +1 -0
- package/{src/run-pm.ts → dist/run-pm.js} +34 -46
- package/dist/run-pm.js.map +1 -0
- package/{src/run-research.ts → dist/run-research.js} +19 -30
- package/dist/run-research.js.map +1 -0
- package/{src/run-scout.ts → dist/run-scout.js} +17 -27
- package/dist/run-scout.js.map +1 -0
- package/dist/run-web.js +93 -0
- package/dist/run-web.js.map +1 -0
- package/dist/scout.js +81 -0
- package/dist/scout.js.map +1 -0
- package/dist/session-cost.js +15 -0
- package/dist/session-cost.js.map +1 -0
- package/dist/stack.js +34 -0
- package/dist/stack.js.map +1 -0
- package/dist/stuck.js +21 -0
- package/dist/stuck.js.map +1 -0
- package/dist/tui/App.js +1269 -0
- package/dist/tui/App.js.map +1 -0
- package/dist/tui/BakeOff.js +95 -0
- package/dist/tui/BakeOff.js.map +1 -0
- package/dist/tui/BoardEditor.js +183 -0
- package/dist/tui/BoardEditor.js.map +1 -0
- package/dist/tui/EditableBoard.js +140 -0
- package/dist/tui/EditableBoard.js.map +1 -0
- package/dist/tui/Frame.js +78 -0
- package/dist/tui/Frame.js.map +1 -0
- package/dist/tui/Intake.js +59 -0
- package/dist/tui/Intake.js.map +1 -0
- package/dist/tui/Kanban.js +75 -0
- package/dist/tui/Kanban.js.map +1 -0
- package/dist/tui/Settings.js +299 -0
- package/dist/tui/Settings.js.map +1 -0
- package/dist/tui/StackPick.js +44 -0
- package/dist/tui/StackPick.js.map +1 -0
- package/dist/tui/WebAccounts.js +131 -0
- package/dist/tui/WebAccounts.js.map +1 -0
- package/dist/tui/components.js +178 -0
- package/dist/tui/components.js.map +1 -0
- package/dist/tui/config.js +129 -0
- package/dist/tui/config.js.map +1 -0
- package/dist/tui/deploy.js +114 -0
- package/dist/tui/deploy.js.map +1 -0
- package/dist/tui/engine.js +710 -0
- package/dist/tui/engine.js.map +1 -0
- package/dist/tui/notify.js +47 -0
- package/dist/tui/notify.js.map +1 -0
- package/dist/tui/panels.js +42 -0
- package/dist/tui/panels.js.map +1 -0
- package/dist/tui/templates.js +106 -0
- package/dist/tui/templates.js.map +1 -0
- package/dist/tui/theme.js +42 -0
- package/dist/tui/theme.js.map +1 -0
- package/dist/tui/validate.js +51 -0
- package/dist/tui/validate.js.map +1 -0
- package/{src/tui.tsx → dist/tui.js} +23 -33
- package/dist/tui.js.map +1 -0
- package/dist/types.js +19 -0
- package/dist/types.js.map +1 -0
- package/dist/web/oauth-anthropic.js +179 -0
- package/dist/web/oauth-anthropic.js.map +1 -0
- package/dist/web/session.js +274 -0
- package/dist/web/session.js.map +1 -0
- package/package.json +5 -4
- package/src/bakeoff.ts +0 -214
- package/src/build-state.ts +0 -47
- package/src/burndown.ts +0 -35
- package/src/calibration.ts +0 -88
- package/src/cost.ts +0 -43
- package/src/council.ts +0 -175
- package/src/demo.ts +0 -106
- package/src/estimate.ts +0 -111
- package/src/executor.ts +0 -169
- package/src/git.ts +0 -72
- package/src/intake.ts +0 -125
- package/src/models.ts +0 -175
- package/src/narrate.ts +0 -87
- package/src/openrouter.ts +0 -119
- package/src/orchestrator.ts +0 -310
- package/src/pm.ts +0 -302
- package/src/preview.ts +0 -206
- package/src/registry-store.ts +0 -41
- package/src/registry.ts +0 -132
- package/src/research.ts +0 -123
- package/src/retro.ts +0 -99
- package/src/roles.ts +0 -401
- package/src/router.ts +0 -120
- package/src/run-bakeoff.ts +0 -77
- package/src/run-build.ts +0 -203
- package/src/run-web.ts +0 -87
- package/src/scout.ts +0 -121
- package/src/session-cost.ts +0 -17
- package/src/stack.ts +0 -46
- package/src/tui/App.tsx +0 -1760
- package/src/tui/BakeOff.tsx +0 -190
- package/src/tui/BoardEditor.tsx +0 -260
- package/src/tui/EditableBoard.tsx +0 -179
- package/src/tui/Frame.tsx +0 -142
- package/src/tui/Intake.tsx +0 -111
- package/src/tui/Kanban.tsx +0 -158
- package/src/tui/Settings.tsx +0 -513
- package/src/tui/StackPick.tsx +0 -79
- package/src/tui/WebAccounts.tsx +0 -197
- package/src/tui/components.tsx +0 -340
- package/src/tui/config.ts +0 -150
- package/src/tui/deploy.ts +0 -137
- package/src/tui/engine.ts +0 -749
- package/src/tui/notify.ts +0 -21
- package/src/tui/panels.tsx +0 -89
- package/src/tui/templates.ts +0 -119
- package/src/tui/theme.ts +0 -44
- package/src/tui/validate.ts +0 -53
- package/src/types.ts +0 -208
- package/src/web/oauth-anthropic.ts +0 -206
- package/src/web/session.ts +0 -299
package/README.md
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
[](https://www.npmjs.com/package/projectinator)
|
|
8
8
|

|
|
9
9
|

|
|
10
|
-

|
|
11
11
|

|
|
12
12
|

|
|
13
13
|
|
|
@@ -30,6 +30,8 @@ npx projectinator # run without installing
|
|
|
30
30
|
npm install -g projectinator # then just: projectinator
|
|
31
31
|
```
|
|
32
32
|
|
|
33
|
+
`projectinator --help` / `--version` work without opening the app; `projectinator doctor` checks your setup.
|
|
34
|
+
|
|
33
35
|
> **It's a CLI, not a library.** `npm install projectinator` (without `-g`) only drops it into a
|
|
34
36
|
> project's `node_modules` — it won't create a runnable command. Use `npx projectinator` or
|
|
35
37
|
> `npm install -g projectinator`. (If `-g` installs but the command isn't found, npm's global
|
|
@@ -67,10 +69,10 @@ Type an idea → it plans → you approve → it builds, tests, and hands you wo
|
|
|
67
69
|
place, every route updates. Run a **bake-off** to pick empirically.
|
|
68
70
|
- **A real pipeline.** PM decomposes → Designer specs → Developer writes files → **Reviewer**
|
|
69
71
|
(cheap, read-only) checks the wiring → Tester **runs the app headless and catches real
|
|
70
|
-
bugs** → feedback loop re-runs the dev on failure.
|
|
72
|
+
bugs** → feedback loop re-runs the dev **one model tier up** on failure.
|
|
71
73
|
- **Multi-file apps.** Vanilla HTML/CSS/JS or **React (CDN, no build)** — your choice.
|
|
72
74
|
- **The cockpit.** A polished terminal UI: editable board, Kanban, standup, per-task cost,
|
|
73
|
-
live budget bar, desktop notification when done.
|
|
75
|
+
live budget bar, desktop notification and optional webhook when done.
|
|
74
76
|
- **Honest cost.** Live spend tracking, per-project budget cap + an alert before the cap,
|
|
75
77
|
**per-task timeout and cost ceiling**, and predicted-vs-actual reporting that sharpens
|
|
76
78
|
itself over real runs.
|
|
@@ -87,11 +89,12 @@ Type an idea → it plans → you approve → it builds, tests, and hands you wo
|
|
|
87
89
|
| 👁 **Live preview** | Local server + auto-reload; ES modules and fetch resolve like production. |
|
|
88
90
|
| 🚀 **Deploy** | One click to Cloudflare Pages, Vercel, or Netlify (their CLI + your login). |
|
|
89
91
|
| 📤 **Export** | Backlog → Markdown, CSV, **Jira** CSV, **Trello** CSV. |
|
|
90
|
-
| 📜 **Git per build** | The workspace is a git repo; one commit per task. History view + **undo a task**. |
|
|
92
|
+
| 📜 **Git per build** | The workspace is a git repo; one commit per task. History view with **per-commit diffs** + **undo a task**. **Transcripts**: read what every role said on every run. |
|
|
91
93
|
| 📊 **Analytics** | Retro (with optional AI narrative), burndown, cost by epic/model, and estimate accuracy that self-calibrates from real runs. |
|
|
92
94
|
| 💾 **Templates** | Save a project's brief as a reusable template; import/share as a file. |
|
|
93
95
|
| ⛔ **Task limits** | Per-task timeout and cost ceiling (Settings → Preferences). A runaway task is aborted, billed for what it spent, and the build halts — resumable. |
|
|
94
96
|
| ✎ **Task notes** | Annotate any task on the board (`n`). Yours only — never sent to a model; included in exports. |
|
|
97
|
+
| 📥 **Import a folder** | Bring an existing site or prototype in as a project; the PM plans changes against its real files. |
|
|
95
98
|
|
|
96
99
|
## How it works
|
|
97
100
|
|
|
@@ -101,7 +104,7 @@ idea
|
|
|
101
104
|
└─ intake? PM asks clarifying questions if the request is vague
|
|
102
105
|
└─ plan mode? Quick (one PM) or Deep (planning council → approve epics)
|
|
103
106
|
└─ decompose → a routed, epic-tagged backlog with a cost estimate
|
|
104
|
-
└─ build toposort deps · design → code → review → test · Reviewer/Tester→Dev feedback loop
|
|
107
|
+
└─ build toposort deps · design → code → review → test · Reviewer/Tester→Dev feedback loop (dev retries one tier up)
|
|
105
108
|
└─ done working files + retro + deploy/export/preview
|
|
106
109
|
```
|
|
107
110
|
|
|
@@ -120,16 +123,32 @@ Real, unedited output from a full build, kept in [`examples/`](examples/):
|
|
|
120
123
|
|
|
121
124
|
## CLI (same engine, for scripting/CI)
|
|
122
125
|
|
|
126
|
+
Everything below uses the same workspace as the cockpit, so headless builds show up in the app.
|
|
127
|
+
|
|
123
128
|
```bash
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
+
projectinator doctor # Node, keys, Chromium, Pi catalog, git
|
|
130
|
+
projectinator build "a tip calculator" --dry-run # plan + estimate only (one PM call)
|
|
131
|
+
projectinator build "a tip calculator" --yes # build without the confirmation prompt
|
|
132
|
+
projectinator build "…" --json --budget 2 --provider anthropic # NDJSON events; cap; lock provider
|
|
133
|
+
projectinator build "…" --task-cap 0.5 --task-timeout 5 # per-task limits (USD / minutes)
|
|
134
|
+
projectinator projects # past builds, status, cost
|
|
135
|
+
projectinator models # the roster as it will run, with prices
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Exit codes: `0` ok · `1` environment problem · `2` bad usage · `3` build halted (resume it in the app).
|
|
139
|
+
|
|
140
|
+
<details>
|
|
141
|
+
<summary>Developer scripts (from a clone)</summary>
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
npm start # the cockpit
|
|
145
|
+
npm run build -- --live --mini # fixed 4-task build, cheap end-to-end proof (~$0.10)
|
|
146
|
+
npm run build -- --live --mini --resume # resume a halted run
|
|
129
147
|
npm run bakeoff -- --capability design "Design a pricing page" # model bake-off
|
|
130
|
-
npm test #
|
|
148
|
+
npm test # 212 tests
|
|
131
149
|
npm run typecheck
|
|
132
150
|
```
|
|
151
|
+
</details>
|
|
133
152
|
|
|
134
153
|
## Configuration & data
|
|
135
154
|
|
package/bin/projectinator.mjs
CHANGED
|
@@ -1,21 +1,66 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// Launcher for `projectinator` / `npx github:smanookian/projectinator`.
|
|
3
|
-
// Runs the
|
|
3
|
+
// Runs the compiled app from dist/ (published), or the TypeScript sources via tsx (dev clone).
|
|
4
4
|
import { spawnSync } from "node:child_process";
|
|
5
|
-
import {
|
|
5
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
6
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
6
7
|
import { dirname, join } from "node:path";
|
|
7
8
|
|
|
8
9
|
const root = dirname(dirname(fileURLToPath(import.meta.url)));
|
|
9
10
|
const entry = join(root, "src", "tui.tsx");
|
|
10
11
|
|
|
11
|
-
|
|
12
|
-
const
|
|
13
|
-
stdio: "inherit",
|
|
14
|
-
cwd: root,
|
|
15
|
-
});
|
|
12
|
+
const args = process.argv.slice(2);
|
|
13
|
+
const { version } = JSON.parse(readFileSync(join(root, "package.json"), "utf8"));
|
|
16
14
|
|
|
17
|
-
if (
|
|
18
|
-
console.
|
|
19
|
-
process.exit(
|
|
15
|
+
if (args.includes("--version") || args.includes("-v")) {
|
|
16
|
+
console.log(version);
|
|
17
|
+
process.exit(0);
|
|
18
|
+
}
|
|
19
|
+
if (args.includes("--help") || args.includes("-h") || args[0] === "help") {
|
|
20
|
+
console.log(`projectinator ${version} — your AI build team in the terminal
|
|
21
|
+
|
|
22
|
+
Usage
|
|
23
|
+
projectinator open the cockpit (TUI)
|
|
24
|
+
projectinator doctor check Node, API keys, Chromium, Pi catalog, git
|
|
25
|
+
projectinator build "<idea>" [...] plan + build headless (--dry-run, --yes, --json,
|
|
26
|
+
--budget, --provider, --concurrency, --task-cap, --task-timeout)
|
|
27
|
+
projectinator projects list past builds with status and cost
|
|
28
|
+
projectinator models the roster as it will run, with prices
|
|
29
|
+
projectinator --version | --help
|
|
30
|
+
|
|
31
|
+
Setup
|
|
32
|
+
Inside the app: Settings → API keys (Anthropic, OpenAI, Gemini, OpenRouter).
|
|
33
|
+
Keys live in ~/.projectinator/config.json (chmod 0600).
|
|
34
|
+
Optional: npx playwright install chromium → lets the tester run the app headless.
|
|
35
|
+
|
|
36
|
+
Exit codes: 0 ok · 1 environment problem · 2 bad usage · 3 build halted
|
|
37
|
+
Docs: https://github.com/smanookian/projectinator#readme`);
|
|
38
|
+
process.exit(0);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const COMMANDS = new Set(["doctor", "build", "projects", "models"]);
|
|
42
|
+
if (args.length && !COMMANDS.has(args[0])) {
|
|
43
|
+
console.error(`projectinator: unknown ${args[0].startsWith("-") ? "option" : "command"} "${args[0]}". Try --help.`);
|
|
44
|
+
process.exit(2);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Published package: run the compiled JS in-process (no tsx, fast start).
|
|
48
|
+
// Dev clone without `npm run compile`: fall back to tsx on the TypeScript sources.
|
|
49
|
+
const dist = join(root, "dist");
|
|
50
|
+
if (existsSync(join(dist, "cli.js")) && existsSync(join(dist, "tui.js"))) {
|
|
51
|
+
process.chdir(root);
|
|
52
|
+
if (args.length) {
|
|
53
|
+
const { main } = await import(pathToFileURL(join(dist, "cli.js")).href);
|
|
54
|
+
process.exitCode = await main(args);
|
|
55
|
+
} else {
|
|
56
|
+
await import(pathToFileURL(join(dist, "tui.js")).href);
|
|
57
|
+
}
|
|
58
|
+
} else {
|
|
59
|
+
const target = args.length ? join(root, "src", "cli.ts") : entry;
|
|
60
|
+
const res = spawnSync(process.execPath, ["--import", "tsx", target, ...args], { stdio: "inherit", cwd: root });
|
|
61
|
+
if (res.error) {
|
|
62
|
+
console.error("Failed to launch Projectinator:", res.error.message, "\n(dev clone: run `npm install` and `npm run compile`, or keep tsx installed)");
|
|
63
|
+
process.exit(1);
|
|
64
|
+
}
|
|
65
|
+
process.exit(res.status ?? 0);
|
|
20
66
|
}
|
|
21
|
-
process.exit(res.status ?? 0);
|
package/dist/bakeoff.js
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
// Model bake-off — the founding idea: run ONE task across several models, then
|
|
2
|
+
// compare cost, latency, and quality so you can pick the best model per role and
|
|
3
|
+
// feed that back into the routing registry.
|
|
4
|
+
//
|
|
5
|
+
// v1 covers TEXT roles (plan, design, test-reasoning) where the deliverable is
|
|
6
|
+
// text a judge can score. Code bake-off (per-candidate sandbox + real test
|
|
7
|
+
// scoring) is a later step.
|
|
8
|
+
import { createAgentSession, defineTool, } from "@earendil-works/pi-coding-agent";
|
|
9
|
+
import { Type } from "typebox";
|
|
10
|
+
import { piRuntime, resolvePiModel } from "./executor.js";
|
|
11
|
+
import { buildRolePrompt } from "./roles.js";
|
|
12
|
+
import { estimateTokens } from "./estimate.js";
|
|
13
|
+
import { addSessionCost } from "./session-cost.js";
|
|
14
|
+
function lastAssistantText(session) {
|
|
15
|
+
const msgs = session.messages;
|
|
16
|
+
for (let i = msgs.length - 1; i >= 0; i--) {
|
|
17
|
+
const m = msgs[i];
|
|
18
|
+
if (m?.role !== "assistant")
|
|
19
|
+
continue;
|
|
20
|
+
const c = m.content;
|
|
21
|
+
if (typeof c === "string")
|
|
22
|
+
return c;
|
|
23
|
+
if (Array.isArray(c)) {
|
|
24
|
+
return c
|
|
25
|
+
.map((p) => (typeof p === "string" ? p : p && typeof p === "object" && "text" in p ? String(p.text) : ""))
|
|
26
|
+
.join("")
|
|
27
|
+
.trim();
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return "";
|
|
31
|
+
}
|
|
32
|
+
const id = (c) => `${c.provider}/${c.model}`;
|
|
33
|
+
/** Run one candidate on the task, capturing output, cost, and latency. */
|
|
34
|
+
async function runCandidate(task, cand) {
|
|
35
|
+
const base = { provider: cand.provider, model: cand.model, output: "", cost: 0, ms: 0, outputTokens: 0 };
|
|
36
|
+
try {
|
|
37
|
+
const runtime = await piRuntime();
|
|
38
|
+
const model = resolvePiModel(runtime, cand.provider, cand.model);
|
|
39
|
+
const { session } = await createAgentSession({
|
|
40
|
+
model,
|
|
41
|
+
modelRuntime: runtime,
|
|
42
|
+
thinkingLevel: "medium",
|
|
43
|
+
noTools: "all",
|
|
44
|
+
});
|
|
45
|
+
try {
|
|
46
|
+
const t0 = Date.now();
|
|
47
|
+
await session.prompt(buildRolePrompt(task, ""));
|
|
48
|
+
const ms = Date.now() - t0;
|
|
49
|
+
const stats = session.getSessionStats();
|
|
50
|
+
addSessionCost(stats.cost);
|
|
51
|
+
const out = {
|
|
52
|
+
...base,
|
|
53
|
+
output: lastAssistantText(session),
|
|
54
|
+
cost: Math.round(stats.cost * 10000) / 10000,
|
|
55
|
+
ms,
|
|
56
|
+
outputTokens: stats.tokens.output,
|
|
57
|
+
};
|
|
58
|
+
if (stats.tokens.total === 0)
|
|
59
|
+
out.error = "returned 0 tokens (invalid key, no credit/balance, or no model access)";
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
finally {
|
|
63
|
+
session.dispose(); // dispose even when prompt() throws (expected for inaccessible models)
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
catch (e) {
|
|
67
|
+
return { ...base, error: e instanceof Error ? e.message : String(e) };
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// ---- judge: score every output on one rubric, forced structured output ----
|
|
71
|
+
const JudgeSchema = Type.Object({
|
|
72
|
+
scores: Type.Array(Type.Object({
|
|
73
|
+
option: Type.String({ description: "the option letter, e.g. A" }),
|
|
74
|
+
score: Type.Number({ description: "0-10 quality for this deliverable" }),
|
|
75
|
+
reason: Type.String({ description: "one sentence" }),
|
|
76
|
+
})),
|
|
77
|
+
winner: Type.String({ description: "the option letter of the best output" }),
|
|
78
|
+
}, { additionalProperties: true });
|
|
79
|
+
function buildJudgeTool() {
|
|
80
|
+
let captured;
|
|
81
|
+
const tool = defineTool({
|
|
82
|
+
name: "submit_scores",
|
|
83
|
+
label: "Submit Scores",
|
|
84
|
+
description: "Submit a 0-10 quality score and one-sentence reason for every option, plus the winning option letter.",
|
|
85
|
+
parameters: JudgeSchema,
|
|
86
|
+
execute: async (_id, params) => {
|
|
87
|
+
captured = params;
|
|
88
|
+
return { content: [{ type: "text", text: `Scored ${params.scores.length} options; winner ${params.winner}.` }], details: {} };
|
|
89
|
+
},
|
|
90
|
+
});
|
|
91
|
+
return { tool, get: () => captured };
|
|
92
|
+
}
|
|
93
|
+
/** Judge anonymised outputs (A, B, C…) on one rubric for the task's capability. */
|
|
94
|
+
async function judge(task, entries, judgeCand) {
|
|
95
|
+
const scored = entries.filter((e) => !e.error && e.output);
|
|
96
|
+
if (scored.length < 2)
|
|
97
|
+
return { scores: [], winner: undefined, judgeId: id(judgeCand) };
|
|
98
|
+
const letters = scored.map((_, i) => String.fromCharCode(65 + i)); // A, B, C…
|
|
99
|
+
const blocks = scored.map((e, i) => `### Option ${letters[i]}\n${e.output}`).join("\n\n");
|
|
100
|
+
const runtime = await piRuntime();
|
|
101
|
+
const model = resolvePiModel(runtime, judgeCand.provider, judgeCand.model);
|
|
102
|
+
const { tool, get } = buildJudgeTool();
|
|
103
|
+
const { session } = await createAgentSession({
|
|
104
|
+
model,
|
|
105
|
+
modelRuntime: runtime,
|
|
106
|
+
thinkingLevel: "medium",
|
|
107
|
+
noTools: "all",
|
|
108
|
+
customTools: [tool],
|
|
109
|
+
tools: ["submit_scores"],
|
|
110
|
+
});
|
|
111
|
+
const prompt = [
|
|
112
|
+
`You are judging ${scored.length} anonymous attempts at the same ${task.capability} task. Be a strict, fair critic.`,
|
|
113
|
+
`Task: ${task.title}`,
|
|
114
|
+
"",
|
|
115
|
+
`Score each option 0-10 on how well it delivers a high-quality ${task.capability} result (correctness, completeness, clarity, usefulness). Then pick the single best.`,
|
|
116
|
+
"Call submit_scores exactly once with a score+reason for EVERY option letter and the winner.",
|
|
117
|
+
"",
|
|
118
|
+
blocks,
|
|
119
|
+
].join("\n");
|
|
120
|
+
try {
|
|
121
|
+
await session.prompt(prompt);
|
|
122
|
+
let raw = get();
|
|
123
|
+
for (let i = 0; i < 2 && !raw; i++) {
|
|
124
|
+
await session.prompt("Call submit_scores now with a score for every option letter and the winner.");
|
|
125
|
+
raw = get();
|
|
126
|
+
}
|
|
127
|
+
addSessionCost(session.getSessionStats().cost);
|
|
128
|
+
if (!raw)
|
|
129
|
+
return { scores: [], winner: undefined, judgeId: id(judgeCand) };
|
|
130
|
+
const byLetter = new Map(letters.map((l, i) => [l, scored[i]]));
|
|
131
|
+
const scores = raw.scores
|
|
132
|
+
.map((s) => {
|
|
133
|
+
const e = byLetter.get(s.option.trim().toUpperCase().slice(0, 1));
|
|
134
|
+
return e ? { model: id(e), score: s.score, reason: s.reason } : undefined;
|
|
135
|
+
})
|
|
136
|
+
.filter((x) => !!x);
|
|
137
|
+
const winEntry = byLetter.get(String(raw.winner).trim().toUpperCase().slice(0, 1));
|
|
138
|
+
return { scores, winner: winEntry ? id(winEntry) : undefined, judgeId: id(judgeCand) };
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
session.dispose();
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
/** Run the full bake-off: every candidate on the task, then judge. */
|
|
145
|
+
export async function runBakeoff(task, candidates, opts = {}) {
|
|
146
|
+
const log = opts.onProgress ?? (() => { });
|
|
147
|
+
const entries = [];
|
|
148
|
+
for (const c of candidates) {
|
|
149
|
+
log(`running ${id(c)}…`);
|
|
150
|
+
const e = await runCandidate(task, c);
|
|
151
|
+
log(e.error ? ` ${id(c)}: ERROR ${e.error}` : ` ${id(c)}: $${e.cost.toFixed(4)} ${(e.ms / 1000).toFixed(1)}s ${e.outputTokens} tok`);
|
|
152
|
+
entries.push(e);
|
|
153
|
+
}
|
|
154
|
+
const judgeCand = opts.judge ?? candidates[0];
|
|
155
|
+
log(`judging with ${id(judgeCand)}…`);
|
|
156
|
+
const { scores, winner, judgeId } = await judge(task, entries, judgeCand);
|
|
157
|
+
return { task, entries, scores, winner, judge: judgeId };
|
|
158
|
+
}
|
|
159
|
+
/** Convenience: build a one-off Task for a capability/difficulty from a prompt. */
|
|
160
|
+
export function bakeoffTask(prompt, capability, difficulty = "medium") {
|
|
161
|
+
return { id: "BAKE", title: prompt, capability, difficulty, dependsOn: [], estTokens: estimateTokens(capability, difficulty) };
|
|
162
|
+
}
|
|
163
|
+
//# sourceMappingURL=bakeoff.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bakeoff.js","sourceRoot":"","sources":["../src/bakeoff.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,iFAAiF;AACjF,4CAA4C;AAC5C,EAAE;AACF,+EAA+E;AAC/E,2EAA2E;AAC3E,4BAA4B;AAE5B,OAAO,EACL,kBAAkB,EAClB,UAAU,GAEX,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,IAAI,EAAe,MAAM,SAAS,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AA+BnD,SAAS,iBAAiB,CAAC,OAAqB;IAC9C,MAAM,IAAI,GAAG,OAAO,CAAC,QAAuD,CAAC;IAC7E,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,EAAE,IAAI,KAAK,WAAW;YAAE,SAAS;QACtC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;QACpB,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,OAAO,CAAC,CAAC;QACpC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACrB,OAAO,CAAC;iBACL,GAAG,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAE,CAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;iBACzI,IAAI,CAAC,EAAE,CAAC;iBACR,IAAI,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,EAAE,GAAG,CAAC,CAAY,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;AAExD,0EAA0E;AAC1E,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,IAAe;IACrD,MAAM,IAAI,GAAiB,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;IACvH,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,SAAS,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,kBAAkB,CAAC;YAC3C,KAAK;YACL,YAAY,EAAE,OAAO;YACrB,aAAa,EAAE,QAAQ;YACvB,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;YAChD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;YACxC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3B,MAAM,GAAG,GAAiB;gBACxB,GAAG,IAAI;gBACP,MAAM,EAAE,iBAAiB,CAAC,OAAO,CAAC;gBAClC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,KAAK;gBAC5C,EAAE;gBACF,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;aAClC,CAAC;YACF,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,KAAK,CAAC;gBAAE,GAAG,CAAC,KAAK,GAAG,wEAAwE,CAAC;YACnH,OAAO,GAAG,CAAC;QACb,CAAC;gBAAS,CAAC;YACT,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,uEAAuE;QAC5F,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IACxE,CAAC;AACH,CAAC;AAED,8EAA8E;AAE9E,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAC7B;IACE,MAAM,EAAE,IAAI,CAAC,KAAK,CAChB,IAAI,CAAC,MAAM,CAAC;QACV,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,2BAA2B,EAAE,CAAC;QACjE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,mCAAmC,EAAE,CAAC;QACxE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,cAAc,EAAE,CAAC;KACrD,CAAC,CACH;IACD,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,sCAAsC,EAAE,CAAC;CAC7E,EACD,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAC/B,CAAC;AAGF,SAAS,cAAc;IACrB,IAAI,QAA8B,CAAC;IACnC,MAAM,IAAI,GAAG,UAAU,CAAC;QACtB,IAAI,EAAE,eAAe;QACrB,KAAK,EAAE,eAAe;QACtB,WAAW,EAAE,uGAAuG;QACpH,UAAU,EAAE,WAAW;QACvB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAgB,EAAE,EAAE;YACvC,QAAQ,GAAG,MAAM,CAAC;YAClB,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,MAAM,CAAC,MAAM,CAAC,MAAM,oBAAoB,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAChI,CAAC;KACF,CAAC,CAAC;IACH,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;AACvC,CAAC;AAED,mFAAmF;AACnF,KAAK,UAAU,KAAK,CAAC,IAAU,EAAE,OAAuB,EAAE,SAAoB;IAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC;IAC3D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;IAExF,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW;IAC9E,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,cAAc,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1F,MAAM,OAAO,GAAG,MAAM,SAAS,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3E,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,cAAc,EAAE,CAAC;IACvC,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,kBAAkB,CAAC;QAC3C,KAAK;QACL,YAAY,EAAE,OAAO;QACrB,aAAa,EAAE,QAAQ;QACvB,OAAO,EAAE,KAAK;QACd,WAAW,EAAE,CAAC,IAAI,CAAC;QACnB,KAAK,EAAE,CAAC,eAAe,CAAC;KACzB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG;QACb,mBAAmB,MAAM,CAAC,MAAM,mCAAmC,IAAI,CAAC,UAAU,kCAAkC;QACpH,SAAS,IAAI,CAAC,KAAK,EAAE;QACrB,EAAE;QACF,iEAAiE,IAAI,CAAC,UAAU,sFAAsF;QACtK,6FAA6F;QAC7F,EAAE;QACF,MAAM;KACP,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7B,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,MAAM,OAAO,CAAC,MAAM,CAAC,6EAA6E,CAAC,CAAC;YACpG,GAAG,GAAG,GAAG,EAAE,CAAC;QACd,CAAC;QACD,cAAc,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;QAE3E,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,MAAM,GAAiB,GAAG,CAAC,MAAM;aACpC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5E,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAmB,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACnF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;IACzF,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,OAAO,EAAE,CAAC;IACpB,CAAC;AACH,CAAC;AAQD,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAU,EAAE,UAAuB,EAAE,OAAuB,EAAE;IAC7F,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACzB,MAAM,CAAC,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACtC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,YAAY,MAAM,CAAC,CAAC;QACzI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,IAAI,UAAU,CAAC,CAAC,CAAE,CAAC;IAC/C,GAAG,CAAC,gBAAgB,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAC1E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC3D,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,UAAsB,EAAE,aAAyB,QAAQ;IACnG,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,cAAc,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC;AACjI,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// Build persistence — checkpoint a run so a halt/crash/cancel can resume without
|
|
2
|
+
// re-paying for finished tasks. The orchestrator itself stays fs-free; this module
|
|
3
|
+
// (and run-build) own the disk I/O.
|
|
4
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
5
|
+
export function newBuildState(id, tasks, idea, mode) {
|
|
6
|
+
return { id, idea, mode, tasks, outcomes: [], totalCost: 0, status: "running" };
|
|
7
|
+
}
|
|
8
|
+
export function saveState(state, path) {
|
|
9
|
+
writeFileSync(path, JSON.stringify(state, null, 2) + "\n");
|
|
10
|
+
}
|
|
11
|
+
export function loadState(path) {
|
|
12
|
+
if (!existsSync(path))
|
|
13
|
+
return undefined;
|
|
14
|
+
try {
|
|
15
|
+
return JSON.parse(readFileSync(path, "utf-8"));
|
|
16
|
+
}
|
|
17
|
+
catch (e) {
|
|
18
|
+
throw new Error(`Bad build state at ${path}: ${e instanceof Error ? e.message : e}`);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
/** Which task ids are already finished (last outcome wins; a failed attempt does not
|
|
22
|
+
* count). Used to skip on resume. */
|
|
23
|
+
export function completedIds(state) {
|
|
24
|
+
return new Set(state.outcomes.filter((o) => !o.error).map((o) => o.taskId));
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=build-state.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build-state.js","sourceRoot":"","sources":["../src/build-state.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,mFAAmF;AACnF,oCAAoC;AAEpC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAqBlE,MAAM,UAAU,aAAa,CAAC,EAAU,EAAE,KAAa,EAAE,IAAa,EAAE,IAA0B;IAChG,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAiB,EAAE,IAAY;IACvD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAe,CAAC;IAC/D,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvF,CAAC;AACH,CAAC;AAED;sCACsC;AACtC,MAAM,UAAU,YAAY,CAAC,KAAiB;IAC5C,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;AAC9E,CAAC"}
|
package/dist/burndown.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// Burndown — tasks remaining and cumulative spend across the build. There are no
|
|
2
|
+
// timestamps in build-state, so the X axis is task-completion order (step 1..N),
|
|
3
|
+
// which is the natural timeline for a build. Retries add a step (and cost) without
|
|
4
|
+
// burning down a task, so they show up as flat-remaining / rising-cost.
|
|
5
|
+
const round2 = (n) => Math.round(n * 100) / 100;
|
|
6
|
+
export function computeBurndown(state) {
|
|
7
|
+
const taskCount = state.tasks.length;
|
|
8
|
+
const done = new Set();
|
|
9
|
+
let cum = 0;
|
|
10
|
+
const steps = [];
|
|
11
|
+
for (const o of state.outcomes) {
|
|
12
|
+
const retry = done.has(o.taskId);
|
|
13
|
+
done.add(o.taskId);
|
|
14
|
+
cum += o.cost;
|
|
15
|
+
steps.push({ taskId: o.taskId, remaining: taskCount - done.size, cumCost: round2(cum), retry });
|
|
16
|
+
}
|
|
17
|
+
return { taskCount, totalCost: round2(cum), steps };
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=burndown.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"burndown.js","sourceRoot":"","sources":["../src/burndown.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,iFAAiF;AACjF,mFAAmF;AACnF,wEAAwE;AAiBxE,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AAExD,MAAM,UAAU,eAAe,CAAC,KAAiB;IAC/C,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACnB,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC;AACtD,CAAC"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// Self-calibrating token estimates. After each real task, we record its measured
|
|
2
|
+
// token usage per (capability, difficulty) AND per (capability, difficulty, model).
|
|
3
|
+
// The router prefers the per-model average (a Haiku and an Opus run of the same task
|
|
4
|
+
// differ a lot), then the generic bucket, then the static table. Persisted globally.
|
|
5
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
6
|
+
import { homedir } from "node:os";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
const MIN_SAMPLES = 2; // trust calibration only after a couple of runs
|
|
9
|
+
const MAX_N = 20; // cap so old runs don't dominate
|
|
10
|
+
function calPath() {
|
|
11
|
+
return join(homedir(), ".projectinator", "calibration.json");
|
|
12
|
+
}
|
|
13
|
+
const key = (c, d, model) => (model ? `${c}/${d}/${model}` : `${c}/${d}`);
|
|
14
|
+
function load() {
|
|
15
|
+
try {
|
|
16
|
+
if (!existsSync(calPath()))
|
|
17
|
+
return {};
|
|
18
|
+
return JSON.parse(readFileSync(calPath(), "utf-8"));
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return {};
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function save(cal) {
|
|
25
|
+
try {
|
|
26
|
+
mkdirSync(join(homedir(), ".projectinator"), { recursive: true });
|
|
27
|
+
writeFileSync(calPath(), JSON.stringify(cal, null, 2) + "\n");
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
/* best effort — never break a build on a calibration write */
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function fold(cal, k, inputTotal, output, cachedFraction, ms) {
|
|
34
|
+
const prev = cal[k];
|
|
35
|
+
if (!prev) {
|
|
36
|
+
cal[k] = { input: inputTotal, output, cachedFraction, n: 1, ...(ms ? { ms } : {}) };
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const n = Math.min(prev.n, MAX_N);
|
|
40
|
+
cal[k] = {
|
|
41
|
+
input: (prev.input * n + inputTotal) / (n + 1),
|
|
42
|
+
output: (prev.output * n + output) / (n + 1),
|
|
43
|
+
cachedFraction: (prev.cachedFraction * n + cachedFraction) / (n + 1),
|
|
44
|
+
n: prev.n + 1,
|
|
45
|
+
...(ms ? { ms: prev.ms ? (prev.ms * n + ms) / (n + 1) : ms } : prev.ms ? { ms: prev.ms } : {}),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
/** Fold a real measurement into the running averages for its bucket and its model. */
|
|
49
|
+
export function recordActual(capability, difficulty, inputTotal, output, cachedFraction, modelId, durationMs) {
|
|
50
|
+
if (!(inputTotal > 0))
|
|
51
|
+
return;
|
|
52
|
+
const cal = load();
|
|
53
|
+
fold(cal, key(capability, difficulty), inputTotal, output, cachedFraction, durationMs);
|
|
54
|
+
if (modelId)
|
|
55
|
+
fold(cal, key(capability, difficulty, modelId), inputTotal, output, cachedFraction, durationMs);
|
|
56
|
+
save(cal);
|
|
57
|
+
}
|
|
58
|
+
/** Typical wall-clock ms for a bucket (per-model if known), or undefined before any timed run. */
|
|
59
|
+
export function expectedDurationMs(capability, difficulty, modelId) {
|
|
60
|
+
const cal = load();
|
|
61
|
+
const s = (modelId && cal[key(capability, difficulty, modelId)]) || cal[key(capability, difficulty)];
|
|
62
|
+
return s?.ms;
|
|
63
|
+
}
|
|
64
|
+
/** All recorded samples keyed "capability/difficulty[/model]" (for the accuracy view). */
|
|
65
|
+
export function allSamples() {
|
|
66
|
+
return load();
|
|
67
|
+
}
|
|
68
|
+
function fromSample(s) {
|
|
69
|
+
if (!s || s.n < MIN_SAMPLES)
|
|
70
|
+
return undefined;
|
|
71
|
+
return {
|
|
72
|
+
input: Math.round(s.input),
|
|
73
|
+
output: Math.round(s.output),
|
|
74
|
+
cachedInputFraction: Math.min(0.95, Math.max(0, s.cachedFraction)),
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
/** Calibrated estimate for a bucket, once enough samples exist (model-agnostic). */
|
|
78
|
+
export function calibratedTokens(capability, difficulty) {
|
|
79
|
+
return fromSample(load()[key(capability, difficulty)]);
|
|
80
|
+
}
|
|
81
|
+
/** Calibrated estimate for a bucket ON A SPECIFIC MODEL, once enough samples exist.
|
|
82
|
+
* Undefined when that model hasn't run this bucket enough — callers keep their estimate. */
|
|
83
|
+
export function modelCalibratedTokens(capability, difficulty, modelId) {
|
|
84
|
+
return fromSample(load()[key(capability, difficulty, modelId)]);
|
|
85
|
+
}
|
|
86
|
+
//# sourceMappingURL=calibration.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"calibration.js","sourceRoot":"","sources":["../src/calibration.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,oFAAoF;AACpF,qFAAqF;AACrF,qFAAqF;AAErF,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAcjC,MAAM,WAAW,GAAG,CAAC,CAAC,CAAC,gDAAgD;AACvE,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,iCAAiC;AAEnD,SAAS,OAAO;IACd,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,gBAAgB,EAAE,kBAAkB,CAAC,CAAC;AAC/D,CAAC;AACD,MAAM,GAAG,GAAG,CAAC,CAAa,EAAE,CAAa,EAAE,KAAc,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAE3G,SAAS,IAAI;IACX,IAAI,CAAC;QACH,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YAAE,OAAO,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE,OAAO,CAAC,CAAgB,CAAC;IACrE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CAAC,GAAgB;IAC5B,IAAI,CAAC;QACH,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,gBAAgB,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClE,aAAa,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAChE,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;IAChE,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CAAC,GAAgB,EAAE,CAAS,EAAE,UAAkB,EAAE,MAAc,EAAE,cAAsB,EAAE,EAAW;IAChH,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACpB,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;QACpF,OAAO;IACT,CAAC;IACD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAClC,GAAG,CAAC,CAAC,CAAC,GAAG;QACP,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAC5C,cAAc,EAAE,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACpE,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC;QACb,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/F,CAAC;AACJ,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,YAAY,CAC1B,UAAsB,EACtB,UAAsB,EACtB,UAAkB,EAClB,MAAc,EACd,cAAsB,EACtB,OAAgB,EAChB,UAAmB;IAEnB,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC;QAAE,OAAO;IAC9B,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC;IACnB,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,CAAC,CAAC;IACvF,IAAI,OAAO;QAAE,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,CAAC,CAAC;IAC7G,IAAI,CAAC,GAAG,CAAC,CAAC;AACZ,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,kBAAkB,CAAC,UAAsB,EAAE,UAAsB,EAAE,OAAgB;IACjG,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC;IACnB,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IACrG,OAAO,CAAC,EAAE,EAAE,CAAC;AACf,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,UAAU;IACxB,OAAO,IAAI,EAAE,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,CAAqB;IACvC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,WAAW;QAAE,OAAO,SAAS,CAAC;IAC9C,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;QAC1B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5B,mBAAmB,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC;KACnE,CAAC;AACJ,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,gBAAgB,CAAC,UAAsB,EAAE,UAAsB;IAC7E,OAAO,UAAU,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;6FAC6F;AAC7F,MAAM,UAAU,qBAAqB,CAAC,UAAsB,EAAE,UAAsB,EAAE,OAAe;IACnG,OAAO,UAAU,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC"}
|