ultimate-pi 0.2.7 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/harness-eval/SKILL.md +1 -1
- package/.agents/skills/harness-governor/SKILL.md +2 -2
- package/.agents/skills/harness-spec/SKILL.md +1 -1
- package/.pi/PACKAGING.md +3 -2
- package/.pi/extensions/custom-header.ts +0 -17
- package/.pi/extensions/pi-model-router-harness.ts +42 -0
- package/.pi/extensions/policy-gate.ts +18 -0
- package/.pi/extensions/provider-payload-sanitize.ts +66 -0
- package/.pi/extensions/sentrux-rules-sync.ts +0 -18
- package/.pi/harness/README.md +3 -2
- package/.pi/harness/docs/adrs/0004-defer-ci-agent-smoke.md +1 -1
- package/.pi/harness/docs/adrs/0006-sentrux-dual-layer.md +1 -1
- package/.pi/harness/docs/adrs/0009-sentrux-rules-lifecycle.md +2 -2
- package/.pi/harness/evals/smoke/README.md +1 -1
- package/.pi/harness/evolution/README.md +1 -1
- package/.pi/harness/evolution/chaos-drill.md +1 -1
- package/.pi/prompts/harness-setup.md +42 -35
- package/.pi/scripts/README.md +25 -9
- package/.pi/scripts/harness-cli-verify.sh +4 -2
- package/.pi/scripts/harness-seed-project-contracts.mjs +49 -0
- package/.pi/scripts/harness-sync-model-router.mjs +84 -0
- package/.pi/scripts/harness-verify.mjs +5 -3
- package/.pi/scripts/sentrux-rules-sync.mjs +2 -2
- package/.pi/scripts/vendor-sync-pi-model-router.sh +47 -0
- package/.pi/settings.example.json +0 -1
- package/.sentrux/rules.toml +1 -1
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +62 -0
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +8 -0
- package/biome.json +2 -1
- package/package.json +9 -10
- package/vendor/pi-model-router/.prettierignore +4 -0
- package/vendor/pi-model-router/.prettierrc +5 -0
- package/vendor/pi-model-router/AGENTS.md +39 -0
- package/vendor/pi-model-router/LICENSE +21 -0
- package/vendor/pi-model-router/README.md +99 -0
- package/vendor/pi-model-router/UPSTREAM_PIN.md +8 -0
- package/vendor/pi-model-router/docs/ARCHITECTURE.md +54 -0
- package/vendor/pi-model-router/extensions/commands.ts +720 -0
- package/vendor/pi-model-router/extensions/config.ts +348 -0
- package/vendor/pi-model-router/extensions/constants.ts +1 -0
- package/vendor/pi-model-router/extensions/index.ts +457 -0
- package/vendor/pi-model-router/extensions/provider.ts +529 -0
- package/vendor/pi-model-router/extensions/routing.ts +416 -0
- package/vendor/pi-model-router/extensions/state.ts +49 -0
- package/vendor/pi-model-router/extensions/types.ts +86 -0
- package/vendor/pi-model-router/extensions/ui.ts +130 -0
- package/vendor/pi-model-router/model-router.example.json +48 -0
- package/vendor/pi-model-router/package.json +48 -0
- package/vendor/pi-model-router/tsconfig.json +16 -0
- package/.pi/extensions/model-router-bootstrap.ts +0 -174
- package/.sentrux/.harness-rules-meta.json +0 -5
|
@@ -17,7 +17,7 @@ description: Run harness evaluation phase and emit EvalVerdict artifacts. Use wi
|
|
|
17
17
|
2. Gather evidence: tests, diff scope, policy state, debate consensus packet.
|
|
18
18
|
3. Emit verdict via `pi.appendEntry('harness-eval-verdict', { ... })` pattern (session custom entry).
|
|
19
19
|
4. When Sentrux enabled, ensure `harness-sentrux-signal` exists (stub or MCP) per ADR 0006.
|
|
20
|
-
5. Deterministic checks: `
|
|
20
|
+
5. Deterministic checks: `node "$UP_PKG/.pi/scripts/harness-verify.mjs"` (see `.pi/scripts/README.md`) and project test script.
|
|
21
21
|
|
|
22
22
|
## Verdict values
|
|
23
23
|
|
|
@@ -16,8 +16,8 @@ description: Enforce harness governance phases, policy gates, budgets, and promo
|
|
|
16
16
|
1. Read current phase from `/harness-policy-status` or session `harness-policy-state`.
|
|
17
17
|
2. Check ADRs: constitution (0001), eval promotion (0003), Sentrux (0006), drift (0007), rules lifecycle (0009).
|
|
18
18
|
3. For promotion: require eval pass, no abort lock, debate consensus if escalated, Sentrux when `HARNESS_SENTRUX_REQUIRED=true`.
|
|
19
|
-
4. After architecture changes: edit `.pi/harness/sentrux/architecture.manifest.json`, then `
|
|
20
|
-
5. Run `
|
|
19
|
+
4. After architecture changes: edit `.pi/harness/sentrux/architecture.manifest.json`, then `node "$UP_PKG/.pi/scripts/sentrux-rules-sync.mjs" --force` (see `.pi/scripts/README.md` for `UP_PKG`) or `/harness-sentrux-sync`.
|
|
20
|
+
5. Run `node "$UP_PKG/.pi/scripts/harness-verify.mjs"` before claiming release readiness.
|
|
21
21
|
|
|
22
22
|
## Spec Distiller integration
|
|
23
23
|
|
|
@@ -16,7 +16,7 @@ description: Draft or refine harness artifact contracts under .pi/harness/specs.
|
|
|
16
16
|
1. Read `.pi/harness/specs/README.md` for versioning rules (`contract_version`, optional fields only for compatible changes).
|
|
17
17
|
2. Edit or add schema under `.pi/harness/specs/`.
|
|
18
18
|
3. Update affected extensions to emit matching custom entries.
|
|
19
|
-
4. Run `
|
|
19
|
+
4. Run `node "$UP_PKG/.pi/scripts/harness-verify.mjs"` (see `.pi/scripts/README.md`).
|
|
20
20
|
5. Add or update an ADR under `.pi/harness/docs/adrs/` for breaking changes.
|
|
21
21
|
|
|
22
22
|
## Rules
|
package/.pi/PACKAGING.md
CHANGED
|
@@ -12,7 +12,7 @@ Aligned with [pi packages](https://github.com/badlogic/pi-mono/blob/main/package
|
|
|
12
12
|
|
|
13
13
|
Pi does **not** define `scripts`, `agents`, or `providers` in the manifest.
|
|
14
14
|
|
|
15
|
-
- **Harness scripts** → `.pi/scripts/`
|
|
15
|
+
- **Harness scripts** → `.pi/scripts/` — run via `node` / `bash` and `$UP_PKG` (see `.pi/scripts/README.md`); do not require npm script aliases in consumer `package.json`
|
|
16
16
|
- **Subagent agents** → `.pi/agents/**/*.md` (loaded by `@tintinweb/pi-subagents` from the **project** `.pi/agents/`; `/harness-setup` seeds them from the installed package)
|
|
17
17
|
- **Providers** → install via `bundledDependencies` + user settings, not a separate manifest directory
|
|
18
18
|
|
|
@@ -22,6 +22,7 @@ We use an explicit allowlist (not the whole `.pi/` tree) so dev-only artifacts n
|
|
|
22
22
|
|
|
23
23
|
- No `.pi/harness/runs/`, local `model-router.json`, or `firecrawl/.env`
|
|
24
24
|
- Ship `.pi/settings.example.json`, not `.pi/settings.json` (dev checkout uses `".."` local package)
|
|
25
|
+
- Include **`vendor/pi-model-router/`** ([`pi-model-router`](https://github.com/yeliu84/pi-model-router), MIT) — see repo [`THIRD_PARTY_NOTICES.md`](../THIRD_PARTY_NOTICES.md); refresh with `npm run vendor:sync-router`
|
|
25
26
|
|
|
26
27
|
## Settings
|
|
27
28
|
|
|
@@ -34,4 +35,4 @@ We use an explicit allowlist (not the whole `.pi/` tree) so dev-only artifacts n
|
|
|
34
35
|
|
|
35
36
|
Runtime pi extensions are regular `dependencies` (installed by `npm install` when pi installs the package). We do **not** use `bundledDependencies`: bundling pre-installs `node_modules` and breaks `npm install -g` / `pi update` for native modules such as `koffi` (empty stub dir, postinstall fails).
|
|
36
37
|
|
|
37
|
-
`@mariozechner/pi-coding-agent`
|
|
38
|
+
`@mariozechner/pi-coding-agent` (and sibling `@mariozechner/pi-ai`, `pi-tui`, `pi-agent-core` used by the vendored router) are provided by the Pi install / hoisted from the peer; ultimate-pi lists the latter three as `devDependencies` for `npm run check:ts`.
|
|
@@ -90,23 +90,6 @@ function ansiCell(
|
|
|
90
90
|
}
|
|
91
91
|
|
|
92
92
|
async function loadBanner(): Promise<string[]> {
|
|
93
|
-
// #region agent log
|
|
94
|
-
fetch("http://127.0.0.1:7928/ingest/a5d40896-34cb-4f12-97db-df7ada0b22f0", {
|
|
95
|
-
method: "POST",
|
|
96
|
-
headers: {
|
|
97
|
-
"Content-Type": "application/json",
|
|
98
|
-
"X-Debug-Session-Id": "7737a8",
|
|
99
|
-
},
|
|
100
|
-
body: JSON.stringify({
|
|
101
|
-
sessionId: "7737a8",
|
|
102
|
-
hypothesisId: "B",
|
|
103
|
-
location: "custom-header.ts:loadBanner",
|
|
104
|
-
message: "banner path",
|
|
105
|
-
data: { imagePath, cwd: process.cwd() },
|
|
106
|
-
timestamp: Date.now(),
|
|
107
|
-
}),
|
|
108
|
-
}).catch(() => {});
|
|
109
|
-
// #endregion
|
|
110
93
|
const Jimp = getJimpRuntime();
|
|
111
94
|
const image = await Jimp.read(imagePath);
|
|
112
95
|
resizeImageCompat(image, PIXEL_WIDTH, PIXEL_HEIGHT);
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vendored [pi-model-router](https://github.com/yeliu84/pi-model-router), gated behind
|
|
3
|
+
* a project-local `.pi/model-router.json` from `/harness-setup` so the extension
|
|
4
|
+
* (and built-in fallback tiers) never load before harness bootstrap.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
10
|
+
import vendorModelRouter from "../../vendor/pi-model-router/extensions/index.js";
|
|
11
|
+
|
|
12
|
+
function isHarnessRouterReady(cwd: string): boolean {
|
|
13
|
+
const path = join(cwd, ".pi", "model-router.json");
|
|
14
|
+
if (!existsSync(path)) {
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
try {
|
|
18
|
+
const data: unknown = JSON.parse(readFileSync(path, "utf8"));
|
|
19
|
+
if (typeof data !== "object" || data === null) {
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
const profiles = (data as { profiles?: unknown }).profiles;
|
|
23
|
+
return (
|
|
24
|
+
typeof profiles === "object" &&
|
|
25
|
+
profiles !== null &&
|
|
26
|
+
Object.keys(profiles).length > 0
|
|
27
|
+
);
|
|
28
|
+
} catch {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export default function piModelRouterHarness(pi: ExtensionAPI) {
|
|
34
|
+
const cwd = process.cwd();
|
|
35
|
+
if (!isHarnessRouterReady(cwd)) {
|
|
36
|
+
console.warn(
|
|
37
|
+
"[ultimate-pi] Model router disabled until `.pi/model-router.json` exists (generate via /harness-setup Step 3.5).",
|
|
38
|
+
);
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
vendorModelRouter(pi);
|
|
42
|
+
}
|
|
@@ -178,6 +178,24 @@ export default function policyGate(pi: ExtensionAPI) {
|
|
|
178
178
|
pi.on("before_agent_start", async (event) => {
|
|
179
179
|
const bootstrapPrompt = isBootstrapPrompt(event.prompt);
|
|
180
180
|
const abortSignal = hasAbortSignal(event.prompt);
|
|
181
|
+
|
|
182
|
+
// /harness-setup instructions mention `harness-plan` (e.g. gh label text). That
|
|
183
|
+
// substring must not force inferPhase() to "plan" or bootstrap stays blocked.
|
|
184
|
+
if (bootstrapPrompt) {
|
|
185
|
+
state.phase = "execute";
|
|
186
|
+
state.approvedPlan = true;
|
|
187
|
+
state.planId = null;
|
|
188
|
+
state.budgetBypass = true;
|
|
189
|
+
state.aborted = false;
|
|
190
|
+
state.abortReason = null;
|
|
191
|
+
state.abortedAt = null;
|
|
192
|
+
state.updatedAt = nowIso();
|
|
193
|
+
pi.appendEntry("harness-policy-state", state);
|
|
194
|
+
return {
|
|
195
|
+
systemPrompt: `${event.systemPrompt}\n\n[PolicyGate]\nPhase=${state.phase}; ApprovedPlan=${state.approvedPlan}; PlanId=${state.planId ?? "none"}; Aborted=${state.aborted}; Bootstrap=harness-setup.`,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
|
|
181
199
|
if (abortSignal) {
|
|
182
200
|
state.phase = "plan";
|
|
183
201
|
state.approvedPlan = false;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Strip provider-specific fields from LLM request payloads before HTTP send.
|
|
3
|
+
*
|
|
4
|
+
* Strict OpenAI-compatible gateways return 400 when assistant history includes
|
|
5
|
+
* a top-level `reasoning` key (Cursor/thinking transcripts). Pi builds clean
|
|
6
|
+
* chat params; this is a safety net for any extra fields that slip through.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type {
|
|
10
|
+
BeforeProviderRequestEvent,
|
|
11
|
+
ExtensionAPI,
|
|
12
|
+
} from "@mariozechner/pi-coding-agent";
|
|
13
|
+
|
|
14
|
+
const CHAT_MESSAGE_EXTRA_KEYS = [
|
|
15
|
+
"reasoning",
|
|
16
|
+
"reasoning_text",
|
|
17
|
+
"chain_of_thought",
|
|
18
|
+
"chainOfThought",
|
|
19
|
+
"thinking",
|
|
20
|
+
"thought",
|
|
21
|
+
] as const;
|
|
22
|
+
|
|
23
|
+
function stripExtraChatFields(message: unknown): unknown {
|
|
24
|
+
if (
|
|
25
|
+
message === null ||
|
|
26
|
+
typeof message !== "object" ||
|
|
27
|
+
Array.isArray(message)
|
|
28
|
+
) {
|
|
29
|
+
return message;
|
|
30
|
+
}
|
|
31
|
+
const m = message as Record<string, unknown>;
|
|
32
|
+
if (typeof m.role !== "string") {
|
|
33
|
+
return message;
|
|
34
|
+
}
|
|
35
|
+
let touched = false;
|
|
36
|
+
const next = { ...m };
|
|
37
|
+
for (const k of CHAT_MESSAGE_EXTRA_KEYS) {
|
|
38
|
+
if (k in next) {
|
|
39
|
+
delete next[k];
|
|
40
|
+
touched = true;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return touched ? next : message;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function sanitizePayload(payload: unknown): unknown {
|
|
47
|
+
if (payload === null || typeof payload !== "object") {
|
|
48
|
+
return payload;
|
|
49
|
+
}
|
|
50
|
+
const body = payload as Record<string, unknown>;
|
|
51
|
+
const rawMessages = body.messages;
|
|
52
|
+
if (Array.isArray(rawMessages)) {
|
|
53
|
+
const messages = rawMessages.map(stripExtraChatFields);
|
|
54
|
+
if (messages.some((m, i) => m !== rawMessages[i])) {
|
|
55
|
+
return { ...body, messages };
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return payload;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export default function providerPayloadSanitize(pi: ExtensionAPI) {
|
|
63
|
+
pi.on("before_provider_request", (event: BeforeProviderRequestEvent) => {
|
|
64
|
+
return sanitizePayload(event.payload);
|
|
65
|
+
});
|
|
66
|
+
}
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import { spawn } from "node:child_process";
|
|
6
|
-
import { existsSync } from "node:fs";
|
|
7
6
|
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
8
7
|
import { resolveHarnessScript } from "./lib/harness-paths.js";
|
|
9
8
|
|
|
@@ -17,23 +16,6 @@ function resolveSyncScript(): string {
|
|
|
17
16
|
|
|
18
17
|
function runSync(args: string[]): Promise<{ code: number; output: string }> {
|
|
19
18
|
const syncScript = resolveSyncScript();
|
|
20
|
-
// #region agent log
|
|
21
|
-
fetch("http://127.0.0.1:7928/ingest/a5d40896-34cb-4f12-97db-df7ada0b22f0", {
|
|
22
|
-
method: "POST",
|
|
23
|
-
headers: {
|
|
24
|
-
"Content-Type": "application/json",
|
|
25
|
-
"X-Debug-Session-Id": "7737a8",
|
|
26
|
-
},
|
|
27
|
-
body: JSON.stringify({
|
|
28
|
-
sessionId: "7737a8",
|
|
29
|
-
hypothesisId: "C",
|
|
30
|
-
location: "sentrux-rules-sync.ts:runSync",
|
|
31
|
-
message: "sync script path",
|
|
32
|
-
data: { syncScript, cwd: process.cwd(), exists: existsSync(syncScript) },
|
|
33
|
-
timestamp: Date.now(),
|
|
34
|
-
}),
|
|
35
|
-
}).catch(() => {});
|
|
36
|
-
// #endregion
|
|
37
19
|
return new Promise((resolve) => {
|
|
38
20
|
const child = spawn(process.execPath, [syncScript, ...args], {
|
|
39
21
|
cwd: process.cwd(),
|
package/.pi/harness/README.md
CHANGED
|
@@ -17,8 +17,9 @@ This scaffold is intentionally minimal and safe to adopt incrementally.
|
|
|
17
17
|
## Verification
|
|
18
18
|
|
|
19
19
|
```bash
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
UP_PKG="$(node -p "require('path').dirname(require.resolve('ultimate-pi/package.json'))")"
|
|
21
|
+
node "$UP_PKG/.pi/scripts/harness-verify.mjs"
|
|
22
|
+
node "$UP_PKG/.pi/scripts/sentrux-rules-sync.mjs" --force # after editing sentrux/architecture.manifest.json
|
|
22
23
|
```
|
|
23
24
|
|
|
24
25
|
## Governance Extensions
|
|
@@ -15,7 +15,7 @@ Running full agent smoke or A/B harness comparisons in CI has high token cost an
|
|
|
15
15
|
2. Deterministic schema/fixture CI is green for ≥4 weeks.
|
|
16
16
|
3. At least 20 manual harness runs with `harness_run_completed` in PostHog.
|
|
17
17
|
|
|
18
|
-
Phase 2 ships **deterministic** eval fixtures only (`
|
|
18
|
+
Phase 2 ships **deterministic** eval fixtures only (`node "$UP_PKG/.pi/scripts/harness-verify.mjs"`; see `.pi/scripts/README.md`).
|
|
19
19
|
|
|
20
20
|
## Consequences
|
|
21
21
|
|
|
@@ -10,7 +10,7 @@ Evaluator trust requires both programmatic gates (policy, budget, integrity) and
|
|
|
10
10
|
## Decision
|
|
11
11
|
|
|
12
12
|
1. **Rules file:** `.sentrux/rules.toml` synced from manifest — see [ADR 0009](0009-sentrux-rules-lifecycle.md).
|
|
13
|
-
2. **CLI gate:** `
|
|
13
|
+
2. **CLI gate:** `node "$UP_PKG/.pi/scripts/harness-verify.mjs"` fails if `HARNESS_SENTRUX_REQUIRED=true` and no `harness-sentrux-signal` stub/file exists for the run (placeholder until MCP wired). Resolve `$UP_PKG` via [.pi/scripts/README.md](../../../scripts/README.md).
|
|
14
14
|
3. **MCP layer (Q2+):** Evaluator sessions must record at least one Sentrux observation before `harness_eval_verdict` promotion when Sentrux is enabled.
|
|
15
15
|
4. Observations flow through `observation-bus.ts` as `HarnessObservation` envelopes.
|
|
16
16
|
5. PostHog event: `harness_sentrux_signal` with `signal_type` and `score` only — no secrets.
|
|
@@ -11,13 +11,13 @@ Sentrux enforces architecture via [`.sentrux/rules.toml`](https://sentrux.dev/do
|
|
|
11
11
|
|
|
12
12
|
1. **Canonical source:** [`.pi/harness/sentrux/architecture.manifest.json`](../../sentrux/architecture.manifest.json) — layers, boundaries, global constraints.
|
|
13
13
|
2. **Generated artifact:** `.sentrux/rules.toml` — committed to git; managed block between `harness:managed:start/end` markers.
|
|
14
|
-
3. **Sync command:** `
|
|
14
|
+
3. **Sync command:** `node "$UP_PKG/.pi/scripts/sentrux-rules-sync.mjs" --force` (resolve `$UP_PKG` via [.pi/scripts/README.md](../../../scripts/README.md)).
|
|
15
15
|
4. **Pi command:** `/harness-sentrux-sync` via `sentrux-rules-sync.ts` extension.
|
|
16
16
|
5. **When to sync:**
|
|
17
17
|
- `/harness-setup` Step 2.8 (after sentrux install)
|
|
18
18
|
- After editing `architecture.manifest.json`
|
|
19
19
|
- On `agent_end` when harness phase is `plan` or `merge`
|
|
20
|
-
- `
|
|
20
|
+
- `node "$UP_PKG/.pi/scripts/harness-verify.mjs"` fails if manifest hash ≠ last sync (`--check`)
|
|
21
21
|
6. **Custom rules:** TOML outside the managed block is preserved on sync.
|
|
22
22
|
|
|
23
23
|
## Consequences
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# Deterministic harness smoke evals
|
|
2
2
|
|
|
3
|
-
No LLM calls. Used by `
|
|
3
|
+
No LLM calls. Used by `node "$UP_PKG/.pi/scripts/harness-verify.mjs"` and CI (see `.pi/scripts/README.md` for `UP_PKG`).
|
|
4
4
|
|
|
5
5
|
Fixtures validate JSON schemas and golden expectations for trust-layer artifacts.
|
|
@@ -5,7 +5,7 @@ Self-healing and meta-optimization read **JSONL first** (`.pi/harness/runs/*/eve
|
|
|
5
5
|
## Components
|
|
6
6
|
|
|
7
7
|
- `self-healing-rules.json` — pattern → suggested remediation
|
|
8
|
-
- `meta-optimizer.mjs` — scans run index, proposes router/tuning deltas
|
|
8
|
+
- `meta-optimizer.mjs` — scans run index, proposes router/tuning deltas; run `node "$UP_PKG/.pi/harness/evolution/meta-optimizer.mjs"` (see `.pi/scripts/README.md`).
|
|
9
9
|
- `chaos-drill.md` — manual chaos / failure injection checklist
|
|
10
10
|
|
|
11
11
|
PostHog `harness_*` events are for dashboards; JSONL is the optimization source of truth per ADR 0008.
|
|
@@ -18,7 +18,7 @@ Run quarterly or before major harness releases. **No automation in Phase 2.**
|
|
|
18
18
|
## Pass criteria
|
|
19
19
|
|
|
20
20
|
- All five scenarios produce expected custom entries and matching `harness_*` PostHog events within 60s.
|
|
21
|
-
- `
|
|
21
|
+
- `node "$UP_PKG/.pi/scripts/harness-verify.mjs"` still passes after drill (see `.pi/scripts/README.md`).
|
|
22
22
|
|
|
23
23
|
## Rollback
|
|
24
24
|
|
|
@@ -41,6 +41,8 @@ echo "ultimate-pi package: $UP_PKG"
|
|
|
41
41
|
|
|
42
42
|
For extension package names, read **`$UP_PKG/.pi/settings.example.json`** (shipped template). Merge its `packages` array into the **project** `.pi/settings.json` if missing — do not copy the repo-dev `.pi/settings.json` from the package (it may contain `".."` and is not published).
|
|
43
43
|
|
|
44
|
+
If `require.resolve('ultimate-pi/package.json')` fails (bare clone without resolving the package name), run from **this repository root**: `UP_PKG="$(pwd)"`.
|
|
45
|
+
|
|
44
46
|
## Step 0.5 — Graphify (skip if `--skip-graphify`)
|
|
45
47
|
|
|
46
48
|
**Critical:** `graphify . --wiki` and `graphify . --update` are **invalid** CLI (error: `unknown command '.'`). Use only:
|
|
@@ -57,12 +59,16 @@ Run from the **project root** (the external repo root, not ultimate-pi unless th
|
|
|
57
59
|
```bash
|
|
58
60
|
mkdir -p ./raw .pi/harness/specs .pi/harness/runs .pi/harness/incidents .pi/harness/debates
|
|
59
61
|
|
|
60
|
-
#
|
|
61
|
-
|
|
62
|
-
|
|
62
|
+
# Copy JSON schemas + specs README from the package so plan-packet.schema.json exists
|
|
63
|
+
# in the target repo immediately (before graphify or policy-gated planning).
|
|
64
|
+
node "$UP_PKG/.pi/scripts/harness-seed-project-contracts.mjs" "$(pwd)"
|
|
65
|
+
|
|
66
|
+
# Bundled with ultimate-pi harness; $UP_PKG is set in Step 0
|
|
67
|
+
bash "$UP_PKG/.pi/scripts/harness-graphify-bootstrap.sh"
|
|
68
|
+
# Developing ultimate-pi from repo root: UP_PKG="$(pwd)" then same command
|
|
63
69
|
|
|
64
70
|
# Pass --force when $ARGUMENTS contains --force to rebuild an existing graph:
|
|
65
|
-
#
|
|
71
|
+
# bash "$UP_PKG/.pi/scripts/harness-graphify-bootstrap.sh" --force
|
|
66
72
|
```
|
|
67
73
|
|
|
68
74
|
If the bootstrap script is missing, run it from the installed ultimate-pi package (`.pi/scripts/` inside the npm package), or execute equivalent steps manually:
|
|
@@ -220,9 +226,9 @@ If user chose **cloud**, skip all 1.5.x steps. Just note:
|
|
|
220
226
|
Run the bundled verifier from the **project root**. It installs missing npm globals, fixes common **Linux system dependencies** (Chrome libs for `agent-browser`), runs smoke tests, and exits non-zero if a required tool fails.
|
|
221
227
|
|
|
222
228
|
```bash
|
|
223
|
-
|
|
224
|
-
# ultimate-pi checkout:
|
|
225
|
-
# Reinstall everything:
|
|
229
|
+
bash "$UP_PKG/.pi/scripts/harness-cli-verify.sh"
|
|
230
|
+
# ultimate-pi checkout: same (ensure Step 0 set UP_PKG="$(pwd)" or used require.resolve)
|
|
231
|
+
# Reinstall everything: bash "$UP_PKG/.pi/scripts/harness-cli-verify.sh" --force
|
|
226
232
|
```
|
|
227
233
|
|
|
228
234
|
**Required (script must exit 0):** firecrawl-cli, ctx7, biome, ast-grep (`sg`), sentrux (when harness manifest present).
|
|
@@ -237,14 +243,14 @@ sudo apt-get install -y libnss3 libnspr4 libgbm1 libatk1.0-0 libatk-bridge2.0-0
|
|
|
237
243
|
libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 \
|
|
238
244
|
libasound2 libpango-1.0-0 libcairo2 libx11-6 libxcb1 libxext6 fonts-liberation
|
|
239
245
|
agent-browser install --with-deps
|
|
240
|
-
|
|
246
|
+
bash "$UP_PKG/.pi/scripts/harness-cli-verify.sh"
|
|
241
247
|
```
|
|
242
248
|
|
|
243
249
|
**Do not continue** past Step 2 if `harness-cli-verify.sh` exits non-zero.
|
|
244
250
|
|
|
245
251
|
### Manual reference (if script missing in target repo)
|
|
246
252
|
|
|
247
|
-
Use `
|
|
253
|
+
Use `bash "$UP_PKG/.pi/scripts/harness-cli-verify.sh"` (see Step 0 for `UP_PKG`), or install tools individually:
|
|
248
254
|
|
|
249
255
|
### 2.1 — firecrawl-cli (Web Search + Scrape + Crawl + Interact + Download + Parse)
|
|
250
256
|
|
|
@@ -420,10 +426,7 @@ Configure MCP server in `.pi/mcp.json` (see Step 4.3).
|
|
|
420
426
|
|
|
421
427
|
Generate architectural rules from the harness manifest (creates/updates `.sentrux/rules.toml`):
|
|
422
428
|
```bash
|
|
423
|
-
|
|
424
|
-
npm run harness:sentrux-sync
|
|
425
|
-
# From an external project (after pi install npm:ultimate-pi):
|
|
426
|
-
npm run harness:sentrux-sync
|
|
429
|
+
node "$UP_PKG/.pi/scripts/sentrux-rules-sync.mjs" --force
|
|
427
430
|
# Or in pi: /harness-sentrux-sync
|
|
428
431
|
```
|
|
429
432
|
|
|
@@ -441,7 +444,9 @@ sentrux gate --save . 2>/dev/null || echo "Baseline will be saved on first gate
|
|
|
441
444
|
|
|
442
445
|
## Step 3 — Pi Extension Packages
|
|
443
446
|
|
|
444
|
-
Bundled extensions load from the installed `ultimate-pi` package.
|
|
447
|
+
Bundled extensions load from the installed `ultimate-pi` package. **Per-turn model routing** comes from a **vendored** fork of [`yeliu84/pi-model-router`](https://github.com/yeliu84/pi-model-router) in `vendor/pi-model-router/`, wired through [`.pi/extensions/pi-model-router-harness.ts`](.pi/extensions/pi-model-router-harness.ts). The harness **gates** activation on `.pi/model-router.json` (Step **3.5** below) so `router/auto` and built-in tiers such as `openai/gpt-5.4-pro` cannot load prematurely. Attribution: see [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) and `vendor/pi-model-router/UPSTREAM_PIN.md`. Maintainer refresh: `npm run vendor:sync-router`.
|
|
448
|
+
|
|
449
|
+
Optionally install the companion lockfile used in development:
|
|
445
450
|
|
|
446
451
|
```bash
|
|
447
452
|
UP_PKG="$(node -p "require('path').dirname(require.resolve('ultimate-pi/package.json'))")"
|
|
@@ -453,7 +458,7 @@ else
|
|
|
453
458
|
fi
|
|
454
459
|
```
|
|
455
460
|
|
|
456
|
-
Merge extension entries from `$UP_PKG/.pi/settings.example.json` into this project's `.pi/settings.json` `packages` array (add any missing `npm:…` entries; keep existing user packages).
|
|
461
|
+
Merge extension entries from `$UP_PKG/.pi/settings.example.json` into this project's `.pi/settings.json` `packages` array (add any missing `npm:…` entries; keep existing user packages). **Do not add** `npm:@yeliu84/pi-model-router` (superseded by the vendored router).
|
|
457
462
|
|
|
458
463
|
Verify each package:
|
|
459
464
|
|
|
@@ -462,7 +467,8 @@ Verify each package:
|
|
|
462
467
|
| `@posthog/pi` | Analytics event capture | F0 |
|
|
463
468
|
| `pi-lean-ctx` | Context runtime (read/bash/find/grep/MCP bridge) | F0 |
|
|
464
469
|
| `@tintinweb/pi-subagents` | L4 critic sub-agent spawn/control | P16 |
|
|
465
|
-
| `@
|
|
470
|
+
| `@sting8k/pi-vcc` | VCC compaction / conversation memory | Shipped |
|
|
471
|
+
| `pi-model-router` | Vendored (`vendor/`); activates after `.pi/model-router.json` exists | F0 |
|
|
466
472
|
|
|
467
473
|
## Step 3.5 — Model Router Configuration (Dynamic)
|
|
468
474
|
|
|
@@ -475,11 +481,12 @@ The script below:
|
|
|
475
481
|
3. Only writes if file doesn't exist yet (safe to re-run, will skip existing)
|
|
476
482
|
|
|
477
483
|
```bash
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
484
|
+
UP_PKG="$(node -p "require('path').dirname(require.resolve('ultimate-pi/package.json'))")"
|
|
485
|
+
|
|
486
|
+
# Verify vendored extension source ships with ultimate-pi
|
|
487
|
+
ls "$UP_PKG/vendor/pi-model-router/extensions/index.ts" 2>/dev/null \
|
|
488
|
+
&& echo "✓ vendored pi-model-router" \
|
|
489
|
+
|| echo "✗ missing vendor/pi-model-router"
|
|
483
490
|
|
|
484
491
|
# Generate config from detected providers (only if missing)
|
|
485
492
|
if [ -f .pi/model-router.json ]; then
|
|
@@ -587,15 +594,16 @@ console.log(` Medium tier: ${mediumModel}`);
|
|
|
587
594
|
console.log(` Low tier: ${lowModel}`);
|
|
588
595
|
GENDONE
|
|
589
596
|
fi
|
|
590
|
-
```
|
|
591
597
|
|
|
592
|
-
|
|
598
|
+
# Merge router defaults after config exists (never adds npm packages — router is vendored)
|
|
599
|
+
node "$UP_PKG/.pi/scripts/harness-sync-model-router.mjs"
|
|
600
|
+
```
|
|
593
601
|
|
|
594
|
-
|
|
602
|
+
Do NOT block. If generation fails, warn in report and continue (defaults script clears `defaultProvider` if it pointed at `router` while no config file exists).
|
|
595
603
|
|
|
596
|
-
|
|
604
|
+
**Router onboarding** — The vendored extension starts only after `.pi/model-router.json` appears. Running the script above prepares that file plus optional Pi defaults (**`router` / `auto`**) via `harness-sync-model-router.mjs` when `defaultProvider` was unset—then **`/reload`**.
|
|
597
605
|
|
|
598
|
-
|
|
606
|
+
Manual override: **`/router profile auto`** anytime after reload if they changed defaults.
|
|
599
607
|
|
|
600
608
|
## Step 3.6 — Seed `.pi/agents` (pi-subagents)
|
|
601
609
|
|
|
@@ -702,7 +710,7 @@ Created: $(date +%Y-%m-%d)
|
|
|
702
710
|
Re-run CLI verification (must pass unless `--skip-tools`):
|
|
703
711
|
|
|
704
712
|
```bash
|
|
705
|
-
|
|
713
|
+
bash "$UP_PKG/.pi/scripts/harness-cli-verify.sh"
|
|
706
714
|
```
|
|
707
715
|
|
|
708
716
|
Then run the remaining checks:
|
|
@@ -741,10 +749,9 @@ print(f'✓ knowledge graph built ({n} nodes)' if n else '✗ graph.json has 0 n
|
|
|
741
749
|
" 2>/dev/null || echo "✗ no graph built yet"
|
|
742
750
|
graphify hook status 2>/dev/null && echo "✓ graphify git hooks installed" || echo "✗ graphify git hooks not installed"
|
|
743
751
|
|
|
744
|
-
# model router
|
|
745
|
-
ls "$UP_PKG/
|
|
746
|
-
|
|
747
|
-
&& echo "✓ model-router package" || echo "✗ model-router package"
|
|
752
|
+
# vendored model router
|
|
753
|
+
ls "$UP_PKG/vendor/pi-model-router/extensions/index.ts" 2>/dev/null \
|
|
754
|
+
&& echo "✓ vendored pi-model-router" || echo "✗ vendor/pi-model-router missing"
|
|
748
755
|
ls .pi/model-router.json 2>/dev/null && echo "✓ model-router config" || echo "✗ model-router config"
|
|
749
756
|
|
|
750
757
|
# raw folder for graphify sources
|
|
@@ -801,7 +808,7 @@ Output summary table:
|
|
|
801
808
|
|
|
802
809
|
Next steps:
|
|
803
810
|
1. If tools missing: re-run with `--force` or install individually
|
|
804
|
-
2. If graph not built: run `
|
|
811
|
+
2. If graph not built: run `bash "$UP_PKG/.pi/scripts/harness-graphify-bootstrap.sh"` (or `graphify update .` from project root)
|
|
805
812
|
3. If hooks not installed: run `graphify hook install`
|
|
806
813
|
4. If gh not authenticated: `gh auth login`
|
|
807
814
|
5. If self-hosted Firecrawl unhealthy: `docker compose -f firecrawl/docker-compose.yaml logs`
|
|
@@ -811,9 +818,9 @@ Next steps:
|
|
|
811
818
|
## Guard Rails
|
|
812
819
|
|
|
813
820
|
- **Internet required**: Several tools need npm registry access. Block if offline.
|
|
814
|
-
- **CLI verify script**: Step 2 and Step 5
|
|
821
|
+
- **CLI verify script**: Step 2 and Step 5 run `bash "$UP_PKG/.pi/scripts/harness-cli-verify.sh"` — installs npm globals, Linux Chrome system libs for `agent-browser`, and smoke-tests each tool. Block on non-zero exit.
|
|
815
822
|
- **Graphify requires Python 3.10+**: Check `python3 --version`. Block if too old.
|
|
816
|
-
- **Graphify bootstrap is mandatory** (unless `--skip-graphify`): Run `
|
|
823
|
+
- **Graphify bootstrap is mandatory** (unless `--skip-graphify`): Run `bash "$UP_PKG/.pi/scripts/harness-graphify-bootstrap.sh"`. Never use `graphify . --wiki`. Initial setup must run `graphify update .` and verify `graphify-out/graph.json` has nodes.
|
|
817
824
|
- **Python packages (Graphify)**: Before install, detect via PATH, `pip`/`pip3 show graphifyy`, `uv`, or apt. Prefer `uv tool install graphifyy`.
|
|
818
825
|
- **Node.js >= 18 required**: Some pi packages use modern Node APIs.
|
|
819
826
|
- **Docker required for self-hosted**: Step 1.5 needs Docker Engine + Compose. Block if install fails.
|
|
@@ -836,7 +843,7 @@ Next steps:
|
|
|
836
843
|
| Graphify install fails | Show installer output. Retry `uv tool install graphifyy` or `pip3 install --user graphifyy`. Ensure `~/.local/bin` is on PATH. |
|
|
837
844
|
| `graphify update .` fails | Block setup. Corpus may have no code files, or graphify not on PATH. Show stderr. |
|
|
838
845
|
| Invalid `graphify .` usage | Replace with `graphify update .` — the `.` subcommand does not exist. |
|
|
839
|
-
| graphify-out empty / 0 nodes | Re-run `
|
|
846
|
+
| graphify-out empty / 0 nodes | Re-run `bash "$UP_PKG/.pi/scripts/harness-graphify-bootstrap.sh" --force` from project root. |
|
|
840
847
|
| graphify hook install fails | Hooks need `.git/` directory. Verify inside git repo. Manual: `git config core.hooksPath .pi/git-hooks` |
|
|
841
848
|
| firecrawl auth failed | Show manual login instructions. Continue with other tools. |
|
|
842
849
|
| gh not installed | Show GitHub CLI install link. Skip label creation. |
|
package/.pi/scripts/README.md
CHANGED
|
@@ -2,16 +2,32 @@
|
|
|
2
2
|
|
|
3
3
|
These scripts ship inside the `ultimate-pi` npm package under `.pi/scripts/`.
|
|
4
4
|
|
|
5
|
-
Pi's package manifest (`package.json` → `pi`) only loads **extensions**, **skills**, **prompts**, and **themes** — there is no `scripts` field.
|
|
5
|
+
Pi's package manifest (`package.json` → `pi`) only loads **extensions**, **skills**, **prompts**, and **themes** — there is no `scripts` field. **Do not rely on npm `harness:*` scripts** in the consuming project's `package.json`; external repos that install `ultimate-pi` may not define them.
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
- Extensions resolving paths with `resolveHarnessScript()` in `.pi/extensions/lib/harness-paths.ts`
|
|
7
|
+
## Resolve `ultimate-pi` package root (`UP_PKG`)
|
|
9
8
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
9
|
+
**Consumer repo** (`ultimate-pi` in `node_modules`):
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
UP_PKG="$(node -p "require('path').dirname(require.resolve('ultimate-pi/package.json'))")"
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
**Developing this repo** (clone of `ultimate-pi`): from the repo root, `UP_PKG="$(pwd)"` (or the same `require.resolve` after `npm install`).
|
|
16
|
+
|
|
17
|
+
From **Typescript extensions**, use `resolveHarnessScript()` / `getHarnessPackageRoot()` in `.pi/extensions/lib/harness-paths.ts`.
|
|
18
|
+
|
|
19
|
+
## Invocations (from the consuming project root)
|
|
20
|
+
|
|
21
|
+
| Action | Command |
|
|
22
|
+
|--------|---------|
|
|
23
|
+
| Graphify bootstrap | `bash "$UP_PKG/.pi/scripts/harness-graphify-bootstrap.sh"` |
|
|
24
|
+
| CLI tool install + smoke tests | `bash "$UP_PKG/.pi/scripts/harness-cli-verify.sh"` |
|
|
25
|
+
| Deterministic harness checks | `node "$UP_PKG/.pi/scripts/harness-verify.mjs"` |
|
|
26
|
+
| Sentrux rules from manifest | `node "$UP_PKG/.pi/scripts/sentrux-rules-sync.mjs" --force` |
|
|
27
|
+
| Model-router / Pi defaults | `harness-sync-model-router.mjs` (Step 3.5 of `/harness-setup`) |
|
|
28
|
+
| Vendor router sync (this repo only) | `bash .pi/scripts/vendor-sync-pi-model-router.sh` or `npm run vendor:sync-router` |
|
|
29
|
+
| Meta-optimizer (JSONL proposals) | `node "$UP_PKG/.pi/harness/evolution/meta-optimizer.mjs"` |
|
|
30
|
+
|
|
31
|
+
Pass `--force` to shell scripts that support it (e.g. `harness-graphify-bootstrap.sh --force`, `harness-cli-verify.sh --force`).
|
|
16
32
|
|
|
17
33
|
Repo-root `scripts/` (e.g. `regen_graphify_html.py`) is dev-only and excluded from the npm tarball via `.npmignore`.
|
|
@@ -259,8 +259,10 @@ verify_sentrux() {
|
|
|
259
259
|
return
|
|
260
260
|
fi
|
|
261
261
|
sentrux plugin add-standard 2>/dev/null || warn "sentrux plugin add-standard skipped"
|
|
262
|
-
|
|
263
|
-
|
|
262
|
+
_sync_script="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)/sentrux-rules-sync.mjs"
|
|
263
|
+
if [ -f "$_sync_script" ]; then
|
|
264
|
+
node "$_sync_script" --force 2>/dev/null ||
|
|
265
|
+
warn "sentrux rules sync failed (see .pi/scripts/README.md)"
|
|
264
266
|
fi
|
|
265
267
|
if sentrux check . &>/dev/null; then
|
|
266
268
|
pass "sentrux $(sentrux --version 2>/dev/null | head -1)"
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Copy harness JSON contracts (and specs README) from the installed ultimate-pi
|
|
4
|
+
* package into the current project. External repos get `.pi/harness/specs/` before
|
|
5
|
+
* graphify or /harness-plan so paths like plan-packet.schema.json resolve locally.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* node "$UP_PKG/.pi/scripts/harness-seed-project-contracts.mjs" [PROJECT_ROOT]
|
|
9
|
+
*
|
|
10
|
+
* PROJECT_ROOT defaults to process.cwd(). Package root is derived from this file
|
|
11
|
+
* (the script always lives under the shipped ultimate-pi package).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { copyFile, mkdir, readdir } from "node:fs/promises";
|
|
15
|
+
import { join, dirname } from "node:path";
|
|
16
|
+
import { fileURLToPath } from "node:url";
|
|
17
|
+
|
|
18
|
+
const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
|
|
19
|
+
const UP_PKG = join(SCRIPT_DIR, "..", "..");
|
|
20
|
+
const SPEC_SRC = join(UP_PKG, ".pi", "harness", "specs");
|
|
21
|
+
|
|
22
|
+
const projectRoot = process.argv[2] || process.cwd();
|
|
23
|
+
const specDest = join(projectRoot, ".pi", "harness", "specs");
|
|
24
|
+
|
|
25
|
+
async function main() {
|
|
26
|
+
const names = await readdir(SPEC_SRC);
|
|
27
|
+
const toCopy = names.filter(
|
|
28
|
+
(n) => n.endsWith(".schema.json") || n === "README.md",
|
|
29
|
+
);
|
|
30
|
+
if (toCopy.length === 0) {
|
|
31
|
+
console.error(
|
|
32
|
+
"harness-seed-project-contracts: no schema files under",
|
|
33
|
+
SPEC_SRC,
|
|
34
|
+
);
|
|
35
|
+
process.exit(1);
|
|
36
|
+
}
|
|
37
|
+
await mkdir(specDest, { recursive: true });
|
|
38
|
+
for (const name of toCopy) {
|
|
39
|
+
await copyFile(join(SPEC_SRC, name), join(specDest, name));
|
|
40
|
+
}
|
|
41
|
+
console.log(
|
|
42
|
+
`harness-seed-project-contracts: copied ${toCopy.length} file(s) -> ${specDest}`,
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
main().catch((err) => {
|
|
47
|
+
console.error(err);
|
|
48
|
+
process.exit(1);
|
|
49
|
+
});
|