skill-harness 0.25.1 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +355 -35
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -8843,7 +8843,7 @@ var PI_DADDY_RECORD_V1_COMMIT2, PI_DADDY_RECORD_V1_SCHEMA2, PI_DADDY_RECORD_V1_G
|
|
|
8843
8843
|
var init_pi_daddy_record_v1_contract = __esm({
|
|
8844
8844
|
"packages/adapters/src/pi-daddy-record-v1-contract.ts"() {
|
|
8845
8845
|
"use strict";
|
|
8846
|
-
PI_DADDY_RECORD_V1_COMMIT2 = "
|
|
8846
|
+
PI_DADDY_RECORD_V1_COMMIT2 = "fad624ebc465eeb924fe91ba3b063b3851ce30ad";
|
|
8847
8847
|
PI_DADDY_RECORD_V1_SCHEMA2 = {
|
|
8848
8848
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
8849
8849
|
"$id": "https://github.com/mojomanyana/pi-daddy/contracts/ledger-record/v1/record.schema.json",
|
|
@@ -9348,6 +9348,8 @@ var init_pi_daddy_record_v1_contract = __esm({
|
|
|
9348
9348
|
"enum": [
|
|
9349
9349
|
"declined",
|
|
9350
9350
|
"dismissed",
|
|
9351
|
+
"expired",
|
|
9352
|
+
"aborted",
|
|
9351
9353
|
"no-ui",
|
|
9352
9354
|
"error"
|
|
9353
9355
|
]
|
|
@@ -10234,8 +10236,8 @@ function normalizePiDaddyRecordLedgerV12(raw) {
|
|
|
10234
10236
|
const record = parsed;
|
|
10235
10237
|
if (record.seq !== index + 1) throw new Error(`${where}: sequence gap`);
|
|
10236
10238
|
if (record.prev !== previous) throw new Error(`${where}: previous line hash mismatch`);
|
|
10237
|
-
const { digest:
|
|
10238
|
-
if (sha4(canonical3(unsigned)) !==
|
|
10239
|
+
const { digest: digest3, ...unsigned } = record;
|
|
10240
|
+
if (sha4(canonical3(unsigned)) !== digest3) throw new Error(`${where}: record digest mismatch`);
|
|
10239
10241
|
if (ids.has(record.id)) throw new Error(`${where}: duplicate record identity`);
|
|
10240
10242
|
ids.add(record.id);
|
|
10241
10243
|
validate2(PI_DADDY_RECORD_V1_GOVERNANCE_SCHEMA2, record.body, `${where} governance body`);
|
|
@@ -12933,7 +12935,7 @@ function normalizeLegacyGrant(record, index) {
|
|
|
12933
12935
|
const effective = record.effective;
|
|
12934
12936
|
const denied = record.denied;
|
|
12935
12937
|
const gated = record.gatedBlocked;
|
|
12936
|
-
const
|
|
12938
|
+
const digest3 = object3(record.definitionDigest);
|
|
12937
12939
|
const common2 = {
|
|
12938
12940
|
event_version: TRAJECTORY_EVENT_VERSION,
|
|
12939
12941
|
source: "pi-daddy-0.17",
|
|
@@ -12952,7 +12954,7 @@ function normalizeLegacyGrant(record, index) {
|
|
|
12952
12954
|
gate_outcome: record.gateOutcome,
|
|
12953
12955
|
human_denied: record.humanDenied === true,
|
|
12954
12956
|
reason: record.reason,
|
|
12955
|
-
definition_name:
|
|
12957
|
+
definition_name: digest3?.name,
|
|
12956
12958
|
legacy_schema: "pi-daddy-grant-ledger/0.17"
|
|
12957
12959
|
});
|
|
12958
12960
|
const refusal = record.blocked ? legacyRefusalCode(record) : void 0;
|
|
@@ -12962,7 +12964,7 @@ function normalizeLegacyGrant(record, index) {
|
|
|
12962
12964
|
requested_capabilities: requested,
|
|
12963
12965
|
effective_capabilities: effective,
|
|
12964
12966
|
refusal_code: refusal,
|
|
12965
|
-
digests: anyDefined({ definition: string(
|
|
12967
|
+
digests: anyDefined({ definition: string(digest3?.sha256) }),
|
|
12966
12968
|
attributes
|
|
12967
12969
|
});
|
|
12968
12970
|
const events = [
|
|
@@ -13570,7 +13572,7 @@ var init_src = __esm({
|
|
|
13570
13572
|
|
|
13571
13573
|
// packages/pi-extension/src/index.ts
|
|
13572
13574
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
13573
|
-
import { basename as basename4, dirname as dirname10, join as
|
|
13575
|
+
import { basename as basename4, dirname as dirname10, join as join35 } from "node:path";
|
|
13574
13576
|
|
|
13575
13577
|
// packages/pi-extension/src/commands.ts
|
|
13576
13578
|
init_dist();
|
|
@@ -15960,7 +15962,7 @@ async function runViaExtension(opts) {
|
|
|
15960
15962
|
}
|
|
15961
15963
|
|
|
15962
15964
|
// packages/pi-extension/src/commands.ts
|
|
15963
|
-
var USAGE = "usage: /skill-harness run [skill] [--model p:m] [--reps N] [--mode red|green|force] [--canary] [--judge p:m] | judge [run-dir] | review [skill] | coverage [skill] | jev enable|run|status|disable";
|
|
15965
|
+
var USAGE = "usage: /skill-harness run [skill] [--model p:m] [--reps N] [--mode red|green|force] [--canary] [--judge p:m] | judge [run-dir] | review [skill] | coverage [skill] | jev enable [workflow]|run|status|disable";
|
|
15964
15966
|
function parse(argstr) {
|
|
15965
15967
|
const tokens = argstr.trim().length ? argstr.trim().split(/\s+/) : [];
|
|
15966
15968
|
const [sub = "", ...rest] = tokens;
|
|
@@ -17368,7 +17370,7 @@ var RESEARCH_OPTIONAL = {
|
|
|
17368
17370
|
"run-pi": ["pi-node", "auth-path", "timeout-ms"]
|
|
17369
17371
|
};
|
|
17370
17372
|
async function researchCommand(command, flags, helpers, options) {
|
|
17371
|
-
const { cases, jsonFile: jsonFile2, textFile: textFile2, writeNew:
|
|
17373
|
+
const { cases, jsonFile: jsonFile2, textFile: textFile2, writeNew: writeNew3, readLegacyRun } = helpers;
|
|
17372
17374
|
const { emit = console.log, piRunner } = options;
|
|
17373
17375
|
if (command === "fixtures") {
|
|
17374
17376
|
const f = createLearningFixtures({});
|
|
@@ -17387,7 +17389,7 @@ async function researchCommand(command, flags, helpers, options) {
|
|
|
17387
17389
|
"label-evidence.json": refs,
|
|
17388
17390
|
"consents.json": []
|
|
17389
17391
|
}))
|
|
17390
|
-
await
|
|
17392
|
+
await writeNew3(join32(dir, name), value);
|
|
17391
17393
|
emit(
|
|
17392
17394
|
JSON.stringify({
|
|
17393
17395
|
saved: dir,
|
|
@@ -17405,7 +17407,7 @@ async function researchCommand(command, flags, helpers, options) {
|
|
|
17405
17407
|
sessionConsent: await jsonFile2(flags.consent),
|
|
17406
17408
|
experimentEntry: await jsonFile2(flags.entry)
|
|
17407
17409
|
});
|
|
17408
|
-
await
|
|
17410
|
+
await writeNew3(flags.out, result);
|
|
17409
17411
|
emit(JSON.stringify({ saved: resolve15(flags.out), trainingReady: false }));
|
|
17410
17412
|
return;
|
|
17411
17413
|
}
|
|
@@ -17502,7 +17504,7 @@ async function researchCommand(command, flags, helpers, options) {
|
|
|
17502
17504
|
}
|
|
17503
17505
|
const report = compareRuns(cases, labels, manifest, runs);
|
|
17504
17506
|
report.labelEvidenceReview = evidence;
|
|
17505
|
-
await
|
|
17507
|
+
await writeNew3(flags.out, report);
|
|
17506
17508
|
emit(
|
|
17507
17509
|
JSON.stringify({
|
|
17508
17510
|
saved: resolve15(flags.out),
|
|
@@ -17690,8 +17692,9 @@ function normalize3(payload) {
|
|
|
17690
17692
|
}
|
|
17691
17693
|
function deadline(signal) {
|
|
17692
17694
|
return new Promise((_, reject) => {
|
|
17693
|
-
|
|
17694
|
-
signal.
|
|
17695
|
+
const fail = () => reject(signal.reason instanceof Error ? signal.reason : new Error("provider request timed out"));
|
|
17696
|
+
if (signal.aborted) return fail();
|
|
17697
|
+
signal.addEventListener("abort", fail, { once: true });
|
|
17695
17698
|
});
|
|
17696
17699
|
}
|
|
17697
17700
|
async function readBounded(response, timedOut) {
|
|
@@ -17727,7 +17730,7 @@ async function readBounded(response, timedOut) {
|
|
|
17727
17730
|
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
17728
17731
|
}
|
|
17729
17732
|
function failure(startedAt, error) {
|
|
17730
|
-
const publicErrors = ["invalid provider response", "provider request timed out", "provider response exceeded limit"];
|
|
17733
|
+
const publicErrors = ["invalid provider response", "provider request timed out", "provider response exceeded limit", "provider request cancelled"];
|
|
17731
17734
|
const message = error instanceof Error && publicErrors.includes(error.message) ? error.message : "provider request failed";
|
|
17732
17735
|
return {
|
|
17733
17736
|
status: "error",
|
|
@@ -17742,6 +17745,7 @@ async function callProvider(provider, model, example, options = {}) {
|
|
|
17742
17745
|
const startedAt = performance.now();
|
|
17743
17746
|
let controller;
|
|
17744
17747
|
let timer;
|
|
17748
|
+
let cancel;
|
|
17745
17749
|
try {
|
|
17746
17750
|
const request = makeRequest(provider, model, example);
|
|
17747
17751
|
requiredString(options.apiKey, "apiKey", 16 * 1024);
|
|
@@ -17752,8 +17756,14 @@ async function callProvider(provider, model, example, options = {}) {
|
|
|
17752
17756
|
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 3e5) {
|
|
17753
17757
|
throw new TypeError("invalid timeout");
|
|
17754
17758
|
}
|
|
17759
|
+
if (options.signal !== void 0 && !(options.signal instanceof AbortSignal)) {
|
|
17760
|
+
throw new TypeError("signal must be an AbortSignal");
|
|
17761
|
+
}
|
|
17762
|
+
if (options.signal?.aborted) throw new Error("provider request cancelled");
|
|
17755
17763
|
controller = new AbortController();
|
|
17756
|
-
|
|
17764
|
+
cancel = () => controller.abort(new Error("provider request cancelled"));
|
|
17765
|
+
options.signal?.addEventListener("abort", cancel, { once: true });
|
|
17766
|
+
timer = setTimeout(() => controller.abort(new Error("provider request timed out")), timeoutMs);
|
|
17757
17767
|
const timedOut = deadline(controller.signal);
|
|
17758
17768
|
const response = await Promise.race([
|
|
17759
17769
|
(options.fetchImpl ?? fetch)(request.url, {
|
|
@@ -17778,6 +17788,7 @@ async function callProvider(provider, model, example, options = {}) {
|
|
|
17778
17788
|
return failure(startedAt, error);
|
|
17779
17789
|
} finally {
|
|
17780
17790
|
if (timer !== void 0) clearTimeout(timer);
|
|
17791
|
+
if (cancel) options.signal?.removeEventListener("abort", cancel);
|
|
17781
17792
|
controller?.abort();
|
|
17782
17793
|
}
|
|
17783
17794
|
}
|
|
@@ -18630,33 +18641,266 @@ async function main(argv, {
|
|
|
18630
18641
|
);
|
|
18631
18642
|
}
|
|
18632
18643
|
|
|
18644
|
+
// packages/pi-extension/src/jev-packet.ts
|
|
18645
|
+
import { createHash as createHash19 } from "node:crypto";
|
|
18646
|
+
import { closeSync as closeSync2, fsyncSync as fsyncSync2, lstatSync, mkdirSync as mkdirSync5, mkdtempSync as mkdtempSync4, openSync as openSync2, writeFileSync as writeFileSync8 } from "node:fs";
|
|
18647
|
+
import { homedir as homedir6 } from "node:os";
|
|
18648
|
+
import { join as join34 } from "node:path";
|
|
18649
|
+
var JEV_PROVIDER = "jev";
|
|
18650
|
+
var JEV_MODEL = "typesafe/jev-1.13";
|
|
18651
|
+
var JEV_WORKFLOW_LIMIT = 3;
|
|
18652
|
+
var JEV_QUESTION = "Does this handoff account for every explicitly required acceptance check with successful evidence tied to the reported candidate?";
|
|
18653
|
+
var JEV_SOURCE_KIND = "skill-harness-selected-handoff-v1";
|
|
18654
|
+
var digest2 = (text3) => createHash19("sha256").update(text3, "utf8").digest("hex");
|
|
18655
|
+
function requireIdentity(value, name) {
|
|
18656
|
+
if (typeof value !== "string" || !value.trim() || value.length > 256)
|
|
18657
|
+
throw new Error(`${name} must be a nonempty runtime identity of at most 256 characters`);
|
|
18658
|
+
}
|
|
18659
|
+
function prepareHandoff(value) {
|
|
18660
|
+
for (const field of ["candidate", "requirements", "evidence"]) {
|
|
18661
|
+
if (typeof value[field] !== "string" || !value[field].trim() || value[field].length > 32e3)
|
|
18662
|
+
throw new Error(`${field} must be a nonempty bounded string`);
|
|
18663
|
+
if (/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/u.test(value[field]))
|
|
18664
|
+
throw new Error(`${field} must contain valid Unicode`);
|
|
18665
|
+
}
|
|
18666
|
+
const input = JSON.stringify({
|
|
18667
|
+
candidate: value.candidate,
|
|
18668
|
+
requirements: value.requirements,
|
|
18669
|
+
evidence: value.evidence
|
|
18670
|
+
});
|
|
18671
|
+
if (Array.from(input).length > 16e3)
|
|
18672
|
+
throw new Error("The complete selected handoff packet must not exceed 16000 characters");
|
|
18673
|
+
return Object.freeze({ input, question: JEV_QUESTION, inputSha256: digest2(input) });
|
|
18674
|
+
}
|
|
18675
|
+
function createHandoffSource(options) {
|
|
18676
|
+
requireIdentity(options.sessionId, "Pi sessionId");
|
|
18677
|
+
requireIdentity(options.toolCallId, "Pi toolCallId");
|
|
18678
|
+
const source = {
|
|
18679
|
+
schema: 1,
|
|
18680
|
+
kind: JEV_SOURCE_KIND,
|
|
18681
|
+
sessionId: options.sessionId,
|
|
18682
|
+
toolCallId: options.toolCallId,
|
|
18683
|
+
frozenAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18684
|
+
input: options.packet.input,
|
|
18685
|
+
inputSha256: options.packet.inputSha256,
|
|
18686
|
+
question: JEV_QUESTION,
|
|
18687
|
+
provider: JEV_PROVIDER,
|
|
18688
|
+
model: JEV_MODEL,
|
|
18689
|
+
consent: options.consent,
|
|
18690
|
+
authorization: options.authorization,
|
|
18691
|
+
provenance: "tool-selected-input",
|
|
18692
|
+
candidateIdentity: "caller-claimed",
|
|
18693
|
+
sourceBinding: "unassessed",
|
|
18694
|
+
redaction: "unassessed",
|
|
18695
|
+
rights: "unassessed",
|
|
18696
|
+
labelStatus: "unlabeled",
|
|
18697
|
+
trainingEligible: false,
|
|
18698
|
+
exportEligible: false,
|
|
18699
|
+
publicCaptureVerified: false
|
|
18700
|
+
};
|
|
18701
|
+
const bytes = JSON.stringify(source, null, 2) + "\n";
|
|
18702
|
+
return Object.freeze({ source, bytes, sha256: digest2(bytes) });
|
|
18703
|
+
}
|
|
18704
|
+
function privateDirectory(path) {
|
|
18705
|
+
try {
|
|
18706
|
+
mkdirSync5(path, { mode: 448 });
|
|
18707
|
+
} catch (error) {
|
|
18708
|
+
if (error.code !== "EEXIST") throw error;
|
|
18709
|
+
}
|
|
18710
|
+
const stat4 = lstatSync(path);
|
|
18711
|
+
if (!stat4.isDirectory() || stat4.isSymbolicLink())
|
|
18712
|
+
throw new Error("JEV storage must be a private directory, not a link");
|
|
18713
|
+
if (process.platform !== "win32" && ((stat4.mode & 63) !== 0 || process.getuid && stat4.uid !== process.getuid()))
|
|
18714
|
+
throw new Error("JEV storage directory must be owned by the current user with mode 0700");
|
|
18715
|
+
}
|
|
18716
|
+
function writeNew2(path, bytes) {
|
|
18717
|
+
const fd = openSync2(path, "wx", 384);
|
|
18718
|
+
try {
|
|
18719
|
+
writeFileSync8(fd, bytes);
|
|
18720
|
+
fsyncSync2(fd);
|
|
18721
|
+
} finally {
|
|
18722
|
+
closeSync2(fd);
|
|
18723
|
+
}
|
|
18724
|
+
}
|
|
18725
|
+
function retainHandoffSource(source, assertCurrent, storageHome = join34(homedir6(), ".skill-harness")) {
|
|
18726
|
+
assertCurrent();
|
|
18727
|
+
privateDirectory(storageHome);
|
|
18728
|
+
const root = join34(storageHome, "jev-workflow");
|
|
18729
|
+
privateDirectory(root);
|
|
18730
|
+
const directory = mkdtempSync4(join34(root, "selection-"));
|
|
18731
|
+
const path = join34(directory, "selection.json");
|
|
18732
|
+
writeNew2(path, source.bytes);
|
|
18733
|
+
return {
|
|
18734
|
+
path,
|
|
18735
|
+
writeOutcome(outcome) {
|
|
18736
|
+
assertCurrent();
|
|
18737
|
+
writeNew2(join34(directory, "outcome.json"), JSON.stringify({
|
|
18738
|
+
schema: 1,
|
|
18739
|
+
kind: "skill-harness-handoff-advice-v1",
|
|
18740
|
+
source: { path: "selection.json", sha256: source.sha256 },
|
|
18741
|
+
recordedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18742
|
+
outcome,
|
|
18743
|
+
trainingEligible: false,
|
|
18744
|
+
exportEligible: false,
|
|
18745
|
+
labelStatus: "unlabeled"
|
|
18746
|
+
}, null, 2) + "\n");
|
|
18747
|
+
}
|
|
18748
|
+
};
|
|
18749
|
+
}
|
|
18750
|
+
|
|
18633
18751
|
// packages/pi-extension/src/jev-session.ts
|
|
18634
|
-
|
|
18752
|
+
var unmeasuredProvider = () => ({
|
|
18753
|
+
provider: JEV_PROVIDER,
|
|
18754
|
+
requestedModel: JEV_MODEL,
|
|
18755
|
+
resolvedModel: null,
|
|
18756
|
+
usage: { inputTokens: null, outputTokens: null, costUsd: null },
|
|
18757
|
+
latencyMs: null
|
|
18758
|
+
});
|
|
18759
|
+
var missingKeyRemedy = "Set OPENROUTER_API_KEY in the environment that launches Pi, restart Pi, then enable JEV again with fresh session consent.";
|
|
18760
|
+
function createJevController(pi, options = {}) {
|
|
18761
|
+
const run = options.run ?? main;
|
|
18635
18762
|
let epoch = 0;
|
|
18636
18763
|
let active = null;
|
|
18637
18764
|
const clearSession = () => {
|
|
18638
18765
|
epoch++;
|
|
18766
|
+
active?.abort.abort();
|
|
18639
18767
|
active = null;
|
|
18640
18768
|
};
|
|
18641
18769
|
pi.on("session_start", clearSession);
|
|
18642
18770
|
pi.on("session_shutdown", clearSession);
|
|
18643
|
-
|
|
18771
|
+
const currentId = (ctx) => {
|
|
18772
|
+
const id = ctx.sessionManager?.getSessionId();
|
|
18773
|
+
if (active && active.sessionId !== id) clearSession();
|
|
18774
|
+
return id;
|
|
18775
|
+
};
|
|
18776
|
+
const hasKey = () => !!(options.env ?? process.env).OPENROUTER_API_KEY?.trim();
|
|
18777
|
+
const status = (ctx) => {
|
|
18778
|
+
const id = currentId(ctx);
|
|
18779
|
+
const availability = !id ? "missing-session" : !active ? "disabled" : active.mode === "manual" ? "manual-only" : active.blocked ?? (active.busy ? "busy" : active.used >= JEV_WORKFLOW_LIMIT ? "limit-reached" : !hasKey() ? "missing-key" : "ready");
|
|
18780
|
+
return {
|
|
18781
|
+
enabled: active !== null,
|
|
18782
|
+
mode: active?.mode ?? "disabled",
|
|
18783
|
+
storage: active?.consent.decision ?? null,
|
|
18784
|
+
remaining: active?.mode === "workflow" ? JEV_WORKFLOW_LIMIT - active.used : 0,
|
|
18785
|
+
availability,
|
|
18786
|
+
providerReadiness: hasKey() ? "key-present" : "missing-key",
|
|
18787
|
+
advisory: true
|
|
18788
|
+
};
|
|
18789
|
+
};
|
|
18790
|
+
const evaluate = async (value, toolCallId, signal, ctx) => {
|
|
18791
|
+
requireIdentity(toolCallId, "Pi toolCallId");
|
|
18792
|
+
const packet = prepareHandoff(value);
|
|
18793
|
+
const id = currentId(ctx);
|
|
18794
|
+
const unavailable = (reason) => ({
|
|
18795
|
+
advisory: true,
|
|
18796
|
+
status: "unavailable",
|
|
18797
|
+
probability: null,
|
|
18798
|
+
reason,
|
|
18799
|
+
...unmeasuredProvider(),
|
|
18800
|
+
remaining: active?.mode === "workflow" ? JEV_WORKFLOW_LIMIT - active.used : 0
|
|
18801
|
+
});
|
|
18802
|
+
if (!id || !active || active.mode !== "workflow")
|
|
18803
|
+
return unavailable("Explicit /skill-harness jev enable workflow activation is required in this Pi session.");
|
|
18804
|
+
const bound = active;
|
|
18805
|
+
const assertCurrent = () => {
|
|
18806
|
+
if (active !== bound || ctx.sessionManager?.getSessionId() !== bound.sessionId || bound.abort.signal.aborted || signal?.aborted)
|
|
18807
|
+
throw new Error("Session, authorization or tool execution changed");
|
|
18808
|
+
};
|
|
18809
|
+
if (signal?.aborted) return unavailable("Tool execution was cancelled.");
|
|
18810
|
+
const cached = bound.cache.get(packet.inputSha256);
|
|
18811
|
+
if (cached) return { ...cached, remaining: JEV_WORKFLOW_LIMIT - bound.used, reused: true };
|
|
18812
|
+
if (bound.blocked) return unavailable("Workflow advice is suppressed after an error; explicit new activation is required.");
|
|
18813
|
+
if (bound.busy) return unavailable("An advisory request is already in flight; no additional call was made.");
|
|
18814
|
+
if (bound.used >= JEV_WORKFLOW_LIMIT) return unavailable("This activation's advisory call limit is reached.");
|
|
18815
|
+
if (!hasKey()) return unavailable(`OPENROUTER_API_KEY is unavailable; no call was made. ${missingKeyRemedy}`);
|
|
18816
|
+
const abort = new AbortController();
|
|
18817
|
+
const cancel = () => abort.abort();
|
|
18818
|
+
const signals = [bound.abort.signal, signal].filter((s) => s !== void 0);
|
|
18819
|
+
for (const s of signals) {
|
|
18820
|
+
if (s.aborted) cancel();
|
|
18821
|
+
else s.addEventListener("abort", cancel, { once: true });
|
|
18822
|
+
}
|
|
18823
|
+
bound.busy = true;
|
|
18824
|
+
bound.used++;
|
|
18825
|
+
let receipt;
|
|
18826
|
+
try {
|
|
18827
|
+
assertCurrent();
|
|
18828
|
+
const source = createHandoffSource({
|
|
18829
|
+
packet,
|
|
18830
|
+
sessionId: bound.sessionId,
|
|
18831
|
+
toolCallId,
|
|
18832
|
+
consent: bound.consent,
|
|
18833
|
+
authorization: bound.authorization
|
|
18834
|
+
});
|
|
18835
|
+
const retained = bound.consent.decision === "granted" ? retainHandoffSource(source, assertCurrent, options.storageHome) : void 0;
|
|
18836
|
+
receipt = {
|
|
18837
|
+
kind: JEV_SOURCE_KIND,
|
|
18838
|
+
sha256: source.sha256,
|
|
18839
|
+
sessionId: bound.sessionId,
|
|
18840
|
+
toolCallId,
|
|
18841
|
+
retained: !!retained,
|
|
18842
|
+
...retained ? { path: retained.path } : {}
|
|
18843
|
+
};
|
|
18844
|
+
assertCurrent();
|
|
18845
|
+
const result = await (options.provider ?? callProvider)(JEV_PROVIDER, JEV_MODEL, packet, {
|
|
18846
|
+
apiKey: (options.env ?? process.env).OPENROUTER_API_KEY,
|
|
18847
|
+
signal: abort.signal
|
|
18848
|
+
});
|
|
18849
|
+
assertCurrent();
|
|
18850
|
+
if (result.status !== "answered") bound.blocked = "provider-error";
|
|
18851
|
+
const advice = {
|
|
18852
|
+
advisory: true,
|
|
18853
|
+
status: result.status === "answered" ? "answered" : "unavailable",
|
|
18854
|
+
probability: result.status === "answered" ? result.probability : null,
|
|
18855
|
+
provider: JEV_PROVIDER,
|
|
18856
|
+
requestedModel: JEV_MODEL,
|
|
18857
|
+
resolvedModel: result.resolvedModel,
|
|
18858
|
+
usage: result.usage,
|
|
18859
|
+
latencyMs: result.latencyMs,
|
|
18860
|
+
...result.status === "answered" ? {} : { reason: "Provider unavailable; further workflow calls require explicit new activation." },
|
|
18861
|
+
remaining: JEV_WORKFLOW_LIMIT - bound.used,
|
|
18862
|
+
inputSha256: packet.inputSha256,
|
|
18863
|
+
source: receipt
|
|
18864
|
+
};
|
|
18865
|
+
try {
|
|
18866
|
+
retained?.writeOutcome(advice);
|
|
18867
|
+
} catch {
|
|
18868
|
+
assertCurrent();
|
|
18869
|
+
bound.blocked = "storage-error";
|
|
18870
|
+
advice.reason = "Provider outcome received, but outcome retention failed; further workflow calls require explicit new activation.";
|
|
18871
|
+
}
|
|
18872
|
+
bound.cache.set(packet.inputSha256, Object.freeze(advice));
|
|
18873
|
+
return advice;
|
|
18874
|
+
} catch {
|
|
18875
|
+
bound.blocked = "execution-error";
|
|
18876
|
+
return {
|
|
18877
|
+
advisory: true,
|
|
18878
|
+
status: "unavailable",
|
|
18879
|
+
probability: null,
|
|
18880
|
+
...unmeasuredProvider(),
|
|
18881
|
+
reason: "Advice unavailable or cancelled; no approval is implied. Explicit new activation is required for further workflow calls.",
|
|
18882
|
+
remaining: JEV_WORKFLOW_LIMIT - bound.used,
|
|
18883
|
+
inputSha256: packet.inputSha256,
|
|
18884
|
+
...receipt ? { source: receipt } : {}
|
|
18885
|
+
};
|
|
18886
|
+
} finally {
|
|
18887
|
+
bound.busy = false;
|
|
18888
|
+
for (const s of signals) s.removeEventListener("abort", cancel);
|
|
18889
|
+
}
|
|
18890
|
+
};
|
|
18891
|
+
const command = async (args, ctx) => {
|
|
18644
18892
|
const action = args.trim() || "status";
|
|
18645
|
-
const sessionId = ctx
|
|
18893
|
+
const sessionId = currentId(ctx);
|
|
18646
18894
|
if (!sessionId) throw Error("JEV requires the current Pi session identity");
|
|
18647
|
-
|
|
18648
|
-
epoch++;
|
|
18649
|
-
active = null;
|
|
18650
|
-
}
|
|
18895
|
+
requireIdentity(sessionId, "Pi sessionId");
|
|
18651
18896
|
if (action === "status") {
|
|
18652
18897
|
ctx.ui.notify(
|
|
18653
|
-
|
|
18898
|
+
JSON.stringify(status(ctx))
|
|
18654
18899
|
);
|
|
18655
18900
|
return;
|
|
18656
18901
|
}
|
|
18657
18902
|
if (action === "disable") {
|
|
18658
|
-
|
|
18659
|
-
active = null;
|
|
18903
|
+
clearSession();
|
|
18660
18904
|
ctx.ui.notify(
|
|
18661
18905
|
"JEV disabled. Existing selected records remain local; disable storage in their review before export if needed."
|
|
18662
18906
|
);
|
|
@@ -18666,15 +18910,35 @@ function createJevSessionHandler(pi, run = main) {
|
|
|
18666
18910
|
throw Error(
|
|
18667
18911
|
"Interactive JEV activation requires a per-session storage answer. Use the explicit CLI storage option in noninteractive mode."
|
|
18668
18912
|
);
|
|
18669
|
-
if (action === "enable") {
|
|
18670
|
-
|
|
18671
|
-
|
|
18913
|
+
if (action === "enable" || action === "enable workflow") {
|
|
18914
|
+
clearSession();
|
|
18915
|
+
const activation = epoch;
|
|
18916
|
+
const workflow = action === "enable workflow";
|
|
18917
|
+
let authorization = null;
|
|
18918
|
+
if (workflow) {
|
|
18919
|
+
if (!ctx.ui.confirm) throw new Error("Workflow JEV activation requires interactive paid-scope confirmation");
|
|
18920
|
+
const confirmed = await ctx.ui.confirm(
|
|
18921
|
+
"Enable optional JEV handoff advice for THIS Pi session?",
|
|
18922
|
+
`Allow up to ${JEV_WORKFLOW_LIMIT} automatic workflow paid calls to ${JEV_MODEL} through OpenRouter using OPENROUTER_API_KEY. The coordinator may send selected candidate, acceptance requirements and evidence (at most 16000 characters per packet) for the fixed handoff-readiness question when uncertainty remains after deterministic checks, before independent review. Advice never grants approval. This is separate from your Pi subscription; no retry or fallback. Manual jev run calls remain separately confirmed outside this workflow limit.`
|
|
18923
|
+
);
|
|
18924
|
+
if (activation !== epoch || ctx.sessionManager?.getSessionId() !== sessionId || confirmed !== true) return;
|
|
18925
|
+
authorization = {
|
|
18926
|
+
kind: "jev-workflow-paid-scope",
|
|
18927
|
+
interactionId: randomUUID3(),
|
|
18928
|
+
sessionId,
|
|
18929
|
+
recordedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18930
|
+
provider: JEV_PROVIDER,
|
|
18931
|
+
model: JEV_MODEL,
|
|
18932
|
+
question: JEV_QUESTION,
|
|
18933
|
+
maximumCalls: JEV_WORKFLOW_LIMIT
|
|
18934
|
+
};
|
|
18935
|
+
}
|
|
18672
18936
|
const choices = [
|
|
18673
18937
|
"No \u2014 use JEV without retaining data for LoRA",
|
|
18674
18938
|
"Yes \u2014 retain selected decision data for later LoRA review"
|
|
18675
18939
|
];
|
|
18676
18940
|
const answer = await ctx.ui.select(
|
|
18677
|
-
"Store selected data from THIS session for future LoRA dataset review? Storage does not grant training permission.",
|
|
18941
|
+
"Store selected data from THIS session for future LoRA dataset review? Workflow selections are saved under ~/.skill-harness/jev-workflow. Storage does not grant training permission; No does not change Pi native session history.",
|
|
18678
18942
|
choices
|
|
18679
18943
|
);
|
|
18680
18944
|
if (activation !== epoch || ctx.sessionManager?.getSessionId() !== sessionId)
|
|
@@ -18687,14 +18951,24 @@ function createJevSessionHandler(pi, run = main) {
|
|
|
18687
18951
|
recordedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
18688
18952
|
});
|
|
18689
18953
|
pi.appendEntry?.("skill-harness-jev-storage-choice", consent);
|
|
18690
|
-
active = {
|
|
18954
|
+
active = {
|
|
18955
|
+
sessionId,
|
|
18956
|
+
consent,
|
|
18957
|
+
authorization,
|
|
18958
|
+
mode: workflow ? "workflow" : "manual",
|
|
18959
|
+
abort: new AbortController(),
|
|
18960
|
+
used: 0,
|
|
18961
|
+
busy: false,
|
|
18962
|
+
blocked: null,
|
|
18963
|
+
cache: /* @__PURE__ */ new Map()
|
|
18964
|
+
};
|
|
18691
18965
|
ctx.ui.notify(
|
|
18692
|
-
|
|
18966
|
+
(workflow ? `JEV workflow calls authorized for this session (up to ${JEV_WORKFLOW_LIMIT} paid calls). LoRA storage ` : "JEV manual mode activated; each selected paid call still requires confirmation. LoRA storage ") + consent.decision + " for this session only. " + (hasKey() ? "Provider readiness: OPENROUTER_API_KEY is present; credentials and provider access have not been verified. No provider call was made." : `Provider unavailable: OPENROUTER_API_KEY is missing or blank in this Pi process. No provider call was made. ${missingKeyRemedy}`)
|
|
18693
18967
|
);
|
|
18694
18968
|
return;
|
|
18695
18969
|
}
|
|
18696
18970
|
if (action !== "run")
|
|
18697
|
-
throw Error("usage: /skill-harness jev enable | status | disable | run");
|
|
18971
|
+
throw Error("usage: /skill-harness jev enable [workflow] | status | disable | run");
|
|
18698
18972
|
if (!active)
|
|
18699
18973
|
throw Error(
|
|
18700
18974
|
"Enable JEV in this session first: /skill-harness jev enable"
|
|
@@ -18784,13 +19058,59 @@ function createJevSessionHandler(pi, run = main) {
|
|
|
18784
19058
|
}
|
|
18785
19059
|
);
|
|
18786
19060
|
};
|
|
19061
|
+
return { command, status, evaluate };
|
|
19062
|
+
}
|
|
19063
|
+
|
|
19064
|
+
// packages/pi-extension/src/jev-advice.ts
|
|
19065
|
+
import { Type as Type3 } from "typebox";
|
|
19066
|
+
function createJevAdviceTool(controller) {
|
|
19067
|
+
return {
|
|
19068
|
+
name: "jev_advice",
|
|
19069
|
+
label: "JEV handoff advice",
|
|
19070
|
+
description: "Check session-local JEV workflow status without a provider call, or evaluate selected handoff evidence against the fixed acceptance-readiness question. Evaluation requires the user's explicit /skill-harness jev enable workflow activation. Advisory probability never grants approval or replaces tests/review. Select only relevant public or already-redacted evidence; do not include secrets, transcripts, reviewer verdicts or hidden reasoning.",
|
|
19071
|
+
promptGuidelines: [
|
|
19072
|
+
"Use jev_advice action=status to discover whether JEV workflow advice is enabled.",
|
|
19073
|
+
"When enabled and useful uncertainty remains after deterministic checks, optionally evaluate a selected handoff before independent review. Do not call for every feature or retry unavailable advice.",
|
|
19074
|
+
"Keep JEV advice out of the independent reviewer's inputs; act on independently verified issues rather than treating its probability as a verdict."
|
|
19075
|
+
],
|
|
19076
|
+
parameters: Type3.Object({
|
|
19077
|
+
action: Type3.Union([Type3.Literal("status"), Type3.Literal("evaluate")]),
|
|
19078
|
+
candidate: Type3.Optional(Type3.String({ minLength: 1, maxLength: 16e3, description: "Reported candidate identity; this is a claim, not an authenticated Git binding." })),
|
|
19079
|
+
requirements: Type3.Optional(Type3.String({ minLength: 1, maxLength: 16e3, description: "Explicit acceptance checks, before seeing independent review or JEV outcomes." })),
|
|
19080
|
+
evidence: Type3.Optional(Type3.String({ minLength: 1, maxLength: 16e3, description: "Selected decision-time handoff evidence. Complete JSON packet is limited to 16000 characters." }))
|
|
19081
|
+
}, { additionalProperties: false }),
|
|
19082
|
+
async execute(id, params, signal, _onUpdate, ctx) {
|
|
19083
|
+
requireIdentity(id, "Pi toolCallId");
|
|
19084
|
+
if (!params || typeof params !== "object" || Array.isArray(params)) throw new Error("Invalid jev_advice parameters");
|
|
19085
|
+
const p = params;
|
|
19086
|
+
const expected = p.action === "status" ? ["action"] : ["action", "candidate", "requirements", "evidence"];
|
|
19087
|
+
if (Object.keys(p).length !== expected.length || expected.some((key) => !Object.hasOwn(p, key)))
|
|
19088
|
+
throw new Error("jev_advice accepts only its documented status or evaluate fields");
|
|
19089
|
+
let details;
|
|
19090
|
+
if (p.action === "status") details = controller.status(ctx);
|
|
19091
|
+
else if (p.action === "evaluate")
|
|
19092
|
+
details = await controller.evaluate(
|
|
19093
|
+
{ candidate: p.candidate, requirements: p.requirements, evidence: p.evidence },
|
|
19094
|
+
id,
|
|
19095
|
+
signal,
|
|
19096
|
+
ctx
|
|
19097
|
+
);
|
|
19098
|
+
else throw new Error("jev_advice action must be status or evaluate");
|
|
19099
|
+
return { content: [{ type: "text", text: JSON.stringify(details) }], details };
|
|
19100
|
+
}
|
|
19101
|
+
};
|
|
19102
|
+
}
|
|
19103
|
+
function registerJevAdvice(pi, controller) {
|
|
19104
|
+
pi.registerTool(createJevAdviceTool(controller));
|
|
18787
19105
|
}
|
|
18788
19106
|
|
|
18789
19107
|
// packages/pi-extension/src/index.ts
|
|
18790
19108
|
function index_default(pi) {
|
|
18791
19109
|
const moduleDir = dirname10(fileURLToPath2(import.meta.url));
|
|
18792
|
-
const assetsDir = basename4(dirname10(moduleDir)) === "skill-harness" ?
|
|
18793
|
-
|
|
19110
|
+
const assetsDir = basename4(dirname10(moduleDir)) === "skill-harness" ? join35(moduleDir, "..", "assets") : join35(moduleDir, "..", "..", "..", "assets");
|
|
19111
|
+
const jev = createJevController(pi);
|
|
19112
|
+
registerCommand(pi, assetsDir, jev.command);
|
|
19113
|
+
registerJevAdvice(pi, jev);
|
|
18794
19114
|
registerTool(pi);
|
|
18795
19115
|
pi.on("session_shutdown", async () => {
|
|
18796
19116
|
closeReview();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "skill-harness",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.26.1",
|
|
4
4
|
"description": "Test/optimize loop for agent skills \u2014 run spec'd scenarios on pi, LLM-judge, score, review, re-run",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"keywords": [
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"README.md"
|
|
32
32
|
],
|
|
33
33
|
"dependencies": {
|
|
34
|
-
"@skill-harness/cli": "0.
|
|
34
|
+
"@skill-harness/cli": "0.26.1"
|
|
35
35
|
},
|
|
36
36
|
"peerDependencies": {
|
|
37
37
|
"typebox": "*"
|