skill-harness 0.26.1 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +222 -16
- package/package.json +55 -55
package/dist/index.js
CHANGED
|
@@ -8684,6 +8684,9 @@ function validateNumber(schema2, value, path) {
|
|
|
8684
8684
|
if (typeof schema2.minimum === "number" && value < schema2.minimum) {
|
|
8685
8685
|
return [{ path, message: `must be >= ${schema2.minimum}` }];
|
|
8686
8686
|
}
|
|
8687
|
+
if (typeof schema2.maximum === "number" && value > schema2.maximum) {
|
|
8688
|
+
return [{ path, message: `must be <= ${schema2.maximum}` }];
|
|
8689
|
+
}
|
|
8687
8690
|
return [];
|
|
8688
8691
|
}
|
|
8689
8692
|
function bestBranch(root, schemas, branches, value, path) {
|
|
@@ -8805,7 +8808,8 @@ var init_closed_schema = __esm({
|
|
|
8805
8808
|
"minItems",
|
|
8806
8809
|
"maxItems",
|
|
8807
8810
|
"uniqueItems",
|
|
8808
|
-
"exclusiveMinimum"
|
|
8811
|
+
"exclusiveMinimum",
|
|
8812
|
+
"maximum"
|
|
8809
8813
|
]);
|
|
8810
8814
|
KEYWORD_SHAPES = {
|
|
8811
8815
|
$ref: { check: (value) => typeof value === "string", expected: "a string" },
|
|
@@ -8826,6 +8830,7 @@ var init_closed_schema = __esm({
|
|
|
8826
8830
|
maxLength: { check: (value) => typeof value === "number", expected: "a number" },
|
|
8827
8831
|
exclusiveMinimum: { check: (value) => typeof value === "number" && Number.isFinite(value), expected: "a finite number" },
|
|
8828
8832
|
minimum: { check: (value) => typeof value === "number", expected: "a number" },
|
|
8833
|
+
maximum: { check: (value) => typeof value === "number" && Number.isFinite(value), expected: "a finite number" },
|
|
8829
8834
|
uniqueItems: { check: (value) => typeof value === "boolean", expected: "a boolean" },
|
|
8830
8835
|
minItems: { check: (value) => Number.isInteger(value) && Number(value) >= 0, expected: "a non-negative integer" },
|
|
8831
8836
|
maxItems: { check: (value) => Number.isInteger(value) && Number(value) >= 0, expected: "a non-negative integer" },
|
|
@@ -8843,7 +8848,7 @@ var PI_DADDY_RECORD_V1_COMMIT2, PI_DADDY_RECORD_V1_SCHEMA2, PI_DADDY_RECORD_V1_G
|
|
|
8843
8848
|
var init_pi_daddy_record_v1_contract = __esm({
|
|
8844
8849
|
"packages/adapters/src/pi-daddy-record-v1-contract.ts"() {
|
|
8845
8850
|
"use strict";
|
|
8846
|
-
PI_DADDY_RECORD_V1_COMMIT2 = "
|
|
8851
|
+
PI_DADDY_RECORD_V1_COMMIT2 = "ce7ea2e00f371f74fe3119e3f9de7aef672462a1";
|
|
8847
8852
|
PI_DADDY_RECORD_V1_SCHEMA2 = {
|
|
8848
8853
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
8849
8854
|
"$id": "https://github.com/mojomanyana/pi-daddy/contracts/ledger-record/v1/record.schema.json",
|
|
@@ -9172,7 +9177,8 @@ var init_pi_daddy_record_v1_contract = __esm({
|
|
|
9172
9177
|
"prompt",
|
|
9173
9178
|
"session",
|
|
9174
9179
|
"persisted",
|
|
9175
|
-
"inherited"
|
|
9180
|
+
"inherited",
|
|
9181
|
+
"auto"
|
|
9176
9182
|
]
|
|
9177
9183
|
},
|
|
9178
9184
|
"approvalScope": {
|
|
@@ -10090,7 +10096,8 @@ var init_pi_daddy_record_v1_contract = __esm({
|
|
|
10090
10096
|
"trigger": {
|
|
10091
10097
|
"enum": [
|
|
10092
10098
|
"first-delegation",
|
|
10093
|
-
"grants-models"
|
|
10099
|
+
"grants-models",
|
|
10100
|
+
"auto-mode"
|
|
10094
10101
|
]
|
|
10095
10102
|
},
|
|
10096
10103
|
"overrides": {
|
|
@@ -10113,6 +10120,32 @@ var init_pi_daddy_record_v1_contract = __esm({
|
|
|
10113
10120
|
],
|
|
10114
10121
|
"additionalProperties": false
|
|
10115
10122
|
}
|
|
10123
|
+
},
|
|
10124
|
+
"autoMode": {
|
|
10125
|
+
"type": "object",
|
|
10126
|
+
"properties": {
|
|
10127
|
+
"enabled": {
|
|
10128
|
+
"type": "boolean"
|
|
10129
|
+
},
|
|
10130
|
+
"source": {
|
|
10131
|
+
"enum": [
|
|
10132
|
+
"default",
|
|
10133
|
+
"environment",
|
|
10134
|
+
"session"
|
|
10135
|
+
]
|
|
10136
|
+
},
|
|
10137
|
+
"revision": {
|
|
10138
|
+
"type": "integer",
|
|
10139
|
+
"minimum": 0,
|
|
10140
|
+
"maximum": 9007199254740991
|
|
10141
|
+
}
|
|
10142
|
+
},
|
|
10143
|
+
"required": [
|
|
10144
|
+
"enabled",
|
|
10145
|
+
"source",
|
|
10146
|
+
"revision"
|
|
10147
|
+
],
|
|
10148
|
+
"additionalProperties": false
|
|
10116
10149
|
}
|
|
10117
10150
|
},
|
|
10118
10151
|
"required": [
|
|
@@ -10124,7 +10157,41 @@ var init_pi_daddy_record_v1_contract = __esm({
|
|
|
10124
10157
|
"trigger",
|
|
10125
10158
|
"overrides"
|
|
10126
10159
|
],
|
|
10127
|
-
"additionalProperties": false
|
|
10160
|
+
"additionalProperties": false,
|
|
10161
|
+
"oneOf": [
|
|
10162
|
+
{
|
|
10163
|
+
"properties": {
|
|
10164
|
+
"trigger": {
|
|
10165
|
+
"const": "auto-mode"
|
|
10166
|
+
},
|
|
10167
|
+
"outcome": {
|
|
10168
|
+
"const": "changed"
|
|
10169
|
+
},
|
|
10170
|
+
"overrides": {
|
|
10171
|
+
"type": "object",
|
|
10172
|
+
"additionalProperties": false
|
|
10173
|
+
}
|
|
10174
|
+
},
|
|
10175
|
+
"required": [
|
|
10176
|
+
"autoMode"
|
|
10177
|
+
]
|
|
10178
|
+
},
|
|
10179
|
+
{
|
|
10180
|
+
"properties": {
|
|
10181
|
+
"trigger": {
|
|
10182
|
+
"enum": [
|
|
10183
|
+
"first-delegation",
|
|
10184
|
+
"grants-models"
|
|
10185
|
+
]
|
|
10186
|
+
}
|
|
10187
|
+
},
|
|
10188
|
+
"not": {
|
|
10189
|
+
"required": [
|
|
10190
|
+
"autoMode"
|
|
10191
|
+
]
|
|
10192
|
+
}
|
|
10193
|
+
}
|
|
10194
|
+
]
|
|
10128
10195
|
},
|
|
10129
10196
|
"episodeOutcome": {
|
|
10130
10197
|
"type": "object",
|
|
@@ -13571,6 +13638,7 @@ var init_src = __esm({
|
|
|
13571
13638
|
});
|
|
13572
13639
|
|
|
13573
13640
|
// packages/pi-extension/src/index.ts
|
|
13641
|
+
import { readFileSync as readFileSync20 } from "node:fs";
|
|
13574
13642
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
13575
13643
|
import { basename as basename4, dirname as dirname10, join as join35 } from "node:path";
|
|
13576
13644
|
|
|
@@ -13964,7 +14032,8 @@ var V3_SUPPORTED_KEYWORDS = /* @__PURE__ */ new Set([
|
|
|
13964
14032
|
"minItems",
|
|
13965
14033
|
"maxItems",
|
|
13966
14034
|
"uniqueItems",
|
|
13967
|
-
"exclusiveMinimum"
|
|
14035
|
+
"exclusiveMinimum",
|
|
14036
|
+
"maximum"
|
|
13968
14037
|
]);
|
|
13969
14038
|
|
|
13970
14039
|
// packages/adapters/dist/pi-daddy-record-v1.js
|
|
@@ -16855,6 +16924,88 @@ async function writePreparedExport({ prepared, directory }) {
|
|
|
16855
16924
|
return { directory, trainingEligible: prepared.metadata.trainingEligible, files: Object.fromEntries(Object.entries(prepared.files).map(([name, s]) => [name, { sha256: learningDigest(s), bytes: Buffer.byteLength(s) }])) };
|
|
16856
16925
|
}
|
|
16857
16926
|
|
|
16927
|
+
// experiments/decision-shadow/workflow-fixtures.mjs
|
|
16928
|
+
var WORKFLOW_ORACLE_VERSION = "workflow-convergence-oracles-v1";
|
|
16929
|
+
function evaluateWorkflowFixture(family, facts) {
|
|
16930
|
+
if (!facts || typeof facts !== "object" || Array.isArray(facts)) return false;
|
|
16931
|
+
switch (family) {
|
|
16932
|
+
case "workspace":
|
|
16933
|
+
return typeof facts.requestedRoot === "string" && facts.requestedRoot.startsWith("/") && facts.observedRoot === facts.requestedRoot && facts.pathsObserved === true && Array.isArray(facts.requiredPaths) && facts.requiredPaths.length > 0 && Array.isArray(facts.readablePaths) && facts.requiredPaths.every((path) => typeof path === "string" && path.startsWith(facts.requestedRoot + "/") && facts.readablePaths.includes(path));
|
|
16934
|
+
case "identity":
|
|
16935
|
+
return facts.expected?.algorithm === "principal-candidate-v1" && facts.actual?.algorithm === facts.expected.algorithm && typeof facts.expected.id === "string" && /^[a-f0-9]{64}$/.test(facts.expected.id) && facts.actual.id === facts.expected.id;
|
|
16936
|
+
case "evidence":
|
|
16937
|
+
return facts.candidateMatched === true && facts.referencesVerified === true && Array.isArray(facts.obligations) && facts.obligations.some((item) => item?.due === "review") && facts.obligations.every((item) => item && ["review", "finish"].includes(item.due) && typeof item.id === "string" && item.id.length > 0 && (item.due === "review" ? item.state === "verified" : ["pending", "verified"].includes(item.state)));
|
|
16938
|
+
case "routing": {
|
|
16939
|
+
if (facts.observationComplete !== true || !Array.isArray(facts.productFindings) || !Array.isArray(facts.dueEvidenceGaps)) return false;
|
|
16940
|
+
const expected = facts.productFindings.length ? "build" : facts.dueEvidenceGaps.length ? "evidence" : "git-ops";
|
|
16941
|
+
return facts.next === expected;
|
|
16942
|
+
}
|
|
16943
|
+
case "reuse":
|
|
16944
|
+
return facts.operation === "static-inspection" && facts.purpose === "same-check" && facts.previous?.state === "complete" && facts.previous.outputsVerified === true && typeof facts.currentInput === "string" && /^[a-f0-9]{64}$/.test(facts.currentInput) && facts.previous.input === facts.currentInput && facts.previous.environment === facts.currentEnvironment && typeof facts.currentEnvironment === "string" && facts.currentEnvironment.length > 0;
|
|
16945
|
+
default:
|
|
16946
|
+
throw new TypeError("Unknown workflow fixture family");
|
|
16947
|
+
}
|
|
16948
|
+
}
|
|
16949
|
+
function workflowFixtureFamilies() {
|
|
16950
|
+
const root = "/fixture/pilot", hash4 = "a".repeat(64);
|
|
16951
|
+
const workspace = {
|
|
16952
|
+
requestedRoot: root,
|
|
16953
|
+
observedRoot: root,
|
|
16954
|
+
pathsObserved: true,
|
|
16955
|
+
requiredPaths: [`${root}/report.md`],
|
|
16956
|
+
readablePaths: [`${root}/report.md`]
|
|
16957
|
+
};
|
|
16958
|
+
const identity2 = {
|
|
16959
|
+
expected: { algorithm: "principal-candidate-v1", id: hash4 },
|
|
16960
|
+
actual: { algorithm: "principal-candidate-v1", id: hash4 }
|
|
16961
|
+
};
|
|
16962
|
+
const evidence = {
|
|
16963
|
+
candidateMatched: true,
|
|
16964
|
+
referencesVerified: true,
|
|
16965
|
+
obligations: [{ id: "behavior-check", due: "review", state: "verified" }, { id: "archive", due: "finish", state: "pending" }]
|
|
16966
|
+
};
|
|
16967
|
+
const routing = { observationComplete: true, productFindings: [], dueEvidenceGaps: ["missing-report"], next: "evidence" };
|
|
16968
|
+
const reuse = {
|
|
16969
|
+
operation: "static-inspection",
|
|
16970
|
+
purpose: "same-check",
|
|
16971
|
+
currentInput: hash4,
|
|
16972
|
+
currentEnvironment: "node-fixture-v1",
|
|
16973
|
+
previous: { state: "complete", input: hash4, environment: "node-fixture-v1", outputsVerified: true }
|
|
16974
|
+
};
|
|
16975
|
+
return [
|
|
16976
|
+
{
|
|
16977
|
+
family: "workspace",
|
|
16978
|
+
split: "train",
|
|
16979
|
+
question: "Do these explicitly observed roots and readable absolute paths establish access to the requested fixture workspace? Missing access evidence is not a permission-denial finding.",
|
|
16980
|
+
facts: [workspace, { ...workspace, observedRoot: "/fixture/original" }, { ...workspace, readablePaths: [] }, { ...workspace, pathsObserved: false }]
|
|
16981
|
+
},
|
|
16982
|
+
{
|
|
16983
|
+
family: "identity",
|
|
16984
|
+
split: "train",
|
|
16985
|
+
question: "Do these records contain matching full candidate IDs under the same principal-candidate-v1 algorithm? A diff hash or another algorithm is not an interchangeable candidate ID.",
|
|
16986
|
+
facts: [identity2, { ...identity2, actual: { algorithm: "git-diff-sha256", id: hash4 } }, { ...identity2, actual: { algorithm: "principal-candidate-v1", id: "b".repeat(64) } }, { expected: { algorithm: "principal-candidate-v1", id: "aaaaaaa" }, actual: { algorithm: "principal-candidate-v1", id: "aaaaaaa" } }]
|
|
16987
|
+
},
|
|
16988
|
+
{
|
|
16989
|
+
family: "evidence",
|
|
16990
|
+
split: "validation",
|
|
16991
|
+
question: "Are all review-due obligations verified by matching-candidate, verified references? Finish-due obligations may remain pending; a new runtime replay is not required by this fixture rule.",
|
|
16992
|
+
facts: [evidence, { ...evidence, obligations: [{ id: "behavior-check", due: "review", state: "pending" }] }, { ...evidence, candidateMatched: false }, { ...evidence, referencesVerified: false }]
|
|
16993
|
+
},
|
|
16994
|
+
{
|
|
16995
|
+
family: "routing",
|
|
16996
|
+
split: "test",
|
|
16997
|
+
question: "Does Next follow this explicit routing rule: observed product findings -> build, otherwise due evidence gaps -> evidence, otherwise git-ops? Incomplete observations cannot establish routing.",
|
|
16998
|
+
facts: [routing, { ...routing, next: "build" }, { ...routing, productFindings: ["blocked-api-shutdown"], next: "build" }, { ...routing, observationComplete: false }]
|
|
16999
|
+
},
|
|
17000
|
+
{
|
|
17001
|
+
family: "reuse",
|
|
17002
|
+
split: "test",
|
|
17003
|
+
question: "May this completed static inspection be reused for the same check, unchanged input/environment and verified outputs? A new independent review or live runtime observation requires a distinct operation.",
|
|
17004
|
+
facts: [reuse, { ...reuse, previous: { ...reuse.previous, outputsVerified: false } }, { ...reuse, purpose: "independent-review" }, { ...reuse, operation: "live-database-observation" }]
|
|
17005
|
+
}
|
|
17006
|
+
];
|
|
17007
|
+
}
|
|
17008
|
+
|
|
16858
17009
|
// experiments/decision-shadow/learning-fixtures.mjs
|
|
16859
17010
|
var FIXTURE_ORACLE_VERSION = "mechanical-fixture-oracles-v1";
|
|
16860
17011
|
var COMMIT = /^[a-f0-9]{40}$/;
|
|
@@ -16895,9 +17046,10 @@ function evaluateFixture(family, e) {
|
|
|
16895
17046
|
throw new TypeError("Unknown fixture oracle family");
|
|
16896
17047
|
}
|
|
16897
17048
|
}
|
|
16898
|
-
function createLearningFixtures({ recordedAt = "2026-10-08T00:00:00.000Z" } = {}) {
|
|
17049
|
+
function createLearningFixtures({ recordedAt = "2026-10-08T00:00:00.000Z", set: set2 = "mechanical" } = {}) {
|
|
17050
|
+
if (!["mechanical", "workflow"].includes(set2)) throw new TypeError("Fixture set must be mechanical or workflow");
|
|
16899
17051
|
const commit = "a".repeat(40), complete = Object.fromEntries(PHASES.map((p) => [p, "complete"]));
|
|
16900
|
-
const
|
|
17052
|
+
const mechanicalFamilies = [
|
|
16901
17053
|
{ family: "candidate", split: "train", question: "Do the records establish one exact full 40-character lowercase Git candidate shared by candidate, tested and reviewed?", facts: [{ candidate: commit, tested: commit, reviewed: commit }, { candidate: commit, tested: "b".repeat(40), reviewed: commit }, { candidate: commit, tested: commit, reviewed: "c".repeat(40) }, { candidate: "aaaaaaa", tested: "aaaaaaa", reviewed: "aaaaaaa" }] },
|
|
16902
17054
|
{ family: "phases", split: "train", question: "Do the latest snapshots cover exactly every required step with all five named phases complete? Earlier complete phases cannot fill later omissions.", facts: [{ required: ["step-1"], records: [{ step: "step-1", phases: complete }] }, { required: ["step-1", "step-2"], records: [{ step: "step-1", phases: complete }] }, { required: ["step-1"], records: [{ step: "step-1", phases: complete }, { step: "step-extra", phases: complete }] }, { required: ["step-1"], records: [{ step: "step-1", phases: complete }, { step: "step-1", phases: { ...complete, verified: "unknown" } }] }] },
|
|
16903
17055
|
{ family: "findings", split: "train", question: "Does the observed finding history establish that every finding identity (step, source, id) has an explicit latest verified disposition? Omission never clears an earlier finding.", facts: [{ historyObserved: true, records: [{ step: "step-1", findings: [] }] }, { historyObserved: true, records: [{ step: "step-1", findings: [{ id: "f1", source: "report-a", status: "open" }] }, { step: "step-1", findings: [{ id: "f1", source: "report-a", status: "verified" }] }] }, { historyObserved: true, records: [{ step: "step-1", findings: [{ id: "f1", source: "report-a", status: "open" }] }, { step: "step-1", findings: [] }] }, { historyObserved: true, records: [{ step: "step-1", findings: [{ id: "f1", source: "report-a", status: "open" }] }, { step: "step-1", findings: [{ id: "f1", source: "report-b", status: "verified" }] }] }] },
|
|
@@ -16905,15 +17057,20 @@ function createLearningFixtures({ recordedAt = "2026-10-08T00:00:00.000Z" } = {}
|
|
|
16905
17057
|
{ family: "report", split: "test", question: "Does this structured requested delivery comply with the selected saved-full-report/five-line-summary protocol, including its explicit unavailable-persistence blocked exception?", facts: [{ protocol: "saved-full-report-and-five-line-summary", persistence: "available", requestedDelivery: "saved-full-report-and-five-line-summary", reportReference: "required" }, { protocol: "saved-full-report-and-five-line-summary", persistence: "available", requestedDelivery: "complete-report-only-in-final", reportReference: "forbidden" }, { protocol: "saved-full-report-and-five-line-summary", persistence: "available", requestedDelivery: "saved-full-report-and-five-line-summary", reportReference: "omitted" }, { protocol: "saved-full-report-and-five-line-summary", persistence: "unavailable", requestedDelivery: "blocked-no-invented-report", reportReference: "not-saved" }] },
|
|
16906
17058
|
{ family: "cleanup", split: "test", question: "Do these records establish settled cleanup with a matching execution identity and reapedAll true? This does not establish task success or approval.", facts: [{ executionId: "exec:one", state: "settled", receipt: { state: "settled", executionId: "exec:one", reapedAll: true } }, { executionId: "exec:one", state: "settled", receipt: null }, { executionId: "exec:one", state: "settled", receipt: { state: "settled", executionId: "exec:other", reapedAll: true } }, { executionId: "exec:one", state: "settled", receipt: { state: "settled", executionId: "exec:one", reapedAll: false } }] }
|
|
16907
17059
|
];
|
|
17060
|
+
const families = set2 === "workflow" ? workflowFixtureFamilies() : mechanicalFamilies;
|
|
17061
|
+
const evaluate = set2 === "workflow" ? evaluateWorkflowFixture : evaluateFixture;
|
|
17062
|
+
const oracleVersion = set2 === "workflow" ? WORKFLOW_ORACLE_VERSION : FIXTURE_ORACLE_VERSION;
|
|
17063
|
+
const experimentId = set2 === "workflow" ? "workflow-convergence-20-v1" : "mechanical-fixtures-24-v1";
|
|
17064
|
+
const scope = set2 === "workflow" ? "20 synthetic workflow convergence cases; no observed sessions or model-quality claims." : "24 synthetic mechanical plumbing cases; no observed examples or model-quality claims.";
|
|
16908
17065
|
const caseDocument = { schema: 1, cases: families.flatMap((f) => f.facts.map((facts, i) => {
|
|
16909
17066
|
const input = JSON.stringify(facts);
|
|
16910
|
-
return { id: `fixture-${f.family}-${i + 1}`, input, question: f.question, provenance: "synthetic", source: { sha256: learningDigest(input), recordId: `${
|
|
17067
|
+
return { id: `fixture-${f.family}-${i + 1}`, input, question: f.question, provenance: "synthetic", source: { sha256: learningDigest(input), recordId: `${oracleVersion}/${f.family}/${i + 1}` }, visibility: "public" };
|
|
16911
17068
|
})) };
|
|
16912
17069
|
const cases = parseCases(caseDocument), receipts = [], labels = [], entries = [];
|
|
16913
17070
|
for (const c of cases) {
|
|
16914
17071
|
const family = c.id.split("-")[1], f = families.find((f2) => f2.family === family);
|
|
16915
|
-
const value =
|
|
16916
|
-
const receipt = { schema: 1, kind: "independent-decision-label", caseId: c.id, caseHash: c.hash, source: c.source, value, labelKind: "test", actor:
|
|
17072
|
+
const value = evaluate(family, JSON.parse(c.input));
|
|
17073
|
+
const receipt = { schema: 1, kind: "independent-decision-label", caseId: c.id, caseHash: c.hash, source: c.source, value, labelKind: "test", actor: oracleVersion, independent: true, recordedAt, method: { kind: "deterministic-test", id: family, version: oracleVersion + ":" + learningDigest(evaluate.toString()) } };
|
|
16917
17074
|
const bytes = JSON.stringify(receipt, null, 2) + "\n";
|
|
16918
17075
|
receipts.push({ caseId: c.id, filename: c.id + "-label.json", bytes, sha256: learningDigest(bytes) });
|
|
16919
17076
|
labels.push({ caseId: c.id, caseHash: c.hash, value, kind: "test", actor: receipt.actor, evidenceSha256: learningDigest(bytes), independent: true });
|
|
@@ -16921,8 +17078,8 @@ function createLearningFixtures({ recordedAt = "2026-10-08T00:00:00.000Z" } = {}
|
|
|
16921
17078
|
}
|
|
16922
17079
|
const labelDocument = { schema: 1, labels };
|
|
16923
17080
|
parseLabels(labelDocument, cases);
|
|
16924
|
-
const manifest = parseExperiment(cases, { schema: 1, kind: "decision-learning-experiment", id:
|
|
16925
|
-
return { caseDocument, labelDocument, manifest, labelReceipts: receipts, fixtureOnly: true, trainingEligible: false, scope
|
|
17081
|
+
const manifest = parseExperiment(cases, { schema: 1, kind: "decision-learning-experiment", id: experimentId, frozenAt: recordedAt, entries });
|
|
17082
|
+
return { caseDocument, labelDocument, manifest, labelReceipts: receipts, fixtureOnly: true, trainingEligible: false, scope };
|
|
16926
17083
|
}
|
|
16927
17084
|
|
|
16928
17085
|
// experiments/decision-shadow/research.mjs
|
|
@@ -17367,13 +17524,14 @@ var RESEARCH_OPTIONS = {
|
|
|
17367
17524
|
fixtures: ["out"]
|
|
17368
17525
|
};
|
|
17369
17526
|
var RESEARCH_OPTIONAL = {
|
|
17527
|
+
fixtures: ["set"],
|
|
17370
17528
|
"run-pi": ["pi-node", "auth-path", "timeout-ms"]
|
|
17371
17529
|
};
|
|
17372
17530
|
async function researchCommand(command, flags, helpers, options) {
|
|
17373
17531
|
const { cases, jsonFile: jsonFile2, textFile: textFile2, writeNew: writeNew3, readLegacyRun } = helpers;
|
|
17374
17532
|
const { emit = console.log, piRunner } = options;
|
|
17375
17533
|
if (command === "fixtures") {
|
|
17376
|
-
const f = createLearningFixtures({});
|
|
17534
|
+
const f = createLearningFixtures({ set: flags.set ?? "mechanical" });
|
|
17377
17535
|
const dir = resolve15(flags.out);
|
|
17378
17536
|
await mkdir4(dir, { mode: 448 });
|
|
17379
17537
|
const refs = [];
|
|
@@ -18257,7 +18415,7 @@ var help = `Decision research workflow (Node >=20; qualified Pi execution needs
|
|
|
18257
18415
|
score --cases FILE --labels FILE --run RESULT.jsonl [--run RESULT2.jsonl]
|
|
18258
18416
|
corpus --cases FILE --labels FILE --out NEW.json
|
|
18259
18417
|
verify-sources --cases FILE --sources SELECTION.json --evidence-root DIR --out NEW.json
|
|
18260
|
-
fixtures --out NEW_DIR
|
|
18418
|
+
fixtures --out NEW_DIR [--set mechanical|workflow]
|
|
18261
18419
|
validate-experiment --cases FILE --experiment FILE
|
|
18262
18420
|
preview-pi --cases FILE --experiment FILE --model openai-codex:MODEL --thinking LEVEL --split test
|
|
18263
18421
|
run-pi --cases FILE --experiment FILE --model openai-codex:MODEL --thinking LEVEL --split test --pi-package DIR --out NEW.jsonl --allow-subscription [--pi-node PATH] [--auth-path FILE] [--timeout-ms N]
|
|
@@ -19063,6 +19221,41 @@ function createJevController(pi, options = {}) {
|
|
|
19063
19221
|
|
|
19064
19222
|
// packages/pi-extension/src/jev-advice.ts
|
|
19065
19223
|
import { Type as Type3 } from "typebox";
|
|
19224
|
+
var nullableNumber = Type3.Union([Type3.Number(), Type3.Null()]);
|
|
19225
|
+
var nullableString = Type3.Union([Type3.String(), Type3.Null()]);
|
|
19226
|
+
var adviceOutput = Type3.Union([
|
|
19227
|
+
Type3.Object({
|
|
19228
|
+
enabled: Type3.Boolean(),
|
|
19229
|
+
mode: Type3.Union([Type3.Literal("disabled"), Type3.Literal("manual"), Type3.Literal("workflow")]),
|
|
19230
|
+
storage: Type3.Union([Type3.Literal("granted"), Type3.Literal("declined"), Type3.Null()]),
|
|
19231
|
+
remaining: Type3.Integer({ minimum: 0 }),
|
|
19232
|
+
availability: Type3.String(),
|
|
19233
|
+
providerReadiness: Type3.Union([Type3.Literal("key-present"), Type3.Literal("missing-key")]),
|
|
19234
|
+
advisory: Type3.Literal(true)
|
|
19235
|
+
}, { additionalProperties: false }),
|
|
19236
|
+
Type3.Object({
|
|
19237
|
+
advisory: Type3.Literal(true),
|
|
19238
|
+
status: Type3.Union([Type3.Literal("answered"), Type3.Literal("unavailable")]),
|
|
19239
|
+
probability: Type3.Union([Type3.Number({ minimum: 0, maximum: 1 }), Type3.Null()]),
|
|
19240
|
+
provider: Type3.String(),
|
|
19241
|
+
requestedModel: Type3.String(),
|
|
19242
|
+
resolvedModel: nullableString,
|
|
19243
|
+
usage: Type3.Object({ inputTokens: nullableNumber, outputTokens: nullableNumber, costUsd: nullableNumber }, { additionalProperties: false }),
|
|
19244
|
+
latencyMs: nullableNumber,
|
|
19245
|
+
reason: Type3.Optional(Type3.String()),
|
|
19246
|
+
remaining: Type3.Integer({ minimum: 0 }),
|
|
19247
|
+
inputSha256: Type3.Optional(Type3.String()),
|
|
19248
|
+
reused: Type3.Optional(Type3.Boolean()),
|
|
19249
|
+
source: Type3.Optional(Type3.Object({
|
|
19250
|
+
kind: Type3.String(),
|
|
19251
|
+
sha256: Type3.String(),
|
|
19252
|
+
sessionId: Type3.String(),
|
|
19253
|
+
toolCallId: Type3.String(),
|
|
19254
|
+
retained: Type3.Boolean(),
|
|
19255
|
+
path: Type3.Optional(Type3.String())
|
|
19256
|
+
}, { additionalProperties: false }))
|
|
19257
|
+
}, { additionalProperties: false })
|
|
19258
|
+
]);
|
|
19066
19259
|
function createJevAdviceTool(controller) {
|
|
19067
19260
|
return {
|
|
19068
19261
|
name: "jev_advice",
|
|
@@ -19079,6 +19272,7 @@ function createJevAdviceTool(controller) {
|
|
|
19079
19272
|
requirements: Type3.Optional(Type3.String({ minLength: 1, maxLength: 16e3, description: "Explicit acceptance checks, before seeing independent review or JEV outcomes." })),
|
|
19080
19273
|
evidence: Type3.Optional(Type3.String({ minLength: 1, maxLength: 16e3, description: "Selected decision-time handoff evidence. Complete JSON packet is limited to 16000 characters." }))
|
|
19081
19274
|
}, { additionalProperties: false }),
|
|
19275
|
+
outputSchema: adviceOutput,
|
|
19082
19276
|
async execute(id, params, signal, _onUpdate, ctx) {
|
|
19083
19277
|
requireIdentity(id, "Pi toolCallId");
|
|
19084
19278
|
if (!params || typeof params !== "object" || Array.isArray(params)) throw new Error("Invalid jev_advice parameters");
|
|
@@ -19096,7 +19290,7 @@ function createJevAdviceTool(controller) {
|
|
|
19096
19290
|
ctx
|
|
19097
19291
|
);
|
|
19098
19292
|
else throw new Error("jev_advice action must be status or evaluate");
|
|
19099
|
-
return { content: [{ type: "text", text: JSON.stringify(details) }], details };
|
|
19293
|
+
return { content: [{ type: "text", text: JSON.stringify(details) }], details, structuredContent: details };
|
|
19100
19294
|
}
|
|
19101
19295
|
};
|
|
19102
19296
|
}
|
|
@@ -19108,11 +19302,23 @@ function registerJevAdvice(pi, controller) {
|
|
|
19108
19302
|
function index_default(pi) {
|
|
19109
19303
|
const moduleDir = dirname10(fileURLToPath2(import.meta.url));
|
|
19110
19304
|
const assetsDir = basename4(dirname10(moduleDir)) === "skill-harness" ? join35(moduleDir, "..", "assets") : join35(moduleDir, "..", "..", "..", "assets");
|
|
19305
|
+
const packageRoot = basename4(dirname10(moduleDir)) === "skill-harness" ? join35(moduleDir, "..") : join35(moduleDir, "..", "..", "..");
|
|
19306
|
+
let loadedVersion;
|
|
19307
|
+
try {
|
|
19308
|
+
const manifest = JSON.parse(readFileSync20(join35(packageRoot, "package.json"), "utf8"));
|
|
19309
|
+
if (["skill-harness", "skill-harness-monorepo"].includes(manifest.name)) loadedVersion = manifest.version;
|
|
19310
|
+
} catch {
|
|
19311
|
+
}
|
|
19312
|
+
const removeVersionReporter = pi.events?.on("pi-daddy:ecosystem-versions:v1", (request) => {
|
|
19313
|
+
const report = request?.report;
|
|
19314
|
+
if (typeof report === "function" && typeof loadedVersion === "string") report({ id: "skill-harness", version: loadedVersion, root: packageRoot });
|
|
19315
|
+
});
|
|
19111
19316
|
const jev = createJevController(pi);
|
|
19112
19317
|
registerCommand(pi, assetsDir, jev.command);
|
|
19113
19318
|
registerJevAdvice(pi, jev);
|
|
19114
19319
|
registerTool(pi);
|
|
19115
19320
|
pi.on("session_shutdown", async () => {
|
|
19321
|
+
removeVersionReporter?.();
|
|
19116
19322
|
closeReview();
|
|
19117
19323
|
});
|
|
19118
19324
|
}
|
package/package.json
CHANGED
|
@@ -1,58 +1,58 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
2
|
+
"name": "skill-harness",
|
|
3
|
+
"version": "0.27.0",
|
|
4
|
+
"description": "Test/optimize loop for agent skills \u2014 run spec'd scenarios on pi, LLM-judge, score, review, re-run",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"agent-skills",
|
|
8
|
+
"skill",
|
|
9
|
+
"llm",
|
|
10
|
+
"testing",
|
|
11
|
+
"eval",
|
|
12
|
+
"grading",
|
|
13
|
+
"pi",
|
|
14
|
+
"ci",
|
|
15
|
+
"harness"
|
|
16
|
+
],
|
|
17
|
+
"bin": {
|
|
18
|
+
"skill-harness": "bin.js"
|
|
19
|
+
},
|
|
20
|
+
"pi": {
|
|
21
|
+
"extensions": [
|
|
22
|
+
"./dist/index.js"
|
|
23
|
+
]
|
|
24
|
+
},
|
|
25
|
+
"files": [
|
|
26
|
+
"bin.js",
|
|
27
|
+
"dist/index.js",
|
|
28
|
+
"assets/report.template.html",
|
|
29
|
+
"assets/report.grade.js",
|
|
30
|
+
"LICENSE",
|
|
31
|
+
"README.md"
|
|
32
|
+
],
|
|
33
|
+
"dependencies": {
|
|
34
|
+
"@skill-harness/cli": "0.27.0"
|
|
35
|
+
},
|
|
36
|
+
"peerDependencies": {
|
|
37
|
+
"typebox": "*"
|
|
38
|
+
},
|
|
39
|
+
"peerDependenciesMeta": {
|
|
40
|
+
"typebox": {
|
|
41
|
+
"optional": true
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
"devDependencies": {
|
|
45
|
+
"typebox": "^1.1.38"
|
|
46
|
+
},
|
|
47
|
+
"repository": {
|
|
48
|
+
"type": "git",
|
|
49
|
+
"url": "git+https://github.com/mojomanyana/skill-harness.git"
|
|
50
|
+
},
|
|
51
|
+
"license": "MIT",
|
|
52
|
+
"engines": {
|
|
53
|
+
"node": ">=20"
|
|
54
|
+
},
|
|
55
|
+
"scripts": {
|
|
56
|
+
"prepack": "node ../../scripts/release-pack.mjs --guard"
|
|
42
57
|
}
|
|
43
|
-
},
|
|
44
|
-
"devDependencies": {
|
|
45
|
-
"typebox": "^1.1.38"
|
|
46
|
-
},
|
|
47
|
-
"repository": {
|
|
48
|
-
"type": "git",
|
|
49
|
-
"url": "git+https://github.com/mojomanyana/skill-harness.git"
|
|
50
|
-
},
|
|
51
|
-
"license": "MIT",
|
|
52
|
-
"engines": {
|
|
53
|
-
"node": ">=20"
|
|
54
|
-
},
|
|
55
|
-
"scripts": {
|
|
56
|
-
"prepack": "node ../../scripts/release-pack.mjs --guard"
|
|
57
|
-
}
|
|
58
58
|
}
|