statelyai 0.15.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/bin.mjs +1 -1
- package/dist/{cli-CE7KhcOJ.mjs → cli-CqVcBAPG.mjs} +7 -14
- package/dist/index.d.mts +15 -15
- package/dist/index.mjs +1 -1
- package/dist/skills/setup-behavior-ci/SKILL.md +25 -0
- package/dist/skills/setup-behavior-ci/agents/openai.yaml +3 -0
- package/dist/skills/setup-behavior-ci/assets/campaign-adapter.mjs +203 -0
- package/dist/skills/setup-behavior-ci/assets/campaign-producer.mjs +63 -0
- package/dist/skills/setup-behavior-ci/assets/config.json +39 -0
- package/dist/skills/setup-behavior-ci/assets/estimated-model.json +12 -0
- package/dist/skills/setup-behavior-ci/assets/executable-config.json +42 -0
- package/dist/skills/setup-behavior-ci/assets/minimal.campaign.mjs +50 -0
- package/dist/skills/setup-behavior-ci/assets/plan.json +86 -0
- package/dist/skills/setup-behavior-ci/assets/vitest-adapter.mjs +47 -0
- package/dist/skills/setup-behavior-ci/assets/workflow.yml +28 -0
- package/dist/skills/setup-behavior-ci/references/contract.md +31 -0
- package/dist/skills/setup-behavior-ci/references/executable.md +41 -0
- package/dist/testing.d.mts +1 -0
- package/dist/testing.mjs +3 -0
- package/package.json +12 -8
package/README.md
CHANGED
|
@@ -48,9 +48,9 @@ npm install --global statelyai
|
|
|
48
48
|
|
|
49
49
|
<!-- trusted local campaign command from src/testCommand.ts -->
|
|
50
50
|
|
|
51
|
-
`statelyai test ./feature.campaign.mjs` runs graph paths and property testing through
|
|
51
|
+
`statelyai test ./feature.campaign.mjs` runs graph paths and property testing through the shared internal testing runtime. Install `statelyai` in the campaign project and import authoring helpers from `statelyai/testing`. The campaign is trusted JavaScript with ordinary local permissions. The child process is not a sandbox.
|
|
52
52
|
|
|
53
|
-
Use `--report report.json --fixture regression.json` to save results and a concrete counterexample, then `--replay regression.json` after fixing the app. Exit codes are 0 for passed, 1 for a model/app discrepancy, and 2 for inconclusive execution. See [Test application behavior](../../docs/cli/test.md) for budgets and replay, or the [runnable browser example](../
|
|
53
|
+
Use `--report report.json --fixture regression.json` to save results and a concrete counterexample, then `--replay regression.json` after fixing the app. Exit codes are 0 for passed, 1 for a model/app discrepancy, and 2 for inconclusive execution. See [Test application behavior](../../docs/cli/test.md) for budgets and replay, or the [runnable browser example](../mcp/testing/README.md#verify-the-browser-example).
|
|
54
54
|
|
|
55
55
|
## Quick start
|
|
56
56
|
|
package/dist/bin.mjs
CHANGED
|
@@ -9,7 +9,7 @@ import { fileURLToPath, pathToFileURL } from "node:url";
|
|
|
9
9
|
import { z, z as z$1 } from "zod";
|
|
10
10
|
import { Args, Command, Flags, flush, handle, run } from "@oclif/core";
|
|
11
11
|
import { changeSet, createFileStore, createLoop, editorGraphRevision, graphPathGenerationId, graphPathNextCursor, graphPathsPageSchema, parseModelGraph, proposalOutline, stableJson, verdictLines, verify } from "@statelyai/mcp/loop";
|
|
12
|
-
import { campaignExitCode, modelTestReplaySchema, reportReplays, runCampaignFile } from "@statelyai/
|
|
12
|
+
import { campaignExitCode, modelTestReplaySchema, reportReplays, runCampaignFile } from "@statelyai/mcp/testing";
|
|
13
13
|
import { StudioApiError, buildAuthorizeUrl, createPkcePair, createStatelyClient, discoverAuthorizationServer, discoverProtectedResource, exchangeCodeForToken, refreshToken, registerOAuthClient } from "@statelyai/sdk";
|
|
14
14
|
import * as ts$1 from "typescript";
|
|
15
15
|
import ts from "typescript";
|
|
@@ -31,7 +31,7 @@ var __commonJSMin$1 = (cb, mod) => () => (mod || cb((mod = { exports: {} }).expo
|
|
|
31
31
|
|
|
32
32
|
//#endregion
|
|
33
33
|
//#region src/version.ts
|
|
34
|
-
const CLI_VERSION = "0.
|
|
34
|
+
const CLI_VERSION = "0.16.0";
|
|
35
35
|
|
|
36
36
|
//#endregion
|
|
37
37
|
//#region src/behaviorCi.ts
|
|
@@ -1273,7 +1273,7 @@ async function prepareTestOutputs(options) {
|
|
|
1273
1273
|
var TestCommand = class TestCommand extends Command {
|
|
1274
1274
|
static enableJsonFlag = false;
|
|
1275
1275
|
static summary = "Run a trusted local model-based testing campaign.";
|
|
1276
|
-
static description = "Imports and executes a trusted .mjs campaign with ordinary local permissions. This is not a sandbox. Install
|
|
1276
|
+
static description = "Imports and executes a trusted .mjs campaign with ordinary local permissions. This is not a sandbox. Install statelyai in the campaign project and import statelyai/testing. Runs graph paths and property tests; --replay checks a saved counterexample as a regression. Exit 0 passes, 1 is a model/app discrepancy, 2 is inconclusive or an infrastructure failure.";
|
|
1277
1277
|
static args = { campaign: Args.string({
|
|
1278
1278
|
required: true,
|
|
1279
1279
|
description: "Trusted local .mjs campaign module"
|
|
@@ -2659,7 +2659,7 @@ function offsetAt(lineStarts, line, character) {
|
|
|
2659
2659
|
}
|
|
2660
2660
|
|
|
2661
2661
|
//#endregion
|
|
2662
|
-
//#region ../graph-tools/dist/semanticDiff-
|
|
2662
|
+
//#region ../graph-tools/dist/semanticDiff-pFPzuv8z.mjs
|
|
2663
2663
|
/**
|
|
2664
2664
|
* Convert a resolved {@link GraphNode} back into a {@link NodeConfig}.
|
|
2665
2665
|
*
|
|
@@ -5601,7 +5601,7 @@ function createMachine(config, implementations) {
|
|
|
5601
5601
|
}
|
|
5602
5602
|
|
|
5603
5603
|
//#endregion
|
|
5604
|
-
//#region ../graph-tools/dist/validation-
|
|
5604
|
+
//#region ../graph-tools/dist/validation-Du75Lsap.mjs
|
|
5605
5605
|
function resolveProfileData(data, namespace) {
|
|
5606
5606
|
const prefix = `${namespace}.`;
|
|
5607
5607
|
return Object.fromEntries(Object.entries(data).filter(([key]) => key.startsWith(prefix)).map(([key, value]) => [key.slice(prefix.length), value]));
|
|
@@ -6618,12 +6618,7 @@ function graphToMachineConfig(graph, options = {}) {
|
|
|
6618
6618
|
}
|
|
6619
6619
|
function analyzeXStateV5Compatibility(graph) {
|
|
6620
6620
|
const issues = [];
|
|
6621
|
-
const graphPaths = [
|
|
6622
|
-
["data.version", graph.data.version],
|
|
6623
|
-
["data.options", graph.data.options],
|
|
6624
|
-
["data.schemas.input", graph.data.schemas?.input],
|
|
6625
|
-
["data.schemas.output", graph.data.schemas?.output]
|
|
6626
|
-
].filter(([, value]) => value != null);
|
|
6621
|
+
const graphPaths = [["data.version", graph.data.version], ["data.options", graph.data.options]].filter(([, value]) => value != null);
|
|
6627
6622
|
if (graphPaths.length) issues.push({
|
|
6628
6623
|
level: "warning",
|
|
6629
6624
|
message: "XState v6 machine fields are retained but read-only in XState v5.",
|
|
@@ -6658,8 +6653,6 @@ const XSTATE_V5_PROJECTION = {
|
|
|
6658
6653
|
readOnlyPaths: [
|
|
6659
6654
|
"data.version",
|
|
6660
6655
|
"data.options",
|
|
6661
|
-
"data.schemas.input",
|
|
6662
|
-
"data.schemas.output",
|
|
6663
6656
|
"nodes.*.data.initialInput",
|
|
6664
6657
|
"nodes.*.data.timeout",
|
|
6665
6658
|
"nodes.*.data.route",
|
|
@@ -23859,7 +23852,7 @@ function describeCredentialBackend(backend, location) {
|
|
|
23859
23852
|
}
|
|
23860
23853
|
|
|
23861
23854
|
//#endregion
|
|
23862
|
-
//#region ../../node_modules/.pnpm/@statelyai+graph@2.1.0_@statelyai+layout@0.
|
|
23855
|
+
//#region ../../node_modules/.pnpm/@statelyai+graph@2.1.0_@statelyai+layout@0.5.0_cytoscape@3.34.0_d3-force@3.0.0_d3-hiera_b5a11f1847a44c5e54d10b16dab0c7b9/node_modules/@statelyai/graph/dist/index.mjs
|
|
23863
23856
|
const LAYOUT_KEYS = {
|
|
23864
23857
|
node: [
|
|
23865
23858
|
"x",
|
package/dist/index.d.mts
CHANGED
|
@@ -321,7 +321,7 @@ declare function createStatelyProjectConfig(options: {
|
|
|
321
321
|
defaultXStateVersion?: number;
|
|
322
322
|
}): StatelyProjectConfig;
|
|
323
323
|
//#endregion
|
|
324
|
-
//#region ../../node_modules/.pnpm/@statelyai+graph@2.1.0_@statelyai+layout@0.
|
|
324
|
+
//#region ../../node_modules/.pnpm/@statelyai+graph@2.1.0_@statelyai+layout@0.5.0_cytoscape@3.34.0_d3-force@3.0.0_d3-hiera_b5a11f1847a44c5e54d10b16dab0c7b9/node_modules/@statelyai/graph/dist/types-BAEQTwK_.d.mts
|
|
325
325
|
//#region src/types.d.ts
|
|
326
326
|
/**
|
|
327
327
|
* Directedness of a graph or an individual edge.
|
|
@@ -910,8 +910,8 @@ declare const verificationRunReportSchema: z.ZodObject<{
|
|
|
910
910
|
version: z.ZodLiteral<1>;
|
|
911
911
|
suite: z.ZodString;
|
|
912
912
|
status: z.ZodEnum<{
|
|
913
|
-
failed: "failed";
|
|
914
913
|
passed: "passed";
|
|
914
|
+
failed: "failed";
|
|
915
915
|
}>;
|
|
916
916
|
verdict: z.ZodEnum<{
|
|
917
917
|
inconclusive: "inconclusive";
|
|
@@ -1016,8 +1016,8 @@ declare const verificationRunReportSchema: z.ZodObject<{
|
|
|
1016
1016
|
type: z.ZodLiteral<"path-result">;
|
|
1017
1017
|
id: z.ZodString;
|
|
1018
1018
|
status: z.ZodEnum<{
|
|
1019
|
-
failed: "failed";
|
|
1020
1019
|
passed: "passed";
|
|
1020
|
+
failed: "failed";
|
|
1021
1021
|
}>;
|
|
1022
1022
|
witnessedStates: z.ZodArray<z.ZodString>;
|
|
1023
1023
|
witnessedTransitions: z.ZodArray<z.ZodString>;
|
|
@@ -1028,8 +1028,8 @@ declare const verificationRunReportSchema: z.ZodObject<{
|
|
|
1028
1028
|
state: z.ZodString;
|
|
1029
1029
|
event: z.ZodString;
|
|
1030
1030
|
status: z.ZodEnum<{
|
|
1031
|
-
failed: "failed";
|
|
1032
1031
|
passed: "passed";
|
|
1032
|
+
failed: "failed";
|
|
1033
1033
|
}>;
|
|
1034
1034
|
}, z.core.$strip>>>;
|
|
1035
1035
|
divergences: z.ZodArray<z.ZodObject<{
|
|
@@ -1073,8 +1073,8 @@ declare const verificationRunReportSchema: z.ZodObject<{
|
|
|
1073
1073
|
predicateResults: z.ZodArray<z.ZodObject<{
|
|
1074
1074
|
state: z.ZodString;
|
|
1075
1075
|
status: z.ZodEnum<{
|
|
1076
|
-
failed: "failed";
|
|
1077
1076
|
passed: "passed";
|
|
1077
|
+
failed: "failed";
|
|
1078
1078
|
}>;
|
|
1079
1079
|
signals: z.ZodArray<z.ZodString>;
|
|
1080
1080
|
message: z.ZodOptional<z.ZodString>;
|
|
@@ -1088,8 +1088,8 @@ declare const verificationRunReportSchema: z.ZodObject<{
|
|
|
1088
1088
|
predicateResults: z.ZodArray<z.ZodObject<{
|
|
1089
1089
|
state: z.ZodString;
|
|
1090
1090
|
status: z.ZodEnum<{
|
|
1091
|
-
failed: "failed";
|
|
1092
1091
|
passed: "passed";
|
|
1092
|
+
failed: "failed";
|
|
1093
1093
|
}>;
|
|
1094
1094
|
signals: z.ZodArray<z.ZodString>;
|
|
1095
1095
|
message: z.ZodOptional<z.ZodString>;
|
|
@@ -1159,8 +1159,8 @@ declare const verificationRunReportSchema: z.ZodObject<{
|
|
|
1159
1159
|
predicateResults: z.ZodArray<z.ZodObject<{
|
|
1160
1160
|
state: z.ZodString;
|
|
1161
1161
|
status: z.ZodEnum<{
|
|
1162
|
-
failed: "failed";
|
|
1163
1162
|
passed: "passed";
|
|
1163
|
+
failed: "failed";
|
|
1164
1164
|
}>;
|
|
1165
1165
|
signals: z.ZodArray<z.ZodString>;
|
|
1166
1166
|
message: z.ZodOptional<z.ZodString>;
|
|
@@ -1174,8 +1174,8 @@ declare const verificationRunReportSchema: z.ZodObject<{
|
|
|
1174
1174
|
predicateResults: z.ZodArray<z.ZodObject<{
|
|
1175
1175
|
state: z.ZodString;
|
|
1176
1176
|
status: z.ZodEnum<{
|
|
1177
|
-
failed: "failed";
|
|
1178
1177
|
passed: "passed";
|
|
1178
|
+
failed: "failed";
|
|
1179
1179
|
}>;
|
|
1180
1180
|
signals: z.ZodArray<z.ZodString>;
|
|
1181
1181
|
message: z.ZodOptional<z.ZodString>;
|
|
@@ -1229,8 +1229,8 @@ declare const verificationRequestSchema: z.ZodObject<{
|
|
|
1229
1229
|
version: z.ZodLiteral<1>;
|
|
1230
1230
|
suite: z.ZodString;
|
|
1231
1231
|
status: z.ZodEnum<{
|
|
1232
|
-
failed: "failed";
|
|
1233
1232
|
passed: "passed";
|
|
1233
|
+
failed: "failed";
|
|
1234
1234
|
}>;
|
|
1235
1235
|
verdict: z.ZodEnum<{
|
|
1236
1236
|
inconclusive: "inconclusive";
|
|
@@ -1335,8 +1335,8 @@ declare const verificationRequestSchema: z.ZodObject<{
|
|
|
1335
1335
|
type: z.ZodLiteral<"path-result">;
|
|
1336
1336
|
id: z.ZodString;
|
|
1337
1337
|
status: z.ZodEnum<{
|
|
1338
|
-
failed: "failed";
|
|
1339
1338
|
passed: "passed";
|
|
1339
|
+
failed: "failed";
|
|
1340
1340
|
}>;
|
|
1341
1341
|
witnessedStates: z.ZodArray<z.ZodString>;
|
|
1342
1342
|
witnessedTransitions: z.ZodArray<z.ZodString>;
|
|
@@ -1347,8 +1347,8 @@ declare const verificationRequestSchema: z.ZodObject<{
|
|
|
1347
1347
|
state: z.ZodString;
|
|
1348
1348
|
event: z.ZodString;
|
|
1349
1349
|
status: z.ZodEnum<{
|
|
1350
|
-
failed: "failed";
|
|
1351
1350
|
passed: "passed";
|
|
1351
|
+
failed: "failed";
|
|
1352
1352
|
}>;
|
|
1353
1353
|
}, z.core.$strip>>>;
|
|
1354
1354
|
divergences: z.ZodArray<z.ZodObject<{
|
|
@@ -1392,8 +1392,8 @@ declare const verificationRequestSchema: z.ZodObject<{
|
|
|
1392
1392
|
predicateResults: z.ZodArray<z.ZodObject<{
|
|
1393
1393
|
state: z.ZodString;
|
|
1394
1394
|
status: z.ZodEnum<{
|
|
1395
|
-
failed: "failed";
|
|
1396
1395
|
passed: "passed";
|
|
1396
|
+
failed: "failed";
|
|
1397
1397
|
}>;
|
|
1398
1398
|
signals: z.ZodArray<z.ZodString>;
|
|
1399
1399
|
message: z.ZodOptional<z.ZodString>;
|
|
@@ -1407,8 +1407,8 @@ declare const verificationRequestSchema: z.ZodObject<{
|
|
|
1407
1407
|
predicateResults: z.ZodArray<z.ZodObject<{
|
|
1408
1408
|
state: z.ZodString;
|
|
1409
1409
|
status: z.ZodEnum<{
|
|
1410
|
-
failed: "failed";
|
|
1411
1410
|
passed: "passed";
|
|
1411
|
+
failed: "failed";
|
|
1412
1412
|
}>;
|
|
1413
1413
|
signals: z.ZodArray<z.ZodString>;
|
|
1414
1414
|
message: z.ZodOptional<z.ZodString>;
|
|
@@ -1478,8 +1478,8 @@ declare const verificationRequestSchema: z.ZodObject<{
|
|
|
1478
1478
|
predicateResults: z.ZodArray<z.ZodObject<{
|
|
1479
1479
|
state: z.ZodString;
|
|
1480
1480
|
status: z.ZodEnum<{
|
|
1481
|
-
failed: "failed";
|
|
1482
1481
|
passed: "passed";
|
|
1482
|
+
failed: "failed";
|
|
1483
1483
|
}>;
|
|
1484
1484
|
signals: z.ZodArray<z.ZodString>;
|
|
1485
1485
|
message: z.ZodOptional<z.ZodString>;
|
|
@@ -1493,8 +1493,8 @@ declare const verificationRequestSchema: z.ZodObject<{
|
|
|
1493
1493
|
predicateResults: z.ZodArray<z.ZodObject<{
|
|
1494
1494
|
state: z.ZodString;
|
|
1495
1495
|
status: z.ZodEnum<{
|
|
1496
|
-
failed: "failed";
|
|
1497
1496
|
passed: "passed";
|
|
1497
|
+
failed: "failed";
|
|
1498
1498
|
}>;
|
|
1499
1499
|
signals: z.ZodArray<z.ZodString>;
|
|
1500
1500
|
message: z.ZodOptional<z.ZodString>;
|
package/dist/index.mjs
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import { A as formatVerificationJUnit, B as calibrateBehavior, C as run, D as resourceId, E as linkProject, F as readGraphFile, I as verifyReport, L as adoptBehavior, M as briefReport, N as changesReport, O as resolveVerificationRequest, P as pendingReport, R as behaviorCiConfigSchema, S as resolveProjectStatus, T as scanProjectSources, V as checkBehavior, _ as isFileGitDirty, a as discoverMachinesInSource, b as resolveCredentialStatus, c as formatDiscoveredMachines, d as formatProjectStatus, f as formatStoredCredentialSource, g as initProject, h as inferInitProjectName, i as discoverLinkedPullTargets, j as createStatelyProjectConfig, k as verifyProject, l as formatMissingNewMachineDirMessage, m as getEnvCredential, n as buildOAuthLoginStart, o as discoverOAuthLogin, p as getEnvApiKey, r as classifyPushCandidates, s as discoverRemoteProjectMachineTargets, t as COMMANDS, u as formatPlanSummary, v as resolveApiKey, w as scanProjectMachines, x as resolveOpenRootDir, y as resolveConfiguredPullTargets, z as behaviorCiResultsSchema } from "./cli-
|
|
1
|
+
import { A as formatVerificationJUnit, B as calibrateBehavior, C as run, D as resourceId, E as linkProject, F as readGraphFile, I as verifyReport, L as adoptBehavior, M as briefReport, N as changesReport, O as resolveVerificationRequest, P as pendingReport, R as behaviorCiConfigSchema, S as resolveProjectStatus, T as scanProjectSources, V as checkBehavior, _ as isFileGitDirty, a as discoverMachinesInSource, b as resolveCredentialStatus, c as formatDiscoveredMachines, d as formatProjectStatus, f as formatStoredCredentialSource, g as initProject, h as inferInitProjectName, i as discoverLinkedPullTargets, j as createStatelyProjectConfig, k as verifyProject, l as formatMissingNewMachineDirMessage, m as getEnvCredential, n as buildOAuthLoginStart, o as discoverOAuthLogin, p as getEnvApiKey, r as classifyPushCandidates, s as discoverRemoteProjectMachineTargets, t as COMMANDS, u as formatPlanSummary, v as resolveApiKey, w as scanProjectMachines, x as resolveOpenRootDir, y as resolveConfiguredPullTargets, z as behaviorCiResultsSchema } from "./cli-CqVcBAPG.mjs";
|
|
2
2
|
|
|
3
3
|
export { COMMANDS, adoptBehavior, behaviorCiConfigSchema, behaviorCiResultsSchema, briefReport, buildOAuthLoginStart, calibrateBehavior, changesReport, checkBehavior, classifyPushCandidates, createStatelyProjectConfig, discoverLinkedPullTargets, discoverMachinesInSource, discoverOAuthLogin, discoverRemoteProjectMachineTargets, formatDiscoveredMachines, formatMissingNewMachineDirMessage, formatPlanSummary, formatProjectStatus, formatStoredCredentialSource, formatVerificationJUnit, getEnvApiKey, getEnvCredential, inferInitProjectName, initProject, isFileGitDirty, linkProject, pendingReport, readGraphFile, resolveApiKey, resolveConfiguredPullTargets, resolveCredentialStatus, resolveOpenRootDir, resolveProjectStatus, resolveVerificationRequest, resourceId, run, scanProjectMachines, scanProjectSources, verifyProject, verifyReport };
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: setup-behavior-ci
|
|
3
|
+
description: Set up deterministic Stately behavior CI from existing Vitest tests or authored executable model-based campaigns, with reviewed requirements, calibrated mappings, real application adapters and immutable evidence history.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
Set up the selected feature from real source and authoritative intent. CI runs the existing CLI and tests without an LLM. Author the model and bindings for the user; do not ask them to maintain hashes.
|
|
7
|
+
|
|
8
|
+
Choose the mode before setup. Preserve the existing selected-Vitest mode when connecting current tests to an estimated model. For an executable model-based campaign, read [executable mode](references/executable.md), author the model, generators, real adapter and requirement assertions, then apply the shared calibration/adoption rules below. Do not treat estimated graph coverage as executed requirement evidence.
|
|
9
|
+
|
|
10
|
+
This skill ships with `statelyai` at `dist/skills/setup-behavior-ci/SKILL.md`. Resolve its exported `statelyai/skills/setup-behavior-ci/SKILL.md` path from a local install; copy the complete skill directory into the target agent's skill location when installation is requested. Its relative assets/references remain usable outside this monorepo. Repository `.agents/skills/setup-behavior-ci` is the sole maintained source; the package build copies it deterministically.
|
|
11
|
+
|
|
12
|
+
For selected-Vitest mode:
|
|
13
|
+
|
|
14
|
+
1. Discover the feature's existing test files, native runner command/configuration, source dependencies and authoritative requirements. Run the actual bounded Vitest selection once with `--reporter=json --outputFile=<fresh temporary file>`. Read native `testResults[].name` and `assertionResults[].fullName`; copy exact titles into mappings. Map every case in the selected native command, including existing public tests. Compare the complete `(file, fullName)` inventory against config mappings and require exact equality before proceeding. Label ancillary observed smoke checks as observations rather than invented requirements. Preserve existing behavioral assertions. Don't substitute toy tests or invent reporter titles.
|
|
15
|
+
2. Read [the contract](references/contract.md). Generate a short `spec.json` with meaningful stable requirement/check IDs, acceptance criteria, source authority/citations, and `retire: []`. Requirements inferred from implementation must be labeled inference. Estimate the behavior as XGraph JSON in `model.json`; describe states/transitions that matter, including retained edge cases. Entity mappings aid investigation; they do not prove coverage. Review generated meaning against source/spec and existing assertions. The model is an estimate, not executable test truth.
|
|
16
|
+
3. Independently challenge the estimated model with external inputs. Inventory all seven categories in [plan](assets/plan.json): user, host-update, async-result, cancellation, timer, server-event, lifecycle. Trace each relevant state owner and ingress in source, not just the selected tests. Combine operations across owners, such as host replacement after selection, cancellation before completion, disposal during work, and timer expiry during retry. Map discovered obligations to stable requirement/check/entity IDs, or record a cited gap or reasoned not-applicable entry. Keep cited intent, observations, inferences and assumptions distinct. Existing passing tests do not settle requirement conflicts.
|
|
17
|
+
4. Copy/adapt [config](assets/config.json) and the self-contained [Vitest adapter](assets/vitest-adapter.mjs). Select bounded source, test producer, runner configuration, and meaningful dependency files. Paths are repository-relative, including check file paths; command cwd may be a package. Include project/package manifests and lockfiles for the selected dependency scope. The CLI automatically includes root and command-cwd manifests/locks, config/spec/model/adapter/mapped tests and exact baseline files. Materialize external/user intent in checked requirements documents and list authoritative intent files in `plan.authorityInputs`; edits to them must invalidate calibration. Put runner/producer configuration modules in `plan.runnerInputs` so changes require recalibration. Pin CI Node and package-manager versions to the calibrated versions; original and current OS/architecture remain evidence. Limit all selected files together to 200. No broad workspace directory or `node_modules` selection. Keep state/output outside inputs.
|
|
18
|
+
5. Adapt [workflow](assets/workflow.yml) to the repository's package manager, lockfile, installation, CLI pin/build, selected command and paths. Use hosted runners by default, `contents: read`, unconditional artifact upload including hidden files and job summary. Keep `GITHUB_SHA` tied to actual checkout. Never run an LLM, auto-adopt, push, or add auth credentials in CI. Include the actual workflow path and Vitest config in config inputs before staging and checking. Existing trusted self-hosted runner workflows require same-repository trusted branch triggers, not arbitrary fork code.
|
|
19
|
+
6. Stage generated input files with `git add` before the first local run. `baseline: null` bootstraps a public FileStore model and behavior record. Run `statelyai behavior calibrate --config <config> --output <fresh ignored directory>` against the unchanged implementation first. The CLI executes the real native runner and generates a self-contained calibration receipt. Inspect every mismatch against authoritative requirements. Correct a wrong model/test assumption or fix the implementation, then recalibrate. If a known implementation defect must remain red, add a cited, reasoned `implementation-defect` disposition to the plan before recalibration. Unresolved calibration mismatches block adoption. Never copy observed bugs into the requirements automatically. Inspect summary, native report, argv/exit/logs, manifest, model bindings and current gaps. A passing run is exit 0; mapped test failures/missing/skipped checks are 1; invalid evidence is 2. Fix setup errors; do not shrink obligations to make red pass.
|
|
20
|
+
7. Explicitly run `statelyai behavior adopt <inspected output> --config <config>`. Adopt red as well as green records before the next evolution so failures remain in history. Commit generated config/spec/model/adapter/plan, the installed calibration receipt and the complete stateRoot `.statelyai` tree. Adoption updates baseline; no manual hashes. For dirty local runs, bound bytes must still match when adopting. `git add -f` may be needed when repository ignores `.statelyai`. Never commit output logs/runstore.
|
|
21
|
+
8. Verify the generated setup with the built/installed CLI and real tests. Check a deliberate assertion failure in a selected existing test produces exit 1, a missing required mapping cannot pass, and a restored run passes while history still shows failure. Any unmapped selected case, including a passing case, is ERROR and cannot be adopted; fix the mapping inventory before application repair. Never turn that ERROR into a claimed behavioral FAIL. Keep experimental changes isolated. Remote execution is a separate verification step requiring actual run/log/artifact evidence. On a remote red or green run, download and inspect the artifact, adopt it against its exact checked parent, then commit the advanced state before the next run.
|
|
22
|
+
|
|
23
|
+
Normal CI runs `behavior check` with the checked receipt. Changed model/spec/plan/mappings/adapter/producer or runner identity requires explicit native recalibration and inspected adoption. Implementation changes run against the stable calibrated contract. Calibration binds its original revision and files; it does not pin all future app revisions. Schema 1 remains readable but has no setup assurance claim.
|
|
24
|
+
|
|
25
|
+
On later requirement changes, update spec and model estimate, preserve IDs for the same obligation, and use an explicit reasoned retirement to remove one. Omitted requirements/checks remain active. Refresh mappings from actual reporter output when tests change. Adopt refuses divergent ancestry; inspect both refs and rerun from the chosen checked baseline rather than rewriting artifact history.
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
const statuses = new Set(["passed", "discrepancy", "inconclusive"]);
|
|
2
|
+
const nonempty = (value) => typeof value === "string" && value.trim().length > 0;
|
|
3
|
+
const count = (value) => Number.isSafeInteger(value) && value >= 0;
|
|
4
|
+
const fail = (message) => {
|
|
5
|
+
throw new Error(`Invalid campaign evidence: ${message}`);
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
export const executionFullName = "campaign.execution";
|
|
9
|
+
export const assertionFullName = (requirementId, checkId) =>
|
|
10
|
+
`campaign.assertion ${JSON.stringify([requirementId, checkId])}`;
|
|
11
|
+
|
|
12
|
+
export default function normalize(envelope) {
|
|
13
|
+
const { file, report } = envelope ?? {};
|
|
14
|
+
if (
|
|
15
|
+
!nonempty(file) ||
|
|
16
|
+
file.startsWith("/") ||
|
|
17
|
+
file.includes("\\") ||
|
|
18
|
+
file.split("/").some((part) => !part || part === "." || part === "..") ||
|
|
19
|
+
!file.endsWith(".mjs")
|
|
20
|
+
)
|
|
21
|
+
fail("file must be a repository-relative campaign .mjs path");
|
|
22
|
+
if (report?.formatVersion !== 1 || report.mode !== "campaign" || !statuses.has(report.status))
|
|
23
|
+
fail("expected a native full-campaign report");
|
|
24
|
+
if (
|
|
25
|
+
!Array.isArray(report.phases) ||
|
|
26
|
+
report.phases.length !== 2 ||
|
|
27
|
+
report.phases[0]?.phase !== "paths" ||
|
|
28
|
+
report.phases[1]?.phase !== "property" ||
|
|
29
|
+
report.phases.some((phase) => !statuses.has(phase.status))
|
|
30
|
+
)
|
|
31
|
+
fail("expected paths and property phases in order");
|
|
32
|
+
const status = report.phases.some((phase) => phase.status === "inconclusive")
|
|
33
|
+
? "inconclusive"
|
|
34
|
+
: report.phases.some((phase) => phase.status === "discrepancy")
|
|
35
|
+
? "discrepancy"
|
|
36
|
+
: "passed";
|
|
37
|
+
if (status !== report.status) fail("native verdict conflicts with phases");
|
|
38
|
+
|
|
39
|
+
if (
|
|
40
|
+
!nonempty(report.campaign?.id) ||
|
|
41
|
+
!nonempty(report.campaign.intent) ||
|
|
42
|
+
!nonempty(report.campaign.scope) ||
|
|
43
|
+
!nonempty(report.campaign.modelVersion) ||
|
|
44
|
+
!Array.isArray(report.campaign.assumptions) ||
|
|
45
|
+
!report.campaign.assumptions.every(nonempty) ||
|
|
46
|
+
!Array.isArray(report.campaign.evidence) ||
|
|
47
|
+
!report.campaign.evidence.every(nonempty) ||
|
|
48
|
+
!/^[a-f0-9]{64}$/.test(report.provenance?.entrySha256 ?? "") ||
|
|
49
|
+
report.provenance?.hashScope !== "entry-module-only" ||
|
|
50
|
+
!nonempty(report.provenance.node) ||
|
|
51
|
+
!["runner", "xstate", "xstateTest", "fastCheck"].every((key) =>
|
|
52
|
+
nonempty(report.provenance.versions?.[key]),
|
|
53
|
+
) ||
|
|
54
|
+
!Number.isSafeInteger(report.budget?.seed) ||
|
|
55
|
+
!["numRuns", "maxCommands", "pathLimit", "timeoutMs"].every(
|
|
56
|
+
(key) => count(report.budget?.[key]) && report.budget[key] > 0,
|
|
57
|
+
)
|
|
58
|
+
)
|
|
59
|
+
fail("native campaign metadata, provenance or budget missing");
|
|
60
|
+
for (const phase of report.phases) {
|
|
61
|
+
if (phase.status !== "passed" && !nonempty(phase.message)) fail("native diagnostic missing");
|
|
62
|
+
if (phase.status !== "passed" && phase.requirementGaps === undefined) continue;
|
|
63
|
+
const coverage = phase.coverage;
|
|
64
|
+
if (
|
|
65
|
+
!coverage ||
|
|
66
|
+
!count(coverage.totals?.runs) ||
|
|
67
|
+
coverage.totals.runs === 0 ||
|
|
68
|
+
!coverage.exploration ||
|
|
69
|
+
typeof coverage.exploration.truncated !== "boolean" ||
|
|
70
|
+
!Array.isArray(coverage.exploration.truncationReasons) ||
|
|
71
|
+
(coverage.exploration.truncated &&
|
|
72
|
+
(coverage.exploration.truncationReasons.length === 0 ||
|
|
73
|
+
coverage.exploration.truncationReasons.some(
|
|
74
|
+
(reason) => phase.phase !== "property" || reason !== "maximum sequence length reached",
|
|
75
|
+
))) ||
|
|
76
|
+
!Array.isArray(coverage.temporal?.failed) ||
|
|
77
|
+
coverage.temporal.failed.length > 0 ||
|
|
78
|
+
!Array.isArray(coverage.temporal?.inconclusive) ||
|
|
79
|
+
coverage.temporal.inconclusive.length > 0
|
|
80
|
+
)
|
|
81
|
+
fail("passing or coverage-gap native phase lacks completed execution evidence");
|
|
82
|
+
const assertions = [
|
|
83
|
+
"sutComparisons",
|
|
84
|
+
"oracleComparisons",
|
|
85
|
+
"invariantChecks",
|
|
86
|
+
"temporalChecks",
|
|
87
|
+
].map((key) => coverage.totals[key]);
|
|
88
|
+
assertions.push(phase.additionalAssertions ?? 0);
|
|
89
|
+
if (!assertions.every(count) || assertions.reduce((total, value) => total + value, 0) === 0)
|
|
90
|
+
fail("passing or coverage-gap native phase lacks observable assertions");
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const declarations = report.campaign?.requirements;
|
|
94
|
+
if (!Array.isArray(declarations) || declarations.length === 0)
|
|
95
|
+
fail("nonempty campaign.requirements declarations required");
|
|
96
|
+
const requirements = new Map();
|
|
97
|
+
const checks = new Set();
|
|
98
|
+
for (const requirement of declarations) {
|
|
99
|
+
if (
|
|
100
|
+
!nonempty(requirement?.id) ||
|
|
101
|
+
requirements.has(requirement.id) ||
|
|
102
|
+
!Array.isArray(requirement.checks) ||
|
|
103
|
+
requirement.checks.length === 0
|
|
104
|
+
)
|
|
105
|
+
fail("duplicate or malformed requirement declaration");
|
|
106
|
+
const entries = new Map();
|
|
107
|
+
for (const id of requirement.checks) {
|
|
108
|
+
if (!nonempty(id) || checks.has(id)) fail("duplicate or malformed assertion declaration");
|
|
109
|
+
checks.add(id);
|
|
110
|
+
entries.set(id, { passed: 0, failed: 0 });
|
|
111
|
+
}
|
|
112
|
+
requirements.set(requirement.id, entries);
|
|
113
|
+
}
|
|
114
|
+
for (const phase of report.phases) {
|
|
115
|
+
if (!Array.isArray(phase.requirements) || phase.requirements.length !== requirements.size)
|
|
116
|
+
fail(`missing requirement inventory in ${phase.phase}`);
|
|
117
|
+
const seenRequirements = new Set();
|
|
118
|
+
for (const requirement of phase.requirements) {
|
|
119
|
+
const expected = requirements.get(requirement?.id);
|
|
120
|
+
if (
|
|
121
|
+
!expected ||
|
|
122
|
+
seenRequirements.has(requirement.id) ||
|
|
123
|
+
!Array.isArray(requirement.checks) ||
|
|
124
|
+
requirement.checks.length !== expected.size
|
|
125
|
+
)
|
|
126
|
+
fail(`requirement inventory mismatch in ${phase.phase}`);
|
|
127
|
+
seenRequirements.add(requirement.id);
|
|
128
|
+
const seenChecks = new Set();
|
|
129
|
+
for (const check of requirement.checks) {
|
|
130
|
+
const total = expected.get(check?.id);
|
|
131
|
+
if (!total || seenChecks.has(check.id) || !count(check.passed) || !count(check.failed))
|
|
132
|
+
fail(`assertion inventory or counts invalid in ${phase.phase}`);
|
|
133
|
+
if (phase.status === "passed" && check.failed > 0)
|
|
134
|
+
fail("passed phase contains failed assertions");
|
|
135
|
+
seenChecks.add(check.id);
|
|
136
|
+
total.passed += check.passed;
|
|
137
|
+
total.failed += check.failed;
|
|
138
|
+
if (!count(total.passed) || !count(total.failed)) fail("assertion counts overflow");
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
const results = [];
|
|
143
|
+
for (const [requirementId, entries] of requirements) {
|
|
144
|
+
for (const [checkId, counts] of entries) {
|
|
145
|
+
results.push({
|
|
146
|
+
file,
|
|
147
|
+
fullName: assertionFullName(requirementId, checkId),
|
|
148
|
+
outcome: counts.failed ? "failed" : counts.passed ? "passed" : "skipped",
|
|
149
|
+
detail:
|
|
150
|
+
`Assertion ${checkId}: ${counts.passed} passed, ${counts.failed} failed invocations across paths and property, including shrinking.`.slice(
|
|
151
|
+
0,
|
|
152
|
+
4000,
|
|
153
|
+
),
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
const missing = results.filter((result) => result.outcome === "skipped");
|
|
158
|
+
const inconclusive = report.phases.filter((phase) => phase.status === "inconclusive");
|
|
159
|
+
const discrepancy = report.phases.some((phase) => phase.status === "discrepancy");
|
|
160
|
+
const missingIds = [...requirements.values()].flatMap((entries) =>
|
|
161
|
+
[...entries].filter(([, counts]) => counts.passed + counts.failed === 0).map(([id]) => id),
|
|
162
|
+
);
|
|
163
|
+
for (const phase of report.phases) {
|
|
164
|
+
if (
|
|
165
|
+
phase.requirementGaps !== undefined &&
|
|
166
|
+
(discrepancy ||
|
|
167
|
+
phase.phase !== "property" ||
|
|
168
|
+
phase.status !== "inconclusive" ||
|
|
169
|
+
!Array.isArray(phase.requirementGaps) ||
|
|
170
|
+
phase.requirementGaps.length === 0 ||
|
|
171
|
+
results.some((result) => result.outcome === "failed") ||
|
|
172
|
+
new Set(phase.requirementGaps).size !== phase.requirementGaps.length ||
|
|
173
|
+
JSON.stringify([...phase.requirementGaps].sort()) !==
|
|
174
|
+
JSON.stringify([...missingIds].sort()))
|
|
175
|
+
)
|
|
176
|
+
fail("requirement gap marker conflicts with unevaluated assertions");
|
|
177
|
+
}
|
|
178
|
+
const coverageOnly =
|
|
179
|
+
inconclusive.length > 0 && inconclusive.every((phase) => phase.requirementGaps !== undefined);
|
|
180
|
+
const detail = report.phases
|
|
181
|
+
.filter((phase) => phase.status !== "passed")
|
|
182
|
+
.map((phase) => `${phase.phase}: ${phase.message}`)
|
|
183
|
+
.join("\n");
|
|
184
|
+
results.push({
|
|
185
|
+
file,
|
|
186
|
+
fullName: executionFullName,
|
|
187
|
+
outcome:
|
|
188
|
+
inconclusive.length && !coverageOnly
|
|
189
|
+
? "error"
|
|
190
|
+
: discrepancy
|
|
191
|
+
? "failed"
|
|
192
|
+
: missing.length
|
|
193
|
+
? "skipped"
|
|
194
|
+
: "passed",
|
|
195
|
+
detail: (
|
|
196
|
+
detail ||
|
|
197
|
+
(missing.length
|
|
198
|
+
? "Required assertions were not executed."
|
|
199
|
+
: "Native paths and property execution passed.")
|
|
200
|
+
).slice(0, 4000),
|
|
201
|
+
});
|
|
202
|
+
return results;
|
|
203
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { parseArgs } from 'node:util';
|
|
4
|
+
import {
|
|
5
|
+
campaignExitCode,
|
|
6
|
+
modelTestReportSchema,
|
|
7
|
+
reportReplays,
|
|
8
|
+
runCampaignFile,
|
|
9
|
+
} from 'statelyai/testing';
|
|
10
|
+
import normalize from './campaign-adapter.mjs';
|
|
11
|
+
|
|
12
|
+
const { positionals, values } = parseArgs({
|
|
13
|
+
allowPositionals: true,
|
|
14
|
+
options: { report: { type: 'string' }, budget: { type: 'string' } },
|
|
15
|
+
});
|
|
16
|
+
if (positionals.length !== 1 || !values.report)
|
|
17
|
+
throw new Error(
|
|
18
|
+
'Usage: node campaign-producer.mjs <campaign.mjs> --report <file> [--budget <budget.json>]',
|
|
19
|
+
);
|
|
20
|
+
const file = path
|
|
21
|
+
.relative(process.cwd(), path.resolve(positionals[0]))
|
|
22
|
+
.split(path.sep)
|
|
23
|
+
.join('/');
|
|
24
|
+
if (
|
|
25
|
+
!file ||
|
|
26
|
+
file.startsWith('../') ||
|
|
27
|
+
path.isAbsolute(file) ||
|
|
28
|
+
!file.endsWith('.mjs')
|
|
29
|
+
)
|
|
30
|
+
throw new Error(
|
|
31
|
+
'Campaign must be a repository-relative .mjs file; run from repository root.',
|
|
32
|
+
);
|
|
33
|
+
const budget = values.budget
|
|
34
|
+
? JSON.parse(await readFile(values.budget, 'utf8'))
|
|
35
|
+
: undefined;
|
|
36
|
+
const report = modelTestReportSchema.parse(
|
|
37
|
+
await runCampaignFile(file, {
|
|
38
|
+
budget,
|
|
39
|
+
onLog: (text) => process.stderr.write(text),
|
|
40
|
+
}),
|
|
41
|
+
);
|
|
42
|
+
const output = path.resolve(
|
|
43
|
+
process.env.STATELY_BEHAVIOR_OUTPUT ?? path.dirname(values.report),
|
|
44
|
+
);
|
|
45
|
+
const traces = path.join(output, 'traces');
|
|
46
|
+
await mkdir(traces, { recursive: true });
|
|
47
|
+
const writeJson = (filename, value) =>
|
|
48
|
+
writeFile(filename, `${JSON.stringify(value, null, 2)}\n`);
|
|
49
|
+
await writeJson(path.join(traces, 'campaign-report.json'), report);
|
|
50
|
+
await writeJson(path.join(traces, 'campaign-process.json'), {
|
|
51
|
+
nativeExitCode: campaignExitCode(report),
|
|
52
|
+
});
|
|
53
|
+
for (const [index, replay] of reportReplays(report).entries())
|
|
54
|
+
await writeJson(path.join(traces, `replay-${index + 1}.json`), replay);
|
|
55
|
+
await mkdir(path.dirname(path.resolve(values.report)), { recursive: true });
|
|
56
|
+
const envelope = { file, report };
|
|
57
|
+
await writeJson(values.report, envelope);
|
|
58
|
+
const results = normalize(envelope);
|
|
59
|
+
process.exitCode = results.some(
|
|
60
|
+
(result) => result.outcome === 'failed' || result.outcome === 'error',
|
|
61
|
+
)
|
|
62
|
+
? 1
|
|
63
|
+
: 0;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 2,
|
|
3
|
+
"name": "Selected behavior",
|
|
4
|
+
"spec": ".behavior-ci/feature/spec.json",
|
|
5
|
+
"model": ".behavior-ci/feature/model.json",
|
|
6
|
+
"adapter": ".behavior-ci/feature/vitest-adapter.mjs",
|
|
7
|
+
"stateRoot": ".behavior-ci/feature/state",
|
|
8
|
+
"baseline": null,
|
|
9
|
+
"command": {
|
|
10
|
+
"executable": "pnpm",
|
|
11
|
+
"args": [
|
|
12
|
+
"exec",
|
|
13
|
+
"vitest",
|
|
14
|
+
"run",
|
|
15
|
+
"src/feature.test.ts",
|
|
16
|
+
"--reporter=json",
|
|
17
|
+
"--outputFile={report}"
|
|
18
|
+
],
|
|
19
|
+
"cwd": "."
|
|
20
|
+
},
|
|
21
|
+
"inputs": [
|
|
22
|
+
"src/feature.ts",
|
|
23
|
+
"src/feature.test.ts",
|
|
24
|
+
"vitest.config.ts",
|
|
25
|
+
".github/workflows/behavior.yml"
|
|
26
|
+
],
|
|
27
|
+
"checks": [
|
|
28
|
+
{
|
|
29
|
+
"id": "feature.check",
|
|
30
|
+
"file": "src/feature.test.ts",
|
|
31
|
+
"fullName": "exact native reporter fullName"
|
|
32
|
+
}
|
|
33
|
+
],
|
|
34
|
+
"timeoutMs": 120000,
|
|
35
|
+
"assurance": {
|
|
36
|
+
"plan": ".behavior-ci/feature/plan.json",
|
|
37
|
+
"calibration": ".behavior-ci/feature/calibration.json"
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "feature",
|
|
3
|
+
"nodes": [
|
|
4
|
+
{ "id": "feature", "type": "node", "parentId": null, "data": { "key": "feature" } },
|
|
5
|
+
{ "id": "feature.off", "type": "node", "parentId": "feature", "data": { "key": "off" } },
|
|
6
|
+
{ "id": "feature.on", "type": "node", "parentId": "feature", "data": { "key": "on" } }
|
|
7
|
+
],
|
|
8
|
+
"edges": [
|
|
9
|
+
{ "id": "feature.turn-on", "type": "edge", "sourceId": "feature.off", "targetId": "feature.on", "data": { "eventType": "ON" } }
|
|
10
|
+
],
|
|
11
|
+
"data": {}
|
|
12
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 2,
|
|
3
|
+
"name": "Executable campaign behavior",
|
|
4
|
+
"spec": ".behavior-ci/feature/spec.json",
|
|
5
|
+
"model": ".behavior-ci/feature/model.json",
|
|
6
|
+
"adapter": ".behavior-ci/feature/campaign-adapter.mjs",
|
|
7
|
+
"stateRoot": ".behavior-ci/feature/state",
|
|
8
|
+
"baseline": null,
|
|
9
|
+
"command": {
|
|
10
|
+
"executable": "node",
|
|
11
|
+
"args": [
|
|
12
|
+
".behavior-ci/feature/campaign-producer.mjs",
|
|
13
|
+
".behavior-ci/feature/feature.campaign.mjs",
|
|
14
|
+
"--report",
|
|
15
|
+
"{report}"
|
|
16
|
+
],
|
|
17
|
+
"cwd": "."
|
|
18
|
+
},
|
|
19
|
+
"inputs": [
|
|
20
|
+
"src/feature.ts",
|
|
21
|
+
".behavior-ci/feature/feature.campaign.mjs",
|
|
22
|
+
".behavior-ci/feature/sut.mjs",
|
|
23
|
+
".github/workflows/behavior.yml"
|
|
24
|
+
],
|
|
25
|
+
"checks": [
|
|
26
|
+
{
|
|
27
|
+
"id": "feature.assertion",
|
|
28
|
+
"file": ".behavior-ci/feature/feature.campaign.mjs",
|
|
29
|
+
"fullName": "campaign.assertion [\"feature.requirement\",\"feature.assertion\"]"
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"id": "feature.execution",
|
|
33
|
+
"file": ".behavior-ci/feature/feature.campaign.mjs",
|
|
34
|
+
"fullName": "campaign.execution"
|
|
35
|
+
}
|
|
36
|
+
],
|
|
37
|
+
"timeoutMs": 120000,
|
|
38
|
+
"assurance": {
|
|
39
|
+
"plan": ".behavior-ci/feature/plan.json",
|
|
40
|
+
"calibration": ".behavior-ci/feature/calibration.json"
|
|
41
|
+
}
|
|
42
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { createMachine, defineCampaign, defineRequirements, fc } from "statelyai/testing";
|
|
3
|
+
|
|
4
|
+
const requirements = defineRequirements({
|
|
5
|
+
"TOGGLE-001": ["initial.off", "on.turns-on"],
|
|
6
|
+
});
|
|
7
|
+
|
|
8
|
+
export default defineCampaign({
|
|
9
|
+
id: "minimal-toggle",
|
|
10
|
+
intent: "A new toggle starts off; ON turns it on.",
|
|
11
|
+
scope: "API-shape example; replace with the real application adapter.",
|
|
12
|
+
assumptions: ["One ON operation; no persistence or concurrency."],
|
|
13
|
+
evidence: ["The explicit intent above is example-only."],
|
|
14
|
+
modelVersion: "1",
|
|
15
|
+
requirements,
|
|
16
|
+
model: createMachine({
|
|
17
|
+
initial: "off",
|
|
18
|
+
states: {
|
|
19
|
+
off: { on: { ON: () => ({ target: "on" }) } },
|
|
20
|
+
on: {},
|
|
21
|
+
},
|
|
22
|
+
}),
|
|
23
|
+
options: {
|
|
24
|
+
events: {
|
|
25
|
+
ON: [{ case: "valid-on", generate: fc.constant({}) }],
|
|
26
|
+
},
|
|
27
|
+
sut: {
|
|
28
|
+
create() {
|
|
29
|
+
let value = "off";
|
|
30
|
+
return {
|
|
31
|
+
send(event) {
|
|
32
|
+
if (event.type === "ON") value = "on";
|
|
33
|
+
},
|
|
34
|
+
read() {
|
|
35
|
+
return value;
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
states: {
|
|
41
|
+
"*": async (snapshot, session) => {
|
|
42
|
+
const actual = await session.read();
|
|
43
|
+
const check = snapshot.value === "off" ? "initial.off" : "on.turns-on";
|
|
44
|
+
await requirements.assert(check, () => {
|
|
45
|
+
assert.equal(actual, snapshot.value);
|
|
46
|
+
});
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
},
|
|
50
|
+
});
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"runnerInputs": [
|
|
4
|
+
"vitest.config.ts"
|
|
5
|
+
],
|
|
6
|
+
"inputs": [
|
|
7
|
+
{
|
|
8
|
+
"kind": "mapped",
|
|
9
|
+
"category": "user",
|
|
10
|
+
"source": {
|
|
11
|
+
"authority": "cited-requirement",
|
|
12
|
+
"citation": "requirements.md#feature"
|
|
13
|
+
},
|
|
14
|
+
"requirementIds": [
|
|
15
|
+
"feature"
|
|
16
|
+
],
|
|
17
|
+
"checkIds": [
|
|
18
|
+
"feature.check"
|
|
19
|
+
],
|
|
20
|
+
"entityIds": [
|
|
21
|
+
"feature"
|
|
22
|
+
]
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"kind": "gap",
|
|
26
|
+
"category": "host-update",
|
|
27
|
+
"source": {
|
|
28
|
+
"authority": "inference",
|
|
29
|
+
"citation": "Review external state owners in source"
|
|
30
|
+
},
|
|
31
|
+
"reason": "Discover whether host replacement can invalidate current selections"
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"kind": "gap",
|
|
35
|
+
"category": "async-result",
|
|
36
|
+
"source": {
|
|
37
|
+
"authority": "inference",
|
|
38
|
+
"citation": "Review asynchronous completion handlers"
|
|
39
|
+
},
|
|
40
|
+
"reason": "Discover stale and out-of-order completions"
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"kind": "gap",
|
|
44
|
+
"category": "cancellation",
|
|
45
|
+
"source": {
|
|
46
|
+
"authority": "inference",
|
|
47
|
+
"citation": "Review cancellation and abort handlers"
|
|
48
|
+
},
|
|
49
|
+
"reason": "Discover cancellation during pending work"
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"kind": "gap",
|
|
53
|
+
"category": "timer",
|
|
54
|
+
"source": {
|
|
55
|
+
"authority": "inference",
|
|
56
|
+
"citation": "Review timers and deadlines"
|
|
57
|
+
},
|
|
58
|
+
"reason": "Discover expiry and clock-driven transitions"
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"kind": "gap",
|
|
62
|
+
"category": "server-event",
|
|
63
|
+
"source": {
|
|
64
|
+
"authority": "inference",
|
|
65
|
+
"citation": "Review inbound server events"
|
|
66
|
+
},
|
|
67
|
+
"reason": "Discover retries, duplicate deliveries and server updates"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"kind": "gap",
|
|
71
|
+
"category": "lifecycle",
|
|
72
|
+
"source": {
|
|
73
|
+
"authority": "inference",
|
|
74
|
+
"citation": "Review initialization, teardown and restoration"
|
|
75
|
+
},
|
|
76
|
+
"reason": "Discover restart and disposal behavior"
|
|
77
|
+
}
|
|
78
|
+
],
|
|
79
|
+
"coverage": {
|
|
80
|
+
"kind": "selected-checks"
|
|
81
|
+
},
|
|
82
|
+
"dispositions": [],
|
|
83
|
+
"authorityInputs": [
|
|
84
|
+
"requirements.md"
|
|
85
|
+
]
|
|
86
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
|
|
3
|
+
export default function normalize(report, { root }) {
|
|
4
|
+
if (!report || !Array.isArray(report.testResults) || typeof report.success !== "boolean") {
|
|
5
|
+
throw new Error("Expected native Vitest JSON report");
|
|
6
|
+
}
|
|
7
|
+
if (report.numRuntimeErrorTestSuites > 0 || report.unhandledErrors?.length) {
|
|
8
|
+
throw new Error("Vitest reported runtime errors");
|
|
9
|
+
}
|
|
10
|
+
const results = [];
|
|
11
|
+
for (const suite of report.testResults) {
|
|
12
|
+
if (typeof suite.name !== "string" || !Array.isArray(suite.assertionResults)) {
|
|
13
|
+
throw new Error("Malformed Vitest suite");
|
|
14
|
+
}
|
|
15
|
+
if (suite.status === "failed" && suite.assertionResults.length === 0) {
|
|
16
|
+
throw new Error(`Suite failed before tests ran: ${suite.name}`);
|
|
17
|
+
}
|
|
18
|
+
for (const test of suite.assertionResults) {
|
|
19
|
+
if (typeof test.fullName !== "string" || !Array.isArray(test.failureMessages)) {
|
|
20
|
+
throw new Error("Malformed Vitest assertion");
|
|
21
|
+
}
|
|
22
|
+
const outcome = {
|
|
23
|
+
passed: "passed",
|
|
24
|
+
failed: "failed",
|
|
25
|
+
pending: "skipped",
|
|
26
|
+
skipped: "skipped",
|
|
27
|
+
todo: "skipped",
|
|
28
|
+
}[test.status];
|
|
29
|
+
if (!outcome) throw new Error(`Unknown Vitest status: ${test.status}`);
|
|
30
|
+
results.push({
|
|
31
|
+
file: path.relative(root, suite.name).split(path.sep).join("/"),
|
|
32
|
+
fullName: test.fullName,
|
|
33
|
+
outcome,
|
|
34
|
+
detail: test.failureMessages.join("\n").slice(0, 4000) || `Vitest: ${test.status}`,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
const failed = results.filter((r) => r.outcome === "failed").length;
|
|
39
|
+
if (
|
|
40
|
+
report.numFailedTests !== failed ||
|
|
41
|
+
report.numTotalTests !== results.length ||
|
|
42
|
+
report.success === failed > 0
|
|
43
|
+
) {
|
|
44
|
+
throw new Error("Vitest report totals/success disagree with assertions");
|
|
45
|
+
}
|
|
46
|
+
return results;
|
|
47
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: Behavior evidence
|
|
2
|
+
on:
|
|
3
|
+
pull_request:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
permissions:
|
|
6
|
+
contents: read
|
|
7
|
+
jobs:
|
|
8
|
+
behavior:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
steps:
|
|
11
|
+
- uses: actions/checkout@v4
|
|
12
|
+
- uses: actions/setup-node@v4
|
|
13
|
+
with:
|
|
14
|
+
node-version: "24.17.0"
|
|
15
|
+
- uses: pnpm/action-setup@v4
|
|
16
|
+
- run: pnpm install --frozen-lockfile
|
|
17
|
+
- name: Check behavior
|
|
18
|
+
run: pnpm exec statelyai behavior check --config .behavior-ci/feature/config.json --output .scratch/behavior-run
|
|
19
|
+
- name: Job summary
|
|
20
|
+
if: always()
|
|
21
|
+
run: cat .scratch/behavior-run/summary.md >> "$GITHUB_STEP_SUMMARY"
|
|
22
|
+
- uses: actions/upload-artifact@v4
|
|
23
|
+
if: always()
|
|
24
|
+
with:
|
|
25
|
+
name: behavior-${{ github.run_id }}-${{ github.run_attempt }}
|
|
26
|
+
path: .scratch/behavior-run
|
|
27
|
+
include-hidden-files: true
|
|
28
|
+
if-no-files-found: error
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Checked contract
|
|
2
|
+
|
|
3
|
+
The source of truth is `packages/cli/src/behaviorCi.ts` (`behaviorCiConfigSchema`, `behaviorCiResultsSchema`) and the public `@statelyai/mcp/loop` behavior record contract. JSON config is strict; unknown keys fail.
|
|
4
|
+
|
|
5
|
+
New config uses `schemaVersion: 2` and `assurance: {plan,calibration}` plus `name`, `spec`, `model`, `adapter`, `stateRoot`, `baseline`, `command: {executable,args,cwd}`, `inputs`, `checks`, `timeoutMs`. Paths are repository-relative without traversal or symlinks. `cwd: "."` is valid. `baseline` is initially null and subsequently `{machineId,versionId,sha256}` written by adoption. `checks` map stable check `id` to exact repository-relative `file` and native reporter `fullName`. A test maps once. Command argv includes exactly one `{report}` placeholder, usually `--outputFile={report}`. Spawn uses no shell. Timeout range is 100–600000 ms.
|
|
6
|
+
|
|
7
|
+
The `.mjs` module exports a default function `(rawReport, {root,cwd}) => results`, optionally async. Each result is `{file,fullName,outcome,detail}`. Outcomes are `passed`, `failed`, `skipped`, `error`. Detail is nonempty and at most 4000 characters. Keep the adapter self-contained except Node built-ins; its source is part of the binding. Config owns check IDs, never the adapter. The Vitest adapter handles native Vitest JSON. The executable adapter handles the native campaign envelope described in [executable mode](executable.md); this is not a framework registry.
|
|
8
|
+
|
|
9
|
+
`spec.json` contains `requirements` and `retire`. Each requirement is `{id,text,source:{authority,citation},entityIds,checks:[{id,acceptance}]}`. Authority is `cited-requirement`, `observation`, `inference` or `assumption`. Retirement is `{kind:"requirement"|"check",id,reason}`. Core omission retains prior requirements and checks. Record and requirement limits remain core limits, including 100 reports and 200 bound files; this pilot does not compact history automatically.
|
|
10
|
+
|
|
11
|
+
`model.json` is XGraph JSON. Start from the structurally tested [estimated model](../assets/estimated-model.json), then adapt it to the actual feature. Nodes use `parentId` for hierarchy; transitions use `sourceId` and `targetId`, not `source` and `target`. Use the public graph schema for other fields. Generated graph meaning deserves review; structural validity alone is insufficient.
|
|
12
|
+
|
|
13
|
+
Output is summary.md, run.json, the captured config.json, manifest.json, process.json, stdout/stderr, raw report.json and a `store/.statelyai` copy in the existing FileStore format. Run metadata references exact immutable model/record versions. Current verdict covers every retained active check on this run; historical reports and core aggregate gaps are displayed separately. Tests, sources, adapter, model, specification, config, selected manifests/locks and baseline files are hashed before and after execution. Runner metadata includes CLI package version and the SHA-256 of the executing source/bundle; built ignored distribution files need not become tracked application inputs. No transitive completeness claim. Core execution remains reported and review identity unavailable.
|
|
14
|
+
|
|
15
|
+
Inputs must be tracked; local staged/dirty bytes are allowed and labeled. CI requires clean bound inputs and actual HEAD equal to GITHUB_SHA. GitHub env is declared metadata, not authenticated proof. Authenticate the run, commit, logs and downloaded artifact externally before adopting. Artifact checksums detect corruption, not a malicious trusted runner forging an entire artifact.
|
|
16
|
+
|
|
17
|
+
Adoption reads data only; it never imports the downloaded adapter or runs artifact commands. Exact checked parent, record/model digests, checksums, recorded Git bytes (or unchanged local dirty bytes) and immutable file conflicts must validate. Green/red evidence can advance the baseline; setup/error evidence cannot. Repeated adoption of the same next ref is idempotent. CI never changes checked baseline.
|
|
18
|
+
|
|
19
|
+
Setup maps every selected native case, including existing public tests, using exact `(file, fullName)` pairs. A failed selected case outside those mappings produces ERROR, not an adoptable FAIL. Preserve that artifact, complete the mappings without weakening obligations, and rerun before application repair. A mapped native failure is a reported test failure; its detail may describe an assertion, runtime exception or timeout. The CLI does not establish a causal diagnosis from the failure status alone.
|
|
20
|
+
|
|
21
|
+
Schema 2 requires a discovery plan with `schemaVersion: 1`, `runnerInputs`, `authorityInputs`, `inputs`, `coverage` and `dispositions`. List checked authoritative intent documents in nonempty `authorityInputs`; changed intent requires recalibration even when structured spec is unchanged. List command producer/configuration modules in `runnerInputs`; mapped test files and adapter are automatic. All seven input categories must occur: `user`, `host-update`, `async-result`, `cancellation`, `timer`, `server-event`, `lifecycle`. Each entry has `source: {authority,citation}`. A `kind: "mapped"` entry adds nonempty `requirementIds`, `checkIds` and `entityIds`; referenced IDs must exist and selected checks must belong to the listed requirements. A `kind: "gap"` or `"not-applicable"` entry adds a reason. Both coverage modes retain disclosed discovery gaps. Complete bounded path retrieval does not establish complete feature input coverage. Coverage is `{kind:"selected-checks"}` or `{kind:"bounded-paths",artifact,caseMappings:[{pathId,checkIds}]}`. Complete bounded-path claims require complete bound generation evidence and a native mapping for every path; generated paths are not themselves executed cases.
|
|
22
|
+
|
|
23
|
+
`behavior calibrate --config <config> --output <fresh-directory>` runs the same deterministic native process as `check`, then saves `calibration.json`. The receipt includes original manifest, snapshots of original selected bytes, actual process outcome, raw native report, normalized results, contract digest and explicit implementation-defect dispositions. Checksums detect corruption; receipts do not attest that a trusted producer told the truth. `adopt` verifies the receipt against the current contract and local adapter, and installs it at the checked calibration path. It never executes downloaded code or artifact commands. It can import the current trusted repository adapter to verify native normalization. A failed calibration check requires a plan disposition `{checkId,kind:"implementation-defect",reason,source}` before it can be adopted. Skipped/missing/error calibration outcomes cannot establish a baseline. Red history remains red; a disposition never changes a failure to a pass.
|
|
24
|
+
|
|
25
|
+
`check` requires a valid receipt for schema 2. Model, spec, plan, authoritative intent files, mappings, native test producer files, command, adapter or runner identity changes invalidate it. Ordinary implementation edits do not invalidate the contract. The receipt and run store the original checkout root for native report normalization when CI or adoption uses another checkout path. Receipt snapshots keep original revision evidence inspectable even in shallow CI checkouts. Run output includes the used receipt. Schema 1 check/adopt remains compatible but reports assurance unavailable. Schema 2 rejects every extra selected native `(file,fullName)` pair, including passing cases.
|
|
26
|
+
|
|
27
|
+
A bounded-path archive is `{schemaVersion:1,source:{graph,kind:"simple"|"shortest"},pages:[...actual generate_graph_paths structuredContent]}`. Collect every continuation until null. Validation requires identical model, contiguous offsets, canonical cursor chain, matching generation hash, valid modeled states/transitions/events, complete retrieval, reported complete generation and exact path-to-native-check mappings. This is complete retrieval of the declared bounded generator result. Generator completeness is reported, not independently recomputed or attested. Checksums cannot authenticate a malicious trusted producer that rewrites an entire archive. The PR summary states this limit and separately counts generated paths and native passed/failed cases. The runner executable/version, Node version and CLI code digest are contract-bound; platform and architecture remain original/current evidence rather than invalidating the contract when CI uses another OS.
|
|
28
|
+
|
|
29
|
+
Calibration destinations must be isolated from every selected file, selected directory, auto-selected manifest/lockfile, stateRoot and run output. Parent/child overlaps and dot aliases are refused before native execution or receipt installation.
|
|
30
|
+
|
|
31
|
+
The native child receives `STATELY_BEHAVIOR_OUTPUT`, always overridden to its unique absolute run directory. Producers may write diagnostic files below its `traces/` directory on both success and failure. Keep assertion semantics independent of this path. The CLI hashes these files with the report and validates their complete inventory during adoption; extra, removed or changed trace files invalidate the artifact. Limits and symlink rules also apply to traces. Producers that ignore the variable are unchanged.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Executable campaign mode
|
|
2
|
+
|
|
3
|
+
Use for authored model-based tests of real functions, stores, services or UI. Keep an XGraph estimate in `model.json` for the existing behavior record. The executable XState model is a separate trusted `.mjs` program. Reconcile both against cited requirements; neither is automatically ground truth.
|
|
4
|
+
|
|
5
|
+
## Author the campaign
|
|
6
|
+
|
|
7
|
+
Install a pinned `statelyai` dependency in the application project, with compatible Node and package manager pins and a checked lockfile. An installed global CLI does not supply application imports. When validating an unpublished candidate, install the packed candidate and its pinned dependencies, record its SHA-256 and installation provenance, and call the result packed-candidate proof. Do not claim registry availability or publication from a local pack/install.
|
|
8
|
+
|
|
9
|
+
Use the runnable [minimal campaign](../assets/minimal.campaign.mjs) for the native callback and event descriptor syntax. Its in-memory toggle demonstrates API shape only; replace it with the application model and real adapter. Write `defineCampaign({...})`, a real model, native event descriptors and fast-check generators. Import `createMachine`, `defineCampaign`, `defineRequirements` and `fc` from `statelyai/testing`. A descriptor such as `{case: 'invalid-empty', generate: fc.constant({value: ''})}` belongs under its actual model event in `options.events`; add `when` or native `pick` only where source/spec establishes state-dependent applicability. Use application source and authoritative requirements to determine expected states and transitions. Include boundary and invalid payloads, repeated/ignored operations, cancellation, retries, timer/lifecycle interactions, and cross-owner sequences that apply to the feature. Record omissions and assumptions in the discovery plan.
|
|
10
|
+
|
|
11
|
+
Create a `defineRequirements` registry with stable requirement IDs mapped to nonempty arrays of globally unique assertion IDs. Pass that registry as `requirements` to `defineCampaign`. A requirement needing Enter and Blur uses distinct assertion IDs; one must never satisfy the other. Put runtime assertions inside awaited native `invariant`, state callbacks or `sut.check` callbacks:
|
|
12
|
+
|
|
13
|
+
```js
|
|
14
|
+
const requirements = defineRequirements({
|
|
15
|
+
'FORM-001': ['validation.invalid-email'],
|
|
16
|
+
});
|
|
17
|
+
// Within a real assertion callback, after reading the application:
|
|
18
|
+
await requirements.assert('validation.invalid-email', () => {
|
|
19
|
+
assert.equal(observed.validation, 'Enter a valid email');
|
|
20
|
+
});
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
The body must complete with `undefined` on success and throw on failure. Boolean predicates are not assertion bodies. Await or return `requirements.assert`, including nested asynchronous work. Only consumed assertions whose bodies complete within the active native callback count. Ignored, pending and late assertions make the run inconclusive. Assertions outside these callbacks, setup-only assertions, labels and phase PASS do not establish requirement execution. The tracker does not infer promise ownership: a detached assertion that finishes before the callback returns counts its actual result, and a recorded failure still fails the phase even if the detached task catches it. Counts include exploration and shrinking invocations; they are neither distinct cases nor attribution to the final shrunk witness. A saved replay supplies fresh evidence for only that concrete sequence and does not close unrelated full-campaign gaps.
|
|
24
|
+
|
|
25
|
+
Use fresh isolated SUT sessions for every path, trial, shrink and replay. `sut.send` drives the real app; `sut.read` observes it. Keep behavioral assertions in `sut.check`, `invariant`, state callbacks or model projections. Browser locator, navigation, setup and service transport errors remain infrastructure errors. Release contexts, services and resources in `dispose` and campaign `cleanup`. Do not add an Action DSL, replace the native engine or run an LLM in CI.
|
|
26
|
+
|
|
27
|
+
## Bridge into existing behavior CI
|
|
28
|
+
|
|
29
|
+
Copy [campaign-producer.mjs](../assets/campaign-producer.mjs) and [campaign-adapter.mjs](../assets/campaign-adapter.mjs) together into the checked feature directory. The producer imports the local installed package, calls `runCampaignFile`, validates the native report, then writes the `{file, report}` envelope at `--report {report}`. Its `file` is the repository-relative campaign path. Run from the repository root. An optional `--budget <budget.json>` supplies native budget overrides; check and bind that file. The producer writes the native report, native exit code and portable replays below `STATELY_BEHAVIOR_OUTPUT/traces`. Startup, malformed data or missing bindings are setup errors.
|
|
30
|
+
|
|
31
|
+
Adapt [executable-config.json](../assets/executable-config.json). Each declared assertion has exactly one selected `(file, fullName)` pair. For requirement `FORM-001` and assertion `validation.invalid-email`, the stable fullName is `campaign.assertion ["FORM-001","validation.invalid-email"]`. It is generated with `campaign.assertion ${JSON.stringify([requirementId, checkId])}`. Every campaign also has `campaign.execution`, mapped to an ancillary observation check, with its own requirement in `spec.json`. The config check IDs are stable specification IDs; the adapter never assigns them. Generate the complete inventory by importing `assertionFullName` and `executionFullName` from the copied adapter and reading the registry declarations. Then compare with actual adapter output. Selected schema-2 inventory must match exactly, including the execution guard. Missing declarations or phase binding metadata is invalid evidence and cannot pass.
|
|
32
|
+
|
|
33
|
+
The adapter folds actual assertion counts across paths and property. Any failed invocation is `failed`; positive passed count with no failure is `passed`; zero is `skipped`. Every phase must contain the exact declaration/check inventory, including zeros. The separate execution guard preserves projection, invariant, temporal and other unassigned native discrepancies as `failed`; it never assigns them to a requirement arbitrarily. Coverage-only `inconclusive` phases carry validated `requirementGaps` and yield a skipped guard. Other infrastructure or incomplete exploration yields `error`. A phase PASS cannot turn zero assertions into a requirement PASS.
|
|
34
|
+
|
|
35
|
+
The producer exits 1 when normalization contains `failed` or `error`, otherwise 0, including skipped-only results. This follows the existing behavior runner's process consistency contract. Required skips still make `behavior check` FAIL. Native exit 0/1/2 and raw status remain in traces. Producer exit 0 alone does not mean assurance PASS. Calibration cannot be adopted with skipped/error checks; add meaningful missing execution rather than weakening obligations. True failures require the existing cited implementation-defect disposition before red calibration adoption.
|
|
36
|
+
|
|
37
|
+
Set `plan.runnerInputs` to the campaign, every campaign import and adapter implementation, producer, native budget/config, relevant package manifests and lockfiles, and actual workflow. Include the application entry points and feature sources in config inputs. For packed candidates, bind package manifests, lockfile integrity entries and the compact SHA-256 provenance manifest. Do not snapshot large binary tarballs in runner inputs; their base64 bytes can exceed the 16 MiB receipt limit. CI must install with the frozen lockfile so altered tarballs fail integrity validation. Record authoritative intent files in `plan.authorityInputs`. Bind all producer/import dependencies explicitly; the native entry SHA hashes only the entry module. Preserve the existing 200-file limit and output/input separation. Use the shared workflow template with pinned installation and real clean-runner dependencies, including Playwright browsers only when the adapter needs them.
|
|
38
|
+
|
|
39
|
+
## Maintenance handoff
|
|
40
|
+
|
|
41
|
+
Check in a short feature maintenance document explaining the cited intent, executable and estimated model locations, real SUT adapter, event/generator inventory, requirement/assertion mapping, bounded coverage and gaps, commands, toolchain pins, output/replay locations and evolution procedure. Include it in authority inputs only for authoritative intent; runner maintenance instructions alone are not user requirements. Preserve requirement/assertion IDs for unchanged obligations. Review and explicitly retire removed obligations through `spec.retire`; omission never retires them. A changed campaign, import, producer, adapter, plan, manifest, lockfile, workflow, mapping or native runner requires explicit recalibration and inspected adoption. Preserve red history before the next evolution. Verify green, deliberate real assertion red, restored green, unexecuted assertion, missing binding metadata and concrete replay behavior locally. Remote green/red/fix/evolution/handoff claims require actual remote logs and downloaded artifacts against their exact checked parent.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "@statelyai/mcp/testing";
|
package/dist/testing.mjs
ADDED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "statelyai",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.0",
|
|
4
4
|
"description": "Command-line tools for Stately",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"bin": {
|
|
@@ -17,23 +17,27 @@
|
|
|
17
17
|
".": {
|
|
18
18
|
"types": "./dist/index.d.mts",
|
|
19
19
|
"import": "./dist/index.mjs"
|
|
20
|
-
}
|
|
20
|
+
},
|
|
21
|
+
"./testing": {
|
|
22
|
+
"types": "./dist/testing.d.mts",
|
|
23
|
+
"import": "./dist/testing.mjs"
|
|
24
|
+
},
|
|
25
|
+
"./skills/setup-behavior-ci/SKILL.md": "./dist/skills/setup-behavior-ci/SKILL.md"
|
|
21
26
|
},
|
|
22
27
|
"dependencies": {
|
|
23
28
|
"@oclif/core": "^4.10.3",
|
|
24
29
|
"typescript": "^5.9.3",
|
|
25
30
|
"ws": "^8.21.0",
|
|
26
31
|
"zod": "^4.4.1",
|
|
27
|
-
"@statelyai/mcp": "0.
|
|
28
|
-
"@statelyai/sdk": "0.
|
|
29
|
-
"@statelyai/model-testing": "0.2.0"
|
|
32
|
+
"@statelyai/mcp": "0.21.0",
|
|
33
|
+
"@statelyai/sdk": "0.40.0"
|
|
30
34
|
},
|
|
31
35
|
"devDependencies": {
|
|
32
36
|
"@types/ws": "^8.18.1",
|
|
33
37
|
"tsdown": "0.21.0-beta.2",
|
|
34
38
|
"vitest": "^3.2.4",
|
|
35
|
-
"@statelyai/
|
|
36
|
-
"@statelyai/
|
|
39
|
+
"@statelyai/editor-sync": "0.2.7",
|
|
40
|
+
"@statelyai/api-contract": "0.10.0"
|
|
37
41
|
},
|
|
38
42
|
"oclif": {
|
|
39
43
|
"bin": "statelyai",
|
|
@@ -45,7 +49,7 @@
|
|
|
45
49
|
"topicSeparator": " "
|
|
46
50
|
},
|
|
47
51
|
"scripts": {
|
|
48
|
-
"build": "tsdown",
|
|
52
|
+
"build": "tsdown && node scripts/copy-testing-skill.mjs",
|
|
49
53
|
"test": "vitest run"
|
|
50
54
|
}
|
|
51
55
|
}
|