trxa-tool 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/replay-trxa-testplan/SKILL.md +627 -0
- package/LICENSE +120 -0
- package/MANUAL.md +548 -0
- package/README.md +67 -3
- package/STUDIO.md +319 -0
- package/dist/analyze.d.ts +58 -0
- package/dist/analyze.js +122 -0
- package/dist/bpm.d.ts +40 -0
- package/dist/bpm.js +60 -0
- package/dist/build-test-script.d.ts +24 -0
- package/dist/build-test-script.js +40 -0
- package/dist/csv.d.ts +1 -0
- package/dist/csv.js +11 -0
- package/dist/extract.d.ts +14 -0
- package/dist/extract.js +34 -0
- package/dist/fixture-schema.d.ts +12 -0
- package/dist/fixture-schema.js +28 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +814 -0
- package/dist/library-archive.d.ts +37 -0
- package/dist/library-archive.js +106 -0
- package/dist/mapping.d.ts +25 -0
- package/dist/mapping.js +390 -0
- package/dist/normalize.d.ts +21 -0
- package/dist/normalize.js +43 -0
- package/dist/parse.d.ts +3 -0
- package/dist/parse.js +233 -0
- package/dist/plan-runner.d.ts +27 -0
- package/dist/plan-runner.js +36 -0
- package/dist/plan-schema.d.ts +27 -0
- package/dist/plan-schema.js +45 -0
- package/dist/planrun-schema.d.ts +45 -0
- package/dist/planrun-schema.js +87 -0
- package/dist/planrun.d.ts +20 -0
- package/dist/planrun.js +46 -0
- package/dist/report-schema.d.ts +83 -0
- package/dist/report-schema.js +89 -0
- package/dist/report.d.ts +42 -0
- package/dist/report.js +70 -0
- package/dist/schema.d.ts +96 -0
- package/dist/schema.js +10 -0
- package/dist/scrub.d.ts +44 -0
- package/dist/scrub.js +166 -0
- package/dist/suggest-inputs.d.ts +19 -0
- package/dist/suggest-inputs.js +92 -0
- package/dist/suite-runner.d.ts +59 -0
- package/dist/suite-runner.js +198 -0
- package/dist/suite-schema.d.ts +59 -0
- package/dist/suite-schema.js +95 -0
- package/dist/testcase-schema.d.ts +102 -0
- package/dist/testcase-schema.js +132 -0
- package/dist/token-usage.d.ts +38 -0
- package/dist/token-usage.js +95 -0
- package/package.json +58 -6
- package/studio/client-dist/assets/index-CgWhGkEn.js +24 -0
- package/studio/client-dist/assets/index-CwAUMVCL.css +1 -0
- package/studio/client-dist/axtr-icon.svg +5 -0
- package/studio/client-dist/favicon.svg +5 -0
- package/studio/client-dist/index.html +14 -0
- package/studio/server/dist/index.js +1938 -0
package/dist/parse.js
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { XMLParser } from "fast-xml-parser";
|
|
2
|
+
/**
|
|
3
|
+
* Tags that must always parse to an array, even when exactly one occurs in the XML -- fast-xml-
|
|
4
|
+
* parser otherwise collapses a single child into a bare object, and every downstream `.map()`
|
|
5
|
+
* over these (`collectLeaves`, `parseCommandArgument`, `parseFormContexts`) would throw on a
|
|
6
|
+
* recording with only one child/argument. Adding a new element type that can repeat (Scope
|
|
7
|
+
* children, command arguments, etc.) means adding its tag name here too -- there's no other
|
|
8
|
+
* signal that forces array mode.
|
|
9
|
+
* `KeyValueOfstringFormContextyaScvdpM` is the mangled tag fast-xml-parser sees for a WCF
|
|
10
|
+
* `Dictionary<string, FormContext>` entry (`DataContractSerializer`'s generic-type name-mangling
|
|
11
|
+
* convention) -- not verified against other D365FO versions, but it's the .NET serializer's own
|
|
12
|
+
* naming scheme, not something specific to one recording, so unlikely to change independently.
|
|
13
|
+
*/
|
|
14
|
+
const ARRAY_TAGS = new Set([
|
|
15
|
+
"Node",
|
|
16
|
+
"Annotation",
|
|
17
|
+
"CommandArgument",
|
|
18
|
+
"string",
|
|
19
|
+
"KeyValueOfstringFormContextyaScvdpM",
|
|
20
|
+
]);
|
|
21
|
+
function makeParser() {
|
|
22
|
+
return new XMLParser({
|
|
23
|
+
ignoreAttributes: false,
|
|
24
|
+
attributeNamePrefix: "@_",
|
|
25
|
+
removeNSPrefix: true,
|
|
26
|
+
parseTagValue: false,
|
|
27
|
+
parseAttributeValue: false,
|
|
28
|
+
trimValues: true,
|
|
29
|
+
isArray: (tagName) => ARRAY_TAGS.has(tagName),
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
/** True when the parsed element is `{ "@_nil": "true" }` (xsi:nil marker) or otherwise empty. */
|
|
33
|
+
function isNil(v) {
|
|
34
|
+
if (v === undefined || v === null)
|
|
35
|
+
return true;
|
|
36
|
+
if (typeof v === "object" && !Array.isArray(v)) {
|
|
37
|
+
const o = v;
|
|
38
|
+
return o["@_nil"] === "true";
|
|
39
|
+
}
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
/** Reads a simple text-content element, honoring xsi:nil. Preserves strings verbatim (no numeric coercion). */
|
|
43
|
+
function text(v) {
|
|
44
|
+
if (isNil(v))
|
|
45
|
+
return null;
|
|
46
|
+
if (typeof v === "string")
|
|
47
|
+
return v;
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
function textReq(v, field) {
|
|
51
|
+
const t = text(v);
|
|
52
|
+
if (t === null)
|
|
53
|
+
throw new Error(`Expected required text field "${field}" to be present`);
|
|
54
|
+
return t;
|
|
55
|
+
}
|
|
56
|
+
function num(v) {
|
|
57
|
+
const t = text(v);
|
|
58
|
+
if (t === null || t === "")
|
|
59
|
+
return null;
|
|
60
|
+
const n = Number(t);
|
|
61
|
+
return Number.isNaN(n) ? null : n;
|
|
62
|
+
}
|
|
63
|
+
function numReq(v, field) {
|
|
64
|
+
const n = num(v);
|
|
65
|
+
if (n === null)
|
|
66
|
+
throw new Error(`Expected required numeric field "${field}" to be present`);
|
|
67
|
+
return n;
|
|
68
|
+
}
|
|
69
|
+
function bool(v) {
|
|
70
|
+
return text(v) === "true";
|
|
71
|
+
}
|
|
72
|
+
/** Unwraps a collection element that is either nil, a single child, or an array of children. */
|
|
73
|
+
function collection(v, childTag) {
|
|
74
|
+
if (isNil(v))
|
|
75
|
+
return [];
|
|
76
|
+
if (typeof v === "string")
|
|
77
|
+
return []; // self-closed with no attrs -> parsed as ""
|
|
78
|
+
const obj = v;
|
|
79
|
+
const child = obj[childTag];
|
|
80
|
+
if (child === undefined)
|
|
81
|
+
return [];
|
|
82
|
+
return Array.isArray(child) ? child : [child];
|
|
83
|
+
}
|
|
84
|
+
function parseCommandArgument(raw) {
|
|
85
|
+
return {
|
|
86
|
+
IsReference: bool(raw.IsReference),
|
|
87
|
+
Value: textReq(raw.Value, "CommandArgument.Value"),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
function parseFormContext(raw) {
|
|
91
|
+
return {
|
|
92
|
+
FormID: textReq(raw.FormID, "FormContext.FormID"),
|
|
93
|
+
FormName: textReq(raw.FormName, "FormContext.FormName"),
|
|
94
|
+
RecordingName: textReq(raw.RecordingName, "FormContext.RecordingName"),
|
|
95
|
+
HasMetadata: bool(raw.HasMetadata),
|
|
96
|
+
Sequence: numReq(raw.Sequence, "FormContext.Sequence"),
|
|
97
|
+
Annotations: raw.Annotations,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
function parseFormContexts(raw) {
|
|
101
|
+
const entries = collection(raw, "KeyValueOfstringFormContextyaScvdpM");
|
|
102
|
+
const result = {};
|
|
103
|
+
for (const entry of entries) {
|
|
104
|
+
const key = textReq(entry.Key, "FormContexts.Key");
|
|
105
|
+
result[key] = parseFormContext(entry.Value);
|
|
106
|
+
}
|
|
107
|
+
return result;
|
|
108
|
+
}
|
|
109
|
+
function nodeBaseFields(raw) {
|
|
110
|
+
return {
|
|
111
|
+
Id: textReq(raw.Id, "Node.Id"),
|
|
112
|
+
Description: text(raw.Description) ?? "",
|
|
113
|
+
CustomDescription: text(raw.CustomDescription),
|
|
114
|
+
ParentSequence: numReq(raw.ParentSequence, "Node.ParentSequence"),
|
|
115
|
+
Sequence: numReq(raw.Sequence, "Node.Sequence"),
|
|
116
|
+
ScreenshotUri: text(raw.ScreenshotUri),
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
function userActionBaseFields(raw) {
|
|
120
|
+
return {
|
|
121
|
+
...nodeBaseFields(raw),
|
|
122
|
+
GlobalId: textReq(raw.GlobalId, "UserAction.GlobalId"),
|
|
123
|
+
SystemGenerated: bool(raw.SystemGenerated),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
function parseScope(raw) {
|
|
127
|
+
return {
|
|
128
|
+
kind: "Scope",
|
|
129
|
+
...nodeBaseFields(raw),
|
|
130
|
+
Name: text(raw.Name) ?? "",
|
|
131
|
+
IsForm: bool(raw.IsForm),
|
|
132
|
+
IsStepGroup: bool(raw.IsStepGroup),
|
|
133
|
+
ScopeType: text(raw.ScopeType) ?? "",
|
|
134
|
+
Children: collection(raw.Children, "Node").map((n) => parseNode(n)),
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
function parseMenuItemUserAction(raw) {
|
|
138
|
+
return {
|
|
139
|
+
kind: "MenuItemUserAction",
|
|
140
|
+
...userActionBaseFields(raw),
|
|
141
|
+
MenuItemName: textReq(raw.MenuItemName, "MenuItemUserAction.MenuItemName"),
|
|
142
|
+
MenuItemType: text(raw.MenuItemType) ?? "",
|
|
143
|
+
NavigationPath: collection(raw.NavigationPath, "string").map((s) => textReq(s, "NavigationPath.string")),
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
function parseCommandUserAction(raw) {
|
|
147
|
+
return {
|
|
148
|
+
kind: "CommandUserAction",
|
|
149
|
+
...userActionBaseFields(raw),
|
|
150
|
+
FormId: textReq(raw.FormId, "CommandUserAction.FormId"),
|
|
151
|
+
ControlName: text(raw.ControlName) ?? "",
|
|
152
|
+
ControlType: text(raw.ControlType) ?? "",
|
|
153
|
+
ControlLabel: text(raw.ControlLabel) ?? "",
|
|
154
|
+
ControlLabelId: text(raw.ControlLabelId) ?? "",
|
|
155
|
+
CommandName: textReq(raw.CommandName, "CommandUserAction.CommandName"),
|
|
156
|
+
Arguments: collection(raw.Arguments, "CommandArgument").map((a) => parseCommandArgument(a)),
|
|
157
|
+
ListContext: text(raw.ListContext),
|
|
158
|
+
RowIndex: num(raw.RowIndex),
|
|
159
|
+
ReturnType: text(raw.ReturnType) ?? "",
|
|
160
|
+
ReturnValue: text(raw.ReturnValue),
|
|
161
|
+
TaskId: text(raw.TaskId),
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
function parsePropertyUserAction(raw) {
|
|
165
|
+
return {
|
|
166
|
+
kind: "PropertyUserAction",
|
|
167
|
+
...userActionBaseFields(raw),
|
|
168
|
+
FormId: textReq(raw.FormId, "PropertyUserAction.FormId"),
|
|
169
|
+
ControlName: text(raw.ControlName) ?? "",
|
|
170
|
+
ControlType: text(raw.ControlType) ?? "",
|
|
171
|
+
ControlLabel: text(raw.ControlLabel) ?? "",
|
|
172
|
+
ControlLabelId: text(raw.ControlLabelId) ?? "",
|
|
173
|
+
BoundDataSource: text(raw.BoundDataSource) ?? "",
|
|
174
|
+
BoundProperty: text(raw.BoundProperty) ?? "",
|
|
175
|
+
PropertyName: text(raw.PropertyName) ?? "",
|
|
176
|
+
UserActionType: text(raw.UserActionType) ?? "",
|
|
177
|
+
Value: text(raw.Value) ?? "",
|
|
178
|
+
ValueLabel: text(raw.ValueLabel),
|
|
179
|
+
IsReference: bool(raw.IsReference),
|
|
180
|
+
ListContext: text(raw.ListContext),
|
|
181
|
+
RowIndex: num(raw.RowIndex),
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
function parseUnknownUserAction(raw, xsiType) {
|
|
185
|
+
return {
|
|
186
|
+
kind: "Unknown",
|
|
187
|
+
...userActionBaseFields(raw),
|
|
188
|
+
xsiType,
|
|
189
|
+
raw,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
function parseNode(raw) {
|
|
193
|
+
const xsiType = raw["@_type"] ?? "Scope";
|
|
194
|
+
switch (xsiType) {
|
|
195
|
+
case "Scope":
|
|
196
|
+
return parseScope(raw);
|
|
197
|
+
case "MenuItemUserAction":
|
|
198
|
+
return parseMenuItemUserAction(raw);
|
|
199
|
+
case "CommandUserAction":
|
|
200
|
+
return parseCommandUserAction(raw);
|
|
201
|
+
case "PropertyUserAction":
|
|
202
|
+
return parsePropertyUserAction(raw);
|
|
203
|
+
default:
|
|
204
|
+
return parseUnknownUserAction(raw, xsiType);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
/** Parses a `Recording.xml` document into a typed Recording tree. */
|
|
208
|
+
export function parseRecordingXml(xml) {
|
|
209
|
+
const parser = makeParser();
|
|
210
|
+
const doc = parser.parse(xml);
|
|
211
|
+
const root = doc.Recording;
|
|
212
|
+
if (!root) {
|
|
213
|
+
throw new Error("Document does not contain a root <Recording> element");
|
|
214
|
+
}
|
|
215
|
+
const rootScopeRaw = root.RootScope;
|
|
216
|
+
const rootScope = {
|
|
217
|
+
kind: "Scope",
|
|
218
|
+
...nodeBaseFields(rootScopeRaw),
|
|
219
|
+
Name: text(root.Name) ?? "",
|
|
220
|
+
IsForm: false,
|
|
221
|
+
IsStepGroup: false,
|
|
222
|
+
ScopeType: "Root",
|
|
223
|
+
Children: collection(rootScopeRaw.Children, "Node").map((n) => parseNode(n)),
|
|
224
|
+
};
|
|
225
|
+
return {
|
|
226
|
+
CanonicalId: textReq(root.CanonicalId, "Recording.CanonicalId"),
|
|
227
|
+
CurrentSequence: numReq(root.CurrentSequence, "Recording.CurrentSequence"),
|
|
228
|
+
Description: text(root.Description) ?? "",
|
|
229
|
+
Name: textReq(root.Name, "Recording.Name"),
|
|
230
|
+
FormContexts: parseFormContexts(root.FormContexts),
|
|
231
|
+
RootScope: rootScope,
|
|
232
|
+
};
|
|
233
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { PlanRun } from "./planrun-schema.js";
|
|
2
|
+
import { type TestRun } from "./report-schema.js";
|
|
3
|
+
export type SuiteRunStatus = "missing" | "incomplete" | "passed" | "failed";
|
|
4
|
+
export interface SuiteRunSummary {
|
|
5
|
+
suiteId: string;
|
|
6
|
+
/** Snapshotted from `PlanRun.suiteNames`; falls back to `suiteId` for a PlanRun persisted before
|
|
7
|
+
* that field existed. */
|
|
8
|
+
suiteName: string;
|
|
9
|
+
status: SuiteRunStatus;
|
|
10
|
+
testRun: TestRun | null;
|
|
11
|
+
}
|
|
12
|
+
export type PlanOverallStatus = "all-passed" | "has-failures" | "incomplete" | "missing-runs";
|
|
13
|
+
export interface PlanStatus {
|
|
14
|
+
planId: string;
|
|
15
|
+
runId: string;
|
|
16
|
+
suites: SuiteRunSummary[];
|
|
17
|
+
overall: PlanOverallStatus;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Pure aggregation, no filesystem access: given a PlanRun and the TestRun already loaded for each
|
|
21
|
+
* of its snapshotted suites (or null if that suite's run file doesn't exist for this runId),
|
|
22
|
+
* computes a rollup. Deliberately reads `run.suiteIds` -- the suite list as it existed when the
|
|
23
|
+
* run started -- rather than the live TestPlan's `suites`, so a run's status can't retroactively
|
|
24
|
+
* change just because the plan later gained or lost a suite (confirmed live: adding a suite to
|
|
25
|
+
* Accounts Payable flipped an already-passed run to "Missing runs" before this fix).
|
|
26
|
+
*/
|
|
27
|
+
export declare function aggregatePlanStatus(run: PlanRun, testRuns: Map<string, TestRun | null>): PlanStatus;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { TERMINAL_STATUSES } from "./report-schema.js";
|
|
2
|
+
function summarizeSuiteRun(suiteId, suiteName, testRun) {
|
|
3
|
+
if (!testRun)
|
|
4
|
+
return { suiteId, suiteName, status: "missing", testRun: null };
|
|
5
|
+
const statuses = Object.values(testRun.results).map((r) => r.status);
|
|
6
|
+
const allTerminal = statuses.every((s) => TERMINAL_STATUSES.has(s));
|
|
7
|
+
if (!allTerminal)
|
|
8
|
+
return { suiteId, suiteName, status: "incomplete", testRun };
|
|
9
|
+
const hasFailure = statuses.some((s) => s === "failed" || s === "blocked");
|
|
10
|
+
return { suiteId, suiteName, status: hasFailure ? "failed" : "passed", testRun };
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Pure aggregation, no filesystem access: given a PlanRun and the TestRun already loaded for each
|
|
14
|
+
* of its snapshotted suites (or null if that suite's run file doesn't exist for this runId),
|
|
15
|
+
* computes a rollup. Deliberately reads `run.suiteIds` -- the suite list as it existed when the
|
|
16
|
+
* run started -- rather than the live TestPlan's `suites`, so a run's status can't retroactively
|
|
17
|
+
* change just because the plan later gained or lost a suite (confirmed live: adding a suite to
|
|
18
|
+
* Accounts Payable flipped an already-passed run to "Missing runs" before this fix).
|
|
19
|
+
*/
|
|
20
|
+
export function aggregatePlanStatus(run, testRuns) {
|
|
21
|
+
const suites = run.suiteIds.map((suiteId) => summarizeSuiteRun(suiteId, run.suiteNames[suiteId] ?? suiteId, testRuns.get(suiteId) ?? null));
|
|
22
|
+
let overall;
|
|
23
|
+
if (suites.some((s) => s.status === "failed")) {
|
|
24
|
+
overall = "has-failures";
|
|
25
|
+
}
|
|
26
|
+
else if (suites.some((s) => s.status === "incomplete")) {
|
|
27
|
+
overall = "incomplete";
|
|
28
|
+
}
|
|
29
|
+
else if (suites.some((s) => s.status === "missing")) {
|
|
30
|
+
overall = "missing-runs";
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
overall = "all-passed";
|
|
34
|
+
}
|
|
35
|
+
return { planId: run.planId, runId: run.id, suites, overall };
|
|
36
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A TestPlan groups multiple independent Test Suites for combined scheduling/reporting -- e.g.
|
|
3
|
+
* a whole "Accounts Receivable" testing cycle spanning several suites. Deliberately flat: no
|
|
4
|
+
* dependency graph and no input bindings across the suite boundary. That's what a TestSuite's
|
|
5
|
+
* `dependsOn`/`inputBindings` already does *within* one suite; if two suites genuinely need to
|
|
6
|
+
* share data, they were never really two suites -- merge their test cases into one suite
|
|
7
|
+
* instead. A TestPlan only composes already-independently-runnable suites for reporting.
|
|
8
|
+
*
|
|
9
|
+
* This matches TestRail's actual "Test Plan" concept (a plan bundles multiple suites/runs for
|
|
10
|
+
* one release) rather than suite-of-suites nesting.
|
|
11
|
+
*/
|
|
12
|
+
export interface TestPlanEntry {
|
|
13
|
+
suiteId: string;
|
|
14
|
+
/** Path to the suite's JSON file, relative to this plan file. */
|
|
15
|
+
suitePath: string;
|
|
16
|
+
}
|
|
17
|
+
export interface TestPlan {
|
|
18
|
+
id: string;
|
|
19
|
+
name: string;
|
|
20
|
+
description: string;
|
|
21
|
+
/** Array order is the suggested run order; no dependsOn, no bindings across suites. */
|
|
22
|
+
suites: TestPlanEntry[];
|
|
23
|
+
}
|
|
24
|
+
/** Validates a parsed JSON object against the TestPlan shape. Does not check that referenced
|
|
25
|
+
* suite files actually exist or are themselves valid -- that's the `validate-plan` CLI
|
|
26
|
+
* command's job, since it needs filesystem access this pure function doesn't have. */
|
|
27
|
+
export declare function validateTestPlan(value: unknown): TestPlan;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A TestPlan groups multiple independent Test Suites for combined scheduling/reporting -- e.g.
|
|
3
|
+
* a whole "Accounts Receivable" testing cycle spanning several suites. Deliberately flat: no
|
|
4
|
+
* dependency graph and no input bindings across the suite boundary. That's what a TestSuite's
|
|
5
|
+
* `dependsOn`/`inputBindings` already does *within* one suite; if two suites genuinely need to
|
|
6
|
+
* share data, they were never really two suites -- merge their test cases into one suite
|
|
7
|
+
* instead. A TestPlan only composes already-independently-runnable suites for reporting.
|
|
8
|
+
*
|
|
9
|
+
* This matches TestRail's actual "Test Plan" concept (a plan bundles multiple suites/runs for
|
|
10
|
+
* one release) rather than suite-of-suites nesting.
|
|
11
|
+
*/
|
|
12
|
+
function fail(field, reason) {
|
|
13
|
+
throw new Error(`Invalid TestPlan: ${field} ${reason}`);
|
|
14
|
+
}
|
|
15
|
+
function assertString(value, field) {
|
|
16
|
+
if (typeof value !== "string" || value.length === 0)
|
|
17
|
+
fail(field, "must be a non-empty string");
|
|
18
|
+
}
|
|
19
|
+
function validateTestPlanEntry(value, index) {
|
|
20
|
+
if (typeof value !== "object" || value === null)
|
|
21
|
+
fail(`suites[${index}]`, "must be an object");
|
|
22
|
+
const v = value;
|
|
23
|
+
assertString(v.suiteId, `suites[${index}].suiteId`);
|
|
24
|
+
assertString(v.suitePath, `suites[${index}].suitePath`);
|
|
25
|
+
return { suiteId: v.suiteId, suitePath: v.suitePath };
|
|
26
|
+
}
|
|
27
|
+
/** Validates a parsed JSON object against the TestPlan shape. Does not check that referenced
|
|
28
|
+
* suite files actually exist or are themselves valid -- that's the `validate-plan` CLI
|
|
29
|
+
* command's job, since it needs filesystem access this pure function doesn't have. */
|
|
30
|
+
export function validateTestPlan(value) {
|
|
31
|
+
if (typeof value !== "object" || value === null)
|
|
32
|
+
fail("plan", "must be an object");
|
|
33
|
+
const v = value;
|
|
34
|
+
assertString(v.id, "id");
|
|
35
|
+
assertString(v.name, "name");
|
|
36
|
+
assertString(v.description, "description");
|
|
37
|
+
if (!Array.isArray(v.suites))
|
|
38
|
+
fail("suites", "must be an array");
|
|
39
|
+
return {
|
|
40
|
+
id: v.id,
|
|
41
|
+
name: v.name,
|
|
42
|
+
description: v.description,
|
|
43
|
+
suites: v.suites.map(validateTestPlanEntry),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A PlanRun is the thin, persisted record that gives a Test Plan's execution a real identity --
|
|
3
|
+
* `{id, planId, startedAt, suiteIds}`. It deliberately carries no name or purpose of its own: that
|
|
4
|
+
* would duplicate what the TestPlan it belongs to already owns. Every reference tool checked
|
|
5
|
+
* before settling this shape (TestRail, Zephyr, Azure DevOps Test Plans) converges on the same
|
|
6
|
+
* pattern -- Plan-as-anchor, Run-as-dated-instance, distinguished from sibling runs only by when
|
|
7
|
+
* it started.
|
|
8
|
+
*
|
|
9
|
+
* `suiteIds` is a snapshot of the plan's suite composition at the moment the run started --
|
|
10
|
+
* confirmed live that a PlanRun without one breaks: adding "create-new-vendor" as a second suite
|
|
11
|
+
* to the Accounts Payable plan made an already-completed, all-passed run retroactively show as
|
|
12
|
+
* "Missing runs" in Studio, because status was being recomputed against the *current* plan's
|
|
13
|
+
* suite list instead of whatever suites existed when that run actually happened. A run is a
|
|
14
|
+
* historical record; it must not change shape because the plan changed after the fact. See
|
|
15
|
+
* plan-runner.ts's `aggregatePlanStatus`, which reads this snapshot instead of `TestPlan.suites`.
|
|
16
|
+
*
|
|
17
|
+
* `suiteNames` extends the same principle to display: it snapshots each suite's human-readable
|
|
18
|
+
* `name` (suiteId -> name) at that same moment, so Studio can show "Create new vendor" instead of
|
|
19
|
+
* "create-new-vendor" without a live lookup against the suite file -- which, per the same
|
|
20
|
+
* suiteIds rationale, could have been renamed or deleted by the time someone views an old run.
|
|
21
|
+
*
|
|
22
|
+
* Suite-level TestRun files (report-schema.ts) are unchanged by this -- a PlanRun's `id` is the
|
|
23
|
+
* same string already used as the `<runId>` suffix in `testruns/<suiteId>-<runId>.json`. This
|
|
24
|
+
* file's only job is to make that id a real, validated, directly-queryable record instead of an
|
|
25
|
+
* unvalidated convention nobody checks.
|
|
26
|
+
*/
|
|
27
|
+
export interface PlanRun {
|
|
28
|
+
id: string;
|
|
29
|
+
planId: string;
|
|
30
|
+
startedAt: string;
|
|
31
|
+
suiteIds: string[];
|
|
32
|
+
suiteNames: Record<string, string>;
|
|
33
|
+
}
|
|
34
|
+
/** Colons aren't legal in Windows filenames, so `id` is `startedAt` with them swapped for dashes
|
|
35
|
+
* -- still lexically sortable, still human-readable, and trivially derivable from `startedAt`
|
|
36
|
+
* rather than needing a separate id-generation scheme. A caller never invents this string
|
|
37
|
+
* themselves (see `planrun.ts`'s `initPlanRun`), which is the whole point: `runId` used to be an
|
|
38
|
+
* arbitrary, unvalidated string a human or LLM picked by hand. `suites` is captured from the plan
|
|
39
|
+
* as it exists right now, at creation time -- never re-derived later -- and split into the two
|
|
40
|
+
* snapshot fields above. */
|
|
41
|
+
export declare function createPlanRun(planId: string, suites: {
|
|
42
|
+
id: string;
|
|
43
|
+
name: string;
|
|
44
|
+
}[]): PlanRun;
|
|
45
|
+
export declare function validatePlanRun(value: unknown): PlanRun;
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A PlanRun is the thin, persisted record that gives a Test Plan's execution a real identity --
|
|
3
|
+
* `{id, planId, startedAt, suiteIds}`. It deliberately carries no name or purpose of its own: that
|
|
4
|
+
* would duplicate what the TestPlan it belongs to already owns. Every reference tool checked
|
|
5
|
+
* before settling this shape (TestRail, Zephyr, Azure DevOps Test Plans) converges on the same
|
|
6
|
+
* pattern -- Plan-as-anchor, Run-as-dated-instance, distinguished from sibling runs only by when
|
|
7
|
+
* it started.
|
|
8
|
+
*
|
|
9
|
+
* `suiteIds` is a snapshot of the plan's suite composition at the moment the run started --
|
|
10
|
+
* confirmed live that a PlanRun without one breaks: adding "create-new-vendor" as a second suite
|
|
11
|
+
* to the Accounts Payable plan made an already-completed, all-passed run retroactively show as
|
|
12
|
+
* "Missing runs" in Studio, because status was being recomputed against the *current* plan's
|
|
13
|
+
* suite list instead of whatever suites existed when that run actually happened. A run is a
|
|
14
|
+
* historical record; it must not change shape because the plan changed after the fact. See
|
|
15
|
+
* plan-runner.ts's `aggregatePlanStatus`, which reads this snapshot instead of `TestPlan.suites`.
|
|
16
|
+
*
|
|
17
|
+
* `suiteNames` extends the same principle to display: it snapshots each suite's human-readable
|
|
18
|
+
* `name` (suiteId -> name) at that same moment, so Studio can show "Create new vendor" instead of
|
|
19
|
+
* "create-new-vendor" without a live lookup against the suite file -- which, per the same
|
|
20
|
+
* suiteIds rationale, could have been renamed or deleted by the time someone views an old run.
|
|
21
|
+
*
|
|
22
|
+
* Suite-level TestRun files (report-schema.ts) are unchanged by this -- a PlanRun's `id` is the
|
|
23
|
+
* same string already used as the `<runId>` suffix in `testruns/<suiteId>-<runId>.json`. This
|
|
24
|
+
* file's only job is to make that id a real, validated, directly-queryable record instead of an
|
|
25
|
+
* unvalidated convention nobody checks.
|
|
26
|
+
*/
|
|
27
|
+
/** Colons aren't legal in Windows filenames, so `id` is `startedAt` with them swapped for dashes
|
|
28
|
+
* -- still lexically sortable, still human-readable, and trivially derivable from `startedAt`
|
|
29
|
+
* rather than needing a separate id-generation scheme. A caller never invents this string
|
|
30
|
+
* themselves (see `planrun.ts`'s `initPlanRun`), which is the whole point: `runId` used to be an
|
|
31
|
+
* arbitrary, unvalidated string a human or LLM picked by hand. `suites` is captured from the plan
|
|
32
|
+
* as it exists right now, at creation time -- never re-derived later -- and split into the two
|
|
33
|
+
* snapshot fields above. */
|
|
34
|
+
export function createPlanRun(planId, suites) {
|
|
35
|
+
const startedAt = new Date().toISOString();
|
|
36
|
+
return {
|
|
37
|
+
id: startedAt.replace(/:/g, "-"),
|
|
38
|
+
planId,
|
|
39
|
+
startedAt,
|
|
40
|
+
suiteIds: suites.map((s) => s.id),
|
|
41
|
+
suiteNames: Object.fromEntries(suites.map((s) => [s.id, s.name])),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
function fail(field, reason) {
|
|
45
|
+
throw new Error(`Invalid PlanRun: ${field} ${reason}`);
|
|
46
|
+
}
|
|
47
|
+
function assertString(value, field) {
|
|
48
|
+
if (typeof value !== "string" || value.length === 0)
|
|
49
|
+
fail(field, "must be a non-empty string");
|
|
50
|
+
}
|
|
51
|
+
/** Missing `suiteIds` (a PlanRun persisted before this field existed) defaults to an empty
|
|
52
|
+
* snapshot rather than failing validation outright -- an old run still loads and displays, it
|
|
53
|
+
* just can't show a per-suite breakdown, which is honest given the data actually on disk. */
|
|
54
|
+
function assertSuiteIds(value) {
|
|
55
|
+
if (value === undefined)
|
|
56
|
+
return [];
|
|
57
|
+
if (!Array.isArray(value))
|
|
58
|
+
fail("suiteIds", "must be an array of strings");
|
|
59
|
+
for (const [i, entry] of value.entries())
|
|
60
|
+
assertString(entry, `suiteIds[${i}]`);
|
|
61
|
+
return value;
|
|
62
|
+
}
|
|
63
|
+
/** Same missing-field tolerance as `assertSuiteIds`, for the same reason -- a PlanRun persisted
|
|
64
|
+
* before `suiteNames` existed defaults to an empty map; callers fall back to the raw suiteId for
|
|
65
|
+
* display rather than failing to load the run at all. */
|
|
66
|
+
function assertSuiteNames(value) {
|
|
67
|
+
if (value === undefined)
|
|
68
|
+
return {};
|
|
69
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
70
|
+
fail("suiteNames", "must be an object mapping suiteId to name");
|
|
71
|
+
}
|
|
72
|
+
const v = value;
|
|
73
|
+
for (const [id, name] of Object.entries(v))
|
|
74
|
+
assertString(name, `suiteNames.${id}`);
|
|
75
|
+
return v;
|
|
76
|
+
}
|
|
77
|
+
export function validatePlanRun(value) {
|
|
78
|
+
if (typeof value !== "object" || value === null)
|
|
79
|
+
fail("planRun", "must be an object");
|
|
80
|
+
const v = value;
|
|
81
|
+
assertString(v.id, "id");
|
|
82
|
+
assertString(v.planId, "planId");
|
|
83
|
+
assertString(v.startedAt, "startedAt");
|
|
84
|
+
const suiteIds = assertSuiteIds(v.suiteIds);
|
|
85
|
+
const suiteNames = assertSuiteNames(v.suiteNames);
|
|
86
|
+
return { id: v.id, planId: v.planId, startedAt: v.startedAt, suiteIds, suiteNames };
|
|
87
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { type PlanRun } from "./planrun-schema.js";
|
|
2
|
+
export declare function readPlanRun(filePath: string): Promise<PlanRun>;
|
|
3
|
+
/** Every PlanRun file in `dir`, optionally filtered to one plan, newest-started first. Shared by
|
|
4
|
+
* the CLI and Studio server so the two can't drift on what "list the runs for this plan" means.
|
|
5
|
+
* Silently skips a file that fails to parse/validate rather than throwing -- one malformed run
|
|
6
|
+
* shouldn't take down the whole list, same tolerance `testruns.ts`'s listing already has. */
|
|
7
|
+
export declare function listPlanRuns(dir: string, planId?: string): Promise<PlanRun[]>;
|
|
8
|
+
/** Creates and persists a fresh PlanRun into `dir`, auto-naming the file from the generated id --
|
|
9
|
+
* the caller supplies a directory, not a filename, so there's no chance of inventing an arbitrary
|
|
10
|
+
* runId string the way `init-run`'s own `-o <path>` always required. `suites` must be the calling
|
|
11
|
+
* plan's current suites (id + name) -- captured once, here, so this run's status/display never
|
|
12
|
+
* drifts if the plan's suites are later added, removed, or renamed (see planrun-schema.ts's
|
|
13
|
+
* `PlanRun.suiteIds`/`suiteNames`). */
|
|
14
|
+
export declare function initPlanRun(planId: string, suites: {
|
|
15
|
+
id: string;
|
|
16
|
+
name: string;
|
|
17
|
+
}[], dir: string): Promise<{
|
|
18
|
+
planRun: PlanRun;
|
|
19
|
+
filePath: string;
|
|
20
|
+
}>;
|
package/dist/planrun.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { createPlanRun, validatePlanRun } from "./planrun-schema.js";
|
|
4
|
+
export async function readPlanRun(filePath) {
|
|
5
|
+
const raw = await readFile(filePath, "utf-8");
|
|
6
|
+
return validatePlanRun(JSON.parse(raw));
|
|
7
|
+
}
|
|
8
|
+
/** Every PlanRun file in `dir`, optionally filtered to one plan, newest-started first. Shared by
|
|
9
|
+
* the CLI and Studio server so the two can't drift on what "list the runs for this plan" means.
|
|
10
|
+
* Silently skips a file that fails to parse/validate rather than throwing -- one malformed run
|
|
11
|
+
* shouldn't take down the whole list, same tolerance `testruns.ts`'s listing already has. */
|
|
12
|
+
export async function listPlanRuns(dir, planId) {
|
|
13
|
+
let files;
|
|
14
|
+
try {
|
|
15
|
+
files = (await readdir(dir)).filter((f) => f.endsWith(".json"));
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return [];
|
|
19
|
+
}
|
|
20
|
+
const runs = [];
|
|
21
|
+
for (const file of files) {
|
|
22
|
+
try {
|
|
23
|
+
const run = await readPlanRun(path.join(dir, file));
|
|
24
|
+
if (!planId || run.planId === planId)
|
|
25
|
+
runs.push(run);
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
// Skip malformed/unrelated files.
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
runs.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
|
|
32
|
+
return runs;
|
|
33
|
+
}
|
|
34
|
+
/** Creates and persists a fresh PlanRun into `dir`, auto-naming the file from the generated id --
|
|
35
|
+
* the caller supplies a directory, not a filename, so there's no chance of inventing an arbitrary
|
|
36
|
+
* runId string the way `init-run`'s own `-o <path>` always required. `suites` must be the calling
|
|
37
|
+
* plan's current suites (id + name) -- captured once, here, so this run's status/display never
|
|
38
|
+
* drifts if the plan's suites are later added, removed, or renamed (see planrun-schema.ts's
|
|
39
|
+
* `PlanRun.suiteIds`/`suiteNames`). */
|
|
40
|
+
export async function initPlanRun(planId, suites, dir) {
|
|
41
|
+
const planRun = createPlanRun(planId, suites);
|
|
42
|
+
const filePath = path.join(dir, `${planRun.id}.json`);
|
|
43
|
+
await mkdir(dir, { recursive: true });
|
|
44
|
+
await writeFile(filePath, JSON.stringify(planRun, null, 2), "utf-8");
|
|
45
|
+
return { planRun, filePath };
|
|
46
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/** Persisted, durable record of a suite run -- this is the mechanism that answers "keep track
|
|
2
|
+
* of each test case, expected input, and actual output" instead of relying on chat memory. */
|
|
3
|
+
/** "skipped" is assigned automatically by `nextStep`/`cmdResolveNext` when an entry's `runIf`
|
|
4
|
+
* (`suite-schema.ts`) evaluates false -- never something the orchestrating LLM passes to
|
|
5
|
+
* `record-result` manually, same as "blocked". Neither a pass nor a failure: a downstream entry
|
|
6
|
+
* that `dependsOn` a skipped one is propagated to "blocked" (see `nextStep`'s `newlyBlocked`
|
|
7
|
+
* check), not left waiting forever. */
|
|
8
|
+
export type TestCaseStatus = "pending" | "running" | "passed" | "failed" | "blocked" | "skipped";
|
|
9
|
+
/** Statuses a test case never leaves once reached -- used by both `suite-runner.ts` (to decide
|
|
10
|
+
* whether a whole suite run is done) and `plan-runner.ts` (to roll that up across suites).
|
|
11
|
+
* Exported from here, not duplicated in each, so the two can't drift out of sync with each other
|
|
12
|
+
* or with `TestCaseStatus` above. Kept as `Set<string>` rather than `Set<TestCaseStatus>` so
|
|
13
|
+
* callers can `.has()` a possibly-missing status (e.g. `result?.status ?? ""`) without a type
|
|
14
|
+
* error. */
|
|
15
|
+
export declare const TERMINAL_STATUSES: ReadonlySet<string>;
|
|
16
|
+
export interface AssertionResult {
|
|
17
|
+
description: string;
|
|
18
|
+
passed: boolean;
|
|
19
|
+
/** What was actually observed (an OData field value, or the matched/missing UI message). */
|
|
20
|
+
actual: string;
|
|
21
|
+
}
|
|
22
|
+
export interface TestCaseResult {
|
|
23
|
+
status: TestCaseStatus;
|
|
24
|
+
outputs: Record<string, string>;
|
|
25
|
+
assertionResults: AssertionResult[];
|
|
26
|
+
/** Run order this test case actually started in (1-based), assigned once by `resolve-next` the
|
|
27
|
+
* first time it transitions to "running" -- not the suite's declared entry order, which can
|
|
28
|
+
* differ once a DAG has parallel-eligible branches. */
|
|
29
|
+
sequence?: number;
|
|
30
|
+
/** Resolved input values this test case actually ran with -- captured automatically from
|
|
31
|
+
* `resolveInputs`'s output when `resolve-next` marks the test case "running" (deterministic
|
|
32
|
+
* bindings only; doesn't include values invented for `"autogenerated"` bindings, since the CLI
|
|
33
|
+
* never sees those). `record-result` may pass a fuller set (including autogenerated values the
|
|
34
|
+
* orchestrating LLM invented) to override this. */
|
|
35
|
+
inputs?: Record<string, string>;
|
|
36
|
+
/** ISO timestamp captured automatically when `resolve-next` marks this test case "running". */
|
|
37
|
+
startedAt?: string;
|
|
38
|
+
/** Milliseconds between `startedAt` and this result reaching a terminal status -- computed
|
|
39
|
+
* automatically by `recordResult`, not supplied by a caller. */
|
|
40
|
+
durationMs?: number;
|
|
41
|
+
/** Measured, not self-reported, via `record-result --tokens` -- but treat it as a rough
|
|
42
|
+
* order-of-magnitude signal, not a trustworthy number. This CLI has no way to observe token
|
|
43
|
+
* usage itself, so `token-usage.ts`'s `measure-tokens` sources it from Claude Code's own local
|
|
44
|
+
* session transcript (`~/.claude/projects/.../<session>.jsonl`). CONFIRMED UNRELIABLE whenever
|
|
45
|
+
* more than one Claude Code session might be active against this project directory during the
|
|
46
|
+
* measurement window -- live-tested 2026-08-20 running suites in parallel: +24% to +98% (nearly
|
|
47
|
+
* 2x) inflation from cross-session misattribution, cross-checked against each session's own
|
|
48
|
+
* authoritative usage total. That's not a rare edge case; it's close to the default way anyone
|
|
49
|
+
* actually uses this (parallel suites, or a Desktop session open alongside a background job).
|
|
50
|
+
* See `token-usage.ts` for why there's no clean fix available today. Also deliberately a
|
|
51
|
+
* Claude-Code-only bonus, not a core requirement: this whole pipeline (`resolve-next`/
|
|
52
|
+
* `record-result`/the replay skill's instructions) is meant to work with any sufficiently
|
|
53
|
+
* capable agentic LLM driving it, and nothing about pass/fail or captured outputs depends on
|
|
54
|
+
* this field. Running under any other agent -- or under Claude Cowork, which executes on
|
|
55
|
+
* Anthropic's own servers rather than the local machine -- simply leaves it unset, since
|
|
56
|
+
* there's no local session transcript to read either way; that's expected, not a bug. */
|
|
57
|
+
tokensUsed?: number;
|
|
58
|
+
/** Cost in USD for this test case, when the host environment can report one directly (e.g. a
|
|
59
|
+
* future headless `claude -p --output-format json` invocation exposes `total_cost_usd`).
|
|
60
|
+
* Deliberately separate from `tokensUsed`: a $ figure needs a pricing table keyed by model and
|
|
61
|
+
* cache-hit/miss that changes over time, which this CLI has no business maintaining -- so this
|
|
62
|
+
* field exists for whichever layer *does* have an authoritative cost to report, and is left
|
|
63
|
+
* unset otherwise rather than computed from `tokensUsed` with a guessed price. */
|
|
64
|
+
costUsd?: number;
|
|
65
|
+
/** Raw `FormState.Messages` text captured by the orchestrating LLM via `record-result
|
|
66
|
+
* --messages` -- the D365FO-side evidence for what actually happened, distinct from
|
|
67
|
+
* `assertionResults`' pass/fail judgments. A single entry's text can carry a tab-delimited
|
|
68
|
+
* hierarchy (confirmed live: posting blocked by credit management returned one `Warning` as
|
|
69
|
+
* `"Posting\tPosting\tSales order: 000963\tOrder is in credit management."`) -- this is the
|
|
70
|
+
* same content D365FO's web client renders as an indented "Message details" tree; there's no
|
|
71
|
+
* separate data source behind that flyout. The skill converts tabs to `" > "` before passing
|
|
72
|
+
* this flag so the hierarchy survives instead of collapsing into a run-on sentence. */
|
|
73
|
+
messages?: string[];
|
|
74
|
+
}
|
|
75
|
+
export interface TestRun {
|
|
76
|
+
suiteId: string;
|
|
77
|
+
startedAt: string;
|
|
78
|
+
results: Record<string, TestCaseResult>;
|
|
79
|
+
}
|
|
80
|
+
/** Creates a fresh TestRun with every suite entry pre-populated as "pending". */
|
|
81
|
+
export declare function createTestRun(suiteId: string, testCaseIds: string[]): TestRun;
|
|
82
|
+
/** Validates a parsed JSON object against the TestRun shape. */
|
|
83
|
+
export declare function validateTestRun(value: unknown): TestRun;
|