trxa-tool 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/replay-trxa-testplan/SKILL.md +627 -0
- package/LICENSE +120 -0
- package/MANUAL.md +548 -0
- package/README.md +67 -3
- package/STUDIO.md +319 -0
- package/dist/analyze.d.ts +58 -0
- package/dist/analyze.js +122 -0
- package/dist/bpm.d.ts +40 -0
- package/dist/bpm.js +60 -0
- package/dist/build-test-script.d.ts +24 -0
- package/dist/build-test-script.js +40 -0
- package/dist/csv.d.ts +1 -0
- package/dist/csv.js +11 -0
- package/dist/extract.d.ts +14 -0
- package/dist/extract.js +34 -0
- package/dist/fixture-schema.d.ts +12 -0
- package/dist/fixture-schema.js +28 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +814 -0
- package/dist/library-archive.d.ts +37 -0
- package/dist/library-archive.js +106 -0
- package/dist/mapping.d.ts +25 -0
- package/dist/mapping.js +390 -0
- package/dist/normalize.d.ts +21 -0
- package/dist/normalize.js +43 -0
- package/dist/parse.d.ts +3 -0
- package/dist/parse.js +233 -0
- package/dist/plan-runner.d.ts +27 -0
- package/dist/plan-runner.js +36 -0
- package/dist/plan-schema.d.ts +27 -0
- package/dist/plan-schema.js +45 -0
- package/dist/planrun-schema.d.ts +45 -0
- package/dist/planrun-schema.js +87 -0
- package/dist/planrun.d.ts +20 -0
- package/dist/planrun.js +46 -0
- package/dist/report-schema.d.ts +83 -0
- package/dist/report-schema.js +89 -0
- package/dist/report.d.ts +42 -0
- package/dist/report.js +70 -0
- package/dist/schema.d.ts +96 -0
- package/dist/schema.js +10 -0
- package/dist/scrub.d.ts +44 -0
- package/dist/scrub.js +166 -0
- package/dist/suggest-inputs.d.ts +19 -0
- package/dist/suggest-inputs.js +92 -0
- package/dist/suite-runner.d.ts +59 -0
- package/dist/suite-runner.js +198 -0
- package/dist/suite-schema.d.ts +59 -0
- package/dist/suite-schema.js +95 -0
- package/dist/testcase-schema.d.ts +102 -0
- package/dist/testcase-schema.js +132 -0
- package/dist/token-usage.d.ts +38 -0
- package/dist/token-usage.js +95 -0
- package/package.json +58 -6
- package/studio/client-dist/assets/index-CgWhGkEn.js +24 -0
- package/studio/client-dist/assets/index-CwAUMVCL.css +1 -0
- package/studio/client-dist/axtr-icon.svg +5 -0
- package/studio/client-dist/favicon.svg +5 -0
- package/studio/client-dist/index.html +14 -0
- package/studio/server/dist/index.js +1938 -0
package/README.md
CHANGED
|
@@ -1,3 +1,67 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
# TRXA-TOOL
|
|
2
|
+
|
|
3
|
+
Turn D365 Finance & Operations Task Recorder (`.axtr`) recordings into automated, data-driven
|
|
4
|
+
regression tests that an AI agent replays against a live or sandbox environment.
|
|
5
|
+
|
|
6
|
+
TRXA-TOOL does the deterministic work: it parses each recording into an ordered test script,
|
|
7
|
+
lets you define test cases, test data, suites and plans around it, and works out what runs next.
|
|
8
|
+
Your agent (Claude Code, Claude Cowork, Codex, or any agent that can drive MCP tools) does the
|
|
9
|
+
live part: it drives D365FO through the `d365-erp-mcp` server, checks each test's expectations,
|
|
10
|
+
and records pass/fail results you can review later.
|
|
11
|
+
|
|
12
|
+
## What you need
|
|
13
|
+
|
|
14
|
+
- **Node.js** 18.19 or later
|
|
15
|
+
- **An agent runtime that can call MCP tools**, such as [Claude Code](https://claude.com/claude-code),
|
|
16
|
+
Claude Cowork, or [Codex](https://developers.openai.com/codex)
|
|
17
|
+
- **The `d365-erp-mcp` MCP server**, connected to the D365FO environment you want to test
|
|
18
|
+
- **One or more `.axtr` recordings**, recorded with D365FO's Task Recorder or exported from an LCS
|
|
19
|
+
Business Process Modeler library
|
|
20
|
+
|
|
21
|
+
## Install
|
|
22
|
+
|
|
23
|
+
```sh
|
|
24
|
+
npm install -g trxa-tool
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
This installs the `trxa` command.
|
|
28
|
+
|
|
29
|
+
## Quick start
|
|
30
|
+
|
|
31
|
+
Run these from your own project folder:
|
|
32
|
+
|
|
33
|
+
```sh
|
|
34
|
+
trxa init-skill # once per project: installs the replay skill for Claude Code and Codex
|
|
35
|
+
trxa parse "recordings/My recording.axtr" -o testscripts/my-recording.testscript.json
|
|
36
|
+
trxa studio # optional: the browser UI for building test cases, suites and plans
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Then ask your agent to run a suite, for example: *"use the replay-trxa-testplan skill to run
|
|
40
|
+
suites/my-suite.json"*.
|
|
41
|
+
|
|
42
|
+
Run `trxa help` for the full command list.
|
|
43
|
+
|
|
44
|
+
## Documentation
|
|
45
|
+
|
|
46
|
+
These files ship with the package (find them with `npm root -g`, under `trxa-tool/`):
|
|
47
|
+
|
|
48
|
+
- **[MANUAL.md](MANUAL.md)**: step-by-step setup and the full CLI workflow, from recording to
|
|
49
|
+
test run.
|
|
50
|
+
- **[STUDIO.md](STUDIO.md)**: the browser UI.
|
|
51
|
+
|
|
52
|
+
## A note on data
|
|
53
|
+
|
|
54
|
+
A `.axtr` recording captures whatever was on screen when it was made, including real account
|
|
55
|
+
numbers, amounts and names. Test scripts and drafted test inputs carry that data forward. Review
|
|
56
|
+
anything derived from a real recording before sharing or committing it. See MANUAL.md's
|
|
57
|
+
"Data hygiene" section.
|
|
58
|
+
|
|
59
|
+
## Issues
|
|
60
|
+
|
|
61
|
+
Found a bug or a recording the tool can't handle? Open an issue at
|
|
62
|
+
https://github.com/elcanelo/trxa-tool/issues. Please don't attach recordings or test files that
|
|
63
|
+
contain real business data.
|
|
64
|
+
|
|
65
|
+
## License
|
|
66
|
+
|
|
67
|
+
Proprietary. Use is governed by the end-user license agreement in [LICENSE](LICENSE).
|
package/STUDIO.md
ADDED
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
# TRXA-TOOL Studio manual
|
|
2
|
+
|
|
3
|
+
Studio is a local browser UI for the same pipeline [MANUAL.md](MANUAL.md) walks through via the
|
|
4
|
+
CLI: parsing a `.axtr` recording, drafting inputs, editing a Test Case's `outputs[]`/
|
|
5
|
+
`expectations[]`, managing Fixtures, building Suites (including their dependency DAG and
|
|
6
|
+
conditional-skip gates) and Plans, and reviewing Test Runs -- without hand-editing JSON. It runs
|
|
7
|
+
the same code the CLI does, so anything Studio writes is exactly as valid as anything the CLI
|
|
8
|
+
writes, and vice versa -- switch between them freely on the same files.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 1. Starting Studio
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
trxa studio # starts the server + client in the background, opens your browser
|
|
16
|
+
trxa studio stop # shuts down the whole process tree (not just the top-level process)
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Running `studio` again while it's already up just reopens the browser instead of starting a
|
|
20
|
+
second instance -- PID/log tracking lives in `~/.trxa-tool/studio-run/` (a per-user location, independent of
|
|
21
|
+
where the tool is installed).
|
|
22
|
+
|
|
23
|
+
**Which project does it open?** Studio's data (`recordings/`, `testcases/`, `suites/`, etc.) lives
|
|
24
|
+
under a *working folder* -- a real, persisted setting, not just "wherever you happened to run the
|
|
25
|
+
command from." The first time Studio ever starts on a machine, it defaults to and remembers
|
|
26
|
+
whatever directory `trxa studio` was run from; after that, it's whatever you last set it to.
|
|
27
|
+
Change it anytime from the **Settings** tab (gear icon, top right) -- the switch takes effect
|
|
28
|
+
immediately, no restart needed. Install location, working folder, and "wherever you happen to be
|
|
29
|
+
running the command from" are three different things, and only the working folder setting decides
|
|
30
|
+
which project's files Studio reads and writes.
|
|
31
|
+
|
|
32
|
+
Studio binds to `127.0.0.1` only and opens at `http://localhost:5173`. **Never expose it on a
|
|
33
|
+
network** -- it does unauthenticated filesystem writes by design (same trust model as the CLI:
|
|
34
|
+
whoever can run it can write anywhere under the working folder's `testcases/`, `testdata/`,
|
|
35
|
+
`testscripts/` directories).
|
|
36
|
+
|
|
37
|
+
## 2. The six tabs, at a glance
|
|
38
|
+
|
|
39
|
+
| Tab | Purpose |
|
|
40
|
+
|---|---|
|
|
41
|
+
| **Recordings** | Parse a `.axtr` file, draft inputs from it, and create a new Test Case + Fixture pair. |
|
|
42
|
+
| **Test Cases** | Edit an existing Test Case's `outputs[]`/`expectations[]`, or delete it. |
|
|
43
|
+
| **Fixtures** | Full CRUD on `testdata/*.json` Fixtures -- concrete input values for a Test Case. |
|
|
44
|
+
| **Suites** | A drag-and-drop dependency-graph canvas for `suites/*.json` -- `dependsOn`, `inputBindings`, and conditional-skip (`runIf`) gates. |
|
|
45
|
+
| **Plans** | Compose a `plans/*.json` from existing suites, with reorderable run order and a rollup-status viewer. |
|
|
46
|
+
| **Test Runs** | Plan-first, read-only viewer: pick a Plan's run (a real `runs/*.json` PlanRun record), see its suites' rollup, drill into one for per-test-case status, outputs, assertions, live auto-refresh while a run is in progress. |
|
|
47
|
+
|
|
48
|
+
Every tab's search box, and the active tab + open record itself, round-trip through
|
|
49
|
+
`location.hash` -- a refresh, a bookmarked link, or the browser's back/forward buttons all land
|
|
50
|
+
back where you were instead of resetting to the Recordings tab. The header also carries **Export
|
|
51
|
+
library**/**Import library** buttons that bundle or restore the whole `recordings/`/`testcases/`/
|
|
52
|
+
`testdata/`/`testscripts/`/`suites/`/`plans/`/`testruns/` tree as one zip (same as the CLI's
|
|
53
|
+
`export`/`import` commands) -- useful for backing up or moving an installation.
|
|
54
|
+
|
|
55
|
+
Not here yet: creating a Test Case's `inputs[]`/`bindTo` after the fact -- that stays on the
|
|
56
|
+
Recordings page's flow below.
|
|
57
|
+
|
|
58
|
+
## 3. Recordings: parse a `.axtr` file and draft a Test Case + Fixture
|
|
59
|
+
|
|
60
|
+
The left pane lists every `.axtr` file under `recordings/`, grouped into a folder tree matching
|
|
61
|
+
their layout on disk (search flattens it to a filtered list with full paths, since matches from
|
|
62
|
+
different folders no longer need visual grouping once you're searching). Click one to parse it.
|
|
63
|
+
|
|
64
|
+
Once parsed, you'll see the recording's name, step count, and required-privilege count, followed
|
|
65
|
+
by a step table: sequence, description, form, node type, and the suggested replay hint for that step
|
|
66
|
+
(the tool it'd call during replay, a confidence badge, and a note when the mapping needed one).
|
|
67
|
+
Steps with no hint show an "unmapped" badge -- rare, but worth reporting if you see one.
|
|
68
|
+
|
|
69
|
+
Click **Suggest Inputs** to draft candidate inputs from the recording (same output as
|
|
70
|
+
`trxa suggest-inputs`). Each candidate traces back to either a recorded value
|
|
71
|
+
(`PropertyUserAction`) or a low-confidence grid-selection step, and renders as an editable row
|
|
72
|
+
directly under the step it came from -- required candidates are marked with a red `*`. Edit the
|
|
73
|
+
values here before creating the Test Case; whatever you type becomes that draft input's initial
|
|
74
|
+
value in the Fixture you create alongside it.
|
|
75
|
+
|
|
76
|
+
**Caution:** candidate values are copied verbatim from the recording -- if it was made against
|
|
77
|
+
real business data rather than demo data, review/scrub these before creating anything you intend
|
|
78
|
+
to commit. See MANUAL.md's "Data hygiene" section.
|
|
79
|
+
|
|
80
|
+
Once you've reviewed the candidates, fill in the **Create Test Case & Fixture** form:
|
|
81
|
+
|
|
82
|
+
- **Test case ID** -- written to `testcases/<id>.json` and `testscripts/<id>.testscript.json`.
|
|
83
|
+
- **Test case name** -- pre-filled from the recording's own name; edit freely.
|
|
84
|
+
- **Description** -- optional.
|
|
85
|
+
- **Fixture name** -- type a new one or pick from the autocomplete of existing fixture names
|
|
86
|
+
(e.g. `basic`). Written to `testdata/<id>.<name>.json`.
|
|
87
|
+
|
|
88
|
+
**Create Test Case & Fixture** stays disabled until every required candidate has a value and all
|
|
89
|
+
four fields above are filled. On success it writes all three files in one action; if the Test
|
|
90
|
+
Case is created but the Fixture write fails for some reason, the error banner says so explicitly
|
|
91
|
+
-- you can still create the Fixture separately from the Fixtures tab using the same Test Case ID.
|
|
92
|
+
|
|
93
|
+
## 4. Test Cases: edit outputs and expectations
|
|
94
|
+
|
|
95
|
+
The left pane groups every `testcases/*.json` file into a folder tree matching its source
|
|
96
|
+
recording's location under `recordings/` (an entry that fails to parse/validate shows "Invalid:
|
|
97
|
+
<error>" instead of crashing the list; one created before this grouping existed, or from a
|
|
98
|
+
top-level recording, falls into an "Ungrouped" bucket). Click one to load it, along with its
|
|
99
|
+
underlying `testscript.json` (used to populate the step pickers below).
|
|
100
|
+
|
|
101
|
+
If the Test Case has `inputs[]`, they're shown read-only at the top with a reminder of the
|
|
102
|
+
`{{paramName}}` substitution syntax you can use in a filter/value below (e.g.
|
|
103
|
+
`CustomerAccount eq '{{customerAccount}}'`) -- editing inputs themselves stays on the Recordings
|
|
104
|
+
page's flow (§3).
|
|
105
|
+
|
|
106
|
+
### Outputs
|
|
107
|
+
|
|
108
|
+
Each output captures a value after some step, for a later Test Case in a suite to consume (see
|
|
109
|
+
`{ "kind": "testCaseOutput", ... }` bindings in MANUAL.md §7). The **name** field is free text --
|
|
110
|
+
whatever you call it here is what a downstream Test Case's binding refers back to.
|
|
111
|
+
|
|
112
|
+
Pick a **kind**:
|
|
113
|
+
|
|
114
|
+
- **`odata`** -- query the entity via OData: `Entity set`, `Filter` (may use
|
|
115
|
+
`{{paramName}}`), `Field` to read. Preferred when there's a solid, unique business key to
|
|
116
|
+
filter by -- durable, works even after the session ends.
|
|
117
|
+
- **`uiField` (form state)** -- read a value straight off the live form. Pick the step to read it
|
|
118
|
+
**after**; then either:
|
|
119
|
+
- **Control name** -- the exact AOT control name. Free at replay time (no extra query, the
|
|
120
|
+
value is already in that step's own response) and exactly reproducible. Auto-suggested from
|
|
121
|
+
the selected step when it acts on one control; otherwise pick from the autocomplete of every
|
|
122
|
+
control name seen anywhere in this script, or type your own.
|
|
123
|
+
- **Hint** (only used when Control name is blank) -- a plain-English description (e.g. "the
|
|
124
|
+
newly assigned purchase order number shown after clicking New"), for values Task Recorder
|
|
125
|
+
never captured because nobody clicked them -- typically an auto-assigned ID. The replaying
|
|
126
|
+
LLM resolves this against live form state during a run and is expected to write the resolved
|
|
127
|
+
control name back into this field for you, so it stays free/exact on every run after the
|
|
128
|
+
first.
|
|
129
|
+
|
|
130
|
+
### Expectations
|
|
131
|
+
|
|
132
|
+
Each expectation checks something must (or must not) hold after a step, for the Test Case to
|
|
133
|
+
pass. Pick the step to check **after**, then a **kind**:
|
|
134
|
+
|
|
135
|
+
- **`odata`** -- query an entity and compare `Expected field` against `Expected value` with a
|
|
136
|
+
`Comparator` (`equals`/`notEquals`/`contains`/`greaterThan`/`lessThan`).
|
|
137
|
+
- **`uiMessage`** -- check `FormState.Messages` for an entry of the given **Severity**
|
|
138
|
+
(`Info`/`Warning`/`Error`) whose text contains **Pattern** as a plain substring, not a regex --
|
|
139
|
+
leave it blank to match any message of that severity regardless of text.
|
|
140
|
+
|
|
141
|
+
**Watch the `Should exist` checkbox carefully** -- it's easy to get backwards. If your
|
|
142
|
+
expectation reads "no error appeared," you want `Severity: Error` with **`Should exist`
|
|
143
|
+
unchecked**, not checked -- checking it asserts an Error-severity message *must* appear, the
|
|
144
|
+
opposite of what "no error" usually means. (This exact mistake has shipped in a real test case before
|
|
145
|
+
being caught during a live replay -- worth double-checking every time.)
|
|
146
|
+
|
|
147
|
+
### Saving and deleting
|
|
148
|
+
|
|
149
|
+
**Save** writes the file back (validated server-side the same way the CLI's
|
|
150
|
+
`validate-testcase` command would reject a malformed one) and confirms with a toast. **Delete**
|
|
151
|
+
asks for confirmation, then removes only `testcases/<id>.json` -- not the `testscript.json` it
|
|
152
|
+
points to, since that file isn't guaranteed unique to one Test Case (a suite's test cases can
|
|
153
|
+
share a source recording), and not any Fixture/Suite that still references this Test Case's ID
|
|
154
|
+
(those will start failing to resolve; clean them up separately).
|
|
155
|
+
|
|
156
|
+
## 5. Fixtures: manage test data
|
|
157
|
+
|
|
158
|
+
The left pane groups every `testdata/*.json` file by the Test Case it belongs to (same
|
|
159
|
+
"Invalid: ..." handling as Test Cases), so a scenario with several data variants (e.g. three
|
|
160
|
+
vendors × three items) stays organized instead of one long flat list. **+ New fixture** switches
|
|
161
|
+
the detail pane to create mode.
|
|
162
|
+
|
|
163
|
+
For a **new** fixture: pick a **Filename slug** (written to `testdata/<slug>.json` -- convention
|
|
164
|
+
is `<testCaseId>.<name>`, e.g. `create-purchase-order.basic`, but nothing enforces it), a **Test
|
|
165
|
+
case** from the dropdown, and an optional **Label** -- a short free-text description (distinct
|
|
166
|
+
from the filename-safe slug) shown in the fixture's list entry so multiple variants for the same
|
|
167
|
+
Test Case stay distinguishable at a glance. Once you pick a Test Case, the **Values** section
|
|
168
|
+
below renders one field per its `inputs[]` -- label, required marker, placeholder from the
|
|
169
|
+
input's recorded `example`, and its description as a hint underneath.
|
|
170
|
+
|
|
171
|
+
**The Test case a Fixture targets can't be changed after creation** -- the field is locked on an
|
|
172
|
+
existing Fixture; delete and recreate it against a different Test Case instead. This is
|
|
173
|
+
deliberate: a Fixture's Values shape is entirely derived from its Test Case's `inputs[]`, so
|
|
174
|
+
repointing it would silently invalidate every value already entered.
|
|
175
|
+
|
|
176
|
+
**Save**/**Delete** work the same as Test Cases (§4) -- Save confirms via toast, Delete asks for
|
|
177
|
+
confirmation and removes only that one `testdata/*.json` file.
|
|
178
|
+
|
|
179
|
+
## 6. Suites: the dependency graph canvas
|
|
180
|
+
|
|
181
|
+
The left pane lists every `suites/*.json` file. **+ New suite** switches the detail pane to
|
|
182
|
+
create mode, asking for a **Filename slug** (written to `suites/<slug>.json`), a **Suite id** (the
|
|
183
|
+
stable id a Plan references it by -- independent of the filename), and a **Name**.
|
|
184
|
+
|
|
185
|
+
The main panel is a top-to-bottom flow canvas (`@xyflow/react`): one node per entry, arrows
|
|
186
|
+
showing `dependsOn`. **Drag from a node's bottom handle to another node's top handle** to add a
|
|
187
|
+
dependency; select an edge and press **Backspace or Delete** to remove one. Drag a node itself to
|
|
188
|
+
reposition it -- positions are presentation-only (kept in your browser's `localStorage`, not
|
|
189
|
+
written to the suite file), and a **Reset layout** button (shown once you've moved something)
|
|
190
|
+
clears them back to the automatic layered layout. Use the toolbar's dropdown + **Add** to add an
|
|
191
|
+
existing Test Case as a new entry.
|
|
192
|
+
|
|
193
|
+
Click a node to open its inspector:
|
|
194
|
+
|
|
195
|
+
- **Legal entity** -- passed as the company on this entry's Test Case's *first* navigation call
|
|
196
|
+
only; leave blank to stay in whatever company the run is already in.
|
|
197
|
+
- **Depends on** -- read-only list of what you've connected via the canvas.
|
|
198
|
+
- **Run condition** -- an optional conditional-skip gate. Pick one of this entry's own
|
|
199
|
+
dependencies from the **Only run if** dropdown (or "Always run" to clear the condition), then
|
|
200
|
+
an **output** that dependency declares, a **comparator** (`equals`/`notEquals`/`contains`/
|
|
201
|
+
`greaterThan`/`lessThan`), and an **expected value**. If the condition evaluates false at replay
|
|
202
|
+
time, this entry is recorded `"skipped"` instead of running -- not a pass, not a failure. A
|
|
203
|
+
downstream entry that depends on a skipped one is itself reported `"blocked"`, so nothing is
|
|
204
|
+
ever left waiting forever. Only shown for entries that have at least one dependency, since a
|
|
205
|
+
root entry has nothing to gate on.
|
|
206
|
+
- **Input bindings** -- one row per the Test Case's declared `inputs[]`, with a five-way picker:
|
|
207
|
+
**Unbound**, **Fixture** (pick one of the Test Case's fixtures, then a value key from it),
|
|
208
|
+
**Output** (pick an upstream Test Case -- any transitive dependency, not just a direct one --
|
|
209
|
+
and one of its declared outputs), **Literal** (type a value directly into the suite file), or
|
|
210
|
+
**Auto** (Autogenerated -- no value here at all; the orchestrating LLM invents one at replay
|
|
211
|
+
time using the input's own description/example as guidance, typically for a business key that
|
|
212
|
+
just needs to be unique per run).
|
|
213
|
+
|
|
214
|
+
**Save**/**Delete** work the same as other tabs -- Save validates server-side (including the same
|
|
215
|
+
DAG cycle/unknown-id check `validate-suite` runs on the CLI), Delete asks for confirmation and
|
|
216
|
+
removes only that one `suites/*.json` file.
|
|
217
|
+
|
|
218
|
+
## 7. Plans: group suites for combined reporting
|
|
219
|
+
|
|
220
|
+
The left pane lists every `plans/*.json` file. **+ New plan** switches to create mode, asking for
|
|
221
|
+
a **Filename slug**, **Plan id**, **Name**, and **Description**.
|
|
222
|
+
|
|
223
|
+
The editor is a two-pane picker: **Available suites** (with its own search) on the left, each
|
|
224
|
+
with a **+ Add** button; **In this plan (run order)** on the right, each entry with move-earlier/
|
|
225
|
+
move-later/remove controls -- array order is the suggested run order, there's no dependency graph
|
|
226
|
+
between suites (if two suites need to share data, they belong in one suite instead).
|
|
227
|
+
|
|
228
|
+
Below the editor, a **rollup-status viewer** shows a combined pass/fail verdict across every suite
|
|
229
|
+
in the plan for one PlanRun. **Start new run** creates a real, persisted `runs/<id>.json` record
|
|
230
|
+
(`{id, planId, startedAt}`) with an auto-generated id; the **Run** dropdown lists every PlanRun
|
|
231
|
+
that belongs to this plan, newest first, by its actual start time. Run
|
|
232
|
+
each of the plan's suites (by hand or via the replay skill) using that PlanRun's id as the
|
|
233
|
+
`<runId>` in `testruns/<suiteId>-<runId>.json`, exactly as the CLI's `init-plan-run` +
|
|
234
|
+
`init-run` sequence expects (see MANUAL.md §8).
|
|
235
|
+
|
|
236
|
+
For the selected run, each suite gets one of four statuses: **missing** (no
|
|
237
|
+
`testruns/<suiteId>-<runId>.json` exists at all -- that suite hasn't been run yet under this
|
|
238
|
+
PlanRun), **incomplete** (the file exists, but some test case in it hasn't reached a terminal
|
|
239
|
+
status yet), **failed** (every test case reached a terminal status, and at least one is
|
|
240
|
+
`failed`/`blocked`), or **passed** (every test case terminal, none failed or blocked). The plan's
|
|
241
|
+
own **Overall** badge then picks the worst case across all its suites, in that same order: any
|
|
242
|
+
suite `failed` → **has-failures**; else any `incomplete` → **incomplete**; else any `missing` →
|
|
243
|
+
**missing-runs**; else → **all-passed**. This is the same logic the CLI's
|
|
244
|
+
`plan-status` command uses -- a pure, read-only computation over
|
|
245
|
+
already-existing Test Run files, safe to check as often as you like while suites are still
|
|
246
|
+
running, since nothing beyond each suite's own Test Run is ever written.
|
|
247
|
+
|
|
248
|
+
**Save**/**Delete** work the same as Suites.
|
|
249
|
+
|
|
250
|
+
## 8. Test Runs: view results
|
|
251
|
+
|
|
252
|
+
The left pane groups by Plan, not by suite file: each Plan you've created is a header, and under
|
|
253
|
+
it every PlanRun that belongs to it (see §7) is a clickable row showing when it started, newest
|
|
254
|
+
first -- a search box filters by plan name. Selecting a run shows the plan-level rollup (the same
|
|
255
|
+
Overall badge plus per-suite status/counts §7 describes), and clicking one of its suites drills
|
|
256
|
+
into that suite's own detail below.
|
|
257
|
+
|
|
258
|
+
That suite detail shows every test case in it: its status badge (`Passed`/`Failed`/`Blocked`/
|
|
259
|
+
`Skipped`/`Running`/`Pending`), any captured `outputs[]` values, and every `expectations[]`
|
|
260
|
+
assertion with a Pass/Fail badge and the `actual` value that was actually observed (an OData
|
|
261
|
+
field, or the matched/missing UI message) -- the same evidence `replay-trxa-testplan` records
|
|
262
|
+
when it calls `record-result`. A "failures only" toggle filters the per-test-case list down to
|
|
263
|
+
just what needs attention. **While a run is still in progress, this view auto-refreshes itself
|
|
264
|
+
every few seconds** -- no manual re-selecting needed to watch it happen live.
|
|
265
|
+
|
|
266
|
+
If any test case reports `tokensUsed` and/or `costUsd` (see MANUAL.md §7.8's "Token/cost
|
|
267
|
+
reporting"), a **Tokens**/**Cost** tile appears alongside the Passed/Failed/Blocked/Skipped/Total
|
|
268
|
+
summary row, showing the sum across the whole run; the same figures also show per-test-case in
|
|
269
|
+
each result's meta line. Older runs recorded before this existed simply don't show these tiles --
|
|
270
|
+
there's no "$0.00" placeholder for a run that never reported either.
|
|
271
|
+
|
|
272
|
+
**Copy summary** (top-right of an open suite) copies that suite's results to your clipboard as
|
|
273
|
+
both Markdown (for pasting into chat) and rich HTML (for pasting into email/Word/Docs, with status
|
|
274
|
+
colors and Infolog severity preserved). **Export** offers the same content as a downloadable
|
|
275
|
+
`.md` file, or **PDF** via the browser's own print dialog. **Delete** (same row) removes that
|
|
276
|
+
suite's Test Run file entirely after a confirmation prompt -- unlike deleting a Test Case/
|
|
277
|
+
Fixture/Suite/Plan, a Test Run has no other file that references it by id, so this can't break
|
|
278
|
+
anything else. The plan-level rollup view has its own **Delete run**, which only removes the
|
|
279
|
+
`runs/<id>.json` PlanRun record itself -- the underlying suite Test Run files aren't touched, so
|
|
280
|
+
deleting a PlanRun just un-groups suites that already ran, it doesn't erase their results.
|
|
281
|
+
|
|
282
|
+
This tab is otherwise **read-only** -- there's no editing a run's contents, since it's a durable
|
|
283
|
+
record of what a replay session actually did, not something to hand-edit. To produce a new one,
|
|
284
|
+
start it from the Plans tab's rollup viewer (§7), then run each suite live per MANUAL.md §8's
|
|
285
|
+
"Running a plan" procedure (itself the same per-suite `resolve-next`/`record-result` loop from
|
|
286
|
+
§7.7, once per suite); this tab picks it up on its own (the list itself also polls periodically,
|
|
287
|
+
so a run started from another session or the CLI appears without a manual refresh).
|
|
288
|
+
|
|
289
|
+
## 9. Cross-cutting UI notes
|
|
290
|
+
|
|
291
|
+
- **Toasts**: every Save on an editable tab (Recordings/Test Cases/Fixtures/Suites/Plans) confirms
|
|
292
|
+
with a small message that fades in at the top of the screen, holds briefly, then fades out on
|
|
293
|
+
its own -- no need to dismiss it. Test Runs has nothing to save, so it never shows one.
|
|
294
|
+
- **Error banners**: validation failures (missing required fields, a malformed filter, a suite DAG
|
|
295
|
+
cycle, etc.) show as a banner at the top of the detail pane, not a toast -- toasts are for
|
|
296
|
+
success, banners are for something you need to fix.
|
|
297
|
+
- **List/detail split pane**: every tab shares the same layout -- a searchable/scrollable list on
|
|
298
|
+
the left, the selected item's editor (or, for Test Runs, read-only view) on the right. Nothing
|
|
299
|
+
you do in an editable tab's editor is saved until you click Save.
|
|
300
|
+
- **URL/refresh persistence**: the active tab and whichever record you have open are reflected in
|
|
301
|
+
the browser's address bar (`location.hash`), so refreshing the page, following a bookmarked
|
|
302
|
+
link, or using back/forward all restore the same screen instead of resetting to Recordings.
|
|
303
|
+
|
|
304
|
+
## 10. Troubleshooting
|
|
305
|
+
|
|
306
|
+
- **Port 5173 (or 4321) already in use**: something else is already running there -- check
|
|
307
|
+
`~/.trxa-tool/studio-run/` for a stale PID, or run `trxa studio stop` first. If neither
|
|
308
|
+
clears it, something outside Studio's own tracking is holding the port; find and stop it
|
|
309
|
+
manually (Task Manager on Windows, `lsof -i :5173` on macOS/Linux).
|
|
310
|
+
- **A recording/Test Case/Fixture/Suite/Plan doesn't show up**: those lists load once, when you
|
|
311
|
+
open the tab -- refresh if you added a file outside Studio (e.g. via the CLI or by hand) while
|
|
312
|
+
it was open. Test Runs is the exception: both its list and an open run poll periodically on
|
|
313
|
+
their own, so a run started elsewhere (another session, the CLI) appears without a manual
|
|
314
|
+
refresh.
|
|
315
|
+
- **"Invalid: ..." in a list**: that file failed schema validation -- the error message is the
|
|
316
|
+
same one `validate-testcase`/`validate-fixture`/`validate-suite`/`validate-plan` would give you
|
|
317
|
+
on the CLI. Fix it by hand or via the CLI; Studio can't open an invalid file for editing.
|
|
318
|
+
- **Can't delete a dependency connection on the Suites canvas**: click the connection to select
|
|
319
|
+
it, then press **Backspace or Delete**.
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { type ScrubbedValue } from "./scrub.js";
|
|
2
|
+
import { type Confidence } from "./mapping.js";
|
|
3
|
+
export interface StepSummary {
|
|
4
|
+
sequence: number;
|
|
5
|
+
kind: string;
|
|
6
|
+
formName: string | null;
|
|
7
|
+
/** Scrubbed -- literal values are `<<REDACTED:...>>` tokens, never real data. */
|
|
8
|
+
description: string;
|
|
9
|
+
hintTool: string | null;
|
|
10
|
+
hintConfidence: Confidence | null;
|
|
11
|
+
hintNote: string | null;
|
|
12
|
+
}
|
|
13
|
+
export interface RecordingSummary {
|
|
14
|
+
file: string;
|
|
15
|
+
name: string;
|
|
16
|
+
canonicalId: string;
|
|
17
|
+
stepCount: number;
|
|
18
|
+
steps: StepSummary[];
|
|
19
|
+
requiredPrivileges: string[];
|
|
20
|
+
}
|
|
21
|
+
export interface ParseError {
|
|
22
|
+
file: string;
|
|
23
|
+
error: string;
|
|
24
|
+
}
|
|
25
|
+
export interface GapBucket {
|
|
26
|
+
/** e.g. "CommandUserAction:ChangeSelectedIndexInCache:Grid" or "Unknown:ValidationUserAction" */
|
|
27
|
+
key: string;
|
|
28
|
+
count: number;
|
|
29
|
+
sampleRecordings: string[];
|
|
30
|
+
}
|
|
31
|
+
export interface LibraryReport {
|
|
32
|
+
generatedAt: string;
|
|
33
|
+
recordingsAnalyzed: number;
|
|
34
|
+
parseErrors: ParseError[];
|
|
35
|
+
recordings: RecordingSummary[];
|
|
36
|
+
aggregate: {
|
|
37
|
+
nodeKindCounts: Record<string, number>;
|
|
38
|
+
hintConfidenceCounts: {
|
|
39
|
+
high: number;
|
|
40
|
+
medium: number;
|
|
41
|
+
low: number;
|
|
42
|
+
unmapped: number;
|
|
43
|
+
};
|
|
44
|
+
unknownXsiTypes: GapBucket[];
|
|
45
|
+
lowConfidenceGaps: GapBucket[];
|
|
46
|
+
distinctSecurityPrivileges: string[];
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
export interface LibraryAnalysis {
|
|
50
|
+
report: LibraryReport;
|
|
51
|
+
scrubbedValues: ScrubbedValue[];
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Locally parses and scrubs every `.axtr` file in `recordingsDir`, returning a structural-only
|
|
55
|
+
* report (safe to hand to an LLM) plus the full list of stripped literal values (never safe to
|
|
56
|
+
* hand to an LLM -- keep local, see the `analyze-library` CLI command's CSV output).
|
|
57
|
+
*/
|
|
58
|
+
export declare function analyzeLibrary(recordingsDir: string): Promise<LibraryAnalysis>;
|
package/dist/analyze.js
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { readFile, readdir } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { extractAxtr } from "./extract.js";
|
|
4
|
+
import { parseRecordingXml } from "./parse.js";
|
|
5
|
+
import { scrubRecording } from "./scrub.js";
|
|
6
|
+
import { normalizeRecording } from "./normalize.js";
|
|
7
|
+
import { mapStep } from "./mapping.js";
|
|
8
|
+
import { parseBpmPackage, distinctPrivileges } from "./bpm.js";
|
|
9
|
+
/** Cosmetic cap on how many example recording filenames a gap bucket lists in the report --
|
|
10
|
+
* `count` above already tracks the true total, this just keeps `sampleRecordings` from growing
|
|
11
|
+
* unboundedly for a gap that recurs across most of a large library. No significance to the value
|
|
12
|
+
* 5 beyond "a handful is enough to spot-check"; raise it freely if a reviewer wants more examples. */
|
|
13
|
+
const MAX_SAMPLE_RECORDINGS = 5;
|
|
14
|
+
function bumpBucket(map, key, recordingFile) {
|
|
15
|
+
const existing = map.get(key);
|
|
16
|
+
if (existing) {
|
|
17
|
+
existing.count += 1;
|
|
18
|
+
if (!existing.sampleRecordings.includes(recordingFile) && existing.sampleRecordings.length < MAX_SAMPLE_RECORDINGS) {
|
|
19
|
+
existing.sampleRecordings.push(recordingFile);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
else {
|
|
23
|
+
map.set(key, { key, count: 1, sampleRecordings: [recordingFile] });
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
function gapKeyFor(step) {
|
|
27
|
+
const node = step.node;
|
|
28
|
+
if (node.kind === "Unknown")
|
|
29
|
+
return `Unknown:${node.xsiType}`;
|
|
30
|
+
if (node.kind === "CommandUserAction")
|
|
31
|
+
return `CommandUserAction:${node.CommandName}:${node.ControlType}`;
|
|
32
|
+
if (node.kind === "PropertyUserAction")
|
|
33
|
+
return `PropertyUserAction:${node.UserActionType}`;
|
|
34
|
+
return `${node.kind}`;
|
|
35
|
+
}
|
|
36
|
+
async function analyzeOneRecording(axtrPath, fileName) {
|
|
37
|
+
const bytes = await readFile(axtrPath);
|
|
38
|
+
const extracted = await extractAxtr(bytes);
|
|
39
|
+
const recording = parseRecordingXml(extracted.recordingXml);
|
|
40
|
+
const scrubbed = scrubRecording(recording, fileName);
|
|
41
|
+
const script = normalizeRecording(scrubbed.recording);
|
|
42
|
+
const bpm = await parseBpmPackage(extracted.bpmPackageBytes);
|
|
43
|
+
const summary = {
|
|
44
|
+
file: fileName,
|
|
45
|
+
name: script.name,
|
|
46
|
+
canonicalId: script.canonicalId,
|
|
47
|
+
stepCount: script.steps.length,
|
|
48
|
+
steps: script.steps.map((step) => {
|
|
49
|
+
const hint = mapStep(step);
|
|
50
|
+
return {
|
|
51
|
+
sequence: step.sequence,
|
|
52
|
+
kind: step.node.kind,
|
|
53
|
+
formName: step.formName,
|
|
54
|
+
description: step.description,
|
|
55
|
+
hintTool: hint?.tool ?? null,
|
|
56
|
+
hintConfidence: hint?.confidence ?? null,
|
|
57
|
+
hintNote: hint?.note ?? null,
|
|
58
|
+
};
|
|
59
|
+
}),
|
|
60
|
+
requiredPrivileges: bpm ? distinctPrivileges(bpm) : [],
|
|
61
|
+
};
|
|
62
|
+
return { summary, scrubbedValues: scrubbed.values, steps: script.steps };
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Locally parses and scrubs every `.axtr` file in `recordingsDir`, returning a structural-only
|
|
66
|
+
* report (safe to hand to an LLM) plus the full list of stripped literal values (never safe to
|
|
67
|
+
* hand to an LLM -- keep local, see the `analyze-library` CLI command's CSV output).
|
|
68
|
+
*/
|
|
69
|
+
export async function analyzeLibrary(recordingsDir) {
|
|
70
|
+
const entries = (await readdir(recordingsDir)).filter((f) => f.toLowerCase().endsWith(".axtr")).sort();
|
|
71
|
+
const recordings = [];
|
|
72
|
+
const parseErrors = [];
|
|
73
|
+
const scrubbedValues = [];
|
|
74
|
+
const nodeKindCounts = {};
|
|
75
|
+
const hintConfidenceCounts = { high: 0, medium: 0, low: 0, unmapped: 0 };
|
|
76
|
+
const unknownXsiTypes = new Map();
|
|
77
|
+
const lowConfidenceGaps = new Map();
|
|
78
|
+
const privilegeSet = new Set();
|
|
79
|
+
for (const fileName of entries) {
|
|
80
|
+
const axtrPath = path.join(recordingsDir, fileName);
|
|
81
|
+
try {
|
|
82
|
+
const { summary, scrubbedValues: values, steps } = await analyzeOneRecording(axtrPath, fileName);
|
|
83
|
+
recordings.push(summary);
|
|
84
|
+
scrubbedValues.push(...values);
|
|
85
|
+
for (const p of summary.requiredPrivileges)
|
|
86
|
+
privilegeSet.add(p);
|
|
87
|
+
for (const step of steps) {
|
|
88
|
+
nodeKindCounts[step.node.kind] = (nodeKindCounts[step.node.kind] ?? 0) + 1;
|
|
89
|
+
const hint = mapStep(step);
|
|
90
|
+
if (!hint) {
|
|
91
|
+
hintConfidenceCounts.unmapped += 1;
|
|
92
|
+
bumpBucket(lowConfidenceGaps, gapKeyFor(step), fileName);
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
hintConfidenceCounts[hint.confidence] += 1;
|
|
96
|
+
if (hint.confidence === "low")
|
|
97
|
+
bumpBucket(lowConfidenceGaps, gapKeyFor(step), fileName);
|
|
98
|
+
}
|
|
99
|
+
if (step.node.kind === "Unknown") {
|
|
100
|
+
bumpBucket(unknownXsiTypes, step.node.xsiType, fileName);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
catch (err) {
|
|
105
|
+
parseErrors.push({ file: fileName, error: err instanceof Error ? err.message : String(err) });
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
const report = {
|
|
109
|
+
generatedAt: new Date().toISOString(),
|
|
110
|
+
recordingsAnalyzed: recordings.length,
|
|
111
|
+
parseErrors,
|
|
112
|
+
recordings,
|
|
113
|
+
aggregate: {
|
|
114
|
+
nodeKindCounts,
|
|
115
|
+
hintConfidenceCounts,
|
|
116
|
+
unknownXsiTypes: [...unknownXsiTypes.values()].sort((a, b) => b.count - a.count),
|
|
117
|
+
lowConfidenceGaps: [...lowConfidenceGaps.values()].sort((a, b) => b.count - a.count),
|
|
118
|
+
distinctSecurityPrivileges: [...privilegeSet].sort(),
|
|
119
|
+
},
|
|
120
|
+
};
|
|
121
|
+
return { report, scrubbedValues };
|
|
122
|
+
}
|
package/dist/bpm.d.ts
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parses the nested BPMPackage.ax7bpm zip (itself a ZIP, confirmed via inspection) embedded in
|
|
3
|
+
* an .axtr package. Its `Recording/ProcessStep.xml` lists one <Task> per *form* visited (not
|
|
4
|
+
* per user action) with the full security role/duty/privilege matrix required to reach it, plus
|
|
5
|
+
* a Connections graph of form-to-form flow. `Recording/TaskStep.xml` was inspected too and found
|
|
6
|
+
* to duplicate the same generic ordinal descriptions already present in Recording.xml /
|
|
7
|
+
* recording_resource.xml -- it carries no additional information, so it is intentionally not parsed.
|
|
8
|
+
*
|
|
9
|
+
* No authored "expected outcome" / assertion data exists anywhere in this package.
|
|
10
|
+
*/
|
|
11
|
+
export interface BpmSecurityRequirement {
|
|
12
|
+
roleName: string;
|
|
13
|
+
roleDisplayName: string;
|
|
14
|
+
dutyName: string;
|
|
15
|
+
dutyDisplayName: string;
|
|
16
|
+
privilegeName: string;
|
|
17
|
+
privilegeDisplayName: string;
|
|
18
|
+
accessLevel: string;
|
|
19
|
+
isValidRole: boolean;
|
|
20
|
+
}
|
|
21
|
+
export interface BpmTask {
|
|
22
|
+
id: string;
|
|
23
|
+
aotName: string;
|
|
24
|
+
objectType: string;
|
|
25
|
+
configurationKey: string;
|
|
26
|
+
layer: string;
|
|
27
|
+
model: string;
|
|
28
|
+
sequence: number;
|
|
29
|
+
/** Every role/duty/privilege combination known to grant access to this form. */
|
|
30
|
+
security: BpmSecurityRequirement[];
|
|
31
|
+
/** Task Ids this task flows into. */
|
|
32
|
+
connectsTo: string[];
|
|
33
|
+
}
|
|
34
|
+
export interface BpmPackage {
|
|
35
|
+
tasks: BpmTask[];
|
|
36
|
+
}
|
|
37
|
+
/** Parses the nested BPMPackage.ax7bpm bytes extracted from a .axtr file. Returns null if absent. */
|
|
38
|
+
export declare function parseBpmPackage(bytes: Buffer | null): Promise<BpmPackage | null>;
|
|
39
|
+
/** Every distinct privilege that appears across all tasks -- a starting point for a security preflight check. */
|
|
40
|
+
export declare function distinctPrivileges(pkg: BpmPackage): string[];
|