plain-forge 1.0.16 → 1.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/forge/rules/examples/integration-embedded-testing/run_conformance_tests_java.sh +14 -3
- package/forge/rules/integration-embedded-testing.md +9 -7
- package/forge/rules/integration-embedded.md +4 -4
- package/forge/skills/implement-conformance-testing-script/SKILL.md +56 -43
- package/forge/skills/implement-conformance-testing-script/assets/run_conformance_tests_cypress.ps1 +34 -21
- package/forge/skills/implement-conformance-testing-script/assets/run_conformance_tests_golang.ps1 +16 -5
- package/forge/skills/implement-conformance-testing-script/assets/run_conformance_tests_java.sh +22 -11
- package/forge/skills/implement-conformance-testing-script/assets/run_conformance_tests_python.ps1 +13 -2
- package/forge/skills/implement-conformance-testing-script/assets/run_conformance_tests_python.sh +16 -6
- package/forge/skills/implement-prepare-environment-script/SKILL.md +23 -18
- package/forge/skills/implement-prepare-environment-script/assets/prepare_environment_java.sh +8 -4
- package/forge/skills/implement-prepare-environment-script/assets/prepare_environment_python.sh +8 -4
- package/forge/skills/implement-unit-testing-script/SKILL.md +14 -11
- package/forge/skills/implement-unit-testing-script/assets/run_unittests_flutter.ps1 +6 -1
- package/forge/skills/implement-unit-testing-script/assets/run_unittests_golang.ps1 +6 -1
- package/forge/skills/implement-unit-testing-script/assets/run_unittests_java.sh +6 -3
- package/forge/skills/implement-unit-testing-script/assets/run_unittests_python.ps1 +6 -1
- package/forge/skills/implement-unit-testing-script/assets/run_unittests_python.sh +8 -4
- package/forge/skills/implement-unit-testing-script/assets/run_unittests_react.ps1 +6 -2
- package/forge/skills/init-config-file/SKILL.md +5 -5
- package/forge/skills/load-plain-reference/SKILL.md +97 -42
- package/forge/skills/plain-healthcheck/SKILL.md +1 -1
- package/forge/skills/run-codeplain/SKILL.md +12 -3
- package/package.json +1 -1
|
@@ -21,7 +21,18 @@ fi
|
|
|
21
21
|
|
|
22
22
|
current_dir=$(pwd)
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
# Resolve the conformance tests folder to an absolute path ($2 may be relative
|
|
25
|
+
# on older renderers; newer renderers pass it as an absolute path).
|
|
26
|
+
case "$2" in
|
|
27
|
+
/*) CONFORMANCE_TESTS_SOURCE="$2" ;;
|
|
28
|
+
*) CONFORMANCE_TESTS_SOURCE="$current_dir/$2" ;;
|
|
29
|
+
esac
|
|
30
|
+
|
|
31
|
+
# Scratch folder lives in the system temp directory so no build debris is left
|
|
32
|
+
# inside the project. It is removed again when the script exits.
|
|
33
|
+
CONFORMANCE_TESTS_FOLDER="/tmp/java_conformance"
|
|
34
|
+
|
|
35
|
+
trap 'rm -rf "$CONFORMANCE_TESTS_FOLDER"' EXIT
|
|
25
36
|
|
|
26
37
|
cd "$current_dir" 2>/dev/null
|
|
27
38
|
|
|
@@ -43,8 +54,8 @@ else
|
|
|
43
54
|
mkdir -p $CONFORMANCE_TESTS_FOLDER
|
|
44
55
|
fi
|
|
45
56
|
|
|
46
|
-
printf "Copying all files and folders from $
|
|
47
|
-
cp -R $
|
|
57
|
+
printf "Copying all files and folders from $CONFORMANCE_TESTS_SOURCE to $CONFORMANCE_TESTS_FOLDER...\n"
|
|
58
|
+
cp -R "$CONFORMANCE_TESTS_SOURCE"/* $CONFORMANCE_TESTS_FOLDER
|
|
48
59
|
|
|
49
60
|
# Move to the subfolder
|
|
50
61
|
printf "Moving to $CONFORMANCE_TESTS_FOLDER...\n"
|
|
@@ -10,13 +10,13 @@ When an embedded integration ships its three `test_scripts/` (prepare-environmen
|
|
|
10
10
|
|
|
11
11
|
## Staging model (read this first)
|
|
12
12
|
|
|
13
|
-
The three scripts do **not** all stage into the same place. Embedded integrations need a runnable host project, so the prepare and unit-test scripts operate **inside the host codebase itself**; only the conformance script uses a
|
|
13
|
+
The three scripts do **not** all stage into the same place. Embedded integrations need a runnable host project, so the prepare and unit-test scripts operate **inside the host codebase itself**; only the conformance script uses a scratch folder in the system temp directory (`/tmp/<lang>_conformance/`) because the conformance suite lives in a separate project.
|
|
14
14
|
|
|
15
15
|
| Script | Where it operates | What it does to the host source tree |
|
|
16
16
|
|--------|-------------------|---------------------------------------|
|
|
17
17
|
| `prepare_environment_<lang>` | Inside the host codebase | Copies `$1` into the module's package path under the host's source tree, then compiles / installs the host project |
|
|
18
18
|
| `run_unittests_<lang>` | Inside the host codebase | Same copy as above (self-contained), then runs the module's unit tests + lint inside the host |
|
|
19
|
-
| `run_conformance_tests_<lang>` |
|
|
19
|
+
| `run_conformance_tests_<lang>` | `/tmp/<lang>_conformance/` (system temp) | Copies `$2` (the conformance-tests folder) into the scratch directory and runs the conformance suite there, which depends on the host build that `prepare_environment` already installed |
|
|
20
20
|
|
|
21
21
|
This deliberately writes into the host's `src/main/...` and `src/test/...` (or the language equivalent). Two consequences flow from that:
|
|
22
22
|
|
|
@@ -127,11 +127,13 @@ All three scripts are invoked by the `codeplain` renderer with positional argume
|
|
|
127
127
|
- `2` — missing or inaccessible input folder
|
|
128
128
|
- `69` — unrecoverable environment failure (missing toolchain, cannot enter working folder, install failed)
|
|
129
129
|
- Any other non-zero code — propagated verbatim from the underlying build / test tool
|
|
130
|
+
|
|
131
|
+
The renderer passes `$1` (and `$2` for the conformance runner) as **absolute paths** (older renderer versions passed relative ones). Never build a derived path by concatenating a raw argument (`.tmp/$1`, `<lang>_$1`, `$current_dir/$2`) — with an absolute argument that produces a doubled path (`/project//abs/path/...`). Use the argument as-is where a path is needed, take its `basename` when naming a scratch folder, and resolve `$2` to absolute (if it isn't already) before any `cd`
|
|
130
132
|
3. **Echo the toolchain home at the top** — the first thing a developer checks when CI breaks. For Java: `JAVA_HOME`. For Python: the active interpreter (`python --version` / `which python`). For Node: `node --version`. For Go: `go env GOROOT`. Pick the variable whose value most often explains "why does it work locally but not on the CI runner?"
|
|
131
133
|
4. **Respect `VERBOSE=1`.** Gate chatty diagnostic prints behind `if [ "${VERBOSE:-}" = "1" ]` (and the PowerShell equivalent). Errors and key step markers print unconditionally
|
|
132
134
|
5. **Resolve paths relative to the script, not `$PWD`.** Use `"$(cd "$(dirname "$0")/<relative-path-to-anchor>" && pwd)"`. Hard-coded `../../` chains break the moment the renderer's `cwd` changes
|
|
133
135
|
6. **Create destination directories before copying.** `mkdir -p` before any `cp -R` / `rsync` / `robocopy` — most copy commands do not create intermediate directories and fail silently in some shells
|
|
134
|
-
7. **Scope destructive operations narrowly.** Any `rm -rf` (or `Remove-Item -Recurse -Force`) targets only the module's own package path inside the working folder — never the host's `src/`, `target/`, `node_modules/`, `build/`, or `dist/` at the project root. Only `prepare_environment` owns the build-output directory's lifecycle for its
|
|
136
|
+
7. **Scope destructive operations narrowly.** Any `rm -rf` (or `Remove-Item -Recurse -Force`) targets only the module's own package path inside the working folder — never the host's `src/`, `target/`, `node_modules/`, `build/`, or `dist/` at the project root. Only `prepare_environment` owns the build-output directory's lifecycle for its system-temp working folder
|
|
135
137
|
8. **Print where you are before you `cd`.** `echo "Moving to: $DIR"` saves an hour of debugging when paths are wrong. PowerShell: `Write-Host "Moving to: $DIR"`
|
|
136
138
|
|
|
137
139
|
## 1. `prepare_environment_<lang>` — copy into the host, then compile
|
|
@@ -185,7 +187,7 @@ Hard rules:
|
|
|
185
187
|
- **Never use a "clean and test" shortcut** (`mvn clean test`, `npm run rebuild`, …) — the cleanup belongs in `prepare_environment` and would erase whatever it installed
|
|
186
188
|
- **Never run the full test suite** when scoping is possible. A per-module script that runs the entire host's tests wastes minutes on every render iteration
|
|
187
189
|
|
|
188
|
-
## 3. `run_conformance_tests_<lang>` — copy into
|
|
190
|
+
## 3. `run_conformance_tests_<lang>` — copy into the system temp directory, then run the external suite
|
|
189
191
|
|
|
190
192
|
Receives two positional arguments: the renderer's build output folder (`$1`) and the conformance-tests folder (`$2`).
|
|
191
193
|
|
|
@@ -193,8 +195,8 @@ Purpose: run the conformance suite in `$2` against the build that `prepare_envir
|
|
|
193
195
|
|
|
194
196
|
Required steps:
|
|
195
197
|
|
|
196
|
-
1. Validate `$1` and `$2`. **Only `$2` is actually used by the script body today** — but keep `$1` in the signature; the renderer passes both positionally
|
|
197
|
-
2. Stage `$2/*` into a scratch directory
|
|
198
|
+
1. Validate `$1` and `$2`. **Only `$2` is actually used by the script body today** — but keep `$1` in the signature; the renderer passes both positionally. Resolve `$2` to an absolute path if it isn't one already (newer renderers pass it absolute; older ones passed it relative to the invocation directory)
|
|
199
|
+
2. Stage `$2/*` into a scratch directory in the system temp directory (`/tmp/<lang>_conformance/` in Bash, the `GetTempPath()` equivalent in PowerShell). Wipe it first if it exists, and remove it again on exit (`trap 'rm -rf "$DIR"' EXIT` / `finally`). Use a deletion form that handles hidden files and odd shells safely:
|
|
198
200
|
- Bash: `find "$DIR" -mindepth 1 -exec rm -rf {} +` (safer than `rm -rf "$DIR"/*`)
|
|
199
201
|
- PowerShell: `Remove-Item -Recurse -Force "$DIR\*"` after confirming the path exists
|
|
200
202
|
3. `cd` into the scratch directory
|
|
@@ -271,7 +273,7 @@ A working three-script set for a Java / Maven embedded integration lives under [
|
|
|
271
273
|
|
|
272
274
|
- [`prepare_environment_java.sh`](examples/integration-embedded-testing/prepare_environment_java.sh) — copies `$1` into the host's source tree, cleans `target/`, runs `mvn clean install -DskipTests`
|
|
273
275
|
- [`run_unittests_java.sh`](examples/integration-embedded-testing/run_unittests_java.sh) — re-stages into the host and runs `mvn test -Dtest='<module-pkg>.**.*Test' checkstyle:check`
|
|
274
|
-
- [`run_conformance_tests_java.sh`](examples/integration-embedded-testing/run_conformance_tests_java.sh) — copies `$2` into
|
|
276
|
+
- [`run_conformance_tests_java.sh`](examples/integration-embedded-testing/run_conformance_tests_java.sh) — copies `$2` into `/tmp/java_conformance/` (system temp, removed again on exit), builds the conformance project, runs the suite, and parses the Surefire summary line per the strict pass criteria above
|
|
275
277
|
|
|
276
278
|
Use them as a template when adding a new embedded integration in Java / Maven — copy, search-and-replace only the package segment and the `-Dtest` filter, then wire into `config.yaml`. For other languages (Python, Node.js, Go, Rust, …), the `implement-*-testing-script` skills generate the equivalent three-script set following the same contract.
|
|
277
279
|
|
|
@@ -10,7 +10,7 @@ When an integration `.plain` module is **embedded** — meaning the generated co
|
|
|
10
10
|
|
|
11
11
|
Embedded means: the host codebase already exists, has its own language / framework / dependency manager / packaging layout, and the integration must conform to all of that without negotiation.
|
|
12
12
|
|
|
13
|
-
> **For test-script authoring**, also follow [`integration-embedded-testing.md`](integration-embedded-testing.md). It defines the per-script contract (`prepare_environment_<lang>`, `run_unittests_<lang>`, `run_conformance_tests_<lang>`) — staging into the host vs
|
|
13
|
+
> **For test-script authoring**, also follow [`integration-embedded-testing.md`](integration-embedded-testing.md). It defines the per-script contract (`prepare_environment_<lang>`, `run_unittests_<lang>`, `run_conformance_tests_<lang>`) — staging into the host vs the system temp directory, arg validation, exit codes, output parsing, the three `***implementation reqs***` entries the spec must declare so the scripts can be generated. This file (`integration-embedded.md`) only summarizes the test-script wiring; the testing rule is the source of truth.
|
|
14
14
|
|
|
15
15
|
## The host codebase dictates the tech stack (hard rule)
|
|
16
16
|
|
|
@@ -119,7 +119,7 @@ The renderer reads the directives from the spec and the shapes from the linked s
|
|
|
119
119
|
|
|
120
120
|
## Test-script wiring — copy into the host, run tests there
|
|
121
121
|
|
|
122
|
-
Embedded integrations are tested **inside the host codebase itself**. The prepare and unit-test scripts copy the renderer's output (`$1`, i.e. `plain_modules/<module>/`) into the host's source tree at the module's package path, then compile / test the host project in place. Only the conformance script uses a
|
|
122
|
+
Embedded integrations are tested **inside the host codebase itself**. The prepare and unit-test scripts copy the renderer's output (`$1`, i.e. `plain_modules/<module>/`) into the host's source tree at the module's package path, then compile / test the host project in place. Only the conformance script uses a scratch folder in the system temp directory (`/tmp/<lang>_conformance/`), because the conformance suite is a separate project that consumes the host build as a dependency.
|
|
123
123
|
|
|
124
124
|
This matters because the integration's generated code references host symbols by their full import path (e.g. `from host_project.integrations.base import IntegrationContract`). Those imports only resolve cleanly when the test process is rooted in the host's package layout — anything else creates path edge cases that bite later in conformance failures.
|
|
125
125
|
|
|
@@ -127,7 +127,7 @@ See [`integration-embedded-testing.md`](integration-embedded-testing.md) for the
|
|
|
127
127
|
|
|
128
128
|
- **`prepare_environment_<lang>`** copies `$1` into the host's source tree at the module's package path, cleans the host's build-output directory, then runs the host's install / build (e.g. `mvn clean install -DskipTests`). The conformance suite later resolves the integration from the host's local dependency cache
|
|
129
129
|
- **`run_unittests_<lang>`** repeats the same copy into the host (self-contained — must work without `prepare_environment` having run first), then runs the module's unit tests + lint scoped to the module's package
|
|
130
|
-
- **`run_conformance_tests_<lang>`** copies `$2` (the conformance-tests folder) into
|
|
130
|
+
- **`run_conformance_tests_<lang>`** copies `$2` (the conformance-tests folder) into `/tmp/<lang>_conformance/` (system temp), `cd`s in, builds the conformance project, and runs it against the build that `prepare_environment` already installed into the host
|
|
131
131
|
|
|
132
132
|
### Invariants the scripts must enforce
|
|
133
133
|
|
|
@@ -150,7 +150,7 @@ Before declaring an embedded integration done, in addition to the shared checkli
|
|
|
150
150
|
- [ ] Host-package version pins are copied into `***implementation reqs***`
|
|
151
151
|
- [ ] `prepare_environment` copies `$1` into the host's source tree at the module's package path, cleans the host's build-output directory, and runs the host's install / build so the conformance suite can resolve the integration from the local dependency cache
|
|
152
152
|
- [ ] `run_unittests` runs the same copy-into-host sequence (self-contained — does not depend on `prepare_environment` having run) and invokes the host's test runner scoped to the module's package
|
|
153
|
-
- [ ] `run_conformance_tests` copies `$2` into
|
|
153
|
+
- [ ] `run_conformance_tests` copies `$2` into `/tmp/<lang>_conformance/` (system temp), `cd`s in, builds the conformance project, and runs it against the host build that `prepare_environment` already installed
|
|
154
154
|
- [ ] Host codebase root is read from a named env var (default value documented in each script's usage) — never hardcoded; the env var name is captured in the integration's configuration concept
|
|
155
155
|
- [ ] `***implementation reqs***` declares **everything about `:UnitTests:`** — integration source path, `:UnitTests:` source path, `:UnitTests:` package, framework + conventions, lint / static-analysis gate — per [`integration-embedded-testing.md`](integration-embedded-testing.md)
|
|
156
156
|
- [ ] `***test reqs***` declares **everything about `:ConformanceTests:`** — source location, framework + execution command, package, mocking / network policy, pass criteria, build / install needs — per [`integration-embedded-testing.md`](integration-embedded-testing.md)
|
|
@@ -27,8 +27,8 @@ Read both before writing anything — every script you produce must be a faithfu
|
|
|
27
27
|
|
|
28
28
|
A conformance script is structurally very close to a unit-test script (see the sibling skill [`implement-unit-testing-script`](../implement-unit-testing-script/SKILL.md)) but with two important differences:
|
|
29
29
|
|
|
30
|
-
1. **Two positional arguments instead of one.** A conformance script takes both the **build folder** (source under test) and a **separate conformance tests folder** (the tests to execute against that build).
|
|
31
|
-
2. **Tests are loaded from outside the working folder.** The build is staged into
|
|
30
|
+
1. **Two positional arguments instead of one.** A conformance script takes both the **build folder** (source under test) and a **separate conformance tests folder** (the tests to execute against that build). Both arrive as **absolute paths** from current renderers (older renderer versions passed relative ones) — the script must work with both.
|
|
31
|
+
2. **Tests are loaded from outside the working folder.** The build is staged into the working folder `<temp>/<lang>_<basename>` (system temp directory + language id + basename of `$1` — see [Conventions](#conventions)) and the script `cd`s into it, but the test command is pointed at the conformance tests folder, resolved to an **absolute path** up front (`$CONFORMANCE_TESTS_FOLDER` below). Tests are never copied into the staging area.
|
|
32
32
|
|
|
33
33
|
Everything else — toolchain check, build staging, dependency isolation, exit codes — is the same.
|
|
34
34
|
|
|
@@ -50,9 +50,11 @@ That per-spec invocation pattern is what makes the install step expensive. A nai
|
|
|
50
50
|
The two variants are a direct response to this:
|
|
51
51
|
|
|
52
52
|
- **Install-inline** is correct only when N is small (a few specs) or dependencies are cheap. It is self-contained: stage, install, run, repeat from scratch every invocation.
|
|
53
|
-
- **Activate-only** is the production answer. [`prepare_environment_<lang>`](../implement-prepare-environment-script/SKILL.md) runs **once per render** and pays the install cost a single time, populating
|
|
53
|
+
- **Activate-only** is the production answer. [`prepare_environment_<lang>`](../implement-prepare-environment-script/SKILL.md) runs **once per render** and pays the install cost a single time, populating `<temp>/<lang>_<basename>/` with the warmed environment. Each of the N conformance invocations then just **attaches** to that working folder and runs the tests — no install, no compile, just activate-and-go.
|
|
54
54
|
|
|
55
|
-
**
|
|
55
|
+
**The cleanup contract follows from this split:** the **install-inline** variant owns its working folder and removes it on exit (`trap 'rm -rf "$WORKING_FOLDER"' EXIT` / `Remove-Item` in `finally`). The **activate-only** variant **never deletes the working folder** — not on success, not on failure. Prepare owns that folder's lifecycle; if activate-only deleted it, the second of the N invocations would fail the "prepared environment missing" guard and the amortization would be destroyed.
|
|
56
|
+
|
|
57
|
+
**Why picking the right variant matters:** if you emit the install-inline variant alongside an existing prepare script, prepare's work is wiped (by the script's staging `rm -rf` and its exit cleanup) or duplicated (by re-running install) on every run — defeating prepare's whole purpose. Conversely, emitting activate-only without a prepare script means the "verify prepared environment" check fails on every run because nothing has populated the working folder. See [Anti-Patterns](#anti-patterns).
|
|
56
58
|
|
|
57
59
|
## Pick the Shell First
|
|
58
60
|
|
|
@@ -73,36 +75,42 @@ Steps **1–3** and **step 8** are identical in both variants. Steps **4–7** d
|
|
|
73
75
|
|
|
74
76
|
1. **Toolchain check.** Verify that the required language runtime / build tool (and the required version, if any) is installed. If not, print an error and exit with code `69`.
|
|
75
77
|
2. **Argument validation.** Require **two** positional arguments: `<build_folder>` and `<conformance_tests_folder>`. If either is missing, print usage and exit with code `69`.
|
|
76
|
-
3. **
|
|
78
|
+
3. **Resolve the conformance tests folder to an absolute path.** Current renderers pass `$2` as an absolute path; older ones passed it relative to the invocation directory. Resolve it once, up front, into a variable the test command in step 8 uses:
|
|
79
|
+
- Bash: `case "$2" in /*) CONFORMANCE_TESTS_FOLDER="$2" ;; *) CONFORMANCE_TESTS_FOLDER="$(pwd)/$2" ;; esac`
|
|
80
|
+
- PowerShell: `if (-not [System.IO.Path]::IsPathRooted($ConformanceTestsFolder)) { $ConformanceTestsFolder = Join-Path (Get-Location) $ConformanceTestsFolder }`
|
|
81
|
+
|
|
82
|
+
Never concatenate `$current_dir/$2` at the point of use — with an absolute `$2` that produces a doubled path (`/project//abs/path/...`), the classic symptom being `ImportError: Start directory is not importable` pointing at a path that contains the project root twice.
|
|
77
83
|
|
|
78
84
|
### Steps 4–7 — install-inline variant (no prepare script)
|
|
79
85
|
|
|
80
|
-
4. **Working directory setup.** Define a working folder at
|
|
86
|
+
4. **Working directory setup.** Define a working folder at `<temp>/<lang>_<basename>` (`/tmp/<lang>_$(basename "$1")` in Bash, `Join-Path ([System.IO.Path]::GetTempPath()) "<lang>_$(Split-Path $BuildFolder -Leaf)"` in PowerShell). Wipe it (`rm -rf` / `Remove-Item -Recurse -Force`) and recreate it. Register cleanup so the folder is removed when the script exits — `trap 'rm -rf "$WORKING_FOLDER"' EXIT` in Bash, `Remove-Item -Recurse -Force` in the `finally` block in PowerShell. This folder — and **only** this folder — is where every subsequent write must land.
|
|
81
87
|
5. **Copy the build.** Recursively copy everything from `<build_folder>` (`$1`) into the working folder. **Do not** copy the conformance tests — they stay where they are. After this step both `$1` (build folder) and `$2` (conformance tests folder) are treated as **read-only** for the rest of the script.
|
|
82
|
-
6. **Enter the working directory.** `cd` / `Set-Location` into
|
|
83
|
-
7. **Install dependencies into an isolated environment inside
|
|
88
|
+
6. **Enter the working directory.** `cd` / `Set-Location` into `<temp>/<lang>_<basename>`. If that fails, exit with code `69`. All remaining steps run from inside the working folder; they must never write back to `$1` or `$2`.
|
|
89
|
+
7. **Install dependencies into an isolated environment inside `<temp>/<lang>_<basename>`.** Set up a per-working-folder dependency location (a Python venv at `./.venv`, a local `./node_modules`, a project-scoped Maven repo at `./.m2`, etc.) and install/resolve all dependencies into it. **Never** install into the source build folder (`$1`), the conformance tests folder (`$2`), the user's global cache (`~/.m2`, system-wide `pip`, `~/.cargo`, `~/.npm`, ...), or anywhere outside `<temp>/<lang>_<basename>`. If the install command fails, propagate its exit code immediately and **do not** proceed to step 8. See [Dependency isolation (install-inline)](#dependency-isolation-install-inline).
|
|
84
90
|
|
|
85
91
|
### Steps 4–7 — activate-only variant (prepare script exists)
|
|
86
92
|
|
|
87
93
|
4. **Verify the prepared environment.** Both:
|
|
88
|
-
- Check that the working folder
|
|
94
|
+
- Check that the working folder `<temp>/<lang>_<basename>` exists — derived **exactly** the way prepare derives it (system temp dir + `<lang>_` + basename of `$1`).
|
|
89
95
|
- Check that the language's isolation location inside it exists (e.g. `.venv/bin/activate` for Python, `.m2/` for Java, `node_modules/` for Node, `.gocache/` for Go, `.cargo/` for Rust).
|
|
90
96
|
|
|
91
97
|
If either check fails, print a helpful error (`"Error: prepared environment missing — did you run prepare_environment_<lang>.<sh|ps1> first?"`) and exit `69`. **Do not silently fall back to creating it inline** — that would mask a real misconfiguration and turn this script into the install-inline variant in disguise. After this step both `$1` and `$2` are treated as **read-only** for the rest of the script.
|
|
92
|
-
|
|
98
|
+
|
|
99
|
+
**No cleanup registration in this variant.** The working folder belongs to prepare; this script must never delete it — not via a `trap`, not in a `finally` block, not on failure. See [the cleanup contract](#variant-decision-install-inline-vs-activate-only).
|
|
100
|
+
5. **Enter the working directory.** `cd` / `Set-Location` into `<temp>/<lang>_<basename>`. If that fails, exit `69`. All remaining steps run from inside the working folder; they must never write back to `$1` or `$2`.
|
|
93
101
|
6. **Activate the prepared dependency environment.** Per-language:
|
|
94
102
|
- Python: `source .venv/bin/activate` (must succeed; exit `69` on failure).
|
|
95
103
|
- Java: set `MAVEN_LOCAL_REPO="$(pwd)/.m2"` so it can be passed as `-Dmaven.repo.local="$MAVEN_LOCAL_REPO"` to `mvn` in step 8.
|
|
96
104
|
- Node.js / Go / Rust: nothing to activate explicitly — the test command in step 8 just needs to receive the same isolation flag/env var that prepare used (`./node_modules` is found by default; pass `GOMODCACHE` / `CARGO_HOME`).
|
|
97
105
|
|
|
98
|
-
Activation is **always relative to the working folder**, never to `$1` or `$2` — prepare populated
|
|
106
|
+
Activation is **always relative to the working folder**, never to `$1` or `$2` — prepare populated `<temp>/<lang>_<basename>/...`, and that is the only place to attach to.
|
|
99
107
|
7. *(There is no step 7 in this variant — install was prepare's job. Skip straight to step 8.)*
|
|
100
108
|
|
|
101
109
|
### Common step 8 (both variants)
|
|
102
110
|
|
|
103
|
-
8. **Run the conformance tests.** Invoke the language's standard test command, **pointed at `$
|
|
111
|
+
8. **Run the conformance tests.** Invoke the language's standard test command, **pointed at `$CONFORMANCE_TESTS_FOLDER`** (the absolute path resolved in step 3). The script's final exit code is whatever the test command returns — except for the "no tests discovered" case below.
|
|
104
112
|
|
|
105
|
-
The test command is **read-only** with respect to `$
|
|
113
|
+
The test command is **read-only** with respect to `$CONFORMANCE_TESTS_FOLDER`. It loads test files from there, but any artifacts the runner produces (caches, JUnit XML, coverage reports, compiled test classes, etc.) must land inside `<temp>/<lang>_<basename>`, not next to the test files. If your chosen runner defaults to writing output beside the tests, pass an explicit output-directory flag pointing inside the working folder (e.g. `pytest --basetemp=./.pytest_tmp`, `jest --cacheDirectory=./.jest_cache`, Maven `target/` under the working folder via `mvn -f "$CONFORMANCE_TESTS_FOLDER/pom.xml" -Dproject.build.directory="$(pwd)/target" test`).
|
|
106
114
|
|
|
107
115
|
### Read-only inputs — hard rule
|
|
108
116
|
|
|
@@ -110,15 +118,15 @@ A conformance script has **two** read-only inputs: the source build folder (`$1`
|
|
|
110
118
|
|
|
111
119
|
- install dependencies into `$1` or `$2` (no `pip install` inside `$1`/`$2`, no `npm install` inside them, no `mvn install` writing into them, no Cargo build artifacts ending up under them),
|
|
112
120
|
- write a virtualenv / `node_modules` / `.m2` / `.gocache` / `.cargo` directory inside `$1` or `$2`,
|
|
113
|
-
- run the test command with its `cwd` set to `$1` or `$2` (every test command runs from inside
|
|
121
|
+
- run the test command with its `cwd` set to `$1` or `$2` (every test command runs from inside `<temp>/<lang>_<basename>` after the `cd` in step 6 / activate-only step 5),
|
|
114
122
|
- create logs, caches, build outputs, JUnit XML, coverage reports, compiled test classes, or temp files inside `$1` or `$2`.
|
|
115
123
|
|
|
116
124
|
Why each input is read-only:
|
|
117
125
|
|
|
118
|
-
- **`$1` (build folder)** is shared with the renderer (`plain_modules/...` by default) and downstream tooling. Writing into it corrupts the renderer's view of "what was generated" and breaks subsequent renders. The whole point of staging into
|
|
126
|
+
- **`$1` (build folder)** is shared with the renderer (`plain_modules/...` by default) and downstream tooling. Writing into it corrupts the renderer's view of "what was generated" and breaks subsequent renders. The whole point of staging into the system temp directory is so the source folder stays a clean, reproducible artifact of the render.
|
|
119
127
|
- **`$2` (conformance tests folder)** is the user's authored test source — typically checked into version control. Writing into it pollutes the working tree, churns git status, and (with frameworks that auto-discover) can make subsequent runs pick up generated files as if they were tests.
|
|
120
128
|
|
|
121
|
-
If you find yourself about to issue any command whose `cwd` is `$1` or `$2`, or whose target path starts with `$1/` or `$2/`, **stop**. Either move the operation into
|
|
129
|
+
If you find yourself about to issue any command whose `cwd` is `$1` or `$2`, or whose target path starts with `$1/` or `$2/`, **stop**. Either move the operation into `<temp>/<lang>_<basename>`, or you're doing something the script must not do.
|
|
122
130
|
|
|
123
131
|
### "No tests discovered" detection
|
|
124
132
|
|
|
@@ -140,19 +148,20 @@ Shared across both shell flavors **and** both variants:
|
|
|
140
148
|
- `69` — unrecoverable invocation error: missing argument, missing toolchain, can't enter working folder, can't create venv (install-inline), or prepared environment missing/broken (activate-only). Matches the reference scripts' `UNRECOVERABLE_ERROR_EXIT_CODE`.
|
|
141
149
|
- `1` — "no tests discovered" guard tripped (see above).
|
|
142
150
|
- Any other non-zero code — propagated from the underlying test command.
|
|
143
|
-
- **Working folder naming:**
|
|
151
|
+
- **Working folder naming:** `<temp>/<lang>_<basename>` — the system temp directory (`/tmp` in Bash, `[System.IO.Path]::GetTempPath()` in PowerShell) plus a short identifier for the language (`java`, `python`, `node`, `go`, `rust`, ...) and the **basename** of the *first* argument (the build folder — `$(basename "$1")` / `Split-Path $BuildFolder -Leaf`), never the conformance tests folder and never the raw argument (it may be absolute; concatenating it produces broken doubled paths). All dependency installs, build outputs, caches, test runner artifacts, and the test invocation itself live inside this folder. Nothing the script does should touch `$1` after step 5 (install-inline) / step 4 (activate-only), or `$2` at any point.
|
|
152
|
+
- **Cleanup ownership:** the **install-inline** variant removes its working folder on exit, success and failure alike (`trap 'rm -rf "$WORKING_FOLDER"' EXIT` in Bash; `Remove-Item -Recurse -Force` in the PowerShell `finally` block, after `Pop-Location`). The **activate-only** variant **never deletes the working folder** — prepare owns its lifecycle and re-creates it once per render; deleting it here would break every subsequent per-spec invocation.
|
|
144
153
|
- **Logging — be as verbose as possible.** The script is the only thing the operator sees between a `codeplain` render and a green/red conformance result; when it fails, the only forensic evidence anyone has is its stdout/stderr. Treat the script like a production runbook, not a quiet helper. Concretely:
|
|
145
|
-
- **Announce every step before doing it**, with the exact value of every variable involved — the resolved build folder, the resolved conformance-tests folder, the
|
|
154
|
+
- **Announce every step before doing it**, with the exact value of every variable involved — the resolved build folder, the resolved conformance-tests folder, the working folder, the language, the toolchain version detected, the isolation root being activated (install-inline) or verified (activate-only), the variant in use, and the test command about to run. "Running tests" alone is useless; "Running `python -m unittest discover -b -s /abs/path/to/tests` from working folder `/tmp/python_build/` with venv activated at `./.venv` (Python 3.11.6)" is triage-ready.
|
|
146
155
|
- **Echo every non-trivial command before executing it.** In Bash, use `set -x` for the body of the script (or `echo "+ <cmd>"` immediately before each call); in PowerShell, set `$VerbosePreference = 'Continue'` and use `Write-Verbose` / `Write-Host` to print each command line with its arguments.
|
|
147
156
|
- **Print clear section banners.** Each pattern step gets a banner like `===== [6/8] Activate prepared environment =====` (activate-only) or `===== [7/8] Install dependencies =====` (install-inline) so a long log can be navigated by eye, **and** the banner names the variant so the operator immediately knows which branch is executing.
|
|
148
|
-
- **Print resolved absolute paths**, not just the relative names. Operators reading the log on a different machine need to know where the venv / `.m2` / `node_modules` / `.gocache` / `.cargo` actually landed, and
|
|
157
|
+
- **Print resolved absolute paths**, not just the relative names. Operators reading the log on a different machine need to know where the venv / `.m2` / `node_modules` / `.gocache` / `.cargo` actually landed, and what `$2` resolved to.
|
|
149
158
|
- **Print the toolchain's own `--version` output** (and the path it resolved from) during step 1, even on success. This is the single most useful piece of forensic data when conformance passes locally but fails in CI.
|
|
150
159
|
- **Surface the install / activation output verbatim** — do not pipe it to `/dev/null`, do not redirect to a log file inside the working folder. The operator must see every dependency-resolver line (install-inline) or every "venv missing" / "\.m2 missing" guard message (activate-only) in the script's own output stream. Drop the `VERBOSE`-gated wrapping from the Python reference — conformance output is where forensic data is most valuable.
|
|
151
|
-
- **Print which variant the script is** (install-inline vs. activate-only) and **why** — e.g. "activate-only variant: detected
|
|
152
|
-
- **On failure, print what was about to run before exiting** — the exact test command, the working directory, the
|
|
153
|
-
- **Print a final summary line** that names the variant, the test command, the exit code, the
|
|
160
|
+
- **Print which variant the script is** (install-inline vs. activate-only) and **why** — e.g. "activate-only variant: detected `/tmp/python_build/.venv/bin/activate` from a prior `prepare_environment_python.sh` run". When the verify step trips, the operator must see both the path that was checked and the expected source ("did you run `prepare_environment_python.sh <build>` first?").
|
|
161
|
+
- **On failure, print what was about to run before exiting** — the exact test command, the working directory, the resolved `$CONFORMANCE_TESTS_FOLDER`, and the relevant environment variables (`PYTHONPATH`, `NODE_PATH`, `GOMODCACHE`, `CARGO_HOME`, `MAVEN_OPTS`, `PATH`, plus `VIRTUAL_ENV` for Python).
|
|
162
|
+
- **Print a final summary line** that names the variant, the test command, the exit code, the resolved conformance-tests folder, and the working folder so the operator knows exactly what to re-run by hand if needed.
|
|
154
163
|
Verbosity is not noise here — a chatty script that documents itself in its own output is the difference between a 30-second triage and a 30-minute one. Never trade verbosity for terseness.
|
|
155
|
-
- **
|
|
164
|
+
- **Resolve `$2` before any `cd`.** This is the single most common bug in hand-written conformance scripts: using the conformance-tests argument after changing directory. Resolve it to an absolute path in step 3 (absolute → use as-is; relative → prefix the invocation directory) and use only the resolved variable afterwards. The resolved value must also appear in the verbose log (per the logging rule above) so misresolutions of `$2` are visible at a glance.
|
|
156
165
|
|
|
157
166
|
### Dependency isolation (install-inline)
|
|
158
167
|
|
|
@@ -160,17 +169,17 @@ This section applies to **install-inline scripts only.** For activate-only scrip
|
|
|
160
169
|
|
|
161
170
|
The dependency environment must live **inside** `$WORKING_FOLDER` so the test run can't be polluted by — or pollute — the user's global caches. Pick the most idiomatic isolation mechanism for the language:
|
|
162
171
|
|
|
163
|
-
| Language | Isolation mechanism | Install command (run inside `$WORKING_FOLDER`) | Test command (point at `$
|
|
172
|
+
| Language | Isolation mechanism | Install command (run inside `$WORKING_FOLDER`) | Test command (point at `$CONFORMANCE_TESTS_FOLDER`) |
|
|
164
173
|
|---|---|---|---|
|
|
165
|
-
| Python | `venv` at `./.venv` | `python3 -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt` | `python -m unittest discover -b -s "$
|
|
166
|
-
| Node.js | local `./node_modules` (default) | `npm ci` (preferred) or `npm install` | `npx jest --rootDir "$
|
|
167
|
-
| Java | project-scoped Maven repo at `./.m2` | `mvn -Dmaven.repo.local=./.m2 install -DskipTests` (build + install artifact so the test pom can resolve it) | `mvn -f "$
|
|
168
|
-
| Go | module cache at `./.gocache` | `GOMODCACHE="$PWD/.gocache" go mod download` (optional pre-warm) | `GOMODCACHE="$PWD/.gocache" go test "$
|
|
169
|
-
| Rust | cargo home at `./.cargo` | `CARGO_HOME="$PWD/.cargo" cargo fetch` (optional pre-warm) | `CARGO_HOME="$PWD/.cargo" cargo test --manifest-path "$
|
|
174
|
+
| Python | `venv` at `./.venv` | `python3 -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt` | `python -m unittest discover -b -s "$CONFORMANCE_TESTS_FOLDER"` (or `pytest "$CONFORMANCE_TESTS_FOLDER"`) |
|
|
175
|
+
| Node.js | local `./node_modules` (default) | `npm ci` (preferred) or `npm install` | `npx jest --rootDir "$CONFORMANCE_TESTS_FOLDER"` |
|
|
176
|
+
| Java | project-scoped Maven repo at `./.m2` | `mvn -Dmaven.repo.local=./.m2 install -DskipTests` (build + install artifact so the test pom can resolve it) | `mvn -f "$CONFORMANCE_TESTS_FOLDER/pom.xml" -Dmaven.repo.local="$(pwd)/.m2" test` |
|
|
177
|
+
| Go | module cache at `./.gocache` | `GOMODCACHE="$PWD/.gocache" go mod download` (optional pre-warm) | `GOMODCACHE="$PWD/.gocache" go test "$CONFORMANCE_TESTS_FOLDER/..."` |
|
|
178
|
+
| Rust | cargo home at `./.cargo` | `CARGO_HOME="$PWD/.cargo" cargo fetch` (optional pre-warm) | `CARGO_HOME="$PWD/.cargo" cargo test --manifest-path "$CONFORMANCE_TESTS_FOLDER/Cargo.toml"` |
|
|
170
179
|
|
|
171
180
|
Notes:
|
|
172
181
|
|
|
173
|
-
- **Every path in the install command and test command is relative to
|
|
182
|
+
- **Every path in the install command and test command is relative to `<temp>/<lang>_<basename>`.** That's why the script `cd`s into the working folder in step 6 — from that point on, `./.venv`, `./node_modules`, `./.m2`, etc. all resolve under `<temp>/<lang>_<basename>`, never under `$1` or `$2`.
|
|
174
183
|
- **Always pass the isolation flag/env var to both the install command and the test command.** They must agree on where deps live, otherwise the test command will silently fall back to the global cache **or** (worse) write into `$1` / `$2`.
|
|
175
184
|
- **Python is the only ecosystem where the venv is mandatory** to satisfy "into a virtual environment" literally. The others use language-native equivalents that achieve the same isolation.
|
|
176
185
|
- **Propagate the install exit code immediately.** In Bash: `<install cmd> || exit $?`. In PowerShell: check `$LASTEXITCODE` and `exit $LASTEXITCODE` if non-zero.
|
|
@@ -180,13 +189,15 @@ Notes:
|
|
|
180
189
|
|
|
181
190
|
This section applies to **activate-only scripts only.** The isolation location was created by prepare; conformance just needs to attach to it and pass the right flags to the test command.
|
|
182
191
|
|
|
183
|
-
|
|
192
|
+
In the table below, `$WORKING_FOLDER` is `<temp>/<lang>_<basename>` — e.g. `/tmp/python_$(basename "$1")` in Bash.
|
|
193
|
+
|
|
194
|
+
| Language | Verify exists in step 4 | Activate in step 6 | Test command in step 8 (point at `$CONFORMANCE_TESTS_FOLDER`) |
|
|
184
195
|
|---|---|---|---|
|
|
185
|
-
| Python |
|
|
186
|
-
| Node.js |
|
|
187
|
-
| Java |
|
|
188
|
-
| Go |
|
|
189
|
-
| Rust |
|
|
196
|
+
| Python | `$WORKING_FOLDER/.venv/bin/activate` | `source .venv/bin/activate` (after `cd`-ing into the working folder) | `python -m unittest discover -b -s "$CONFORMANCE_TESTS_FOLDER"` |
|
|
197
|
+
| Node.js | `$WORKING_FOLDER/node_modules/` | (nothing) | `npx jest --rootDir "$CONFORMANCE_TESTS_FOLDER"` |
|
|
198
|
+
| Java | `$WORKING_FOLDER/.m2/` | `MAVEN_LOCAL_REPO="$(pwd)/.m2"` | `mvn -f "$CONFORMANCE_TESTS_FOLDER/pom.xml" -Dmaven.repo.local="$MAVEN_LOCAL_REPO" test` |
|
|
199
|
+
| Go | `$WORKING_FOLDER/.gocache/` | `export GOMODCACHE="$(pwd)/.gocache"` | `go test "$CONFORMANCE_TESTS_FOLDER/..."` |
|
|
200
|
+
| Rust | `$WORKING_FOLDER/.cargo/` | `export CARGO_HOME="$(pwd)/.cargo"` | `cargo test --manifest-path "$CONFORMANCE_TESTS_FOLDER/Cargo.toml"` |
|
|
190
201
|
|
|
191
202
|
Notes:
|
|
192
203
|
|
|
@@ -209,7 +220,7 @@ Notes:
|
|
|
209
220
|
- **Exit codes:** use `exit 69` etc. (PowerShell honors them just like Bash).
|
|
210
221
|
- **Toolchain check:** prefer `Get-Command <tool> -ErrorAction SilentlyContinue` and, where a specific version is needed, parse the tool's `--version` output.
|
|
211
222
|
- **Filesystem:** use `Test-Path`, `Remove-Item -Recurse -Force`, `New-Item -ItemType Directory`, `Copy-Item -Recurse`, `Set-Location`. Quote paths to handle spaces.
|
|
212
|
-
- **
|
|
223
|
+
- **Resolve `$ConformanceTestsFolder` to absolute** with `[System.IO.Path]::IsPathRooted` + `Join-Path (Get-Location) ...` **before** any `Set-Location` call.
|
|
213
224
|
- **No `chmod` step needed.** If execution policy is likely to block the script, mention `Set-ExecutionPolicy -Scope CurrentUser RemoteSigned` to the user — don't bake it into the script.
|
|
214
225
|
|
|
215
226
|
## Workflow
|
|
@@ -229,19 +240,21 @@ Notes:
|
|
|
229
240
|
|
|
230
241
|
## Anti-Patterns
|
|
231
242
|
|
|
232
|
-
- **(Hard mistake) Don't install into, build into, or otherwise write to the source build folder (`$1`) or the conformance tests folder (`$2`).** Both arguments are read-only input. Every install, cache, build artifact, log, JUnit XML, coverage report, compiled test class, and temp file must land in
|
|
233
|
-
- **Don't
|
|
243
|
+
- **(Hard mistake) Don't install into, build into, or otherwise write to the source build folder (`$1`) or the conformance tests folder (`$2`).** Both arguments are read-only input. Every install, cache, build artifact, log, JUnit XML, coverage report, compiled test class, and temp file must land in `<temp>/<lang>_<basename>`. This includes never running `pip install`, `npm install`, `mvn install`, or `cargo build` with `$1` or `$2` as their `cwd` or target; never letting a venv / `node_modules` / `.m2` / `.gocache` / `.cargo` directory appear inside `$1` or `$2`; and never running the test command from inside either folder. The whole point of staging into the system temp directory is so the build folder remains a clean artifact of the render and the conformance tests folder remains a clean tree under the user's version control — writing to either one corrupts those guarantees.
|
|
244
|
+
- **(Hard mistake) Don't build any path by concatenating a raw argument.** `$1` and `$2` arrive as absolute paths from current renderers. Patterns like `WORKING_FOLDER=.tmp/$1`, `WORKING_FOLDER=<lang>_$1`, or `"$current_dir/$2"` silently produce doubled paths (`/project//abs/path/...`) — the classic symptom is `ImportError: Start directory is not importable` or "folder does not exist" pointing at a path that contains the project root twice. Always derive the working folder from `$(basename "$1")` / `Split-Path -Leaf`, and use the `$CONFORMANCE_TESTS_FOLDER` resolved in step 3.
|
|
245
|
+
- **(Hard mistake — activate-only) Don't delete the working folder.** No `trap 'rm -rf …' EXIT`, no `Remove-Item` in `finally`, no cleanup on failure. The working folder is prepare's deliverable and is attached to N times per render; deleting it after the first invocation makes every later invocation fail the "prepared environment missing" guard. Cleanup belongs to the install-inline variant only.
|
|
246
|
+
- **Don't emit the install-inline variant when a `prepare_environment_<lang>` script already exists.** The conformance script's staging `rm -rf` and exit cleanup will wipe everything prepare did, and the inline install will redo it from scratch on every run. Always run the [Variant decision](#variant-decision-install-inline-vs-activate-only) check first.
|
|
234
247
|
- **Don't emit the activate-only variant when no prepare script exists.** The "verify prepared environment" check will fail on every run because nothing has populated the working folder.
|
|
235
248
|
- **Don't silently fall back from activate-only to install-inline** when the prepared environment is missing. Exit `69` with a clear error so the misconfiguration is visible. Silent fallback hides the real bug and produces inconsistent behavior between runs.
|
|
236
|
-
- **Don't copy the conformance tests folder into
|
|
237
|
-
- **Don't compute the test path after `cd`.**
|
|
249
|
+
- **Don't copy the conformance tests folder into the working folder.** Only the build folder is staged (and only in install-inline). The test folder is read in place from `$CONFORMANCE_TESTS_FOLDER`.
|
|
250
|
+
- **Don't compute the test path after `cd`.** Resolve `$2` to an absolute path in step 3; otherwise a relative `$2` will be resolved against the working folder and silently miss the tests.
|
|
238
251
|
- **Don't skip the "no tests discovered" check.** A conformance suite that finds zero tests and exits `0` is the worst possible failure mode — it looks like success in CI.
|
|
239
252
|
- **Don't skip the toolchain check**, even when "everyone has it installed" — exit code `69` is what the calling system relies on to detect a missing runtime.
|
|
240
|
-
- **Don't reuse the source folder in place** (install-inline). Always copy into
|
|
253
|
+
- **Don't reuse the source folder in place** (install-inline). Always copy into `<temp>/<lang>_<basename>` first; the renderer relies on this isolation.
|
|
241
254
|
- **Don't change the exit-code contract.** Other parts of the system branch on `69` and `1` specifically — and these codes must be identical between the Bash and PowerShell variants.
|
|
242
255
|
- **Don't write a cross-shell hybrid** (e.g. a `.sh` that detects PowerShell, or vice versa). Ship one script per shell, named with the appropriate extension.
|
|
243
256
|
- **Don't install dependencies into the user's global location** (`~/.m2`, system-wide `pip`, `~/.cargo`, etc.) in the install-inline variant. Always isolate inside `$WORKING_FOLDER` so concurrent runs and other projects can't interfere.
|
|
244
257
|
- **Don't run the test command without first verifying the install / activation succeeded.** A failed install (or missing prepared env) followed by a "test" run produces misleading errors that look like test failures.
|
|
245
258
|
- **Don't forget to update `config.yaml`.** After creating the conformance test script, always add or update the `conformance-tests-script:` key in `config.yaml` to reference the new script. Without this entry, the `codeplain` renderer won't know where to find the conformance test script.
|
|
246
259
|
- **Don't forget to link the script as a resource in the base `.plain` files.** A script that is referenced only from `config.yaml` is invisible to the spec contract — the renderer treats `conformance-tests-script:` as a build-system pointer, not as part of the test-req contract the model reads. Use the `add-resource` skill to add a markdown link to the script from the `.plain` module that owns the conformance test reqs (see Workflow step 9). Omitting the linked resource means the model authors and reviews conformance-test code without ever seeing the actual runner it has to satisfy — and for activate-only scripts, the model also loses sight of which isolation paths must already be in place by the time tests run.
|
|
247
|
-
- **Don't write a terse script.** Silent steps, `>/dev/null` redirects on the install / activation output, missing `--version` prints, absent variant banners, an un-logged
|
|
260
|
+
- **Don't write a terse script.** Silent steps, `>/dev/null` redirects on the install / activation output, missing `--version` prints, absent variant banners, an un-logged resolved `$2`, and a missing final summary all make the script harder to debug than the tests it is running. Follow the *Logging — be as verbose as possible* rule under [Conventions](#conventions) literally: every step announces itself, every command is echoed, the variant is named in its banner, the resolved `$CONFORMANCE_TESTS_FOLDER` is printed, every failure prints what was about to run and where, and the final summary names the variant, test command, exit code, and working folder.
|
package/forge/skills/implement-conformance-testing-script/assets/run_conformance_tests_cypress.ps1
CHANGED
|
@@ -69,6 +69,17 @@ function Cleanup {
|
|
|
69
69
|
if ($script:build_output -and (Test-Path $script:build_output)) {
|
|
70
70
|
Remove-Item $script:build_output -Force -ErrorAction SilentlyContinue
|
|
71
71
|
}
|
|
72
|
+
|
|
73
|
+
# Step out of any temp subfolder we may still be in before deleting it
|
|
74
|
+
Set-Location -Path ([System.IO.Path]::GetTempPath()) -ErrorAction SilentlyContinue
|
|
75
|
+
|
|
76
|
+
# Remove temporary build subfolders if they exist
|
|
77
|
+
if ($script:NODE_SUBFOLDER -and (Test-Path $script:NODE_SUBFOLDER)) {
|
|
78
|
+
Remove-Item -Path $script:NODE_SUBFOLDER -Recurse -Force -ErrorAction SilentlyContinue
|
|
79
|
+
}
|
|
80
|
+
if ($script:NODE_CONFORMANCE_TESTS_SUBFOLDER -and (Test-Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER)) {
|
|
81
|
+
Remove-Item -Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER -Recurse -Force -ErrorAction SilentlyContinue
|
|
82
|
+
}
|
|
72
83
|
}
|
|
73
84
|
|
|
74
85
|
# Check for and kill any existing Node server from previous runs
|
|
@@ -98,20 +109,21 @@ if ($args[2] -eq "-v" -or $args[2] -eq "--verbose") {
|
|
|
98
109
|
$current_dir = Get-Location
|
|
99
110
|
|
|
100
111
|
try {
|
|
101
|
-
# Define the path to the subfolder
|
|
102
|
-
$
|
|
112
|
+
# Define the path to the subfolder. It lives in the system temp directory;
|
|
113
|
+
# $BuildFolder may be an absolute path, so only its leaf name is used.
|
|
114
|
+
$script:NODE_SUBFOLDER = Join-Path ([System.IO.Path]::GetTempPath()) "node_$(Split-Path $BuildFolder -Leaf)"
|
|
103
115
|
|
|
104
116
|
# Running React application
|
|
105
|
-
Write-Host "### Step 1: Starting the React application in folder $NODE_SUBFOLDER..."
|
|
117
|
+
Write-Host "### Step 1: Starting the React application in folder $script:NODE_SUBFOLDER..."
|
|
106
118
|
|
|
107
119
|
if ($env:VERBOSE -eq "1") {
|
|
108
|
-
Write-Host "Preparing Node subfolder: $NODE_SUBFOLDER"
|
|
120
|
+
Write-Host "Preparing Node subfolder: $script:NODE_SUBFOLDER"
|
|
109
121
|
}
|
|
110
122
|
|
|
111
123
|
# Check if the node subfolder exists
|
|
112
|
-
if (Test-Path $NODE_SUBFOLDER) {
|
|
124
|
+
if (Test-Path $script:NODE_SUBFOLDER) {
|
|
113
125
|
# Delete all files and folders except "node_modules", "plain_modules", and "package-lock.json"
|
|
114
|
-
Get-ChildItem -Path $NODE_SUBFOLDER -Force |
|
|
126
|
+
Get-ChildItem -Path $script:NODE_SUBFOLDER -Force |
|
|
115
127
|
Where-Object {
|
|
116
128
|
$_.Name -ne "node_modules" -and
|
|
117
129
|
$_.Name -ne "plain_modules" -and
|
|
@@ -126,18 +138,18 @@ try {
|
|
|
126
138
|
Write-Host "Subfolder does not exist. Creating it..."
|
|
127
139
|
}
|
|
128
140
|
|
|
129
|
-
New-Item -ItemType Directory -Path $NODE_SUBFOLDER -Force | Out-Null
|
|
141
|
+
New-Item -ItemType Directory -Path $script:NODE_SUBFOLDER -Force | Out-Null
|
|
130
142
|
}
|
|
131
143
|
|
|
132
|
-
Copy-Item -Path "$BuildFolder/*" -Destination $NODE_SUBFOLDER -Recurse -Force
|
|
144
|
+
Copy-Item -Path "$BuildFolder/*" -Destination $script:NODE_SUBFOLDER -Recurse -Force
|
|
133
145
|
|
|
134
146
|
# Move to the subfolder
|
|
135
|
-
if (-not (Test-Path $NODE_SUBFOLDER)) {
|
|
136
|
-
Write-Host "Error: Node build folder '$NODE_SUBFOLDER' does not exist."
|
|
147
|
+
if (-not (Test-Path $script:NODE_SUBFOLDER)) {
|
|
148
|
+
Write-Host "Error: Node build folder '$script:NODE_SUBFOLDER' does not exist."
|
|
137
149
|
exit $UNRECOVERABLE_ERROR_EXIT_CODE
|
|
138
150
|
}
|
|
139
151
|
|
|
140
|
-
Push-Location $NODE_SUBFOLDER
|
|
152
|
+
Push-Location $script:NODE_SUBFOLDER
|
|
141
153
|
|
|
142
154
|
# Temporarily allow stderr output without throwing (npm may write warnings to stderr)
|
|
143
155
|
# ForEach-Object converts ErrorRecord objects (from stderr) to plain strings to avoid verbose error formatting
|
|
@@ -245,17 +257,18 @@ try {
|
|
|
245
257
|
# Move back to the original directory
|
|
246
258
|
Set-Location $current_dir
|
|
247
259
|
|
|
248
|
-
# Define the path to the conformance tests subfolder
|
|
249
|
-
|
|
260
|
+
# Define the path to the conformance tests subfolder. It lives in the system
|
|
261
|
+
# temp directory; $ConformanceTestsFolder may be absolute, so only its leaf name is used.
|
|
262
|
+
$script:NODE_CONFORMANCE_TESTS_SUBFOLDER = Join-Path ([System.IO.Path]::GetTempPath()) "node_$(Split-Path $ConformanceTestsFolder -Leaf)"
|
|
250
263
|
|
|
251
264
|
if ($env:VERBOSE -eq "1") {
|
|
252
|
-
Write-Host "Preparing conformance tests Node subfolder: $NODE_CONFORMANCE_TESTS_SUBFOLDER"
|
|
265
|
+
Write-Host "Preparing conformance tests Node subfolder: $script:NODE_CONFORMANCE_TESTS_SUBFOLDER"
|
|
253
266
|
}
|
|
254
267
|
|
|
255
268
|
# Check if the conformance tests node subfolder exists
|
|
256
|
-
if (Test-Path $NODE_CONFORMANCE_TESTS_SUBFOLDER) {
|
|
269
|
+
if (Test-Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER) {
|
|
257
270
|
# Delete all files and folders except "node_modules", "plain_modules", and "package-lock.json"
|
|
258
|
-
Get-ChildItem -Path $NODE_CONFORMANCE_TESTS_SUBFOLDER -Force |
|
|
271
|
+
Get-ChildItem -Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER -Force |
|
|
259
272
|
Where-Object {
|
|
260
273
|
$_.Name -ne "node_modules" -and
|
|
261
274
|
$_.Name -ne "plain_modules" -and
|
|
@@ -270,18 +283,18 @@ try {
|
|
|
270
283
|
Write-Host "Subfolder does not exist. Creating it..."
|
|
271
284
|
}
|
|
272
285
|
|
|
273
|
-
New-Item -ItemType Directory -Path $NODE_CONFORMANCE_TESTS_SUBFOLDER -Force | Out-Null
|
|
286
|
+
New-Item -ItemType Directory -Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER -Force | Out-Null
|
|
274
287
|
}
|
|
275
288
|
|
|
276
|
-
Copy-Item -Path "$ConformanceTestsFolder/*" -Destination $NODE_CONFORMANCE_TESTS_SUBFOLDER -Recurse -Force
|
|
289
|
+
Copy-Item -Path "$ConformanceTestsFolder/*" -Destination $script:NODE_CONFORMANCE_TESTS_SUBFOLDER -Recurse -Force
|
|
277
290
|
|
|
278
291
|
# Move to the subfolder with Cypress tests
|
|
279
|
-
if (-not (Test-Path $NODE_CONFORMANCE_TESTS_SUBFOLDER)) {
|
|
280
|
-
Write-Host "Error: conformance tests Node folder '$NODE_CONFORMANCE_TESTS_SUBFOLDER' does not exist."
|
|
292
|
+
if (-not (Test-Path $script:NODE_CONFORMANCE_TESTS_SUBFOLDER)) {
|
|
293
|
+
Write-Host "Error: conformance tests Node folder '$script:NODE_CONFORMANCE_TESTS_SUBFOLDER' does not exist."
|
|
281
294
|
exit $UNRECOVERABLE_ERROR_EXIT_CODE
|
|
282
295
|
}
|
|
283
296
|
|
|
284
|
-
Push-Location $NODE_CONFORMANCE_TESTS_SUBFOLDER
|
|
297
|
+
Push-Location $script:NODE_CONFORMANCE_TESTS_SUBFOLDER
|
|
285
298
|
|
|
286
299
|
# Temporarily allow stderr output without throwing (npm may write warnings to stderr)
|
|
287
300
|
# ForEach-Object converts ErrorRecord objects (from stderr) to plain strings to avoid verbose error formatting
|
package/forge/skills/implement-conformance-testing-script/assets/run_conformance_tests_golang.ps1
CHANGED
|
@@ -23,7 +23,15 @@ $ConformanceTestsFolder = $args[1]
|
|
|
23
23
|
|
|
24
24
|
$current_dir = Get-Location
|
|
25
25
|
|
|
26
|
-
|
|
26
|
+
# Resolve conformance tests folder to an absolute path so it can be used
|
|
27
|
+
# from the build subfolder (where we'll Push-Location to next).
|
|
28
|
+
if (-not [System.IO.Path]::IsPathRooted($ConformanceTestsFolder)) {
|
|
29
|
+
$ConformanceTestsFolder = Join-Path $current_dir $ConformanceTestsFolder
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
# Working folder lives in the system temp directory. $BuildFolder may be an
|
|
33
|
+
# absolute path, so only its leaf name is used to build the working folder name.
|
|
34
|
+
$GO_BUILD_SUBFOLDER = Join-Path ([System.IO.Path]::GetTempPath()) "go_$(Split-Path $BuildFolder -Leaf)"
|
|
27
35
|
|
|
28
36
|
if ($env:VERBOSE -eq "1") {
|
|
29
37
|
Write-Host "Preparing Go build subfolder: $GO_BUILD_SUBFOLDER"
|
|
@@ -60,9 +68,9 @@ try {
|
|
|
60
68
|
go get
|
|
61
69
|
|
|
62
70
|
# Move to conformance tests folder
|
|
63
|
-
Set-Location
|
|
71
|
+
Set-Location $ConformanceTestsFolder
|
|
64
72
|
if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) {
|
|
65
|
-
Write-Host "Error: Conformance tests folder '$
|
|
73
|
+
Write-Host "Error: Conformance tests folder '$ConformanceTestsFolder' does not exist."
|
|
66
74
|
exit $UNRECOVERABLE_ERROR_EXIT_CODE
|
|
67
75
|
}
|
|
68
76
|
|
|
@@ -75,7 +83,7 @@ try {
|
|
|
75
83
|
}
|
|
76
84
|
|
|
77
85
|
# Move back to build directory
|
|
78
|
-
Set-Location
|
|
86
|
+
Set-Location $GO_BUILD_SUBFOLDER
|
|
79
87
|
|
|
80
88
|
# Execute Go lang conformance tests
|
|
81
89
|
Write-Host "Running Golang conformance tests...`n"
|
|
@@ -83,7 +91,7 @@ try {
|
|
|
83
91
|
# Temporarily allow stderr output without throwing (Go may write to stderr)
|
|
84
92
|
# ForEach-Object converts ErrorRecord objects (from stderr) to plain strings to avoid verbose error formatting
|
|
85
93
|
$ErrorActionPreference = 'Continue'
|
|
86
|
-
$output = go run
|
|
94
|
+
$output = go run (Join-Path $ConformanceTestsFolder "conformance_tests.go") 2>&1 | ForEach-Object { if ($_ -is [System.Management.Automation.ErrorRecord]) { $_.Exception.Message } else { $_ } } | Out-String
|
|
87
95
|
$exit_code = $LASTEXITCODE
|
|
88
96
|
$ErrorActionPreference = 'Stop'
|
|
89
97
|
|
|
@@ -97,4 +105,7 @@ try {
|
|
|
97
105
|
exit $exit_code
|
|
98
106
|
} finally {
|
|
99
107
|
Pop-Location
|
|
108
|
+
if (Test-Path $GO_BUILD_SUBFOLDER) {
|
|
109
|
+
Remove-Item -Path $GO_BUILD_SUBFOLDER -Recurse -Force -ErrorAction SilentlyContinue
|
|
110
|
+
}
|
|
100
111
|
}
|