plain-forge 1.0.5 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -1
- package/forge/rules/examples/integration-embedded-testing/prepare_environment_java.sh +51 -0
- package/forge/rules/examples/integration-embedded-testing/run_conformance_tests_java.sh +99 -0
- package/forge/rules/examples/integration-embedded-testing/run_unittests_java.sh +51 -0
- package/forge/rules/impl-reqs.md +6 -0
- package/forge/rules/integration-embedded-testing.md +285 -0
- package/forge/rules/integration-embedded.md +47 -42
- package/forge/rules/integration-standalone.md +9 -7
- package/forge/rules/integrations.md +33 -0
- package/forge/rules/test-reqs.md +6 -0
- package/forge/skills/add-implementation-requirement/SKILL.md +9 -1
- package/forge/skills/forge-plain/SKILL.md +14 -8
- package/forge/skills/init-plain-project/SKILL.md +6 -4
- package/forge/skills/load-plain-reference/SKILL.md +1 -1
- package/package.json +1 -1
- package/forge/docs/.gitkeep +0 -0
package/README.md
CHANGED
|
@@ -182,7 +182,6 @@ plain-forge keeps a single canonical source of truth under `forge/` and uses tin
|
|
|
182
182
|
forge/ # canonical, runtime-neutral content
|
|
183
183
|
skills/ # all skills used during spec writing
|
|
184
184
|
rules/ # workspace rules for spec validation
|
|
185
|
-
docs/ # shared docs (PLAIN_REFERENCE.md, etc.)
|
|
186
185
|
|
|
187
186
|
runtimes/ # per-runtime adapters
|
|
188
187
|
claude/
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
# Export Java 21
|
|
4
|
+
#export JAVA_HOME="/opt/homebrew/opt/openjdk@21/libexec/openjdk.jdk/Contents/Home"
|
|
5
|
+
#export PATH="$JAVA_HOME/bin:$PATH"
|
|
6
|
+
echo "JAVA_HOME: $JAVA_HOME"
|
|
7
|
+
|
|
8
|
+
# Check if build folder name is provided
|
|
9
|
+
if [ -z "$1" ]; then
|
|
10
|
+
printf "Error: No build folder name provided.\n"
|
|
11
|
+
printf "Usage: $0 <build_folder_name>\n"
|
|
12
|
+
exit 1
|
|
13
|
+
fi
|
|
14
|
+
|
|
15
|
+
JAVA_BUILD_SUBFOLDER=../../../
|
|
16
|
+
|
|
17
|
+
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
|
|
18
|
+
printf "Copying generated code to main project folder: $JAVA_BUILD_SUBFOLDER\n"
|
|
19
|
+
fi
|
|
20
|
+
|
|
21
|
+
# Check if the main project folder exists
|
|
22
|
+
if [ ! -d "$JAVA_BUILD_SUBFOLDER" ]; then
|
|
23
|
+
echo "Error: Main project folder '$JAVA_BUILD_SUBFOLDER' does not exist."
|
|
24
|
+
exit 2
|
|
25
|
+
fi
|
|
26
|
+
|
|
27
|
+
# clean existing code from the main project folder
|
|
28
|
+
rm -rf $JAVA_BUILD_SUBFOLDER/<placeholder for integration code>/*
|
|
29
|
+
rm -rf $JAVA_BUILD_SUBFOLDER/<placeholder for integration code tests>/*
|
|
30
|
+
|
|
31
|
+
# copy generated code to the main project folder
|
|
32
|
+
# FIX: added the mkdir lines otherwise it complains that the cp destintation directory does not exist
|
|
33
|
+
mkdir -p $JAVA_BUILD_SUBFOLDER/<placeholder for integration code>
|
|
34
|
+
mkdir -p $JAVA_BUILD_SUBFOLDER/<placeholder for integration code tests>
|
|
35
|
+
cp -R $1/<placeholder for integration code>/* $JAVA_BUILD_SUBFOLDER/<<placeholder for integration code>
|
|
36
|
+
cp -R $1/<placeholder for integration code tests>/* $JAVA_BUILD_SUBFOLDER/<placeholder for integration code tests>
|
|
37
|
+
|
|
38
|
+
# Move to the subfolder
|
|
39
|
+
echo "Moving to: $JAVA_BUILD_SUBFOLDER"
|
|
40
|
+
cd "$JAVA_BUILD_SUBFOLDER" 2>/dev/null
|
|
41
|
+
|
|
42
|
+
if [ $? -ne 0 ]; then
|
|
43
|
+
printf "Error: Java build folder '$JAVA_BUILD_SUBFOLDER' does not exist.\n"
|
|
44
|
+
exit 2
|
|
45
|
+
fi
|
|
46
|
+
|
|
47
|
+
# Remove target directory to avoid conflicts
|
|
48
|
+
rm -rf ./target || echo "Warning: some files may be locked"
|
|
49
|
+
|
|
50
|
+
echo "Runinng maven install in the build folder..."
|
|
51
|
+
mvn clean install -Dspring-boot.repackage.skip=true -DskipTests
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
# Export Java 21
|
|
4
|
+
#export JAVA_HOME="/opt/homebrew/opt/openjdk@21/libexec/openjdk.jdk/Contents/Home"
|
|
5
|
+
#export PATH="$JAVA_HOME/bin:$PATH"
|
|
6
|
+
echo "JAVA_HOME: $JAVA_HOME"
|
|
7
|
+
|
|
8
|
+
# Check if build folder name is provided
|
|
9
|
+
if [ -z "$1" ]; then
|
|
10
|
+
printf "Error: No build folder name provided.\n"
|
|
11
|
+
printf "Usage: $0 <build_folder_name> <conformance_tests_folder>\n"
|
|
12
|
+
exit 1
|
|
13
|
+
fi
|
|
14
|
+
|
|
15
|
+
# Check if conformance tests folder name is provided
|
|
16
|
+
if [ -z "$2" ]; then
|
|
17
|
+
printf "Error: No conformance tests folder name provided.\n"
|
|
18
|
+
printf "Usage: $0 <build_folder_name> <conformance_tests_folder>\n"
|
|
19
|
+
exit 1
|
|
20
|
+
fi
|
|
21
|
+
|
|
22
|
+
current_dir=$(pwd)
|
|
23
|
+
|
|
24
|
+
CONFORMANCE_TESTS_FOLDER=".tmp/java_conformance"
|
|
25
|
+
|
|
26
|
+
cd "$current_dir" 2>/dev/null
|
|
27
|
+
|
|
28
|
+
printf "Preparing Java conformance tests subfolder: $CONFORMANCE_TESTS_FOLDER\n"
|
|
29
|
+
|
|
30
|
+
# Check if the go build subfolder exists
|
|
31
|
+
if [ -d "$CONFORMANCE_TESTS_FOLDER" ]; then
|
|
32
|
+
# Find and delete all files and folders
|
|
33
|
+
find "$CONFORMANCE_TESTS_FOLDER" -mindepth 1 -exec rm -rf {} +
|
|
34
|
+
|
|
35
|
+
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
|
|
36
|
+
printf "Cleanup completed.\n"
|
|
37
|
+
fi
|
|
38
|
+
else
|
|
39
|
+
# print the current directory
|
|
40
|
+
printf "Current directory: $(pwd)\n"
|
|
41
|
+
|
|
42
|
+
printf "Subfolder does not exist. Creating it...\n"
|
|
43
|
+
mkdir -p $CONFORMANCE_TESTS_FOLDER
|
|
44
|
+
fi
|
|
45
|
+
|
|
46
|
+
printf "Copying all files and folders from $2 to $CONFORMANCE_TESTS_FOLDER...\n"
|
|
47
|
+
cp -R $2/* $CONFORMANCE_TESTS_FOLDER
|
|
48
|
+
|
|
49
|
+
# Move to the subfolder
|
|
50
|
+
printf "Moving to $CONFORMANCE_TESTS_FOLDER...\n"
|
|
51
|
+
cd "$CONFORMANCE_TESTS_FOLDER" 2>/dev/null
|
|
52
|
+
|
|
53
|
+
if [ $? -ne 0 ]; then
|
|
54
|
+
printf "Error: Java conformance tests folder '$CONFORMANCE_TESTS_FOLDER' does not exist.\n"
|
|
55
|
+
exit 2
|
|
56
|
+
fi
|
|
57
|
+
|
|
58
|
+
echo "Running maven install in the conformance tests folder..."
|
|
59
|
+
mvn clean install -DskipTests
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
if [ $? -ne 0 ]; then
|
|
63
|
+
exit 2
|
|
64
|
+
fi
|
|
65
|
+
|
|
66
|
+
echo "Running maven test in the conformance tests folder..."
|
|
67
|
+
|
|
68
|
+
output=$(mvn test --no-transfer-progress 2>&1)
|
|
69
|
+
exit_code=$?
|
|
70
|
+
|
|
71
|
+
echo "Finished running maven test in the conformance tests folder..."
|
|
72
|
+
|
|
73
|
+
# Check if no tests were run
|
|
74
|
+
if echo "$output" | grep -q 'Tests run: [1-9][0-9]*, Failures: 0, Errors: 0, Skipped: 0'; then
|
|
75
|
+
echo "All tests passed"
|
|
76
|
+
echo "$output"
|
|
77
|
+
exit $exit_code
|
|
78
|
+
else
|
|
79
|
+
# Check if no tests were run
|
|
80
|
+
if echo "$output" | grep -q 'Tests run: 0, Failures: 0, Errors: 0, Skipped: 0'; then
|
|
81
|
+
echo "Tests run: 0"
|
|
82
|
+
echo "Error: No tests were executed (Tests run: 0)"
|
|
83
|
+
echo "You are seeing this cause the following appeared in the output: Tests run: 0, Failures: 0, Errors: 0, Skipped: 0"
|
|
84
|
+
else
|
|
85
|
+
echo "Some tests failed, had errors, or were skipped. This is not allowed. All tests must pass."
|
|
86
|
+
fi
|
|
87
|
+
fi
|
|
88
|
+
|
|
89
|
+
# If there was an error, print the output and exit with the error code
|
|
90
|
+
if [ $exit_code -ne 0 ]; then
|
|
91
|
+
echo "Error: Maven test failed in the conformance tests folder with exit code $exit_code..."
|
|
92
|
+
echo "$output"
|
|
93
|
+
exit $exit_code
|
|
94
|
+
fi
|
|
95
|
+
|
|
96
|
+
echo "Finished running conformance tests..."
|
|
97
|
+
echo "$output"
|
|
98
|
+
# Echo the original exit code of the unittest command
|
|
99
|
+
exit $exit_code
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
# Export Java 21
|
|
4
|
+
#export JAVA_HOME="/opt/homebrew/opt/openjdk@21/libexec/openjdk.jdk/Contents/Home"
|
|
5
|
+
#export PATH="$JAVA_HOME/bin:$PATH"
|
|
6
|
+
echo "JAVA_HOME: $JAVA_HOME"
|
|
7
|
+
|
|
8
|
+
# Check if subfolder name is provided
|
|
9
|
+
if [ -z "$1" ]; then
|
|
10
|
+
echo "Error: No subfolder name provided."
|
|
11
|
+
echo "Usage: $0 <subfolder_name>"
|
|
12
|
+
exit 1
|
|
13
|
+
fi
|
|
14
|
+
# FIX: Changed this line from "../../" to "../../.." to go up three levels to the main project folder
|
|
15
|
+
MAIN_PROJECT_FOLDER="$(cd "$(dirname "$0")/../../.." && pwd)"
|
|
16
|
+
|
|
17
|
+
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
|
|
18
|
+
printf "Copying generated code to main project folder: $MAIN_PROJECT_FOLDER\n"
|
|
19
|
+
fi
|
|
20
|
+
|
|
21
|
+
# Check if the main project folder exists
|
|
22
|
+
if [ ! -d "$MAIN_PROJECT_FOLDER" ]; then
|
|
23
|
+
echo "Error: Main project folder '$MAIN_PROJECT_FOLDER' does not exist."
|
|
24
|
+
exit 2
|
|
25
|
+
fi
|
|
26
|
+
|
|
27
|
+
# clean existing code from the main project folder
|
|
28
|
+
rm -rf $MAIN_PROJECT_FOLDER/<placeholder for integration code>/*
|
|
29
|
+
rm -rf $MAIN_PROJECT_FOLDER/<placeholder for integration code tests>/*
|
|
30
|
+
|
|
31
|
+
# copy generated code to the main project folder
|
|
32
|
+
# FIX: added the mkdir lines otherwise it complains that the cp destintation directory does not exist
|
|
33
|
+
mkdir -p $MAIN_PROJECT_FOLDER/<placeholder for integration code>
|
|
34
|
+
mkdir -p $MAIN_PROJECT_FOLDER/<placeholder for integration code tests>
|
|
35
|
+
cp -R $1/<placeholder for integration code>/* $MAIN_PROJECT_FOLDER/<placeholder for integration code>
|
|
36
|
+
cp -R $1/<placeholder for integration code tests>/* $MAIN_PROJECT_FOLDER/<placeholder for integration code tests>
|
|
37
|
+
|
|
38
|
+
# Move to the subfolder
|
|
39
|
+
# FIX: added this line to print the current directory to help the LLM
|
|
40
|
+
echo "Moving to: $MAIN_PROJECT_FOLDER"
|
|
41
|
+
cd "$MAIN_PROJECT_FOLDER" 2>/dev/null
|
|
42
|
+
|
|
43
|
+
if [ $? -ne 0 ]; then
|
|
44
|
+
printf "Error: Java build folder '$MAIN_PROJECT_FOLDER' does not exist.\n"
|
|
45
|
+
exit 2
|
|
46
|
+
fi
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# Execute all Java unittests in the subfolder
|
|
50
|
+
echo "Running Java unittests in $1..."
|
|
51
|
+
mvn test -Dtest='<placeholder for integration code tests package>.**.*Test' checkstyle:check
|
package/forge/rules/impl-reqs.md
CHANGED
|
@@ -12,6 +12,12 @@ When writing or editing an `***implementation reqs***` section in a `.plain` fil
|
|
|
12
12
|
- Observable behavior (endpoints, business rules, user-facing features) belongs in `***functional specs***`
|
|
13
13
|
- Internal structure, technology choices, and coding guidance belong here
|
|
14
14
|
|
|
15
|
+
## `:UnitTests:` lives here (hard rule)
|
|
16
|
+
- **Everything** about `:UnitTests:` goes in `***implementation reqs***` — paths, approach, packages, framework, conventions, fixtures, mocking policy, file layout, naming, lint / static-analysis gates
|
|
17
|
+
- `:UnitTests:` are part of the generated codebase (they sit inside `plain_modules/<module>/` alongside the implementation), so requirements that shape them are implementation reqs by definition
|
|
18
|
+
- The unit-test generator reads **only** `***implementation reqs***` — anything about `:UnitTests:` placed elsewhere (e.g. `***test reqs***`) is silently ignored
|
|
19
|
+
- Author each `:UnitTests:` requirement via `add-implementation-requirement` and phrase it in terms of `:UnitTests:` so the partition stays visible at a glance
|
|
20
|
+
|
|
15
21
|
## What belongs here
|
|
16
22
|
- Technology choices: language, framework, runtime version
|
|
17
23
|
- Architectural constraints: patterns, layering, dependency rules
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Rules for authoring the three test scripts that an embedded ***plain integration ships
|
|
3
|
+
globs: "**/*.plain"
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Rules for embedded-integration test scripts
|
|
7
|
+
|
|
8
|
+
When an embedded integration ships its three `test_scripts/` (prepare-environment, unit, conformance) — whether you author them by hand or via the `implement-*-testing-script` skills — these rules apply on top of the shared testing-script rules in PLAIN_REFERENCE.md (exit-code conventions, the activate-only vs install-inline conformance distinction, `VERBOSE=1`, etc.).
|
|
9
|
+
|
|
10
|
+
## Staging model (read this first)
|
|
11
|
+
|
|
12
|
+
The three scripts do **not** all stage into the same place. Embedded integrations need a runnable host project, so the prepare and unit-test scripts operate **inside the host codebase itself**; only the conformance script uses a `.tmp/` scratch folder because the conformance suite lives in a separate project.
|
|
13
|
+
|
|
14
|
+
| Script | Where it operates | What it does to the host source tree |
|
|
15
|
+
|--------|-------------------|---------------------------------------|
|
|
16
|
+
| `prepare_environment_<lang>` | Inside the host codebase | Copies `$1` into the module's package path under the host's source tree, then compiles / installs the host project |
|
|
17
|
+
| `run_unittests_<lang>` | Inside the host codebase | Same copy as above (self-contained), then runs the module's unit tests + lint inside the host |
|
|
18
|
+
| `run_conformance_tests_<lang>` | `.tmp/<lang>_conformance/` | Copies `$2` (the conformance-tests folder) into the scratch directory and runs the conformance suite there, which depends on the host build that `prepare_environment` already installed |
|
|
19
|
+
|
|
20
|
+
This deliberately writes into the host's `src/main/...` and `src/test/...` (or the language equivalent). Two consequences flow from that:
|
|
21
|
+
|
|
22
|
+
- **The host codebase root must come from a configured env var** (`HOST_CODEBASE_ROOT` is the conventional name; declare it in the integration's configuration concept). Scripts never hardcode it
|
|
23
|
+
- **The destructive-op scoping rule below becomes critical.** `rm -rf` must target only the module's own package path under the host's source tree, never the host's `src/main/`, `target/`, `node_modules/`, or `build/` at the project root
|
|
24
|
+
|
|
25
|
+
This rule covers the **mechanics each script must obey** regardless of language: argument handling, exit codes, idempotency, path resolution, output parsing, and the "what NOT to put here" guard rails. The language-specific install / test commands come from the skills; the contract below is invariant.
|
|
26
|
+
|
|
27
|
+
## What the `.plain` spec must declare
|
|
28
|
+
|
|
29
|
+
The three scripts are generated from facts in the integration's spec. For an embedded integration, those facts cannot be inferred — they have to be **declared explicitly** in the right section, partitioned by which predefined concept they describe:
|
|
30
|
+
|
|
31
|
+
- **Everything about `:UnitTests:`** — paths, approach, packages, framework, conventions, fixtures, mocking policy — lives in `***implementation reqs***`. Unit tests are part of the generated codebase, so requirements that shape them are implementation reqs (see [`impl-reqs.md`](impl-reqs.md))
|
|
32
|
+
- **Everything about `:ConformanceTests:`** — paths, approach, packages, framework, execution command, pass criteria, mocking policy — lives in `***test reqs***`. Conformance tests are external to the generated codebase, so requirements that shape them are test reqs (see [`test-reqs.md`](test-reqs.md))
|
|
33
|
+
|
|
34
|
+
Authors use `add-implementation-requirement` for the first group and `add-test-requirement` for the second. The two groups are parallel — each predefined concept owns a complete authoring story in its own section.
|
|
35
|
+
|
|
36
|
+
### In `***implementation reqs***` — everything about `:UnitTests:`
|
|
37
|
+
|
|
38
|
+
These reqs feed `run_unittests_<lang>`. `prepare_environment_<lang>` is **not** part of the unit-test workflow — it exists for conformance (see the next subsection) and reads its own facts from `***test reqs***`. At minimum, declare:
|
|
39
|
+
|
|
40
|
+
1. **Integration source path inside the host** — where `$1/<source>/*` gets copied to. Example: `src/main/java/com/example/integrations/foo`
|
|
41
|
+
2. **`:UnitTests:` source path inside the host** — where `$1/<tests>/*` gets copied to, and where the test runner discovers unit tests. Example: `src/test/java/com/example/integrations/foo`
|
|
42
|
+
3. **Fully qualified `:UnitTests:` package** — the package the test runner uses to scope discovery via its filter argument. Example: `com.example.integrations.foo`
|
|
43
|
+
4. **`:UnitTests:` framework and conventions** — JUnit / pytest / Jest / Go's `testing` / etc., plus naming conventions (`*Test`, `test_*`, `*.test.ts`, …), fixture / mock / assertion style, file layout inside the test package
|
|
44
|
+
5. **Quality gates that run alongside `:UnitTests:`** — Checkstyle / ESLint / Pylint / Ruff / `go vet` / `cargo clippy` — whatever the host project requires
|
|
45
|
+
|
|
46
|
+
Author the location facts as one tight group, phrased in terms of `:UnitTests:`:
|
|
47
|
+
|
|
48
|
+
```plain
|
|
49
|
+
- :UnitTests: of :Implementation: live in `src/test/java/com/example/integrations/foo` inside the host codebase.
|
|
50
|
+
- The corresponding integration source code lives in `src/main/java/com/example/integrations/foo`.
|
|
51
|
+
- The fully qualified package used for :UnitTests: discovery is `com.example.integrations.foo`.
|
|
52
|
+
- :UnitTests: use JUnit 5 with the host's Checkstyle profile applied via `mvn checkstyle:check`.
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
These facts feed directly into the prepare and unit-test script bodies:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
# clean existing code from the host
|
|
59
|
+
rm -rf $MAIN_PROJECT_FOLDER/<integration source path>/*
|
|
60
|
+
rm -rf $MAIN_PROJECT_FOLDER/<:UnitTests: source path>/*
|
|
61
|
+
|
|
62
|
+
# create destinations and copy generated code into the host
|
|
63
|
+
mkdir -p $MAIN_PROJECT_FOLDER/<integration source path>
|
|
64
|
+
mkdir -p $MAIN_PROJECT_FOLDER/<:UnitTests: source path>
|
|
65
|
+
cp -R $1/<integration source path>/* $MAIN_PROJECT_FOLDER/<integration source path>
|
|
66
|
+
cp -R $1/<:UnitTests: source path>/* $MAIN_PROJECT_FOLDER/<:UnitTests: source path>
|
|
67
|
+
|
|
68
|
+
# run :UnitTests: scoped to the integration's package
|
|
69
|
+
mvn test -Dtest='<:UnitTests: package>.**.*Test' checkstyle:check
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
### In `***test reqs***` — everything about `:ConformanceTests:`
|
|
73
|
+
|
|
74
|
+
These reqs feed `run_conformance_tests_<lang>`. At minimum, declare:
|
|
75
|
+
|
|
76
|
+
1. **`:ConformanceTests:` source location** — where the conformance suite lives in the project (typically a sibling folder, e.g. `conformance_tests/<module>/`); the renderer passes the resolved path as `$2`
|
|
77
|
+
2. **`:ConformanceTests:` framework and execution command** — `mvn test --no-transfer-progress`, `pytest`, `npm test`, `go test ./...`, etc., with any flags / profiles the project requires
|
|
78
|
+
3. **Fully qualified `:ConformanceTests:` package** (or path / pattern) used to scope discovery, if the runner needs one
|
|
79
|
+
4. **`:ConformanceTests:` network and secrets policy** — by default the suite runs against the **live provider** (see [`integrations.md`](integrations.md) → *`:ConformanceTests:` always run against the live integration*). Declare the env-var names the script reads (e.g. `<PROVIDER>_API_KEY`), whether the script loads a `.env` file before running, and any specific endpoints that are mocked because they can't be exercised live safely (429, forced 5xx)
|
|
80
|
+
5. **`:ConformanceTests:` pass criteria** — the strict criteria from [*Pass criteria (strict)*](#pass-criteria-strict): at least one test ran AND zero failures / errors / skipped. Reaffirm this in the spec so the renderer knows the runner must parse the test tool's stdout
|
|
81
|
+
6. **`:ConformanceTests:` build / install needs** — anything the conformance project needs before `mvn test` (or equivalent) will work: dependencies, fixtures, schema files, generated stubs
|
|
82
|
+
|
|
83
|
+
Author the conformance facts as one or more entries, phrased in terms of `:ConformanceTests:`:
|
|
84
|
+
|
|
85
|
+
```plain
|
|
86
|
+
- :ConformanceTests: of :Implementation: live in `conformance_tests/foo/` and are implemented with JUnit 5 + Maven.
|
|
87
|
+
- The fully qualified package used for :ConformanceTests: discovery is `com.example.integrations.foo.conformance`.
|
|
88
|
+
- :ConformanceTests: are run via `mvn test --no-transfer-progress`; the host's Surefire plugin must be installed.
|
|
89
|
+
- :ConformanceTests: run against the live :ProviderName: sandbox — no mocking of provider calls.
|
|
90
|
+
- The conformance script reads `<PROVIDER>_API_KEY` (and any additional secrets) from the shell or from a `.env` file at the project root and fails fast (exit `69`) if any required var is missing after the optional `.env` load.
|
|
91
|
+
- The 429 (rate-limit) and forced-5xx paths use a local mock for that specific endpoint; every other path is live.
|
|
92
|
+
- :ConformanceTests: pass only when the Surefire summary line shows at least one test ran with zero failures, zero errors, and zero skipped.
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
These facts feed directly into the conformance script body:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
# stage :ConformanceTests: source into the scratch directory
|
|
99
|
+
find "$DIR" -mindepth 1 -exec rm -rf {} +
|
|
100
|
+
cp -R $2/* $DIR
|
|
101
|
+
cd $DIR
|
|
102
|
+
|
|
103
|
+
# build the conformance project, then run :ConformanceTests:
|
|
104
|
+
mvn clean install -DskipTests
|
|
105
|
+
output=$(mvn test --no-transfer-progress 2>&1)
|
|
106
|
+
|
|
107
|
+
# parse Surefire summary against the declared pass criteria, then exit accordingly
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
### Rules common to both sections
|
|
111
|
+
|
|
112
|
+
- **The paths must agree across reqs.** The `:UnitTests:` source path, the `:UnitTests:` package, and the integration source path describe the same module from three angles. Same for the `:ConformanceTests:` location and its package. A mismatch (e.g. test path `src/test/java/com/example/foo` but test package `com.example.bar`) silently produces a green build with stale or zero tests
|
|
113
|
+
- **Paths are host-relative**, not absolute. `MAIN_PROJECT_FOLDER` comes from `HOST_CODEBASE_ROOT` (declared in the configuration concept); the paths above join onto it
|
|
114
|
+
- **The renderer's output folder `$1` mirrors the same layout.** `$1/<integration source path>/*` exists because the renderer emits the generated code into the same package directories it'll be copied into — the `cp -R` is a straight overlay, not a path translation
|
|
115
|
+
- **Each fact lives in one place.** If two `***implementation reqs***` entries declare slightly different `:UnitTests:` paths (or two `***test reqs***` entries declare slightly different `:ConformanceTests:` packages), the renderer can't tell which to use. Author each group as a tight cluster (one bullet with sub-bullets) so a future reviewer sees them together
|
|
116
|
+
- **Never duplicate a fact across sections.** `:UnitTests:` facts belong **only** in `***implementation reqs***`; `:ConformanceTests:` facts belong **only** in `***test reqs***`. The two groups never overlap — the script generators read each from its own section
|
|
117
|
+
- **For non-Java languages**, the facts have language-specific equivalents — Python: `src/foo/`, `tests/foo/`, `tests.foo`; Node: `src/foo/`, `test/foo/`, `test/foo/**/*.test.ts`. The `implement-*-testing-script` skills know the mapping per language, but the spec still has to declare the host-relative paths and packages so the skill knows what to put in the script bodies
|
|
118
|
+
|
|
119
|
+
## Common contract (all three scripts)
|
|
120
|
+
|
|
121
|
+
All three scripts are invoked by the `codeplain` renderer with positional arguments. They MUST:
|
|
122
|
+
|
|
123
|
+
1. **Be POSIX-bash (`.sh`) on macOS / Linux, PowerShell (`.ps1`) on Windows.** Executable, idempotent — re-running with the same inputs produces the same result
|
|
124
|
+
2. **Validate every positional argument up front.** Exit with a clear `Usage:` line on bad args. Conventional exit codes (consistent with the shared testing-script rules):
|
|
125
|
+
- `1` — bad arguments / usage error (and "no tests discovered" for the conformance runner)
|
|
126
|
+
- `2` — missing or inaccessible input folder
|
|
127
|
+
- `69` — unrecoverable environment failure (missing toolchain, cannot enter working folder, install failed)
|
|
128
|
+
- Any other non-zero code — propagated verbatim from the underlying build / test tool
|
|
129
|
+
3. **Echo the toolchain home at the top** — the first thing a developer checks when CI breaks. For Java: `JAVA_HOME`. For Python: the active interpreter (`python --version` / `which python`). For Node: `node --version`. For Go: `go env GOROOT`. Pick the variable whose value most often explains "why does it work locally but not on the CI runner?"
|
|
130
|
+
4. **Respect `VERBOSE=1`.** Gate chatty diagnostic prints behind `if [ "${VERBOSE:-}" = "1" ]` (and the PowerShell equivalent). Errors and key step markers print unconditionally
|
|
131
|
+
5. **Resolve paths relative to the script, not `$PWD`.** Use `"$(cd "$(dirname "$0")/<relative-path-to-anchor>" && pwd)"`. Hard-coded `../../` chains break the moment the renderer's `cwd` changes
|
|
132
|
+
6. **Create destination directories before copying.** `mkdir -p` before any `cp -R` / `rsync` / `robocopy` — most copy commands do not create intermediate directories and fail silently in some shells
|
|
133
|
+
7. **Scope destructive operations narrowly.** Any `rm -rf` (or `Remove-Item -Recurse -Force`) targets only the module's own package path inside the working folder — never the host's `src/`, `target/`, `node_modules/`, `build/`, or `dist/` at the project root. Only `prepare_environment` owns the build-output directory's lifecycle for its `.tmp/` working folder
|
|
134
|
+
8. **Print where you are before you `cd`.** `echo "Moving to: $DIR"` saves an hour of debugging when paths are wrong. PowerShell: `Write-Host "Moving to: $DIR"`
|
|
135
|
+
|
|
136
|
+
## 1. `prepare_environment_<lang>` — copy into the host, then compile
|
|
137
|
+
|
|
138
|
+
Receives one positional argument: the renderer's build output folder (e.g. `plain_modules/<module>/`).
|
|
139
|
+
|
|
140
|
+
Purpose: stage the generated code **into the host codebase at the module's package path**, then run the host's install / build so that downstream test projects (specifically the conformance suite) can depend on it from the local dependency cache.
|
|
141
|
+
|
|
142
|
+
Required steps:
|
|
143
|
+
|
|
144
|
+
1. Validate `$1` (the renderer's output folder); fail with usage if missing
|
|
145
|
+
2. Resolve the host codebase root from the env var declared in the `host-codebase` concept (`HOST_CODEBASE_ROOT` is the conventional name; surface it in `--help` / usage)
|
|
146
|
+
3. `rm -rf` the module's package directories under the host's source tree — both main / source AND test directories. Both, because stale tests in the target package cause confusing failures
|
|
147
|
+
4. `mkdir -p` the destination directories
|
|
148
|
+
5. Copy `$1/src/...` (or the language-equivalent layout) into the host's source tree at the module's package path
|
|
149
|
+
6. `cd` into the host codebase root, verify with an explicit exit-code check (`if [ $? -ne 0 ]` / `if (! $?) { ... }`)
|
|
150
|
+
7. Clean the host's build-output directory before compiling — stale class / object files from a previous build cause confusing link-time mismatches when the contract has changed. Maven: `rm -rf ./target`. Cargo: `cargo clean -p <pkg>`. Go: `go clean -cache` (usually unnecessary, but mention if applicable). For npm projects with incremental builds disabled, wipe the relevant `dist/` / `build/` folder
|
|
151
|
+
8. Run the host's standard install / build command — but **skip the fat-artifact and skip tests** when the build system supports it:
|
|
152
|
+
- Maven: `mvn clean install -DskipTests` (plus any host-specific repackage-skip flag)
|
|
153
|
+
- npm / pnpm / yarn: `npm ci` (no test script; dependency install only)
|
|
154
|
+
- Python: `pip install -e .` (then any extra integration dependencies)
|
|
155
|
+
- Go: `go mod tidy && go build ./...`
|
|
156
|
+
- Cargo: `cargo fetch && cargo build --tests --no-run`
|
|
157
|
+
9. Exit `0` on success; propagate the underlying build tool's exit code on failure
|
|
158
|
+
|
|
159
|
+
**Why this script exists separately.** Conformance tests live in a separate project and resolve the integration's code as a dependency from a local cache (`~/.m2`, `node_modules`, the language equivalent). Without an explicit install step, conformance tests link against whatever was last built — which may not be the renderer's most recent output.
|
|
160
|
+
|
|
161
|
+
## 2. `run_unittests_<lang>` — copy into the host, then run unit tests there
|
|
162
|
+
|
|
163
|
+
Receives one positional argument: the renderer's build output folder.
|
|
164
|
+
|
|
165
|
+
Purpose: stage the generated code **into the host codebase** (the same way `prepare_environment` does) and run the module's unit tests + language-appropriate quality gates against the host project from inside the host root.
|
|
166
|
+
|
|
167
|
+
Required steps:
|
|
168
|
+
|
|
169
|
+
1. Validate `$1`
|
|
170
|
+
2. Resolve the host codebase root from the configured env var
|
|
171
|
+
3. `rm -rf` the module's main + test package directories under the host's source tree
|
|
172
|
+
4. `mkdir -p` and copy `$1/src/...` into place — **same staging as `prepare_environment`**
|
|
173
|
+
5. `cd` into the host codebase root
|
|
174
|
+
6. Run the test command **scoped to the module's package only**:
|
|
175
|
+
- Maven: `mvn test -Dtest='<fully.qualified.package>.**.*Test' <lint-check>:check`
|
|
176
|
+
- pytest: `pytest tests/<module>/` (or whatever path matches the module)
|
|
177
|
+
- Jest: `npm test -- --testPathPattern '<module>/'`
|
|
178
|
+
- Go: `go test ./<module>/...`
|
|
179
|
+
7. Always include the host's lint / static-analysis gate in the same invocation when the build system has one (Checkstyle, ESLint, Pylint / Ruff, `go vet`, `cargo clippy`, …). Running tests without the lint gate produces a false-green when style violations would otherwise block the build
|
|
180
|
+
|
|
181
|
+
Hard rules:
|
|
182
|
+
|
|
183
|
+
- **Yes, this duplicates `prepare_environment`'s copy step.** That's intentional — `run_unittests` must be self-contained per the shared testing-script rules (there is no activate-only variant for unit tests). A user invoking the unit-test script independently must not need `prepare_environment` to have run first
|
|
184
|
+
- **Never use a "clean and test" shortcut** (`mvn clean test`, `npm run rebuild`, …) — the cleanup belongs in `prepare_environment` and would erase whatever it installed
|
|
185
|
+
- **Never run the full test suite** when scoping is possible. A per-module script that runs the entire host's tests wastes minutes on every render iteration
|
|
186
|
+
|
|
187
|
+
## 3. `run_conformance_tests_<lang>` — copy into `.tmp/`, then run the external suite
|
|
188
|
+
|
|
189
|
+
Receives two positional arguments: the renderer's build output folder (`$1`) and the conformance-tests folder (`$2`).
|
|
190
|
+
|
|
191
|
+
Purpose: run the conformance suite in `$2` against the build that `prepare_environment` produced. The conformance suite is a separate project that consumes the integration as a dependency.
|
|
192
|
+
|
|
193
|
+
Required steps:
|
|
194
|
+
|
|
195
|
+
1. Validate `$1` and `$2`. **Only `$2` is actually used by the script body today** — but keep `$1` in the signature; the renderer passes both positionally
|
|
196
|
+
2. Stage `$2/*` into a scratch directory under `.tmp/<lang>_conformance/`. Wipe it first if it exists. Use a deletion form that handles hidden files and odd shells safely:
|
|
197
|
+
- Bash: `find "$DIR" -mindepth 1 -exec rm -rf {} +` (safer than `rm -rf "$DIR"/*`)
|
|
198
|
+
- PowerShell: `Remove-Item -Recurse -Force "$DIR\*"` after confirming the path exists
|
|
199
|
+
3. `cd` into the scratch directory
|
|
200
|
+
4. Compile / set up the conformance project — exit early if this fails:
|
|
201
|
+
- Maven: `mvn clean install -DskipTests`
|
|
202
|
+
- npm: `npm ci`
|
|
203
|
+
- Python: `pip install -r requirements.txt`
|
|
204
|
+
- Go / Cargo: standard build
|
|
205
|
+
5. Run the conformance test command and **capture stdout to a variable**:
|
|
206
|
+
- Bash: `output=$(mvn test --no-transfer-progress 2>&1)` (or the language equivalent)
|
|
207
|
+
- PowerShell: `$output = & <cmd> 2>&1 | Out-String`
|
|
208
|
+
6. **Parse the captured output strictly** (see *Pass criteria* below) — don't trust the test tool's exit code alone
|
|
209
|
+
7. On the failure path: print the captured output unconditionally so the renderer (and the user) can see what failed
|
|
210
|
+
8. On the success path: print the output only if `VERBOSE=1`
|
|
211
|
+
|
|
212
|
+
### Secrets and `.env` handling
|
|
213
|
+
|
|
214
|
+
`:ConformanceTests:` run against the live provider (per [`integrations.md`](integrations.md) → *`:ConformanceTests:` always run against the live integration*), so the conformance script needs the user's credentials at runtime.
|
|
215
|
+
|
|
216
|
+
- **Credentials are read from environment variables** named in `***test reqs***` and the auth concept. Never from a literal, never from a file checked into the repo
|
|
217
|
+
- **The script may optionally load a `.env` file** before running the suite. Typical pattern in Bash:
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
ENV_FILE="${ENV_FILE:-$MAIN_PROJECT_FOLDER/.env}"
|
|
221
|
+
if [ -f "$ENV_FILE" ]; then
|
|
222
|
+
echo "Loading env from $ENV_FILE"
|
|
223
|
+
set -a; . "$ENV_FILE"; set +a
|
|
224
|
+
fi
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
PowerShell uses the equivalent `Get-Content` + `Set-Item Env:` loop. Absence of `.env` is **not** an error — CI provides the same vars directly through the shell
|
|
228
|
+
- **Verify required env vars exist after the optional `.env` load** and fail fast if any are missing:
|
|
229
|
+
|
|
230
|
+
```bash
|
|
231
|
+
for var in PROVIDER_API_KEY PROVIDER_ACCOUNT_ID; do
|
|
232
|
+
if [ -z "${!var:-}" ]; then
|
|
233
|
+
printf "Error: %s is required for :ConformanceTests:\n" "$var" >&2
|
|
234
|
+
exit 69
|
|
235
|
+
fi
|
|
236
|
+
done
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
- **Export the resolved vars** (already in scope thanks to `set -a`) so child processes started by the test runner inherit them — Maven, pytest, npm, Go all read env vars from their parent process
|
|
240
|
+
- **Never log credential values.** Echo the env-var **name** when reporting "loaded", not its value. Redact in any error path that dumps captured output
|
|
241
|
+
- **Document the secret names twice** — once in the auth concept (for the runtime), once in `***test reqs***` for `:ConformanceTests:` (for the script). They must be the same names; a divergence means the script reads different credentials than the runtime does, and conformance silently tests the wrong account
|
|
242
|
+
|
|
243
|
+
### Pass criteria (strict)
|
|
244
|
+
|
|
245
|
+
These are the **only** valid pass criteria. Anything outside this list is a failure:
|
|
246
|
+
|
|
247
|
+
- At least one test ran AND zero failures AND zero errors AND zero skipped. Maven example: `Tests run: [1-9][0-9]*, Failures: 0, Errors: 0, Skipped: 0`. Each language's test runner has an equivalent summary line — match it with a regex, not by substring
|
|
248
|
+
- Exit with the test tool's exit code on the success path (almost always `0`)
|
|
249
|
+
|
|
250
|
+
Failure cases — each must be detected explicitly:
|
|
251
|
+
|
|
252
|
+
- `Tests run: 0` (or the language equivalent) → **failure**. The renderer must never be told "green" when nothing ran. Exit `1` per the shared testing-script rules (the "no tests discovered" exit code)
|
|
253
|
+
- Any non-zero `Failures:`, `Errors:`, or `Skipped:` count → **failure**. Dump the captured output and propagate the test tool's exit code
|
|
254
|
+
- The test tool exited non-zero but the summary line shows `Tests run: 0` → still a `1` (no-tests-discovered), not the tool's own exit code
|
|
255
|
+
|
|
256
|
+
**Why parse stdout instead of trusting the test tool's exit code.** Every major test runner returns `0` when zero test classes match a filter. Maven Surefire, pytest with no collected tests, `go test ./pkg/that/has/no/_test.go/files`, Jest with an empty pattern — all green-by-default. The renderer's contract is "tests must run AND must pass" — the exit code alone cannot express that.
|
|
257
|
+
|
|
258
|
+
## Cross-cutting rules
|
|
259
|
+
|
|
260
|
+
- **No secrets in scripts.** Read credentials from env vars the renderer (or the user's shell) already sets. If a required env var is missing, fail fast with a clear `Error: $FOO is required for ...` and exit `69`. Never hardcode an API key, never read from a checked-in file
|
|
261
|
+
- **Mirror the package / module path exactly.** A typo in the package segment (e.g. one wrong directory in `com/example/foo/...` or `host_pkg/foo/bar/...`) makes `rm -rf` silently do nothing and the subsequent copy put files in the wrong place. Both produce a **green** build with stale code. Derive the package path from the `host-codebase` concept; never retype it inline
|
|
262
|
+
- **Don't add cleverness to the script that belongs in the build.** If you're tempted to `grep` the test output for more than the pass/fail summary, change the test or the test-runner config instead. The script is a thin contract — keep it that way
|
|
263
|
+
- **Don't use shortcut profiles that skip quality gates.** Maven's `-Pfast`, Gradle's `-x check`, npm's `--ignore-scripts`, pytest's `--no-cov`, `-DskipTests` outside `prepare_environment` — all defeat the purpose of these scripts. The build's full quality gate (tests + lint + coverage thresholds) must run for unit and conformance
|
|
264
|
+
- **Failure messages are for humans.** `printf "Error: <what failed> at <where>\n"` then exit with the right code. No silent failures, no `|| true` to paper over errors, no `2>/dev/null` on commands whose stderr matters
|
|
265
|
+
- **Use existing scripts as templates.** When you add a new embedded integration, copy from a known-good module, then search-and-replace only the package path and the test-filter argument. Resist the urge to "improve" structure mid-port — pattern-matching across many modules is a feature, not a bug
|
|
266
|
+
|
|
267
|
+
## Reference implementation (Java + Maven)
|
|
268
|
+
|
|
269
|
+
A working three-script set for a Java / Maven embedded integration lives under [`examples/integration-embedded-testing/`](examples/integration-embedded-testing/) next to this file:
|
|
270
|
+
|
|
271
|
+
- [`prepare_environment_java.sh`](examples/integration-embedded-testing/prepare_environment_java.sh) — copies `$1` into the host's source tree, cleans `target/`, runs `mvn clean install -DskipTests`
|
|
272
|
+
- [`run_unittests_java.sh`](examples/integration-embedded-testing/run_unittests_java.sh) — re-stages into the host and runs `mvn test -Dtest='<module-pkg>.**.*Test' checkstyle:check`
|
|
273
|
+
- [`run_conformance_tests_java.sh`](examples/integration-embedded-testing/run_conformance_tests_java.sh) — copies `$2` into `.tmp/java_conformance/`, builds the conformance project, runs the suite, and parses the Surefire summary line per the strict pass criteria above
|
|
274
|
+
|
|
275
|
+
Use them as a template when adding a new embedded integration in Java / Maven — copy, search-and-replace only the package segment and the `-Dtest` filter, then wire into `config.yaml`. For other languages (Python, Node.js, Go, Rust, …), the `implement-*-testing-script` skills generate the equivalent three-script set following the same contract.
|
|
276
|
+
|
|
277
|
+
## Adding a script for a new module
|
|
278
|
+
|
|
279
|
+
1. Copy the three scripts from a working embedded-integration module into the new module's directory
|
|
280
|
+
2. Search-and-replace the package segment everywhere it appears — including the test-filter argument (the most commonly missed spot)
|
|
281
|
+
3. `chmod +x` all three (and verify ACLs on Windows for the `.ps1` siblings)
|
|
282
|
+
4. Wire them into the module's `config.yaml` via the `unittests-script`, `conformance-tests-script`, and `prepare-environment-script` keys. Use `init-config-file` to assemble the canonical form
|
|
283
|
+
5. Smoke-test by running each script manually with the renderer's output folder as `$1` (and the conformance-tests folder as `$2` for the conformance runner). Confirm exit codes match what the renderer expects
|
|
284
|
+
|
|
285
|
+
For non-Java languages, the plain-forge skills [`implement-prepare-environment-script`](../skills/implement-prepare-environment-script/SKILL.md), [`implement-unit-testing-script`](../skills/implement-unit-testing-script/SKILL.md), and [`implement-conformance-testing-script`](../skills/implement-conformance-testing-script/SKILL.md) adapt the contract above to Python, Node.js, Go, Rust, Flutter, and others. Always route script creation through these skills so the cross-cutting rules above are applied uniformly.
|
|
@@ -9,6 +9,8 @@ When an integration `.plain` module is **embedded** — meaning the generated co
|
|
|
9
9
|
|
|
10
10
|
Embedded means: the host codebase already exists, has its own language / framework / dependency manager / packaging layout, and the integration must conform to all of that without negotiation.
|
|
11
11
|
|
|
12
|
+
> **For test-script authoring**, also follow [`integration-embedded-testing.md`](integration-embedded-testing.md). It defines the per-script contract (`prepare_environment_<lang>`, `run_unittests_<lang>`, `run_conformance_tests_<lang>`) — staging into the host vs `.tmp/`, arg validation, exit codes, output parsing, the three `***implementation reqs***` entries the spec must declare so the scripts can be generated, and a Java / Maven reference implementation. This file (`integration-embedded.md`) only summarizes the test-script wiring; the testing rule is the source of truth.
|
|
13
|
+
|
|
12
14
|
## The host codebase dictates the tech stack (hard rule)
|
|
13
15
|
|
|
14
16
|
- Language, framework, dependency manager, packaging layout, coding standards, error model, logging library, and architecture are **inherited** from the host — they are **never chosen** by the integration spec
|
|
@@ -16,6 +18,29 @@ Embedded means: the host codebase already exists, has its own language / framewo
|
|
|
16
18
|
- If a Phase 3 (`forge-plain`) tech-stack question seems to push back on a host rule, treat the host as ground truth and rewrite the question
|
|
17
19
|
- Implementation reqs added in Phase 3 are **transcribed** from the host stack verbatim — host language and exact version, host framework + version, dependency manager and manifest path, packaging layout, host conventions the contract must follow, and every host-package version the contract pins
|
|
18
20
|
|
|
21
|
+
## Project layout — the integration lives under `plain/` at the host root
|
|
22
|
+
|
|
23
|
+
The embedded integration's `.plain` specs (and the rest of the ***plain project — `template/`, `resources/`, `test_scripts/`, `config.yaml`, generated `plain_modules/`) live in a single top-level `plain/` folder at the host repository root, alongside the host's own source tree:
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
<host repo>/
|
|
27
|
+
├── docs/
|
|
28
|
+
├── plain/ # ← the embedded ***plain integration lives here
|
|
29
|
+
│ ├── <module>.plain
|
|
30
|
+
│ ├── template/
|
|
31
|
+
│ ├── resources/
|
|
32
|
+
│ ├── test_scripts/
|
|
33
|
+
│ ├── plain_modules/ # generated; gitignored
|
|
34
|
+
│ └── config.yaml
|
|
35
|
+
└── src/ # host source tree (Java, Python, etc.)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
This keeps the integration self-contained in one directory the user can `cd` into to render, while leaving the host's own source layout untouched.
|
|
39
|
+
|
|
40
|
+
- If the host **already** has a `plain/` directory, adopt it verbatim — see step 1 of *Discover before you ask* below
|
|
41
|
+
- If the host does **not** yet have one, create `plain/` at the repo root; do not invent a different name (`specs/`, `plain_specs/`, …)
|
|
42
|
+
- The three test scripts still `cd` into `$HOST_CODEBASE_ROOT` (the host repo root) to compile and run tests against the host project; the `plain/` folder is the integration's **authoring** root, not its **runtime** root
|
|
43
|
+
|
|
19
44
|
## Discover before you ask
|
|
20
45
|
|
|
21
46
|
Run host discovery **before** the first Phase 1 question. Treat the results as ground truth for everything that follows.
|
|
@@ -91,47 +116,26 @@ The renderer reads the directives from the spec and the shapes from the linked s
|
|
|
91
116
|
- If two reqs are in tension (one from the host, one newly authored), the host wins; rewrite or drop the newly authored req
|
|
92
117
|
- Do not author a req that re-declares something the host already enforces — that's a maintenance burden with no benefit
|
|
93
118
|
|
|
94
|
-
## Test-script wiring —
|
|
119
|
+
## Test-script wiring — copy into the host, run tests there
|
|
95
120
|
|
|
96
|
-
Embedded integrations are tested **inside
|
|
121
|
+
Embedded integrations are tested **inside the host codebase itself**. The prepare and unit-test scripts copy the renderer's output (`$1`, i.e. `plain_modules/<module>/`) into the host's source tree at the module's package path, then compile / test the host project in place. Only the conformance script uses a `.tmp/` scratch folder, because the conformance suite is a separate project that consumes the host build as a dependency.
|
|
97
122
|
|
|
98
123
|
This matters because the integration's generated code references host symbols by their full import path (e.g. `from host_project.integrations.base import IntegrationContract`). Those imports only resolve cleanly when the test process is rooted in the host's package layout — anything else creates path edge cases that bite later in conformance failures.
|
|
99
124
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
Both scripts stage their own working copy under `.tmp/<lang>_<arg>/` per the shared testing-script rules (input folders are read-only). Inside that working folder:
|
|
103
|
-
|
|
104
|
-
1. **Copy the host codebase into the working folder.** Use a recursive copy (`rsync -a --delete`, `cp -R`, or `robocopy`) so each test run starts from a clean, identical host tree
|
|
105
|
-
2. **Overlay `plain_modules/<module>/` into the host's package tree at the target package path** recorded in the `host-codebase` concept (e.g. `plain_modules/integrations/<provider>/` → `<host_copy>/host_project/integrations/<provider>/`). Use a copy that overwrites — the generated module replaces any same-named files in the host copy
|
|
106
|
-
3. **Install dependencies inside the merged tree.** The host's own manifest (`pyproject.toml` / `package.json` / `go.mod` / …) drives the install. The integration's extra dependencies are layered on top by either (a) the renderer having written them into the host's manifest already, or (b) the script installing them explicitly after the host install
|
|
107
|
-
4. **Run the test command from inside the merged tree** — `cwd` is `<host_copy>`, and the test runner discovers tests using the host's normal layout
|
|
108
|
-
|
|
109
|
-
### Per-script responsibilities
|
|
110
|
-
|
|
111
|
-
- **`prepare_environment_<lang>`** performs the full merge once per render (host copy + plain_modules overlay + dependency install + any build artifacts) into `.tmp/<lang>_<arg>/`. The N subsequent `run_conformance_tests_<lang>` invocations attach to this populated folder (activate-only variant — see the shared testing-script rules)
|
|
112
|
-
- **`run_unittests_<lang>`** performs its **own** merge into its **own** `.tmp/<lang>_<arg>/` working folder — it does not share `prepare`'s folder, and it does not depend on `prepare` having run. The host copy + overlay + install steps are duplicated inside the unit-test script for self-containedness
|
|
113
|
-
- **`run_conformance_tests_<lang>`** does **not** re-merge. It `cd`s into the working folder that `prepare_environment` populated and runs the conformance command against the merged tree
|
|
114
|
-
|
|
115
|
-
### Language-specific merge primitives
|
|
116
|
-
|
|
117
|
-
| Language | Host copy | Overlay | Dependency install | Test invocation (inside merged tree) |
|
|
118
|
-
|----------|-----------|---------|--------------------|--------------------------------------|
|
|
119
|
-
| Python | `rsync -a <host>/ .tmp/python_<arg>/` | `rsync -a plain_modules/<module>/ .tmp/python_<arg>/<host_pkg_path>/` | `cd .tmp/python_<arg> && pip install -e .` (then integration extras) | `cd .tmp/python_<arg> && pytest …` |
|
|
120
|
-
| Node.js | `rsync -a <host>/ .tmp/node_<arg>/` | `rsync -a plain_modules/<module>/ .tmp/node_<arg>/<host_pkg_path>/` | `cd .tmp/node_<arg> && npm ci` (then integration extras) | `cd .tmp/node_<arg> && npm test …` |
|
|
121
|
-
| Go | `rsync -a <host>/ .tmp/go_<arg>/` | `rsync -a plain_modules/<module>/ .tmp/go_<arg>/<host_pkg_path>/` | `cd .tmp/go_<arg> && go mod tidy` | `cd .tmp/go_<arg> && go test ./…` |
|
|
122
|
-
| Java / Kotlin | `rsync -a <host>/ .tmp/java_<arg>/` | `rsync -a plain_modules/<module>/ .tmp/java_<arg>/<host_pkg_path>/` | `cd .tmp/java_<arg> && mvn -q -DskipTests install` | `cd .tmp/java_<arg> && mvn test …` |
|
|
123
|
-
| Rust | `rsync -a <host>/ .tmp/rust_<arg>/` | `rsync -a plain_modules/<module>/ .tmp/rust_<arg>/<host_pkg_path>/` | `cd .tmp/rust_<arg> && cargo fetch` | `cd .tmp/rust_<arg> && cargo test …` |
|
|
125
|
+
See [`integration-embedded-testing.md`](integration-embedded-testing.md) for the full per-script contract (arg validation, exit codes, idempotency, output parsing, pass criteria, cross-cutting rules). The summary that belongs in *this* file:
|
|
124
126
|
|
|
125
|
-
|
|
127
|
+
- **`prepare_environment_<lang>`** copies `$1` into the host's source tree at the module's package path, cleans the host's build-output directory, then runs the host's install / build (e.g. `mvn clean install -DskipTests`). The conformance suite later resolves the integration from the host's local dependency cache
|
|
128
|
+
- **`run_unittests_<lang>`** repeats the same copy into the host (self-contained — must work without `prepare_environment` having run first), then runs the module's unit tests + lint scoped to the module's package
|
|
129
|
+
- **`run_conformance_tests_<lang>`** copies `$2` (the conformance-tests folder) into `.tmp/<lang>_conformance/`, `cd`s in, builds the conformance project, and runs it against the build that `prepare_environment` already installed into the host
|
|
126
130
|
|
|
127
131
|
### Invariants the scripts must enforce
|
|
128
132
|
|
|
129
133
|
- **Host root is a parameter, not a literal.** No script may hardcode an absolute host path. Read the host root from an env var (e.g. `HOST_CODEBASE_ROOT`) with a sensible default matching the user's layout (e.g. `../host_project`). Surface the env var in each script's `--help` / usage banner. Capture this env var in the integration's configuration concept so it has exactly one declared name across specs, scripts, and runtime
|
|
130
|
-
- **
|
|
131
|
-
- **
|
|
132
|
-
- **
|
|
133
|
-
- **
|
|
134
|
-
-
|
|
134
|
+
- **Everything about `:UnitTests:` is declared in `***implementation reqs***`** — paths, approach, packages, framework, conventions. The prepare and unit-test scripts derive their copy destinations and test-filter argument from these reqs. See [`integration-embedded-testing.md`](integration-embedded-testing.md) for the exact reqs the spec must author (phrased in terms of `:UnitTests:`)
|
|
135
|
+
- **Everything about `:ConformanceTests:` is declared in `***test reqs***`** — paths, approach, packages, framework, execution command, pass criteria, mocking policy. The conformance script derives its build and run steps from these reqs. See [`integration-embedded-testing.md`](integration-embedded-testing.md) for the exact reqs (phrased in terms of `:ConformanceTests:`)
|
|
136
|
+
- **The two groups never overlap.** `:UnitTests:` facts belong only in `***implementation reqs***`; `:ConformanceTests:` facts belong only in `***test reqs***`. Neither lives in the `host-codebase` concept
|
|
137
|
+
- **Destructive ops are scoped to the module's own package path** under the host's source tree. `rm -rf` never touches the host's `src/main/`, `target/`, `node_modules/`, `build/`, or `dist/` at the project root. Only the module-specific package directories are wiped
|
|
138
|
+
- **Each script is idempotent.** Re-running the same script with the same `$1` yields the same result
|
|
135
139
|
|
|
136
140
|
## Embedded-specific completion checklist
|
|
137
141
|
|
|
@@ -143,13 +147,14 @@ Before declaring an embedded integration done, in addition to the shared checkli
|
|
|
143
147
|
- [ ] Every host file the integration touches is linked at its **original host-relative path** — no host file has been copied into `resources/host/` or anywhere else, and no host file contents are inlined in any spec
|
|
144
148
|
- [ ] `forge-plain` Phase 2's tech-stack decisions are transcribed verbatim from the host (no independent stack choices)
|
|
145
149
|
- [ ] Host-package version pins are copied into `***implementation reqs***`
|
|
146
|
-
- [ ] `prepare_environment` copies the host
|
|
147
|
-
- [ ] `run_unittests` runs the same host-
|
|
148
|
-
- [ ] `run_conformance_tests` `
|
|
149
|
-
- [ ] Host codebase root is read from a named env var (default value documented in each script's usage) — never hardcoded
|
|
150
|
-
- [ ]
|
|
151
|
-
- [ ]
|
|
152
|
-
- [ ]
|
|
150
|
+
- [ ] `prepare_environment` copies `$1` into the host's source tree at the module's package path, cleans the host's build-output directory, and runs the host's install / build so the conformance suite can resolve the integration from the local dependency cache
|
|
151
|
+
- [ ] `run_unittests` runs the same copy-into-host sequence (self-contained — does not depend on `prepare_environment` having run) and invokes the host's test runner scoped to the module's package
|
|
152
|
+
- [ ] `run_conformance_tests` copies `$2` into `.tmp/<lang>_conformance/`, `cd`s in, builds the conformance project, and runs it against the host build that `prepare_environment` already installed
|
|
153
|
+
- [ ] Host codebase root is read from a named env var (default value documented in each script's usage) — never hardcoded; the env var name is captured in the integration's configuration concept
|
|
154
|
+
- [ ] `***implementation reqs***` declares **everything about `:UnitTests:`** — integration source path, `:UnitTests:` source path, `:UnitTests:` package, framework + conventions, lint / static-analysis gate — per [`integration-embedded-testing.md`](integration-embedded-testing.md)
|
|
155
|
+
- [ ] `***test reqs***` declares **everything about `:ConformanceTests:`** — source location, framework + execution command, package, mocking / network policy, pass criteria, build / install needs — per [`integration-embedded-testing.md`](integration-embedded-testing.md)
|
|
156
|
+
- [ ] Neither group is duplicated across sections: `:UnitTests:` facts never appear in `***test reqs***`, `:ConformanceTests:` facts never appear in `***implementation reqs***`, and neither lives in the `host-codebase` concept
|
|
157
|
+
- [ ] Every `rm -rf` in the scripts is scoped to the module's own package directory under the host's source tree — never targets the host's `src/main/`, `target/`, `node_modules/`, `build/`, or `dist/` at the project root
|
|
153
158
|
|
|
154
159
|
## Anti-patterns specific to embedded integrations
|
|
155
160
|
|
|
@@ -160,6 +165,6 @@ Before declaring an embedded integration done, in addition to the shared checkli
|
|
|
160
165
|
- **Hardcoding the host codebase path in any spec or script.** Read it from the env var declared in the configuration concept
|
|
161
166
|
- **Asking the user to design the integration's tech stack.** Read it from the host's manifest files
|
|
162
167
|
- **Authoring an integration spec that contradicts an existing integration in the same host** without first surfacing the conflict and getting explicit user confirmation
|
|
163
|
-
- **Wiring tests with `PYTHONPATH` / `NODE_PATH` / Go `replace` directives instead of physically
|
|
164
|
-
- **
|
|
165
|
-
- **
|
|
168
|
+
- **Wiring tests with `PYTHONPATH` / `NODE_PATH` / Go `replace` directives instead of physically copying `$1` into the host's package tree.** The import-stitching approach is forbidden — every prepare / unit test starts by copying the generated module into the host's source tree at the module's package path
|
|
169
|
+
- **Letting an `rm -rf` in a test script reach the host's `src/main/`, `target/`, `node_modules/`, `build/`, or `dist/` at the project root.** Destructive ops are scoped strictly to the module's own package directory. A wrong package path in the script silently does nothing and copies files in the wrong place — both produce a green build with stale code
|
|
170
|
+
- **Running conformance tests against a stale local dependency cache.** `prepare_environment` must run before conformance for the conformance project to resolve the integration from `~/.m2` (or the language equivalent). If conformance gets invoked without a fresh prepare, dependent on which build last hit the cache, the suite tests the wrong code
|
|
@@ -100,15 +100,17 @@ Separate from the provider's API version (which lives in the provider OpenAPI fi
|
|
|
100
100
|
|
|
101
101
|
Capture as a contract-version concept; pin the version in every published schema.
|
|
102
102
|
|
|
103
|
-
## Testing — live
|
|
103
|
+
## Testing — live conformance, secrets from env, webhooks
|
|
104
104
|
|
|
105
|
-
|
|
105
|
+
`:ConformanceTests:` for a standalone integration **run against the live provider** (see [`integrations.md`](integrations.md) → *`:ConformanceTests:` always run against the live integration*). The testing strategy is captured as `***test reqs***` entries authored via `add-test-requirement`:
|
|
106
106
|
|
|
107
|
-
- **
|
|
108
|
-
- **
|
|
109
|
-
- **
|
|
110
|
-
- **
|
|
111
|
-
- **
|
|
107
|
+
- **Conformance is live by default.** No VCR cassettes, no prerecorded responses for the calls under test, no mock servers. A green conformance run that never touched the provider proves nothing. Recorded responses under `resources/fixtures/` exist for unit tests and for grounding the OpenAPI schemas — not for conformance
|
|
108
|
+
- **Secrets come from environment variables.** Pin every credential as an env-var name (e.g. `<PROVIDER>_API_KEY`, `<PROVIDER>_CLIENT_ID` + `<PROVIDER>_CLIENT_SECRET`) in `***test reqs***` and in the auth concept. Use the names the provider's docs use so users don't have to translate
|
|
109
|
+
- **The user supplies values via `.env` or the shell.** The project ships `.env.example` (gitignored `.env` for real values). CI provides the same env-var names from its secret store. The conformance script may optionally `source` a `.env` from the project root if one exists; it must verify every required var is set after that optional load and fail fast (exit `69`) on missing vars
|
|
110
|
+
- **Sandbox credentials in CI.** Name where credentials come from (CI secret store, dedicated test tenant), the rotation / leak-response policy, and the env-var names CI must set
|
|
111
|
+
- **Webhook tests** (if webhooks are in scope) must cover signature verification end-to-end — including invalid signatures and replay attempts. Signing keys are env vars, like every other secret
|
|
112
|
+
- **Rate-limit (429) tests.** The 429 path must **not** exhaust the live API's quota — use a local mock for that specific endpoint, document the exception in `***test reqs***`. Every other path remains live
|
|
113
|
+
- **Idempotency tests.** Run the same mutating call twice (with the same idempotency key) against the live sandbox and assert the same response
|
|
112
114
|
|
|
113
115
|
## Standalone-specific completion checklist
|
|
114
116
|
|
|
@@ -51,6 +51,39 @@ Documentation lies — it goes stale, omits undocumented fields, describes a dif
|
|
|
51
51
|
- **Only `GET` / `HEAD` / `OPTIONS` on the cross-check.** Mutating calls (`POST`, `PATCH`, `PUT`, `DELETE`) require explicit per-call user confirmation and must target a sandbox account.
|
|
52
52
|
- **Credentials are never written to `.plain` files or summaries.** Reference them by env-var name only.
|
|
53
53
|
|
|
54
|
+
## `:ConformanceTests:` always run against the live integration (hard rule)
|
|
55
|
+
|
|
56
|
+
Integrations exist to talk to a real third-party (or internal) API. Their `:ConformanceTests:` therefore **always run against the live integration** — no VCR cassettes, no recorded fixtures, no `nock` / `WireMock` / `MSW` mocks for the calls under test. A "green" conformance run that never touched the provider proves nothing about the integration.
|
|
57
|
+
|
|
58
|
+
- The integration's `:ConformanceTests:` **must** make real network calls to the provider (typically a sandbox / staging environment, occasionally production for read-only paths). Fixtures under `resources/fixtures/` exist for unit tests and for grounding the schemas in the OpenAPI file — they are **not** a substitute for live conformance
|
|
59
|
+
- The only exceptions are paths that **cannot** be exercised live safely:
|
|
60
|
+
- **Rate-limit (429) tests** — must not exhaust the live quota; use a local mock for that specific endpoint
|
|
61
|
+
- **Deliberately destructive failure modes** (forced 5xx) the provider doesn't let you trigger — same; mock the specific endpoint
|
|
62
|
+
Document each exception explicitly in `***test reqs***`; everything else is live by default
|
|
63
|
+
|
|
64
|
+
### Secrets come from the environment
|
|
65
|
+
|
|
66
|
+
Live conformance needs credentials. The integration spec must pin every credential as **an env var name, never a literal value**, and the conformance script must read those env vars at runtime:
|
|
67
|
+
|
|
68
|
+
- **Author the env-var names in `***test reqs***`** (using `:ConformanceTests:`) and again in the auth concept. Examples: `STRIPE_API_KEY`, `GITHUB_TOKEN`, `SALESFORCE_CLIENT_ID` + `SALESFORCE_CLIENT_SECRET`. Use names that match what the provider's own docs use, so a user copying values from the provider console doesn't need to translate
|
|
69
|
+
- **The user supplies the values out-of-band**, either:
|
|
70
|
+
- in a `.env` file at the project root (`.env` is gitignored; the project ships `.env.example` with the names but no values), or
|
|
71
|
+
- exported in the shell that invokes the test scripts (CI uses the same names from its secret store)
|
|
72
|
+
- **`run_conformance_tests_<lang>` reads the env vars at runtime.** If a required var is missing, the script must fail fast with `Error: <NAME> is required for :ConformanceTests:` and exit `69` (per the testing-script exit-code conventions). Never default to a placeholder, never use a value baked into the script
|
|
73
|
+
- **The script may optionally load a `.env` file** before running the suite — typical pattern is to look for `.env` in the project root and `source` / `dotenv -e` it if present, but never fail when it's absent (since CI provides the vars directly via the shell). If a `.env` loader is used, the script must verify each required var is set **after** loading, not before
|
|
74
|
+
- **The conformance suite reads the same env-var names** the integration's runtime reads — so a credential that works at runtime is the same credential that exercises the suite
|
|
75
|
+
- **Credentials never appear in `.plain` files, commits, summaries, logs, or fixtures.** The cross-check (see *Live API must be cross-checked*) already requires redacting credentials from saved fixtures; the same rule applies to conformance logs
|
|
76
|
+
|
|
77
|
+
The `.plain` spec should make this discoverable. A minimal `***test reqs***` block for an integration looks like:
|
|
78
|
+
|
|
79
|
+
```plain
|
|
80
|
+
- :ConformanceTests: run against the live :ProviderName: sandbox — no mocking of provider calls.
|
|
81
|
+
- Credentials are read from the environment, never from a file checked into the repo.
|
|
82
|
+
- The conformance script reads `<PROVIDER>_API_KEY` (and any additional secrets) from the shell or from a `.env` file at the project root.
|
|
83
|
+
- The script must verify every required env var is set after the optional `.env` load and fail fast with a clear error if any is missing.
|
|
84
|
+
- The 429 (rate-limit) and forced-5xx paths use a local mock for that specific endpoint; every other path is live.
|
|
85
|
+
```
|
|
86
|
+
|
|
54
87
|
## Embedded vs standalone — pick the shape early
|
|
55
88
|
|
|
56
89
|
Every integration is either **embedded** (lives as a library/module inside an existing host codebase) or **standalone** (a service, daemon, CLI, scheduled job, or container). The choice is captured as a concept (`integration-shape: embedded | standalone`) so later specs can reference it.
|
package/forge/rules/test-reqs.md
CHANGED
|
@@ -13,6 +13,12 @@ When writing or editing a `***test reqs***` section in a `.plain` file, always f
|
|
|
13
13
|
- Acceptance tests are nested under individual functional specs, not here
|
|
14
14
|
- Behavioral requirements go in `***functional specs***`, not here
|
|
15
15
|
|
|
16
|
+
## `:ConformanceTests:` lives here (hard rule)
|
|
17
|
+
- **Everything** about `:ConformanceTests:` goes in `***test reqs***` — paths, approach, packages, framework, execution command, mocking / network policy, fixtures, pass criteria, environment prerequisites
|
|
18
|
+
- `:ConformanceTests:` live outside the generated codebase (typically in a separate project under `conformance_tests/<module>/`), so requirements that shape them belong in test reqs by definition
|
|
19
|
+
- The conformance-test generator reads **only** `***test reqs***` — anything about `:ConformanceTests:` placed elsewhere (e.g. `***implementation reqs***`) is silently ignored
|
|
20
|
+
- Author each `:ConformanceTests:` requirement via `add-test-requirement` and phrase it in terms of `:ConformanceTests:` so the partition stays visible at a glance
|
|
21
|
+
|
|
16
22
|
## What belongs here
|
|
17
23
|
- Test framework: which framework to use (e.g., pytest, Unittest, xUnit)
|
|
18
24
|
- Execution method: the command to run the tests
|
|
@@ -38,7 +38,15 @@ Implementation reqs are free-form instructions that steer code generation. Commo
|
|
|
38
38
|
|
|
39
39
|
- **Behavior and features** — those go in `***functional specs***`
|
|
40
40
|
- **Concept definitions** — those go in `***definitions***`
|
|
41
|
-
- **
|
|
41
|
+
- **Conformance-test instructions** — those go in `***test reqs***`. Note: **unit-test instructions belong HERE**, not in `***test reqs***` — see *Unit-test guidance belongs here* below
|
|
42
|
+
|
|
43
|
+
## Unit-test guidance belongs here
|
|
44
|
+
|
|
45
|
+
**Everything about `:UnitTests:` goes in `***implementation reqs***`** — paths, approach, packages, framework (JUnit / pytest / Jest / Go's `testing` / …), conventions, fixtures, mocking policy, file layout, naming, lint / static-analysis gates. Unit tests are part of the generated codebase, so requirements that shape them are implementation reqs by definition.
|
|
46
|
+
|
|
47
|
+
- The unit-test generator reads **only** `***implementation reqs***`; anything about `:UnitTests:` placed in `***test reqs***` is silently ignored
|
|
48
|
+
- Phrase each `:UnitTests:` requirement in terms of the predefined `:UnitTests:` concept so the partition stays visible at a glance
|
|
49
|
+
- `***test reqs***` is exclusively for `:ConformanceTests:` — see [`add-test-requirement`](../add-test-requirement/SKILL.md)
|
|
42
50
|
|
|
43
51
|
## Key Principle: HOW vs WHAT
|
|
44
52
|
|
|
@@ -183,13 +183,19 @@ prepare-environment-script: test_scripts/prepare_environment_<language>.<sh|ps1>
|
|
|
183
183
|
|
|
184
184
|
Use `.sh` on macOS/Linux and `.ps1` on Windows, matching what testing scripts. Preserve any existing fields in a `config.yaml` you are updating.
|
|
185
185
|
|
|
186
|
+
**Hard partition reminder.** Throughout this phase:
|
|
187
|
+
|
|
188
|
+
- **Everything about `:UnitTests:`** (framework, layout, packages, conventions, execution command, coverage, mocking policy — every fact) is authored into `***implementation reqs***` via `add-implementation-requirement`. The unit-test generator reads only that section
|
|
189
|
+
- **Everything about `:ConformanceTests:`** (framework, layout, packages, execution command, mocking policy, environment prereqs — every fact) is authored into `***test reqs***` via `add-test-requirement`. The conformance-test generator reads only that section
|
|
190
|
+
- A topic that mixes both kinds of facts is split: unit facts go to impl reqs, conformance facts go to test reqs. They never share a bullet
|
|
191
|
+
|
|
186
192
|
Walk through these topics in order, running ask → author → review for each. Skip a topic only if it genuinely doesn't apply, and say so explicitly:
|
|
187
193
|
|
|
188
|
-
1. **
|
|
189
|
-
- Author: a framework requirement in `***
|
|
194
|
+
1. **Unit-test framework** — e.g. pytest, Jest, JUnit, Go's `testing` package. If the user has no preference, suggest one that fits the language chosen in Phase 2.
|
|
195
|
+
- Author: a `:UnitTests:` framework requirement in `***implementation reqs***` at the appropriate scope (template if shared, otherwise on the module) — e.g. "`:UnitTests:` should use pytest" plus "`:UnitTests:` are run via `pytest tests/`". Generate `run_unittests` (and any framework config files it needs, e.g. `pytest.ini`, `jest.config.js`) via `implement-unit-testing-script`. Add the `unittests-script:` entry to the relevant `config.yaml`(s), creating each file if it doesn't exist yet.
|
|
190
196
|
- Review: the framework req, the generated script paths, and the new `config.yaml` entry.
|
|
191
|
-
2. **
|
|
192
|
-
- Author: a
|
|
197
|
+
2. **Unit-test types and architecture mapping** — unit tests and integration tests. Which combinations does the user want? How do tests map to the architectural layers established in Phase 2 (e.g. one test module per service, repository tests with an in-memory store, etc.)?
|
|
198
|
+
- Author: a `:UnitTests:` scope / architecture requirement in `***implementation reqs***` describing which types are in scope and how they map to the architecture — phrased in terms of `:UnitTests:` so the partition is visible.
|
|
193
199
|
- Review: that requirement.
|
|
194
200
|
3. **Conformance testing** — explicitly ask whether conformance/end-to-end tests should be part of the project. Conformance testing drives whether `run_conformance_tests` is generated and whether `***acceptance tests***` are authored. If the user is unsure, briefly explain the tradeoff (extra scripts + per-spec acceptance tests vs. lighter setup) and let them choose.
|
|
195
201
|
- Author (if yes):
|
|
@@ -203,11 +209,11 @@ Walk through these topics in order, running ask → author → review for each.
|
|
|
203
209
|
- Author (if yes): `prepare_environment` via `implement-prepare-environment-script`; add the `prepare-environment-script:` entry to the relevant `config.yaml`(s); if the script's responsibilities are non-trivial and worth pinning in the spec, also add a brief `***test reqs***` entry describing what `prepare_environment` is responsible for.
|
|
204
210
|
- Author (if no): record the decision; skip the script and the config entry.
|
|
205
211
|
- Review: the script (if any), the new config entry (if any), and the test req (if any).
|
|
206
|
-
5. **Test layout & conventions** — directory layout
|
|
207
|
-
- Author: layout/convention requirements in `***test reqs***`
|
|
212
|
+
5. **Test layout & conventions** — directory layout, naming conventions, fixtures / mocks strategy, anything that constrains the *shape* of test code beyond what topics 1–4 already established. Ask about both kinds of tests where applicable; keep their facts in separate reqs in separate sections.
|
|
213
|
+
- Author: `:UnitTests:` layout / convention requirements in `***implementation reqs***`; `:ConformanceTests:` layout / convention requirements in `***test reqs***` (only when conformance is enabled). Phrase each one with the predefined concept it shapes so the partition is visible.
|
|
208
214
|
- Review: each requirement snippet.
|
|
209
|
-
6. **Execution & tooling** — how tests are run (commands, runners, options), coverage targets, CI integration, any environment setup tests rely on beyond `prepare_environment`. If the agreed execution command or options differ from what the script generated in topic 1 (or 3, or 4) currently uses, update the affected script(s) now.
|
|
210
|
-
- Author: execution requirements in `***
|
|
215
|
+
6. **Execution & tooling** — how tests are run (commands, runners, options), coverage targets, CI integration, any environment setup tests rely on beyond `prepare_environment`. Split by concept the same way as topic 5. If the agreed execution command or options differ from what the script generated in topic 1 (or 3, or 4) currently uses, update the affected script(s) now.
|
|
216
|
+
- Author: `:UnitTests:` execution requirements in `***implementation reqs***`; `:ConformanceTests:` execution requirements in `***test reqs***`. Update any affected scripts under `test_scripts/`.
|
|
211
217
|
- Review: each requirement snippet and any modified script.
|
|
212
218
|
7. **Other testing constraints** — performance/load expectations, deterministic seeds, network isolation, secrets handling, anything stack-wide that constrains *how* tests are written and that hasn't already been covered.
|
|
213
219
|
- Author: each constraint as its own requirement at the appropriate scope.
|
|
@@ -58,15 +58,17 @@ Use **AskUserQuestion** for one tight batch covering:
|
|
|
58
58
|
|
|
59
59
|
Create the import module with `create-import-module`. It must contain:
|
|
60
60
|
|
|
61
|
-
- `***implementation reqs***` — the base stack as requirements
|
|
61
|
+
- `***implementation reqs***` — the base stack as requirements **and everything about `:UnitTests:`**:
|
|
62
62
|
- Programming language and version.
|
|
63
63
|
- Primary framework (if any).
|
|
64
64
|
- Dependency / package manager.
|
|
65
65
|
- Project kind as a constraint (e.g. ":Implementation: should be a REST API service.").
|
|
66
|
+
- `:UnitTests:` framework (pytest / Jest / JUnit / Go's `testing` / …) and the command used to run them — phrased in terms of `:UnitTests:` so the partition is explicit.
|
|
66
67
|
- Anything else the user added in the free-form catch-all.
|
|
67
|
-
- `***test reqs***` — the base testing rules:
|
|
68
|
-
-
|
|
69
|
-
-
|
|
68
|
+
- `***test reqs***` — the base **conformance**-testing rules (only added if conformance testing is enabled):
|
|
69
|
+
- ":ConformanceTests: must be implemented and executed - do not skip tests."
|
|
70
|
+
- The `:ConformanceTests:` framework and the command used to run them.
|
|
71
|
+
- **Do NOT** put unit-test framework / command here — that lives in `***implementation reqs***` above.
|
|
70
72
|
|
|
71
73
|
Do **not** add a `***definitions***` section to `template/base.plain`. This skill does not author any concepts. Use only the predefined concepts (`:Implementation:`, `:ConformanceTests:`) in the reqs — no project-specific concepts like `:AppName:` or `:App:`. Do not declare `required_concepts` either.
|
|
72
74
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: load-plain-reference
|
|
3
3
|
description: >-
|
|
4
|
-
Loads the full ***plain language reference into context: syntax, section types
|
|
4
|
+
Loads the full ***plain language reference into context (PLAIN_REFERENCE.md): syntax, section types
|
|
5
5
|
(definitions, implementation reqs, test reqs, functional specs, acceptance tests),
|
|
6
6
|
concept notation, frontmatter (import/requires/required_concepts/exported_concepts),
|
|
7
7
|
templates, linked resources, module model, and authoring best practices. Use whenever
|
package/package.json
CHANGED
package/forge/docs/.gitkeep
DELETED
|
File without changes
|