testeiya 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PIPELINES.md +332 -0
- package/README.md +114 -1
- package/dist/prompt/testomat.io.js +2 -2
- package/dist/prompt/testomat.io.js.map +1 -1
- package/dist/src/args.js +104 -1
- package/dist/src/args.js.map +1 -1
- package/dist/src/checkpoint.js +0 -21
- package/dist/src/checkpoint.js.map +1 -1
- package/dist/src/cli.js +42 -1
- package/dist/src/cli.js.map +1 -1
- package/dist/src/git.js +68 -0
- package/dist/src/git.js.map +1 -0
- package/dist/src/mcp.js +4 -20
- package/dist/src/mcp.js.map +1 -1
- package/dist/src/output.js +16 -3
- package/dist/src/output.js.map +1 -1
- package/dist/src/pipeline/command-stage.js +27 -0
- package/dist/src/pipeline/command-stage.js.map +1 -0
- package/dist/src/pipeline/diagram.js +103 -0
- package/dist/src/pipeline/diagram.js.map +1 -0
- package/dist/src/pipeline/gate.js +59 -0
- package/dist/src/pipeline/gate.js.map +1 -0
- package/dist/src/pipeline/launcher.js +74 -0
- package/dist/src/pipeline/launcher.js.map +1 -0
- package/dist/src/pipeline/log.js +70 -0
- package/dist/src/pipeline/log.js.map +1 -0
- package/dist/src/pipeline/pipeline-file.js +242 -0
- package/dist/src/pipeline/pipeline-file.js.map +1 -0
- package/dist/src/pipeline/pipeline.js +415 -0
- package/dist/src/pipeline/pipeline.js.map +1 -0
- package/dist/src/pipeline/report.js +61 -0
- package/dist/src/pipeline/report.js.map +1 -0
- package/dist/src/pipeline/spec.js +2 -0
- package/dist/src/pipeline/spec.js.map +1 -0
- package/dist/src/pipeline/stage.js +150 -0
- package/dist/src/pipeline/stage.js.map +1 -0
- package/dist/src/pipeline/state.js +70 -0
- package/dist/src/pipeline/state.js.map +1 -0
- package/dist/src/pipeline/task-stage.js +84 -0
- package/dist/src/pipeline/task-stage.js.map +1 -0
- package/dist/src/pipeline/variables.js +41 -0
- package/dist/src/pipeline/variables.js.map +1 -0
- package/dist/src/run.js +6 -3
- package/dist/src/run.js.map +1 -1
- package/mcp-tools.json +1092 -2110
- package/package.json +7 -5
- package/pipelines/api-testing.yml +39 -0
- package/pipelines/issue-test-cases.yml +41 -0
- package/pipelines/pr-review.yml +43 -0
- package/pipelines/regression.yml +39 -0
- package/pipelines/web-testing.yml +50 -0
- package/prompt/testomat.io.ts +5 -5
- package/skills/skills.lock.json +2 -2
- package/skills/testomatio/test-management/detect-useless-tests/SKILL.md +1 -1
- package/skills/testomatio/test-management/improve-test-cases/SKILL.md +1 -1
- package/skills/testomatio/test-management/qa-test-code-coverage/SKILL.md +2 -2
- package/skills/testomatio/test-management/qa-write-test-cases/SKILL.md +3 -3
- package/skills/testomatio/test-management/scan-automation-project/SKILL.md +8 -8
- package/skills/testomatio/test-management/sync-test-cases-with-tms/SKILL.md +9 -9
- package/skills/testomatio/test-management/sync-test-cases-with-tms/references/TESTOMATIO_CLI.md +4 -4
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testeiya",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "AI testing agent — QA-focused coding agent for manual and automated tests",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
"files": [
|
|
16
16
|
"dist",
|
|
17
17
|
"prompt",
|
|
18
|
+
"pipelines",
|
|
19
|
+
"PIPELINES.md",
|
|
18
20
|
"skills",
|
|
19
21
|
"mcp-tools.json",
|
|
20
22
|
"README.md",
|
|
@@ -47,17 +49,17 @@
|
|
|
47
49
|
"@earendil-works/pi-ai": "^0.84.3",
|
|
48
50
|
"@earendil-works/pi-coding-agent": "^0.84.3",
|
|
49
51
|
"@earendil-works/pi-tui": "^0.84.3",
|
|
50
|
-
"@testomatio/mcp": "^
|
|
52
|
+
"@testomatio/mcp": "^3.1.0",
|
|
51
53
|
"chalk": "^5.6.2",
|
|
52
54
|
"dedent": "^1.7.2",
|
|
53
55
|
"pi-mcp-adapter": "^2.21.1",
|
|
54
|
-
"typebox": "1.3.7"
|
|
56
|
+
"typebox": "1.3.7",
|
|
57
|
+
"yaml": "^2.9.0"
|
|
55
58
|
},
|
|
56
59
|
"devDependencies": {
|
|
57
60
|
"@types/node": "^22.19.19",
|
|
58
61
|
"esbuild": "^0.28.1",
|
|
59
62
|
"tsx": "^4.23.12",
|
|
60
|
-
"typescript": "^5.9.3"
|
|
61
|
-
"yaml": "^2.9.0"
|
|
63
|
+
"typescript": "^5.9.3"
|
|
62
64
|
}
|
|
63
65
|
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: api-testing
|
|
2
|
+
description: Find the API endpoints a pull request changed, test each with Explorbot, explain what broke.
|
|
3
|
+
|
|
4
|
+
gates:
|
|
5
|
+
api:
|
|
6
|
+
env: [EXPLORBOT_URL, EXPLORBOT_API_SPEC]
|
|
7
|
+
message: No API to test. Set EXPLORBOT_URL to the API base and EXPLORBOT_API_SPEC to its OpenAPI spec.
|
|
8
|
+
explorbot-ai:
|
|
9
|
+
env: [EXPLORBOT_AI_PROVIDER]
|
|
10
|
+
message: Explorbot has no AI provider. Set EXPLORBOT_AI_PROVIDER and its key.
|
|
11
|
+
|
|
12
|
+
env:
|
|
13
|
+
EXPLORBOT_OUTPUT: ${PIPELINE_DIR}/explorbot
|
|
14
|
+
EXPLORBOT_NO_BANNER: "1"
|
|
15
|
+
|
|
16
|
+
stages:
|
|
17
|
+
- id: review
|
|
18
|
+
session: qa
|
|
19
|
+
task: |
|
|
20
|
+
Review this pull request as a QA engineer. /qa-review-pr
|
|
21
|
+
List every API endpoint it adds or changes in endpoints.txt, one path per line, like /users/{id}.
|
|
22
|
+
Leave the file empty when no endpoint changed.
|
|
23
|
+
outputs: [endpoints.txt]
|
|
24
|
+
|
|
25
|
+
- id: explore
|
|
26
|
+
when: [api, explorbot-ai]
|
|
27
|
+
each_line: endpoints.txt
|
|
28
|
+
run: npx -y explorbot api explore "$ITEM" --style psycho
|
|
29
|
+
outputs: [explorbot/]
|
|
30
|
+
timeout: 10m
|
|
31
|
+
|
|
32
|
+
- id: results
|
|
33
|
+
session: qa
|
|
34
|
+
when: [api, explorbot-ai]
|
|
35
|
+
task: |
|
|
36
|
+
Explorbot results are under explorbot/: plans, request logs, and a session report. Read them.
|
|
37
|
+
Say which failures are real defects in this change: a broken contract, missing validation, a wrong status.
|
|
38
|
+
Fail the run if any is a real defect.
|
|
39
|
+
inputs: [endpoints.txt, explorbot/]
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
name: issue-test-cases
|
|
2
|
+
description: Turn an issue or epic into test cases, and sync them to Testomat.io.
|
|
3
|
+
|
|
4
|
+
gates:
|
|
5
|
+
issue:
|
|
6
|
+
env: [ISSUE_URL]
|
|
7
|
+
message: No issue. Set ISSUE_URL to the issue or epic to cover.
|
|
8
|
+
testomatio:
|
|
9
|
+
env: [TESTOMATIO, TESTOMATIO_PROJECT_ID]
|
|
10
|
+
message: No Testomat.io project. Set TESTOMATIO and TESTOMATIO_PROJECT_ID.
|
|
11
|
+
|
|
12
|
+
when: [issue]
|
|
13
|
+
|
|
14
|
+
stages:
|
|
15
|
+
- id: requirements
|
|
16
|
+
session: analyst
|
|
17
|
+
task: |
|
|
18
|
+
Read the issue at ${ISSUE_URL}: its body, its comments, and every linked issue.
|
|
19
|
+
An epic has sub-issues. Read each one. On GitHub, `gh issue view <url> --comments`
|
|
20
|
+
reads an issue and `gh api repos/<owner>/<repo>/issues/<number>/sub_issues` lists sub-issues.
|
|
21
|
+
Review the requirements as a QA engineer. /qa-requirement-reviewer
|
|
22
|
+
Write the acceptance criteria to criteria.md, one section per sub-issue or feature area.
|
|
23
|
+
End it with the open questions and gaps you found.
|
|
24
|
+
The code here may not implement the issue yet. Read it only to use the real names of screens and terms.
|
|
25
|
+
outputs: [criteria.md]
|
|
26
|
+
|
|
27
|
+
- id: test-cases
|
|
28
|
+
session: writer
|
|
29
|
+
task: |
|
|
30
|
+
Write test cases from criteria.md. /qa-write-test-cases
|
|
31
|
+
Put them in test-cases/, one markdown file per suite, one suite per sub-issue or feature area.
|
|
32
|
+
Cover every criterion. A case that depends on an open question says so in its description.
|
|
33
|
+
inputs: [criteria.md]
|
|
34
|
+
outputs: [test-cases/]
|
|
35
|
+
|
|
36
|
+
- id: sync
|
|
37
|
+
when: [testomatio]
|
|
38
|
+
each_file: test-cases/*.md
|
|
39
|
+
task: |
|
|
40
|
+
Sync the test cases in ${ITEM} to Testomat.io. /sync-test-cases-with-tms
|
|
41
|
+
Skip a case that already exists there.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
name: pr-review
|
|
2
|
+
description: Review a pull request, propose test cases for it, and sync them to Testomat.io.
|
|
3
|
+
|
|
4
|
+
gates:
|
|
5
|
+
code-changed:
|
|
6
|
+
changes:
|
|
7
|
+
min_files: 3
|
|
8
|
+
ignore: ["**/*.md", "**/*.lock", "docs/**"]
|
|
9
|
+
message: Fewer than 3 code files changed.
|
|
10
|
+
testomatio:
|
|
11
|
+
env: [TESTOMATIO, TESTOMATIO_PROJECT_ID]
|
|
12
|
+
message: No Testomat.io project. Set TESTOMATIO and TESTOMATIO_PROJECT_ID.
|
|
13
|
+
|
|
14
|
+
when: [code-changed]
|
|
15
|
+
|
|
16
|
+
stages:
|
|
17
|
+
- id: requirements
|
|
18
|
+
session: reviewer
|
|
19
|
+
task: |
|
|
20
|
+
Work out what this pull request is meant to do. /qa-pr-requirements-analyzer
|
|
21
|
+
Write the acceptance criteria to criteria.md.
|
|
22
|
+
outputs: [criteria.md]
|
|
23
|
+
|
|
24
|
+
- id: review
|
|
25
|
+
session: reviewer
|
|
26
|
+
task: |
|
|
27
|
+
Review this pull request as a QA engineer against criteria.md. /qa-review-pr
|
|
28
|
+
inputs: [criteria.md]
|
|
29
|
+
|
|
30
|
+
- id: test-cases
|
|
31
|
+
session: writer
|
|
32
|
+
task: |
|
|
33
|
+
Write test cases for this pull request from criteria.md and review.md. /qa-write-test-cases
|
|
34
|
+
Put them in test-cases/, one markdown file per suite.
|
|
35
|
+
inputs: [criteria.md, review.md]
|
|
36
|
+
outputs: [test-cases/]
|
|
37
|
+
|
|
38
|
+
- id: sync
|
|
39
|
+
when: [testomatio]
|
|
40
|
+
each_file: test-cases/*.md
|
|
41
|
+
task: |
|
|
42
|
+
Sync the test cases in ${ITEM} to Testomat.io. /sync-test-cases-with-tms
|
|
43
|
+
Skip a case that already exists there.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: regression
|
|
2
|
+
description: Run the tests a change can break, from the coverage map, then explain the failures.
|
|
3
|
+
|
|
4
|
+
gates:
|
|
5
|
+
coverage:
|
|
6
|
+
file: [coverage.tests.yml]
|
|
7
|
+
message: No coverage.tests.yml. Create one with /qa-test-code-coverage to run only affected tests.
|
|
8
|
+
testomatio:
|
|
9
|
+
env: [TESTOMATIO]
|
|
10
|
+
message: No Testomat.io project. Set TESTOMATIO to the project's API key.
|
|
11
|
+
local:
|
|
12
|
+
run: test -z "$REMOTE"
|
|
13
|
+
message: Tests run on the Testomat.io CI profile. No local log to read.
|
|
14
|
+
test-command:
|
|
15
|
+
env: [TEST_COMMAND]
|
|
16
|
+
message: No test command. Set TEST_COMMAND, e.g. "npx playwright test".
|
|
17
|
+
|
|
18
|
+
when: [coverage]
|
|
19
|
+
|
|
20
|
+
env:
|
|
21
|
+
BASE_BRANCH: ${BASE_BRANCH:-origin/main}
|
|
22
|
+
|
|
23
|
+
stages:
|
|
24
|
+
- id: tests
|
|
25
|
+
when: [testomatio]
|
|
26
|
+
run: |
|
|
27
|
+
npx -y @testomatio/reporter run --kind detect --warn \
|
|
28
|
+
--filter "coverage:file=coverage.tests.yml,diff=$BASE_BRANCH" \
|
|
29
|
+
${REMOTE:+--remote "$REMOTE"} ${TEST_COMMAND:+"$TEST_COMMAND"}
|
|
30
|
+
timeout: 30m
|
|
31
|
+
|
|
32
|
+
- id: failures
|
|
33
|
+
when: [local, test-command]
|
|
34
|
+
session: triage
|
|
35
|
+
task: |
|
|
36
|
+
Read the test run in tests.log. For each failure, say if this change caused it,
|
|
37
|
+
if it is flaky, or if the test is out of date. /debug-fix-failed-flaky-autotests
|
|
38
|
+
Do not edit any file. Fail the run if this change caused a failure.
|
|
39
|
+
inputs: [tests.log]
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
name: web-testing
|
|
2
|
+
description: Review a pull request, test the change in a browser with Explorbot, explain what broke.
|
|
3
|
+
|
|
4
|
+
gates:
|
|
5
|
+
staging:
|
|
6
|
+
env: [EXPLORBOT_URL]
|
|
7
|
+
message: No staging URL. Set EXPLORBOT_URL to test in a browser.
|
|
8
|
+
browser:
|
|
9
|
+
run: ls "${PLAYWRIGHT_BROWSERS_PATH:-$HOME/.cache/ms-playwright}" | grep -q chromium
|
|
10
|
+
message: No browser. Run npx playwright install --with-deps chromium.
|
|
11
|
+
explorbot-ai:
|
|
12
|
+
env: [EXPLORBOT_AI_PROVIDER]
|
|
13
|
+
message: Explorbot has no AI provider. Set EXPLORBOT_AI_PROVIDER and its key.
|
|
14
|
+
|
|
15
|
+
env:
|
|
16
|
+
EXPLORBOT_OUTPUT: ${PIPELINE_DIR}/explorbot
|
|
17
|
+
EXPLORBOT_NO_BANNER: "1"
|
|
18
|
+
|
|
19
|
+
stages:
|
|
20
|
+
- id: review
|
|
21
|
+
session: qa
|
|
22
|
+
task: |
|
|
23
|
+
Review this pull request as a QA engineer. /qa-review-pr
|
|
24
|
+
Write the user-facing scenarios worth checking in a browser to scenarios.md.
|
|
25
|
+
outputs: [scenarios.md]
|
|
26
|
+
|
|
27
|
+
- id: write-plan
|
|
28
|
+
session: qa
|
|
29
|
+
when: [staging]
|
|
30
|
+
task: |
|
|
31
|
+
Turn scenarios.md into an Explorbot test plan for ${EXPLORBOT_URL}. /explorbot-plan
|
|
32
|
+
Write the plan to plan.md. Keep only scenarios a browser can check.
|
|
33
|
+
inputs: [scenarios.md]
|
|
34
|
+
outputs: [plan.md]
|
|
35
|
+
|
|
36
|
+
- id: explore
|
|
37
|
+
when: [staging, browser, explorbot-ai]
|
|
38
|
+
run: npx -y explorbot test "$PIPELINE_DIR/plan.md" '*'
|
|
39
|
+
inputs: [plan.md]
|
|
40
|
+
outputs: [explorbot/]
|
|
41
|
+
timeout: 30m
|
|
42
|
+
|
|
43
|
+
- id: results
|
|
44
|
+
session: qa
|
|
45
|
+
when: [staging, browser, explorbot-ai]
|
|
46
|
+
task: |
|
|
47
|
+
Explorbot results are under explorbot/. Find its session report and read it.
|
|
48
|
+
Say which failed scenarios are real defects in this change and which are test noise.
|
|
49
|
+
Fail the run if any is a real defect.
|
|
50
|
+
inputs: [plan.md, explorbot/]
|
package/prompt/testomat.io.ts
CHANGED
|
@@ -39,18 +39,18 @@ export function testomatio(options: TestomatioOptions): string {
|
|
|
39
39
|
|
|
40
40
|
function dynamicData(tms: TmsAccess): string {
|
|
41
41
|
if (tms === "mcp-proxy") {
|
|
42
|
-
return "Get them through the
|
|
42
|
+
return "Get them through the Testomat.io tools, one per entity (`tests`, `runs`, `testruns`, `issues`, …); pick the operation with its `command` param. The `mcp` tool searches them when the name is unclear.";
|
|
43
43
|
}
|
|
44
44
|
if (tms === "cli-only") {
|
|
45
45
|
return "There are no Testomat.io tools in this session. Get them from the REST API: `curl` with the `TESTOMATIO` token as the Authorization header against `$TESTOMATIO_URL/api/v2`. Keep to documented endpoints; never invent paths or parameters.";
|
|
46
46
|
}
|
|
47
|
-
return "Get them through the `testomatio-<slug>` MCP tools, one set per project.";
|
|
47
|
+
return "Get them through the `testomatio-<slug>` MCP tools, one set per project and one tool per entity; pick the operation with its `command` param.";
|
|
48
48
|
}
|
|
49
49
|
|
|
50
50
|
/**
|
|
51
|
-
* `mcp-proxy`: one `mcp`
|
|
52
|
-
* all; `check-tests` and REST through the shell. `mcp-direct`:
|
|
53
|
-
*
|
|
51
|
+
* `mcp-proxy`: one tool per entity plus the `mcp` search tool. `cli-only`: no
|
|
52
|
+
* tools at all; `check-tests` and REST through the shell. `mcp-direct`: one tool
|
|
53
|
+
* per entity, prefixed `testomatio-<slug>`; the desktop app's one-shot mode.
|
|
54
54
|
*/
|
|
55
55
|
export type TmsAccess = "mcp-direct" | "mcp-proxy" | "cli-only";
|
|
56
56
|
|
package/skills/skills.lock.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
{
|
|
4
4
|
"source": "testomatio/skills",
|
|
5
5
|
"ref": null,
|
|
6
|
-
"sha": "
|
|
6
|
+
"sha": "f971b5264682260d0257e2e21f1dfa20836fe8d3",
|
|
7
7
|
"folder": "testomatio",
|
|
8
8
|
"skills": [
|
|
9
9
|
"automate-manual-test-cases",
|
|
@@ -76,7 +76,7 @@
|
|
|
76
76
|
{
|
|
77
77
|
"source": "microsoft/playwright-cli/tree/main/skills/playwright-cli",
|
|
78
78
|
"ref": null,
|
|
79
|
-
"sha": "
|
|
79
|
+
"sha": "b1a5903c5b633b8c9f843ad7b629c2a78437040c",
|
|
80
80
|
"folder": "playwright",
|
|
81
81
|
"skills": [
|
|
82
82
|
"playwright-cli"
|
|
@@ -3,7 +3,7 @@ name: detect-useless-tests
|
|
|
3
3
|
description: Find useless tests within one testing level (unit, e2e, manual) and list them by area, file, and line for review. Use when asked "which tests can we delete?", "find useless tests", or "what in our suite is not worth keeping?".
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Detect Useless
|
|
6
|
+
# Detect Useless Tests
|
|
7
7
|
|
|
8
8
|
Go through tests of the same level (unit, e2e, manual) and detect useless ones.
|
|
9
9
|
User must provide exact locations of tests to check to identify the level of tests in pyramid.
|
|
@@ -24,7 +24,7 @@ Analyze and improve existing markdown manual test cases to make them clearer for
|
|
|
24
24
|
|
|
25
25
|
## Step 1: Locate Test Files
|
|
26
26
|
|
|
27
|
-
- Find markdown test files (`.test.md` or `.md`) in common locations: `./
|
|
27
|
+
- Find markdown test files (`.test.md` or `.md`) in common locations: `./test-cases`, `./tests/manual`, `./docs/tests`.
|
|
28
28
|
- If none found, scan the whole project for `.md` files with test case content.
|
|
29
29
|
- If still none, ask the user for the test directory path.
|
|
30
30
|
|
|
@@ -36,7 +36,7 @@ Analyze the project's tests — manual markdown cases and automated e2e tests
|
|
|
36
36
|
- Project Overview — languages, frameworks, complexity (drives the subagent split in Step 4).
|
|
37
37
|
- Map whatever exists: both kinds go into one coverage file; a single kind is fine too.
|
|
38
38
|
- ❓ If the scan finds no tests at all (it checks the `.testeiya/` cache too), ask the user:
|
|
39
|
-
1. Pull manual cases from Testomat.io — have `sync-test-cases-with-tms` pull into the cache: `npx check-tests pull -d .testeiya/
|
|
39
|
+
1. Pull manual cases from Testomat.io — have `sync-test-cases-with-tms` pull into the cache: `npx check-tests pull -d .testeiya/test-cases`, add `.testeiya/` to `.gitignore` if missing, re-run the scan.
|
|
40
40
|
2. Clone the automated tests repo: `git clone <url> .testeiya/e2e-tests`, add `.testeiya/` to `.gitignore` if missing, re-run the scan.
|
|
41
41
|
3. Point to a directory the scan missed, then re-run the scan there.
|
|
42
42
|
4. Stop.
|
|
@@ -165,7 +165,7 @@ npx @testomatio/reporter run --kind manual \
|
|
|
165
165
|
- Coverage gaps — source features no test maps to. On approval, propose new cases (delegate to `qa-write-test-cases`).
|
|
166
166
|
- Dead tests — tests whose features no longer exist in source.
|
|
167
167
|
- Answer questions like "do we have tests for X?" from the inventory.
|
|
168
|
-
- Editing pulled manual cases: edit them in `.testeiya/
|
|
168
|
+
- Editing pulled manual cases: edit them in `.testeiya/test-cases/` and push back with `sync-test-cases-with-tms`.
|
|
169
169
|
|
|
170
170
|
## References
|
|
171
171
|
|
|
@@ -61,7 +61,7 @@ Information sources:
|
|
|
61
61
|
- Design mockups (Figma, Miro, etc.)
|
|
62
62
|
- Existing test cases (TestRail, Testomat.io, etc.)
|
|
63
63
|
- Source code:
|
|
64
|
-
- manual test files: `.test.md` or `.md` in folders like `
|
|
64
|
+
- manual test files: `.test.md` or `.md` in folders like `test-cases`, `tests`, `manual`, `qa`, `spec`
|
|
65
65
|
- automated test files (spec, test, cy files, etc.)
|
|
66
66
|
- project structure
|
|
67
67
|
|
|
@@ -218,8 +218,8 @@ If the user originally asked only for a checklist, first ask: "Do you want to ge
|
|
|
218
218
|
Where to store test cases:
|
|
219
219
|
|
|
220
220
|
- Empty or manual-tests-only workspace → project root
|
|
221
|
-
- End-to-end testing project → `
|
|
222
|
-
- Any other case → `.testeiya/
|
|
221
|
+
- End-to-end testing project → `test-cases/` directory
|
|
222
|
+
- Any other case → `.testeiya/test-cases/` (ensure `.testeiya/` exists and is git-ignored)
|
|
223
223
|
|
|
224
224
|
Generation rules:
|
|
225
225
|
|
|
@@ -36,7 +36,7 @@ All persistent QA metadata lives in `.testeiya/` in the project root.
|
|
|
36
36
|
- `<name>/`: a linked or cloned external project, named after its source. Read-only reference.
|
|
37
37
|
- `requirements/`: user stories and acceptance criteria (pdfs, docs, images).
|
|
38
38
|
- `docs/`: feature explanations, test planning and strategy files. Prose for humans, never data dumps.
|
|
39
|
-
- `
|
|
39
|
+
- `test-cases/`: markdown test cases pulled from Testomat.io.
|
|
40
40
|
- `auto-tests/`: references to the relevant automated (e2e) tests.
|
|
41
41
|
- `exploratory/`: explorbot setup.
|
|
42
42
|
- `project-info.json`: the project's settings as Testomat.io reports them, when cached.
|
|
@@ -53,13 +53,13 @@ Rules:
|
|
|
53
53
|
| This project has | Store pulled data in |
|
|
54
54
|
| ------------------ | ---------------------- |
|
|
55
55
|
| **Source code** (`src/` folder) or **big project** | `.testeiya/…` |
|
|
56
|
-
| **Only test infrastructure** (e2e dirs like `tests/`, `playwright/`, `cypress/` but no `src/`) | `
|
|
57
|
-
| **Empty / manual-only** | `
|
|
56
|
+
| **Only test infrastructure** (e2e dirs like `tests/`, `playwright/`, `cypress/` but no `src/`) | `test-cases/` or `e2e-tests/` |
|
|
57
|
+
| **Empty / manual-only** | `test-cases/` |
|
|
58
58
|
|
|
59
59
|
Detection logic:
|
|
60
60
|
1. Has `src/` folder or is a monorepo? → Use `.testeiya/`
|
|
61
|
-
2. Has e2e test dirs (`tests/`, `playwright/`, `cypress/`, `e2e/`)? → Use tracked folder (`
|
|
62
|
-
3. Otherwise → Default to `
|
|
61
|
+
2. Has e2e test dirs (`tests/`, `playwright/`, `cypress/`, `e2e/`)? → Use tracked folder (`test-cases/`, `e2e-tests/`)
|
|
62
|
+
3. Otherwise → Default to `test-cases/`
|
|
63
63
|
|
|
64
64
|
- The rule covers *pulled* data only. Output a skill *produces* and the user wants — a `coverage.*.yml`, new `*.test.md` files — goes in the repo as normal.
|
|
65
65
|
- On a source repo with no manual tests, this skill changes no tracked file: it only creates the self-ignored `.testeiya/` directory.
|
|
@@ -107,7 +107,7 @@ A `*.test.md` file is a suite. Each test inside it is a `<!-- test ... -->` bloc
|
|
|
107
107
|
- `type: automated`: a **reference** to an automated test (see below).
|
|
108
108
|
- No `type:` line at all: treat as manual. This is common, so never count manual tests by grepping for `type: manual`; it silently undercounts.
|
|
109
109
|
|
|
110
|
-
Know the counts before you answer anything about project scope, coverage or progress. Read them straight from the files; this mirrors how Testomat.io parses them. `find .` also looks inside `.testeiya/
|
|
110
|
+
Know the counts before you answer anything about project scope, coverage or progress. Read them straight from the files; this mirrors how Testomat.io parses them. `find .` also looks inside `.testeiya/test-cases/`, so a re-run after a pull finds the cached cases instead of reporting "no manual tests":
|
|
111
111
|
|
|
112
112
|
```bash
|
|
113
113
|
# one line per suite or test: SUITE <title>, or <type> <title>
|
|
@@ -127,7 +127,7 @@ find . -name "*.test.md" -exec awk '
|
|
|
127
127
|
|
|
128
128
|
Pipe the output into `grep -c '\[manual\]'` or `grep -c '\[automated\]'` to count one kind.
|
|
129
129
|
|
|
130
|
-
- If the only `.test.md` files are under `.testeiya/
|
|
130
|
+
- If the only `.test.md` files are under `.testeiya/test-cases/`, say so in the inventory: they came from Testomat.io, not from this repo.
|
|
131
131
|
- A source checkout with no `*.test.md` files has nothing to count. Say so; do not estimate.
|
|
132
132
|
|
|
133
133
|
### Automated tests
|
|
@@ -186,7 +186,7 @@ Return one structured markdown result directly. **Do NOT save to a file.**
|
|
|
186
186
|
## Test Inventory
|
|
187
187
|
|
|
188
188
|
- **Automated Tests:** 10 files (Playwright)
|
|
189
|
-
- **Manual Tests:** 37 cases (`.testeiya/
|
|
189
|
+
- **Manual Tests:** 37 cases (`.testeiya/test-cases/`, pulled from Testomat.io)
|
|
190
190
|
- **Automated References:** 12 (implementation lives in another repository)
|
|
191
191
|
|
|
192
192
|
### Manual Tests (20 of 37 shown)
|
|
@@ -38,18 +38,18 @@ Downloads test scenarios from Testomat.io and saves them as local Markdown files
|
|
|
38
38
|
|
|
39
39
|
Directory selection — if the `testDir` input is provided or the user passes `-d <path>`, use that path. Otherwise:
|
|
40
40
|
- Workspace is empty or for manual tests only: pull to root.
|
|
41
|
-
- End-to-end testing project: pull to the `
|
|
42
|
-
- Any other case: pull to `.testeiya/
|
|
41
|
+
- End-to-end testing project: pull to the `test-cases` directory.
|
|
42
|
+
- Any other case: pull to `.testeiya/test-cases` (ensure `.testeiya/` exists and is gitignored).
|
|
43
43
|
|
|
44
44
|
Cache-folder rules:
|
|
45
|
-
- Default pull target is the gitignored cache `.testeiya/
|
|
46
|
-
- **Pulled cases must never land in a tracked folder** (`
|
|
45
|
+
- Default pull target is the gitignored cache `.testeiya/test-cases/`. Add `.testeiya/` to the project `.gitignore` if it is not there yet.
|
|
46
|
+
- **Pulled cases must never land in a tracked folder** (`test-cases/`, etc.) — that pollutes the repo.
|
|
47
47
|
- Being gitignored does not block editing the files or pushing them back.
|
|
48
48
|
- If the repo already keeps its `*.test.md` files in a tracked folder, don't pull — work with them in place, or pass `-d <that folder>` for an in-place refresh.
|
|
49
49
|
- This matches `scan-automation-project`, which pulls the *code* into `.testeiya/code/` when it runs inside a manual-tests repo.
|
|
50
50
|
|
|
51
51
|
```bash
|
|
52
|
-
npx check-tests pull -d .testeiya/
|
|
52
|
+
npx check-tests pull -d .testeiya/test-cases
|
|
53
53
|
```
|
|
54
54
|
|
|
55
55
|
- To pull only specific suites (user names a suite or gives suite IDs): `npx check-tests pull --suite-ids "@S12345678,@S87654321"`.
|
|
@@ -62,7 +62,7 @@ Uploads local Markdown tests to Testomat.io.
|
|
|
62
62
|
Pre-push file filtering — push only test case files, never project docs or requirements:
|
|
63
63
|
- Include files matching `*.test.md`, or Markdown files with valid `<!-- test ... -->` blocks.
|
|
64
64
|
- Ignore all other Markdown (`README.md`, `CHANGELOG.md`, documentation).
|
|
65
|
-
- If test cases live alongside other files, copy or move the test files into `.testeiya/
|
|
65
|
+
- If test cases live alongside other files, copy or move the test files into `.testeiya/test-cases/` first.
|
|
66
66
|
|
|
67
67
|
Pre-push validation:
|
|
68
68
|
1. At least one `*.test.md` file exists.
|
|
@@ -85,13 +85,13 @@ labels: ...
|
|
|
85
85
|
npx check-tests push --files login.test.md checkout.test.md
|
|
86
86
|
|
|
87
87
|
# Testeiya cache folder
|
|
88
|
-
npx check-tests push -d .testeiya/
|
|
88
|
+
npx check-tests push -d .testeiya/test-cases
|
|
89
89
|
```
|
|
90
90
|
|
|
91
91
|
- **When the files to push are known** (e.g. just produced by `qa-write-test-cases` / `improve-test-cases`), **pass them explicitly via `--files`** (alias `-f`). Without `--files` the CLI falls back to the default glob `**/*.test.md`, which may pick up unrelated files.
|
|
92
92
|
- Quote glob patterns. Paths resolve relative to `--dir`.
|
|
93
|
-
- Use `-d` when `.testeiya` or `
|
|
94
|
-
- **Push only the test cases directory** (e.g. `.testeiya/
|
|
93
|
+
- Use `-d` when `.testeiya` or `test-cases` directories exist.
|
|
94
|
+
- **Push only the test cases directory** (e.g. `.testeiya/test-cases`, not `.testeiya/`).
|
|
95
95
|
- More push options and examples: [TESTOMATIO_CLI.md](./references/TESTOMATIO_CLI.md).
|
|
96
96
|
|
|
97
97
|
## Summary Output
|
package/skills/testomatio/test-management/sync-test-cases-with-tms/references/TESTOMATIO_CLI.md
CHANGED
|
@@ -85,11 +85,11 @@ npx check-tests pull [options]
|
|
|
85
85
|
# Export tests to current directory
|
|
86
86
|
npx check-tests pull
|
|
87
87
|
|
|
88
|
-
# Export tests to
|
|
89
|
-
npx check-tests pull -d
|
|
88
|
+
# Export tests to test-cases folder
|
|
89
|
+
npx check-tests pull -d test-cases
|
|
90
90
|
|
|
91
91
|
# Keep source structure
|
|
92
|
-
npx check-tests pull -d
|
|
92
|
+
npx check-tests pull -d test-cases --keep-structure
|
|
93
93
|
|
|
94
94
|
# Pull specific suites only
|
|
95
95
|
npx check-tests pull --suite-ids "@S12345678,@S87654321"
|
|
@@ -117,7 +117,7 @@ npx check-tests push
|
|
|
117
117
|
npx check-tests push -f docs/login.test.md docs/checkout.test.md
|
|
118
118
|
|
|
119
119
|
# Glob (quoted)
|
|
120
|
-
npx check-tests push --files "
|
|
120
|
+
npx check-tests push --files "test-cases/**/*.test.md"
|
|
121
121
|
|
|
122
122
|
# Multiple globs
|
|
123
123
|
npx check-tests push -f "smoke/**/*.test.md" "regression/**/*.test.md"
|