testeiya 0.3.9 → 0.3.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/README.md +353 -27
  2. package/dist/prompt/index.js +6 -2
  3. package/dist/prompt/index.js.map +1 -1
  4. package/dist/prompt/print.js +8 -0
  5. package/dist/prompt/print.js.map +1 -1
  6. package/dist/prompt/system-prompt.js +14 -4
  7. package/dist/prompt/system-prompt.js.map +1 -1
  8. package/dist/prompt/testomatio.js +3 -3
  9. package/dist/prompt/tools.js +28 -7
  10. package/dist/prompt/tools.js.map +1 -1
  11. package/dist/src/args.js +198 -46
  12. package/dist/src/args.js.map +1 -1
  13. package/dist/src/cli.js +64 -20
  14. package/dist/src/cli.js.map +1 -1
  15. package/dist/src/doctor.js +171 -0
  16. package/dist/src/doctor.js.map +1 -0
  17. package/dist/src/env.js +39 -5
  18. package/dist/src/env.js.map +1 -1
  19. package/dist/src/mcp.js +3 -3
  20. package/dist/src/mcp.js.map +1 -1
  21. package/dist/src/model.js +15 -10
  22. package/dist/src/model.js.map +1 -1
  23. package/dist/src/models.js +38 -0
  24. package/dist/src/models.js.map +1 -0
  25. package/dist/src/output.js +216 -0
  26. package/dist/src/output.js.map +1 -0
  27. package/dist/src/run.js +141 -27
  28. package/dist/src/run.js.map +1 -1
  29. package/dist/src/session.js +46 -25
  30. package/dist/src/session.js.map +1 -1
  31. package/dist/src/sessions.js +72 -0
  32. package/dist/src/sessions.js.map +1 -0
  33. package/dist/src/skills.js +130 -0
  34. package/dist/src/skills.js.map +1 -0
  35. package/package.json +7 -3
  36. package/prompt/index.ts +11 -2
  37. package/prompt/print.ts +9 -0
  38. package/prompt/system-prompt.ts +18 -4
  39. package/prompt/testomatio.ts +3 -3
  40. package/prompt/tools.ts +30 -7
  41. package/skills/codeceptjs/ci-fix-tests/SKILL.md +119 -0
  42. package/skills/codeceptjs/codeceptjs-auth/SKILL.md +141 -0
  43. package/skills/codeceptjs/codeceptjs-exploration/SKILL.md +100 -0
  44. package/skills/codeceptjs/codeceptjs-fundamentals/SKILL.md +196 -0
  45. package/skills/codeceptjs/codeceptjs-run-analysis/SKILL.md +83 -0
  46. package/skills/codeceptjs/debugging-codeceptjs-tests/SKILL.md +126 -0
  47. package/skills/codeceptjs/migrate-codeceptjs-4/SKILL.md +109 -0
  48. package/skills/codeceptjs/migrate-cypress-to-codeceptjs/SKILL.md +295 -0
  49. package/skills/codeceptjs/migrate-protractor-to-codeceptjs/SKILL.md +339 -0
  50. package/skills/codeceptjs/migrate-selenium-java-to-codeceptjs/SKILL.md +459 -0
  51. package/skills/codeceptjs/migrate-testcafe-to-codeceptjs/SKILL.md +338 -0
  52. package/skills/codeceptjs/refactoring-codeceptjs-tests/SKILL.md +78 -0
  53. package/skills/codeceptjs/writing-codeceptjs-tests/SKILL.md +102 -0
  54. package/skills/playwright/playwright-best-practices/SKILL.md +303 -0
  55. package/skills/playwright/playwright-best-practices/advanced/authentication-flows.md +360 -0
  56. package/skills/playwright/playwright-best-practices/advanced/authentication.md +871 -0
  57. package/skills/playwright/playwright-best-practices/advanced/clock-mocking.md +364 -0
  58. package/skills/playwright/playwright-best-practices/advanced/mobile-testing.md +409 -0
  59. package/skills/playwright/playwright-best-practices/advanced/multi-context.md +288 -0
  60. package/skills/playwright/playwright-best-practices/advanced/multi-user.md +393 -0
  61. package/skills/playwright/playwright-best-practices/advanced/network-advanced.md +452 -0
  62. package/skills/playwright/playwright-best-practices/advanced/third-party.md +464 -0
  63. package/skills/playwright/playwright-best-practices/architecture/pom-vs-fixtures.md +363 -0
  64. package/skills/playwright/playwright-best-practices/architecture/test-architecture.md +369 -0
  65. package/skills/playwright/playwright-best-practices/architecture/when-to-mock.md +383 -0
  66. package/skills/playwright/playwright-best-practices/browser-apis/browser-apis.md +391 -0
  67. package/skills/playwright/playwright-best-practices/browser-apis/iframes.md +403 -0
  68. package/skills/playwright/playwright-best-practices/browser-apis/service-workers.md +504 -0
  69. package/skills/playwright/playwright-best-practices/browser-apis/websockets.md +403 -0
  70. package/skills/playwright/playwright-best-practices/core/annotations.md +424 -0
  71. package/skills/playwright/playwright-best-practices/core/assertions-waiting.md +361 -0
  72. package/skills/playwright/playwright-best-practices/core/configuration.md +452 -0
  73. package/skills/playwright/playwright-best-practices/core/fixtures-hooks.md +417 -0
  74. package/skills/playwright/playwright-best-practices/core/global-setup.md +434 -0
  75. package/skills/playwright/playwright-best-practices/core/locators.md +242 -0
  76. package/skills/playwright/playwright-best-practices/core/page-object-model.md +315 -0
  77. package/skills/playwright/playwright-best-practices/core/projects-dependencies.md +453 -0
  78. package/skills/playwright/playwright-best-practices/core/test-data.md +492 -0
  79. package/skills/playwright/playwright-best-practices/core/test-suite-structure.md +361 -0
  80. package/skills/playwright/playwright-best-practices/core/test-tags.md +298 -0
  81. package/skills/playwright/playwright-best-practices/debugging/console-errors.md +420 -0
  82. package/skills/playwright/playwright-best-practices/debugging/debugging.md +504 -0
  83. package/skills/playwright/playwright-best-practices/debugging/error-testing.md +360 -0
  84. package/skills/playwright/playwright-best-practices/debugging/flaky-tests.md +496 -0
  85. package/skills/playwright/playwright-best-practices/frameworks/angular.md +530 -0
  86. package/skills/playwright/playwright-best-practices/frameworks/nextjs.md +469 -0
  87. package/skills/playwright/playwright-best-practices/frameworks/react.md +531 -0
  88. package/skills/playwright/playwright-best-practices/frameworks/vue.md +574 -0
  89. package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/ci-cd.md +468 -0
  90. package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/docker.md +283 -0
  91. package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/github-actions.md +546 -0
  92. package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/gitlab.md +397 -0
  93. package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/other-providers.md +521 -0
  94. package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/parallel-sharding.md +371 -0
  95. package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/performance.md +453 -0
  96. package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/reporting.md +424 -0
  97. package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/test-coverage.md +497 -0
  98. package/skills/playwright/playwright-best-practices/testing-patterns/accessibility.md +359 -0
  99. package/skills/playwright/playwright-best-practices/testing-patterns/api-testing.md +719 -0
  100. package/skills/playwright/playwright-best-practices/testing-patterns/browser-extensions.md +506 -0
  101. package/skills/playwright/playwright-best-practices/testing-patterns/canvas-webgl.md +493 -0
  102. package/skills/playwright/playwright-best-practices/testing-patterns/component-testing.md +500 -0
  103. package/skills/playwright/playwright-best-practices/testing-patterns/drag-drop.md +576 -0
  104. package/skills/playwright/playwright-best-practices/testing-patterns/electron.md +509 -0
  105. package/skills/playwright/playwright-best-practices/testing-patterns/file-operations.md +377 -0
  106. package/skills/playwright/playwright-best-practices/testing-patterns/file-upload-download.md +562 -0
  107. package/skills/playwright/playwright-best-practices/testing-patterns/forms-validation.md +561 -0
  108. package/skills/playwright/playwright-best-practices/testing-patterns/graphql-testing.md +331 -0
  109. package/skills/playwright/playwright-best-practices/testing-patterns/i18n.md +508 -0
  110. package/skills/playwright/playwright-best-practices/testing-patterns/performance-testing.md +476 -0
  111. package/skills/playwright/playwright-best-practices/testing-patterns/security-testing.md +430 -0
  112. package/skills/playwright/playwright-best-practices/testing-patterns/visual-regression.md +634 -0
  113. package/skills/playwright/playwright-cli/SKILL.md +420 -0
  114. package/skills/playwright/playwright-cli/references/element-attributes.md +23 -0
  115. package/skills/playwright/playwright-cli/references/playwright-tests.md +39 -0
  116. package/skills/playwright/playwright-cli/references/request-mocking.md +87 -0
  117. package/skills/playwright/playwright-cli/references/running-code.md +241 -0
  118. package/skills/playwright/playwright-cli/references/session-management.md +225 -0
  119. package/skills/playwright/playwright-cli/references/storage-state.md +275 -0
  120. package/skills/playwright/playwright-cli/references/test-generation.md +433 -0
  121. package/skills/playwright/playwright-cli/references/tracing.md +139 -0
  122. package/skills/playwright/playwright-cli/references/video-recording.md +143 -0
  123. package/skills/skills.lock.json +41 -37
  124. package/skills/skills.yaml +1 -0
  125. package/skills/testomatio/explorbot/explorbot-fundamentals/SKILL.md +76 -0
  126. package/skills/testomatio/explorbot/explorbot-fundamentals/references/no-install.md +36 -0
  127. package/skills/testomatio/explorbot/explorbot-plan/SKILL.md +108 -0
  128. package/skills/testomatio/explorbot/explorbot-setup/SKILL.md +86 -0
  129. package/skills/testomatio/explorbot/prima/SKILL.md +87 -0
  130. package/skills/testomatio/qa-process/qa-explain-behavior/SKILL.md +111 -0
  131. package/skills/testomatio/qa-process/qa-explain-behavior/references/answer-shapes.md +116 -0
  132. package/skills/testomatio/qa-process/qa-explain-behavior/references/examples.md +157 -0
  133. package/skills/testomatio/qa-process/qa-lead-strategy-advisor/SKILL.md +91 -0
  134. package/skills/testomatio/qa-process/qa-lead-strategy-advisor/references/output-format.md +71 -0
  135. package/skills/testomatio/qa-process/testing-workflow/SKILL.md +227 -0
  136. package/skills/testomatio/test-automation/automate-manual-test-cases/CLAUDE.md +56 -0
  137. package/skills/testomatio/test-automation/automate-manual-test-cases/SKILL.md +242 -0
  138. package/skills/testomatio/test-automation/automate-manual-test-cases/references/CODECEPTJS_BEST_PRACTICES.md +182 -0
  139. package/skills/testomatio/test-automation/automate-manual-test-cases/references/FINAL_SUMMARY_TEMPLATE.md +22 -0
  140. package/skills/testomatio/test-automation/automate-manual-test-cases/references/PLAYWRIGHT_BEST_PRACTICES.md +90 -0
  141. package/skills/testomatio/test-automation/automate-manual-test-cases/references/POM_BEST_PRACTICES.md +53 -0
  142. package/skills/testomatio/test-automation/automate-manual-test-cases/references/TEST_DATA_MANAGEMENT.md +52 -0
  143. package/skills/testomatio/test-automation/debug-fix-failed-flaky-autotests/SKILL.md +107 -0
  144. package/skills/testomatio/test-automation/debug-fix-failed-flaky-autotests/references/DEBUGGING_QUICK_REFERENCE.md +113 -0
  145. package/skills/testomatio/test-automation/qa-automation-test-consolidation/SKILL.md +29 -0
  146. package/skills/testomatio/test-automation/qa-data-seeder/SKILL.md +83 -0
  147. package/skills/testomatio/test-automation/run-tests-with-testomatio-reporter/SKILL.md +153 -0
  148. package/skills/testomatio/test-automation/setup-change-aware-pr-testing/SKILL.md +190 -0
  149. package/skills/testomatio/test-automation/setup-ci-automation/SKILL.md +76 -0
  150. package/skills/testomatio/test-automation/testomat-allure-adapter/SKILL.md +289 -0
  151. package/skills/testomatio/test-management/detect-duplicate-test-cases/SKILL.md +86 -0
  152. package/skills/testomatio/test-management/detect-duplicate-test-cases/references/DUPLICATE_INSTRUCTIONS.md +54 -0
  153. package/skills/testomatio/test-management/improve-test-cases/SKILL.md +114 -0
  154. package/skills/testomatio/test-management/improve-test-cases/references/TESTOMAT_MARKDOWN_EXAMPLE.md +66 -0
  155. package/skills/testomatio/test-management/pull-request-diff-analyzer/SKILL.md +141 -0
  156. package/skills/testomatio/test-management/qa-e2e-tests-reporting/SKILL.md +354 -0
  157. package/skills/testomatio/test-management/qa-e2e-tests-reporting/references/TESTOMATIO_ARTIFACTS.md +82 -0
  158. package/skills/testomatio/test-management/qa-e2e-tests-reporting/references/TESTOMATIO_HTML_REPORT.md +37 -0
  159. package/skills/testomatio/test-management/qa-e2e-tests-reporting/references/TESTOMATIO_REPORTERS_CONFIG.md +118 -0
  160. package/skills/testomatio/test-management/qa-pr-requirements-analyzer/SKILL.md +149 -0
  161. package/skills/testomatio/test-management/qa-pr-requirements-analyzer/references/summary-example.md +54 -0
  162. package/skills/testomatio/test-management/qa-requirement-reviewer/SKILL.md +126 -0
  163. package/skills/testomatio/test-management/qa-requirement-reviewer/references/requirements_reviewer_examples.md +128 -0
  164. package/skills/testomatio/test-management/qa-split-testing-levels-pyramid/SKILL.md +55 -0
  165. package/skills/testomatio/test-management/qa-sprint-report-by-testomatio/SKILL.md +220 -0
  166. package/skills/testomatio/test-management/qa-sprint-report-by-testomatio/references/qa-sprint-report.md +126 -0
  167. package/skills/testomatio/test-management/qa-test-code-coverage/SKILL.md +174 -0
  168. package/skills/testomatio/test-management/qa-test-code-coverage/references/COVERAGE_FILE_FORMAT.md +138 -0
  169. package/skills/testomatio/test-management/qa-test-code-coverage/references/E2E_FRAMEWORKS.md +65 -0
  170. package/skills/testomatio/test-management/qa-test-code-coverage/scripts/check-coverage.mjs +40 -0
  171. package/skills/testomatio/test-management/qa-thinking/SKILL.md +36 -0
  172. package/skills/testomatio/test-management/qa-write-test-cases/SKILL.md +267 -0
  173. package/skills/testomatio/test-management/qa-write-test-cases/references/test-case-format.md +387 -0
  174. package/skills/testomatio/test-management/qa-write-test-cases/references/testomat-tms-guide.md +72 -0
  175. package/skills/testomatio/test-management/qa-write-test-cases/references/writing-rule.md +159 -0
  176. package/skills/testomatio/test-management/scan-automation-project/SKILL.md +152 -0
  177. package/skills/testomatio/test-management/sync-test-cases-with-tms/SKILL.md +107 -0
  178. package/skills/testomatio/test-management/sync-test-cases-with-tms/references/TESTOMATIO_CLI.md +140 -0
  179. package/skills/testomatio/test-management/testomatio-mcp/SKILL.md +195 -0
  180. package/skills/testomatio/test-management/testomatio-mcp/references/MCP_SETUP.md +177 -0
@@ -0,0 +1,152 @@
1
+ ---
2
+ name: scan-automation-project
3
+ description: Scan project source code to inventory languages, frameworks, and existing tests (manual `*.test.md` and automated test files). Use this skill whenever analyzing a codebase for test planning, detecting test frameworks, or preparing for test automation. Specifically, when the user mentions "scan project", "what tests exist", "analyze codebase", "detect frameworks", "test matrix", or needs a codebase inventory before a QA workflow.
4
+ license: MIT
5
+ metadata:
6
+ author: Testomat.io
7
+ version: 1.0.0
8
+ ---
9
+
10
+ # Scan Automation Project
11
+
12
+ Scan project source code and return a QA-focused inventory: languages, frameworks, and existing tests.
13
+
14
+ **Shallow scan only.** No full import resolution, no line-by-line parsing, no reading full test implementations. High-level structure only.
15
+
16
+ ## Step 1: Locate Source Code
17
+
18
+ - If the project root has non-empty source folders or files, go to Step 2.
19
+ - If **no source** files are found, stop and ask the user:
20
+
21
+ ```
22
+ ❓ No application source code detected (folder may be empty or missing).
23
+ Where is the source code?
24
+ 1. Use local project from another folder (I will create a symlink)
25
+ 2. Clone from Git repo
26
+ 3. I don't know yet (stop here)
27
+ ```
28
+
29
+ - Wait for the user's reply.
30
+ - If the user gives a Git repo or another folder, clone or symlink it into `.testeiya/code/`. No-op if it is already there.
31
+
32
+ ### Rule for pulled data
33
+
34
+ **When you need to pull external data** (manual test cases, app code, e2e tests from another repo) into this project:
35
+
36
+ | This project has | Store pulled data in | Gitignored? |
37
+ | ------------------ | ---------------------- | ----------- |
38
+ | **Source code** (`src/` folder) or **big project** | `.testeiya/…` | **Yes** |
39
+ | **Only test infrastructure** (e2e dirs like `tests/`, `playwright/`, `cypress/` but no `src/`) | `manual-tests/` or `e2e-tests/` | No |
40
+ | **Empty / manual-only** | `manual-tests/` | No |
41
+
42
+ Detection logic:
43
+ 1. Has `src/` folder or is a monorepo? → Use `.testeiya/`
44
+ 2. Has e2e test dirs (`tests/`, `playwright/`, `cypress/`, `e2e/`)? → Use tracked folder (`manual-tests/`, `e2e-tests/`)
45
+ 3. Otherwise → Default to `manual-tests/`
46
+
47
+ - Any skill that creates `.testeiya/...` must also add `.testeiya/` to the project's `.gitignore` — but only if it is not there yet. Never add it twice.
48
+ - The rule covers *pulled* data only. Output a skill *produces* and the user wants — a `coverage.*.yml`, new `*.test.md` files — goes in the repo as normal.
49
+ - On a source repo with no manual tests, this skill changes no tracked file: it only creates the gitignored `.testeiya/` directory and, if needed, adds one line to `.gitignore`.
50
+ - **Only touch `.testeiya/` and tracked folders — never pollute the repo with cache in a wrong location.**
51
+
52
+ ## Step 2: Project Analysis
53
+
54
+ Collect source file paths only. **Do NOT read file contents.**
55
+
56
+ Include:
57
+ - Application source files.
58
+ - View/template files: `.html`/`.htm`, `.vue`, `.svelte`, `.hbs`/`.handlebars`, `.ejs`, `.pug`/`.jade`, `.mustache`, `.liquid`, `.erb`, `.haml`, `.slim`, `.blade.php`, `.twig`, `.j2`/`.jinja`/`.jinja2`, `.cshtml`/`.razor`, `.jsp`. They are application source — coverage skills map UI changes through them.
59
+
60
+ Exclude:
61
+ - Dependencies, build output, coverage, reports, caches, config, lock, and environment files.
62
+ - Paths ignored by `.gitignore` — but **not** `.testeiya/code/`. In a manual-tests repo the app code lives there; scan it as source.
63
+ - Testeiya internal files (e.g. `session-factory.ts`, `system-prompt.ts`).
64
+ - **If in doubt**, exclude.
65
+
66
+ From the file list:
67
+ - Detect languages and frameworks. Collect one `frameworks` list with ALL application and testing frameworks.
68
+ - Extract the project name from a project config file (`package.json`, `Cargo.toml`, `pom.xml`, ...); fall back to the root directory name.
69
+ - Rate complexity by source file count:
70
+
71
+ | File Count | Complexity |
72
+ |------------|--------------|
73
+ | 1-30 | `small` |
74
+ | 31-150 | `moderate` |
75
+ | 151-500 | `large` |
76
+ | 500+ | `very-large` |
77
+
78
+ ## Step 3: Test Inventory (optional)
79
+
80
+ Detect existing tests, automated and manual. Stay shallow.
81
+
82
+ Automated tests:
83
+ - Detect frameworks via config files (`jest.config.*`, `playwright.config.*`, `vitest.config.*`, `pytest.ini`, `pom.xml`, ...), project dependencies, and test file patterns (`*.test.*`, `*.spec.*`, `*_test.*`).
84
+ - For each framework: identify its test file pattern and count matching files.
85
+
86
+ Manual tests:
87
+ - Find all `.test.md` files and parse test titles. `find .` also looks inside `.testeiya/manual-tests/`, so a re-run after a pull finds the cached cases instead of reporting "no manual tests":
88
+
89
+ ```bash
90
+ find . -name "*.test.md" -exec awk '
91
+ /^<!-- test/ { in_block=1; kind="TEST"; next }
92
+ /^<!-- suite/ { in_block=1; kind="SUITE"; next }
93
+ in_block && /^-->/ { in_block=0; expect=1; next }
94
+ expect && /^#+[[:space:]]+/ {
95
+ title=$0; sub(/^#+[[:space:]]+/, "", title)
96
+ if (kind == "SUITE") printf "SUITE: %s\n", title
97
+ else printf "|- %s\n", title
98
+ expect=0
99
+ }
100
+ ' {} +
101
+ ```
102
+
103
+ - If the only `.test.md` files are under `.testeiya/manual-tests/`, say so in the inventory: they came from Testomat.io, not from this repo.
104
+
105
+ ## Step 4: Output
106
+
107
+ Return one structured markdown result directly. **Do NOT save to a file.**
108
+
109
+ ```markdown
110
+ # Project Overview
111
+
112
+ - **Project Name:** acme-web-app
113
+ - **Description:** A React-based customer dashboard with an Express API.
114
+ - **Languages:** TypeScript, SQL
115
+ - **Frameworks:** React, Express, Jest, Playwright
116
+ - **Complexity:** small (12 files)
117
+
118
+ ## Test Inventory
119
+
120
+ - **Automated Tests:** 10 files
121
+ - **Manual Tests:** 37 cases
122
+
123
+ ### Manual Tests (20 of 37 shown)
124
+
125
+ - SUITE: Authentication
126
+ |- User can login
127
+ |- User can reset password
128
+ - SUITE: Billing
129
+ |- User can view invoice
130
+ ...and 17 more
131
+
132
+ ### Automated Tests (10 of 10 shown)
133
+
134
+ - home.page.spec.ts
135
+ ...
136
+ ```
137
+
138
+ Field rules:
139
+ - **Description:** 1-2 sentences based only on detected source code and folder structure (include the domain area if that makes sense).
140
+ - **Complexity:** one of `small` | `moderate` | `large` | `very-large`, plus the file count.
141
+ - **Manual Tests:** preserve hierarchy as plain strings — SUITE items as parent bullets, test titles (`|-`) as nested children.
142
+ - **Automated Tests:** list of detected test files.
143
+ - All values must come from observable files. Do NOT guess missing data or add fields not shown above.
144
+
145
+ Sections:
146
+ - If Step 3 was skipped or found no tests, omit `## Test Inventory` and note that the project contains no tests; the `# Project Overview` section is still useful for next steps.
147
+ - If one test type is absent, note it with a blockquote, e.g. ``> No manual tests (`.test.md`) found in the project.``
148
+
149
+ Test listing truncation:
150
+ - Show at most the first 20 entries per list, in original file order.
151
+ - Mark truncation in the heading (`(20 of 37 shown)`) and with `...and N more`.
152
+ - Do not print full test listings beyond that.
@@ -0,0 +1,107 @@
1
+ ---
2
+ name: sync-test-cases-with-tms
3
+ description: Synchronize test scenarios and cases between a local project and Testomat.io. Use this skill whenever the user wants to pull/export/download tests from Testomat.io; or push/import/sync new or updated test cases back to the TMS in corresponding `*.test.md` format. Supports custom directories, markdown test format and advanced import/export workflows.
4
+ inputs:
5
+ testDir:
6
+ description: "Target directory for pulled tests"
7
+ required: false
8
+ license: MIT
9
+ metadata:
10
+ author: Testomat.io
11
+ version: 1.0.0
12
+ ---
13
+
14
+ # Sync Test Cases with TMS
15
+
16
+ Sync Markdown test cases (`*.test.md`) between the local project and Testomat.io.
17
+
18
+ Use when the user wants to:
19
+ - Pull/export/download tests from Testomat.io to local Markdown files.
20
+ - Push/upload/import local Markdown tests to Testomat.io.
21
+ - Bulk edit: pull cases -> edit locally -> push back to TMS.
22
+
23
+ ## Setup
24
+
25
+ - Check for the `TESTOMATIO` token: skill input -> `.env` in project root -> ask the user.
26
+ - Save the token to `.env`: `TESTOMATIO=tstmt_xxxxx`.
27
+ - If the user has no key: get it in Testomat.io under **Settings → Project → Project Reporting API key** (`https://app.testomat.io/projects/<project-id>/settings/project`).
28
+
29
+ ## Running the CLI
30
+
31
+ - Run `check-tests` via `npx`. Do not install it as a project dependency.
32
+ - **The first `check-tests` call in the session must be `npx check-tests@latest …`** to resolve the latest version. All later calls in the same session use plain `npx check-tests …`.
33
+ - **Use only options documented here, in [TESTOMATIO_CLI.md](./references/TESTOMATIO_CLI.md), or listed by `npx check-tests --help`. Do not invent options** (e.g. `--pattern`, `--force`).
34
+
35
+ ## Pull
36
+
37
+ Downloads test scenarios from Testomat.io and saves them as local Markdown files.
38
+
39
+ Directory selection — if the `testDir` input is provided or the user passes `-d <path>`, use that path. Otherwise:
40
+ - Workspace is empty or for manual tests only: pull to root.
41
+ - End-to-end testing project: pull to the `manual-tests` directory.
42
+ - Any other case: pull to `.testeiya/manual-tests` (ensure `.testeiya/` exists and is gitignored).
43
+
44
+ Cache-folder rules:
45
+ - Default pull target is the gitignored cache `.testeiya/manual-tests/`. Add `.testeiya/` to the project `.gitignore` if it is not there yet.
46
+ - **Pulled cases must never land in a tracked folder** (`manual-tests/`, etc.) — that pollutes the repo.
47
+ - Being gitignored does not block editing the files or pushing them back.
48
+ - If the repo already keeps its `*.test.md` files in a tracked folder, don't pull — work with them in place, or pass `-d <that folder>` for an in-place refresh.
49
+ - This matches `scan-automation-project`, which pulls the *code* into `.testeiya/code/` when it runs inside a manual-tests repo.
50
+
51
+ ```bash
52
+ npx check-tests pull -d .testeiya/manual-tests
53
+ ```
54
+
55
+ - To pull only specific suites (user names a suite or gives suite IDs): `npx check-tests pull --suite-ids "@S12345678,@S87654321"`.
56
+ - More pull options: [TESTOMATIO_CLI.md](./references/TESTOMATIO_CLI.md).
57
+
58
+ ## Push
59
+
60
+ Uploads local Markdown tests to Testomat.io.
61
+
62
+ Pre-push file filtering — push only test case files, never project docs or requirements:
63
+ - Include files matching `*.test.md`, or Markdown files with valid `<!-- test ... -->` blocks.
64
+ - Ignore all other Markdown (`README.md`, `CHANGELOG.md`, documentation).
65
+ - If test cases live alongside other files, copy or move the test files into `.testeiya/manual-tests/` first.
66
+
67
+ Pre-push validation:
68
+ 1. At least one `*.test.md` file exists.
69
+ 2. Each file contains a valid test block:
70
+
71
+ ```md
72
+ <!-- test
73
+ priority: ...
74
+ creator: ...
75
+ tags: ...
76
+ labels: ...
77
+ -->
78
+ # ... (test case title)
79
+
80
+ ... (test case description)
81
+ ```
82
+
83
+ ```bash
84
+ # Specific files (preferred when known)
85
+ npx check-tests push --files login.test.md checkout.test.md
86
+
87
+ # Testeiya cache folder
88
+ npx check-tests push -d .testeiya/manual-tests
89
+ ```
90
+
91
+ - **When the files to push are known** (e.g. just produced by `qa-write-test-cases` / `improve-test-cases`), **pass them explicitly via `--files`** (alias `-f`). Without `--files` the CLI falls back to the default glob `**/*.test.md`, which may pick up unrelated files.
92
+ - Quote glob patterns. Paths resolve relative to `--dir`.
93
+ - Use `-d` when `.testeiya` or `manual-tests` directories exist.
94
+ - **Push only the test cases directory** (e.g. `.testeiya/manual-tests`, not `.testeiya/`).
95
+ - More push options and examples: [TESTOMATIO_CLI.md](./references/TESTOMATIO_CLI.md).
96
+
97
+ ## Summary Output
98
+
99
+ After syncing, print a short log-style summary:
100
+
101
+ ```
102
+ Sync Complete:
103
+ - Action: pull/push
104
+ - Directory: <path>
105
+ - Tests synced: 15
106
+ - Status: Success
107
+ ```
@@ -0,0 +1,140 @@
1
+ # Testomat.io CLI Documentation
2
+
3
+ This document provides comprehensive information about the `check-tests` CLI commands, which synchronizes test scenarios, and interacts with Testomat.io test cases from your local project with TMS.
4
+
5
+ ---
6
+
7
+ ## Prerequisites
8
+
9
+ ### Run check-tests via npx (always latest)
10
+
11
+ Invoke `check-tests` through `npx` so users automatically pick up the newest published version:
12
+
13
+ ```bash
14
+ # First invocation in the agent session — forces resolve of latest
15
+ npx check-tests@latest <command>
16
+
17
+ # Subsequent invocations in the same session could be used without @latest
18
+ npx check-tests <command>
19
+ ```
20
+
21
+ Do not install `check-tests` as a project dependency. The first call (with `@latest` version specified) refreshes the npx cache; next calls in the same session reuse it without an extra registry round-trip. A new agent session re-triggers the `@latest` check so users use the latest version over time.
22
+
23
+ ---
24
+
25
+ ## Environment Variables
26
+
27
+ The tool supports loading environment variables from `.env` files using dotenv.
28
+
29
+ ### Testomat.io Configuration
30
+
31
+ | Variable | Description | Required |
32
+ | ------------------------ | ------------------------------------------------------ | ------------------------------------- |
33
+ | `TESTOMATIO` | API key for Testomat.io (format: tstmt_xxxxx) | Yes (for sync operations) |
34
+ | `TESTOMATIO_URL` | Testomat.io server URL | No (default: https://app.testomat.io) |
35
+ | `TESTOMATIO_WORKDIR` | Working directory for relative file paths | No |
36
+ | `TESTOMATIO_PREPEND_DIR` | Directory to prepend to test paths | No |
37
+ | `TESTOMATIO_LABELS` | Comma-separated labels. Supports `label:value` | No |
38
+
39
+ ### Configuration File
40
+
41
+ Save credentials to `.env` file:
42
+
43
+ ```env
44
+ TESTOMATIO=tstmt_xxxxx
45
+ TESTOMATIO_URL=https://app.testomat.io
46
+ ...
47
+ ```
48
+
49
+ ---
50
+
51
+ ## CLI Options
52
+
53
+ ### Basic "check-tests" Options
54
+
55
+ | Option | Description | Default |
56
+ | --------------------- | ------------------------------------------- | ----------------------------|
57
+ | `-h, --help` | Display help information | - |
58
+ | `-d, --dir <dir>` | Test directory to scan | Current dir (default: `.`) |
59
+ | `--suite-ids <ids>` | Comma-separated suite IDs to pull (e.g. `@S12345678, @S456r4342`) | - |
60
+
61
+ ### Testomat.io Integration Specific Project Options
62
+
63
+ | Option | Description | Default |
64
+ | ----------------------------- | ------------------------------------------------------------- | ------- |
65
+ | `--update-ids` | Update test and suite with Testomat.io IDs | false |
66
+ | `--keep-structure` | Prefer structure of source code over structure in Testomat.io | false |
67
+ | `--no-empty` | Remove empty suites after import | false |
68
+ | `--clean-ids` | Remove Testomat.io IDs from test and suite | false |
69
+
70
+ ---
71
+
72
+ ## Commands
73
+
74
+ ### Pull
75
+
76
+ Retrieve the latest test scenarios from Testomat.io and save them as Markdown files locally.
77
+
78
+ ```bash
79
+ npx check-tests pull [options]
80
+ ```
81
+
82
+ **Examples:**
83
+
84
+ ```bash
85
+ # Export tests to current directory
86
+ npx check-tests pull
87
+
88
+ # Export tests to manual-tests folder
89
+ npx check-tests pull -d manual-tests
90
+
91
+ # Keep source structure
92
+ npx check-tests pull -d manual-tests --keep-structure
93
+
94
+ # Pull specific suites only
95
+ npx check-tests pull --suite-ids "@S12345678,@S87654321"
96
+ ```
97
+
98
+ ### Push
99
+
100
+ Send local Markdown test updates to Testomat.io. (Equivalent to `check-tests manual <files> --update-ids`.)
101
+
102
+ ```bash
103
+ npx check-tests push [options]
104
+ ```
105
+
106
+ `--files` (alias `-f`) accepts file paths, glob patterns, or a mix; defaults to `**/*.test.md`. Paths resolve relative to `--dir`. Quote globs.
107
+
108
+ **Best practice:** when the files to push are known, list them explicitly via `--files` rather than relying on the default glob.
109
+
110
+ **Examples:**
111
+
112
+ ```bash
113
+ # Default glob (**/*.test.md)
114
+ npx check-tests push
115
+
116
+ # Specific files
117
+ npx check-tests push -f docs/login.test.md docs/checkout.test.md
118
+
119
+ # Glob (quoted)
120
+ npx check-tests push --files "manual-tests/**/*.test.md"
121
+
122
+ # Multiple globs
123
+ npx check-tests push -f "smoke/**/*.test.md" "regression/**/*.test.md"
124
+
125
+ # With labels
126
+ TESTOMATIO_LABELS=smoke,updated npx check-tests push
127
+ ```
128
+
129
+ ---
130
+
131
+ ## Quick Reference
132
+
133
+ | Action | Command |
134
+ | --------------- | -------------------------------------------------------------- |
135
+ | Pull tests | `npx check-tests pull -d <dir>` |
136
+ | Push files | `npx check-tests push --files <file1.test.md> <file2.test.md>` |
137
+ | Push glob | `npx check-tests push --files "<dir>/**/*.test.md"` |
138
+ | Push directory | `npx check-tests push -d <dir>` (glob: `**/*.test.md`) |
139
+ | With labels | `TESTOMATIO_LABELS=smoke npx check-tests push` |
140
+ | Keep structure | `npx check-tests pull --keep-structure` |
@@ -0,0 +1,195 @@
1
+ ---
2
+ name: testomatio-mcp
3
+ description: Configures and leverages the Testomat.io MCP server for test management analytics, run analysis, failure investigation, and defect triage. Use when the user needs to connect to Testomat.io via MCP, analyze test runs, cluster failures, investigate root causes.
4
+ license: MIT
5
+ metadata:
6
+ author: Testomat.io
7
+ version: 1.0.0
8
+ ---
9
+
10
+ # Testomatio MCP
11
+
12
+ Set up access to Testomat.io via MCP and run QA analysis workflows: run analysis, failure clustering, root-cause investigation, plan analysis, defect triage.
13
+
14
+ ## When to Use
15
+
16
+ - Detect or set up the Testomat.io MCP server for an AI agent (Claude, Cursor, OpenCode, etc.).
17
+ - Analyze test runs: recent results, success rates, failure trends.
18
+ - Investigate failures: group similar failures, find patterns.
19
+ - Find the root cause when many tests fail with the same issue.
20
+ - Create defects from failed tests in the user's issue tracker (e.g. Jira).
21
+
22
+ ## MCP Tools
23
+
24
+ - Test: `tests_list`, `tests_get`, `tests_search`
25
+ - Test Plan: `plans_list`, `plans_get`, `plans_search`
26
+ - Run: `runs_list`, `runs_get`, `runs_search`
27
+ - Testrun (results within a run): `testruns_list`, `testruns_get`
28
+ - Suite: `suites_list`, `suites_get`, `suites_search`
29
+ - Label: `labels_list`, `labels_get`, `labels_create`, `labels_update`, `labels_delete`. Labels may be scoped (`key:value`). Attach or swap them on entities via the `link` parameter — [MCP Setup Reference](./references/MCP_SETUP.md).
30
+ - Tag: `tags_list`, `tags_get`, `tags_search`
31
+
32
+ **CRUD tools (create, update, delete) exist for all entities. Use them only when explicitly needed for targeted updates.**
33
+
34
+ ## Rules
35
+
36
+ - **Prefer local tests over MCP.** If the repo contains markdown test cases (`*.test.md`) or automated test files, read them from the filesystem (or load them via the `sync-test-cases-with-tms` skill) instead of using MCP for full test discovery and analysis.
37
+ - Use MCP for:
38
+ - reading and analyzing run reports
39
+ - analytics and reporting
40
+ - test plan management
41
+ - tests that live exclusively in Testomat.io (no local copies), or quick test case searches (`tests_list`, `tests_search`)
42
+ - targeted searches and point updates to remote test cases
43
+ - Filter `runs_list`, `plans_list`, `testruns_list` with TQL (Testomat.io Query Language). Operators and examples: [MCP Setup Reference](./references/MCP_SETUP.md).
44
+
45
+ ## Setup
46
+
47
+ ### Step 1: Detect Existing Configuration
48
+
49
+ Check the user's AI agent config files for a `testomatio` or `mcp.testomatio` entry:
50
+ - `opencode.json`
51
+ - `.cursor/mcp.json`
52
+ - `claude_desktop_config.json`
53
+ - other, based on the user's local agent
54
+
55
+ Validate:
56
+ - the server is enabled (when the client supports enable/disable flags)
57
+ - `TESTOMATIO_PROJECT_TOKEN` and `TESTOMATIO_PROJECT_ID` are present
58
+
59
+ Configured and enabled → go to Workflows. Missing, incomplete, or disabled → Step 2.
60
+
61
+ ### Step 2: Configure MCP
62
+
63
+ Before configuring:
64
+ - Prefer confirming the project contains no more than 1000 tests.
65
+ - If project size cannot be determined, warn the user about MCP performance limits on large projects.
66
+ - **Use only targeted or filtered operations when explicitly requested.**
67
+
68
+ Credential priority for `TESTOMATIO_PROJECT_TOKEN` and `TESTOMATIO_PROJECT_ID`:
69
+ 1. Existing environment variables.
70
+ 2. Existing MCP configuration.
71
+ 3. User-provided values.
72
+
73
+ Per-agent config formats (OpenCode, Cursor, Claude Desktop) and where to obtain credentials: [MCP Setup Reference](./references/MCP_SETUP.md).
74
+
75
+ **Rules:**
76
+ - **Do not overwrite unrelated MCP server configurations.** Merge changes carefully with existing config content.
77
+ - **Do not expose project tokens, secrets, or credentials in chat responses.**
78
+
79
+ After successful configuration, confirm with a summary:
80
+
81
+ ```text
82
+ Testomat.io MCP configuration completed successfully.
83
+
84
+ AI Agent: Cursor
85
+ Config Updated: `.cursor/mcp.json`
86
+ MCP Server: Enabled
87
+ Authentication: Verified
88
+
89
+ Available workflows:
90
+ - Test run analysis
91
+ - Failure clustering
92
+ - Root-cause investigation
93
+ - Plan analysis & defect triage
94
+ ```
95
+
96
+ ## Workflows
97
+
98
+ Common patterns, not a complete list — adapt and extend to the user's needs. All assume the MCP server is enabled.
99
+
100
+ ### Workflow 1: Analyze Test Runs
101
+
102
+ Goal: understand the health of recent test execution.
103
+
104
+ 1. Call `runs_list` with `tql` filters (e.g. `finished`, `failed`, `with_defect`); narrow scope by status, environment, or plan.
105
+ 2. Select a run of interest and call `runs_get` for detailed metadata.
106
+ 3. Call `testruns_list` with `run_id` and `filter_status: failed` to extract failed results.
107
+ 4. Summarize: total tests, passed, failed, skipped, environments involved.
108
+ - Compare success/failure ratios across runs to spot regressions.
109
+
110
+ ### Workflow 2: Failure Investigation & Error Clustering
111
+
112
+ Goal: determine what went wrong and which areas are affected.
113
+
114
+ 1. Call `testruns_list` (`filter_status: failed`, `run_id`) to get failed results.
115
+ 2. For each failed testrun, call `tests_get` to map it to its suite, priority, tags, and linked issues.
116
+ 3. Cluster failures:
117
+ - by `suite_id` → identify the suite with the highest failure count
118
+ - by `message` substring → find tests failing with the same exception or assertion
119
+ 4. Present clusters ranked by count, with suite and message breakdowns and affected test titles.
120
+
121
+ ### Workflow 3: Plan Analysis & Root-Cause Detection
122
+
123
+ Goal: when many tests fail with the same issue, explain the root cause.
124
+
125
+ 1. Call `plans_get` (if a plan is linked to the run) to understand the intended scope; check whether failures correlate with specific plan sections.
126
+ 2. Call `runs_list` with `tql` targeting the same plan or environment to get historical runs.
127
+ 3. Compare failures across runs:
128
+ - increasing failure count → new regression
129
+ - same failures across multiple runs → stable bug or flaky infrastructure
130
+ - spike in a single suite or tag → localized code change, feature regression
131
+ - scattered across unrelated suites → systemic issue (infrastructure, config, dependency)
132
+ 4. Call `tests_get` on affected tests to check `code`, `state`, and linked issues.
133
+ 5. Report a root-cause hypothesis.
134
+
135
+ Example — user asks: "Check why 20 tests failed in the last run — one suite or a systemic issue?"
136
+
137
+ ```text
138
+ 1. Detecting existing Testomat.io MCP configuration:
139
+ - Found `.cursor/mcp.json`
140
+ - Found `testomatio` MCP server entry
141
+ - MCP server status: enabled
142
+
143
+ 2. Verifying MCP availability...
144
+ - Authentication successful
145
+ - Testomat.io tools available
146
+
147
+ 3. Analyzing latest test run...
148
+ - Retrieved last completed run
149
+ - Total failed tests: 20
150
+
151
+ 4. Failure clustering summary:
152
+ - 16/20 failures belong to suite: "Checkout / Payments"
153
+ - 14 failures share the same error: "Timeout waiting for payment confirmation"
154
+ - First failure timestamp indicates failures started simultaneously
155
+
156
+ 5. Probable root cause:
157
+ - The failures appear systemic rather than test-specific.
158
+ - Most failures point to instability or outage in the payment gateway service used during checkout flows.
159
+ ```
160
+
161
+ ### Workflow 4: Defect Creation from Failed Tests
162
+
163
+ Goal: turn recurring failures into trackable defects.
164
+
165
+ 1. Run Workflow 2 to cluster failures by message.
166
+ 2. If a cluster contains 5+ tests with the same failure:
167
+ - propose creating a defect (e.g. Jira issue) to the user
168
+ - draft the defect summary: title, description, affected test titles, failure messages, run link, environment
169
+ - if a Jira MCP is available, offer to create the ticket via the Jira skill
170
+ 3. After defect creation, call `tests_issues_link` for each affected test to link it to the defect.
171
+
172
+ ### Workflow 5: Update Labels & Tags on Tests
173
+
174
+ Goal: set, change, or remove labels and tags on tests — for example, update a status label that drives a dashboard chart.
175
+
176
+ 1. Call `labels_list` (or `project_info`) to read the exact label values that exist in the project, including the full `key:value` form of scoped labels. Do not guess label strings.
177
+ 2. Optionally find the tests to update with `tests_list` + TQL: `label == 'regression:yes'`.
178
+ 3. Update with `tests_update` (or `tests_create`) using the `link` array — one `{ action, type, value }` entry per change. To change a value (e.g. `regression:no` → `regression:yes`), send a `remove` and an `add` in the same call. The same shape works for `tag`, `custom_field`, `milestone`, `issue`, and `jira`.
179
+
180
+ Link format and a full swap example: [MCP Setup Reference](./references/MCP_SETUP.md).
181
+
182
+ ## Quick Commands
183
+
184
+ | Action | MCP Tool | Key Parameters |
185
+ |---------------------|-----------------|----------------|
186
+ | List failed results | `testruns_list` | `run_id`, `filter_status: failed` |
187
+ | List runs | `runs_list` | `tql`, `page`, `per_page` |
188
+ | Get run details | `runs_get` | `run_id` |
189
+ | Get test details | `tests_get` | `test_id` |
190
+ | Get plan details | `plans_get` | `plan_id` |
191
+ | Link issue to test | `tests_issues_link` | `test_id`, `url` or `jira_id` |
192
+ | List labels | `labels_list` | `page`, `per_page` |
193
+ | Set / swap label on test | `tests_update` | `test_id`, `link` |
194
+ | Find tests by label | `tests_list` | `tql: label == '...'` |
195
+ | Check server status | `system_ping` | none |