testeiya 0.3.9 → 0.3.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +353 -27
- package/dist/prompt/index.js +6 -2
- package/dist/prompt/index.js.map +1 -1
- package/dist/prompt/print.js +8 -0
- package/dist/prompt/print.js.map +1 -1
- package/dist/prompt/system-prompt.js +14 -4
- package/dist/prompt/system-prompt.js.map +1 -1
- package/dist/prompt/testomatio.js +3 -3
- package/dist/prompt/tools.js +28 -7
- package/dist/prompt/tools.js.map +1 -1
- package/dist/src/args.js +198 -46
- package/dist/src/args.js.map +1 -1
- package/dist/src/cli.js +64 -20
- package/dist/src/cli.js.map +1 -1
- package/dist/src/doctor.js +171 -0
- package/dist/src/doctor.js.map +1 -0
- package/dist/src/env.js +39 -5
- package/dist/src/env.js.map +1 -1
- package/dist/src/mcp.js +3 -3
- package/dist/src/mcp.js.map +1 -1
- package/dist/src/model.js +15 -10
- package/dist/src/model.js.map +1 -1
- package/dist/src/models.js +38 -0
- package/dist/src/models.js.map +1 -0
- package/dist/src/output.js +216 -0
- package/dist/src/output.js.map +1 -0
- package/dist/src/run.js +141 -27
- package/dist/src/run.js.map +1 -1
- package/dist/src/session.js +46 -25
- package/dist/src/session.js.map +1 -1
- package/dist/src/sessions.js +72 -0
- package/dist/src/sessions.js.map +1 -0
- package/dist/src/skills.js +130 -0
- package/dist/src/skills.js.map +1 -0
- package/package.json +7 -3
- package/prompt/index.ts +11 -2
- package/prompt/print.ts +9 -0
- package/prompt/system-prompt.ts +18 -4
- package/prompt/testomatio.ts +3 -3
- package/prompt/tools.ts +30 -7
- package/skills/codeceptjs/ci-fix-tests/SKILL.md +119 -0
- package/skills/codeceptjs/codeceptjs-auth/SKILL.md +141 -0
- package/skills/codeceptjs/codeceptjs-exploration/SKILL.md +100 -0
- package/skills/codeceptjs/codeceptjs-fundamentals/SKILL.md +196 -0
- package/skills/codeceptjs/codeceptjs-run-analysis/SKILL.md +83 -0
- package/skills/codeceptjs/debugging-codeceptjs-tests/SKILL.md +126 -0
- package/skills/codeceptjs/migrate-codeceptjs-4/SKILL.md +109 -0
- package/skills/codeceptjs/migrate-cypress-to-codeceptjs/SKILL.md +295 -0
- package/skills/codeceptjs/migrate-protractor-to-codeceptjs/SKILL.md +339 -0
- package/skills/codeceptjs/migrate-selenium-java-to-codeceptjs/SKILL.md +459 -0
- package/skills/codeceptjs/migrate-testcafe-to-codeceptjs/SKILL.md +338 -0
- package/skills/codeceptjs/refactoring-codeceptjs-tests/SKILL.md +78 -0
- package/skills/codeceptjs/writing-codeceptjs-tests/SKILL.md +102 -0
- package/skills/playwright/playwright-best-practices/SKILL.md +303 -0
- package/skills/playwright/playwright-best-practices/advanced/authentication-flows.md +360 -0
- package/skills/playwright/playwright-best-practices/advanced/authentication.md +871 -0
- package/skills/playwright/playwright-best-practices/advanced/clock-mocking.md +364 -0
- package/skills/playwright/playwright-best-practices/advanced/mobile-testing.md +409 -0
- package/skills/playwright/playwright-best-practices/advanced/multi-context.md +288 -0
- package/skills/playwright/playwright-best-practices/advanced/multi-user.md +393 -0
- package/skills/playwright/playwright-best-practices/advanced/network-advanced.md +452 -0
- package/skills/playwright/playwright-best-practices/advanced/third-party.md +464 -0
- package/skills/playwright/playwright-best-practices/architecture/pom-vs-fixtures.md +363 -0
- package/skills/playwright/playwright-best-practices/architecture/test-architecture.md +369 -0
- package/skills/playwright/playwright-best-practices/architecture/when-to-mock.md +383 -0
- package/skills/playwright/playwright-best-practices/browser-apis/browser-apis.md +391 -0
- package/skills/playwright/playwright-best-practices/browser-apis/iframes.md +403 -0
- package/skills/playwright/playwright-best-practices/browser-apis/service-workers.md +504 -0
- package/skills/playwright/playwright-best-practices/browser-apis/websockets.md +403 -0
- package/skills/playwright/playwright-best-practices/core/annotations.md +424 -0
- package/skills/playwright/playwright-best-practices/core/assertions-waiting.md +361 -0
- package/skills/playwright/playwright-best-practices/core/configuration.md +452 -0
- package/skills/playwright/playwright-best-practices/core/fixtures-hooks.md +417 -0
- package/skills/playwright/playwright-best-practices/core/global-setup.md +434 -0
- package/skills/playwright/playwright-best-practices/core/locators.md +242 -0
- package/skills/playwright/playwright-best-practices/core/page-object-model.md +315 -0
- package/skills/playwright/playwright-best-practices/core/projects-dependencies.md +453 -0
- package/skills/playwright/playwright-best-practices/core/test-data.md +492 -0
- package/skills/playwright/playwright-best-practices/core/test-suite-structure.md +361 -0
- package/skills/playwright/playwright-best-practices/core/test-tags.md +298 -0
- package/skills/playwright/playwright-best-practices/debugging/console-errors.md +420 -0
- package/skills/playwright/playwright-best-practices/debugging/debugging.md +504 -0
- package/skills/playwright/playwright-best-practices/debugging/error-testing.md +360 -0
- package/skills/playwright/playwright-best-practices/debugging/flaky-tests.md +496 -0
- package/skills/playwright/playwright-best-practices/frameworks/angular.md +530 -0
- package/skills/playwright/playwright-best-practices/frameworks/nextjs.md +469 -0
- package/skills/playwright/playwright-best-practices/frameworks/react.md +531 -0
- package/skills/playwright/playwright-best-practices/frameworks/vue.md +574 -0
- package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/ci-cd.md +468 -0
- package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/docker.md +283 -0
- package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/github-actions.md +546 -0
- package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/gitlab.md +397 -0
- package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/other-providers.md +521 -0
- package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/parallel-sharding.md +371 -0
- package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/performance.md +453 -0
- package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/reporting.md +424 -0
- package/skills/playwright/playwright-best-practices/infrastructure-ci-cd/test-coverage.md +497 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/accessibility.md +359 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/api-testing.md +719 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/browser-extensions.md +506 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/canvas-webgl.md +493 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/component-testing.md +500 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/drag-drop.md +576 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/electron.md +509 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/file-operations.md +377 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/file-upload-download.md +562 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/forms-validation.md +561 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/graphql-testing.md +331 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/i18n.md +508 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/performance-testing.md +476 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/security-testing.md +430 -0
- package/skills/playwright/playwright-best-practices/testing-patterns/visual-regression.md +634 -0
- package/skills/playwright/playwright-cli/SKILL.md +420 -0
- package/skills/playwright/playwright-cli/references/element-attributes.md +23 -0
- package/skills/playwright/playwright-cli/references/playwright-tests.md +39 -0
- package/skills/playwright/playwright-cli/references/request-mocking.md +87 -0
- package/skills/playwright/playwright-cli/references/running-code.md +241 -0
- package/skills/playwright/playwright-cli/references/session-management.md +225 -0
- package/skills/playwright/playwright-cli/references/storage-state.md +275 -0
- package/skills/playwright/playwright-cli/references/test-generation.md +433 -0
- package/skills/playwright/playwright-cli/references/tracing.md +139 -0
- package/skills/playwright/playwright-cli/references/video-recording.md +143 -0
- package/skills/skills.lock.json +41 -37
- package/skills/skills.yaml +1 -0
- package/skills/testomatio/explorbot/explorbot-fundamentals/SKILL.md +76 -0
- package/skills/testomatio/explorbot/explorbot-fundamentals/references/no-install.md +36 -0
- package/skills/testomatio/explorbot/explorbot-plan/SKILL.md +108 -0
- package/skills/testomatio/explorbot/explorbot-setup/SKILL.md +86 -0
- package/skills/testomatio/explorbot/prima/SKILL.md +87 -0
- package/skills/testomatio/qa-process/qa-explain-behavior/SKILL.md +111 -0
- package/skills/testomatio/qa-process/qa-explain-behavior/references/answer-shapes.md +116 -0
- package/skills/testomatio/qa-process/qa-explain-behavior/references/examples.md +157 -0
- package/skills/testomatio/qa-process/qa-lead-strategy-advisor/SKILL.md +91 -0
- package/skills/testomatio/qa-process/qa-lead-strategy-advisor/references/output-format.md +71 -0
- package/skills/testomatio/qa-process/testing-workflow/SKILL.md +227 -0
- package/skills/testomatio/test-automation/automate-manual-test-cases/CLAUDE.md +56 -0
- package/skills/testomatio/test-automation/automate-manual-test-cases/SKILL.md +242 -0
- package/skills/testomatio/test-automation/automate-manual-test-cases/references/CODECEPTJS_BEST_PRACTICES.md +182 -0
- package/skills/testomatio/test-automation/automate-manual-test-cases/references/FINAL_SUMMARY_TEMPLATE.md +22 -0
- package/skills/testomatio/test-automation/automate-manual-test-cases/references/PLAYWRIGHT_BEST_PRACTICES.md +90 -0
- package/skills/testomatio/test-automation/automate-manual-test-cases/references/POM_BEST_PRACTICES.md +53 -0
- package/skills/testomatio/test-automation/automate-manual-test-cases/references/TEST_DATA_MANAGEMENT.md +52 -0
- package/skills/testomatio/test-automation/debug-fix-failed-flaky-autotests/SKILL.md +107 -0
- package/skills/testomatio/test-automation/debug-fix-failed-flaky-autotests/references/DEBUGGING_QUICK_REFERENCE.md +113 -0
- package/skills/testomatio/test-automation/qa-automation-test-consolidation/SKILL.md +29 -0
- package/skills/testomatio/test-automation/qa-data-seeder/SKILL.md +83 -0
- package/skills/testomatio/test-automation/run-tests-with-testomatio-reporter/SKILL.md +153 -0
- package/skills/testomatio/test-automation/setup-change-aware-pr-testing/SKILL.md +190 -0
- package/skills/testomatio/test-automation/setup-ci-automation/SKILL.md +76 -0
- package/skills/testomatio/test-automation/testomat-allure-adapter/SKILL.md +289 -0
- package/skills/testomatio/test-management/detect-duplicate-test-cases/SKILL.md +86 -0
- package/skills/testomatio/test-management/detect-duplicate-test-cases/references/DUPLICATE_INSTRUCTIONS.md +54 -0
- package/skills/testomatio/test-management/improve-test-cases/SKILL.md +114 -0
- package/skills/testomatio/test-management/improve-test-cases/references/TESTOMAT_MARKDOWN_EXAMPLE.md +66 -0
- package/skills/testomatio/test-management/pull-request-diff-analyzer/SKILL.md +141 -0
- package/skills/testomatio/test-management/qa-e2e-tests-reporting/SKILL.md +354 -0
- package/skills/testomatio/test-management/qa-e2e-tests-reporting/references/TESTOMATIO_ARTIFACTS.md +82 -0
- package/skills/testomatio/test-management/qa-e2e-tests-reporting/references/TESTOMATIO_HTML_REPORT.md +37 -0
- package/skills/testomatio/test-management/qa-e2e-tests-reporting/references/TESTOMATIO_REPORTERS_CONFIG.md +118 -0
- package/skills/testomatio/test-management/qa-pr-requirements-analyzer/SKILL.md +149 -0
- package/skills/testomatio/test-management/qa-pr-requirements-analyzer/references/summary-example.md +54 -0
- package/skills/testomatio/test-management/qa-requirement-reviewer/SKILL.md +126 -0
- package/skills/testomatio/test-management/qa-requirement-reviewer/references/requirements_reviewer_examples.md +128 -0
- package/skills/testomatio/test-management/qa-split-testing-levels-pyramid/SKILL.md +55 -0
- package/skills/testomatio/test-management/qa-sprint-report-by-testomatio/SKILL.md +220 -0
- package/skills/testomatio/test-management/qa-sprint-report-by-testomatio/references/qa-sprint-report.md +126 -0
- package/skills/testomatio/test-management/qa-test-code-coverage/SKILL.md +174 -0
- package/skills/testomatio/test-management/qa-test-code-coverage/references/COVERAGE_FILE_FORMAT.md +138 -0
- package/skills/testomatio/test-management/qa-test-code-coverage/references/E2E_FRAMEWORKS.md +65 -0
- package/skills/testomatio/test-management/qa-test-code-coverage/scripts/check-coverage.mjs +40 -0
- package/skills/testomatio/test-management/qa-thinking/SKILL.md +36 -0
- package/skills/testomatio/test-management/qa-write-test-cases/SKILL.md +267 -0
- package/skills/testomatio/test-management/qa-write-test-cases/references/test-case-format.md +387 -0
- package/skills/testomatio/test-management/qa-write-test-cases/references/testomat-tms-guide.md +72 -0
- package/skills/testomatio/test-management/qa-write-test-cases/references/writing-rule.md +159 -0
- package/skills/testomatio/test-management/scan-automation-project/SKILL.md +152 -0
- package/skills/testomatio/test-management/sync-test-cases-with-tms/SKILL.md +107 -0
- package/skills/testomatio/test-management/sync-test-cases-with-tms/references/TESTOMATIO_CLI.md +140 -0
- package/skills/testomatio/test-management/testomatio-mcp/SKILL.md +195 -0
- package/skills/testomatio/test-management/testomatio-mcp/references/MCP_SETUP.md +177 -0
package/README.md
CHANGED
|
@@ -1,73 +1,399 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/testomatio/testeiya-app/main/assets/testeiya-cli-logo.png" alt="Testeiya" width="220">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
1
5
|
# Testeiya
|
|
2
6
|
|
|
7
|
+
[](https://www.npmjs.com/package/testeiya)
|
|
8
|
+
[](LICENSE)
|
|
9
|
+
|
|
3
10
|
An agent, app, and the goddess of testing.
|
|
4
11
|
|
|
5
|
-
Testeiya is
|
|
12
|
+
Testeiya is an autonomous QA agent. It reacts to triggers — a pull request, a new
|
|
13
|
+
issue, a failed test run, a deploy — runs exactly one analysis, and delivers its
|
|
14
|
+
verdict where your team works: a PR comment, a markdown report, or a Testomat.io
|
|
15
|
+
project. There is no interactive mode and no human in the loop. That is the point:
|
|
16
|
+
you wire it into CI once, and every event that should get QA thinking gets it.
|
|
6
17
|
|
|
7
|
-
|
|
18
|
+
It ships as a desktop app, a web app, and the command-line agent in this repository.
|
|
8
19
|
|
|
9
20
|
| Folder | What it is |
|
|
10
21
|
|---|---|
|
|
11
|
-
| `prompt/` |
|
|
12
|
-
| `skills/` | The manifest of skills the agent can invoke
|
|
22
|
+
| `prompt/` | System-prompt fragments: the agent's role, rules, tool guidance, Testomat.io operating rules, and the report contract |
|
|
23
|
+
| `skills/` | The manifest of skills the agent can invoke. Every folder is fetched from its own upstream repository |
|
|
13
24
|
| `src/` | The `testeiya` command-line agent (Node) |
|
|
14
25
|
|
|
15
|
-
The desktop and web harness
|
|
26
|
+
The desktop and web harness is not open source. That covers the servers, session management, sync, and UI.
|
|
16
27
|
|
|
17
|
-
## Install
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
Requires Node 22.19 or newer, an LLM provider key, and a model.
|
|
18
31
|
|
|
19
32
|
```bash
|
|
20
|
-
npx testeiya
|
|
21
|
-
--model openrouter/anthropic/claude-sonnet-5 --output report.md
|
|
33
|
+
npx testeiya doctor
|
|
22
34
|
```
|
|
23
35
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
Requires Node 22.19 or newer, a model, and an LLM provider key.
|
|
36
|
+
`doctor` reports which key won, which skills loaded, and whether Testomat.io is reachable — without spending a token.
|
|
27
37
|
|
|
28
|
-
|
|
38
|
+
The key comes from the environment (`OPENROUTER_API_KEY`, `ANTHROPIC_API_KEY`,
|
|
39
|
+
`OPENAI_API_KEY`, `GEMINI_API_KEY`), from `~/.testeiya/.env`, or from
|
|
40
|
+
`~/.testeiya/auth.json`. That last file is the one the desktop app's Settings
|
|
41
|
+
dialog writes, so configuring it once covers both.
|
|
29
42
|
|
|
30
|
-
|
|
43
|
+
There is no default model. Name one with `--model <provider>/<id>` or
|
|
44
|
+
`TESTEIYA_MODEL`. CI usually has neither set, so a run that resolves no model
|
|
45
|
+
exits `2` rather than picking one for you.
|
|
31
46
|
|
|
32
47
|
```bash
|
|
33
|
-
testeiya --help # every flag
|
|
34
48
|
export TESTEIYA_MODEL=openrouter/anthropic/claude-sonnet-5
|
|
35
|
-
testeiya
|
|
36
|
-
|
|
49
|
+
testeiya models anthropic # list what your key can reach
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## How it runs
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
testeiya task "<task>" # run one task, deliver a report, exit
|
|
56
|
+
testeiya ask "<question>" # answer a question, no report
|
|
57
|
+
testeiya skills # list the skills bundled with this package
|
|
58
|
+
testeiya sessions # list saved sessions for this folder
|
|
37
59
|
```
|
|
38
60
|
|
|
39
|
-
|
|
61
|
+
The agent runs one task and exits. Progress goes to stderr, so a run drops into CI as-is:
|
|
62
|
+
|
|
63
|
+
| Exit code | Meaning |
|
|
64
|
+
|---|---|
|
|
65
|
+
| `0` | pass, or a positive verdict from the agent |
|
|
66
|
+
| `1` | failed run, or a negative verdict — findings that need a human |
|
|
67
|
+
| `2` | bad usage |
|
|
68
|
+
| `130` | interrupted |
|
|
69
|
+
|
|
70
|
+
Pass `--exit-zero` when a negative verdict must not fail the job. A broken run
|
|
71
|
+
still exits `1`, bad usage still exits `2`, and the verdict is still in the
|
|
72
|
+
report and in the run envelope for anything that wants to gate on it.
|
|
73
|
+
|
|
74
|
+
A task can come from stdin too:
|
|
40
75
|
|
|
41
76
|
```bash
|
|
42
|
-
|
|
77
|
+
cat issue-42.md | testeiya task --output report.md
|
|
43
78
|
```
|
|
44
79
|
|
|
45
|
-
|
|
80
|
+
Every command takes `--json` for machine-readable output of the run envelope
|
|
81
|
+
(verdict, reason, tokens, session id). `testeiya --help` lists all options;
|
|
82
|
+
`testeiya help` is the full guide.
|
|
83
|
+
|
|
84
|
+
## Scenarios
|
|
85
|
+
|
|
86
|
+
Each scenario below is a single unattended run. Nobody answers questions; the
|
|
87
|
+
agent reads what the trigger gives it and acts on its own judgement. If its
|
|
88
|
+
verdict is negative, the job fails — that failure *is* the signal.
|
|
89
|
+
|
|
90
|
+
### Grill a pull request
|
|
91
|
+
|
|
92
|
+
Every PR gets a QA review before merge. The agent loads the branch's diff,
|
|
93
|
+
analyzes it through the `qa-thinking` skill — edge cases, negative flows,
|
|
94
|
+
abuses, data-consistency risks — and posts the findings as a PR comment:
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
git fetch origin "$PR_BRANCH"
|
|
98
|
+
git checkout "$PR_BRANCH"
|
|
99
|
+
testeiya task "Review this pull request as a QA engineer. What could go wrong?" \
|
|
100
|
+
--output gh:pr-comment --output review.json
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Exit code `1` means the agent found real risk, so you can gate the merge on it.
|
|
104
|
+
|
|
105
|
+
### Write test cases from a pull request
|
|
106
|
+
|
|
107
|
+
Same trigger, different deliverable: the `qa-write-test-cases` skill turns the
|
|
108
|
+
change into test cases in Testomat.io markdown format, written as a build
|
|
109
|
+
artifact ready to commit or import:
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
testeiya task "Create test cases covering the changes in this pull request" \
|
|
113
|
+
--output testcases/
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
With `TESTOMATIO` and the project id set, add
|
|
117
|
+
`sync them to Testomat.io` to the task and they land straight in the TMS via
|
|
118
|
+
`sync-test-cases-with-tms`.
|
|
119
|
+
|
|
120
|
+
### Create test cases from a new issue
|
|
121
|
+
|
|
122
|
+
A requirements text arrives from the issue tracker — piped in, no human
|
|
123
|
+
summarizing it first. The agent writes a checklist and test cases from it:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
gh issue view 57 --json title,body -q '.title + "\n\n" + .body' \
|
|
127
|
+
| testeiya task --output testcases/issue-57.md
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### Explore a deployed app with Explorbot
|
|
131
|
+
|
|
132
|
+
After a deploy, point [Explorbot](https://github.com/testomatio/explorbot) — the
|
|
133
|
+
autonomous browser-testing CLI the agent drives through the `explorbot-*`
|
|
134
|
+
skills — at the staging URL. It researches, plans, and tests the live app in
|
|
135
|
+
its own browser, then reports what broke:
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
testeiya task "Run explorbot against https://staging.example.com, max 10 tests, report failures" \
|
|
139
|
+
--output explorbot-report.md
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
### Repair failing tests after a red build
|
|
143
|
+
|
|
144
|
+
On a failed CodeceptJS run, the `ci-fix-tests` skill attempts safe fixes only —
|
|
145
|
+
locator drift, missing waits — reruns just the failing scenarios, rolls back any
|
|
146
|
+
edit that did not help, and always writes `output/ci-fix.md` for the next job:
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
testeiya task "Fix the failed CodeceptJS tests using ci-fix-tests. No refactors." \
|
|
150
|
+
--no-session
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
### Audit the test suite on a schedule
|
|
154
|
+
|
|
155
|
+
Nightly, the agent walks the Testomat.io project and reports gaps — suites with
|
|
156
|
+
no tests, cases gone stale against the current code, coverage holes:
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
TESTOMATIO=tstmt_xxx testeiya task \
|
|
160
|
+
"Audit this project: which suites have no automated tests, and which manual cases look automatable?" \
|
|
161
|
+
--output audits/$(date +%F).md
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
## CI setup
|
|
165
|
+
|
|
166
|
+
Testeiya needs three things in any CI system: a provider key, a model name, and
|
|
167
|
+
the checkout of whatever the task reads. Everything else is standard.
|
|
168
|
+
|
|
169
|
+
### GitHub Actions
|
|
170
|
+
|
|
171
|
+
GitHub runners ship the [GitHub CLI](https://cli.github.com), so posting back to
|
|
172
|
+
the PR is one flag. `GITHUB_TOKEN` authenticates it; keep the LLM key in
|
|
173
|
+
repository secrets.
|
|
174
|
+
|
|
175
|
+
```yaml
|
|
176
|
+
name: qa-review
|
|
177
|
+
on:
|
|
178
|
+
pull_request:
|
|
179
|
+
types: [opened, synchronize]
|
|
180
|
+
|
|
181
|
+
permissions:
|
|
182
|
+
pull-requests: write
|
|
183
|
+
contents: read
|
|
184
|
+
|
|
185
|
+
jobs:
|
|
186
|
+
grill:
|
|
187
|
+
runs-on: ubuntu-latest
|
|
188
|
+
env:
|
|
189
|
+
GH_TOKEN: ${{ github.token }}
|
|
190
|
+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
|
191
|
+
TESTEIYA_MODEL: openrouter/anthropic/claude-sonnet-5
|
|
192
|
+
# optional, for scenarios that touch Testomat.io
|
|
193
|
+
TESTOMATIO: ${{ secrets.TESTOMATIO }}
|
|
194
|
+
TESTOMATIO_PROJECT_ID: my-project
|
|
195
|
+
steps:
|
|
196
|
+
- uses: actions/checkout@v4
|
|
197
|
+
with:
|
|
198
|
+
fetch-depth: 0
|
|
199
|
+
- uses: actions/setup-node@v4
|
|
200
|
+
with:
|
|
201
|
+
node-version: 22
|
|
202
|
+
- run: npx testeiya@latest doctor
|
|
203
|
+
- run: |
|
|
204
|
+
npx testeiya@latest task \
|
|
205
|
+
"Review this pull request as a QA engineer. What could go wrong?" \
|
|
206
|
+
--output gh:pr-comment --output review.json
|
|
207
|
+
- uses: actions/upload-artifact@v4
|
|
208
|
+
if: always()
|
|
209
|
+
with:
|
|
210
|
+
name: review
|
|
211
|
+
path: review.json
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
The same shape works for the other scenarios: trigger on `issues` and pipe the
|
|
215
|
+
body in for test-case generation, or run on a `schedule` for the nightly audit.
|
|
216
|
+
|
|
217
|
+
### GitLab CI
|
|
218
|
+
|
|
219
|
+
GitLab runners do not ship `gh`, so the report goes to job artifacts — visible
|
|
220
|
+
in the merge request pipeline page. Exit code `1` fails the job and blocks the
|
|
221
|
+
merge when you mark it required.
|
|
222
|
+
|
|
223
|
+
```yaml
|
|
224
|
+
qa-review:
|
|
225
|
+
image: node:22
|
|
226
|
+
rules:
|
|
227
|
+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
228
|
+
variables:
|
|
229
|
+
TESTEIYA_MODEL: openrouter/anthropic/claude-sonnet-5
|
|
230
|
+
TESTOMATIO_PROJECT_ID: my-project
|
|
231
|
+
script:
|
|
232
|
+
- git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
|
|
233
|
+
- npx testeiya@latest doctor
|
|
234
|
+
- npx testeiya@latest task
|
|
235
|
+
"Review this merge request as a QA engineer. What could go wrong?"
|
|
236
|
+
--output review/report.md --output review/envelope.json
|
|
237
|
+
artifacts:
|
|
238
|
+
when: always
|
|
239
|
+
paths:
|
|
240
|
+
- review/
|
|
241
|
+
parallel:
|
|
242
|
+
# secrets go to Settings > CI/CD > Variables:
|
|
243
|
+
# OPENROUTER_API_KEY (masked), TESTOMATIO (masked, protected)
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
For posting back to the merge request, hand the envelope's report to GitLab's
|
|
247
|
+
API in a follow-up step, or install `gh` plus a token mirror if the project is
|
|
248
|
+
also on GitHub.
|
|
249
|
+
|
|
250
|
+
### Anywhere else
|
|
251
|
+
|
|
252
|
+
Any scheduler that can run a container works — the contract is just stdin,
|
|
253
|
+
stdout, and an exit code:
|
|
254
|
+
|
|
255
|
+
```bash
|
|
256
|
+
echo "Audit the checkout suite for gaps" | testeiya task --output audit.md
|
|
257
|
+
case $? in
|
|
258
|
+
0) echo "clean" ;;
|
|
259
|
+
1) echo "findings need review" ;;
|
|
260
|
+
esac
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
## Where the report goes
|
|
264
|
+
|
|
265
|
+
`--output` names a destination and can repeat. Without one the report goes to stdout.
|
|
266
|
+
|
|
267
|
+
```bash
|
|
268
|
+
testeiya task "<task>" --output report.md --output run.json --output gh:pr-comment
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
| Destination | What happens |
|
|
272
|
+
|---|---|
|
|
273
|
+
| `report.md` | the agent writes the report there |
|
|
274
|
+
| `run.json` | the run envelope: verdict, reason, report, tokens, session id |
|
|
275
|
+
| `gh:pr-comment` | posted on this branch's pull request |
|
|
276
|
+
| `gh:pr#123` | posted on that pull request |
|
|
277
|
+
|
|
278
|
+
Posting uses the GitHub CLI and resolves the PR number from `GITHUB_EVENT_PATH`,
|
|
279
|
+
`GITHUB_REF`, or `gh pr view` — so inside Actions it just works. Every
|
|
280
|
+
destination is checked before the run starts, so a missing `gh` costs no tokens.
|
|
281
|
+
|
|
282
|
+
`--footer "<text>"` adds a line under the report and `--header "<text>"` adds
|
|
283
|
+
one above. Both go wherever the report goes: stdout, the file, the posted
|
|
284
|
+
comment. A footer is what turns a posted report into a conversation: tell the
|
|
285
|
+
reader how to answer, and let the workflow feed their reply back into the same
|
|
286
|
+
session.
|
|
287
|
+
|
|
288
|
+
```bash
|
|
289
|
+
testeiya task "Review this pull request" \
|
|
290
|
+
--output gh:pr-comment \
|
|
291
|
+
--footer "> You can reply to this comment by typing /testeiya"
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
Write no footer of your own and the report is signed:
|
|
295
|
+
|
|
296
|
+
> *🧚🏻♀️ Provided by [Testeiya QA Agent](https://testomat.ai/testeiya) & claude-sonnet-5*
|
|
297
|
+
|
|
298
|
+
Pass `--no-default-footer`, or set `TESTEIYA_NO_DEFAULT_FOOTER`, to drop it.
|
|
299
|
+
|
|
300
|
+
## Sessions
|
|
301
|
+
|
|
302
|
+
Runs are saved under `~/.testeiya`, so a follow-up picks up where the last one
|
|
303
|
+
stopped. A resumed run reuses its session's model.
|
|
304
|
+
|
|
305
|
+
```bash
|
|
306
|
+
testeiya task "Review the checkout suite" --name checkout-review --output report.md
|
|
307
|
+
testeiya task "Now write the missing cases" -c
|
|
308
|
+
testeiya sessions
|
|
309
|
+
testeiya task "<task>" --resume <id>
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
Pass `--name` to label a session and `--no-session` to save nothing.
|
|
313
|
+
|
|
314
|
+
In CI, `--session <label>` is the one to reach for: it continues the session
|
|
315
|
+
with that label, and starts it the first time, so a job that runs again and
|
|
316
|
+
again needs no "does it exist yet" branch. Give each thread its own label, and
|
|
317
|
+
carry `~/.testeiya` between rounds with the runner's cache. `--no-session` keeps
|
|
318
|
+
runners stateless when continuity is not wanted.
|
|
319
|
+
|
|
320
|
+
```bash
|
|
321
|
+
testeiya task "Review the new commits" --session "pr-42" --output gh:pr-comment
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
## Testomat.io
|
|
325
|
+
|
|
326
|
+
Set `TESTOMATIO` to a project API key. The agent can then read and write that
|
|
327
|
+
project's tests, suites, runs and plans through `check-tests` and the REST API.
|
|
328
|
+
|
|
329
|
+
```bash
|
|
330
|
+
TESTOMATIO=tstmt_xxx testeiya task --project my-project "Which suites have no tests?"
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
Add the project id and the agent also gets the Testomat.io MCP tools. The id
|
|
334
|
+
comes from `--project` or from `TESTOMATIO_PROJECT_ID`; the MCP server needs it,
|
|
335
|
+
because a token alone does not say which project to talk to. `TESTOMATIO_URL`
|
|
336
|
+
points at a self-hosted instance.
|
|
46
337
|
|
|
47
338
|
## Skills
|
|
48
339
|
|
|
49
|
-
A skill is a folder with a `SKILL.md`. The agent
|
|
340
|
+
A skill is a folder with a `SKILL.md`. The agent sees them all and reaches for
|
|
341
|
+
the ones a task calls for. Name one with a slash to make it certain:
|
|
342
|
+
|
|
343
|
+
```bash
|
|
344
|
+
testeiya task "Review this pull request as a QA engineer /qa-thinking"
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
That skill is loaded in front of the task before the run starts, so it does not
|
|
348
|
+
depend on the model deciding to open it. A name the package does not ship is
|
|
349
|
+
left as plain text, which keeps a task safe to build from someone else's words —
|
|
350
|
+
a `/word` in a pull request comment stays a word.
|
|
351
|
+
|
|
352
|
+
The set is vendored from upstream repositories and moves with every release, so
|
|
353
|
+
ask your own install rather than a list in a README:
|
|
354
|
+
|
|
355
|
+
```bash
|
|
356
|
+
testeiya skills # every bundled skill: name and what it is for
|
|
357
|
+
testeiya skills playwright # filter by name, category or description
|
|
358
|
+
testeiya skills --json # [{name, group, description}]
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
Categories today: QA process, test management, test automation, Explorbot,
|
|
362
|
+
Playwright, CodeceptJS.
|
|
50
363
|
|
|
51
|
-
|
|
364
|
+
Sources are declared in `skills/skills.yaml` and pinned in
|
|
365
|
+
`skills/skills.lock.json`. The vendored folders are deliberately not committed —
|
|
366
|
+
they belong to their authors, under their own licences. A clone has the manifest
|
|
367
|
+
and nothing else; `node scripts/vendor-skills.js` fills the tree. Every release
|
|
368
|
+
runs it, so the published `testeiya` package ships each skill as current on
|
|
369
|
+
release day.
|
|
52
370
|
|
|
53
|
-
|
|
371
|
+
`skillsOverride` in `src/session.ts` keeps only what is found under that tree,
|
|
372
|
+
so an arbitrary clone cannot hand the model its own skills. To add yours, point
|
|
373
|
+
`additionalSkillPaths` at your folder. [EXTENDING.md](EXTENDING.md) covers both hooks.
|
|
54
374
|
|
|
55
|
-
To propose a new source, add its line to `skills/skills.yaml
|
|
375
|
+
To propose a new source, add its line to `skills/skills.yaml`. See [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
56
376
|
|
|
57
377
|
## Building your own agent
|
|
58
378
|
|
|
59
|
-
The CLI is a thin composition over [pi](https://pi.dev):
|
|
379
|
+
The CLI is a thin composition over [pi](https://pi.dev): about 1,500 lines wiring
|
|
380
|
+
the SDK to the prompt and skills here. A fork can add pi extensions and custom
|
|
381
|
+
tools, or swap the one-shot run loop for pi's full interactive TUI.
|
|
382
|
+
[EXTENDING.md](EXTENDING.md) walks through both.
|
|
60
383
|
|
|
61
384
|
## Contributing
|
|
62
385
|
|
|
63
|
-
Prompt wording is exactly what an outside contributor can improve, and a change
|
|
386
|
+
Prompt wording is exactly what an outside contributor can improve, and a change
|
|
387
|
+
to it changes how the agent behaves for everyone. Read
|
|
388
|
+
[CONTRIBUTING.md](CONTRIBUTING.md) first. It covers what belongs here and what
|
|
389
|
+
belongs upstream, in the repository that owns a given skill.
|
|
64
390
|
|
|
65
391
|
## Issues
|
|
66
392
|
|
|
67
393
|
This repository is also the public issue tracker for both surfaces:
|
|
68
394
|
|
|
69
|
-
-
|
|
70
|
-
-
|
|
395
|
+
- Testeiya Desktop app, the packaged desktop application
|
|
396
|
+
- Testeiya CLI, the command-line agent in `src/`
|
|
71
397
|
|
|
72
398
|
## Licence
|
|
73
399
|
|
package/dist/prompt/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { getSystemPrompt } from "./system-prompt.js";
|
|
2
2
|
import { testomatioTms, testomatioConnection, testomatioNotConnected, projectSettings, } from "./testomatio.js";
|
|
3
|
-
import { nonInteractive, reportOutput } from "./print.js";
|
|
3
|
+
import { briefAnswer, nonInteractive, reportOutput } from "./print.js";
|
|
4
4
|
import { contextPromptSection, } from "./context.js";
|
|
5
5
|
export function buildSystemPrompt(options) {
|
|
6
6
|
const mode = options?.mode ?? "tui";
|
|
@@ -15,6 +15,8 @@ export function buildSystemPrompt(options) {
|
|
|
15
15
|
interactive,
|
|
16
16
|
toolBullets: options?.toolBullets,
|
|
17
17
|
rules: options?.rules,
|
|
18
|
+
connectedClis: options?.connectedClis,
|
|
19
|
+
connectedMcps: options?.connectedMcps,
|
|
18
20
|
}),
|
|
19
21
|
];
|
|
20
22
|
const tokenSlugs = options?.tokens ? Object.keys(options.tokens) : [];
|
|
@@ -48,7 +50,9 @@ export function buildSystemPrompt(options) {
|
|
|
48
50
|
else if (mode === "web" || mode === "print") {
|
|
49
51
|
parts.push(testomatioNotConnected());
|
|
50
52
|
}
|
|
51
|
-
// The
|
|
53
|
+
// The answer contract goes last so it is the final instruction the model reads.
|
|
54
|
+
if (options?.brief)
|
|
55
|
+
parts.push(briefAnswer);
|
|
52
56
|
if (options?.outputFile)
|
|
53
57
|
parts.push(reportOutput(options.outputFile));
|
|
54
58
|
return parts.join("\n\n");
|
package/dist/prompt/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../prompt/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,sBAAsB,EACtB,eAAe,GAEhB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../prompt/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,sBAAsB,EACtB,eAAe,GAEhB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AACvE,OAAO,EACL,oBAAoB,GAGrB,MAAM,cAAc,CAAC;AAGtB,MAAM,UAAU,iBAAiB,CAAC,OAA6B;IAC7D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,WAAW,GAAG,IAAI,KAAK,OAAO,CAAC;IAErC,+EAA+E;IAC/E,gFAAgF;IAChF,6EAA6E;IAC7E,oEAAoE;IACpE,8CAA8C;IAC9C,MAAM,KAAK,GAAa;QACtB,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5B,WAAW;YACX,WAAW,EAAE,OAAO,EAAE,WAAW;YACjC,KAAK,EAAE,OAAO,EAAE,KAAK;YACrB,aAAa,EAAE,OAAO,EAAE,aAAa;YACrC,aAAa,EAAE,OAAO,EAAE,aAAa;SACtC,CAAC;KACH,CAAC;IAEF,MAAM,UAAU,GAAG,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACtE,yEAAyE;IACzE,MAAM,cAAc,GAAG,OAAO,EAAE,UAAU,EAAE,cAAc,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;IACpF,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,IAAI,YAAY,CAAC;IACzC,6EAA6E;IAC7E,6EAA6E;IAC7E,qEAAqE;IACrE,IAAI,cAAc,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5E,IAAI,IAAI,KAAK,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACjD,8EAA8E;IAC9E,2EAA2E;IAC3E,yEAAyE;IACzE,KAAK,MAAM,OAAO,IAAI,OAAO,EAAE,QAAQ,IAAI,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEnE,IAAI,OAAO,EAAE,aAAa,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,8BAA8B,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC;QACvE,KAAK,CAAC,IAAI,CACR,oBAAoB,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,EAAE,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC,CACjF,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,EAAE,WAAW,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,cAAc,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9F,CAAC;SAAM,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,gFAAgF;IAChF,IAAI,OAAO,EAAE,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5C,IAAI,OAAO,EAAE,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAEtE,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC"}
|
package/dist/prompt/print.js
CHANGED
|
@@ -19,4 +19,12 @@ export function reportOutput(path) {
|
|
|
19
19
|
</final-report>
|
|
20
20
|
`;
|
|
21
21
|
}
|
|
22
|
+
export const briefAnswer = dedent `
|
|
23
|
+
<answer>
|
|
24
|
+
* You were asked a question, not given a task. Answer it.
|
|
25
|
+
* Lead with the answer in one line, then the evidence you checked.
|
|
26
|
+
* A few sentences. No report file, no headings, no plan.
|
|
27
|
+
* Say plainly when what you found does not settle the question.
|
|
28
|
+
</answer>
|
|
29
|
+
`;
|
|
22
30
|
//# sourceMappingURL=print.js.map
|
package/dist/prompt/print.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"print.js","sourceRoot":"","sources":["../../prompt/print.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAA;;;;;;;;;CASnC,CAAC;AAEF,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,MAAM,CAAA;;4CAE6B,IAAI;;;;;CAK/C,CAAC;AACF,CAAC"}
|
|
1
|
+
{"version":3,"file":"print.js","sourceRoot":"","sources":["../../prompt/print.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAA;;;;;;;;;CASnC,CAAC;AAEF,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,MAAM,CAAA;;4CAE6B,IAAI;;;;;CAK/C,CAAC;AACF,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,MAAM,CAAA;;;;;;;CAOhC,CAAC"}
|
|
@@ -9,6 +9,8 @@ export function getSystemPrompt(cwd, options) {
|
|
|
9
9
|
let extraRules = "";
|
|
10
10
|
for (const rule of options?.rules ?? [])
|
|
11
11
|
extraRules += ` * ${rule}\n`;
|
|
12
|
+
const cliList = (options?.connectedClis ?? []).join(", ") || "none connected yet";
|
|
13
|
+
const mcpList = (options?.connectedMcps ?? []).join(", ") || "none connected yet";
|
|
12
14
|
return dedent `
|
|
13
15
|
<role>
|
|
14
16
|
You are Testeiya, an AI agent that helps with QA tasks.
|
|
@@ -87,6 +89,16 @@ export function getSystemPrompt(cwd, options) {
|
|
|
87
89
|
|
|
88
90
|
${tools({ extra: options?.toolBullets })}
|
|
89
91
|
|
|
92
|
+
<connections>
|
|
93
|
+
Testeiya connects external tools for you — CLI tools and MCP servers. This session has:
|
|
94
|
+
|
|
95
|
+
* **Connected CLIs:** ${cliList}
|
|
96
|
+
* **Connected MCP servers:** ${mcpList}
|
|
97
|
+
|
|
98
|
+
* **Missing Connection — Ask, Never Improvise:** When a task needs a tool that is not in the lists above, STOP and ask the user to connect it in Testeiya (Settings → Connections). Never reach the service sideways: no raw REST/GraphQL calls against its API, no scraping credentials from dotfiles or env dumps, no installing binaries on your own.
|
|
99
|
+
* If a task needs both a CLI and an MCP server, ask once for both and say that Testeiya supports them both as connections — the user installs them side by side there.
|
|
100
|
+
</connections>
|
|
101
|
+
|
|
90
102
|
<goals>
|
|
91
103
|
You help in variety of tasks related to software testing, including writing test cases, analyzing test results, and providing feedback.
|
|
92
104
|
You have skills to perform QA tasks on user demand.
|
|
@@ -98,10 +110,8 @@ export function getSystemPrompt(cwd, options) {
|
|
|
98
110
|
* Assisting with manual test execution
|
|
99
111
|
* Manage tests with TMS Testomat.io
|
|
100
112
|
* Setting up CI pipelines for continuous testing
|
|
101
|
-
* Analyzing requirements
|
|
102
|
-
*
|
|
103
|
-
* Launching automated exploratory tests via explorbot
|
|
104
|
-
* Analyzing test results and providing feedback
|
|
113
|
+
* Analyzing requirements and issues
|
|
114
|
+
* Analyzing test results
|
|
105
115
|
|
|
106
116
|
This is what you can do.
|
|
107
117
|
</goals>
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"system-prompt.js","sourceRoot":"","sources":["../../prompt/system-prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAE/C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,UAAU,eAAe,CAAC,GAAY,EAAE,OAAuB;IACnE,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,IAAI,IAAI,CAAC;IACjD,IAAI,mBAAmB,GAAG,qCAAqC,CAAC;IAChE,IAAI,CAAC,WAAW;QAAE,mBAAmB,GAAG,uCAAuC,CAAC;IAChF,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,KAAK,IAAI,EAAE;QAAE,UAAU,IAAI,SAAS,IAAI,IAAI,CAAC;IACzE,OAAO,MAAM,CAAA;;;;;;;;;;;qBAWM,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE;oEAC2B,iBAAiB;yEACZ,iBAAiB;uCACnD,iBAAiB;wEACgB,iBAAiB;0BAC/D,iBAAiB,0GAA0G,iBAAiB,0DAA0D,iBAAiB;;yHAExH,iBAAiB;;;;2CAI/F,iBAAiB;UAClD,iBAAiB;;;;;;;gWAOqU,iBAAiB;2FACtR,iBAAiB,sJAAsJ,iBAAiB;;6EAEtM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAyCxF,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;;IAExC,KAAK,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC
|
|
1
|
+
{"version":3,"file":"system-prompt.js","sourceRoot":"","sources":["../../prompt/system-prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAE/C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,UAAU,eAAe,CAAC,GAAY,EAAE,OAAuB;IACnE,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,IAAI,IAAI,CAAC;IACjD,IAAI,mBAAmB,GAAG,qCAAqC,CAAC;IAChE,IAAI,CAAC,WAAW;QAAE,mBAAmB,GAAG,uCAAuC,CAAC;IAChF,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,KAAK,IAAI,EAAE;QAAE,UAAU,IAAI,SAAS,IAAI,IAAI,CAAC;IACzE,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC;IAClF,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC;IAClF,OAAO,MAAM,CAAA;;;;;;;;;;;qBAWM,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE;oEAC2B,iBAAiB;yEACZ,iBAAiB;uCACnD,iBAAiB;wEACgB,iBAAiB;0BAC/D,iBAAiB,0GAA0G,iBAAiB,0DAA0D,iBAAiB;;yHAExH,iBAAiB;;;;2CAI/F,iBAAiB;UAClD,iBAAiB;;;;;;;gWAOqU,iBAAiB;2FACtR,iBAAiB,sJAAsJ,iBAAiB;;6EAEtM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAyCxF,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;;IAExC,KAAK,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;;;;;4BAKd,OAAO;mCACA,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;gCA2BV,WAAW,CAAC,CAAC,CAAC,sEAAsE,CAAC,CAAC,CAAC,6EAA6E;;;;;;;;;;;;;;;;;;;EAmBlM,UAAU,6HAA6H,mBAAmB;;;;;gFAK5E,iBAAiB;;;;GAI9F,CAAC;AACJ,CAAC;AAeD;;;;;;;;;;;;;;;;;;;;EAoBE"}
|
|
@@ -32,7 +32,7 @@ const tmsViaDirectTools = dedent `
|
|
|
32
32
|
| Runs, testruns, plans, labels, issues, analytics | MCP \`testomatio-*\` tools |
|
|
33
33
|
| Creating/updating tests or suites | edit the markdown file, then \`npx check-tests push\` |
|
|
34
34
|
| Creating/updating runs, linking issues | MCP create/update tools |
|
|
35
|
-
| Launching automated or mixed Run remotely on CI | Use npx @testomatio/reporter run --remote <profile>, raise error if no CI profiles configured
|
|
35
|
+
| Launching automated or mixed Run remotely on CI | Use npx -y @testomatio/reporter@latest run --remote <profile>, raise error if no CI profiles configured
|
|
36
36
|
| Running automated Run | Use local test runner with testomatio reporter attached to report to current project
|
|
37
37
|
|
|
38
38
|
---
|
|
@@ -69,7 +69,7 @@ const tmsViaProxy = dedent `
|
|
|
69
69
|
| Runs, testruns, plans, labels, issues, analytics | \`mcp({ search: … })\`, then \`mcp({ tool: … })\` |
|
|
70
70
|
| Creating/updating tests or suites | edit the markdown file, then \`npx check-tests push\` |
|
|
71
71
|
| Creating/updating runs, linking issues | the matching \`mcp\` create/update operation |
|
|
72
|
-
| Launching automated or mixed Run remotely on CI | Use npx @testomatio/reporter run --remote <profile>, raise error if no CI profiles configured |
|
|
72
|
+
| Launching automated or mixed Run remotely on CI | Use npx -y @testomatio/reporter@latest run --remote <profile>, raise error if no CI profiles configured |
|
|
73
73
|
| Running automated Run | Use local test runner with testomatio reporter attached to report to current project |
|
|
74
74
|
|
|
75
75
|
---
|
|
@@ -101,7 +101,7 @@ const tmsViaCli = dedent `
|
|
|
101
101
|
| Suite hierarchy / structure / which files exist | \`ls\` / \`find\` |
|
|
102
102
|
| Runs, testruns, plans, labels, issues, analytics | \`curl\` against \`/api/v2\` |
|
|
103
103
|
| Creating/updating tests or suites | edit the markdown file, then \`npx check-tests push\` |
|
|
104
|
-
| Launching automated or mixed Run remotely on CI | Use npx @testomatio/reporter run --remote <profile>, raise error if no CI profiles configured |
|
|
104
|
+
| Launching automated or mixed Run remotely on CI | Use npx -y @testomatio/reporter@latest run --remote <profile>, raise error if no CI profiles configured |
|
|
105
105
|
| Running automated Run | Use local test runner with testomatio reporter attached to report to current project |
|
|
106
106
|
|
|
107
107
|
---
|
package/dist/prompt/tools.js
CHANGED
|
@@ -9,11 +9,19 @@ import dedent from 'dedent';
|
|
|
9
9
|
*/
|
|
10
10
|
export function tools(options) {
|
|
11
11
|
const extra = (options?.extra ?? []).join('');
|
|
12
|
-
let
|
|
13
|
-
if (
|
|
14
|
-
|
|
12
|
+
let routing = ' - GitHub: issues and projects through MCP.';
|
|
13
|
+
if (onPath('gh')) {
|
|
14
|
+
routing +=
|
|
15
15
|
'\n - Pull requests and repositories: `gh` in `bash`. When a PR question needs issue detail, query the issue through MCP — never infer it from a PR title.';
|
|
16
16
|
}
|
|
17
|
+
if (onPath('acli')) {
|
|
18
|
+
routing +=
|
|
19
|
+
'\n - Jira in bulk (many issues at once), boards, sprints and filters: `acli` in `bash`, per the `atlassian` skill. Confluence and attachments have no `acli` command and stay on MCP.';
|
|
20
|
+
}
|
|
21
|
+
if (onPath('glab')) {
|
|
22
|
+
routing +=
|
|
23
|
+
'\n - GitLab releases, labels, milestones, snippets, CI artifacts, and any endpoint with no MCP tool: `glab` in `bash`. Issues, merge requests, pipelines and code search stay on MCP, which covers them well.';
|
|
24
|
+
}
|
|
17
25
|
return dedent `
|
|
18
26
|
<available-tools>
|
|
19
27
|
You have these tools available:
|
|
@@ -25,7 +33,7 @@ ${extra} * **Internal Skills:** For QA-related tasks, check available skills fi
|
|
|
25
33
|
* **External Integrations:** Use MCP tools when they provide superior data or specialized integrations.
|
|
26
34
|
- Primary MCP (Testomat.io): Use Testomat.io MCP tools to extend functionality.
|
|
27
35
|
- Secondary MCP (Jira, GitHub, etc.): Invoke only when user explicitly asks or when remote context exploration is required.
|
|
28
|
-
${
|
|
36
|
+
${routing}
|
|
29
37
|
|
|
30
38
|
<tool-governance>
|
|
31
39
|
* **Prioritize Specificity:** Use \`read\` instead of \`cat\`, and \`edit\` instead of \`sed\`. Dedicated tools provide better error tracking and safety.
|
|
@@ -36,14 +44,27 @@ ${github}
|
|
|
36
44
|
</available-tools>
|
|
37
45
|
`;
|
|
38
46
|
}
|
|
39
|
-
/** Is
|
|
40
|
-
function
|
|
47
|
+
/** Is this CLI on PATH? Never tell the agent to use a binary this machine lacks. */
|
|
48
|
+
function onPath(bin) {
|
|
41
49
|
for (const dir of (process.env.PATH ?? '').split(delimiter)) {
|
|
42
50
|
if (!dir)
|
|
43
51
|
continue;
|
|
44
|
-
if (existsSync(join(dir,
|
|
52
|
+
if (existsSync(join(dir, bin)) || existsSync(join(dir, `${bin}.exe`)))
|
|
45
53
|
return true;
|
|
46
54
|
}
|
|
47
55
|
return false;
|
|
48
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* CLIs Testeiya knows how to connect, probed on PATH — the same trio routed in
|
|
59
|
+
* `<available-tools>` above. Shared with the prompt's connections section so
|
|
60
|
+
* the two can never disagree about whether `gh` etc. exist.
|
|
61
|
+
*/
|
|
62
|
+
export const KNOWN_CLIS = [
|
|
63
|
+
{ bin: 'gh', label: 'gh (GitHub CLI)' },
|
|
64
|
+
{ bin: 'acli', label: 'acli (Atlassian CLI)' },
|
|
65
|
+
{ bin: 'glab', label: 'glab (GitLab CLI)' },
|
|
66
|
+
];
|
|
67
|
+
export function pathClis() {
|
|
68
|
+
return KNOWN_CLIS.filter((c) => onPath(c.bin)).map((c) => c.label);
|
|
69
|
+
}
|
|
49
70
|
//# sourceMappingURL=tools.js.map
|
package/dist/prompt/tools.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../../prompt/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B;;;;;GAKG;AACH,MAAM,UAAU,KAAK,CAAC,OAA8B;IAClD,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9C,IAAI,
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../../prompt/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B;;;;;GAKG;AACH,MAAM,UAAU,KAAK,CAAC,OAA8B;IAClD,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9C,IAAI,OAAO,GAAG,gDAAgD,CAAC;IAC/D,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACjB,OAAO;YACL,8JAA8J,CAAC;IACnK,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QACnB,OAAO;YACL,0LAA0L,CAAC;IAC/L,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QACnB,OAAO;YACL,kNAAkN,CAAC;IACvN,CAAC;IACD,OAAO,MAAM,CAAA;;;;;;;EAOb,KAAK;;;;EAIL,OAAO;;;;;;;;;CASR,CAAC;AACF,CAAC;AAED,oFAAoF;AACpF,SAAS,MAAM,CAAC,GAAW;IACzB,KAAK,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5D,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;IACrF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAqC;IAC1D,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,iBAAiB,EAAE;IACvC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,sBAAsB,EAAE;IAC9C,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE;CAC5C,CAAC;AAEF,MAAM,UAAU,QAAQ;IACtB,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC"}
|