retestkit 1.4.1 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. package/README.md +59 -40
  2. package/dist/config.js +8 -8
  3. package/dist/config.js.map +1 -1
  4. package/dist/logger.js +1 -1
  5. package/dist/logger.js.map +1 -1
  6. package/dist/prompts/index.d.ts +1 -1
  7. package/dist/prompts/index.d.ts.map +1 -1
  8. package/dist/prompts/index.js +21 -21
  9. package/dist/prompts/index.js.map +1 -1
  10. package/dist/prompts/templates/mcp/retest-crawl.md +7 -0
  11. package/{src/prompts/templates/mcp/webtest-discover-flows.md → dist/prompts/templates/mcp/retest-discover-flows.md} +1 -1
  12. package/{src/prompts/templates/mcp/webtest-discover.md → dist/prompts/templates/mcp/retest-discover.md} +2 -2
  13. package/dist/prompts/templates/mcp/retest-full-workflow.md +12 -0
  14. package/{src/prompts/templates/mcp/webtest-generate-tests.md → dist/prompts/templates/mcp/retest-generate-tests.md} +1 -1
  15. package/{src/prompts/templates/mcp/webtest-run-test.md → dist/prompts/templates/mcp/retest-run-test.md} +1 -1
  16. package/{src/prompts/templates/mcp/webtest-start.md → dist/prompts/templates/mcp/retest-start.md} +1 -1
  17. package/{src → dist}/prompts/templates/sampling/system-prefix.md +1 -1
  18. package/dist/resources/index.js +7 -7
  19. package/dist/resources/index.js.map +1 -1
  20. package/dist/schemas/config.js +2 -2
  21. package/dist/schemas/config.js.map +1 -1
  22. package/dist/security/index.js +1 -1
  23. package/dist/security/index.js.map +1 -1
  24. package/dist/server.js +3 -3
  25. package/dist/server.js.map +1 -1
  26. package/dist/test-utils/mock-context.js +22 -22
  27. package/dist/test-utils/mock-context.js.map +1 -1
  28. package/dist/tools/index.d.ts +1 -1
  29. package/dist/tools/index.d.ts.map +1 -1
  30. package/dist/tools/index.js +5 -5
  31. package/dist/tools/index.js.map +1 -1
  32. package/dist/tools/retest/crawl.d.ts.map +1 -0
  33. package/dist/tools/{webtest → retest}/crawl.js +7 -7
  34. package/dist/tools/retest/crawl.js.map +1 -0
  35. package/dist/tools/retest/discover-features.d.ts.map +1 -0
  36. package/dist/tools/{webtest → retest}/discover-features.js +6 -6
  37. package/dist/tools/retest/discover-features.js.map +1 -0
  38. package/dist/tools/retest/discover-flows.d.ts.map +1 -0
  39. package/dist/tools/{webtest → retest}/discover-flows.js +6 -6
  40. package/dist/tools/retest/discover-flows.js.map +1 -0
  41. package/dist/tools/retest/generate-tests.d.ts.map +1 -0
  42. package/dist/tools/{webtest → retest}/generate-tests.js +5 -5
  43. package/dist/tools/retest/generate-tests.js.map +1 -0
  44. package/dist/tools/retest/index.d.ts.map +1 -0
  45. package/dist/tools/retest/index.js.map +1 -0
  46. package/dist/tools/retest/run-test-case.d.ts.map +1 -0
  47. package/dist/tools/{webtest → retest}/run-test-case.js +3 -3
  48. package/dist/tools/retest/run-test-case.js.map +1 -0
  49. package/dist/tools/retest/schemas.d.ts.map +1 -0
  50. package/dist/tools/retest/schemas.js.map +1 -0
  51. package/dist/tools/retest/start-analysis.d.ts.map +1 -0
  52. package/dist/tools/{webtest → retest}/start-analysis.js +5 -5
  53. package/dist/tools/retest/start-analysis.js.map +1 -0
  54. package/dist/workspace/index.js +8 -8
  55. package/dist/workspace/index.js.map +1 -1
  56. package/dist/workspace/types.d.ts +2 -2
  57. package/dist/workspace/types.d.ts.map +1 -1
  58. package/package.json +6 -2
  59. package/.claude/commands/openspec/apply.md +0 -23
  60. package/.claude/commands/openspec/archive.md +0 -27
  61. package/.claude/commands/openspec/proposal.md +0 -28
  62. package/.gemini/commands/openspec/apply.toml +0 -21
  63. package/.gemini/commands/openspec/archive.toml +0 -25
  64. package/.gemini/commands/openspec/proposal.toml +0 -26
  65. package/.github/prompts/openspec-apply.prompt.md +0 -22
  66. package/.github/prompts/openspec-archive.prompt.md +0 -26
  67. package/.github/prompts/openspec-proposal.prompt.md +0 -27
  68. package/.github/workflows/release.yml +0 -33
  69. package/.kilocode/workflows/openspec-apply.md +0 -17
  70. package/.kilocode/workflows/openspec-archive.md +0 -21
  71. package/.kilocode/workflows/openspec-proposal.md +0 -22
  72. package/.mcp.json +0 -23
  73. package/.opencode/command/openspec-apply.md +0 -25
  74. package/.opencode/command/openspec-archive.md +0 -28
  75. package/.opencode/command/openspec-proposal.md +0 -30
  76. package/.roo/commands/openspec-apply.md +0 -20
  77. package/.roo/commands/openspec-archive.md +0 -24
  78. package/.roo/commands/openspec-proposal.md +0 -25
  79. package/.vscode/mcp.json +0 -23
  80. package/AGENTS.md +0 -18
  81. package/CLAUDE.md +0 -18
  82. package/dist/tools/webtest/crawl.d.ts.map +0 -1
  83. package/dist/tools/webtest/crawl.js.map +0 -1
  84. package/dist/tools/webtest/discover-features.d.ts.map +0 -1
  85. package/dist/tools/webtest/discover-features.js.map +0 -1
  86. package/dist/tools/webtest/discover-flows.d.ts.map +0 -1
  87. package/dist/tools/webtest/discover-flows.js.map +0 -1
  88. package/dist/tools/webtest/generate-tests.d.ts.map +0 -1
  89. package/dist/tools/webtest/generate-tests.js.map +0 -1
  90. package/dist/tools/webtest/index.d.ts.map +0 -1
  91. package/dist/tools/webtest/index.js.map +0 -1
  92. package/dist/tools/webtest/run-test-case.d.ts.map +0 -1
  93. package/dist/tools/webtest/run-test-case.js.map +0 -1
  94. package/dist/tools/webtest/schemas.d.ts.map +0 -1
  95. package/dist/tools/webtest/schemas.js.map +0 -1
  96. package/dist/tools/webtest/start-analysis.d.ts.map +0 -1
  97. package/dist/tools/webtest/start-analysis.js.map +0 -1
  98. package/openspec/AGENTS.md +0 -456
  99. package/openspec/changes/archive/2025-12-18-add-hybrid-artifact-paths/proposal.md +0 -33
  100. package/openspec/changes/archive/2025-12-18-add-hybrid-artifact-paths/specs/webtest-resources/spec.md +0 -27
  101. package/openspec/changes/archive/2025-12-18-add-hybrid-artifact-paths/specs/webtest-tools/spec.md +0 -304
  102. package/openspec/changes/archive/2025-12-18-add-hybrid-artifact-paths/tasks.md +0 -43
  103. package/openspec/changes/archive/2025-12-18-add-mcp-server-foundation/design.md +0 -209
  104. package/openspec/changes/archive/2025-12-18-add-mcp-server-foundation/proposal.md +0 -41
  105. package/openspec/changes/archive/2025-12-18-add-mcp-server-foundation/specs/mcp-server-core/spec.md +0 -183
  106. package/openspec/changes/archive/2025-12-18-add-mcp-server-foundation/tasks.md +0 -112
  107. package/openspec/changes/archive/2025-12-18-add-webtest-orchestrator/design.md +0 -333
  108. package/openspec/changes/archive/2025-12-18-add-webtest-orchestrator/proposal.md +0 -66
  109. package/openspec/changes/archive/2025-12-18-add-webtest-orchestrator/specs/mcp-server-core/spec.md +0 -129
  110. package/openspec/changes/archive/2025-12-18-add-webtest-orchestrator/specs/webtest-lifecycle/spec.md +0 -138
  111. package/openspec/changes/archive/2025-12-18-add-webtest-orchestrator/specs/webtest-logging/spec.md +0 -211
  112. package/openspec/changes/archive/2025-12-18-add-webtest-orchestrator/specs/webtest-prompts/spec.md +0 -157
  113. package/openspec/changes/archive/2025-12-18-add-webtest-orchestrator/specs/webtest-resources/spec.md +0 -213
  114. package/openspec/changes/archive/2025-12-18-add-webtest-orchestrator/specs/webtest-sampling/spec.md +0 -257
  115. package/openspec/changes/archive/2025-12-18-add-webtest-orchestrator/specs/webtest-tools/spec.md +0 -501
  116. package/openspec/changes/archive/2025-12-18-add-webtest-orchestrator/tasks.md +0 -264
  117. package/openspec/changes/archive/2025-12-18-allow-analysis-of-incomplete-crawls/proposal.md +0 -24
  118. package/openspec/changes/archive/2025-12-18-allow-analysis-of-incomplete-crawls/specs/webtest-tools/spec.md +0 -80
  119. package/openspec/changes/archive/2025-12-18-allow-analysis-of-incomplete-crawls/tasks.md +0 -8
  120. package/openspec/changes/archive/2025-12-18-fix-crawl-loop-stability/design.md +0 -90
  121. package/openspec/changes/archive/2025-12-18-fix-crawl-loop-stability/proposal.md +0 -28
  122. package/openspec/changes/archive/2025-12-18-fix-crawl-loop-stability/specs/webtest-sampling/spec.md +0 -90
  123. package/openspec/changes/archive/2025-12-18-fix-crawl-loop-stability/tasks.md +0 -33
  124. package/openspec/changes/archive/2025-12-18-use-markdown-artifacts/design.md +0 -558
  125. package/openspec/changes/archive/2025-12-18-use-markdown-artifacts/proposal.md +0 -119
  126. package/openspec/changes/archive/2025-12-18-use-markdown-artifacts/specs/webtest-resources/spec.md +0 -109
  127. package/openspec/changes/archive/2025-12-18-use-markdown-artifacts/specs/webtest-tools/spec.md +0 -121
  128. package/openspec/changes/archive/2025-12-18-use-markdown-artifacts/tasks.md +0 -133
  129. package/openspec/changes/extract-prompts-to-markdown/design.md +0 -86
  130. package/openspec/changes/extract-prompts-to-markdown/proposal.md +0 -50
  131. package/openspec/changes/extract-prompts-to-markdown/specs/webtest-prompts/spec.md +0 -74
  132. package/openspec/changes/extract-prompts-to-markdown/tasks.md +0 -40
  133. package/openspec/changes/refactor-webtest-naming/design.md +0 -95
  134. package/openspec/changes/refactor-webtest-naming/proposal.md +0 -66
  135. package/openspec/changes/refactor-webtest-naming/specs/webtest-prompts/spec.md +0 -79
  136. package/openspec/changes/refactor-webtest-naming/specs/webtest-resources/spec.md +0 -80
  137. package/openspec/changes/refactor-webtest-naming/specs/webtest-sampling/spec.md +0 -122
  138. package/openspec/changes/refactor-webtest-naming/specs/webtest-tools/spec.md +0 -113
  139. package/openspec/changes/refactor-webtest-naming/tasks.md +0 -119
  140. package/openspec/changes/rename-package-to-retest/proposal.md +0 -52
  141. package/openspec/changes/rename-package-to-retest/specs/mcp-server-core/spec.md +0 -53
  142. package/openspec/changes/rename-package-to-retest/specs/retest-lifecycle/spec.md +0 -68
  143. package/openspec/changes/rename-package-to-retest/specs/retest-logging/spec.md +0 -35
  144. package/openspec/changes/rename-package-to-retest/specs/retest-prompts/spec.md +0 -159
  145. package/openspec/changes/rename-package-to-retest/specs/retest-resources/spec.md +0 -251
  146. package/openspec/changes/rename-package-to-retest/specs/retest-sampling/spec.md +0 -99
  147. package/openspec/changes/rename-package-to-retest/specs/retest-tools/spec.md +0 -295
  148. package/openspec/changes/rename-package-to-retest/tasks.md +0 -71
  149. package/openspec/project.md +0 -31
  150. package/openspec/specs/mcp-server-core/spec.md +0 -178
  151. package/openspec/specs/webtest-lifecycle/spec.md +0 -136
  152. package/openspec/specs/webtest-logging/spec.md +0 -209
  153. package/openspec/specs/webtest-prompts/spec.md +0 -155
  154. package/openspec/specs/webtest-resources/spec.md +0 -248
  155. package/openspec/specs/webtest-sampling/spec.md +0 -344
  156. package/openspec/specs/webtest-tools/spec.md +0 -282
  157. package/release.config.js +0 -9
  158. package/src/config.test.ts +0 -96
  159. package/src/config.ts +0 -32
  160. package/src/elicitation/index.test.ts +0 -399
  161. package/src/elicitation/index.ts +0 -171
  162. package/src/elicitation/types.ts +0 -68
  163. package/src/index.ts +0 -83
  164. package/src/lifecycle/index.test.ts +0 -260
  165. package/src/lifecycle/index.ts +0 -101
  166. package/src/logger.redaction.test.ts +0 -322
  167. package/src/logger.test.ts +0 -123
  168. package/src/logger.ts +0 -229
  169. package/src/playwright-client/index.ts +0 -392
  170. package/src/playwright-client/types.ts +0 -99
  171. package/src/progress/index.test.ts +0 -327
  172. package/src/progress/index.ts +0 -170
  173. package/src/progress/types.ts +0 -25
  174. package/src/prompts/index.test.ts +0 -451
  175. package/src/prompts/index.ts +0 -246
  176. package/src/prompts/loader.test.ts +0 -100
  177. package/src/prompts/loader.ts +0 -59
  178. package/src/prompts/templates/mcp/webtest-crawl.md +0 -7
  179. package/src/prompts/templates/mcp/webtest-full-workflow.md +0 -12
  180. package/src/resources/index.ts +0 -250
  181. package/src/resources/subscriptions.ts +0 -37
  182. package/src/sampling/index.test.ts +0 -414
  183. package/src/sampling/index.ts +0 -286
  184. package/src/sampling/prompts.ts +0 -194
  185. package/src/sampling/types.ts +0 -60
  186. package/src/schemas/config.ts +0 -39
  187. package/src/security/index.test.ts +0 -441
  188. package/src/security/index.ts +0 -361
  189. package/src/security/security-scenarios.test.ts +0 -468
  190. package/src/server.ts +0 -211
  191. package/src/test-utils/index.ts +0 -6
  192. package/src/test-utils/mock-context.ts +0 -426
  193. package/src/test-utils/mock-playwright-client.ts +0 -422
  194. package/src/tools/index.ts +0 -11
  195. package/src/tools/webtest/crawl.test.ts +0 -834
  196. package/src/tools/webtest/crawl.ts +0 -901
  197. package/src/tools/webtest/discover-features.ts +0 -412
  198. package/src/tools/webtest/discover-flows.ts +0 -408
  199. package/src/tools/webtest/generate-tests.test.ts +0 -532
  200. package/src/tools/webtest/generate-tests.ts +0 -425
  201. package/src/tools/webtest/index.ts +0 -7
  202. package/src/tools/webtest/integration.test.ts +0 -536
  203. package/src/tools/webtest/run-test-case.test.ts +0 -659
  204. package/src/tools/webtest/run-test-case.ts +0 -508
  205. package/src/tools/webtest/schemas.ts +0 -201
  206. package/src/tools/webtest/start-analysis.test.ts +0 -151
  207. package/src/tools/webtest/start-analysis.ts +0 -158
  208. package/src/transports/http.ts +0 -19
  209. package/src/transports/index.ts +0 -30
  210. package/src/transports/stdio.ts +0 -7
  211. package/src/types/capabilities.test.ts +0 -193
  212. package/src/types/capabilities.ts +0 -50
  213. package/src/types/context.ts +0 -21
  214. package/src/types/tool.ts +0 -11
  215. package/src/workspace/index.ts +0 -945
  216. package/src/workspace/markdown.ts +0 -272
  217. package/src/workspace/types.ts +0 -186
  218. package/tests/integration/server.test.ts +0 -89
  219. package/tests/integration/tools.test.ts +0 -99
  220. package/tsconfig.json +0 -20
  221. package/vitest.config.ts +0 -9
  222. package/vitest.integration.config.ts +0 -10
  223. /package/{src → dist}/prompts/templates/sampling/crawl-action.md +0 -0
  224. /package/{src → dist}/prompts/templates/sampling/feature-discovery.md +0 -0
  225. /package/{src → dist}/prompts/templates/sampling/flow-discovery.md +0 -0
  226. /package/{src → dist}/prompts/templates/sampling/page-content-wrapper.md +0 -0
  227. /package/{src → dist}/prompts/templates/sampling/test-evaluation.md +0 -0
  228. /package/{src → dist}/prompts/templates/sampling/test-generation.md +0 -0
  229. /package/dist/tools/{webtest → retest}/crawl.d.ts +0 -0
  230. /package/dist/tools/{webtest → retest}/discover-features.d.ts +0 -0
  231. /package/dist/tools/{webtest → retest}/discover-flows.d.ts +0 -0
  232. /package/dist/tools/{webtest → retest}/generate-tests.d.ts +0 -0
  233. /package/dist/tools/{webtest → retest}/index.d.ts +0 -0
  234. /package/dist/tools/{webtest → retest}/index.js +0 -0
  235. /package/dist/tools/{webtest → retest}/run-test-case.d.ts +0 -0
  236. /package/dist/tools/{webtest → retest}/schemas.d.ts +0 -0
  237. /package/dist/tools/{webtest → retest}/schemas.js +0 -0
  238. /package/dist/tools/{webtest → retest}/start-analysis.d.ts +0 -0
@@ -1,95 +0,0 @@
1
- ## Context
2
-
3
- The testing-mcp server provides tools for AI-powered web application testing. The current naming has evolved organically and lacks consistency:
4
-
5
- 1. `webtest_analyze_app` does two things: identifies app structure AND identifies flows
6
- 2. `webtest_run_tests` (plural) only runs one test at a time
7
- 3. Terminology in prompts/outputs mixes "analysis", "flows", "entities" inconsistently
8
- 4. File structure uses `analysis/` directory which doesn't reflect new "features" terminology
9
-
10
- This refactor establishes a consistent vocabulary and separates concerns for better discoverability.
11
-
12
- ## Goals / Non-Goals
13
-
14
- ### Goals
15
- - Establish consistent naming convention across all user-facing surfaces
16
- - Separate feature discovery from flow discovery for flexibility
17
- - Make tool names match their actual behavior (singular vs plural)
18
- - Create intuitive file structure that reflects the discovery hierarchy
19
-
20
- ### Non-Goals
21
- - Backward compatibility (pre-1.0, breaking changes acceptable)
22
- - Changing core functionality/algorithms
23
- - Adding new test capabilities beyond naming
24
-
25
- ## Decisions
26
-
27
- ### Decision 1: Two-phase discovery (features then flows)
28
-
29
- **What**: Split `analyze_app` into `discover_features` and `discover_flows`
30
-
31
- **Why**:
32
- - Features are coarse-grained (Auth, Cart, Search)
33
- - Flows are fine-grained within features (Login, Register, Password Reset within Auth)
34
- - Users may want to discover flows for just one feature
35
- - Better sampling efficiency (smaller prompts per call)
36
-
37
- **Alternatives considered**:
38
- - Keep single tool with `mode` parameter - rejected (less intuitive)
39
- - Keep current name but add `discover_flows` - rejected (inconsistent naming)
40
-
41
- ### Decision 2: Feature-based directory structure
42
-
43
- **What**: Move from `analysis/{app-analysis,flows}.md` to `features.md` + `features/<slug>/flows.md`
44
-
45
- **Why**:
46
- - Reflects the hierarchical relationship (features contain flows)
47
- - Enables per-feature flow files (better for large apps)
48
- - Natural organization for humans browsing workspace
49
-
50
- **Structure**:
51
- ```
52
- webtest-workspaces/{analysisId}/
53
- index.md # Workspace metadata
54
- features.md # Discovered features
55
- features/
56
- authentication/
57
- flows.md # Flows for authentication feature
58
- shopping-cart/
59
- flows.md # Flows for cart feature
60
- tests/
61
- tests.md # Generated test cases
62
- runs/
63
- {runId}/
64
- report.md # Test run results
65
- ```
66
-
67
- ### Decision 3: Singular `run_test` vs plural `run_tests`
68
-
69
- **What**: Rename `webtest_run_tests` to `webtest_run_test`
70
-
71
- **Why**: The tool executes ONE test case per invocation. Plural name suggests batch execution which doesn't exist.
72
-
73
- ### Decision 4: Orchestrator prompt `webtest-discover`
74
-
75
- **What**: Rename `webtest-analyze` prompt to `webtest-discover`, which orchestrates both feature and flow discovery.
76
-
77
- **Why**:
78
- - "Discover" is the unified verb for this phase
79
- - Prompt can call `discover_features`, then `discover_flows` for each feature
80
- - User still has simple one-command workflow for full discovery
81
-
82
- ## Risks / Trade-offs
83
-
84
- ### Risk: Breaking existing integrations
85
- **Mitigation**: Clear changelog, version bump, documentation update
86
-
87
- ### Risk: Increased complexity with two discovery tools
88
- **Mitigation**: `webtest-discover` prompt abstracts this; power users benefit from granularity
89
-
90
- ### Trade-off: More files in workspace
91
- **Accepted**: Better organization outweighs slightly more files
92
-
93
- ## Open Questions
94
-
95
- None - all decisions finalized for this proposal.
@@ -1,66 +0,0 @@
1
- # Change: Refactor WebTest Naming for Consistency and Clarity
2
-
3
- ## Why
4
-
5
- The current naming across the MCP server is inconsistent and confusing. `webtest_analyze_app` produces both app analysis and flows in one step, but the mental model should separate **feature discovery** from **flow discovery**. Additionally, `webtest_run_tests` (plural) only runs a single test, which is misleading. A consistent, intuitive naming convention will improve discoverability and user experience.
6
-
7
- ## What Changes
8
-
9
- ### Tools (MCP `tools/list`)
10
-
11
- | Current | New | Rationale |
12
- |---------|-----|-----------|
13
- | `webtest_init` | `webtest_init` | No change - already clear |
14
- | `webtest_crawl_app` | `webtest_crawl_app` | No change - already clear |
15
- | `webtest_analyze_app` | `webtest_discover_features` | **BREAKING** - Renamed to reflect actual output: features discovery. Now outputs `features.md` |
16
- | *(new)* | `webtest_discover_flows` | **NEW** - Discovers flows for a specific feature/module, outputs `features/<feature-slug>/flows.md` |
17
- | `webtest_generate_tests` | `webtest_generate_tests` | No change - already clear |
18
- | `webtest_run_tests` | `webtest_run_test` | **BREAKING** - Singular form matches behavior (runs ONE test) |
19
-
20
- ### Prompts (MCP `prompts/list`)
21
-
22
- | Current | New | Rationale |
23
- |---------|-----|-----------|
24
- | `webtest-start` | `webtest-start` | No change |
25
- | `webtest-crawl` | `webtest-crawl` | No change |
26
- | `webtest-analyze` | `webtest-discover` | **BREAKING** - Orchestrates both feature and flow discovery |
27
- | `webtest-generate-tests` | `webtest-generate-tests` | No change |
28
- | `webtest-run-test` | `webtest-run-test` | No change (already singular) |
29
- | `webtest-full-workflow` | `webtest-full-workflow` | No change |
30
-
31
- ### Resources (file/URI structure)
32
-
33
- | Current | New | Rationale |
34
- |---------|-----|-----------|
35
- | `analysis/app-analysis.md` | `features.md` | Renamed to match tool output |
36
- | `analysis/flows.md` | `features/<slug>/flows.md` | Moved under feature structure |
37
- | All other paths | *(unchanged)* | - |
38
-
39
- ### Vocabulary Standardization
40
-
41
- Throughout prompts, notifications, logs, and output messages:
42
-
43
- | Term | Meaning |
44
- |------|---------|
45
- | **feature** | A distinct capability/module of the app (e.g., "Authentication", "Shopping Cart") |
46
- | **flow** | A user journey within a feature (e.g., "Login with email", "Add item to cart") |
47
- | **crawl** | Automated exploration of the app |
48
- | **discover** | AI-powered analysis to identify features or flows |
49
- | **test** | A single executable test case |
50
- | **run** | Execution of a single test |
51
-
52
- ## Impact
53
-
54
- - Affected specs: `webtest-tools`, `webtest-prompts`, `webtest-resources`, `webtest-sampling`
55
- - Affected code:
56
- - `src/tools/webtest/analyze-app.ts` -> `discover-features.ts`
57
- - `src/tools/webtest/run-test-case.ts` (tool name change)
58
- - `src/prompts/index.ts`
59
- - `src/resources/index.ts`
60
- - `src/workspace/index.ts` (new feature/flow directory structure)
61
- - `src/sampling/prompts.ts` (vocabulary updates)
62
- - All test files referencing renamed tools
63
-
64
- ## Migration
65
-
66
- This is a breaking change for existing clients. No backward compatibility is provided since this is a pre-1.0 release.
@@ -1,79 +0,0 @@
1
- ## MODIFIED Requirements
2
-
3
- ### Requirement: Prompt Template Registration
4
-
5
- The system SHALL register prompt templates with the MCP server for client discovery.
6
-
7
- #### Scenario: Prompts are listed on prompts/list
8
-
9
- - **GIVEN** a client calls `prompts/list`
10
- - **WHEN** the response is returned
11
- - **THEN** it SHALL include prompts:
12
- - `webtest-start` (Start web testing analysis)
13
- - `webtest-crawl` (Crawl to satisfy focus)
14
- - `webtest-discover` (Discover features and flows)
15
- - `webtest-generate-tests` (Generate test cases)
16
- - `webtest-run-test` (Run a test case)
17
-
18
- ## RENAMED Requirements
19
-
20
- - FROM: `### Requirement: Analyze Prompt`
21
- - TO: `### Requirement: Discover Prompt`
22
-
23
- ## MODIFIED Requirements
24
-
25
- ### Requirement: Discover Prompt
26
-
27
- The system SHALL provide a prompt template to discover features and flows from crawled data.
28
-
29
- #### Scenario: Discover prompt is invoked
30
-
31
- - **GIVEN** client invokes `prompts/get` with name `webtest-discover`
32
- - **WHEN** the prompt is returned
33
- - **THEN** it SHALL include:
34
- - Description: "Discover features and flows from crawled web application data"
35
- - Arguments: `analysisId` (required), `crawlId` (optional), `featureSlug` (optional)
36
-
37
- #### Scenario: Discover prompt orchestrates full discovery
38
-
39
- - **GIVEN** client invokes discover prompt with only `analysisId`
40
- - **WHEN** the prompt messages are returned
41
- - **THEN** it SHALL instruct to:
42
- - First call `webtest_discover_features` to identify features
43
- - Then call `webtest_discover_flows` for each discovered feature
44
-
45
- #### Scenario: Discover prompt supports single feature mode
46
-
47
- - **GIVEN** client invokes discover prompt with `analysisId` and `featureSlug`
48
- - **WHEN** the prompt messages are returned
49
- - **THEN** it SHALL instruct to call `webtest_discover_flows` only for the specified feature
50
-
51
- ## MODIFIED Requirements
52
-
53
- ### Requirement: Prompt Chaining Guidance
54
-
55
- The system SHALL include guidance in prompt outputs for the recommended workflow sequence.
56
-
57
- #### Scenario: Start analysis prompt suggests next step
58
-
59
- - **GIVEN** client uses start analysis prompt
60
- - **WHEN** the prompt messages are returned
61
- - **THEN** they SHALL include guidance: "After initialization, use webtest-crawl to explore the application"
62
-
63
- #### Scenario: Crawl prompt suggests next step
64
-
65
- - **GIVEN** client uses crawl prompt
66
- - **WHEN** the prompt messages are returned
67
- - **THEN** they SHALL include guidance: "After crawl completes, use webtest-discover to identify features and flows"
68
-
69
- #### Scenario: Discover prompt suggests next step
70
-
71
- - **GIVEN** client uses discover prompt
72
- - **WHEN** the prompt messages are returned
73
- - **THEN** they SHALL include guidance: "After discovery, use webtest-generate-tests to create test cases"
74
-
75
- #### Scenario: Generate tests prompt suggests next step
76
-
77
- - **GIVEN** client uses generate tests prompt
78
- - **WHEN** the prompt messages are returned
79
- - **THEN** they SHALL include guidance: "After test generation, use webtest-run-test to execute individual test cases"
@@ -1,80 +0,0 @@
1
- ## MODIFIED Requirements
2
-
3
- ### Requirement: Resource URI Scheme
4
-
5
- The system SHALL expose all webtest artifacts using a `webtest://` URI scheme with hierarchical paths, using markdown format for all human-readable artifacts.
6
-
7
- #### Scenario: Analysis root resource is accessible
8
-
9
- - **GIVEN** an analysis has been started with analysisId "abc123"
10
- - **WHEN** client requests resource `webtest://abc123/`
11
- - **THEN** it SHALL return the analysis `index.md` metadata as markdown with YAML frontmatter
12
-
13
- #### Scenario: Features resource is accessible
14
-
15
- - **GIVEN** feature discovery has completed
16
- - **WHEN** client requests `webtest://abc123/features.md`
17
- - **THEN** it SHALL return the features list as markdown with YAML frontmatter containing structured feature definitions
18
-
19
- #### Scenario: Feature-specific flows are accessible
20
-
21
- - **GIVEN** flow discovery has completed for feature "authentication"
22
- - **WHEN** client requests `webtest://abc123/features/authentication/flows.md`
23
- - **THEN** it SHALL return user flows for that feature as markdown with YAML frontmatter
24
-
25
- #### Scenario: Crawl index resource is accessible
26
-
27
- - **GIVEN** a crawl has completed with crawlId "crawl-001"
28
- - **WHEN** client requests resource `webtest://abc123/crawls/crawl-001/index.md`
29
- - **THEN** it SHALL return the crawl index as markdown with YAML frontmatter containing page list and metadata
30
-
31
- #### Scenario: Page artifacts are accessible by type
32
-
33
- - **GIVEN** a page was captured with pageId "page-001"
34
- - **WHEN** client requests `webtest://abc123/crawls/crawl-001/pages/page-001/screenshot.png`
35
- - **THEN** it SHALL return the screenshot image
36
- - **AND** `snapshot.md` returns accessibility tree as formatted markdown with YAML frontmatter
37
- - **AND** `dom.html` returns HTML content
38
-
39
- #### Scenario: Tests are accessible
40
-
41
- - **GIVEN** generate_tests has completed
42
- - **WHEN** client requests `webtest://abc123/tests/tests.md`
43
- - **THEN** it SHALL return the test cases as markdown with YAML frontmatter containing structured test definitions
44
-
45
- #### Scenario: Test run report is accessible
46
-
47
- - **GIVEN** a test run has completed with runId "run-001"
48
- - **WHEN** client requests `webtest://abc123/runs/run-001/report.md`
49
- - **THEN** it SHALL return the test execution report as markdown with YAML frontmatter containing structured results
50
-
51
- ## MODIFIED Requirements
52
-
53
- ### Requirement: Resource Template Registration
54
-
55
- The system SHALL register resource templates with the MCP server for discovery, using markdown extensions for all index and report resources.
56
-
57
- #### Scenario: Templates are listed on resources/list
58
-
59
- - **GIVEN** a client calls `resources/list`
60
- - **WHEN** the response is returned
61
- - **THEN** it SHALL include templates for:
62
- - `webtest://{analysisId}/index.md` (Analysis index)
63
- - `webtest://{analysisId}/features.md` (Discovered features)
64
- - `webtest://{analysisId}/features/{featureSlug}/flows.md` (Feature flows)
65
- - `webtest://{analysisId}/crawls/{crawlId}/index.md` (Crawl index)
66
- - `webtest://{analysisId}/crawls/{crawlId}/checkpoint.md` (Crawl checkpoint)
67
- - `webtest://{analysisId}/crawls/{crawlId}/pages/{pageId}/snapshot.md` (Page snapshot)
68
- - `webtest://{analysisId}/crawls/{crawlId}/pages/{pageId}/screenshot.png` (Page screenshot)
69
- - `webtest://{analysisId}/crawls/{crawlId}/pages/{pageId}/dom.html` (Page DOM)
70
- - `webtest://{analysisId}/tests/tests.md` (Test definitions)
71
- - `webtest://{analysisId}/runs/{runId}/report.md` (Test run report)
72
- - `webtest://{analysisId}/runs/{runId}/steps/{stepNumber}/snapshot.md` (Step snapshot)
73
- - `webtest://{analysisId}/runs/{runId}/steps/{stepNumber}/screenshot.png` (Step screenshot)
74
-
75
- ## REMOVED Requirements
76
-
77
- ### Requirement: Legacy Analysis Resource Paths
78
-
79
- **Reason**: Replaced by feature-based structure
80
- **Migration**: Use `features.md` instead of `analysis/app-analysis.md`, use `features/<slug>/flows.md` instead of `analysis/flows.md`
@@ -1,122 +0,0 @@
1
- ## RENAMED Requirements
2
-
3
- - FROM: `### Requirement: Analysis Sampling`
4
- - TO: `### Requirement: Feature Discovery Sampling`
5
-
6
- ## MODIFIED Requirements
7
-
8
- ### Requirement: Feature Discovery Sampling
9
-
10
- The system SHALL use sampling to discover application features from crawled pages.
11
-
12
- #### Scenario: Feature discovery sampling prompt is constructed
13
-
14
- - **GIVEN** `webtest_discover_features` tool is invoked
15
- - **WHEN** the sampling prompt is built
16
- - **THEN** it SHALL include the crawl summary
17
- - **AND** page snapshots from key pages
18
- - **AND** instructions to identify app purpose, type, and distinct features/modules
19
-
20
- #### Scenario: Feature discovery sampling returns structured features
21
-
22
- - **GIVEN** a feature discovery sampling request completes
23
- - **WHEN** the response is parsed
24
- - **THEN** it SHALL conform to the features schema:
25
- ```json
26
- {
27
- "appPurpose": "string",
28
- "appType": "string",
29
- "features": [{
30
- "slug": "string",
31
- "name": "string",
32
- "description": "string",
33
- "entities": ["string"],
34
- "entryPoints": ["string"]
35
- }],
36
- "securityObservations": ["string"],
37
- "accessibilityObservations": ["string"]
38
- }
39
- ```
40
-
41
- ## ADDED Requirements
42
-
43
- ### Requirement: Flow Discovery Sampling
44
-
45
- The system SHALL use sampling to discover user flows within a specific feature.
46
-
47
- #### Scenario: Flow discovery sampling prompt is constructed
48
-
49
- - **GIVEN** `webtest_discover_flows` tool is invoked with a feature slug
50
- - **WHEN** the sampling prompt is built
51
- - **THEN** it SHALL include the feature definition from `features.md`
52
- - **AND** page snapshots relevant to that feature
53
- - **AND** instructions to identify user flows within the feature
54
-
55
- #### Scenario: Flow discovery sampling returns structured flows
56
-
57
- - **GIVEN** a flow discovery sampling request completes
58
- - **WHEN** the response is parsed
59
- - **THEN** it SHALL conform to the flows schema:
60
- ```json
61
- {
62
- "featureSlug": "string",
63
- "flows": [{
64
- "id": "string",
65
- "name": "string",
66
- "description": "string",
67
- "entryPoint": "string",
68
- "steps": ["string"]
69
- }],
70
- "suggestedAssertions": ["string"]
71
- }
72
- ```
73
-
74
- #### Scenario: Flow discovery uses feature context
75
-
76
- - **GIVEN** flow discovery is scoped to feature "authentication"
77
- - **WHEN** the sampling prompt is built
78
- - **THEN** it SHALL explicitly scope the AI to only identify flows within that feature
79
- - **AND** reference the feature's entities and entry points
80
-
81
- ## MODIFIED Requirements
82
-
83
- ### Requirement: Test Generation Sampling
84
-
85
- The system SHALL use sampling to generate test cases from discovered features and flows.
86
-
87
- #### Scenario: Test generation sampling prompt is constructed
88
-
89
- - **GIVEN** `webtest_generate_tests` tool is invoked
90
- - **WHEN** the sampling prompt is built
91
- - **THEN** it SHALL include the features definition from `features.md`
92
- - **AND** flow definitions from `features/<slug>/flows.md` files
93
- - **AND** test strategy preferences (count, types)
94
-
95
- #### Scenario: Test generation sampling returns test cases
96
-
97
- - **GIVEN** a test generation sampling request completes
98
- - **WHEN** the response is parsed
99
- - **THEN** it SHALL conform to the test case schema:
100
- ```json
101
- {
102
- "tests": [{
103
- "id": "string",
104
- "name": "string",
105
- "purpose": "string",
106
- "featureSlug": "string",
107
- "flowId": "string",
108
- "preconditions": ["string"],
109
- "steps": [{
110
- "action": "string",
111
- "expected": "string"
112
- }],
113
- "priority": "string"
114
- }]
115
- }
116
- ```
117
-
118
- #### Scenario: Test generation references features and flows
119
-
120
- - **GIVEN** test cases are generated
121
- - **WHEN** the output is structured
122
- - **THEN** each test case SHALL include `featureSlug` and `flowId` to trace its origin
@@ -1,113 +0,0 @@
1
- ## RENAMED Requirements
2
-
3
- - FROM: `### Requirement: webtest_analyze_app Tool`
4
- - TO: `### Requirement: webtest_discover_features Tool`
5
-
6
- ## MODIFIED Requirements
7
-
8
- ### Requirement: webtest_discover_features Tool
9
-
10
- The system SHALL provide a `webtest_discover_features` tool that identifies application features/modules from crawl data, outputting results in markdown format.
11
-
12
- #### Scenario: Discover features writes features.md
13
-
14
- - **GIVEN** feature discovery is complete
15
- - **WHEN** output is generated
16
- - **THEN** it SHALL write `features.md` resource to workspace root
17
- - **AND** the file SHALL contain markdown with YAML frontmatter listing discovered features
18
-
19
- #### Scenario: Discover features identifies app structure
20
-
21
- - **GIVEN** crawl data is available
22
- - **WHEN** `webtest_discover_features` is called
23
- - **THEN** it SHALL identify:
24
- - Application purpose and type
25
- - Distinct features/modules (e.g., Authentication, Shopping Cart, Search)
26
- - Entities and data models per feature
27
- - Security and accessibility observations
28
-
29
- #### Scenario: Discover features outputs URIs
30
-
31
- - **GIVEN** feature discovery is complete
32
- - **WHEN** tool returns
33
- - **THEN** it SHALL include `featuresFilePath`, `featuresUri` pointing to `features.md`
34
-
35
- #### Scenario: Discover features suggests next steps
36
-
37
- - **GIVEN** feature discovery is complete
38
- - **WHEN** tool returns
39
- - **THEN** it SHALL include `nextSteps` array suggesting to run `webtest_discover_flows` for each feature
40
-
41
- ## ADDED Requirements
42
-
43
- ### Requirement: webtest_discover_flows Tool
44
-
45
- The system SHALL provide a `webtest_discover_flows` tool that identifies user flows within a specific feature, outputting results in markdown format.
46
-
47
- #### Scenario: Discover flows requires feature parameter
48
-
49
- - **GIVEN** the tool is called
50
- - **WHEN** parameters are validated
51
- - **THEN** it SHALL require `analysisId` and `featureSlug` parameters
52
- - **AND** `featureSlug` SHALL match a feature from `features.md`
53
-
54
- #### Scenario: Discover flows writes feature-specific flows.md
55
-
56
- - **GIVEN** flow discovery is complete for feature "authentication"
57
- - **WHEN** output is generated
58
- - **THEN** it SHALL write to `features/authentication/flows.md`
59
- - **AND** the file SHALL contain markdown with YAML frontmatter listing flows for that feature
60
-
61
- #### Scenario: Discover flows identifies user journeys
62
-
63
- - **GIVEN** crawl data and feature context are available
64
- - **WHEN** `webtest_discover_flows` is called
65
- - **THEN** it SHALL identify:
66
- - User flows within the feature (e.g., Login, Register, Password Reset)
67
- - Entry points for each flow
68
- - Step sequences for each flow
69
- - Assertions that should hold
70
-
71
- #### Scenario: Discover flows outputs URIs
72
-
73
- - **GIVEN** flow discovery is complete
74
- - **WHEN** tool returns
75
- - **THEN** it SHALL include `flowsFilePath`, `flowsUri` pointing to `features/<slug>/flows.md`
76
-
77
- #### Scenario: Discover flows handles unknown feature
78
-
79
- - **GIVEN** the tool is called with a `featureSlug` not in `features.md`
80
- - **WHEN** validation occurs
81
- - **THEN** it SHALL return an error listing available features
82
-
83
- ## RENAMED Requirements
84
-
85
- - FROM: `### Requirement: webtest_run_tests Tool`
86
- - TO: `### Requirement: webtest_run_test Tool`
87
-
88
- ## MODIFIED Requirements
89
-
90
- ### Requirement: webtest_run_test Tool
91
-
92
- The system SHALL provide a `webtest_run_test` tool that executes a single test case with evidence capture, storing all results in markdown format.
93
-
94
- #### Scenario: Run test captures evidence
95
-
96
- - **GIVEN** a step is executed
97
- - **WHEN** evidence is captured
98
- - **THEN** it SHALL take screenshot after action (stored as PNG)
99
- - **AND** capture accessibility snapshot (stored as `snapshot.md` with formatted tree and YAML frontmatter)
100
- - **AND** store with step identifier
101
-
102
- #### Scenario: Run test outputs report
103
-
104
- - **GIVEN** test execution completes
105
- - **WHEN** output is generated
106
- - **THEN** it SHALL write `report.md` with pass/fail summary, step details, evidence links, and YAML frontmatter containing structured run data
107
- - **AND** there SHALL NOT be a separate `index.json` or `artifacts.json` file
108
-
109
- #### Scenario: Run test returns URIs
110
-
111
- - **GIVEN** execution is complete
112
- - **WHEN** tool returns
113
- - **THEN** it SHALL include `testRunId`, `reportFilePath`, `reportUri` pointing to `report.md`
@@ -1,119 +0,0 @@
1
- ## 1. Workspace Structure Changes
2
-
3
- - [ ] 1.1 Update `WorkspaceManager` types for new feature structure
4
- - Add `features: Feature[]` to workspace index
5
- - Add `saveFeatures()` method
6
- - Add `saveFlows(featureSlug)` method
7
- - Remove `saveAnalysis()` method (replaced by above)
8
-
9
- - [ ] 1.2 Create feature directory structure in workspace
10
- - `features.md` at workspace root
11
- - `features/<slug>/flows.md` per feature
12
- - Update `getWorkspacePath()` to support feature paths
13
-
14
- - [ ] 1.3 Update workspace markdown generation
15
- - Implement `generateFeaturesMarkdown()`
16
- - Implement `generateFlowsMarkdown(featureSlug)`
17
- - Update YAML frontmatter schemas
18
-
19
- ## 2. Tool Renaming and Implementation
20
-
21
- - [ ] 2.1 Rename `analyze-app.ts` to `discover-features.ts`
22
- - Rename tool from `webtest_analyze_app` to `webtest_discover_features`
23
- - Update description to reflect feature discovery focus
24
- - Update output to reference `features.md` instead of `app-analysis.md`
25
- - Remove flow generation (moved to separate tool)
26
- - Add `nextSteps` suggesting `webtest_discover_flows` calls
27
-
28
- - [ ] 2.2 Create new `discover-flows.ts` tool
29
- - Implement `webtest_discover_flows` tool
30
- - Accept `analysisId` and `featureSlug` parameters
31
- - Validate feature exists in `features.md`
32
- - Output to `features/<slug>/flows.md`
33
- - Build sampling prompt with feature context
34
-
35
- - [ ] 2.3 Rename tool in `run-test-case.ts`
36
- - Change tool name from `webtest_run_tests` to `webtest_run_test`
37
- - Update description (singular)
38
- - No functional changes
39
-
40
- - [ ] 2.4 Update `src/tools/webtest/index.ts` exports
41
- - Export `discover-features.ts`
42
- - Export `discover-flows.ts`
43
- - Remove `analyze-app.ts` export
44
-
45
- ## 3. Sampling Prompt Updates
46
-
47
- - [ ] 3.1 Rename and update `buildAnalysisPrompt` in `sampling/prompts.ts`
48
- - Rename to `buildFeatureDiscoveryPrompt`
49
- - Update prompt text to focus on features/modules
50
- - Update expected output schema
51
-
52
- - [ ] 3.2 Create `buildFlowDiscoveryPrompt` in `sampling/prompts.ts`
53
- - Accept feature context as parameter
54
- - Scope prompt to single feature
55
- - Define flows schema for output
56
-
57
- - [ ] 3.3 Update `buildTestGenerationPrompt`
58
- - Reference `features.md` and `flows.md` in prompt
59
- - Add `featureSlug` and `flowId` to expected test output
60
-
61
- ## 4. Schema Updates
62
-
63
- - [ ] 4.1 Update `schemas.ts` for new data structures
64
- - Add `FeatureSchema` (slug, name, description, entities, entryPoints)
65
- - Add `FeaturesOutputSchema` (array of features)
66
- - Add `FlowDiscoveryOutputSchema` (scoped to feature)
67
- - Deprecate `AppAnalysisSchema`
68
-
69
- ## 5. Prompt Template Updates
70
-
71
- - [ ] 5.1 Rename `webtest-analyze` prompt to `webtest-discover` in `prompts/index.ts`
72
- - Update prompt name and description
73
- - Update message to orchestrate feature+flow discovery
74
- - Support optional `featureSlug` for single-feature mode
75
-
76
- - [ ] 5.2 Update prompt chaining guidance
77
- - Crawl -> Discover (not Analyze)
78
- - Discover -> Generate Tests
79
- - Update vocabulary in all prompt messages
80
-
81
- - [ ] 5.3 Update tool references in all prompts
82
- - `webtest_analyze_app` -> `webtest_discover_features`
83
- - `webtest_run_tests` -> `webtest_run_test`
84
-
85
- ## 6. Resource Manager Updates
86
-
87
- - [ ] 6.1 Update resource listing in `resources/index.ts`
88
- - Add `features.md` resource
89
- - Add `features/<slug>/flows.md` resources per feature
90
- - Remove `analysis/app-analysis.md` and `analysis/flows.md` references
91
-
92
- - [ ] 6.2 Update resource descriptions
93
- - Use consistent "feature" and "flow" vocabulary
94
-
95
- ## 7. Test Updates
96
-
97
- - [ ] 7.1 Rename test file `analyze-app.test.ts` to `discover-features.test.ts`
98
- - Update all test descriptions
99
- - Update tool name references
100
-
101
- - [ ] 7.2 Create `discover-flows.test.ts`
102
- - Test feature validation
103
- - Test flow output structure
104
- - Test missing feature error handling
105
-
106
- - [ ] 7.3 Update `run-test-case.test.ts`
107
- - Update tool name in tests
108
- - No functional test changes needed
109
-
110
- - [ ] 7.4 Update integration tests
111
- - Update tool names in `integration.test.ts`
112
- - Test full workflow with new tool names
113
-
114
- ## 8. Documentation and Validation
115
-
116
- - [ ] 8.1 Run all unit tests and fix failures
117
- - [ ] 8.2 Run integration tests and fix failures
118
- - [ ] 8.3 Verify build succeeds
119
- - [ ] 8.4 Update specs after implementation (archive step)