vscode-autotest 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +276 -0
- package/dist/cli/index.d.ts +9 -0
- package/dist/cli/index.js +69 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/drivers/vscodeDriver.d.ts +179 -0
- package/dist/drivers/vscodeDriver.js +886 -0
- package/dist/drivers/vscodeDriver.js.map +1 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -0
- package/dist/operators/actionResolver.d.ts +23 -0
- package/dist/operators/actionResolver.js +186 -0
- package/dist/operators/actionResolver.js.map +1 -0
- package/dist/operators/llmClient.d.ts +33 -0
- package/dist/operators/llmClient.js +117 -0
- package/dist/operators/llmClient.js.map +1 -0
- package/dist/operators/planParser.d.ts +10 -0
- package/dist/operators/planParser.js +92 -0
- package/dist/operators/planParser.js.map +1 -0
- package/dist/operators/stepVerifier.d.ts +34 -0
- package/dist/operators/stepVerifier.js +182 -0
- package/dist/operators/stepVerifier.js.map +1 -0
- package/dist/operators/testRunner.d.ts +29 -0
- package/dist/operators/testRunner.js +181 -0
- package/dist/operators/testRunner.js.map +1 -0
- package/dist/types.d.ts +137 -0
- package/dist/types.js +5 -0
- package/dist/types.js.map +1 -0
- package/package.json +60 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 wenytang-ms
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
# VSCode AutoTest
|
|
2
|
+
|
|
3
|
+
AI 驱动的 VSCode 扩展 E2E 测试框架。
|
|
4
|
+
|
|
5
|
+
用户提供 YAML 格式的 Test Plan → 框架自动启动 VSCode → 执行操作 → 验证结果。
|
|
6
|
+
|
|
7
|
+
> **目标**:让 Copilot CLI 直接读取 wiki 中的 Test Plan,自动完成 VSCode 扩展的端到端测试。
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## 快速开始
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
# 安装依赖
|
|
15
|
+
npm install
|
|
16
|
+
|
|
17
|
+
# 编译
|
|
18
|
+
npm run build
|
|
19
|
+
|
|
20
|
+
# 验证 test plan 格式
|
|
21
|
+
npx autotest validate test-plans/java-maven.yaml
|
|
22
|
+
|
|
23
|
+
# 执行测试(含截图 + JSON 报告)
|
|
24
|
+
npx autotest run test-plans/java-maven.yaml --output results.json
|
|
25
|
+
|
|
26
|
+
# 自定义截图输出目录
|
|
27
|
+
npx autotest run test-plans/java-maven.yaml --output results.json --screenshots ./my-shots
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
### 前置条件
|
|
31
|
+
|
|
32
|
+
- Node.js ≥ 18
|
|
33
|
+
- JDK 已安装(测试 Java 扩展时需要)
|
|
34
|
+
- vscode-java 仓库已 clone 到本地(test plan 中引用了其测试项目)
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## 核心架构
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
┌─────────────────────────────────────────────────────┐
|
|
42
|
+
│ Test Plan (YAML / Markdown) │
|
|
43
|
+
│ 描述测试步骤 + 预期结果,不包含硬编码 locator │
|
|
44
|
+
└──────────────────┬──────────────────────────────────┘
|
|
45
|
+
│ planParser.ts
|
|
46
|
+
▼
|
|
47
|
+
┌─────────────────────────────────────────────────────┐
|
|
48
|
+
│ TestRunner (编排层) │
|
|
49
|
+
│ 启动 VSCode → 逐步执行 → 截图 → 报告 │
|
|
50
|
+
│ │
|
|
51
|
+
│ ┌──────────────────┐ ┌─────────────────────────┐ │
|
|
52
|
+
│ │ ActionResolver │ │ StepVerifier │ │
|
|
53
|
+
│ │ 自然语言 → Driver │ │ 确定性验证 + LLM 验证 │ │
|
|
54
|
+
│ │ (16 种 regex) │ │ (6 种策略) │ │
|
|
55
|
+
│ └────────┬─────────┘ └──────────┬──────────────┘ │
|
|
56
|
+
│ │ │ │
|
|
57
|
+
│ │ ┌────────┴────────┐ │
|
|
58
|
+
│ │ │ LLMClient │ │
|
|
59
|
+
│ │ │ Azure OpenAI │ │
|
|
60
|
+
│ │ │ screenshot → │ │
|
|
61
|
+
│ │ │ pass/fail │ │
|
|
62
|
+
│ │ └─────────────────┘ │
|
|
63
|
+
└───────────┼──────────────────────────────────────────┘
|
|
64
|
+
▼
|
|
65
|
+
┌─────────────────────────────────────────────────────┐
|
|
66
|
+
│ VscodeDriver (操作原语 SDK) │
|
|
67
|
+
│ 基于 Playwright Electron + @vscode/test-electron │
|
|
68
|
+
│ 工作区隔离 · 事件驱动等待 · 进程生命周期管理 │
|
|
69
|
+
└──────────────────┬──────────────────────────────────┘
|
|
70
|
+
│
|
|
71
|
+
▼
|
|
72
|
+
┌─────────────────────────────────────────────────────┐
|
|
73
|
+
│ Playwright Electron Runtime │
|
|
74
|
+
│ 启动 VSCode 进程,提供 Page 对象 │
|
|
75
|
+
└─────────────────────────────────────────────────────┘
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
---
|
|
79
|
+
|
|
80
|
+
## 编写 Test Plan
|
|
81
|
+
|
|
82
|
+
### 基本结构
|
|
83
|
+
|
|
84
|
+
```yaml
|
|
85
|
+
name: "我的测试"
|
|
86
|
+
setup:
|
|
87
|
+
extension: "my-extension"
|
|
88
|
+
extensionPath: "./path/to/extension"
|
|
89
|
+
vscodeVersion: "stable"
|
|
90
|
+
workspace: "./test-workspace" # 相对于 test plan 文件的路径
|
|
91
|
+
timeout: 60
|
|
92
|
+
settings: # 可选:注入 VSCode settings
|
|
93
|
+
editor.fontSize: 14
|
|
94
|
+
|
|
95
|
+
steps:
|
|
96
|
+
- id: "step-1"
|
|
97
|
+
action: "执行命令 My Extension: Hello"
|
|
98
|
+
verifyNotification: "Hello World!"
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
> **路径解析**:`workspace` 和 `extensionPath` 相对于 test plan 文件所在目录解析,不依赖 CWD。
|
|
102
|
+
|
|
103
|
+
### 支持的 Action(自然语言 → Driver 映射)
|
|
104
|
+
|
|
105
|
+
| Action 写法 | 映射操作 | 示例 |
|
|
106
|
+
|-------------|---------|------|
|
|
107
|
+
| `执行命令 XXX` / `run command XXX` | Command Palette 执行 | `执行命令 Java: Force Java Compilation` |
|
|
108
|
+
| `打开文件 XXX` / `open file XXX` | Quick Open 打开文件(含重试) | `打开文件 Foo.java` |
|
|
109
|
+
| `点击侧边栏 XXX tab` | 切换侧边栏 Tab | `点击侧边栏 Explorer tab` |
|
|
110
|
+
| `展开/点击 XXX 节点` | TreeView 操作 | `展开 APIs 节点` |
|
|
111
|
+
| `选择 XXX 选项` | Palette 选项选择 | `选择 Full 选项` |
|
|
112
|
+
| `等待 N 秒` / `wait N seconds` | 等待指定时间 | `等待 5 秒` |
|
|
113
|
+
| `saveFile` / `保存文件` | Ctrl+S 保存 | `saveFile` |
|
|
114
|
+
| `goToLine N` / `跳转到行 N` | Ctrl+G 跳转行 | `goToLine 15` |
|
|
115
|
+
| `goToEndOfLine` / `跳转到行尾` | End 键 | `goToEndOfLine` |
|
|
116
|
+
| `waitForLanguageServer` | 等待 LS 就绪 | `waitForLanguageServer` |
|
|
117
|
+
| `typeInEditor XXX` | 在编辑器输入文本 | `typeInEditor System.out.println()` |
|
|
118
|
+
| `typeAndTriggerSnippet XXX` | 输入并触发代码片段 | `typeAndTriggerSnippet class` |
|
|
119
|
+
| `navigateToError N` | 跳转到第 N 个错误 | `navigateToError 1` |
|
|
120
|
+
| `applyCodeAction XXX` | 执行 Code Action | `applyCodeAction Import 'ArrayList'` |
|
|
121
|
+
| `triggerCompletion` | 触发代码补全 | `triggerCompletion` |
|
|
122
|
+
| `insertLineInFile <path> <line> <text>` | 磁盘修改文件 + 编辑器重载 | `insertLineInFile src/Foo.java 2 import java.util.*;` |
|
|
123
|
+
|
|
124
|
+
### 支持的验证方式
|
|
125
|
+
|
|
126
|
+
| 字段 | 类型 | 说明 |
|
|
127
|
+
|------|------|------|
|
|
128
|
+
| `verify` | string | 自然语言描述预期结果(LLM 截图验证,需配置 Azure OpenAI) |
|
|
129
|
+
| `verifyFile` | object | 文件存在性 / 内容匹配 |
|
|
130
|
+
| `verifyNotification` | string | 通知消息匹配 |
|
|
131
|
+
| `verifyEditor` | object | 编辑器内容(检查 Monaco model + 可见 DOM) |
|
|
132
|
+
| `verifyProblems` | object | Problems 面板错误/警告计数(支持 `atLeast` 模式 + 轮询等待) |
|
|
133
|
+
| `verifyCompletion` | object | 代码补全列表验证 |
|
|
134
|
+
|
|
135
|
+
---
|
|
136
|
+
|
|
137
|
+
## 测试隔离与截图
|
|
138
|
+
|
|
139
|
+
### 工作区隔离
|
|
140
|
+
|
|
141
|
+
每次运行自动将 `workspace` 复制到固定临时目录 (`autotest-workspace/`),测试结束后清理。原始工作区**永远不会被修改**。
|
|
142
|
+
|
|
143
|
+
### 截图策略
|
|
144
|
+
|
|
145
|
+
每个步骤自动截图,保存到 `./screenshots/`(可通过 `--screenshots` 自定义):
|
|
146
|
+
|
|
147
|
+
| 步骤状态 | 截图文件 |
|
|
148
|
+
|---------|---------|
|
|
149
|
+
| ✅ pass | `{stepId}_after.png` |
|
|
150
|
+
| ❌ fail / error | `{stepId}_before.png` + `{stepId}_after.png` 或 `{stepId}_error.png` |
|
|
151
|
+
|
|
152
|
+
### 进程管理
|
|
153
|
+
|
|
154
|
+
- 每次启动前自动清空 VSCode user-data 目录,防止旧窗口恢复
|
|
155
|
+
- Ctrl+C 中断时自动关闭 VSCode 进程
|
|
156
|
+
- `close()` 带重试机制处理 Windows 文件锁
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
## LLM 验证(可选)
|
|
161
|
+
|
|
162
|
+
`verify` 字段支持自然语言描述,框架会将截图发送给 Azure OpenAI GPT-4o 进行视觉判断:
|
|
163
|
+
|
|
164
|
+
```yaml
|
|
165
|
+
- id: "check-ls"
|
|
166
|
+
action: "waitForLanguageServer"
|
|
167
|
+
verify: "状态栏显示 Java 语言服务器已就绪(👍 图标)" # ← LLM 看截图判断
|
|
168
|
+
verifyProblems: # ← 确定性验证照常执行
|
|
169
|
+
errors: 0
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
配置环境变量启用:
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
export AZURE_OPENAI_ENDPOINT=https://<resource>.openai.azure.com/
|
|
176
|
+
export AZURE_OPENAI_API_KEY=<key>
|
|
177
|
+
export AZURE_OPENAI_DEPLOYMENT=gpt-4o # 可选,默认 gpt-4o
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
- 未配置时 `verify` 字段自动跳过,确定性验证不受影响
|
|
181
|
+
- `--no-llm` 强制跳过 LLM 验证
|
|
182
|
+
|
|
183
|
+
---
|
|
184
|
+
|
|
185
|
+
## Copilot CLI 集成
|
|
186
|
+
|
|
187
|
+
本项目提供 `AGENTS.md`,Copilot CLI 在此目录下可直接运行测试:
|
|
188
|
+
|
|
189
|
+
```
|
|
190
|
+
> 运行 java-maven 测试
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
Copilot CLI 会自动执行 `npx autotest run`,读取结果和截图,分析失败原因。
|
|
194
|
+
|
|
195
|
+
详见 [AGENTS.md](AGENTS.md)。
|
|
196
|
+
|
|
197
|
+
---
|
|
198
|
+
|
|
199
|
+
## 现有 Test Plan
|
|
200
|
+
|
|
201
|
+
| 文件 | 步数 | 来源 | 场景 | 状态 |
|
|
202
|
+
|------|------|------|------|------|
|
|
203
|
+
| `java-maven.yaml` | 8 | wiki Maven | LS → 补全 → 导航 → 编辑 → 诊断 | ✅ 8/8 |
|
|
204
|
+
| `java-maven-multimodule.yaml` | 5 | wiki Maven Multimodule | LS → 两模块补全 | ✅ 5/5 |
|
|
205
|
+
| `java-maven-java25.yaml` | 6 | wiki Maven Java 25 | JDK 25 → LS → 补全 → 编辑 | ✅ 6/6 |
|
|
206
|
+
| `java-gradle.yaml` | 7 | wiki Gradle | LS → 补全 → 导航 → 编辑 | ✅ 7/7 |
|
|
207
|
+
| `java-gradle-java25.yaml` | 6 | wiki Gradle Java 25 | JDK 25 → LS → 补全 → 编辑 | ✅ 6/6 |
|
|
208
|
+
| `java-basic-editing.yaml` | 8 | wiki Basic #1-5 | LS → class 骨架 → call() 方法 → 编译 | ✅ 8/8 |
|
|
209
|
+
| `java-basic-extended.yaml` | 8 | wiki Basic #6-8 | 补全 → Import → Rename | ✅ 8/8 |
|
|
210
|
+
| `java-new-file-snippet.yaml` | 4 | wiki Basic #9 | 创建新 Java 文件 → class 骨架 | ✅ 4/4 |
|
|
211
|
+
| `java-single-file.yaml` | 6 | wiki Single file | LS → 补全 → 编辑 | ✅ 6/6 |
|
|
212
|
+
| `java-debugger.yaml` | 6 | wiki Debugger | 断点 → 启动调试 → 停止 | ✅ 6/6 |
|
|
213
|
+
| `java-test-runner.yaml` | 6 | wiki Test Runner | 测试面板 → 运行全部 → CodeLens | ✅ 6/6 |
|
|
214
|
+
| `java-maven-resolve-type.yaml` | 10 | wiki Maven for Java | 未知类型 → Hover → Code Action | ✅ 10/10 |
|
|
215
|
+
| `java-dependency-viewer.yaml` | 4 | wiki Dependency Viewer | 依赖视图 → 节点展开 | ✅ 4/4 |
|
|
216
|
+
| `java-extension-pack.yaml` | 3 | wiki Extension Pack | Configure Classpath 命令 | ✅ 3/3 |
|
|
217
|
+
| `java-fresh-import.yaml` | 3 | wiki Fresh Import | Spring Petclinic → LS → 补全 | 🔲 需要 clone 项目 |
|
|
218
|
+
|
|
219
|
+
### Wiki 场景覆盖情况
|
|
220
|
+
|
|
221
|
+
| Wiki 场景 | 状态 | 阻碍 |
|
|
222
|
+
|-----------|------|------|
|
|
223
|
+
| Basic #1-5 | ✅ 已有 test plan | — |
|
|
224
|
+
| Basic #6-8 (补全/Import/Rename) | ✅ 已有 test plan | — |
|
|
225
|
+
| Basic #9 (New Java File snippet) | ✅ 已有 test plan | — |
|
|
226
|
+
| Maven | ✅ 已有 test plan | — |
|
|
227
|
+
| Maven Multimodule | ✅ 已有 test plan | — |
|
|
228
|
+
| Gradle | ✅ 已有 test plan | — |
|
|
229
|
+
| Maven Java 25 | ✅ 已有 test plan | — |
|
|
230
|
+
| Gradle Java 25 | ✅ 已有 test plan | — |
|
|
231
|
+
| Single file | ✅ 已有 test plan | — |
|
|
232
|
+
| Single file without workspace | ❌ 未覆盖 | 需要拖拽文件(无 Driver 支持) |
|
|
233
|
+
| Fresh import (spring-petclinic) | ✅ 已有 test plan | 需要提前 clone 项目 |
|
|
234
|
+
| Debugger for Java | ✅ 已有 test plan | — |
|
|
235
|
+
| Java Test Runner | ✅ 已有 test plan | — |
|
|
236
|
+
| Maven for Java | ✅ 已有 test plan | — |
|
|
237
|
+
| Java Dependency Viewer | ✅ 已有 test plan | — |
|
|
238
|
+
| Java Extension Pack | ✅ 已有 test plan | webview 内部交互有限 |
|
|
239
|
+
|
|
240
|
+
---
|
|
241
|
+
|
|
242
|
+
## 项目结构
|
|
243
|
+
|
|
244
|
+
```
|
|
245
|
+
autotest/
|
|
246
|
+
├── src/
|
|
247
|
+
│ ├── drivers/
|
|
248
|
+
│ │ └── vscodeDriver.ts # Playwright VSCode 操作原语 (35+ 方法)
|
|
249
|
+
│ ├── operators/
|
|
250
|
+
│ │ ├── actionResolver.ts # 自然语言 Action → Driver 调用 (16 种 regex)
|
|
251
|
+
│ │ ├── stepVerifier.ts # 确定性验证 + LLM 验证 (6 种策略)
|
|
252
|
+
│ │ ├── llmClient.ts # Azure OpenAI 客户端 (截图 → pass/fail)
|
|
253
|
+
│ │ ├── planParser.ts # YAML Test Plan 解析器(路径相对于 plan 文件)
|
|
254
|
+
│ │ └── testRunner.ts # 编排引擎(启动 → 执行 → 截图 → 报告)
|
|
255
|
+
│ ├── cli/
|
|
256
|
+
│ │ └── index.ts # CLI 入口 (run / validate / --no-llm)
|
|
257
|
+
│ ├── types.ts # 核心类型定义
|
|
258
|
+
│ └── index.ts # SDK 导出
|
|
259
|
+
├── test-plans/ # YAML 测试计划
|
|
260
|
+
├── test-results/ # 测试输出(每个 plan 一个子目录)
|
|
261
|
+
│ └── <plan-name>/
|
|
262
|
+
│ ├── results.json
|
|
263
|
+
│ └── screenshots/
|
|
264
|
+
├── AGENTS.md # Copilot CLI 集成指南
|
|
265
|
+
├── docs/
|
|
266
|
+
│ ├── architecture.md # 架构文档
|
|
267
|
+
│ ├── implementation-plan.md # 实现计划
|
|
268
|
+
│ └── ROADMAP.md # 路线图
|
|
269
|
+
└── package.json
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
## 相关文档
|
|
273
|
+
|
|
274
|
+
- [架构设计](docs/architecture.md)
|
|
275
|
+
- [实现计划](docs/implementation-plan.md)
|
|
276
|
+
- [路线图](docs/ROADMAP.md)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* AutoTest CLI — AI-driven VSCode extension E2E testing tool.
|
|
4
|
+
*
|
|
5
|
+
* Usage:
|
|
6
|
+
* autotest run <test-plan.yaml> Execute a test plan
|
|
7
|
+
* autotest validate <test-plan.yaml> Validate test plan format
|
|
8
|
+
*/
|
|
9
|
+
import { Command } from "commander";
|
|
10
|
+
import * as path from "node:path";
|
|
11
|
+
import { loadTestPlan, validateTestPlanFile } from "../operators/planParser.js";
|
|
12
|
+
import { TestRunner } from "../operators/testRunner.js";
|
|
13
|
+
const program = new Command();
|
|
14
|
+
program
|
|
15
|
+
.name("autotest")
|
|
16
|
+
.description("AI-driven VSCode extension E2E testing framework")
|
|
17
|
+
.version("0.1.0");
|
|
18
|
+
program
|
|
19
|
+
.command("run <plan>")
|
|
20
|
+
.description("Execute a test plan against VSCode")
|
|
21
|
+
.option("--attach <port>", "Connect to an existing VSCode via CDP port")
|
|
22
|
+
.option("--interactive", "Step-by-step execution with manual confirmation")
|
|
23
|
+
.option("--output <dir>", "Output directory (default: ./test-results/<plan-name>)")
|
|
24
|
+
.option("--no-llm", "Skip LLM verification (auto-pass all verify fields)")
|
|
25
|
+
.action(async (planPath, opts) => {
|
|
26
|
+
try {
|
|
27
|
+
const plan = loadTestPlan(planPath);
|
|
28
|
+
console.log(`📋 Test Plan: ${plan.name}`);
|
|
29
|
+
console.log(` Extension: ${plan.setup.extension}`);
|
|
30
|
+
console.log(` Steps: ${plan.steps.length}`);
|
|
31
|
+
// Derive output dir from plan file name: test-results/<plan-name>/
|
|
32
|
+
const planName = path.basename(planPath, path.extname(planPath));
|
|
33
|
+
const outputDir = opts.output
|
|
34
|
+
? path.resolve(opts.output)
|
|
35
|
+
: path.resolve("test-results", planName);
|
|
36
|
+
const runner = new TestRunner(plan, { outputDir, noLLM: opts.llm === false });
|
|
37
|
+
// Ensure VSCode is closed even if the process is interrupted (Ctrl+C)
|
|
38
|
+
const cleanup = async () => {
|
|
39
|
+
console.log("\n🛑 Interrupted — closing VSCode...");
|
|
40
|
+
await runner.cleanup();
|
|
41
|
+
process.exit(130);
|
|
42
|
+
};
|
|
43
|
+
process.on("SIGINT", cleanup);
|
|
44
|
+
process.on("SIGTERM", cleanup);
|
|
45
|
+
const report = await runner.run();
|
|
46
|
+
// Exit code based on results
|
|
47
|
+
process.exit(report.summary.failed + report.summary.errors > 0 ? 1 : 0);
|
|
48
|
+
}
|
|
49
|
+
catch (e) {
|
|
50
|
+
console.error(`❌ Error: ${e.message}`);
|
|
51
|
+
process.exit(1);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
program
|
|
55
|
+
.command("validate <plan>")
|
|
56
|
+
.description("Validate a test plan YAML file")
|
|
57
|
+
.action((planPath) => {
|
|
58
|
+
const result = validateTestPlanFile(planPath);
|
|
59
|
+
if (result.valid) {
|
|
60
|
+
console.log(`✅ Test plan is valid: ${planPath}`);
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
console.error(`❌ Invalid test plan: ${planPath}`);
|
|
64
|
+
result.errors.forEach((e) => console.error(` - ${e}`));
|
|
65
|
+
process.exit(1);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
program.parse();
|
|
69
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":";AACA;;;;;;GAMG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAChF,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAExD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,kDAAkD,CAAC;KAC/D,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,YAAY,CAAC;KACrB,WAAW,CAAC,oCAAoC,CAAC;KACjD,MAAM,CAAC,iBAAiB,EAAE,4CAA4C,CAAC;KACvE,MAAM,CAAC,eAAe,EAAE,iDAAiD,CAAC;KAC1E,MAAM,CAAC,gBAAgB,EAAE,wDAAwD,CAAC;KAClF,MAAM,CAAC,UAAU,EAAE,qDAAqD,CAAC;KACzE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAgF,EAAE,EAAE;IACnH,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAE9C,mEAAmE;QACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjE,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM;YAC3B,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;YAC3B,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC;QAE9E,sEAAsE;QACtE,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE;YACzB,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;YACpD,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC,CAAC;QACF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAE/B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QAElC,6BAA6B;QAC7B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,YAAa,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,iBAAiB,CAAC;KAC1B,WAAW,CAAC,gCAAgC,CAAC;KAC7C,MAAM,CAAC,CAAC,QAAgB,EAAE,EAAE;IAC3B,MAAM,MAAM,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAC9C,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VscodeDriver — Core driver for launching and controlling VSCode via Playwright.
|
|
3
|
+
*
|
|
4
|
+
* Provides stable operation primitives categorized by reliability:
|
|
5
|
+
* - Level 1 (🟢): Command-based — uses VSCode command system, extremely stable
|
|
6
|
+
* - Level 2 (🟡): Role-based — uses Accessibility roles, stable across versions
|
|
7
|
+
* - Level 3 (🟠): Snapshot-based — AI reads A11y tree to decide, fully dynamic
|
|
8
|
+
*/
|
|
9
|
+
import { type Page } from "@playwright/test";
|
|
10
|
+
import type { A11yNode, Diagnostic, VscodeDriverOptions } from "../types.js";
|
|
11
|
+
export declare class VscodeDriver {
|
|
12
|
+
private app;
|
|
13
|
+
private page;
|
|
14
|
+
private options;
|
|
15
|
+
/** Temp copy of workspace — cleaned up on close() */
|
|
16
|
+
private tempWorkspaceDir;
|
|
17
|
+
constructor(options?: VscodeDriverOptions);
|
|
18
|
+
launch(): Promise<void>;
|
|
19
|
+
/** Close all visible notification toasts */
|
|
20
|
+
dismissAllNotifications(): Promise<void>;
|
|
21
|
+
close(): Promise<void>;
|
|
22
|
+
getPage(): Page;
|
|
23
|
+
/** Execute a VSCode command via Command Palette (F1 → type → Enter) */
|
|
24
|
+
runCommandFromPalette(label: string): Promise<void>;
|
|
25
|
+
/** Open a file via Quick Open (Ctrl+P). Retries if the file indexer isn't ready. */
|
|
26
|
+
openFile(filePath: string): Promise<void>;
|
|
27
|
+
/** Get the content of the active editor */
|
|
28
|
+
getEditorContent(): Promise<string>;
|
|
29
|
+
/** Check if the active editor contains the specified text (checks model + visible DOM) */
|
|
30
|
+
editorContains(text: string): Promise<boolean>;
|
|
31
|
+
/** Save the active file (Ctrl+S) */
|
|
32
|
+
saveFile(): Promise<void>;
|
|
33
|
+
/** Go to a specific line number (Ctrl+G) */
|
|
34
|
+
goToLine(line: number): Promise<void>;
|
|
35
|
+
/** Move cursor to end of current line */
|
|
36
|
+
goToEndOfLine(): Promise<void>;
|
|
37
|
+
/** Execute a keyboard shortcut */
|
|
38
|
+
pressKeys(keys: string): Promise<void>;
|
|
39
|
+
/** Run a command in the integrated terminal */
|
|
40
|
+
runInTerminal(command: string): Promise<void>;
|
|
41
|
+
/** Activate a side tab by name (e.g., "Explorer", "Extensions", "API Center") */
|
|
42
|
+
activeSideTab(tabName: string): Promise<void>;
|
|
43
|
+
/** Check if a side tab is visible */
|
|
44
|
+
isSideTabVisible(tabName: string): Promise<boolean>;
|
|
45
|
+
/** Click a tree item by its display name */
|
|
46
|
+
clickTreeItem(name: string): Promise<void>;
|
|
47
|
+
/** Check if a tree item is visible */
|
|
48
|
+
isTreeItemVisible(name: string): Promise<boolean>;
|
|
49
|
+
/** Select an option by name in the Command Palette dropdown */
|
|
50
|
+
selectPaletteOption(optionText: string): Promise<void>;
|
|
51
|
+
/** Select an option by index in the Command Palette dropdown */
|
|
52
|
+
selectPaletteOptionByIndex(index: number): Promise<void>;
|
|
53
|
+
/** Get all current notification messages */
|
|
54
|
+
getNotifications(): Promise<string[]>;
|
|
55
|
+
/** Get status bar text */
|
|
56
|
+
getStatusBarText(): Promise<string>;
|
|
57
|
+
/** Get the Accessibility tree snapshot of the current window */
|
|
58
|
+
snapshot(): Promise<A11yNode>;
|
|
59
|
+
/** Get a DOM HTML snapshot */
|
|
60
|
+
domSnapshot(): Promise<string>;
|
|
61
|
+
/** Take a screenshot and return as buffer */
|
|
62
|
+
screenshot(outputPath?: string): Promise<Buffer>;
|
|
63
|
+
/** Click any element by role and name (generic) */
|
|
64
|
+
clickByRole(role: string, name: string): Promise<void>;
|
|
65
|
+
/** Click any element containing specific text */
|
|
66
|
+
clickByText(text: string): Promise<void>;
|
|
67
|
+
/** Type text into the active editor at the cursor position */
|
|
68
|
+
typeInEditor(text: string): Promise<void>;
|
|
69
|
+
/** Select all text in the active editor */
|
|
70
|
+
selectAllInEditor(): Promise<void>;
|
|
71
|
+
/** Replace entire editor content with new text */
|
|
72
|
+
setEditorContent(content: string): Promise<void>;
|
|
73
|
+
/**
|
|
74
|
+
* Type a snippet trigger word and select the snippet from completion list.
|
|
75
|
+
*/
|
|
76
|
+
typeAndTriggerSnippet(triggerWord: string): Promise<void>;
|
|
77
|
+
/**
|
|
78
|
+
* Wait for the Java Language Server to become ready.
|
|
79
|
+
* Polls the status bar for the LS ready indicator.
|
|
80
|
+
*/
|
|
81
|
+
waitForLanguageServer(timeoutMs?: number): Promise<boolean>;
|
|
82
|
+
/**
|
|
83
|
+
* Get the count of errors and warnings in the Problems panel.
|
|
84
|
+
*/
|
|
85
|
+
getProblemsCount(): Promise<{
|
|
86
|
+
errors: number;
|
|
87
|
+
warnings: number;
|
|
88
|
+
}>;
|
|
89
|
+
/** Navigate to the next problem (error/warning) in the editor */
|
|
90
|
+
navigateToNextError(): Promise<void>;
|
|
91
|
+
/** Navigate to a specific error by index (1-based) */
|
|
92
|
+
navigateToError(index: number): Promise<void>;
|
|
93
|
+
/**
|
|
94
|
+
* Trigger Code Action menu at current cursor and select an action by label.
|
|
95
|
+
*/
|
|
96
|
+
applyCodeAction(label: string): Promise<void>;
|
|
97
|
+
/**
|
|
98
|
+
* Trigger code completion (IntelliSense) at the current cursor position.
|
|
99
|
+
* Returns the visible completion items as an array of label strings.
|
|
100
|
+
*/
|
|
101
|
+
triggerCompletion(): Promise<string[]>;
|
|
102
|
+
/** Dismiss the current completion widget */
|
|
103
|
+
dismissCompletion(): Promise<void>;
|
|
104
|
+
/** Check if an element with given role and name is visible */
|
|
105
|
+
isElementVisible(role: string, name: string): Promise<boolean>;
|
|
106
|
+
/** Get text content of an element by role and name */
|
|
107
|
+
getElementText(role: string, name: string): Promise<string>;
|
|
108
|
+
/** Get Problems panel diagnostics */
|
|
109
|
+
getProblems(): Promise<Diagnostic[]>;
|
|
110
|
+
/** Check if a file exists in the workspace */
|
|
111
|
+
fileExists(filePath: string): Promise<boolean>;
|
|
112
|
+
/** Check if a file contains specific text */
|
|
113
|
+
fileContains(filePath: string, text: string): Promise<boolean>;
|
|
114
|
+
/** Read file content */
|
|
115
|
+
readFile(filePath: string): Promise<string>;
|
|
116
|
+
/** Get the workspace root path (temp copy) */
|
|
117
|
+
getWorkspacePath(): string | null;
|
|
118
|
+
/**
|
|
119
|
+
* Insert a line into a file on disk at the specified line number (1-based).
|
|
120
|
+
* The file must already be open in the editor. After modifying on disk,
|
|
121
|
+
* reverts the editor to pick up changes — the LS stays active and re-analyzes quickly.
|
|
122
|
+
*/
|
|
123
|
+
insertLineInFile(relativePath: string, lineNumber: number, text: string): Promise<void>;
|
|
124
|
+
/** Revert the current file to its on-disk state */
|
|
125
|
+
revertFile(): Promise<void>;
|
|
126
|
+
/** Start a debug session (F5 or via command) */
|
|
127
|
+
startDebugSession(): Promise<void>;
|
|
128
|
+
/** Stop the current debug session (Shift+F5) */
|
|
129
|
+
stopDebugSession(): Promise<void>;
|
|
130
|
+
/** Set a breakpoint at a specific line in the current file */
|
|
131
|
+
setBreakpoint(line: number): Promise<void>;
|
|
132
|
+
/** Wait for the debugger to hit a breakpoint */
|
|
133
|
+
waitForBreakpointHit(timeoutMs?: number): Promise<boolean>;
|
|
134
|
+
/** Debug step over (F10) */
|
|
135
|
+
debugStepOver(): Promise<void>;
|
|
136
|
+
/** Debug step into (F11) */
|
|
137
|
+
debugStepInto(): Promise<void>;
|
|
138
|
+
/** Debug step out (Shift+F11) */
|
|
139
|
+
debugStepOut(): Promise<void>;
|
|
140
|
+
/** Get variable values from the Variables panel */
|
|
141
|
+
getDebugVariables(): Promise<Array<{
|
|
142
|
+
name: string;
|
|
143
|
+
value: string;
|
|
144
|
+
}>>;
|
|
145
|
+
/** Get Debug Console output text */
|
|
146
|
+
getDebugConsoleOutput(): Promise<string>;
|
|
147
|
+
/** Open the Test Explorer view */
|
|
148
|
+
openTestExplorer(): Promise<void>;
|
|
149
|
+
/** Run all tests via command */
|
|
150
|
+
runAllTests(): Promise<void>;
|
|
151
|
+
/** Wait for test execution to complete by polling the test status bar */
|
|
152
|
+
waitForTestComplete(timeoutMs?: number): Promise<boolean>;
|
|
153
|
+
/** Get test results summary from the Test Explorer */
|
|
154
|
+
getTestResults(): Promise<{
|
|
155
|
+
passed: number;
|
|
156
|
+
failed: number;
|
|
157
|
+
total: number;
|
|
158
|
+
}>;
|
|
159
|
+
/** Click a CodeLens link by its text */
|
|
160
|
+
clickCodeLens(label: string): Promise<void>;
|
|
161
|
+
/** Hover on a symbol in the editor to trigger hover provider */
|
|
162
|
+
hoverOnText(text: string): Promise<void>;
|
|
163
|
+
/** Get the content of the hover popup */
|
|
164
|
+
getHoverContent(): Promise<string>;
|
|
165
|
+
/** Click an action link inside the hover popup */
|
|
166
|
+
clickHoverAction(label: string): Promise<void>;
|
|
167
|
+
/** Dismiss the hover popup */
|
|
168
|
+
dismissHover(): Promise<void>;
|
|
169
|
+
/** Right-click a tree item and select a context menu option */
|
|
170
|
+
contextMenuOnTreeItem(itemName: string, menuLabel: string): Promise<void>;
|
|
171
|
+
/** Create a new file via Explorer right-click menu */
|
|
172
|
+
createNewFileViaExplorer(parentFolder: string, fileName: string): Promise<void>;
|
|
173
|
+
/** Open the Java Dependencies view */
|
|
174
|
+
openDependencyExplorer(): Promise<void>;
|
|
175
|
+
/** Expand a chain of tree nodes (e.g., ["Sources", "src", "main"]) */
|
|
176
|
+
expandTreePath(names: string[]): Promise<void>;
|
|
177
|
+
/** Wait for a specified duration (seconds) */
|
|
178
|
+
wait(seconds: number): Promise<void>;
|
|
179
|
+
}
|