testdriverai 7.8.0-test.4 → 7.8.0-test.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/agent/index.js +6 -5
- package/agent/lib/commands.js +3 -2
- package/agent/lib/http.js +144 -0
- package/agent/lib/sandbox.js +326 -164
- package/agent/lib/sdk.js +4 -2
- package/agent/lib/system.js +25 -65
- package/ai/skills/testdriver-cache/SKILL.md +221 -0
- package/ai/skills/testdriver-errors/SKILL.md +246 -0
- package/ai/skills/testdriver-events/SKILL.md +356 -0
- package/ai/skills/testdriver-mcp/SKILL.md +7 -0
- package/ai/skills/testdriver-provision/SKILL.md +331 -0
- package/ai/skills/testdriver-redraw/SKILL.md +214 -0
- package/ai/skills/testdriver-running-tests/SKILL.md +1 -1
- package/ai/skills/testdriver-screenshots/SKILL.md +184 -0
- package/docs/_data/examples-manifest.json +46 -46
- package/docs/changelog.mdx +155 -3
- package/docs/docs.json +44 -37
- package/docs/images/content/vscode/v7-chat.png +0 -0
- package/docs/images/content/vscode/v7-choose-agent.png +0 -0
- package/docs/images/content/vscode/v7-full.png +0 -0
- package/docs/images/content/vscode/v7-onboarding.png +0 -0
- package/docs/v7/cache.mdx +223 -0
- package/docs/v7/copilot/auto-healing.mdx +265 -0
- package/docs/v7/copilot/creating-tests.mdx +156 -0
- package/docs/v7/copilot/github.mdx +143 -0
- package/docs/v7/copilot/running-tests.mdx +149 -0
- package/docs/v7/copilot/setup.mdx +143 -0
- package/docs/v7/enterprise.mdx +3 -110
- package/docs/v7/errors.mdx +248 -0
- package/docs/v7/events.mdx +358 -0
- package/docs/v7/examples/ai.mdx +1 -1
- package/docs/v7/examples/assert.mdx +1 -1
- package/docs/v7/examples/captcha-api.mdx +1 -1
- package/docs/v7/examples/chrome-extension.mdx +1 -1
- package/docs/v7/examples/drag-and-drop.mdx +1 -1
- package/docs/v7/examples/element-not-found.mdx +1 -1
- package/docs/v7/examples/exec-output.mdx +85 -0
- package/docs/v7/examples/exec-pwsh.mdx +83 -0
- package/docs/v7/examples/focus-window.mdx +62 -0
- package/docs/v7/examples/hover-image.mdx +1 -1
- package/docs/v7/examples/hover-text.mdx +1 -1
- package/docs/v7/examples/installer.mdx +1 -1
- package/docs/v7/examples/launch-vscode-linux.mdx +1 -1
- package/docs/v7/examples/match-image.mdx +1 -1
- package/docs/v7/examples/press-keys.mdx +1 -1
- package/docs/v7/examples/scroll-keyboard.mdx +1 -1
- package/docs/v7/examples/scroll-until-image.mdx +1 -1
- package/docs/v7/examples/scroll-until-text.mdx +1 -1
- package/docs/v7/examples/scroll.mdx +1 -1
- package/docs/v7/examples/type.mdx +1 -1
- package/docs/v7/examples/windows-installer.mdx +1 -1
- package/docs/v7/{cloud.mdx → hosted.mdx} +43 -5
- package/docs/v7/mcp.mdx +9 -0
- package/docs/v7/provision.mdx +333 -0
- package/docs/v7/quickstart.mdx +30 -2
- package/docs/v7/redraw.mdx +216 -0
- package/docs/v7/running-tests.mdx +1 -1
- package/docs/v7/screenshots.mdx +186 -0
- package/docs/v7/self-hosted.mdx +127 -44
- package/interfaces/logger.js +0 -12
- package/interfaces/vitest-plugin.mjs +53 -43
- package/lib/core/Dashcam.js +30 -23
- package/lib/environments.json +18 -0
- package/lib/github-comment.mjs +58 -40
- package/lib/resolve-channel.js +4 -3
- package/lib/sentry.js +5 -0
- package/{examples → manual}/drag-and-drop.test.mjs +1 -1
- package/mcp-server/dist/server.mjs +4 -0
- package/mcp-server/src/server.ts +5 -0
- package/package.json +3 -3
- package/sdk.js +3 -3
- package/setup/aws/install-dev-runner.sh +79 -0
- package/setup/aws/spawn-runner.sh +134 -0
- package/vitest.config.mjs +20 -32
- package/vitest.runner.config.mjs +33 -0
- /package/{examples → manual}/flake-diffthreshold-001.test.mjs +0 -0
- /package/{examples → manual}/flake-diffthreshold-01.test.mjs +0 -0
- /package/{examples → manual}/flake-diffthreshold-05.test.mjs +0 -0
- /package/{examples → manual}/flake-noredraw-cache.test.mjs +0 -0
- /package/{examples → manual}/flake-noredraw-nocache.test.mjs +0 -0
- /package/{examples → manual}/flake-redraw-cache.test.mjs +0 -0
- /package/{examples → manual}/flake-redraw-nocache.test.mjs +0 -0
- /package/{examples → manual}/flake-rocket-match.test.mjs +0 -0
- /package/{examples → manual}/flake-shared.mjs +0 -0
- /package/{examples → manual}/no-provision.test.mjs +0 -0
- /package/{examples → manual}/scroll-until-text.test.mjs +0 -0
|
@@ -13,6 +13,7 @@ const channelConfig = require("../lib/resolve-channel.js");
|
|
|
13
13
|
|
|
14
14
|
// Import Sentry for error reporting
|
|
15
15
|
const Sentry = require("@sentry/node");
|
|
16
|
+
const chalk = require("chalk");
|
|
16
17
|
|
|
17
18
|
// Track if Sentry has been initialized
|
|
18
19
|
let sentryInitialized = false;
|
|
@@ -23,7 +24,7 @@ let sentryInitialized = false;
|
|
|
23
24
|
*/
|
|
24
25
|
function initializeSentry() {
|
|
25
26
|
if (sentryInitialized) return;
|
|
26
|
-
|
|
27
|
+
|
|
27
28
|
// Respect telemetry opt-out
|
|
28
29
|
if (process.env.TD_TELEMETRY === "false") {
|
|
29
30
|
return;
|
|
@@ -31,7 +32,7 @@ function initializeSentry() {
|
|
|
31
32
|
|
|
32
33
|
try {
|
|
33
34
|
const version = resolveTestDriverVersion() || "unknown";
|
|
34
|
-
|
|
35
|
+
|
|
35
36
|
Sentry.init({
|
|
36
37
|
dsn:
|
|
37
38
|
process.env.SENTRY_DSN ||
|
|
@@ -53,12 +54,12 @@ function initializeSentry() {
|
|
|
53
54
|
// Filter out events that should not be reported to Sentry
|
|
54
55
|
beforeSend(event, hint) {
|
|
55
56
|
const error = hint.originalException;
|
|
56
|
-
|
|
57
|
+
|
|
57
58
|
// Don't send user-cancelled errors
|
|
58
59
|
if (error && error.message && error.message.includes("User cancelled")) {
|
|
59
60
|
return null;
|
|
60
61
|
}
|
|
61
|
-
|
|
62
|
+
|
|
62
63
|
// Don't send test failures - these are expected behavior, not bugs in the SDK
|
|
63
64
|
// Test failures indicate the test found a problem, which is the intended use case
|
|
64
65
|
if (event.exception?.values) {
|
|
@@ -67,28 +68,33 @@ function initializeSentry() {
|
|
|
67
68
|
if (exception.type === "TestFailure") {
|
|
68
69
|
return null;
|
|
69
70
|
}
|
|
70
|
-
|
|
71
|
+
|
|
71
72
|
// Filter out common user code errors (ReferenceError, TypeError from user tests)
|
|
72
73
|
// Only report if the error originates from TestDriver SDK code, not user test code
|
|
73
74
|
const isUserCodeError = exception.stacktrace?.frames?.some(frame => {
|
|
74
75
|
const filename = frame.filename || frame.abs_path || "";
|
|
75
76
|
// Check if error is from user test files (not from SDK internals)
|
|
76
|
-
return filename.includes("/tests/") ||
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
77
|
+
return filename.includes("/tests/") ||
|
|
78
|
+
filename.includes("/test/") ||
|
|
79
|
+
filename.includes(".test.") ||
|
|
80
|
+
filename.includes(".spec.");
|
|
80
81
|
});
|
|
81
|
-
|
|
82
|
+
|
|
82
83
|
if (isUserCodeError && (exception.type === "ReferenceError" || exception.type === "TypeError")) {
|
|
83
84
|
return null;
|
|
84
85
|
}
|
|
86
|
+
|
|
87
|
+
// Filter out ElementNotFoundError - expected test outcome, not a crash
|
|
88
|
+
if (exception.type === "ElementNotFoundError") {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
85
91
|
}
|
|
86
92
|
}
|
|
87
|
-
|
|
93
|
+
|
|
88
94
|
return event;
|
|
89
95
|
},
|
|
90
96
|
});
|
|
91
|
-
|
|
97
|
+
|
|
92
98
|
sentryInitialized = true;
|
|
93
99
|
logger.debug("Sentry initialized for vitest");
|
|
94
100
|
} catch (err) {
|
|
@@ -118,17 +124,17 @@ async function flushSentry(timeout = 2000) {
|
|
|
118
124
|
function resolveTestDriverVersion() {
|
|
119
125
|
try {
|
|
120
126
|
return require("../package.json").version;
|
|
121
|
-
} catch {}
|
|
127
|
+
} catch { }
|
|
122
128
|
|
|
123
129
|
try {
|
|
124
130
|
const cwdRequire = createRequire(path.join(process.cwd(), "package.json"));
|
|
125
131
|
return cwdRequire("testdriverai/package.json").version;
|
|
126
|
-
} catch {}
|
|
132
|
+
} catch { }
|
|
127
133
|
|
|
128
134
|
try {
|
|
129
135
|
const pkgPath = path.join(process.cwd(), "node_modules", "testdriverai", "package.json");
|
|
130
136
|
return JSON.parse(fs.readFileSync(pkgPath, "utf8")).version;
|
|
131
|
-
} catch {}
|
|
137
|
+
} catch { }
|
|
132
138
|
|
|
133
139
|
return null;
|
|
134
140
|
}
|
|
@@ -149,19 +155,19 @@ function resolveVitestVersion() {
|
|
|
149
155
|
// Strategy 1: createRequire from import.meta.url (standard CJS interop)
|
|
150
156
|
try {
|
|
151
157
|
return require("vitest/package.json").version;
|
|
152
|
-
} catch {}
|
|
158
|
+
} catch { }
|
|
153
159
|
|
|
154
160
|
// Strategy 2: createRequire from process.cwd() (works when import.meta.url is rewritten)
|
|
155
161
|
try {
|
|
156
162
|
const cwdRequire = createRequire(path.join(process.cwd(), "package.json"));
|
|
157
163
|
return cwdRequire("vitest/package.json").version;
|
|
158
|
-
} catch {}
|
|
164
|
+
} catch { }
|
|
159
165
|
|
|
160
166
|
// Strategy 3: read directly from node_modules on disk
|
|
161
167
|
try {
|
|
162
168
|
const vitestPkgPath = path.join(process.cwd(), "node_modules", "vitest", "package.json");
|
|
163
169
|
return JSON.parse(fs.readFileSync(vitestPkgPath, "utf8")).version;
|
|
164
|
-
} catch {}
|
|
170
|
+
} catch { }
|
|
165
171
|
|
|
166
172
|
return null;
|
|
167
173
|
}
|
|
@@ -176,7 +182,7 @@ function checkVitestVersion() {
|
|
|
176
182
|
if (!version) {
|
|
177
183
|
throw new Error(
|
|
178
184
|
"TestDriver requires Vitest to be installed. " +
|
|
179
|
-
|
|
185
|
+
"Please install it: npm install vitest@latest",
|
|
180
186
|
);
|
|
181
187
|
}
|
|
182
188
|
|
|
@@ -184,7 +190,7 @@ function checkVitestVersion() {
|
|
|
184
190
|
if (major < MINIMUM_VITEST_VERSION) {
|
|
185
191
|
throw new Error(
|
|
186
192
|
`TestDriver requires Vitest >= ${MINIMUM_VITEST_VERSION}.0.0, but found ${version}. ` +
|
|
187
|
-
|
|
193
|
+
`Please upgrade Vitest: npm install vitest@latest`,
|
|
188
194
|
);
|
|
189
195
|
}
|
|
190
196
|
}
|
|
@@ -374,7 +380,7 @@ export async function authenticateWithApiKey(apiKey, apiRoot) {
|
|
|
374
380
|
// Network-level error (fetch failed entirely)
|
|
375
381
|
const networkError = new Error(
|
|
376
382
|
`Unable to reach TestDriver API at ${apiRoot}. ` +
|
|
377
|
-
|
|
383
|
+
"Check your internet connection and try again.",
|
|
378
384
|
);
|
|
379
385
|
networkError.code = "NETWORK_ERROR";
|
|
380
386
|
networkError.isNetworkError = true;
|
|
@@ -394,8 +400,8 @@ export async function authenticateWithApiKey(apiKey, apiRoot) {
|
|
|
394
400
|
if (response.status === 401) {
|
|
395
401
|
const authError = new Error(
|
|
396
402
|
data.message ||
|
|
397
|
-
|
|
398
|
-
|
|
403
|
+
"Invalid API key. Please check your TD_API_KEY and try again. " +
|
|
404
|
+
"Get your API key at https://console.testdriver.ai/team",
|
|
399
405
|
);
|
|
400
406
|
authError.code = data.error || "INVALID_API_KEY";
|
|
401
407
|
authError.isAuthError = true;
|
|
@@ -406,7 +412,7 @@ export async function authenticateWithApiKey(apiKey, apiRoot) {
|
|
|
406
412
|
if (response.status >= 500) {
|
|
407
413
|
const serverError = new Error(
|
|
408
414
|
data.message ||
|
|
409
|
-
|
|
415
|
+
`TestDriver API is currently unavailable (HTTP ${response.status}). Please try again later.`,
|
|
410
416
|
);
|
|
411
417
|
serverError.code = data.error || "API_UNAVAILABLE";
|
|
412
418
|
serverError.isServerError = true;
|
|
@@ -426,7 +432,7 @@ export async function authenticateWithApiKey(apiKey, apiRoot) {
|
|
|
426
432
|
// Other HTTP errors
|
|
427
433
|
throw new Error(
|
|
428
434
|
`Authentication failed: ${response.status} ${response.statusText}` +
|
|
429
|
-
|
|
435
|
+
(data.message ? ` - ${data.message}` : ""),
|
|
430
436
|
);
|
|
431
437
|
}
|
|
432
438
|
|
|
@@ -840,6 +846,9 @@ class TestDriverReporter {
|
|
|
840
846
|
logger.debug("API key present:", !!pluginState.apiKey);
|
|
841
847
|
logger.debug("API root:", pluginState.apiRoot);
|
|
842
848
|
|
|
849
|
+
// Environment info is printed by the SDK when each test initializes,
|
|
850
|
+
// so we skip the duplicate banner here in the reporter.
|
|
851
|
+
|
|
843
852
|
// Check if we should enable the reporter
|
|
844
853
|
if (!pluginState.apiKey) {
|
|
845
854
|
logger.warn("No API key provided, skipping test recording");
|
|
@@ -1135,7 +1144,7 @@ class TestDriverReporter {
|
|
|
1135
1144
|
const error = result.errors[0];
|
|
1136
1145
|
errorMessage = error.message;
|
|
1137
1146
|
errorStack = error.stack;
|
|
1138
|
-
|
|
1147
|
+
|
|
1139
1148
|
// Note: We do NOT report test failures to Sentry.
|
|
1140
1149
|
// Test failures are expected behavior (they indicate a test found a bug).
|
|
1141
1150
|
// We only want actual SDK crashes and exceptions reported to Sentry.
|
|
@@ -1205,8 +1214,9 @@ class TestDriverReporter {
|
|
|
1205
1214
|
|
|
1206
1215
|
console.log("");
|
|
1207
1216
|
console.log(
|
|
1208
|
-
`🔗 Test Report: ${consoleUrl}/runs/${testRunDbId}/${testCaseDbId}
|
|
1217
|
+
chalk.cyan(`🔗 Test Report: ${consoleUrl}/runs/${testRunDbId}/${testCaseDbId}`),
|
|
1209
1218
|
);
|
|
1219
|
+
console.log("");
|
|
1210
1220
|
|
|
1211
1221
|
// If there were retries, list all per-attempt dashcam URLs for debugging
|
|
1212
1222
|
if (hasRetries) {
|
|
@@ -1218,7 +1228,7 @@ class TestDriverReporter {
|
|
|
1218
1228
|
}
|
|
1219
1229
|
}
|
|
1220
1230
|
}
|
|
1221
|
-
|
|
1231
|
+
|
|
1222
1232
|
// Output parseable format for docs generation (examples only)
|
|
1223
1233
|
if (testFile.startsWith("examples/")) {
|
|
1224
1234
|
const testFileName = path.basename(testFile);
|
|
@@ -1244,12 +1254,20 @@ class TestDriverReporter {
|
|
|
1244
1254
|
* @returns {string} The corresponding web console URL
|
|
1245
1255
|
*/
|
|
1246
1256
|
function getConsoleUrl(apiRoot) {
|
|
1247
|
-
//
|
|
1248
|
-
if (process.env.
|
|
1257
|
+
// Explicit override — use TD_CONSOLE_URL when deliberately set
|
|
1258
|
+
if (process.env.TD_CONSOLE_URL) return process.env.TD_CONSOLE_URL;
|
|
1249
1259
|
|
|
1250
1260
|
if (!apiRoot) return "https://console.testdriver.ai";
|
|
1251
1261
|
|
|
1252
|
-
//
|
|
1262
|
+
// Fly.io: swap "-api" for "-web" in the hostname
|
|
1263
|
+
// e.g. preview-138-api.fly.dev -> preview-138-web.fly.dev
|
|
1264
|
+
// td-test-api.fly.dev -> td-test-web.fly.dev
|
|
1265
|
+
const flyMatch = apiRoot.match(/https:\/\/([\w-]+)-api\.fly\.dev/);
|
|
1266
|
+
if (flyMatch) {
|
|
1267
|
+
return `https://${flyMatch[1]}-web.fly.dev`;
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
// Known channel API URLs -> console equivalents
|
|
1253
1271
|
// e.g. https://api-canary.testdriver.ai -> https://console-canary.testdriver.ai
|
|
1254
1272
|
for (const url of Object.values(channelConfig.channels)) {
|
|
1255
1273
|
if (url === apiRoot) {
|
|
@@ -1257,27 +1275,19 @@ function getConsoleUrl(apiRoot) {
|
|
|
1257
1275
|
}
|
|
1258
1276
|
}
|
|
1259
1277
|
|
|
1260
|
-
// Local development
|
|
1278
|
+
// Local development
|
|
1261
1279
|
if (apiRoot.includes("ngrok.io") || apiRoot.includes("trycloudflare.com") || apiRoot.includes("localhost")) {
|
|
1262
|
-
return
|
|
1280
|
+
return "http://localhost:3001";
|
|
1263
1281
|
}
|
|
1264
1282
|
|
|
1265
|
-
// Render PR previews
|
|
1266
|
-
// canary-api-pr-123.onrender.com -> canary-web-pr-123.onrender.com
|
|
1267
|
-
// testdriver-api-i4m4-pr-123.onrender.com -> web-i4m4-pr-123.onrender.com
|
|
1283
|
+
// Render PR previews (legacy)
|
|
1268
1284
|
const renderPrMatch = apiRoot.match(/https:\/\/([\w-]+)-api(-[\w]+)?(-pr-\d+)?\.onrender\.com/);
|
|
1269
1285
|
if (renderPrMatch) {
|
|
1270
1286
|
const [, prefix, suffix, prSuffix] = renderPrMatch;
|
|
1271
|
-
|
|
1272
|
-
if (prefix === 'testdriver' && suffix) {
|
|
1273
|
-
webPrefix = 'web' + suffix;
|
|
1274
|
-
} else {
|
|
1275
|
-
webPrefix = prefix + '-web';
|
|
1276
|
-
}
|
|
1287
|
+
const webPrefix = (prefix === 'testdriver' && suffix) ? 'web' + suffix : prefix + '-web';
|
|
1277
1288
|
return `https://${webPrefix}${prSuffix || ''}.onrender.com`;
|
|
1278
1289
|
}
|
|
1279
1290
|
|
|
1280
|
-
// Other tunnels or unknown hosts: return as-is
|
|
1281
1291
|
return apiRoot;
|
|
1282
1292
|
}
|
|
1283
1293
|
|
package/lib/core/Dashcam.js
CHANGED
|
@@ -104,7 +104,7 @@ class Dashcam {
|
|
|
104
104
|
return `https://console-${envMatch[1]}.testdriver.ai`;
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
-
// Production: API on
|
|
107
|
+
// Production: API on custom domain or v6 -> Console on testdriver.ai
|
|
108
108
|
if (
|
|
109
109
|
apiRoot.includes("api.testdriver.ai") ||
|
|
110
110
|
apiRoot.includes("v6.testdriver.ai")
|
|
@@ -117,21 +117,31 @@ class Dashcam {
|
|
|
117
117
|
return "http://localhost:3001";
|
|
118
118
|
}
|
|
119
119
|
|
|
120
|
+
// Fly.io PR previews: map API app to Web app
|
|
121
|
+
// pr-123-api.fly.dev -> pr-123-web.fly.dev
|
|
122
|
+
const flyPrMatch = apiRoot.match(/https:\/\/(pr-\d+)-api\.fly\.dev/);
|
|
123
|
+
if (flyPrMatch) {
|
|
124
|
+
const [, prPrefix] = flyPrMatch;
|
|
125
|
+
return `https://${prPrefix}-web.fly.dev`;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Fly.io environment apps: test-api.fly.dev -> test-web.fly.dev
|
|
129
|
+
const flyEnvMatch = apiRoot.match(/https:\/\/([\w-]+)-api\.fly\.dev/);
|
|
130
|
+
if (flyEnvMatch) {
|
|
131
|
+
const [, prefix] = flyEnvMatch;
|
|
132
|
+
return `https://${prefix}-web.fly.dev`;
|
|
133
|
+
}
|
|
134
|
+
|
|
120
135
|
// Render PR previews: map API service to Web service
|
|
121
136
|
// canary-api-pr-123.onrender.com -> canary-web-pr-123.onrender.com
|
|
122
137
|
// testdriver-api-i4m4-pr-123.onrender.com -> web-i4m4-pr-123.onrender.com
|
|
123
138
|
const renderPrMatch = apiRoot.match(/https:\/\/([\w-]+)-api(-[\w]+)?(-pr-\d+)?\.onrender\.com/);
|
|
124
139
|
if (renderPrMatch) {
|
|
125
140
|
const [, prefix, suffix, prSuffix] = renderPrMatch;
|
|
126
|
-
// Map API naming to Web naming:
|
|
127
|
-
// canary-api -> canary-web
|
|
128
|
-
// testdriver-api-i4m4 -> web-i4m4
|
|
129
141
|
let webPrefix;
|
|
130
142
|
if (prefix === 'testdriver' && suffix) {
|
|
131
|
-
// testdriver-api-i4m4 -> web-i4m4
|
|
132
143
|
webPrefix = 'web' + suffix;
|
|
133
144
|
} else {
|
|
134
|
-
// canary-api -> canary-web
|
|
135
145
|
webPrefix = prefix + '-web';
|
|
136
146
|
}
|
|
137
147
|
return `https://${webPrefix}${prSuffix || ''}.onrender.com`;
|
|
@@ -147,20 +157,13 @@ class Dashcam {
|
|
|
147
157
|
* @private
|
|
148
158
|
*/
|
|
149
159
|
async _getDashcamPath() {
|
|
150
|
-
const shell = this._getShell();
|
|
151
160
|
|
|
152
161
|
if (this.client.os === "windows") {
|
|
153
162
|
return "C:\\Program Files\\nodejs\\dashcam.cmd";
|
|
163
|
+
} else {
|
|
164
|
+
return "/usr/bin/dashcam";
|
|
154
165
|
}
|
|
155
166
|
|
|
156
|
-
const npmPrefix = await this.client.exec(
|
|
157
|
-
shell,
|
|
158
|
-
"npm prefix -g",
|
|
159
|
-
40000,
|
|
160
|
-
process.env.TD_DEBUG == "true" ? false : true,
|
|
161
|
-
);
|
|
162
|
-
|
|
163
|
-
return npmPrefix.trim() + "/bin/dashcam";
|
|
164
167
|
}
|
|
165
168
|
|
|
166
169
|
/**
|
|
@@ -289,13 +292,17 @@ class Dashcam {
|
|
|
289
292
|
const apiRoot = this._getApiRoot();
|
|
290
293
|
|
|
291
294
|
if (this.client.os === "windows") {
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
295
|
+
try {
|
|
296
|
+
const addLogOutput = await this.client.exec(
|
|
297
|
+
shell,
|
|
298
|
+
`$env:TD_API_ROOT="${apiRoot}"; & "${dashcamPath}" logs --add --type=web --pattern="${pattern}" --name="${name}"`,
|
|
299
|
+
120000,
|
|
300
|
+
process.env.TD_DEBUG == "true" ? false : true,
|
|
301
|
+
);
|
|
302
|
+
this._log("debug", "Add web log tracking output:", addLogOutput);
|
|
303
|
+
} catch (err) {
|
|
304
|
+
this._log("warn", "Add web log tracking failed:", err.message);
|
|
305
|
+
}
|
|
299
306
|
} else {
|
|
300
307
|
const addLogOutput = await this.client.exec(
|
|
301
308
|
shell,
|
|
@@ -449,7 +456,7 @@ class Dashcam {
|
|
|
449
456
|
this.recording = false;
|
|
450
457
|
|
|
451
458
|
// Extract the /replay/... path from CLI output and reconstruct the URL
|
|
452
|
-
// using getConsoleUrl(). The CLI may return a wrong domain
|
|
459
|
+
// using getConsoleUrl(). The CLI may return a wrong domain
|
|
453
460
|
// so we always rewrite the base URL to match the current environment.
|
|
454
461
|
if (output) {
|
|
455
462
|
// Match /replay/{id} with optional query params from any URL or broken prefix
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{
|
|
2
|
+
"dev": {
|
|
3
|
+
"apiRoot": "https://api-dev.testdriver.ai",
|
|
4
|
+
"consoleUrl": "https://console-dev.testdriver.ai"
|
|
5
|
+
},
|
|
6
|
+
"test": {
|
|
7
|
+
"apiRoot": "https://api-test.testdriver.ai",
|
|
8
|
+
"consoleUrl": "https://console-test.testdriver.ai"
|
|
9
|
+
},
|
|
10
|
+
"canary": {
|
|
11
|
+
"apiRoot": "https://api-canary.testdriver.ai",
|
|
12
|
+
"consoleUrl": "https://console-canary.testdriver.ai"
|
|
13
|
+
},
|
|
14
|
+
"stable": {
|
|
15
|
+
"apiRoot": "https://api.testdriver.ai",
|
|
16
|
+
"consoleUrl": "https://console.testdriver.ai"
|
|
17
|
+
}
|
|
18
|
+
}
|
package/lib/github-comment.mjs
CHANGED
|
@@ -49,7 +49,7 @@ function generateTestResultsTable(testCases, testRunUrl) {
|
|
|
49
49
|
|
|
50
50
|
// Filter out skipped tests
|
|
51
51
|
const nonSkippedTests = testCases.filter(test => test.status !== 'skipped');
|
|
52
|
-
|
|
52
|
+
|
|
53
53
|
if (nonSkippedTests.length === 0) {
|
|
54
54
|
return '_No test cases to display (all tests were skipped)_';
|
|
55
55
|
}
|
|
@@ -62,12 +62,12 @@ function generateTestResultsTable(testCases, testRunUrl) {
|
|
|
62
62
|
const name = test.testName || 'Unknown';
|
|
63
63
|
const file = test.testFile || 'unknown';
|
|
64
64
|
const duration = formatDuration(test.duration || 0);
|
|
65
|
-
|
|
65
|
+
|
|
66
66
|
// Use test run context URL instead of direct replay URL
|
|
67
67
|
let replay = '-';
|
|
68
68
|
if (test.replayUrl) {
|
|
69
69
|
const linkUrl = (test.id && testRunUrl) ? `${testRunUrl}/${test.id}` : test.replayUrl;
|
|
70
|
-
|
|
70
|
+
|
|
71
71
|
// Extract replay ID and generate GIF URL
|
|
72
72
|
const replayId = extractReplayId(test.replayUrl);
|
|
73
73
|
if (replayId) {
|
|
@@ -97,28 +97,28 @@ function generateTestResultsTable(testCases, testRunUrl) {
|
|
|
97
97
|
*/
|
|
98
98
|
function generateExceptionsSection(testCases, testRunUrl) {
|
|
99
99
|
const failedTests = testCases.filter(t => t.status === 'failed' && t.errorMessage);
|
|
100
|
-
|
|
100
|
+
|
|
101
101
|
if (failedTests.length === 0) {
|
|
102
102
|
return '';
|
|
103
103
|
}
|
|
104
104
|
|
|
105
105
|
let section = '\n## 🔴 Failures\n\n';
|
|
106
|
-
|
|
106
|
+
|
|
107
107
|
for (const test of failedTests) {
|
|
108
108
|
section += `### ${test.testName}\n\n`;
|
|
109
109
|
section += `**File:** \`${test.testFile}\`\n\n`;
|
|
110
|
-
|
|
110
|
+
|
|
111
111
|
// Use test run context URL instead of direct replay URL
|
|
112
112
|
if (test.id && testRunUrl) {
|
|
113
113
|
section += `**📹 [Watch Replay](${testRunUrl}/${test.id})**\n\n`;
|
|
114
114
|
} else if (test.replayUrl) {
|
|
115
115
|
section += `**📹 [Watch Replay](${test.replayUrl})**\n\n`;
|
|
116
116
|
}
|
|
117
|
-
|
|
117
|
+
|
|
118
118
|
section += '```\n';
|
|
119
119
|
section += test.errorMessage || 'Unknown error';
|
|
120
120
|
section += '\n```\n\n';
|
|
121
|
-
|
|
121
|
+
|
|
122
122
|
if (test.errorStack) {
|
|
123
123
|
section += '<details>\n';
|
|
124
124
|
section += '<summary>Stack Trace</summary>\n\n';
|
|
@@ -140,22 +140,22 @@ function generateExceptionsSection(testCases, testRunUrl) {
|
|
|
140
140
|
*/
|
|
141
141
|
function generateReplaySection(testCases, testRunUrl) {
|
|
142
142
|
const testsWithReplays = testCases.filter(t => t.replayUrl);
|
|
143
|
-
|
|
143
|
+
|
|
144
144
|
if (testsWithReplays.length === 0) {
|
|
145
145
|
return '';
|
|
146
146
|
}
|
|
147
147
|
|
|
148
148
|
let section = '\n## 🎥 Dashcam Replays\n\n';
|
|
149
|
-
|
|
149
|
+
|
|
150
150
|
for (const test of testsWithReplays) {
|
|
151
151
|
section += `### ${test.testName}\n\n`;
|
|
152
|
-
|
|
152
|
+
|
|
153
153
|
// Determine the link URL - prefer test run context
|
|
154
154
|
let linkUrl = test.replayUrl;
|
|
155
155
|
if (test.id && testRunUrl) {
|
|
156
156
|
linkUrl = `${testRunUrl}/${test.id}`;
|
|
157
157
|
}
|
|
158
|
-
|
|
158
|
+
|
|
159
159
|
// Extract replay ID from URL for GIF embed
|
|
160
160
|
const replayId = extractReplayId(test.replayUrl);
|
|
161
161
|
if (replayId) {
|
|
@@ -177,7 +177,7 @@ function generateReplaySection(testCases, testRunUrl) {
|
|
|
177
177
|
*/
|
|
178
178
|
function extractReplayId(url) {
|
|
179
179
|
if (!url) return null;
|
|
180
|
-
|
|
180
|
+
|
|
181
181
|
// Match pattern: /replay/{id} or /replay/{id}?params
|
|
182
182
|
const match = url.match(/\/replay\/([^?/#]+)/);
|
|
183
183
|
return match ? match[1] : null;
|
|
@@ -190,7 +190,7 @@ function extractReplayId(url) {
|
|
|
190
190
|
*/
|
|
191
191
|
function extractShareKey(url) {
|
|
192
192
|
if (!url) return null;
|
|
193
|
-
|
|
193
|
+
|
|
194
194
|
// Match pattern: ?share=KEY or &share=KEY
|
|
195
195
|
const match = url.match(/[?&]share=([^&#]+)/);
|
|
196
196
|
return match ? match[1] : null;
|
|
@@ -204,32 +204,50 @@ function extractShareKey(url) {
|
|
|
204
204
|
*/
|
|
205
205
|
function getReplayGifUrl(replayUrl, replayId) {
|
|
206
206
|
// Determine the API base URL based on the replay URL
|
|
207
|
+
// Replay URLs use console domains; GIF endpoints live on the corresponding API domain
|
|
207
208
|
let apiBaseUrl;
|
|
208
|
-
|
|
209
|
+
|
|
209
210
|
if (replayUrl.includes('app.dashcam.io')) {
|
|
210
211
|
// Production dashcam uses Heroku API
|
|
211
212
|
apiBaseUrl = 'https://testdriverai-v6-c96fc597be11.herokuapp.com';
|
|
212
|
-
} else if (replayUrl.includes('console.testdriver.ai')) {
|
|
213
|
-
// TestDriver console
|
|
214
|
-
apiBaseUrl = 'https://api.testdriver.ai';
|
|
215
213
|
} else if (replayUrl.includes('localhost')) {
|
|
216
214
|
// Local development
|
|
217
215
|
apiBaseUrl = 'http://localhost:1337';
|
|
218
216
|
} else {
|
|
219
|
-
//
|
|
220
|
-
|
|
221
|
-
|
|
217
|
+
// Map console URLs → API URLs for all environments
|
|
218
|
+
// console-test.testdriver.ai → api-test.testdriver.ai
|
|
219
|
+
// console-canary.testdriver.ai → api-canary.testdriver.ai
|
|
220
|
+
// console.testdriver.ai → api.testdriver.ai
|
|
221
|
+
const consoleEnvMatch = replayUrl.match(/https:\/\/console-(test|canary)\.testdriver\.ai/);
|
|
222
|
+
if (consoleEnvMatch) {
|
|
223
|
+
apiBaseUrl = `https://api-${consoleEnvMatch[1]}.testdriver.ai`;
|
|
224
|
+
} else if (replayUrl.includes('console.testdriver.ai')) {
|
|
225
|
+
apiBaseUrl = 'https://api.testdriver.ai';
|
|
226
|
+
}
|
|
227
|
+
// Fly.io: map web app → API app
|
|
228
|
+
// pr-123-web.fly.dev → pr-123-api.fly.dev
|
|
229
|
+
// td-test-web.fly.dev → td-test-api.fly.dev
|
|
230
|
+
else {
|
|
231
|
+
const flyWebMatch = replayUrl.match(/https:\/\/([\w-]+)-web\.fly\.dev/);
|
|
232
|
+
if (flyWebMatch) {
|
|
233
|
+
apiBaseUrl = `https://${flyWebMatch[1]}-api.fly.dev`;
|
|
234
|
+
} else {
|
|
235
|
+
// Fallback: extract base URL from replay URL as-is
|
|
236
|
+
const urlObj = new URL(replayUrl);
|
|
237
|
+
apiBaseUrl = `${urlObj.protocol}//${urlObj.host}`;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
222
240
|
}
|
|
223
|
-
|
|
241
|
+
|
|
224
242
|
// Extract share key if present
|
|
225
243
|
const shareKey = extractShareKey(replayUrl);
|
|
226
|
-
|
|
244
|
+
|
|
227
245
|
// Build GIF URL with shareKey parameter
|
|
228
246
|
let gifUrl = `${apiBaseUrl}/replay/${replayId}/gif`;
|
|
229
247
|
if (shareKey) {
|
|
230
248
|
gifUrl += `?shareKey=${shareKey}`;
|
|
231
249
|
}
|
|
232
|
-
|
|
250
|
+
|
|
233
251
|
return gifUrl;
|
|
234
252
|
}
|
|
235
253
|
|
|
@@ -257,9 +275,9 @@ export function generateGitHubComment(testRunData, testCases = []) {
|
|
|
257
275
|
// Header with overall status
|
|
258
276
|
const statusEmoji = getStatusEmoji(status);
|
|
259
277
|
const statusColor = status === 'passed' ? '🟢' : status === 'failed' ? '🔴' : '🟡';
|
|
260
|
-
|
|
278
|
+
|
|
261
279
|
let comment = `# ${statusColor} TestDriver Test Results\n\n`;
|
|
262
|
-
|
|
280
|
+
|
|
263
281
|
// Compact summary line
|
|
264
282
|
comment += `**Status:** ${statusEmoji} ${status.toUpperCase()}`;
|
|
265
283
|
comment += ` • **Duration:** ${formatDuration(duration)}`;
|
|
@@ -270,23 +288,23 @@ export function generateGitHubComment(testRunData, testCases = []) {
|
|
|
270
288
|
comment += `, ${skippedTests} skipped`;
|
|
271
289
|
}
|
|
272
290
|
comment += `\n\n`;
|
|
273
|
-
|
|
291
|
+
|
|
274
292
|
// Exceptions section (only if there are failures) - show first
|
|
275
293
|
comment += generateExceptionsSection(testCases, testRunUrl);
|
|
276
|
-
|
|
294
|
+
|
|
277
295
|
// Test results table (now includes embedded GIFs)
|
|
278
296
|
comment += '## 📝 Test Results\n\n';
|
|
279
297
|
comment += generateTestResultsTable(testCases, testRunUrl);
|
|
280
|
-
|
|
298
|
+
|
|
281
299
|
// Link to full test run (below table)
|
|
282
300
|
if (testRunUrl) {
|
|
283
301
|
comment += `\n[📋 View Full Test Run](${testRunUrl})\n`;
|
|
284
302
|
}
|
|
285
|
-
|
|
303
|
+
|
|
286
304
|
// Footer
|
|
287
305
|
comment += '\n---\n';
|
|
288
306
|
comment += `<sub>Generated by [TestDriver](https://testdriver.ai) • Run ID: \`${runId}\`</sub>\n`;
|
|
289
|
-
|
|
307
|
+
|
|
290
308
|
return comment;
|
|
291
309
|
}
|
|
292
310
|
|
|
@@ -303,15 +321,15 @@ export function generateGitHubComment(testRunData, testCases = []) {
|
|
|
303
321
|
*/
|
|
304
322
|
export async function postGitHubComment(options) {
|
|
305
323
|
const { token, owner, repo, prNumber, commitSha, body } = options;
|
|
306
|
-
|
|
324
|
+
|
|
307
325
|
if (!token) {
|
|
308
326
|
throw new Error('GitHub token is required');
|
|
309
327
|
}
|
|
310
|
-
|
|
328
|
+
|
|
311
329
|
if (!owner || !repo) {
|
|
312
330
|
throw new Error('Repository owner and name are required');
|
|
313
331
|
}
|
|
314
|
-
|
|
332
|
+
|
|
315
333
|
if (!prNumber && !commitSha) {
|
|
316
334
|
throw new Error('Either prNumber or commitSha must be provided');
|
|
317
335
|
}
|
|
@@ -351,7 +369,7 @@ export async function postGitHubComment(options) {
|
|
|
351
369
|
*/
|
|
352
370
|
export async function updateGitHubComment(options) {
|
|
353
371
|
const { token, owner, repo, commentId, body } = options;
|
|
354
|
-
|
|
372
|
+
|
|
355
373
|
if (!token || !owner || !repo || !commentId) {
|
|
356
374
|
throw new Error('Token, owner, repo, and commentId are required');
|
|
357
375
|
}
|
|
@@ -364,7 +382,7 @@ export async function updateGitHubComment(options) {
|
|
|
364
382
|
comment_id: commentId,
|
|
365
383
|
body,
|
|
366
384
|
});
|
|
367
|
-
|
|
385
|
+
|
|
368
386
|
return response.data;
|
|
369
387
|
}
|
|
370
388
|
|
|
@@ -379,7 +397,7 @@ export async function updateGitHubComment(options) {
|
|
|
379
397
|
*/
|
|
380
398
|
export async function findExistingComment(options) {
|
|
381
399
|
const { token, owner, repo, prNumber } = options;
|
|
382
|
-
|
|
400
|
+
|
|
383
401
|
if (!token || !owner || !repo || !prNumber) {
|
|
384
402
|
return null;
|
|
385
403
|
}
|
|
@@ -410,11 +428,11 @@ export async function findExistingComment(options) {
|
|
|
410
428
|
*/
|
|
411
429
|
export async function postOrUpdateTestResults(testRunData, testCases, githubOptions) {
|
|
412
430
|
const commentBody = generateGitHubComment(testRunData, testCases);
|
|
413
|
-
|
|
431
|
+
|
|
414
432
|
// Try to find and delete existing comment to keep it at the end
|
|
415
433
|
if (githubOptions.prNumber) {
|
|
416
434
|
const existingComment = await findExistingComment(githubOptions);
|
|
417
|
-
|
|
435
|
+
|
|
418
436
|
if (existingComment) {
|
|
419
437
|
// Delete the old comment
|
|
420
438
|
const octokit = new Octokit({ auth: githubOptions.token });
|
|
@@ -425,7 +443,7 @@ export async function postOrUpdateTestResults(testRunData, testCases, githubOpti
|
|
|
425
443
|
});
|
|
426
444
|
}
|
|
427
445
|
}
|
|
428
|
-
|
|
446
|
+
|
|
429
447
|
// Always create a new comment (will be at the end of the thread)
|
|
430
448
|
return await postGitHubComment({
|
|
431
449
|
...githubOptions,
|
package/lib/resolve-channel.js
CHANGED
|
@@ -9,12 +9,13 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
const semver = require("semver");
|
|
12
|
+
const environments = require("./environments.json");
|
|
12
13
|
|
|
13
14
|
const CHANNELS = {
|
|
14
15
|
dev: "http://localhost:1337",
|
|
15
|
-
test:
|
|
16
|
-
canary:
|
|
17
|
-
latest:
|
|
16
|
+
test: environments.test.apiRoot,
|
|
17
|
+
canary: environments.canary.apiRoot,
|
|
18
|
+
latest: environments.stable.apiRoot,
|
|
18
19
|
};
|
|
19
20
|
|
|
20
21
|
function resolveActiveChannel() {
|