testdriverai 6.0.16-canary.e337e67.0 → 6.0.16-canary.f0eefe0.0

package/agent/events.js

@@ -33,7 +33,6 @@ const events = {
   vm: {
     show: "vm:show",
   },
-  narration: "narration",
   status: "status",
   log: {
     markdown: {
@@ -45,6 +44,7 @@ const events = {
     log: "log:log",
     warn: "log:warn",
     debug: "log:debug",
+    narration: "log:narration",
   },
   command: {
     start: "command:start",

package/agent/index.js

@@ -158,7 +158,7 @@ class TestDriverAgent extends EventEmitter2 {
   // single function to handle all program exits
   // allows us to save the current state, run lifecycle hooks, and track analytics
   async exit(failed = true, shouldSave = false, shouldRunPostrun = false) {
-    this.emitter.emit(events.log.log, theme.dim("exiting..."), true);
+    this.emitter.emit(events.log.narration, theme.dim("exiting..."), true);
 
     // Clean up redraw interval
     if (this.redraw && this.redraw.cleanup) {
@@ -192,7 +192,7 @@ class TestDriverAgent extends EventEmitter2 {
 
   // fatal errors always exit the program
   // this ensure we log the error, summarize it, and exit cleanly
-  async dieOnFatal(error) {
+  async dieOnFatal(error, skipPostrun = false) {
     // Show error with source context if available
     const errorContext = this.sourceMapper.getErrorWithSourceContext(error);
     if (errorContext) {
@@ -204,9 +204,13 @@ class TestDriverAgent extends EventEmitter2 {
       );
     }
 
-    await this.summarize(error.message);
-    // Always run postrun lifecycle script, even for fatal errors
-    return await this.exit(true, false, true);
+    if (skipPostrun) {
+      this.exit(true);
+    } else {
+      await this.summarize(error.message);
+      // Always run postrun lifecycle script, even for fatal errors
+      return await this.exit(true, false, true);
+    }
   }
 
   // creates a new "thread" in which the AI is given an error
@@ -286,7 +290,7 @@ class TestDriverAgent extends EventEmitter2 {
       image = null;
     }
 
-    this.emitter.emit(events.log.log, theme.dim("thinking..."), true);
+    this.emitter.emit(events.log.narration, theme.dim("thinking..."), true);
 
     const streamId = `error-${Date.now()}`;
     this.emitter.emit(events.log.markdown.start, streamId);
@@ -328,14 +332,14 @@ class TestDriverAgent extends EventEmitter2 {
 
     if (this.checkCount >= this.checkLimit) {
       this.emitter.emit(
-        events.log.log,
+        events.log.narration,
         theme.red("Exploratory loop detected. Exiting."),
       );
       await this.summarize("Check loop detected.");
       return await this.exit(true);
     }
 
-    this.emitter.emit(events.log.log, theme.dim("checking..."));
+    this.emitter.emit(events.log.narration, theme.dim("checking..."));
 
     // check asks the ai if the task is complete
     let thisScreenshot = await this.system.captureScreenBase64(1, false, true);
@@ -579,8 +583,6 @@ class TestDriverAgent extends EventEmitter2 {
           "check thinks more needs to be done",
         );
 
-        this.emitter.emit(events.log.log, theme.dim("not done yet!"));
-
         return await this.aiExecute(response, validateAndLoop);
       } else {
         this.emitter.emit(events.log.debug, "seems complete, returning");
@@ -715,7 +717,7 @@ class TestDriverAgent extends EventEmitter2 {
       }
     }
 
-    this.emitter.emit(events.log.log, theme.dim("thinking..."), true);
+    this.emitter.emit(events.log.narration, theme.dim("thinking..."), true);
 
     let response = `\`\`\`yaml
 commands:
@@ -743,7 +745,7 @@ commands:
 
     this.tasks.push(currentTask);
 
-    this.emitter.emit(events.log.log, theme.dim("thinking..."), true);
+    this.emitter.emit(events.log.narration, theme.dim("thinking..."), true);
 
     this.lastScreenshot = await this.system.captureScreenBase64();
 
@@ -785,7 +787,7 @@ commands:
   async generate(type, count, baseYaml, skipYaml = false) {
     this.emitter.emit(events.log.debug, "generate called, %s", type);
 
-    this.emitter.emit(events.log.log, theme.dim("thinking..."), true);
+    this.emitter.emit(events.log.narration, theme.dim("thinking..."), true);
 
     if (baseYaml && !skipYaml) {
       await this.runLifecycle("prerun");
@@ -862,7 +864,7 @@ commands:
 
   // this is the functinoality for "undo"
   async popFromHistory(fullStep) {
-    this.emitter.emit(events.log.log, theme.dim("undoing..."), true);
+    this.emitter.emit(events.log.narration, theme.dim("undoing..."), true);
 
     if (this.executionHistory.length) {
       if (fullStep) {
@@ -942,12 +944,16 @@ ${yml}
   async summarize(error = null) {
     this.analytics.track("summarize");
 
-    this.emitter.emit(events.log.log, theme.dim("reviewing test..."), true);
+    this.emitter.emit(
+      events.log.narration,
+      theme.dim("reviewing test..."),
+      true,
+    );
 
     // let text = prompts.summarize(tasks, error);
     let image = await this.system.captureScreenBase64();
 
-    this.emitter.emit(events.log.log, theme.dim("summarizing..."), true);
+    this.emitter.emit(events.log.narration, theme.dim("summarizing..."), true);
 
     const streamId = `summarize-${Date.now()}`;
     this.emitter.emit(events.log.markdown.start, streamId);
@@ -1120,7 +1126,7 @@ ${regression}
       timestamp: fileStartTime,
     });
 
-    this.emitter.emit(events.log.log, theme.cyan(`running ${file}...`));
+    this.emitter.emit(events.log.narration, theme.cyan(`running ${file}...`));
 
     let ymlObj = await this.loadYML(file);
 
@@ -1512,8 +1518,8 @@ ${regression}
     if (this.sandboxId && !this.config.CI && !createNew) {
       // Attempt to connect to known instance
       this.emitter.emit(
-        events.log.log,
-        theme.dim(`- connecting to sandbox ${this.sandboxId}...`),
+        events.log.narration,
+        theme.dim(`connecting to sandbox ${this.sandboxId}...`),
       );
 
       try {
@@ -1535,17 +1541,20 @@ ${regression}
       }
     }
 
-    this.emitter.emit(events.log.log, theme.dim(`- creating new sandbox...`));
+    this.emitter.emit(
+      events.log.narration,
+      theme.dim(`creating new sandbox...`),
+    );
     this.emitter.emit(
       events.log.log,
-      theme.dim(`  (this can take between 10 - 240 seconds)`),
+      theme.dim(`this can take between 10 - 240 seconds`),
     );
     // We don't have resiliency/retries baked in, so let's at least give it 1 attempt
     // to see if that fixes the issue.
     let newSandbox = await this.createNewSandbox().catch(() => {
       this.emitter.emit(
-        events.log.log,
-        theme.dim(`  (double-checking sandbox availability)`),
+        events.log.narration,
+        theme.dim(`double-checking sandbox availability`),
       );
 
       return this.createNewSandbox();
@@ -1572,7 +1581,7 @@ ${regression}
       if (!debuggerStarted) {
         debuggerStarted = true; // Prevent multiple starts, especially when running test in parallel
         this.emitter.emit(
-          events.log.log,
+          events.log.narration,
           theme.green(`Starting debugger server...`),
         );
         debuggerProcess = await createDebuggerProcess(
@@ -1584,9 +1593,12 @@ ${regression}
       this.emitter.emit(events.log.log, `This is beta software!`);
       this.emitter.emit(
         events.log.log,
-        theme.yellow(`Join our Forums for help`),
+        theme.yellow(`Join our Discord for help`),
+      );
+      this.emitter.emit(
+        events.log.log,
+        `https://discord.com/invite/cWDFW8DzPm`,
       );
-      this.emitter.emit(events.log.log, `https://forums.testdriver.ai`);
 
       // make testdriver directory if it doesn't exist
       let testdriverFolder = path.join(this.workingDir);
@@ -1690,16 +1702,33 @@ ${regression}
 
   async connectToSandboxService() {
     this.emitter.emit(
-      events.log.log,
-      theme.gray(`- establishing connection...`),
+      events.log.narration,
+      theme.dim(`establishing connection...`),
     );
-    await this.sandbox.boot(this.config.TD_API_ROOT);
-    this.emitter.emit(events.log.log, theme.gray(`- authenticating...`));
-    await this.sandbox.auth(this.config.TD_API_KEY);
+    let ableToBoot = await this.sandbox.boot(this.config.TD_API_ROOT);
+
+    if (!ableToBoot) {
+      return await this.dieOnFatal(
+        `Unable to connect to TestDriver sandbox service at ${this.config.TD_API_ROOT}.
+Please check your network connection, TD_API_KEY, or the service status.`,
+        true,
+      );
+    }
+
+    this.emitter.emit(events.log.narration, theme.dim(`authenticating...`));
+    let ableToAuth = await this.sandbox.auth(this.config.TD_API_KEY);
+
+    if (!ableToAuth) {
+      return await this.dieOnFatal(
+        `Unable to authorize with TestDriver sandbox service at ${this.config.TD_API_ROOT}.
+Please check your network connection, TD_API_KEY, or the service status.`,
+        true,
+      );
+    }
   }
 
   async connectToSandboxDirect(sandboxId, persist = false) {
-    this.emitter.emit(events.log.log, theme.gray(`- connecting...`));
+    this.emitter.emit(events.log.narration, theme.dim(`connecting...`));
     let instance = await this.sandbox.connect(sandboxId, persist);
     return instance;
   }

package/agent/lib/commander.js

@@ -80,20 +80,20 @@ commands:
       // this will actually interpret the command and execute it
       switch (object.command) {
         case "type":
-          emitter.emit(events.narration, `typing ${object.text}`);
+          emitter.emit(events.log.narration, `typing ${object.text}`);
           emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands.type(object.text, object.delay);
           break;
         case "press-keys":
           emitter.emit(
-            events.narration,
+            events.log.narration,
             `pressing keys ${object.keys.join(",")}`,
           );
           emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["press-keys"](object.keys);
           break;
         case "scroll":
-          emitter.emit(events.narration, `scrolling ${object.direction}`);
+          emitter.emit(events.log.narration, `scrolling ${object.direction}`);
           emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands.scroll(
             object.direction,
@@ -102,17 +102,20 @@ commands:
           );
           break;
         case "wait":
-          emitter.emit(events.narration, `waiting ${object.timeout} seconds`);
+          emitter.emit(
+            events.log.narration,
+            `waiting ${object.timeout} seconds`,
+          );
           emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands.wait(object.timeout);
           break;
         case "click":
-          emitter.emit(events.narration, `${object.action}`);
+          emitter.emit(events.log.narration, `${object.action}`);
           emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["click"](object.x, object.y, object.action);
           break;
         case "hover":
-          emitter.emit(events.narration, `moving mouse`);
+          emitter.emit(events.log.narration, `moving mouse`);
           emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["hover"](object.x, object.y);
           break;
@@ -121,7 +124,10 @@ commands:
           response = await commands["drag"](object.x, object.y);
           break;
         case "hover-text":
-          emitter.emit(events.narration, `searching for ${object.description}`);
+          emitter.emit(
+            events.log.narration,
+            `searching for ${object.description}`,
+          );
           emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["hover-text"](
             object.text,
@@ -132,7 +138,7 @@ commands:
           break;
         case "hover-image":
           emitter.emit(
-            events.narration,
+            events.log.narration,
             `searching for image of ${object.description}`,
           );
           emitter.emit(events.log.log, generator.jsonToManual(object));
@@ -144,13 +150,16 @@ commands:
         case "match-image":
           emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(
-            events.narration,
+            events.log.narration,
             `${object.action} image ${object.path}`,
           );
           response = await commands["match-image"](object.path, object.action);
           break;
         case "wait-for-image":
-          emitter.emit(events.narration, `waiting for ${object.description}`);
+          emitter.emit(
+            events.log.narration,
+            `waiting for ${object.description}`,
+          );
           emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["wait-for-image"](
             object.description,
@@ -158,7 +167,7 @@ commands:
           );
           break;
         case "wait-for-text":
-          emitter.emit(events.narration, `waiting for ${object.text}`);
+          emitter.emit(events.log.narration, `waiting for ${object.text}`);
           emitter.emit(events.log.log, generator.jsonToManual(object));
           copy.text = "*****";
           response = await commands["wait-for-text"](
@@ -168,7 +177,7 @@ commands:
           );
           break;
         case "scroll-until-text":
-          emitter.emit(events.narration, `scrolling until ${object.text}`);
+          emitter.emit(events.log.narration, `scrolling until ${object.text}`);
           emitter.emit(events.log.log, generator.jsonToManual(object));
           copy.text = "*****";
           response = await commands["scroll-until-text"](
@@ -181,7 +190,7 @@ commands:
           break;
         case "scroll-until-image": {
           const needle = object.description || object.path;
-          emitter.emit(events.narration, `scrolling until ${needle}`);
+          emitter.emit(events.log.narration, `scrolling until ${needle}`);
           emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["scroll-until-image"](
             object.description,
@@ -193,7 +202,7 @@ commands:
           break;
         }
         case "focus-application":
-          emitter.emit(events.narration, `focusing ${object.name}`);
+          emitter.emit(events.log.narration, `focusing ${object.name}`);
           emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["focus-application"](object.name);
           break;
@@ -205,12 +214,12 @@ commands:
           break;
         }
         case "assert":
-          emitter.emit(events.narration, `asserting ${object.expect}`);
+          emitter.emit(events.log.narration, `asserting ${object.expect}`);
           emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands.assert(object.expect, object.async);
           break;
         case "exec":
-          emitter.emit(events.narration, `exec`);
+          emitter.emit(events.log.narration, `exec`);
           emitter.emit(
             events.log.log,
             generator.jsonToManual({

package/agent/lib/commands.js

@@ -190,7 +190,7 @@ const createCommands = (
       }
     };
 
-    emitter.emit(events.log.log, `thinking...`);
+    emitter.emit(events.log.narration, `thinking...`);
 
     if (async) {
       await sdk
@@ -276,7 +276,7 @@ const createCommands = (
     }
 
     emitter.emit(
-      "log:debug",
+      events.log.narration,
       theme.dim(`${action} ${button} clicking at ${x}, ${y}...`),
       true,
     );
@@ -347,7 +347,7 @@ const createCommands = (
 
       description = description ? description.toString() : null;
 
-      emitter.emit(events.log.log, theme.dim("thinking..."), true);
+      emitter.emit(events.log.narration, theme.dim("thinking..."), true);
 
       let response = await sdk.req(
         "hover/text",
@@ -375,7 +375,7 @@ const createCommands = (
     // uses our api to find all images on screen
     "hover-image": async (description, action = "click") => {
       // take a screenshot
-      emitter.emit(events.log.log, theme.dim("thinking..."), true);
+      emitter.emit(events.log.narration, theme.dim("thinking..."), true);
 
       let response = await sdk.req(
         "hover/image",
@@ -446,7 +446,7 @@ const createCommands = (
     },
     "wait-for-image": async (description, timeout = 10000) => {
       emitter.emit(
-        events.log.log,
+        events.log.narration,
         theme.dim(
           `waiting for an image matching description "${description}"...`,
         ),
@@ -467,7 +467,7 @@ const createCommands = (
         durationPassed = new Date().getTime() - startTime;
         if (!passed) {
           emitter.emit(
-            events.log.log,
+            events.log.narration,
             theme.dim(
               `${niceSeconds(durationPassed)} seconds have passed without finding an image matching the description "${description}"`,
             ),
@@ -479,7 +479,7 @@ const createCommands = (
 
       if (passed) {
         emitter.emit(
-          events.log.log,
+          events.log.narration,
           theme.dim(
             `An image matching the description "${description}" found!`,
           ),
@@ -496,7 +496,7 @@ const createCommands = (
       await redraw.start();
 
       emitter.emit(
-        events.log.log,
+        events.log.narration,
         theme.dim(`waiting for text: "${text}"...`),
         true,
       );
@@ -525,7 +525,7 @@ const createCommands = (
         durationPassed = new Date().getTime() - startTime;
         if (!passed) {
           emitter.emit(
-            events.log.log,
+            events.log.narration,
             theme.dim(
               `${niceSeconds(durationPassed)} seconds have passed without finding "${text}"`,
             ),
@@ -536,7 +536,7 @@ const createCommands = (
       }
 
       if (passed) {
-        emitter.emit(events.log.log, theme.dim(`"${text}" found!`), true);
+        emitter.emit(events.log.narration, theme.dim(`"${text}" found!`), true);
         return;
       } else {
         throw new MatchError(
@@ -554,7 +554,7 @@ const createCommands = (
       await redraw.start();
 
       emitter.emit(
-        events.log.log,
+        events.log.narration,
         theme.dim(`scrolling for text: "${text}"...`),
         true,
       );
@@ -598,7 +598,7 @@ const createCommands = (
         passed = response.data;
         if (!passed) {
           emitter.emit(
-            events.log.log,
+            events.log.narration,
             theme.dim(
               `scrolling ${direction} ${incrementDistance}px. ${scrollDistance + incrementDistance}/${maxDistance}px scrolled...`,
             ),
@@ -610,7 +610,7 @@ const createCommands = (
       }
 
       if (passed) {
-        emitter.emit(events.log.log, theme.dim(`"${text}" found!`), true);
+        emitter.emit(events.log.narration, theme.dim(`"${text}" found!`), true);
         return;
       } else {
         throw new MatchError(
@@ -638,7 +638,7 @@ const createCommands = (
       }
 
       emitter.emit(
-        events.log.log,
+        events.log.narration,
         theme.dim(`scrolling for an image matching "${needle}"...`),
         true,
       );
@@ -665,7 +665,7 @@ const createCommands = (
 
         if (!passed) {
           emitter.emit(
-            events.log.log,
+            events.log.narration,
             theme.dim(`scrolling ${direction} ${incrementDistance} pixels...`),
             true,
           );
@@ -675,7 +675,11 @@ const createCommands = (
       }
 
       if (passed) {
-        emitter.emit(events.log.log, theme.dim(`"${needle}" found!`), true);
+        emitter.emit(
+          events.log.narration,
+          theme.dim(`"${needle}" found!`),
+          true,
+        );
         return;
       } else {
         throw new CommandError(
@@ -705,7 +709,7 @@ const createCommands = (
       return await assert(assertion, true, async);
     },
     exec: async (language, code, timeout, silent = false) => {
-      emitter.emit(events.log.log, theme.dim(`calling exec...`), true);
+      emitter.emit(events.log.narration, theme.dim(`calling exec...`), true);
 
       emitter.emit(events.log.log, code);
 
@@ -738,10 +742,10 @@ const createCommands = (
           return result.out?.stdout?.trim();
         }
       } else if (language == "js") {
-        emitter.emit(events.log.log, theme.dim(`running js...`), true);
+        emitter.emit(events.log.narration, theme.dim(`running js...`), true);
 
         emitter.emit(
-          events.log.log,
+          events.log.narration,
           theme.dim(`running value of \`${plat}\` in local JS vm...`),
           true,
         );

package/agent/lib/sandbox.js

@@ -84,10 +84,6 @@ const createSandbox = (emitter, analytics) => {
         this.socket.on("close", () => {
           clearInterval(this.heartbeat);
           // Emit a clear error event for API key issues
-          emitter.emit(events.error.fatal, {
-            message: "Socket closed. Check your API KEY (TD_API_KEY)",
-            code: "API_KEY_MISSING_OR_INVALID",
-          });
           reject();
           this.apiSocketConnected = false;
         });

package/docs/docs.json

@@ -220,8 +220,8 @@
         "href": "https://github.com/testdriverai/testdriverai"
       },
       {
-        "label": "Forums",
-        "href": "https://forums.testdriver.ai"
+        "label": "Discord",
+        "href": "https://discord.com/invite/cWDFW8DzPm"
       },
       {
         "label": "Report an Issue",

package/docs/guide/protips.mdx

@@ -13,6 +13,7 @@ When executing the `hover-image` and `hover-text` command, TestDriver may use th
 
 TestDriver may not accurately locate matches in certain cases. This can lead to unexpected behavior when using the `hover-image` or `hover-text` command. For example, if you specify a description that's too generic or similar to other images on the screen, TestDriver may not be able to identify the correct image to interact with. To improve accuracy, use specific and unique descriptions for images and text.
 When executing the `hover-image` and `hover-text` commands, TestDriver uses the provided description to identify the target image or text. If the description is too generic or matches multiple elements on the screen, TestDriver may not be able to determine which one to interact with, leading to unexpected behavior. To improve accuracy, always use specific and unique descriptions for both images and text.
+
 ### Avoid using `hover-text` for single characters (ex: `1`, `>`, `|`)
 
 To improve accuracy of our model, we do not include single characters or symbols in the matching data. This means that `hover-text` may not work as expected for these cases. If you need to interact with single characters or symbols, consider using the `match-image` command with a screenshot of the target element instead.
@@ -29,14 +30,14 @@ Assertions use the current state of the page to determine if the step passes or
 
 ## Reporting issues
 
-If you encounter any issues while using TestDriver, please report them to us. You can do this by creating a new issue in the [Feedback Forum](https://forums.testdriver.ai/c/feedback/10). We appreciate your feedback and will work to resolve any problems as quickly as possible.
+If you encounter any issues while using TestDriver, please report them to us. You can do this by creating a new issue in the [discord](https://discord.com/invite/cWDFW8DzPm). We appreciate your feedback and will work to resolve any problems as quickly as possible.
 
 <Card
   title="Report an Issue"
   icon="arrow-turn-down-right"
   iconType="duotone"
   horizontal
-  href="https://forums.testdriver.ai/c/feedback/10"
+  href="https://discord.com/invite/cWDFW8DzPm"
 >
   We'd love to hear from you!
 </Card>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "6.0.16-canary.e337e67.0",
+  "version": "6.0.16-canary.f0eefe0.0",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {

package/curl.sh

@@ -1,13 +0,0 @@
-curl -X POST "https://forums.testdriver.ai/users" \
-  -H "Content-Type: application/json" \
-  -H "Api-Key: YOUR_ADMIN_API_KEY" \
-  -H "Api-Username: admin" \
-  -d '{
-    "name": "Jane Doe",
-    "email": "jane.doe@example.com",
-    "username": "janedoe",
-    "active": false,
-    "approved": true,
-    "email_verified": false,
-    "staged": false
-  }'

package/debug-ast.js

@@ -1,38 +0,0 @@
-#!/usr/bin/env node
-
-const yamlAst = require("@stoplight/yaml-ast-parser");
-const yaml = require("js-yaml");
-const fs = require("fs");
-
-function debugAST() {
-  const testFile =
-    "/Users/ianjennings/Development/testdriverai/test-source-mapping.yaml";
-  const yamlContent = fs.readFileSync(testFile, "utf-8");
-
-  console.log("YAML Content:");
-  console.log(yamlContent);
-  console.log("\n" + "=".repeat(50) + "\n");
-
-  const ast = yamlAst.load(yamlContent);
-
-  console.log("AST Structure:");
-  console.log(
-    JSON.stringify(
-      ast,
-      (key, value) => {
-        if (key === "parent") return "[parent]"; // Avoid circular reference
-        return value;
-      },
-      2,
-    ),
-  );
-
-  console.log("\n" + "=".repeat(50) + "\n");
-
-  // Test with js-yaml for comparison
-  const yamlObj = yaml.load(yamlContent);
-  console.log("js-yaml result:");
-  console.log(JSON.stringify(yamlObj, null, 2));
-}
-
-debugAST();