wtagent 0.4.0 → 1.0.0

package/src/image/providers/chatgpt-image-driver.js

@@ -0,0 +1,211 @@
+import fs from "node:fs/promises";
+import { BrowserImageDriver } from "../browser-image-driver.js";
+
+const DOWNLOAD_NAME_PATTERN = /download(?: image)?|下载(?:图片)?|下載(?:圖片)?/i;
+const MIN_IMAGE_EDGE = 256;
+
+export function preferredImageUrl(image) {
+  const candidates = [image?.source, image?.href]
+    .filter((value) => typeof value === "string" && value.length > 0);
+  if (candidates.length === 0) {
+    return null;
+  }
+  const durableHttps = candidates.find((value) => {
+    try {
+      const parsed = new URL(value);
+      return parsed.protocol === "https:"
+        && !parsed.hostname.endsWith("chatgpt.com");
+    } catch {
+      return false;
+    }
+  });
+  return durableHttps
+    ?? candidates.find((value) => value.startsWith("https://"))
+    ?? candidates[0];
+}
+
+export class ChatGPTImageDriver extends BrowserImageDriver {
+  capabilities() {
+    return Object.freeze({
+      generate: true,
+      referenceImages: true,
+      aspectRatios: ["auto", "1:1", "3:2", "2:3", "16:9", "9:16"],
+      imageCount: 1,
+    });
+  }
+
+  async generate(request, execution = {}) {
+    await this.ensureChatPage();
+    const baseline = new Set(
+      (await this.chatImages().catch(() => [])).map((image) => preferredImageUrl(image)),
+    );
+    return await this.performTurn(request, execution, async () => {
+      const image = await this.waitForChatImage(request.timeoutMs, baseline);
+      if (image.type === "text") return image;
+      const downloaded = await this.downloadChatImage(
+        image,
+        Math.min(request.timeoutMs, 60_000),
+      );
+      return {
+        sourcePath: downloaded.temporaryPath,
+        cleanup: async () => fs.rm(downloaded.temporaryDir, { recursive: true, force: true }),
+        provenance: {
+          provider: this.provider,
+          providerFileId: this.remoteFileId(image.source)
+            ?? this.remoteFileId(downloaded.downloadUrl),
+          suggestedFilename: downloaded.suggestedFilename,
+        },
+      };
+    });
+  }
+
+  async imagesFromLocator(locator) {
+    if (await locator.count().catch(() => 0) === 0) return [];
+    return await locator.evaluate((root, minEdge) => {
+      const toAbsolute = (value) => {
+        if (!value) return "";
+        try {
+          return new URL(value, location.href).href;
+        } catch {
+          return value;
+        }
+      };
+      const fromSrcset = (srcset) => {
+        let best = "";
+        let bestWidth = 0;
+        for (const part of String(srcset ?? "").split(",")) {
+          const [url, descriptor] = part.trim().split(/\s+/, 2);
+          const width = descriptor?.endsWith("w")
+            ? Number.parseInt(descriptor, 10)
+            : 0;
+          if (url && width >= bestWidth) {
+            best = url;
+            bestWidth = width;
+          }
+        }
+        return best;
+      };
+      const images = [...root.querySelectorAll("img")].map((image) => {
+        const source = toAbsolute(
+          image.currentSrc
+          || fromSrcset(image.srcset)
+          || image.src
+          || image.getAttribute("data-src")
+          || "",
+        );
+        const href = toAbsolute(image.closest("a")?.href || "");
+        return {
+          source,
+          href,
+          width: image.naturalWidth || 0,
+          height: image.naturalHeight || 0,
+        };
+      }).filter((image) => (
+        (image.source || image.href)
+        && image.width >= minEdge
+        && image.height >= minEdge
+      ));
+
+      for (const link of root.querySelectorAll("a[href]")) {
+        const href = toAbsolute(link.href);
+        if (
+          !href
+          || images.some((image) => image.href === href || image.source === href)
+        ) {
+          continue;
+        }
+        if (
+          /oaiusercontent/i.test(href)
+          || /\.(png|jpe?g|webp)(?:\?|$)/i.test(href)
+        ) {
+          images.push({
+            source: href,
+            href,
+            width: minEdge,
+            height: minEdge,
+          });
+        }
+      }
+      return images;
+    }, MIN_IMAGE_EDGE).catch(() => []);
+  }
+
+  async currentAssistantImages() {
+    return await this.imagesFromLocator(this.adapter.assistantMessages().last());
+  }
+
+  async chatImages() {
+    return await this.imagesFromLocator(
+      this.adapter.page.locator("main").first(),
+    );
+  }
+
+  async waitForChatImage(timeoutMs, baseline = new Set()) {
+    const deadline = Date.now() + timeoutMs;
+    let candidate = null;
+    let stableSince = 0;
+    const textState = {};
+    while (Date.now() < deadline) {
+      const images = (await this.currentAssistantImages().catch(() => []))
+        .filter((image) => !baseline.has(preferredImageUrl(image)));
+      if (images.length > 1) {
+        throw new Error(
+          "ChatGPT produced multiple uncorrelated chat images; refusing to guess which artifact belongs to this request.",
+        );
+      }
+      if (images.length === 1) {
+        const next = preferredImageUrl(images[0]);
+        if (candidate !== next) {
+          candidate = next;
+          stableSince = Date.now();
+        } else if (Date.now() - stableSince >= 1_500) {
+          return { ...images[0], source: next };
+        }
+      } else {
+        candidate = null;
+        stableSince = 0;
+      }
+      if (images.length === 0) {
+        const text = await this.completedTextResponse(textState);
+        if (text) return text;
+      } else {
+        textState.candidate = null;
+      }
+      await this.adapter.page.waitForTimeout(1_000);
+    }
+    throw new Error(`ChatGPT did not publish a new image within ${Math.round(timeoutMs / 1000)} seconds.`);
+  }
+
+  async downloadChatImage(image, timeoutMs) {
+    await this.adapter.page.bringToFront().catch(() => null);
+    const url = preferredImageUrl(image);
+    if (url) {
+      try {
+        return await this.saveFromUrl(this.adapter.page, url, timeoutMs);
+      } catch {
+        // The in-chat URL can be a preview or expire; try a download control next.
+      }
+    }
+
+    const response = this.adapter.assistantMessages().last();
+    const buttonCandidates = [
+      response.getByRole("button", { name: DOWNLOAD_NAME_PATTERN }),
+      response.locator('button[aria-label*="download" i]'),
+    ];
+    for (const locator of buttonCandidates) {
+      if (await locator.first().isVisible().catch(() => false)) {
+        try {
+          return await this.saveDownload(
+            this.adapter.page,
+            locator.first(),
+            Math.min(timeoutMs, 30_000),
+          );
+        } catch {
+          break;
+        }
+      }
+    }
+    throw new Error("ChatGPT generated an image but no download control or image URL was found.");
+  }
+
+}

package/src/image/providers/gemini-image-driver.js

@@ -0,0 +1,100 @@
+import fs from "node:fs/promises";
+import { BrowserImageDriver } from "../browser-image-driver.js";
+
+export class GeminiImageDriver extends BrowserImageDriver {
+  capabilities() {
+    return Object.freeze({
+      generate: true,
+      referenceImages: true,
+      aspectRatios: ["auto", "1:1", "3:2", "2:3", "4:3", "3:4", "16:9", "9:16"],
+      imageCount: 1,
+    });
+  }
+
+  async generatedImages() {
+    const response = this.adapter.assistantMessages().last();
+    if (await response.count().catch(() => 0) === 0) return [];
+    return await response.locator("img").evaluateAll((images) => images.map((image) => ({
+      source: image.currentSrc || image.src || "",
+      alt: image.alt || "",
+      width: image.naturalWidth || 0,
+      height: image.naturalHeight || 0,
+    })).filter((image) => image.source && image.width >= 256 && image.height >= 256));
+  }
+
+  async waitForGeneratedImage(timeoutMs, baseline = new Set()) {
+    const deadline = Date.now() + timeoutMs;
+    let stableSource = null;
+    let stableSince = 0;
+    const textState = {};
+    while (Date.now() < deadline) {
+      const images = (await this.generatedImages())
+        .filter((image) => !baseline.has(image.source));
+      if (images.length === 1) {
+        if (stableSource !== images[0].source) {
+          stableSource = images[0].source;
+          stableSince = Date.now();
+        } else if (Date.now() - stableSince >= 2_000) {
+          return images[0];
+        }
+      } else if (images.length > 1) {
+        throw new Error(
+          "Gemini returned multiple images for a single-output request; refusing to choose one implicitly.",
+        );
+      }
+      if (images.length === 0) {
+        const text = await this.completedTextResponse(textState);
+        if (text) return text;
+      } else {
+        textState.candidate = null;
+      }
+      await this.adapter.page.waitForTimeout(1_000);
+    }
+    throw new Error(`Gemini did not finish an image within ${Math.round(timeoutMs / 1000)} seconds.`);
+  }
+
+  async downloadGeneratedImage(timeoutMs, generatedImage = null) {
+    const response = this.adapter.assistantMessages().last();
+    const scope = response;
+    const buttonCandidates = [
+      scope.getByRole("button", { name: /download(?: image)?|下载(?:图片)?|下載(?:圖片)?/i }),
+      scope.locator('button[aria-label*="download" i]'),
+      scope.locator('button[data-test-id*="download" i]'),
+    ];
+    for (const locator of buttonCandidates) {
+      if (await locator.first().isVisible().catch(() => false)) {
+        try {
+          return await this.saveDownload(this.adapter.page, locator.first(), timeoutMs);
+        } catch {
+          break;
+        }
+      }
+    }
+    if (generatedImage?.source) {
+      return await this.saveFromUrl(this.adapter.page, generatedImage.source, timeoutMs);
+    }
+    throw new Error("Gemini generated an image but no image download control was found.");
+  }
+
+  async generate(request, execution = {}) {
+    await this.ensureChatPage();
+    const before = new Set((await this.generatedImages()).map((item) => item.source));
+    return await this.performTurn(request, execution, async () => {
+      const generatedImage = await this.waitForGeneratedImage(request.timeoutMs, before);
+      if (generatedImage.type === "text") return generatedImage;
+      const downloaded = await this.downloadGeneratedImage(
+        Math.min(request.timeoutMs, 60_000),
+        generatedImage,
+      );
+      return {
+        sourcePath: downloaded.temporaryPath,
+        cleanup: async () => fs.rm(downloaded.temporaryDir, { recursive: true, force: true }),
+        provenance: {
+          provider: this.provider,
+          providerFileId: this.remoteFileId(downloaded.downloadUrl),
+          suggestedFilename: downloaded.suggestedFilename,
+        },
+      };
+    });
+  }
+}

package/src/image/providers/grok-image-driver.js

@@ -0,0 +1,193 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { BrowserImageDriver } from "../browser-image-driver.js";
+
+const GROK_ASSET_HOST = "assets.grok.com";
+const GROK_GENERATED_IMAGE_PATH = /^\/users\/[^/]+\/generated\/([0-9a-f-]{16,})\/image\.(?:jpe?g|png|webp)$/i;
+const IMAGE_CONTENT_TYPE = /^image\/(?:jpeg|jpg|png|webp)(?:;|$)/i;
+
+export function parseGrokGeneratedImageUrl(value) {
+  try {
+    const url = new URL(value);
+    if (url.protocol !== "https:" || url.hostname !== GROK_ASSET_HOST) return null;
+    const match = url.pathname.match(GROK_GENERATED_IMAGE_PATH);
+    if (!match) return null;
+    return {
+      id: match[1],
+      key: `${url.origin}${url.pathname}`,
+      url: url.href,
+    };
+  } catch {
+    return null;
+  }
+}
+
+function responseUrl(response) {
+  return typeof response?.url === "function" ? response.url() : response?.url;
+}
+
+function responseHeaders(response) {
+  return typeof response?.headers === "function" ? response.headers() : response?.headers;
+}
+
+export class GrokImageDriver extends BrowserImageDriver {
+  constructor(options) {
+    super(options);
+    this.responseStabilityMs = options.responseStabilityMs ?? 2_000;
+  }
+
+  capabilities() {
+    return Object.freeze({
+      generate: true,
+      referenceImages: false,
+      aspectRatios: ["auto", "1:1", "3:2", "2:3", "16:9", "9:16"],
+      imageCount: 1,
+    });
+  }
+
+  trackGeneratedImageResponses() {
+    const page = this.adapter.page;
+    const images = [];
+    const keys = new Set();
+    let armed = false;
+    let lastAddedAt = 0;
+    const listener = (response) => {
+      if (!armed) return;
+      const parsed = parseGrokGeneratedImageUrl(responseUrl(response));
+      if (!parsed || keys.has(parsed.key)) return;
+      const ok = typeof response?.ok === "function" ? response.ok() : response?.ok;
+      const contentType = String(responseHeaders(response)?.["content-type"] ?? "");
+      if (!ok || !IMAGE_CONTENT_TYPE.test(contentType)) return;
+      keys.add(parsed.key);
+      images.push(parsed);
+      lastAddedAt = Date.now();
+    };
+    page.on("response", listener);
+    return {
+      arm: () => { armed = true; },
+      images: () => images.slice(),
+      lastAddedAt: () => lastAddedAt,
+      stop: () => page.off("response", listener),
+    };
+  }
+
+  async currentAssistantImages() {
+    const response = this.adapter.assistantMessages().last();
+    if (await response.count().catch(() => 0) === 0) return [];
+    return await response.locator("img").evaluateAll((images) => images.map((image) => ({
+      source: image.currentSrc || image.src || "",
+      alt: image.alt || "",
+      width: image.naturalWidth || 0,
+      height: image.naturalHeight || 0,
+    })).filter((image) => image.source && image.width >= 256 && image.height >= 256));
+  }
+
+  async waitForGeneratedImage(timeoutMs, tracker, baseline = new Set()) {
+    const deadline = Date.now() + timeoutMs;
+    let candidateKey = null;
+    let candidateSince = 0;
+    const textState = {};
+    while (Date.now() < deadline) {
+      const responses = tracker.images();
+      const responseByKey = new Map(responses.map((image) => [image.key, image]));
+      const rendered = (await this.currentAssistantImages().catch(() => []))
+        .map((image) => ({ ...image, parsed: parseGrokGeneratedImageUrl(image.source) }))
+        .filter((image) => (
+          image.parsed
+          && !baseline.has(image.parsed.key)
+          && responseByKey.has(image.parsed.key)
+        ));
+      const keys = [...new Set(rendered.map((image) => image.parsed.key))];
+      if (
+        keys.length > 0
+        && Date.now() - tracker.lastAddedAt() >= this.responseStabilityMs
+      ) {
+        const signature = keys.join("|");
+        if (candidateKey !== signature) {
+          candidateKey = signature;
+          candidateSince = Date.now();
+        } else if (Date.now() - candidateSince >= this.responseStabilityMs) {
+          const parsed = responseByKey.get(keys[0]);
+          return {
+            source: parsed.url,
+            providerFileId: parsed.id,
+            generatedImageCount: keys.length,
+            selectedImageIndex: 1,
+          };
+        }
+      }
+      if (keys.length === 0) {
+        const text = await this.completedTextResponse(textState);
+        if (text) return text;
+      } else {
+        textState.candidate = null;
+      }
+      await this.adapter.page.waitForTimeout(500);
+    }
+    throw new Error(`Grok did not finish an image within ${Math.round(timeoutMs / 1000)} seconds.`);
+  }
+
+  async downloadGeneratedImage(image) {
+    const temporaryDir = await fs.mkdtemp(path.join(os.tmpdir(), "wtagent-image-"));
+    const sourceUrl = new URL(image.source);
+    const extension = path.extname(sourceUrl.pathname).toLowerCase() || ".jpg";
+    const temporaryPath = path.join(temporaryDir, `generated-image${extension}`);
+    try {
+      const response = await this.adapter.page.request.get(image.source, {
+        timeout: 60_000,
+      });
+      if (!response.ok()) {
+        throw new Error(`Grok image download failed with HTTP ${response.status()}.`);
+      }
+      await fs.writeFile(temporaryPath, await response.body());
+      return {
+        temporaryDir,
+        temporaryPath,
+        suggestedFilename: path.basename(sourceUrl.pathname) || `generated-image${extension}`,
+        downloadUrl: image.source,
+      };
+    } catch (error) {
+      await fs.rm(temporaryDir, { recursive: true, force: true }).catch(() => null);
+      throw error;
+    }
+  }
+
+  async generate(request, execution = {}) {
+    if (request.referenceImages.length > 0) {
+      throw new Error("The current Grok image driver does not support reference images.");
+    }
+    await this.ensureChatPage();
+    const baseline = new Set(
+      (await this.currentAssistantImages().catch(() => []))
+        .map((image) => parseGrokGeneratedImageUrl(image.source)?.key)
+        .filter(Boolean),
+    );
+    const tracker = this.trackGeneratedImageResponses();
+    try {
+      tracker.arm();
+      return await this.performTurn(request, execution, async () => {
+        const image = await this.waitForGeneratedImage(
+          request.timeoutMs,
+          tracker,
+          baseline,
+        );
+        if (image.type === "text") return image;
+        const downloaded = await this.downloadGeneratedImage(image);
+        return {
+          sourcePath: downloaded.temporaryPath,
+          cleanup: async () => fs.rm(downloaded.temporaryDir, { recursive: true, force: true }),
+          provenance: {
+            provider: this.provider,
+            providerFileId: image.providerFileId,
+            suggestedFilename: downloaded.suggestedFilename,
+            generatedImageCount: image.generatedImageCount,
+            selectedImageIndex: image.selectedImageIndex,
+          },
+        };
+      });
+    } finally {
+      tracker.stop();
+    }
+  }
+}

package/src/policy/policy-engine.js

@@ -9,6 +9,7 @@ const PATH_FIELDS = Object.freeze({
   "fs.search": ["path"],
   "terminal.exec": ["cwd"],
   "process.start": ["cwd"],
+  "image.generate": ["output_path", "reference_images"],
 });
 
 const PRIVILEGED_PROGRAMS = new Set(["sudo", "su", "doas", "runas"]);
@@ -268,11 +269,15 @@ export class PolicyEngine {
     let allowOutside = false;
 
     for (const field of PATH_FIELDS[toolCall.name] ?? []) {
-      const rawPath = toolCall.args[field] || ".";
-      const resolved = await resolveToolPath(context.projectRoot, rawPath);
-      if (!resolved.inside) {
-        reasons.push(`${field} is outside the selected project: ${resolved.path}`);
-        allowOutside = true;
+      const values = Array.isArray(toolCall.args[field])
+        ? toolCall.args[field]
+        : [toolCall.args[field] || "."];
+      for (const rawPath of values) {
+        const resolved = await resolveToolPath(context.projectRoot, rawPath);
+        if (!resolved.inside) {
+          reasons.push(`${field} is outside the selected project: ${resolved.path}`);
+          allowOutside = true;
+        }
       }
     }
 

package/src/protocol/envelope.js

@@ -0,0 +1,74 @@
+// Locate transport tags without mistaking quoted attributes, comments, or
+// CDATA payloads for markup. This is a boundary scanner, not an XML validator.
+// Malformed markup is still passed to the strict protocol parser.
+export function* xmlTags(value) {
+  const text = String(value ?? "");
+  let index = 0;
+  while ((index = text.indexOf("<", index)) >= 0) {
+    const start = index;
+    const special = text.startsWith("<![CDATA[", index)
+      ? [9, "]]>"]
+      : text.startsWith("<!--", index)
+        ? [4, "-->"]
+        : text.startsWith("<?", index) ? [2, "?>"] : null;
+    if (special) {
+      const end = text.indexOf(special[1], index + special[0]);
+      if (end < 0) return;
+      index = end + special[1].length;
+      continue;
+    }
+    const match = /^<(\/)?([A-Za-z_][\w:.-]*)(?=[\s/>])/.exec(text.slice(index));
+    if (!match) {
+      index += 1;
+      continue;
+    }
+    index += match[0].length;
+    let quote = null;
+    for (; index < text.length; index += 1) {
+      const character = text[index];
+      if (quote) {
+        if (character === quote) quote = null;
+      } else if (character === '"' || character === "'") {
+        quote = character;
+      } else if (character === "<" || character === ">") {
+        break;
+      }
+    }
+    // Keep an unfinished opening tag visible to the boundary finder; otherwise
+    // a tool inside a malformed outer root could be mistaken for a bare call.
+    if (index >= text.length || text[index] === "<") {
+      yield { name: match[2], closing: Boolean(match[1]), selfClosing: false, start, end: null };
+      if (index >= text.length) return;
+      continue;
+    }
+    index += 1;
+    yield {
+      name: match[2],
+      closing: Boolean(match[1]),
+      selfClosing: /\/\s*>$/.test(text.slice(start, index)),
+      start,
+      end: index,
+    };
+  }
+}
+
+export function findXmlElement(text, name = "agent_response") {
+  let start = null;
+  let depth = 0;
+  for (const tag of xmlTags(text)) {
+    if (tag.name !== name) continue;
+    if (start == null) {
+      if (tag.closing) continue;
+      start = tag.start;
+    }
+    if (tag.end == null) return { start, end: null };
+    if (tag.closing) depth -= 1;
+    else if (!tag.selfClosing) depth += 1;
+    if (depth === 0) return { start, end: tag.end };
+  }
+  return start == null ? null : { start, end: null };
+}
+
+export function hasCompleteAgentEnvelope(text) {
+  return findXmlElement(text)?.end != null;
+}

package/src/protocol/markers.js

@@ -21,6 +21,22 @@ export const DEFAULT_SYSTEM_REMINDER = [
   "Write large code or file payloads as CDATA (<![CDATA[...]]>) inside <content>, <new_text>, or <message> — never raw code directly inside the envelope — and never include the sequence ]]> inside CDATA content.",
 ].join(" ");
 
+// A native image is an assistant response, never a local XML tool call.
+export const NATIVE_IMAGE_CONTRACT = [
+  "When the task needs generated artwork, generate and output the image directly using your native image-generation capability.",
+  "Image responses are exempt from the XML format: output the image itself, without an XML wrapper or drawing code.",
+  "The Runtime automatically downloads images from your completed reply and returns verified local paths; then continue the user's task using those paths.",
+  "There is no local image-generation tool. This replaces any image-tool instructions from earlier in this conversation.",
+  "If generation returns only text, that text will be returned as the provider result so you can explain the limitation or ask the user; do not claim an image was saved.",
+].join(" ");
+
+export const NATIVE_IMAGE_SYSTEM_REMINDER = [
+  NATIVE_IMAGE_CONTRACT,
+  "For text answers and local operations, use exactly one complete <agent_response> XML envelope in one xml code fence.",
+  "Use done=true for a deliverable answer or a question that requires user input; use done=false with one local tool request when needed.",
+  "Use CDATA for code or long payloads. Local tools execute only after the Runtime returns their results.",
+].join(" ");
+
 export function wrapSystemPrompt(text) {
   return `<${SYSTEM_PROMPT_TAG}>\n${String(text ?? "")}\n</${SYSTEM_PROMPT_TAG}>`;
 }