wtagent 0.3.0 → 1.0.0

package/src/cli/prompt-input.js

@@ -849,6 +849,74 @@ export async function promptForText(config, options = {}) {
   });
 }
 
+// A non-permission startup pause: Enter and expiry both continue. Keep this
+// separate from text/approval prompts, which must never accept on a timer.
+export async function promptForContinue({ message, countdown, timeoutMs = 5_000 }, {
+  inputStream = process.stdin,
+  outputStream = process.stdout,
+} = {}) {
+  if (inputStream.readableEnded || inputStream.destroyed) return null;
+  return await new Promise((resolve, reject) => {
+    const previousRaw = Boolean(inputStream.isRaw);
+    const deadline = Date.now() + timeoutMs;
+    let timer;
+    let settled = false;
+    const render = (seconds) => {
+      const text = countdown(seconds);
+      if (outputStream.isTTY) {
+        // Keep the changing status on one physical terminal row.
+        let visible = "";
+        for (const ch of text) {
+          if (displayWidth(visible + ch) >= (outputStream.columns || 80)) break;
+          visible += ch;
+        }
+        outputStream.write(`\r\u001b[2K${visible}`);
+      } else {
+        outputStream.write(`${text}\n`);
+      }
+    };
+    const finish = (value, error) => {
+      if (settled) return;
+      settled = true;
+      clearInterval(timer);
+      inputStream.removeListener("data", onData);
+      inputStream.removeListener("end", onEnd);
+      inputStream.removeListener("close", onEnd);
+      inputStream.removeListener("error", onError);
+      inputStream.setRawMode?.(previousRaw);
+      inputStream.pause?.();
+      if (value) render(0);
+      if (outputStream.isTTY) outputStream.write("\n");
+      if (error) reject(error);
+      else resolve(value);
+    };
+    const onData = (chunk) => {
+      const text = String(chunk);
+      if (/[\u0003\u0004]/.test(text)) finish(null);
+      else if (/[\r\n]/.test(text)) finish(true);
+    };
+    const onEnd = () => finish(null);
+    const onError = (error) => finish(null, error);
+    inputStream.on("data", onData);
+    inputStream.once("end", onEnd);
+    inputStream.once("close", onEnd);
+    inputStream.once("error", onError);
+    try {
+      inputStream.setRawMode?.(true);
+      outputStream.write(`${message}\n`);
+      render(Math.ceil(timeoutMs / 1_000));
+      timer = setInterval(() => {
+        const seconds = Math.max(0, Math.ceil((deadline - Date.now()) / 1_000));
+        if (seconds === 0) finish(true);
+        else render(seconds);
+      }, 1_000);
+      inputStream.resume?.();
+    } catch (error) {
+      finish(null, error);
+    }
+  });
+}
+
 export async function promptForSelect(config, options = {}) {
   return await promptWithCleanExit(config, {
     ...options,

package/src/image/browser-image-driver.js

@@ -0,0 +1,238 @@
+import fs from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import os from "node:os";
+import path from "node:path";
+import { ImageProviderDriver } from "./image-provider-driver.js";
+import { appendSystemReminder } from "../protocol/markers.js";
+
+const IMAGE_OUTBOUND_CORRELATION_PREFIX =
+  " Opaque WTAgent transport correlation ID (do not repeat): ";
+
+function parseRemoteFileId(value) {
+  try {
+    return new URL(value).searchParams.get("id");
+  } catch {
+    return null;
+  }
+}
+
+function extensionForContentType(contentType, fallback = ".png") {
+  const type = String(contentType ?? "").toLowerCase();
+  if (type.includes("jpeg") || type.includes("jpg")) return ".jpg";
+  if (type.includes("webp")) return ".webp";
+  if (type.includes("png")) return ".png";
+  return fallback;
+}
+
+export class BrowserImageDriver extends ImageProviderDriver {
+  async ensureChatPage() {
+    if (!this.adapter.page) {
+      throw new Error(`${this.provider} chat page is unavailable.`);
+    }
+    const kind = this.adapter.classifyConversationUrl?.(this.adapter.page.url());
+    if (!["fresh", "restorable", "provisional"].includes(kind)) {
+      throw new Error(`${this.provider} is not on a normal chat page.`);
+    }
+    const deadline = Date.now() + 30_000;
+    while (Date.now() < deadline) {
+      if (await this.composer()) return;
+      await this.adapter.page.waitForTimeout(500);
+    }
+    throw new Error(`${this.provider} chat composer is unavailable.`);
+  }
+
+  async composer() {
+    for (const locator of this.adapter.composerLocators()) {
+      if (await locator.isVisible().catch(() => false)) return locator;
+    }
+    return null;
+  }
+
+  promptText(request) {
+    const lines = [
+      "Generate exactly one image now. Use your native image-generation capability and output the image itself. Do not write drawing code or respond with only a text description.",
+      `Image requirements:\n${request.prompt}`,
+    ];
+    if (request.aspectRatio !== "auto") {
+      lines.push(`Aspect ratio: ${request.aspectRatio}.`);
+    }
+    return lines.join("\n\n");
+  }
+
+  // Poll alongside image detection, without consuming the assistant boundary.
+  // The normal auxiliary-turn completion path still verifies/checkpoints it.
+  // A refusal, clarification, or other finished text is a provider result too;
+  // do not classify it using language-specific refusal keywords.
+  async completedTextResponse(state) {
+    const adapter = this.adapter;
+    if (typeof adapter.isNewAssistantIdentity !== "function") return null;
+    const reset = () => { state.candidate = null; return null; };
+    const messages = adapter.assistantMessages();
+    if (await messages.count() === 0) return reset();
+    const message = messages.last();
+    const identity = await adapter.messageIdentity(message);
+    const text = await adapter.assistantText(message);
+    if (!text.trim() || !adapter.isNewAssistantIdentity({ ...identity, text })) {
+      return reset();
+    }
+    if (await adapter.isAssistantGenerating(message)) return reset();
+    for (const locator of adapter.stopButtonLocators()) {
+      if (await locator.first().isVisible()) return reset();
+    }
+    // An image can mount before its pixels/URL are ready. Do not return a
+    // caption while that image is still loading or awaiting correlation.
+    const hasImage = await message.locator("img").evaluateAll((images) => images.some(
+      (image) => !image.complete || (image.naturalWidth >= 256 && image.naturalHeight >= 256),
+    ));
+    if (hasImage) return reset();
+    const key = JSON.stringify([identity.id, identity.turn, text]);
+    if (state.candidate !== key) {
+      state.candidate = key;
+      state.since = Date.now();
+      return null;
+    }
+    const stableMs = adapter.hasReliableCompletionSignal()
+      ? 2_000
+      : Math.max(10_000, adapter.truncatedEnvelopeGraceMs());
+    if (Date.now() - state.since < stableMs) return null;
+    return {
+      type: "text",
+      text,
+      provenance: {
+        provider: this.provider,
+        assistantMessageId: identity.id ?? null,
+        assistantTurn: identity.turn ?? null,
+      },
+    };
+  }
+
+  async performTurn(request, execution, waitForCompletion) {
+    const prompt = this.promptText(request);
+    if (this.sharedConversation) {
+      if (typeof execution?.runAuxiliaryTurn !== "function") {
+        throw new Error(
+          `${this.provider} image generation in the active chat requires Runtime auxiliary-turn support.`,
+        );
+      }
+      return await execution.runAuxiliaryTurn({
+        text: prompt,
+        files: request.referenceImages,
+        timeoutMs: request.timeoutMs,
+        waitForCompletion,
+      });
+    }
+
+    const outboundId = randomUUID();
+    const message = appendSystemReminder(
+      prompt,
+      "This is an internal WTAgent provider operation. Perform only the requested operation; "
+        + `do not emit the WTAgent XML protocol.${IMAGE_OUTBOUND_CORRELATION_PREFIX}${outboundId}.`,
+    );
+    let value;
+    try {
+      const sendResult = await this.adapter.sendMessage(message, {
+        files: request.referenceImages,
+        maxBytes: null,
+        outboundId,
+        allowAssistantContinuation: false,
+        requireAttachments: request.referenceImages.length > 0,
+      });
+      value = await waitForCompletion({
+        adapter: this.adapter,
+        page: this.adapter.page,
+        sendResult,
+        timeoutMs: request.timeoutMs,
+      });
+      await this.adapter.captureAuxiliaryTurnCompletion?.({
+        timeoutMs: Math.min(request.timeoutMs, 30_000),
+      });
+      return value;
+    } catch (error) {
+      await value?.cleanup?.().catch(() => null);
+      if (this.adapter.getLastSendStatus?.() !== "not-submitted") {
+        error.completionUnknown = true;
+      }
+      throw error;
+    }
+  }
+
+  async saveDownload(page, button, timeoutMs) {
+    const [download] = await Promise.all([
+      page.waitForEvent("download", { timeout: timeoutMs }),
+      button.click(),
+    ]);
+    const temporaryDir = await fs.mkdtemp(path.join(os.tmpdir(), "wtagent-image-"));
+    const suggestedFilename = path.basename(
+      download.suggestedFilename() || "generated-image",
+    );
+    const temporaryPath = path.join(
+      temporaryDir,
+      suggestedFilename,
+    );
+    try {
+      await download.saveAs(temporaryPath);
+      return {
+        temporaryDir,
+        temporaryPath,
+        suggestedFilename,
+        downloadUrl: download.url(),
+      };
+    } catch (error) {
+      await fs.rm(temporaryDir, { recursive: true, force: true }).catch(() => null);
+      throw error;
+    }
+  }
+
+  async saveFromUrl(page, url, timeoutMs) {
+    if (!url) {
+      throw new Error("No image URL was available to download.");
+    }
+
+    let body;
+    let contentType = "";
+    if (url.startsWith("blob:") || url.startsWith("data:")) {
+      const result = await page.evaluate(async (src) => {
+        const response = await fetch(src);
+        const bytes = new Uint8Array(await response.arrayBuffer());
+        let binary = "";
+        const chunk = 0x8000;
+        for (let index = 0; index < bytes.length; index += chunk) {
+          binary += String.fromCharCode(...bytes.subarray(index, index + chunk));
+        }
+        return {
+          base64: btoa(binary),
+          contentType: response.headers.get("content-type") || "",
+        };
+      }, url);
+      body = Buffer.from(result.base64, "base64");
+      contentType = result.contentType;
+    } else {
+      const response = await page.request.get(url, { timeout: timeoutMs });
+      if (!response.ok()) {
+        throw new Error(`Image URL download failed with HTTP ${response.status()}.`);
+      }
+      body = Buffer.from(await response.body());
+      contentType = response.headers()["content-type"] ?? "";
+    }
+
+    const temporaryDir = await fs.mkdtemp(path.join(os.tmpdir(), "wtagent-image-"));
+    const suggestedFilename = `generated-image${extensionForContentType(contentType)}`;
+    const temporaryPath = path.join(temporaryDir, suggestedFilename);
+    try {
+      await fs.writeFile(temporaryPath, body);
+      return {
+        temporaryDir,
+        temporaryPath,
+        suggestedFilename,
+        downloadUrl: url,
+      };
+    } catch (error) {
+      await fs.rm(temporaryDir, { recursive: true, force: true }).catch(() => null);
+      throw error;
+    }
+  }
+
+  remoteFileId(url) {
+    return parseRemoteFileId(url);
+  }
+}

package/src/image/browser-image-session-pool.js

@@ -0,0 +1,96 @@
+import { createWebAdapter, getProviderProfileDir, resolveProvider } from "../browser/provider-registry.js";
+
+export class BrowserImageSessionPool {
+  constructor({
+    primaryAdapter,
+    primaryProvider,
+    appDataDir,
+    chromePath,
+    debug = false,
+    minimized = true,
+    adapterFactory = createWebAdapter,
+  }) {
+    this.primaryAdapter = primaryAdapter;
+    this.primaryProvider = primaryProvider;
+    this.appDataDir = appDataDir;
+    this.chromePath = chromePath;
+    this.debug = debug;
+    this.minimized = minimized;
+    this.adapterFactory = adapterFactory;
+    this.resources = new Map();
+  }
+
+  async adapterFor(providerId) {
+    if (providerId === this.primaryProvider.id) {
+      if (!this.primaryAdapter.context || !this.primaryAdapter.page) {
+        throw new Error(
+          `${this.primaryProvider.label} browser context is not ready for image generation.`,
+        );
+      }
+      // Image generation is an ordinary turn in the active conversation. Do
+      // not clone the adapter or create a second Page: doing so would split the
+      // provider's visible history from Runtime's message identities.
+      return {
+        adapter: this.primaryAdapter,
+        sharedConversation: true,
+        close: async () => {},
+      };
+    }
+
+    const cached = this.resources.get(providerId);
+    if (cached) return await cached;
+    const pending = this.#create(providerId);
+    this.resources.set(providerId, pending);
+    try {
+      return await pending;
+    } catch (error) {
+      this.resources.delete(providerId);
+      throw error;
+    }
+  }
+
+  async #create(providerId) {
+    const provider = resolveProvider(providerId);
+    const adapter = this.adapterFactory({
+      provider,
+      profileDir: getProviderProfileDir(this.appDataDir, providerId),
+      chromePath: this.chromePath,
+      debug: this.debug,
+      minimized: this.minimized,
+      cancelOnEsc: false,
+    });
+
+    await adapter.launch();
+    let authState = await adapter.getAuthState();
+    if (authState !== "authenticated") {
+      // A freshly attached SPA can expose neither its login control nor its
+      // composer for a few seconds. Give it the same bounded authentication
+      // grace as the primary Runtime before declaring the profile logged out.
+      await adapter.waitForManualLogin({ timeoutMs: 8_000 }).catch(() => null);
+      authState = await adapter.getAuthState();
+    }
+    if (authState !== "authenticated") {
+      await adapter.close().catch(() => null);
+      throw new Error(
+        `${provider.label} is not logged in. Run "wtagent login --model ${providerId}" first.`,
+      );
+    }
+    await adapter.startConversation(null);
+    adapter.ownsWindow = true;
+    return {
+      adapter,
+      sharedConversation: false,
+      close: async () => adapter.close(),
+    };
+  }
+
+  async close() {
+    const resources = await Promise.allSettled([...this.resources.values()]);
+    this.resources.clear();
+    for (const result of resources) {
+      if (result.status === "fulfilled") {
+        await result.value.close().catch(() => null);
+      }
+    }
+  }
+}

package/src/image/image-generation-service.js

@@ -0,0 +1,101 @@
+import { ArtifactStore } from "../artifacts/artifact-store.js";
+
+export class ImageGenerationService {
+  constructor({ drivers, defaultProvider = null, driverResolver = null }) {
+    this.drivers = new Map(Object.entries(drivers ?? {}));
+    this.defaultProvider = defaultProvider;
+    this.driverResolver = driverResolver;
+  }
+
+  providerIds() {
+    return [...this.drivers.keys()];
+  }
+
+  async resolveDriver(provider = "auto") {
+    const requested = provider === "auto"
+      ? (this.defaultProvider ?? this.providerIds()[0])
+      : provider;
+    const driver = this.drivers.get(requested)
+      ?? await this.driverResolver?.(requested);
+    if (driver && !this.drivers.has(requested)) {
+      this.drivers.set(requested, driver);
+    }
+    if (!driver) {
+      throw new Error(
+        `Image provider "${requested}" is unavailable. Available providers: ${this.providerIds().join(", ") || "none"}.`,
+      );
+    }
+    return driver;
+  }
+
+  async generate(request, context) {
+    const driver = await this.resolveDriver(request.provider);
+    const capabilities = driver.capabilities();
+    if (!capabilities.generate) {
+      throw new Error(`${driver.provider} does not support image generation.`);
+    }
+    if (request.referenceImages.length > 0 && !capabilities.referenceImages) {
+      throw new Error(`${driver.provider} does not support reference images.`);
+    }
+    let aspectRatio = request.aspectRatio;
+    let requestedAspectRatio = null;
+    if (
+      aspectRatio !== "auto"
+      && capabilities.aspectRatios.length > 0
+      && !capabilities.aspectRatios.includes(aspectRatio)
+    ) {
+      requestedAspectRatio = aspectRatio;
+      aspectRatio = "auto";
+    }
+
+    await driver.adapter?.restoreWindow?.();
+    await driver.adapter?.page?.bringToFront?.().catch(() => null);
+    let produced;
+    try {
+      produced = await driver.generate(
+        { ...request, aspectRatio },
+        {
+          runAuxiliaryTurn: context.runAuxiliaryTurn,
+        },
+      );
+    } finally {
+      if (driver.adapter?.ownsWindow) {
+        await driver.adapter.minimizeWindow?.();
+      }
+    }
+    // No artifact exists for a completed textual provider reply. Preserve it
+    // as a normal result so Runtime can acknowledge the completed browser turn.
+    if (produced.type === "text") return produced;
+    try {
+      const store = new ArtifactStore({
+        projectRoot: context.projectRoot,
+        allowOutside: context.allowOutside,
+      });
+      try {
+        return await store.saveImage(
+          produced.sourcePath,
+          request.outputPath,
+          {
+            ...produced.provenance,
+            ...(requestedAspectRatio
+              ? { aspectRatio, requestedAspectRatio }
+              : {}),
+          },
+        );
+      } catch (error) {
+        // The provider operation has already completed by the time a driver
+        // returns. A validation or local write failure must therefore use the
+        // same no-replay semantics as a failed download: blindly retrying the
+        // tool could spend quota and create another remote image.
+        error.completionUnknown = true;
+        error.meta = {
+          ...(error.meta ?? {}),
+          provenance: produced.provenance,
+        };
+        throw error;
+      }
+    } finally {
+      await produced.cleanup?.().catch(() => null);
+    }
+  }
+}

package/src/image/image-provider-driver.js

@@ -0,0 +1,22 @@
+// Provider contract for browser-backed image generation. Concrete drivers own
+// website-specific navigation, completion detection, and artifact correlation.
+export class ImageProviderDriver {
+  constructor({ provider, adapter, sharedConversation = false }) {
+    this.provider = provider;
+    this.adapter = adapter;
+    this.sharedConversation = sharedConversation;
+  }
+
+  capabilities() {
+    return Object.freeze({
+      generate: true,
+      referenceImages: false,
+      aspectRatios: [],
+      imageCount: 1,
+    });
+  }
+
+  async generate(_request, _execution = {}) {
+    throw new Error(`${this.provider} does not implement image generation.`);
+  }
+}

package/src/image/native-image-receiver.js

@@ -0,0 +1,66 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { ArtifactStore, inspectImageBuffer } from "../artifacts/artifact-store.js";
+import { createImageDriver, listImageProviderIds } from "./provider-registry.js";
+import { parseGrokGeneratedImageUrl } from "./providers/grok-image-driver.js";
+
+// Receive the ordinary assistant turn. This class never sends a prompt or
+// generates an image, and is deliberately absent from the local tool catalog.
+export class NativeImageReceiver {
+  static forAdapter(provider, adapter) {
+    return listImageProviderIds().includes(provider)
+      ? new NativeImageReceiver({ provider, adapter }) : null;
+  }
+
+  constructor({ provider, adapter }) {
+    this.provider = provider;
+    this.adapter = adapter;
+    this.driver = createImageDriver({ provider, adapter });
+  }
+
+  async read(message) {
+    const snapshot = await message.locator("img").evaluateAll((elements) => ({
+      pending: elements.some((image) => !image.complete),
+      images: elements.filter((image) => image.isConnected && image.complete
+        && image.naturalWidth >= 256 && image.naturalHeight >= 256)
+        .map((image) => ({
+          source: image.currentSrc || image.src,
+          width: image.naturalWidth,
+          height: image.naturalHeight,
+        })),
+    }));
+    const seen = new Set();
+    snapshot.images = snapshot.images.filter((image) => {
+      if (!image.source || seen.has(image.source)) return false;
+      if (this.provider === "grok" && !parseGrokGeneratedImageUrl(image.source)) return false;
+      seen.add(image.source);
+      return true;
+    });
+    return snapshot;
+  }
+
+  async save(images, { projectRoot, handoffId, assistantMessageId }) {
+    const store = new ArtifactStore({ projectRoot });
+    // The handoff UUID names this turn, so retries reuse the same destination
+    // without overwriting artifacts from another turn or a user-selected path.
+    const directory = path.join("artifacts", "native-images",
+      createHash("sha256").update(handoffId).digest("hex").slice(0, 24));
+    const artifacts = [];
+    for (const [index, image] of images.entries()) {
+      const downloaded = await this.driver.saveFromUrl(this.adapter.page, image.source, 60_000);
+      try {
+        const inspected = inspectImageBuffer(await fs.readFile(downloaded.temporaryPath));
+        artifacts.push(await store.saveImage(downloaded.temporaryPath,
+          path.join(directory, `image-${index + 1}${inspected.extension}`), {
+            provider: this.provider,
+            assistantMessageId,
+            source: image.source,
+          }));
+      } finally {
+        await fs.rm(downloaded.temporaryDir, { recursive: true, force: true });
+      }
+    }
+    return artifacts;
+  }
+}

package/src/image/provider-registry.js

@@ -0,0 +1,23 @@
+import { ChatGPTImageDriver } from "./providers/chatgpt-image-driver.js";
+import { GeminiImageDriver } from "./providers/gemini-image-driver.js";
+import { GrokImageDriver } from "./providers/grok-image-driver.js";
+
+export const IMAGE_PROVIDERS = Object.freeze({
+  chatgpt: Object.freeze({ id: "chatgpt", driver: ChatGPTImageDriver }),
+  gemini: Object.freeze({ id: "gemini", driver: GeminiImageDriver }),
+  grok: Object.freeze({ id: "grok", driver: GrokImageDriver }),
+});
+
+export function listImageProviderIds() {
+  return Object.keys(IMAGE_PROVIDERS);
+}
+
+export function createImageDriver({ provider, adapter, sharedConversation = false }) {
+  const record = IMAGE_PROVIDERS[provider];
+  if (!record) {
+    throw new Error(
+      `Provider "${provider}" has no image driver. Supported image providers: ${listImageProviderIds().join(", ")}.`,
+    );
+  }
+  return new record.driver({ provider, adapter, sharedConversation });
+}