replicas-computer 0.3.31 → 0.3.33

package/dist/index.js

@@ -1907,13 +1907,22 @@ async function computerBrowserStateCommand(path, options = {}) {
   }));
   console.log(JSON.stringify({ targetId: page.id ?? "unknown", ...observation }, null, 2));
 }
-var DESKTOP_POINT_JS = `
+var VIEWPORT_ORIGIN_JS = `
+    const viewportOrigin = () => {
+      const zoom = window.devicePixelRatio;
+      const border = Math.max(0, (window.outerWidth - window.innerWidth * zoom) / 2);
+      return {
+        left: window.screenX + border,
+        top: window.screenY + Math.max(0, window.outerHeight - window.innerHeight * zoom - border),
+        zoom,
+      };
+    };`;
+var DESKTOP_POINT_JS = `${VIEWPORT_ORIGIN_JS}
     const desktopPoint = (rect) => {
-      const borderX = Math.max(0, (window.outerWidth - window.innerWidth) / 2);
-      const topChrome = Math.max(0, window.outerHeight - window.innerHeight - borderX);
+      const { left, top, zoom } = viewportOrigin();
       return {
-        x: Math.round(window.screenX + borderX + rect.x + rect.width / 2),
-        y: Math.round(window.screenY + topChrome + rect.y + rect.height / 2),
+        x: Math.round(left + (rect.x + rect.width / 2) * zoom),
+        y: Math.round(top + (rect.y + rect.height / 2) * zoom),
       };
     };`;
 function parseBrowserRef(value) {
@@ -3316,7 +3325,7 @@ async function computerStatusCommand() {
 }
 
 // src/version.ts
-var COMPUTER_USE_VERSION = "0.3.31";
+var COMPUTER_USE_VERSION = "0.3.33";
 var COMPUTER_USE_PROTOCOL_VERSION = 1;
 
 // src/program.ts
@@ -3908,6 +3917,42 @@ function superviseEncoder(options) {
 // src/ice-servers.ts
 import { readFileSync as readFileSync7 } from "fs";
 
+// ../shared/src/urls.ts
+var DASHBOARD_PATH = "/dashboard";
+var WEB_APP_PATHS = {
+  home: "/",
+  onboarding: "/onboarding",
+  workspaces: "/workspace",
+  environmentSetup: "/workspace/setup",
+  automations: "/automations",
+  environment: "/environment",
+  analytics: "/analytics",
+  review: "/review",
+  dashboard: DASHBOARD_PATH,
+  agents: `${DASHBOARD_PATH}/agents`,
+  integrations: `${DASHBOARD_PATH}/integrations`,
+  integrationsGithub: `${DASHBOARD_PATH}/integrations/github`,
+  integrationsGitlab: `${DASHBOARD_PATH}/integrations/gitlab`,
+  integrationsSlack: `${DASHBOARD_PATH}/integrations/slack`,
+  settings: `${DASHBOARD_PATH}/settings`,
+  billing: `${DASHBOARD_PATH}/billing`,
+  members: `${DASHBOARD_PATH}/members`,
+  learnings: `${DASHBOARD_PATH}/learnings`,
+  "audit-logs": `${DASHBOARD_PATH}/audit-logs`,
+  accountAgents: `${DASHBOARD_PATH}/account/agents`,
+  accountProfile: `${DASHBOARD_PATH}/account/profile`,
+  accountConnections: `${DASHBOARD_PATH}/account/connections`,
+  accountIntegrations: `${DASHBOARD_PATH}/account/integrations`,
+  accountApiKeys: `${DASHBOARD_PATH}/account/api-keys`,
+  accountPreferences: `${DASHBOARD_PATH}/account/preferences`,
+  fullscreenWorkspaces: "/workspaces"
+};
+var LEGACY_WEB_APP_PATHS = {
+  app: "/app",
+  home: "/home",
+  dashboardWorkspaces: `${DASHBOARD_PATH}/workspaces`
+};
+
 // ../shared/src/desktop-stream.ts
 function isStreamIceServer(value) {
   return isRecord(value) && typeof value.urls === "string" && (value.username === void 0 || typeof value.username === "string") && (value.credential === void 0 || typeof value.credential === "string");
@@ -3971,6 +4016,10 @@ function isDesktopWebRtcFailure(value) {
     "codec"
   ].every((key) => typeof value[key] === "string" && value[key].length <= 200) && ["elapsedMs", "candidateErrors", "framesDecoded", "packetsReceived", "packetsLost", "bytesReceived", "lastFrameAgeMs", "encoderFps"].every((key) => typeof value[key] === "number" && Number.isFinite(value[key]));
 }
+var isDesktopRect = (value) => isRecord(value) && ["x", "y", "width", "height"].every((key) => typeof value[key] === "number" && Number.isFinite(value[key]));
+function isDesktopElement(value) {
+  return isRecord(value) && isDesktopRect(value.rect) && ["url", "tag", "selector"].every((key) => typeof value[key] === "string") && ["text", "label", "source"].every((key) => value[key] === void 0 || typeof value[key] === "string") && (value.components === void 0 || Array.isArray(value.components) && value.components.every((name) => typeof name === "string"));
+}
 
 // ../shared/src/runtime-env.ts
 var REPLICAS_RUNTIME_ENV_ALIASES = {
@@ -4935,6 +4984,92 @@ var InputBridge = class {
   }
 };
 
+// src/desktop-inspect.ts
+var PAGE_GEOMETRY = `(() => {${VIEWPORT_ORIGIN_JS}
+  return { visible: document.visibilityState === 'visible', focused: document.hasFocus(), innerWidth, innerHeight, ...viewportOrigin() };
+})()`;
+var DESCRIBE_ELEMENT = `function () {
+  let el = this.nodeType === 1 ? this : this.parentElement;
+  if (!el) return null;
+  while (el.ownerSVGElement) el = el.ownerSVGElement;
+  el = el.closest('a, button, label, summary, [role="button"], [role="link"], [role="tab"], [role="menuitem"], [role="option"], [role="checkbox"], [role="switch"]') || el;
+  const clip = (value, max) => {
+    const text = String(value || '').replace(/\\s+/g, ' ').trim();
+    return !text ? undefined : text.length > max ? text.slice(0, max - 1) + '\u2026' : text;
+  };
+  const step = (node) => {
+    if (node.id) return '#' + CSS.escape(node.id);
+    const same = node.parentElement ? [...node.parentElement.children].filter((child) => child.localName === node.localName) : [];
+    return node.localName + [...node.classList].filter((name) => /^[\\w-]+$/.test(name)).slice(0, 2).map((name) => '.' + name).join('')
+      + (same.length > 1 ? ':nth-of-type(' + (same.indexOf(node) + 1) + ')' : '');
+  };
+  const steps = [];
+  for (let node = el; node && node !== document.documentElement && steps.length < 4; node = node.parentElement) {
+    steps.unshift(step(node));
+    if (node.id) break;
+  }
+  const components = [];
+  let source;
+  try {
+    const key = Object.keys(el).find((name) => name.startsWith('__reactFiber$') || name.startsWith('__reactInternalInstance$'));
+    const typeName = (type, depth = 0) => typeof type === 'function' ? type.displayName || type.name
+      : type && depth < 3 ? type.displayName || typeName(type.render, depth + 1) || typeName(type.type, depth + 1) : undefined;
+    const host = key && el[key];
+    const next = host && host._debugOwner ? (fiber) => fiber._debugOwner || fiber.owner : (fiber) => fiber.return;
+    for (let fiber = host && next(host); fiber && components.length < 4; fiber = next(fiber)) {
+      const name = typeof fiber.tag !== 'number' ? fiber.name : [0, 1, 11, 14, 15].includes(fiber.tag) ? typeName(fiber.type) : undefined;
+      if (typeof name === 'string' && name.length > 1 && !/^_c\\d*$/.test(name) && components.at(-1) !== name) components.push(name);
+    }
+    const debugSource = host && host._debugSource;
+    if (debugSource && debugSource.fileName) source = debugSource.fileName + ':' + debugSource.lineNumber;
+  } catch {}
+  const rect = el.getBoundingClientRect();
+  let x = rect.x, y = rect.y;
+  try {
+    for (let view = window; view.frameElement; view = view.parent) {
+      const frame = view.frameElement.getBoundingClientRect();
+      x += frame.x + view.frameElement.clientLeft;
+      y += frame.y + view.frameElement.clientTop;
+    }
+  } catch {}
+  return {
+    url: location.href,
+    tag: el.localName,
+    selector: steps.join(' > '),
+    rect: { x, y, width: rect.width, height: rect.height },
+    text: clip(el.innerText || (el.type === 'password' ? '' : el.value) || el.textContent, 120),
+    label: clip(el.getAttribute('aria-label') || el.getAttribute('title') || el.getAttribute('alt') || el.getAttribute('placeholder'), 80),
+    components: components.length ? components : undefined,
+    source,
+  };
+}`;
+async function inspectPage(session, x, y) {
+  const page = chromeCallValue(await sendChromeSessionCommand(session, "Runtime.evaluate", { expression: PAGE_GEOMETRY, returnByValue: true }));
+  if (!isRecord(page) || page.visible !== true) return null;
+  const [left, top, zoom, innerWidth, innerHeight] = [page.left, page.top, page.zoom, page.innerWidth, page.innerHeight].map(Number);
+  const pageX = (x - left) / zoom;
+  const pageY = (y - top) / zoom;
+  if (!(pageX >= 0 && pageY >= 0 && pageX < innerWidth && pageY < innerHeight)) return null;
+  const focused = page.focused === true;
+  const located = await sendChromeSessionCommand(session, "DOM.getNodeForLocation", { x: Math.floor(pageX), y: Math.floor(pageY) });
+  const resolved = await sendChromeSessionCommand(session, "DOM.resolveNode", { backendNodeId: located.backendNodeId });
+  const objectId = isRecord(resolved.object) ? resolved.object.objectId : void 0;
+  if (typeof objectId !== "string") return { focused, element: null };
+  const described = chromeCallValue(await sendChromeSessionCommand(session, "Runtime.callFunctionOn", { objectId, functionDeclaration: DESCRIBE_ELEMENT, returnByValue: true }));
+  if (!isRecord(described) || !isRecord(described.rect)) return { focused, element: null };
+  const { x: rectX, y: rectY, width: width2, height: height2 } = described.rect;
+  const element = {
+    ...described,
+    rect: { x: left + Number(rectX) * zoom, y: top + Number(rectY) * zoom, width: Number(width2) * zoom, height: Number(height2) * zoom }
+  };
+  return { focused, element: isDesktopElement(element) ? element : null };
+}
+async function inspectDesktopPoint(x, y) {
+  const pages = (await getChromePages()).flatMap((page) => page.webSocketDebuggerUrl ? [page.webSocketDebuggerUrl] : []);
+  const hits = (await Promise.all(pages.map((url) => withChromeSession(url, (session) => inspectPage(session, x, y)).catch(() => null)))).filter((hit) => hit !== null);
+  return hits.find((hit) => hit.focused)?.element ?? null;
+}
+
 // src/media-server.ts
 var width = Number.parseInt(process.env.REPLICAS_DESKTOP_WIDTH || String(DESKTOP_VIEWER_WIDTH), 10);
 var height = Number.parseInt(process.env.REPLICAS_DESKTOP_HEIGHT || String(DESKTOP_VIEWER_HEIGHT), 10);
@@ -5842,6 +5977,19 @@ async function runMediaServer() {
       });
       return;
     }
+    if (url.pathname === "/inspect") {
+      const x = Number(url.searchParams.get("x"));
+      const y = Number(url.searchParams.get("y"));
+      response.setHeader("cache-control", "no-store");
+      if (!(x >= 0 && x < width && y >= 0 && y < height)) {
+        response.statusCode = 400;
+        response.end("x and y must be a point on the desktop");
+        return;
+      }
+      response.setHeader("content-type", "application/json");
+      void inspectDesktopPoint(x, y).catch(() => null).then((element) => response.end(JSON.stringify(element)));
+      return;
+    }
     const requested = url.pathname === "/" ? "stream-index.html" : url.pathname.slice(1);
     const path = resolve3(webroot, requested);
     if (!path.startsWith(resolve3(webroot) + "/") || !existsSync8(path) || !statSync2(path).isFile()) {

package/dist/stream-viewer.js

@@ -3,6 +3,43 @@ function isRecord(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
+// ../shared/src/urls.ts
+var DEFAULT_WEB_APP_URL = "https://app.replicas.dev";
+var DASHBOARD_PATH = "/dashboard";
+var WEB_APP_PATHS = {
+  home: "/",
+  onboarding: "/onboarding",
+  workspaces: "/workspace",
+  environmentSetup: "/workspace/setup",
+  automations: "/automations",
+  environment: "/environment",
+  analytics: "/analytics",
+  review: "/review",
+  dashboard: DASHBOARD_PATH,
+  agents: `${DASHBOARD_PATH}/agents`,
+  integrations: `${DASHBOARD_PATH}/integrations`,
+  integrationsGithub: `${DASHBOARD_PATH}/integrations/github`,
+  integrationsGitlab: `${DASHBOARD_PATH}/integrations/gitlab`,
+  integrationsSlack: `${DASHBOARD_PATH}/integrations/slack`,
+  settings: `${DASHBOARD_PATH}/settings`,
+  billing: `${DASHBOARD_PATH}/billing`,
+  members: `${DASHBOARD_PATH}/members`,
+  learnings: `${DASHBOARD_PATH}/learnings`,
+  "audit-logs": `${DASHBOARD_PATH}/audit-logs`,
+  accountAgents: `${DASHBOARD_PATH}/account/agents`,
+  accountProfile: `${DASHBOARD_PATH}/account/profile`,
+  accountConnections: `${DASHBOARD_PATH}/account/connections`,
+  accountIntegrations: `${DASHBOARD_PATH}/account/integrations`,
+  accountApiKeys: `${DASHBOARD_PATH}/account/api-keys`,
+  accountPreferences: `${DASHBOARD_PATH}/account/preferences`,
+  fullscreenWorkspaces: "/workspaces"
+};
+var LEGACY_WEB_APP_PATHS = {
+  app: "/app",
+  home: "/home",
+  dashboardWorkspaces: `${DASHBOARD_PATH}/workspaces`
+};
+
 // ../shared/src/desktop-stream.ts
 function isStreamIceServer(value) {
   return isRecord(value) && typeof value.urls === "string" && (value.username === void 0 || typeof value.username === "string") && (value.credential === void 0 || typeof value.credential === "string");
@@ -50,6 +87,18 @@ var DESKTOP_WEBRTC_FAILURE_REASONS = {
 };
 var DESKTOP_QUALITY_MESSAGE = "replicas-desktop-quality";
 var DESKTOP_CONNECTION_MESSAGE = "replicas-desktop-connection";
+var DESKTOP_ANNOTATE_MESSAGE = "replicas-desktop-annotate";
+function isDesktopAnnotateOrigin(origin, viewerHostname) {
+  if (origin === new URL(DEFAULT_WEB_APP_URL).origin) return true;
+  return viewerHostname.split(".").at(-1) === "localhost" && URL.canParse(origin) && new URL(origin).hostname === "localhost";
+}
+function isDesktopAnnotateRequest(value) {
+  return isRecord(value) && value.type === DESKTOP_ANNOTATE_MESSAGE && typeof value.id === "number" && (value.action === "capture" || value.action === "inspect" && Number.isFinite(value.x) && Number.isFinite(value.y));
+}
+var isDesktopRect = (value) => isRecord(value) && ["x", "y", "width", "height"].every((key) => typeof value[key] === "number" && Number.isFinite(value[key]));
+function isDesktopElement(value) {
+  return isRecord(value) && isDesktopRect(value.rect) && ["url", "tag", "selector"].every((key) => typeof value[key] === "string") && ["text", "label", "source"].every((key) => value[key] === void 0 || typeof value[key] === "string") && (value.components === void 0 || Array.isArray(value.components) && value.components.every((name) => typeof name === "string"));
+}
 
 // src/input-validation.ts
 var finite = (value) => typeof value === "number" && Number.isFinite(value);
@@ -280,6 +329,20 @@ var announceTransport = () => {
   if (window.parent === window) return;
   window.parent.postMessage({ type: DESKTOP_TRANSPORT_MESSAGE, transport: transport ?? "connecting", reason: transportReason, path: route.path, codec: route.codec, failure, recovery }, "*");
 };
+addEventListener("message", (event) => {
+  if (window.parent === window || event.source !== window.parent || !isDesktopAnnotateOrigin(event.origin, location.hostname) || !isDesktopAnnotateRequest(event.data)) return;
+  const request2 = event.data;
+  const reply = (result) => window.parent.postMessage({ type: DESKTOP_ANNOTATE_MESSAGE, id: request2.id, ...result }, event.origin);
+  if (request2.action === "capture") {
+    void fetch("/snapshot", { cache: "no-store" }).then((response) => response.ok ? response.blob() : null).catch(() => null).then((image) => reply({ image }));
+    return;
+  }
+  void fetch(`/inspect?x=${Math.round(request2.x)}&y=${Math.round(request2.y)}`).then((response) => response.ok ? response.json() : Promise.resolve(null)).catch(() => null).then((element) => reply({ element: isDesktopElement(element) ? element : null }));
+});
+var parentOrigin = URL.canParse(document.referrer) ? new URL(document.referrer).origin : null;
+if (window.parent !== window && parentOrigin && isDesktopAnnotateOrigin(parentOrigin, location.hostname)) {
+  window.parent.postMessage({ type: DESKTOP_ANNOTATE_MESSAGE, ready: true }, parentOrigin);
+}
 var average = (sum, count) => count ? Math.round(sum * 10 / count) / 10 : void 0;
 var reportConnection = (ended) => {
   if (window.parent === window || !config) return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-computer",
-  "version": "0.3.31",
+  "version": "0.3.33",
   "description": "Replicas workspace computer-use runtime",
   "type": "module",
   "main": "dist/index.js",