unbrowse 6.2.5 → 6.3.0

package/dist/cli.js

@@ -31,7 +31,7 @@ var __promiseAll = (args) => Promise.all(args);
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "6.2.5", BUILD_GIT_SHA = "842fe8f374d6", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi4yLjUiLCJnaXRfc2hhIjoiODQyZmU4ZjM3NGQ2IiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUA4NDJmZThmMzc0ZDYiLCJpc3N1ZWRfYXQiOiIyMDI2LTA1LTAxVDAzOjE5OjI1LjE4OVoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "qYak9fhJCHowq94BznDE7aDxbTn53oRrfywJe503s8M", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "6.3.0", BUILD_GIT_SHA = "ebf3580e8a7b", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi4zLjAiLCJnaXRfc2hhIjoiZWJmMzU4MGU4YTdiIiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUBlYmYzNTgwZThhN2IiLCJpc3N1ZWRfYXQiOiIyMDI2LTA1LTAxVDA1OjU0OjI4LjgyNloifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "eROrMsDX6qJfzkUSaa6C9my4wf18yIDN6v0qG57deps", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash } from "crypto";
@@ -1559,6 +1559,11 @@ var RETRYABLE_STATUSES;
 var init_retry = __esm(() => {
   RETRYABLE_STATUSES = new Set([500, 502, 503, 504, 429]);
 });
+
+// ../../src/execution/probe.ts
+var init_probe = __esm(() => {
+  init_logger();
+});
 // ../../src/extraction/index.ts
 import * as cheerio from "cheerio";
 var STRIP_TAGS, CHROME_TAGS;
@@ -1733,6 +1738,7 @@ var init_execution = __esm(async () => {
   init_client();
   init_client();
   init_retry();
+  init_probe();
   init_domain();
   init_extraction();
   init_graph();
@@ -4566,6 +4572,8 @@ function slimTrace(obj) {
     out.provider = obj.provider;
   if ("result" in obj)
     out.result = obj.result;
+  if (Array.isArray(obj.decision_trace))
+    out.decision_trace = obj.decision_trace;
   if (obj.available_endpoints)
     out.available_endpoints = obj.available_endpoints;
   if (obj.impact)

package/dist/mcp.js

@@ -226,11 +226,11 @@ import { dirname, join, parse } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "6.2.5";
-var BUILD_GIT_SHA = "842fe8f374d6";
+var BUILD_RELEASE_VERSION = "6.3.0";
+var BUILD_GIT_SHA = "ebf3580e8a7b";
 var BUILD_CODE_HASH = "5d9ebf619c61";
-var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi4yLjUiLCJnaXRfc2hhIjoiODQyZmU4ZjM3NGQ2IiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUA4NDJmZThmMzc0ZDYiLCJpc3N1ZWRfYXQiOiIyMDI2LTA1LTAxVDAzOjE5OjI1LjE4OVoifQ";
-var BUILD_RELEASE_MANIFEST_SIGNATURE = "qYak9fhJCHowq94BznDE7aDxbTn53oRrfywJe503s8M";
+var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi4zLjAiLCJnaXRfc2hhIjoiZWJmMzU4MGU4YTdiIiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUBlYmYzNTgwZThhN2IiLCJpc3N1ZWRfYXQiOiIyMDI2LTA1LTAxVDA1OjU0OjI4LjgyNloifQ";
+var BUILD_RELEASE_MANIFEST_SIGNATURE = "eROrMsDX6qJfzkUSaa6C9my4wf18yIDN6v0qG57deps";
 var BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts

package/dist/server.js

@@ -3911,13 +3911,6 @@ function extractRscDataEndpoints(body) {
 }
 
 // ../../src/reverse-engineer/index.ts
-var exports_reverse_engineer = {};
-__export(exports_reverse_engineer, {
-  minePathTemplates: () => minePathTemplates,
-  extractGraphQLOperationName: () => extractGraphQLOperationName,
-  extractEndpoints: () => extractEndpoints,
-  extractAuthHeaders: () => extractAuthHeaders
-});
 import { nanoid as nanoid2 } from "nanoid";
 import { createHash } from "node:crypto";
 function stableEndpointId(method, urlTemplate) {
@@ -4639,7 +4632,8 @@ function extractEndpoints(requests, wsMessages, context, traceSink) {
       reliability_score: 0.5,
       response_schema,
       trigger_url: context?.pageUrl,
-      ...pathBindingCandidates.length > 0 ? { _path_binding_candidates: pathBindingCandidates } : {}
+      ...pathBindingCandidates.length > 0 ? { _path_binding_candidates: pathBindingCandidates } : {},
+      ...buildProvenRecipe(req, computedUrlTemplate) ? { proven_recipe: buildProvenRecipe(req, computedUrlTemplate) } : {}
     };
     endpoint = resolveEndpointPathBindings(endpoint);
     endpoint.semantic = inferEndpointSemantic(endpoint, {
@@ -4860,6 +4854,58 @@ function sanitizeHeaders(headers) {
     return !isSensitiveHeader(k);
   }));
 }
+function pickReplayHeaders(headers) {
+  const out = {};
+  for (const [k, v] of Object.entries(headers ?? {})) {
+    const lower = k.toLowerCase();
+    if (lower.startsWith("sec-fetch-"))
+      continue;
+    if (lower.startsWith("sec-ch-ua"))
+      continue;
+    if (lower.startsWith("if-"))
+      continue;
+    if (lower === "cookie" || lower === "authorization" || lower === "content-length" || lower === "host" || lower === "origin" || lower === "referer" || lower === "user-agent" || lower === "accept-encoding" || lower === "cache-control" || lower === "pragma" || lower === "connection" || lower === "te" || lower === "upgrade-insecure-requests" || lower === "x-csrf-token" || lower === "x-xsrf-token") {
+      continue;
+    }
+    out[k] = v;
+  }
+  return out;
+}
+function buildProvenRecipe(req, urlTemplate) {
+  if (req.response_status < 200 || req.response_status >= 300)
+    return;
+  if (!req.response_body)
+    return;
+  const ct = (req.response_headers?.["content-type"] ?? "").toLowerCase();
+  const bodyLen = Buffer.byteLength(req.response_body);
+  let json_top_keys;
+  if (ct.includes("application/json") || ct.includes("+json")) {
+    try {
+      const parsed = JSON.parse(stripJsonPrefix(req.response_body));
+      if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+        json_top_keys = Object.keys(parsed).slice(0, 8);
+      }
+    } catch {}
+  }
+  let body = undefined;
+  if (req.method !== "GET" && req.method !== "HEAD" && req.request_body) {
+    body = tryParseBody(req.request_body) ?? req.request_body;
+  }
+  return {
+    method: req.method,
+    url_template: urlTemplate,
+    headers: pickReplayHeaders(req.request_headers),
+    ...body !== undefined ? { body } : {},
+    response_signal: {
+      status: req.response_status,
+      ...ct ? { content_type: ct } : {},
+      byte_length_min: Math.floor(bodyLen * 0.5),
+      byte_length_max: Math.ceil(bodyLen * 2),
+      ...json_top_keys ? { json_top_keys } : {}
+    },
+    captured_at: req.timestamp || new Date().toISOString()
+  };
+}
 function extractAuthHeaders(requests) {
   const authHeaders = {};
   for (const req of requests) {
@@ -7285,7 +7331,7 @@ var init_capture = __esm(async () => {
 });
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "6.2.5", BUILD_GIT_SHA = "842fe8f374d6", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi4yLjUiLCJnaXRfc2hhIjoiODQyZmU4ZjM3NGQ2IiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUA4NDJmZThmMzc0ZDYiLCJpc3N1ZWRfYXQiOiIyMDI2LTA1LTAxVDAzOjE5OjI1LjE4OVoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "qYak9fhJCHowq94BznDE7aDxbTn53oRrfywJe503s8M", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "6.3.0", BUILD_GIT_SHA = "ebf3580e8a7b", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi4zLjAiLCJnaXRfc2hhIjoiZWJmMzU4MGU4YTdiIiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUBlYmYzNTgwZThhN2IiLCJpc3N1ZWRfYXQiOiIyMDI2LTA1LTAxVDA1OjU0OjI4LjgyNloifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "eROrMsDX6qJfzkUSaa6C9my4wf18yIDN6v0qG57deps", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash as createHash2 } from "crypto";
@@ -10951,6 +10997,151 @@ var init_retry = __esm(() => {
   RETRYABLE_STATUSES = new Set([500, 502, 503, 504, 429]);
 });
 
+// ../../src/execution/probe.ts
+async function probeUrl(url, opts = {}) {
+  const headers = {
+    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    accept: "*/*",
+    "accept-language": "en-US,en;q=0.9",
+    ...opts.headers ?? {}
+  };
+  if (opts.cookies && opts.cookies.length > 0) {
+    headers["cookie"] = opts.cookies.map((c) => {
+      const v = c.value.startsWith('"') && c.value.endsWith('"') ? c.value.slice(1, -1) : c.value;
+      return `${c.name}=${v}`;
+    }).join("; ");
+  }
+  const headTimeout = opts.timeout_ms ?? 1500;
+  const rangedTimeout = opts.timeout_ms ?? 1000;
+  const start2 = Date.now();
+  try {
+    const res = await fetchWithTimeout(url, { method: "HEAD", headers, redirect: "follow" }, headTimeout);
+    if (res.status !== 405 && res.status !== 501) {
+      const ct = (res.headers.get("content-type") || "").toLowerCase();
+      const lenHeader = res.headers.get("content-length");
+      const byte_length = lenHeader && Number.isFinite(Number(lenHeader)) ? Number(lenHeader) : undefined;
+      return {
+        status: res.status,
+        content_type: ct || undefined,
+        byte_length,
+        ms: Date.now() - start2,
+        method_used: "HEAD"
+      };
+    }
+    log("probe", `HEAD ${url} returned ${res.status}; trying ranged GET`);
+  } catch (err) {
+    log("probe", `HEAD ${url} failed: ${err.message}; trying ranged GET`);
+  }
+  const start22 = Date.now();
+  try {
+    const res = await fetchWithTimeout(url, {
+      method: "GET",
+      headers: { ...headers, range: "bytes=0-0" },
+      redirect: "follow"
+    }, rangedTimeout);
+    const ct = (res.headers.get("content-type") || "").toLowerCase();
+    const range = res.headers.get("content-range");
+    const lenHeader = res.headers.get("content-length");
+    let byte_length;
+    if (range) {
+      const slash = range.split("/")[1];
+      const total = Number(slash);
+      if (Number.isFinite(total))
+        byte_length = total;
+    }
+    if (byte_length === undefined && lenHeader && Number.isFinite(Number(lenHeader))) {
+      const n = Number(lenHeader);
+      if (n > 1)
+        byte_length = n;
+    }
+    try {
+      await res.arrayBuffer();
+    } catch {}
+    return {
+      status: res.status === 206 ? 200 : res.status,
+      content_type: ct || undefined,
+      byte_length,
+      ms: Date.now() - start2 + (Date.now() - start22),
+      method_used: "GET-1byte"
+    };
+  } catch (err) {
+    return {
+      status: 0,
+      ms: Date.now() - start2,
+      error: err.message || "network_error",
+      method_used: "GET-1byte"
+    };
+  }
+}
+async function fetchWithTimeout(url, init, timeoutMs) {
+  const ctrl = new AbortController;
+  const timer = setTimeout(() => ctrl.abort(), timeoutMs);
+  try {
+    return await fetch(url, { ...init, signal: ctrl.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+function decideFromProbe(input) {
+  const { probe, has_trigger_url, intent_wants_dom } = input;
+  const { status, content_type = "", byte_length } = probe;
+  if (status >= 400) {
+    return {
+      strategy: "return-error",
+      reason: `probe status ${status}; returning to caller`
+    };
+  }
+  if (status === 0) {
+    return {
+      strategy: "browser",
+      reason: `probe network error: ${probe.error ?? "unknown"}`
+    };
+  }
+  const isJson = JSON_LIKE.test(content_type);
+  const isHtml = HTML_LIKE.test(content_type);
+  const bodyLarge = (byte_length ?? 0) >= SMALL_HTML_BYTES;
+  if (isJson) {
+    return {
+      strategy: "server",
+      reason: `probe ${status} + ${content_type} — direct fetchable`
+    };
+  }
+  if (isHtml && bodyLarge) {
+    return {
+      strategy: "server",
+      reason: `probe ${status} + html ${byte_length}B — server-rendered, fetch + extract`
+    };
+  }
+  if (isHtml && !bodyLarge && has_trigger_url) {
+    return {
+      strategy: "trigger-intercept",
+      reason: `probe ${status} + html ${byte_length ?? "?"}B — likely SPA shell, trigger-intercept`
+    };
+  }
+  if (isHtml && !bodyLarge) {
+    return {
+      strategy: "browser",
+      reason: `probe ${status} + html ${byte_length ?? "?"}B SPA shell, no trigger_url — browser`
+    };
+  }
+  if (intent_wants_dom) {
+    return {
+      strategy: "browser",
+      reason: `intent wants DOM, content-type ${content_type || "unknown"} — browser`
+    };
+  }
+  return {
+    strategy: "server",
+    reason: `probe ${status}, content-type ${content_type || "unknown"} — try server`
+  };
+}
+var SMALL_HTML_BYTES = 5000, JSON_LIKE, HTML_LIKE;
+var init_probe = __esm(() => {
+  init_logger();
+  JSON_LIKE = /(application\/json|application\/[\w.+-]+\+json|text\/csv|application\/xml|text\/xml)/i;
+  HTML_LIKE = /text\/html|application\/xhtml\+xml/i;
+});
+
 // ../../src/intent-match.ts
 function isRecord(value) {
   return !!value && typeof value === "object" && !Array.isArray(value);
@@ -16115,15 +16306,6 @@ function normalizeReplayHeaders(...bags) {
   }
   return normalized;
 }
-function shouldFallbackToBrowserReplay(data, endpoint, intent, contextUrl) {
-  const replayUrl = resolveExecutionUrlTemplate(endpoint, contextUrl);
-  if (!isDocumentLikeUrl(replayUrl))
-    return false;
-  if (typeof data === "string")
-    return isHtml(data) || isSpaShell(data);
-  const assessment = assessIntentResult(data, intent);
-  return assessment.verdict === "fail";
-}
 function buildSampleRequestFromUrl(url) {
   try {
     return Object.fromEntries(sanitizeNavigationQueryParams(new URL(url)).searchParams.entries());
@@ -17089,124 +17271,6 @@ async function executeBrowserCapture(skill, params, options) {
     learned_skill: learned
   };
 }
-async function tryHttpFetch(url, authHeaders, cookies) {
-  try {
-    const headers = {
-      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-      "Accept-Language": "en-US,en;q=0.9",
-      "Cache-Control": "no-cache",
-      ...authHeaders
-    };
-    if (cookies && cookies.length > 0) {
-      headers["Cookie"] = cookies.map((c) => `${c.name}=${c.value}`).join("; ");
-    }
-    const controller = new AbortController;
-    const timeout = setTimeout(() => controller.abort(), 1e4);
-    const res = await fetch(url, {
-      headers,
-      signal: controller.signal,
-      redirect: "follow"
-    });
-    clearTimeout(timeout);
-    if (res.status !== 200)
-      return null;
-    const ct = res.headers.get("content-type") ?? "";
-    if (!ct.includes("text/html") && !ct.includes("application/xhtml"))
-      return null;
-    const html = await res.text();
-    if (!html || html.length < 1024)
-      return null;
-    return { html, final_url: res.url || url };
-  } catch {
-    return null;
-  }
-}
-function flattenExtracted(data) {
-  if (!Array.isArray(data))
-    return data;
-  const first = data[0];
-  if (first && typeof first === "object" && "type" in first && "data" in first && "relevance_score" in first) {
-    return data.reduce((best, cur) => (cur.relevance_score ?? 0) > (best.relevance_score ?? 0) ? cur : best).data;
-  }
-  return data;
-}
-async function executeDomExtractionEndpoint(endpoint, url, intent, authHeaders, cookies) {
-  const ssrResult = await tryHttpFetch(url, authHeaders, cookies);
-  if (ssrResult) {
-    const ssrExtracted = extractFromDOMWithHint(ssrResult.html, intent, endpoint.dom_extraction);
-    if (ssrExtracted.data) {
-      const ssrQuality = validateExtractionQuality(ssrExtracted.data, ssrExtracted.confidence, intent);
-      if (ssrQuality.valid) {
-        const ssrSemantic = assessIntentResult(ssrExtracted.data, intent);
-        if (ssrSemantic.verdict !== "fail") {
-          console.log(`[ssr-fast] hit — extracted via HTTP fetch`);
-          return {
-            data: flattenExtracted(ssrExtracted.data),
-            status: 200,
-            trace_id: nanoid6()
-          };
-        }
-      }
-    }
-    console.log(`[ssr-fast] miss, falling back to browser`);
-  } else {
-    console.log(`[ssr-fast] miss, falling back to browser`);
-  }
-  const captured = await captureSession(url, authHeaders, cookies, intent);
-  if (captured.requests.length > 0) {
-    const { extractEndpoints: extractEps } = await Promise.resolve().then(() => (init_reverse_engineer(), exports_reverse_engineer));
-    const apiEndpoints = extractEps(captured.requests, undefined, { pageUrl: url, finalUrl: captured.final_url });
-    const jsonEndpoints = apiEndpoints.filter((ep) => ep.response_schema && !ep.dom_extraction);
-    if (jsonEndpoints.length > 0) {
-      const best = jsonEndpoints[0];
-      const matchingReq = captured.requests.find((r) => r.url.includes(best.url_template.split("?")[0].split("{")[0]) && r.response_body && r.response_status >= 200 && r.response_status < 400);
-      if (matchingReq?.response_body) {
-        try {
-          const data = JSON.parse(matchingReq.response_body);
-          console.log(`[dom-exec] found API response from browser capture: ${matchingReq.url.substring(0, 80)}`);
-          return { data, status: matchingReq.response_status, trace_id: nanoid6() };
-        } catch {}
-      }
-    }
-  }
-  const html = captured.html ?? "";
-  const extracted = extractFromDOMWithHint(html, intent, endpoint.dom_extraction);
-  if (extracted.data) {
-    const quality = validateExtractionQuality(extracted.data, extracted.confidence, intent);
-    if (!quality.valid) {
-      return {
-        data: {
-          error: "low_quality_dom_extraction",
-          message: `Structured DOM extraction was rejected: ${quality.quality_note ?? "low quality extraction"}`
-        },
-        status: 422,
-        trace_id: nanoid6()
-      };
-    }
-    const semanticAssessment = assessIntentResult(extracted.data, intent);
-    if (semanticAssessment.verdict === "fail") {
-      return {
-        data: {
-          error: "low_quality_dom_extraction",
-          message: `Structured DOM extraction was rejected: ${semanticAssessment.reason}`
-        },
-        status: 422,
-        trace_id: nanoid6()
-      };
-    }
-    return {
-      data: flattenExtracted(extracted.data),
-      status: 200,
-      trace_id: nanoid6()
-    };
-  }
-  return {
-    data: html,
-    status: 200,
-    trace_id: nanoid6()
-  };
-}
 async function executeEndpoint(skill, endpoint, params = {}, projection, options) {
   endpoint = annotateEndpointPolicy(endpoint);
   if (endpoint.policy?.requires_live_session) {
@@ -17786,175 +17850,109 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
   const hasAuth = cookies.length > 0 || Object.keys(authHeaders).length > 0;
   const preferredWorkflowStrategy = workflowRecipe?.steps[0]?.strategy ? translateWorkflowStrategy(workflowRecipe.steps[0].strategy) : undefined;
   let workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy;
-  if (endpoint.dom_extraction && isSafe) {
-    if (hasStructuredReplay) {
-      result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
-      if (shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
-        result = await executeDomExtractionEndpoint(endpoint, url, options?.intent ?? skill.intent_signature, authHeaders, cookies);
-      }
-    } else {
-      result = await executeDomExtractionEndpoint(endpoint, url, options?.intent ?? skill.intent_signature, authHeaders, cookies);
-    }
-  } else if (hasAuth) {
-    let strategy;
-    const endpointStrategy = preferredWorkflowStrategy ?? endpoint.exec_strategy;
-    if (hasStructuredReplay) {
-      result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
-      if (result.status >= 200 && result.status < 400 && !shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
-        strategy = "server";
-        workflowChosenStrategy = "server";
-      } else if (endpoint.trigger_url && isSafe) {
-        let triggerUrl = endpoint.trigger_url;
-        if (Object.keys(mergedParams).length > 0) {
-          try {
-            const tu = new URL(endpoint.trigger_url);
-            for (const [k, v] of Object.entries(mergedParams)) {
-              if (v != null && !reservedMetaParams.has(k)) {
-                tu.searchParams.set(k, String(v));
-              }
-            }
-            triggerUrl = tu.toString();
-          } catch {}
-        }
-        result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
-        const isSelfFetchableFirst = endpoint.method === "GET" && /\.(json)(\?|$)|\/api\//i.test(url);
-        if (result.status === 0 && isSelfFetchableFirst) {
-          log("exec", `trigger-intercept timed out; trying serverFetch for self-fetchable ${url}`);
-          result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
-          if (result.status >= 200 && result.status < 400) {
-            strategy = "server";
-            workflowChosenStrategy = "server";
-          } else {
-            strategy = "trigger-intercept";
-            workflowChosenStrategy = "trigger-intercept";
-          }
-        } else {
-          strategy = "trigger-intercept";
-          workflowChosenStrategy = "trigger-intercept";
-        }
-      } else {
-        result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
-        strategy = "browser";
-        workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
-      }
-    } else if (endpointStrategy === "server") {
-      result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
-      if (shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
-        result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
-        strategy = "browser";
-        workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
-      } else {
-        strategy = "server";
-        workflowChosenStrategy = "server";
-      }
-    } else if (endpointStrategy === "trigger-intercept" && endpoint.trigger_url && isSafe) {
-      let triggerUrl = endpoint.trigger_url;
-      if (Object.keys(mergedParams).length > 0) {
-        try {
-          const tu = new URL(endpoint.trigger_url);
-          for (const [k, v] of Object.entries(mergedParams)) {
-            if (v != null && !reservedMetaParams.has(k))
-              tu.searchParams.set(k, String(v));
-          }
-          triggerUrl = tu.toString();
-        } catch {}
-      }
-      log("exec", `using learned strategy trigger-intercept via ${triggerUrl}`);
-      result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
-      const isSelfFetchable = endpoint.method === "GET" && /\.(json)(\?|$)|\/api\//i.test(url);
-      if (result.status === 0 && isSelfFetchable) {
-        log("exec", `trigger-intercept timed out; trying serverFetch for self-fetchable ${url}`);
+  const decisionTrace = [];
+  let recipeMatched = false;
+  if (endpoint.proven_recipe && shouldReplayRecipe(endpoint.proven_recipe, url)) {
+    const recipeStart = Date.now();
+    const recipeResult = await replayRecipe(endpoint.proven_recipe, url, cookies, authHeaders, mergedParams);
+    const matchVerdict = matchResponseSignal(recipeResult, endpoint.proven_recipe.response_signal);
+    decisionTrace.push({
+      step: "recipe_replay",
+      method: endpoint.proven_recipe.method,
+      status: recipeResult.status,
+      match: matchVerdict.match,
+      ...matchVerdict.match ? {} : { reason: matchVerdict.reason ?? "unknown" },
+      ms: Date.now() - recipeStart
+    });
+    if (matchVerdict.match) {
+      result = recipeResult;
+      recipeMatched = true;
+      workflowChosenStrategy = workflowChosenStrategy ?? "recipe-replay";
+    }
+  }
+  if (!recipeMatched) {
+    const probeCookies = cookies.map((c) => ({ name: c.name, value: c.value }));
+    const probe = await probeUrl(url, {
+      cookies: probeCookies,
+      headers: { ...authHeaders }
+    });
+    decisionTrace.push({
+      step: "probe",
+      method: probe.method_used,
+      status: probe.status,
+      content_type: probe.content_type,
+      byte_length: probe.byte_length,
+      ms: probe.ms,
+      ...probe.error ? { error: probe.error } : {}
+    });
+    const decision = decideFromProbe({
+      probe,
+      has_trigger_url: !!endpoint.trigger_url,
+      intent_wants_dom: !!endpoint.dom_extraction
+    });
+    decisionTrace.push({
+      step: "decision",
+      strategy: decision.strategy,
+      reason: decision.reason
+    });
+    switch (decision.strategy) {
+      case "server": {
         result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
-        if (result.status >= 200 && result.status < 400) {
-          strategy = "server";
-          workflowChosenStrategy = "server";
-        }
-      } else {
-        strategy = "trigger-intercept";
-        workflowChosenStrategy = "trigger-intercept";
+        decisionTrace.push({ step: "server_fetch", status: result.status });
+        workflowChosenStrategy = workflowChosenStrategy ?? "server";
+        break;
       }
-    } else if (endpointStrategy === "browser") {
-      if (shouldIgnoreLearnedBrowserStrategy(endpoint, url)) {
-        result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
-        if (result.status >= 200 && result.status < 400 && !shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
-          strategy = "server";
-          workflowChosenStrategy = "server";
-        } else {
-          log("exec", `server replay rejected stale learned browser strategy for ${endpoint.endpoint_id}; falling back to browser`);
+      case "trigger-intercept": {
+        if (!endpoint.trigger_url || !isSafe) {
           result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
-          strategy = "browser";
-          workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
-        }
-      } else {
-        log("exec", `using learned strategy browser`);
-        result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
-        strategy = "browser";
-        workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
-      }
-    } else {
-      try {
-        result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
-        if (result.status >= 200 && result.status < 400) {
-          const isApiEndpoint = /\/(api|graphql)\b/i.test(endpoint.url_template) || /\.(json)(\?|$)/.test(endpoint.url_template);
-          if (!isApiEndpoint && shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
-            result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
-            strategy = "browser";
-            workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
-          } else {
-            strategy = "server";
-            workflowChosenStrategy = "server";
-          }
+          decisionTrace.push({ step: "browser_fallback", reason: "no trigger_url or unsafe method", status: result.status });
+          workflowChosenStrategy = workflowChosenStrategy ?? "browser-fetch";
         } else {
-          log("exec", `server fetch returned ${result.status}, falling back`);
-          if (endpoint.trigger_url && isSafe) {
-            let triggerUrl = endpoint.trigger_url;
-            if (Object.keys(mergedParams).length > 0) {
-              try {
-                const tu = new URL(endpoint.trigger_url);
-                for (const [k, v] of Object.entries(mergedParams)) {
-                  if (v != null && !reservedMetaParams.has(k))
-                    tu.searchParams.set(k, String(v));
+          let triggerUrl = endpoint.trigger_url;
+          if (Object.keys(mergedParams).length > 0) {
+            try {
+              const tu = new URL(endpoint.trigger_url);
+              for (const [k, v] of Object.entries(mergedParams)) {
+                if (v != null && !reservedMetaParams.has(k)) {
+                  tu.searchParams.set(k, String(v));
                 }
-                triggerUrl = tu.toString();
-              } catch {}
-            }
-            result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
-            strategy = "trigger-intercept";
-            workflowChosenStrategy = "trigger-intercept";
-          } else {
-            result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
-            strategy = "browser";
-            workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
+              }
+              triggerUrl = tu.toString();
+            } catch {}
           }
+          result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
+          decisionTrace.push({ step: "trigger_intercept", trigger_url: triggerUrl, status: result.status });
+          workflowChosenStrategy = "trigger-intercept";
         }
-      } catch {
+        break;
+      }
+      case "browser": {
         result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
-        strategy = "browser";
-        workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
+        decisionTrace.push({ step: "browser", status: result.status });
+        workflowChosenStrategy = workflowChosenStrategy ?? (workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch");
+        break;
       }
-    }
-    if (strategy && result.status >= 200 && result.status < 400 && strategy !== endpoint.exec_strategy) {
-      log("exec", `learned exec_strategy=${strategy} for endpoint ${endpoint.endpoint_id}`);
-      endpoint.exec_strategy = strategy;
-      try {
-        cachePublishedSkill(skill, options?.client_scope);
-      } catch (e) {
-        log("exec", `failed to cache strategy: ${e}`);
+      case "return-error": {
+        result = {
+          status: probe.status,
+          data: {
+            error: `http_${probe.status}`,
+            message: `Probe returned status ${probe.status}; returned to caller without escalating.`,
+            probe_method: probe.method_used,
+            ...probe.content_type ? { content_type: probe.content_type } : {}
+          },
+          trace_id: nanoid6()
+        };
+        decisionTrace.push({ step: "return_error", status: probe.status });
+        workflowChosenStrategy = workflowChosenStrategy ?? "server";
+        break;
       }
-    }
-  } else if (isSafe) {
-    try {
-      result = await withRetry(() => serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride), (r) => isRetryableStatus(r.status));
-      if (typeof result.data === "string" && isHtml(result.data)) {
-        if (isSpaShell(result.data)) {
-          result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
-        }
+      default: {
+        result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
+        decisionTrace.push({ step: "browser_default", status: result.status });
+        workflowChosenStrategy = workflowChosenStrategy ?? "browser-fetch";
       }
-    } catch {
-      result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
     }
-  } else {
-    result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
   }
   if (workflowRecipe && workflowArtifact && needsWorkflowTokenRefresh(result.status)) {
     const refreshed = await refreshAuthFromBrowser(epDomain);
@@ -17983,6 +17981,7 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
     success: status >= 200 && status < 300,
     status_code: status
   });
+  trace.decision_trace = decisionTrace;
   if (!trace.success) {
     trace.error = status === 0 ? `HTTP 0 — network failure or browser fetch was blocked (DNS, TLS, CORS, anti-bot, or kuri tab error). Try \`unbrowse go\` to open a live session, then re-run.` : status === 404 ? `HTTP 404 — endpoint may be stale. Re-run via POST /v1/intent/resolve to get fresh endpoints.` : `HTTP ${status}`;
     const isEmptyData = data == null || typeof data === "object" && !Array.isArray(data) && Object.keys(data).length === 0;
@@ -18163,7 +18162,8 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
   }
   return {
     trace,
-    result: resultData
+    result: resultData,
+    decision_trace: decisionTrace
   };
 }
 function templatizeQueryParams(url) {
@@ -18205,6 +18205,67 @@ function interpolate(template, params) {
 function interpolateObj(obj, params) {
   return JSON.parse(JSON.stringify(obj).replace(/"(\{(\w+)\})"/g, (_, _full, k) => params[k] != null ? JSON.stringify(params[k]) : `"{${k}}"`));
 }
+function shouldReplayRecipe(_recipe, substitutedUrl) {
+  return !/\{[a-z0-9_]+\}/i.test(substitutedUrl);
+}
+async function replayRecipe(recipe, url, cookies, authHeaders, params) {
+  const headers = { ...recipe.headers, ...authHeaders };
+  if (cookies.length > 0) {
+    headers["cookie"] = cookies.map((c) => {
+      const v = c.value.startsWith('"') && c.value.endsWith('"') ? c.value.slice(1, -1) : c.value;
+      return `${c.name}=${v}`;
+    }).join("; ");
+  }
+  let body;
+  if (recipe.body !== undefined && recipe.method !== "GET" && recipe.method !== "HEAD") {
+    if (typeof recipe.body === "string") {
+      body = recipe.body;
+    } else if (recipe.body && typeof recipe.body === "object") {
+      const interpolated = interpolateObj(recipe.body, params);
+      body = JSON.stringify(interpolated);
+    }
+  }
+  try {
+    const res = await fetch(url, {
+      method: recipe.method,
+      headers,
+      body,
+      redirect: "follow"
+    });
+    const text = await res.text();
+    let data = text;
+    try {
+      data = JSON.parse(text);
+    } catch {}
+    return { status: res.status, data, trace_id: nanoid6() };
+  } catch (err) {
+    return {
+      status: 0,
+      data: { error: err.message || "network_error" },
+      trace_id: nanoid6()
+    };
+  }
+}
+function matchResponseSignal(result, signal) {
+  if (result.status !== signal.status) {
+    return { match: false, reason: `status_changed: ${signal.status} → ${result.status}` };
+  }
+  const bodyLen = typeof result.data === "string" ? Buffer.byteLength(result.data) : Buffer.byteLength(JSON.stringify(result.data ?? null));
+  if (signal.byte_length_min !== undefined && bodyLen < signal.byte_length_min) {
+    return { match: false, reason: `body_shrunk: ${bodyLen}B < min ${signal.byte_length_min}B` };
+  }
+  if (signal.byte_length_max !== undefined && bodyLen > signal.byte_length_max) {
+    return { match: false, reason: `body_grew: ${bodyLen}B > max ${signal.byte_length_max}B` };
+  }
+  if (signal.json_top_keys && result.data && typeof result.data === "object" && !Array.isArray(result.data)) {
+    const actual = new Set(Object.keys(result.data));
+    const missing = signal.json_top_keys.filter((k) => !actual.has(k));
+    if (missing.length > 0) {
+      return { match: false, reason: `missing_top_keys: ${missing.slice(0, 3).join(",")}` };
+    }
+  }
+  return { match: true };
+}
 function stem(word) {
   if (word.endsWith("ies") && word.length > 4)
     return word.slice(0, -3) + "y";
@@ -19122,6 +19183,7 @@ var init_execution = __esm(async () => {
   init_client2();
   init_client2();
   init_retry();
+  init_probe();
   init_domain();
   init_extraction();
   init_graph();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "6.2.5",
+  "version": "6.3.0",
   "description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
   "type": "module",
   "bin": {