unbrowse 6.11.0 → 6.12.0

package/dist/cli.js

@@ -20,7 +20,7 @@ var __promiseAll = (args) => Promise.all(args);
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "6.11.0", BUILD_GIT_SHA = "9193343b3660", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi4xMS4wIiwiZ2l0X3NoYSI6IjkxOTMzNDNiMzY2MCIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAOTE5MzM0M2IzNjYwIiwiaXNzdWVkX2F0IjoiMjAyNi0wNS0xMVQxOToyNjo1Mi40MDdaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "Ai1ymvce0ORL9m3DFa9v14tFbAbeyy7XUfTDakeoHkA", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai", BUILD_DEFAULT_PROFILE = "";
+var BUILD_RELEASE_VERSION = "6.12.0", BUILD_GIT_SHA = "cbea913e06cf", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi4xMi4wIiwiZ2l0X3NoYSI6ImNiZWE5MTNlMDZjZiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAY2JlYTkxM2UwNmNmIiwiaXNzdWVkX2F0IjoiMjAyNi0wNS0xMVQyMDo1MzoyNC4xOTBaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "mNh0iIBPQusrleK1qHzy32E9JY3RBhGJuNzOJiQ0_GU", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai", BUILD_DEFAULT_PROFILE = "";
 
 // ../../src/version.ts
 import { createHash } from "crypto";
@@ -1802,13 +1802,6 @@ var init_payments = __esm(() => {
   PRICING_API_URL = process.env.UNBROWSE_BACKEND_URL ?? DEFAULT_BACKEND_URL;
 });
 
-// ../../src/execution/robots.ts
-var TTL_MS2, cache;
-var init_robots = __esm(() => {
-  TTL_MS2 = 24 * 60 * 60 * 1000;
-  cache = new Map;
-});
-
 // ../../src/site-policy.ts
 var MUTATION_METHODS, KNOWN_SESSION_BOUND_PARAMS;
 var init_site_policy = __esm(() => {
@@ -1887,7 +1880,6 @@ var init_execution = __esm(async () => {
   init_version();
   init_search_forms();
   init_payments();
-  init_robots();
   init_site_policy();
   init_artifact();
   init_compile();
@@ -5723,8 +5715,7 @@ async function cmdResolve(flags) {
         params: { endpoint_id: endpointId, ...extraParams },
         intent,
         projection: { raw: true },
-        ...flags["confirm-third-party-terms"] ? { confirm_third_party_terms: true } : {},
-        ...flags["skip-robots"] ? { skip_robots_check: true } : {}
+        ...flags["confirm-third-party-terms"] ? { confirm_third_party_terms: true } : {}
       };
     }, endpointNeedsThirdPartyTermsConfirmation = function(endpoint) {
       return endpoint.requires_third_party_terms_confirmation === true;
@@ -5787,8 +5778,6 @@ async function cmdResolve(flags) {
       body.confirm_third_party_terms = true;
     if (flags["force-capture"])
       body.force_capture = true;
-    if (flags["skip-robots"])
-      body.skip_robots_check = true;
     if (flags["require-proof"])
       body.require_proof = true;
     const budgetFlag = flags.budget;
@@ -5959,8 +5948,6 @@ async function cmdRun(args, flags) {
       body.dry_run = true;
     if (flags["confirm-third-party-terms"])
       body.confirm_third_party_terms = true;
-    if (flags["skip-robots"])
-      body.skip_robots_check = true;
     const budgetFlag = flags.budget;
     if (typeof budgetFlag === "string") {
       const parsed = parseInt(budgetFlag, 10);
@@ -5976,8 +5963,7 @@ async function cmdRun(args, flags) {
       params: { endpoint_id: endpointId, url, ...extraParams },
       intent,
       projection: { raw: true },
-      ...flags["confirm-third-party-terms"] ? { confirm_third_party_terms: true } : {},
-      ...flags["skip-robots"] ? { skip_robots_check: true } : {}
+      ...flags["confirm-third-party-terms"] ? { confirm_third_party_terms: true } : {}
     };
   }
   function resolveSkillIdFrom(result) {
@@ -6386,8 +6372,6 @@ async function cmdExecute(flags) {
       body.confirm_unsafe = true;
     if (flags["confirm-third-party-terms"])
       body.confirm_third_party_terms = true;
-    if (flags["skip-robots"])
-      body.skip_robots_check = true;
     body.projection = { raw: true };
     let result = await withPendingNotice(api2("POST", `/v1/skills/${skillId}/execute`, body), "Still working. This endpoint may require browser replay or first-time auth/capture setup.");
     if (isResolveSuccessResult(result)) {

package/dist/mcp.js

@@ -230,11 +230,11 @@ import { dirname, join, parse } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "6.11.0";
-var BUILD_GIT_SHA = "9193343b3660";
+var BUILD_RELEASE_VERSION = "6.12.0";
+var BUILD_GIT_SHA = "cbea913e06cf";
 var BUILD_CODE_HASH = "5d9ebf619c61";
-var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi4xMS4wIiwiZ2l0X3NoYSI6IjkxOTMzNDNiMzY2MCIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAOTE5MzM0M2IzNjYwIiwiaXNzdWVkX2F0IjoiMjAyNi0wNS0xMVQxOToyNjo1Mi40MDdaIn0";
-var BUILD_RELEASE_MANIFEST_SIGNATURE = "Ai1ymvce0ORL9m3DFa9v14tFbAbeyy7XUfTDakeoHkA";
+var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi4xMi4wIiwiZ2l0X3NoYSI6ImNiZWE5MTNlMDZjZiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAY2JlYTkxM2UwNmNmIiwiaXNzdWVkX2F0IjoiMjAyNi0wNS0xMVQyMDo1MzoyNC4xOTBaIn0";
+var BUILD_RELEASE_MANIFEST_SIGNATURE = "mNh0iIBPQusrleK1qHzy32E9JY3RBhGJuNzOJiQ0_GU";
 var BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 var BUILD_DEFAULT_PROFILE = "";
 
@@ -1854,7 +1854,6 @@ var tools = [
         dry_run: { type: "boolean", description: "Preview unsafe calls without applying them." },
         confirm_third_party_terms: { type: "boolean", description: "Explicitly confirm policy-sensitive third-party terms risk for flagged domains/actions." },
         force_capture: { type: "boolean", description: "Bypass cache and re-capture the exact URL." },
-        skip_robots: { type: "boolean", description: "Bypass robots.txt compliance check." },
         raw: { type: "boolean", description: "Keep raw projection enabled. Default true." },
         schema: { type: "boolean", description: "Return a schema tree instead of data." },
         path: { type: "string", description: "Drill into the result before returning it, e.g. data.items[] ." },
@@ -1889,8 +1888,6 @@ var tools = [
         body.confirm_third_party_terms = true;
       if (args.force_capture === true)
         body.force_capture = true;
-      if (args.skip_robots === true)
-        body.skip_robots_check = true;
       let result = await api2("POST", "/v1/intent/resolve", body);
       const authError = resolveNestedError(result);
       if (authError === "auth_required") {
@@ -1924,7 +1921,6 @@ var tools = [
         dry_run: { type: "boolean", description: "Preview unsafe calls without applying them." },
         confirm_unsafe: { type: "boolean", description: "Confirm mutation if the endpoint is unsafe." },
         confirm_third_party_terms: { type: "boolean", description: "Explicitly confirm policy-sensitive third-party terms risk for flagged domains/actions." },
-        skip_robots: { type: "boolean", description: "Bypass robots.txt compliance check." },
         raw: { type: "boolean", description: "Keep raw projection enabled. Default true." },
         schema: { type: "boolean", description: "Return a schema tree instead of data." },
         path: { type: "string", description: "Drill into the result before returning it, e.g. data.items[] ." },
@@ -1954,8 +1950,6 @@ var tools = [
         body.confirm_unsafe = true;
       if (args.confirm_third_party_terms === true)
         body.confirm_third_party_terms = true;
-      if (args.skip_robots === true)
-        body.skip_robots_check = true;
       const result = await api2("POST", `/v1/skills/${args.skill}/execute`, body);
       const nestedError = resolveNestedError(result);
       recordImpactForTool("execute", result, args);

package/dist/server.js

@@ -7661,7 +7661,7 @@ var init_capture = __esm(async () => {
 });
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "6.11.0", BUILD_GIT_SHA = "9193343b3660", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi4xMS4wIiwiZ2l0X3NoYSI6IjkxOTMzNDNiMzY2MCIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAOTE5MzM0M2IzNjYwIiwiaXNzdWVkX2F0IjoiMjAyNi0wNS0xMVQxOToyNjo1Mi40MDdaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "Ai1ymvce0ORL9m3DFa9v14tFbAbeyy7XUfTDakeoHkA", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai", BUILD_DEFAULT_PROFILE = "";
+var BUILD_RELEASE_VERSION = "6.12.0", BUILD_GIT_SHA = "cbea913e06cf", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi4xMi4wIiwiZ2l0X3NoYSI6ImNiZWE5MTNlMDZjZiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAY2JlYTkxM2UwNmNmIiwiaXNzdWVkX2F0IjoiMjAyNi0wNS0xMVQyMDo1MzoyNC4xOTBaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "mNh0iIBPQusrleK1qHzy32E9JY3RBhGJuNzOJiQ0_GU", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai", BUILD_DEFAULT_PROFILE = "";
 
 // ../../src/version.ts
 import { createHash as createHash2 } from "crypto";
@@ -10011,8 +10011,6 @@ function classifyFailure(error) {
     return "empty_response";
   if (e.includes("missing") || e.includes("dependency"))
     return "dependency_missing";
-  if (e.includes("robots") || e.includes("disallowed"))
-    return "robots_disallowed";
   return "unknown";
 }
 function emitRouteTrace(params) {
@@ -15160,116 +15158,6 @@ var init_payments = __esm(() => {
   PRICING_API_URL = process.env.UNBROWSE_BACKEND_URL ?? DEFAULT_BACKEND_URL;
 });
 
-// ../../src/execution/robots.ts
-function parseRobotsTxt(text) {
-  const groups = [];
-  let current = null;
-  for (const rawLine of text.split(/\r?\n/)) {
-    const line = rawLine.replace(/#.*$/, "").trim();
-    if (!line) {
-      current = null;
-      continue;
-    }
-    const colon = line.indexOf(":");
-    if (colon === -1)
-      continue;
-    const field = line.slice(0, colon).trim().toLowerCase();
-    const value = line.slice(colon + 1).trim();
-    if (field === "user-agent") {
-      if (!current) {
-        current = { agents: [], disallow: [], allow: [] };
-        groups.push(current);
-      }
-      current.agents.push(value.toLowerCase());
-    } else if (field === "disallow") {
-      if (current && value)
-        current.disallow.push(value);
-    } else if (field === "allow") {
-      if (current && value)
-        current.allow.push(value);
-    } else if (field === "crawl-delay") {
-      if (current)
-        current.crawlDelay = parseFloat(value);
-    } else {
-      current = null;
-    }
-  }
-  return groups;
-}
-function selectRules(groups, agent) {
-  const lower = agent.toLowerCase();
-  const exact = groups.find((g) => g.agents.some((a) => a === lower));
-  if (exact)
-    return exact;
-  return groups.find((g) => g.agents.includes("*")) ?? null;
-}
-function pathMatches(path5, prefix) {
-  if (prefix.endsWith("$")) {
-    return path5 === prefix.slice(0, -1);
-  }
-  return path5.startsWith(prefix);
-}
-function longestMatch(path5, patterns) {
-  let best = -1;
-  for (const p of patterns) {
-    const base = p.endsWith("$") ? p.slice(0, -1) : p;
-    if (pathMatches(path5, p) && base.length > best)
-      best = base.length;
-  }
-  return best;
-}
-async function fetchRules(origin) {
-  const now = Date.now();
-  const cached = cache.get(origin);
-  if (cached && now - cached.fetchedAt < TTL_MS2)
-    return cached.rules;
-  try {
-    const res = await fetch(`${origin}/robots.txt`, {
-      headers: { "user-agent": USER_AGENT },
-      redirect: "follow",
-      signal: AbortSignal.timeout(5000)
-    });
-    if (!res.ok) {
-      cache.set(origin, { rules: [], fetchedAt: now });
-      return [];
-    }
-    const text = await res.text();
-    const rules = parseRobotsTxt(text);
-    cache.set(origin, { rules, fetchedAt: now });
-    return rules;
-  } catch {
-    cache.set(origin, { rules: [], fetchedAt: now });
-    return [];
-  }
-}
-async function isAllowedByRobots(url) {
-  let origin;
-  let pathname;
-  try {
-    const parsed = new URL(url);
-    origin = parsed.origin;
-    pathname = parsed.pathname || "/";
-  } catch {
-    return true;
-  }
-  const groups = await fetchRules(origin);
-  const rules = selectRules(groups, USER_AGENT);
-  if (!rules)
-    return true;
-  const allowLen = longestMatch(pathname, rules.allow);
-  const disallowLen = longestMatch(pathname, rules.disallow);
-  if (disallowLen < 0)
-    return true;
-  if (allowLen >= disallowLen)
-    return true;
-  return false;
-}
-var USER_AGENT = "unbrowse", TTL_MS2, cache;
-var init_robots = __esm(() => {
-  TTL_MS2 = 24 * 60 * 60 * 1000;
-  cache = new Map;
-});
-
 // ../../src/site-policy.ts
 function detectSessionBoundParams(endpoint) {
   if (!endpoint.query || typeof endpoint.query !== "object")
@@ -18885,11 +18773,6 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
     }
   }
   const hasAuthContext = cookies.length > 0 || Object.keys(authHeaders).length > 0 || !!skill.auth_profile_ref || endpoint.semantic?.auth_required === true;
-  if (!hasAuthContext) {
-    const allowed = await isAllowedByRobots(url);
-    if (!allowed)
-      log("exec", `robots.txt would block ${url} (not enforced)`);
-  }
   const serverFetch = async (extraHeaders = {}, bodyOverride = body) => {
     const endpointHeaders = normalizeReplayHeaders(endpoint.headers_template);
     const sessionHeaders = normalizeReplayHeaders(authHeaders);
@@ -20607,7 +20490,6 @@ var init_execution = __esm(async () => {
   init_version();
   init_search_forms();
   init_payments();
-  init_robots();
   init_site_policy();
   init_artifact();
   init_compile();
@@ -30047,7 +29929,7 @@ async function registerRoutes(app) {
   });
   app.post("/v1/intent/resolve", { config: { rateLimit: ROUTE_LIMITS["/v1/intent/resolve"] } }, async (req, reply) => {
     const clientScope = clientScopeFor(req);
-    const { intent, params, context, projection, confirm_unsafe, confirm_third_party_terms, dry_run, force_capture, skip_robots_check, visual_context, budget_ms, require_proof } = req.body;
+    const { intent, params, context, projection, confirm_unsafe, confirm_third_party_terms, dry_run, force_capture, visual_context, budget_ms, require_proof } = req.body;
     if (!intent)
       return reply.code(400).send({ error: "intent required" });
     let inflightFlushResult = null;
@@ -30070,7 +29952,7 @@ async function registerRoutes(app) {
       }
     }
     try {
-      const result = await resolveAndExecute(intent, params ?? {}, context, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, force_capture, skip_robots_check, client_scope: clientScope, budget_ms, require_proof });
+      const result = await resolveAndExecute(intent, params ?? {}, context, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, force_capture, client_scope: clientScope, budget_ms, require_proof });
       const res = attachAgentOutcomeHints({ ...result }, {
         skill: result.skill,
         endpointId: result.trace.endpoint_id,
@@ -30578,7 +30460,7 @@ async function registerRoutes(app) {
   app.post("/v1/skills/:skill_id/execute", { config: { rateLimit: ROUTE_LIMITS["/v1/skills/:skill_id/execute"] } }, async (req, reply) => {
     const clientScope = clientScopeFor(req);
     const { skill_id } = req.params;
-    const { params, projection, confirm_unsafe, confirm_third_party_terms, dry_run, intent, context_url, skip_robots_check } = req.body;
+    const { params, projection, confirm_unsafe, confirm_third_party_terms, dry_run, intent, context_url } = req.body;
     let skill = getRecentLocalSkill(skill_id, clientScope);
     if (!skill) {
       const { findExistingSkillForDomain: findLocal } = await Promise.resolve().then(() => (init_client2(), exports_client2));
@@ -30600,7 +30482,7 @@ async function registerRoutes(app) {
       ...context_url && typeof params?.url !== "string" ? { url: context_url } : {}
     };
     try {
-      let execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent, contextUrl: context_url, client_scope: clientScope });
+      let execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, intent, contextUrl: context_url, client_scope: clientScope });
       if (!execResult.trace.success && execResult.result?.error === "endpoint_not_found" && typeof execParams.endpoint_id === "string") {
         let recovered = false;
         const freshSkill = await getSkill2(skill_id, clientScope);
@@ -30635,7 +30517,7 @@ async function registerRoutes(app) {
           }
         }
         if (recovered) {
-          execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent, contextUrl: context_url, client_scope: clientScope });
+          execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, intent, contextUrl: context_url, client_scope: clientScope });
         }
       }
       saveTrace(execResult.trace);
@@ -30648,7 +30530,7 @@ async function registerRoutes(app) {
       if (execResult.trace.status_code === 404 && skill.domain && skill.intent_signature && skill.execution_type !== "browser-capture") {
         try {
           const recoveryUrl = context_url || typeof execParams.url === "string" && execParams.url || skill.endpoints.find((endpoint) => typeof endpoint.trigger_url === "string" && endpoint.trigger_url)?.trigger_url || `https://${skill.domain}`;
-          const freshResult = await resolveAndExecute(intent || skill.intent_signature, { ...execParams, url: recoveryUrl }, { url: recoveryUrl }, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent: intent || skill.intent_signature, client_scope: clientScope });
+          const freshResult = await resolveAndExecute(intent || skill.intent_signature, { ...execParams, url: recoveryUrl }, { url: recoveryUrl }, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, intent: intent || skill.intent_signature, client_scope: clientScope });
           saveTrace(freshResult.trace);
           if (freshResult.trace?.skill_id && freshResult.trace?.endpoint_id) {
             recordExecution(freshResult.trace.skill_id, freshResult.trace.endpoint_id, freshResult.trace, skill).catch(() => {});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "6.11.0",
+  "version": "6.12.0",
   "description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
   "type": "module",
   "bin": {

package/vendor/kuri/manifest.json

@@ -6,7 +6,7 @@
   "binaries": {
     "darwin-arm64": {
       "zig_target": "aarch64-macos",
-      "sha256": "6accd23464efaa2da57fe09636d82950b544126f800dc31d282110a238eaf003",
+      "sha256": "f1c110bfe6546c3434acd45b9f487efafaa18e09effa98e8cc6745152b6deb80",
       "source": "ci-built"
     },
     "darwin-x64": {