unbrowse 3.2.3 → 3.3.0

package/dist/cli.js

@@ -31,7 +31,7 @@ var __promiseAll = (args) => Promise.all(args);
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.2.2", BUILD_GIT_SHA = "150cce0d751e", BUILD_CODE_HASH = "1488fc1d92b7", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4yLjIiLCJnaXRfc2hhIjoiMTUwY2NlMGQ3NTFlIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0AxNTBjY2UwZDc1MWUiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDExOjU4OjM1LjkxNFoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "Lfph_KbGZrv74O5Pnop_b_0GcSTspB27fOgkCMZK6tk", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.3.0", BUILD_GIT_SHA = "d503455236d3", BUILD_CODE_HASH = "1488fc1d92b7", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4zLjAiLCJnaXRfc2hhIjoiZDUwMzQ1NTIzNmQzIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0BkNTAzNDU1MjM2ZDMiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDE2OjQyOjAxLjcyOFoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "QZKL6cxuZOauUsIdM3tms9r0wyOad9mfa9StUKd4v44", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash } from "crypto";
@@ -3387,7 +3387,8 @@ async function cmdResolve(flags) {
         params: { endpoint_id: endpointId, ...extraParams },
         intent,
         projection: { raw: true },
-        ...flags["confirm-third-party-terms"] ? { confirm_third_party_terms: true } : {}
+        ...flags["confirm-third-party-terms"] ? { confirm_third_party_terms: true } : {},
+        ...flags["skip-robots"] ? { skip_robots_check: true } : {}
       };
     }, endpointNeedsThirdPartyTermsConfirmation = function(endpoint) {
       return endpoint.requires_third_party_terms_confirmation === true;
@@ -3419,6 +3420,8 @@ async function cmdResolve(flags) {
       body.confirm_third_party_terms = true;
     if (flags["force-capture"])
       body.force_capture = true;
+    if (flags["skip-robots"])
+      body.skip_robots_check = true;
     body.projection = { raw: true };
     const startedAt = Date.now();
     async function resolveOnce(message = "Still working. Searching cached routes...") {
@@ -3628,6 +3631,8 @@ async function cmdExecute(flags) {
       body.confirm_unsafe = true;
     if (flags["confirm-third-party-terms"])
       body.confirm_third_party_terms = true;
+    if (flags["skip-robots"])
+      body.skip_robots_check = true;
     body.projection = { raw: true };
     let result = await withPendingNotice(api2("POST", `/v1/skills/${skillId}/execute`, body), "Still working. This endpoint may require browser replay or first-time auth/capture setup.");
     if (isResolveSuccessResult(result)) {

package/dist/mcp.js

@@ -225,11 +225,11 @@ import { dirname, join, parse } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.2.2";
-var BUILD_GIT_SHA = "150cce0d751e";
+var BUILD_RELEASE_VERSION = "3.3.0";
+var BUILD_GIT_SHA = "d503455236d3";
 var BUILD_CODE_HASH = "1488fc1d92b7";
-var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4yLjIiLCJnaXRfc2hhIjoiMTUwY2NlMGQ3NTFlIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0AxNTBjY2UwZDc1MWUiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDExOjU4OjM1LjkxNFoifQ";
-var BUILD_RELEASE_MANIFEST_SIGNATURE = "Lfph_KbGZrv74O5Pnop_b_0GcSTspB27fOgkCMZK6tk";
+var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4zLjAiLCJnaXRfc2hhIjoiZDUwMzQ1NTIzNmQzIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0BkNTAzNDU1MjM2ZDMiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDE2OjQyOjAxLjcyOFoifQ";
+var BUILD_RELEASE_MANIFEST_SIGNATURE = "QZKL6cxuZOauUsIdM3tms9r0wyOad9mfa9StUKd4v44";
 var BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
@@ -1658,6 +1658,7 @@ var tools = [
         dry_run: { type: "boolean", description: "Preview unsafe calls without applying them." },
         confirm_third_party_terms: { type: "boolean", description: "Explicitly confirm policy-sensitive third-party terms risk for flagged domains/actions." },
         force_capture: { type: "boolean", description: "Bypass cache and re-capture the exact URL." },
+        skip_robots: { type: "boolean", description: "Bypass robots.txt compliance check." },
         raw: { type: "boolean", description: "Keep raw projection enabled. Default true." },
         schema: { type: "boolean", description: "Return a schema tree instead of data." },
         path: { type: "string", description: "Drill into the result before returning it, e.g. data.items[] ." },
@@ -1692,6 +1693,8 @@ var tools = [
         body.confirm_third_party_terms = true;
       if (args.force_capture === true)
         body.force_capture = true;
+      if (args.skip_robots === true)
+        body.skip_robots_check = true;
       let result = await api2("POST", "/v1/intent/resolve", body);
       const authError = resolveNestedError(result);
       if (authError === "auth_required") {
@@ -1725,6 +1728,7 @@ var tools = [
         dry_run: { type: "boolean", description: "Preview unsafe calls without applying them." },
         confirm_unsafe: { type: "boolean", description: "Confirm mutation if the endpoint is unsafe." },
         confirm_third_party_terms: { type: "boolean", description: "Explicitly confirm policy-sensitive third-party terms risk for flagged domains/actions." },
+        skip_robots: { type: "boolean", description: "Bypass robots.txt compliance check." },
         raw: { type: "boolean", description: "Keep raw projection enabled. Default true." },
         schema: { type: "boolean", description: "Return a schema tree instead of data." },
         path: { type: "string", description: "Drill into the result before returning it, e.g. data.items[] ." },
@@ -1754,6 +1758,8 @@ var tools = [
         body.confirm_unsafe = true;
       if (args.confirm_third_party_terms === true)
         body.confirm_third_party_terms = true;
+      if (args.skip_robots === true)
+        body.skip_robots_check = true;
       const result = await api2("POST", `/v1/skills/${args.skill}/execute`, body);
       const nestedError = resolveNestedError(result);
       recordImpactForTool("execute", result, args);

package/dist/server.js

@@ -6227,12 +6227,17 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
       }
     }
     let harEntries = [];
-    try {
-      const harPromise = harStop(tabId);
-      const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve({ entries: [] }), 5000));
-      const harResult = await Promise.race([harPromise, timeoutPromise]);
-      harEntries = harResult.entries ?? [];
-    } catch {}
+    if (cdpRequestMap.size === 0) {
+      try {
+        const harResult = await Promise.race([
+          harStop(tabId),
+          new Promise((r) => setTimeout(() => r({ entries: [] }), 5000))
+        ]);
+        harEntries = harResult.entries ?? [];
+      } catch {}
+    } else {
+      harStop(tabId).catch(() => {});
+    }
     if (cdpPendingBodies.size > 0) {
       await new Promise((r) => setTimeout(r, 1500));
     }
@@ -6879,7 +6884,7 @@ var init_capture = __esm(async () => {
 });
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.2.2", BUILD_GIT_SHA = "150cce0d751e", BUILD_CODE_HASH = "1488fc1d92b7", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4yLjIiLCJnaXRfc2hhIjoiMTUwY2NlMGQ3NTFlIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0AxNTBjY2UwZDc1MWUiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDExOjU4OjM1LjkxNFoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "Lfph_KbGZrv74O5Pnop_b_0GcSTspB27fOgkCMZK6tk", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.3.0", BUILD_GIT_SHA = "d503455236d3", BUILD_CODE_HASH = "1488fc1d92b7", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4zLjAiLCJnaXRfc2hhIjoiZDUwMzQ1NTIzNmQzIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0BkNTAzNDU1MjM2ZDMiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDE2OjQyOjAxLjcyOFoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "QZKL6cxuZOauUsIdM3tms9r0wyOad9mfa9StUKd4v44", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash } from "crypto";
@@ -13237,6 +13242,8 @@ function getEndpointPolicy(endpoint) {
     return;
   if (matched.require_for_mutations && !MUTATION_METHODS.has(endpoint.method))
     return;
+  if (endpoint.idempotency === "safe" || endpoint.idempotency === "read")
+    return;
   return {
     requires_third_party_terms_confirmation: true,
     policy_domain: matched.domain,
@@ -25197,11 +25204,11 @@ async function registerRoutes(app) {
   });
   app.post("/v1/intent/resolve", { config: { rateLimit: ROUTE_LIMITS["/v1/intent/resolve"] } }, async (req, reply) => {
     const clientScope = clientScopeFor(req);
-    const { intent, params, context, projection, confirm_unsafe, confirm_third_party_terms, dry_run, force_capture } = req.body;
+    const { intent, params, context, projection, confirm_unsafe, confirm_third_party_terms, dry_run, force_capture, skip_robots_check } = req.body;
     if (!intent)
       return reply.code(400).send({ error: "intent required" });
     try {
-      const result = await resolveAndExecute(intent, params ?? {}, context, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, force_capture, client_scope: clientScope });
+      const result = await resolveAndExecute(intent, params ?? {}, context, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, force_capture, skip_robots_check, client_scope: clientScope });
       const res = attachAgentOutcomeHints({ ...result }, {
         skill: result.skill,
         endpointId: result.trace.endpoint_id,
@@ -25398,7 +25405,7 @@ async function registerRoutes(app) {
   app.post("/v1/skills/:skill_id/execute", { config: { rateLimit: ROUTE_LIMITS["/v1/skills/:skill_id/execute"] } }, async (req, reply) => {
     const clientScope = clientScopeFor(req);
     const { skill_id } = req.params;
-    const { params, projection, confirm_unsafe, confirm_third_party_terms, dry_run, intent, context_url } = req.body;
+    const { params, projection, confirm_unsafe, confirm_third_party_terms, dry_run, intent, context_url, skip_robots_check } = req.body;
     let skill = getRecentLocalSkill(skill_id, clientScope);
     if (!skill) {
       const { findExistingSkillForDomain: findLocal } = await Promise.resolve().then(() => (init_client2(), exports_client2));
@@ -25420,7 +25427,7 @@ async function registerRoutes(app) {
       ...context_url && typeof params?.url !== "string" ? { url: context_url } : {}
     };
     try {
-      const execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, intent, contextUrl: context_url, client_scope: clientScope });
+      const execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent, contextUrl: context_url, client_scope: clientScope });
       saveTrace(execResult.trace);
       if (execResult.trace.endpoint_id) {
         recordExecution(skill.skill_id, execResult.trace.endpoint_id, execResult.trace, skill).catch(() => {});
@@ -25431,7 +25438,7 @@ async function registerRoutes(app) {
       if (execResult.trace.status_code === 404 && skill.domain && skill.intent_signature && skill.execution_type !== "browser-capture") {
         try {
           const recoveryUrl = context_url || typeof execParams.url === "string" && execParams.url || skill.endpoints.find((endpoint) => typeof endpoint.trigger_url === "string" && endpoint.trigger_url)?.trigger_url || `https://${skill.domain}`;
-          const freshResult = await resolveAndExecute(intent || skill.intent_signature, { ...execParams, url: recoveryUrl }, { url: recoveryUrl }, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, intent: intent || skill.intent_signature, client_scope: clientScope });
+          const freshResult = await resolveAndExecute(intent || skill.intent_signature, { ...execParams, url: recoveryUrl }, { url: recoveryUrl }, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent: intent || skill.intent_signature, client_scope: clientScope });
           saveTrace(freshResult.trace);
           if (freshResult.trace?.skill_id && freshResult.trace?.endpoint_id) {
             recordExecution(freshResult.trace.skill_id, freshResult.trace.endpoint_id, freshResult.trace, skill).catch(() => {});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "3.2.3",
+  "version": "3.3.0",
   "description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
   "type": "module",
   "bin": {