unbrowse 6.5.2 → 6.6.0-preview.0

package/dist/cli.js

@@ -31,7 +31,7 @@ var __promiseAll = (args) => Promise.all(args);
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "6.5.2", BUILD_GIT_SHA = "2bedcaef3363", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi41LjIiLCJnaXRfc2hhIjoiMmJlZGNhZWYzMzYzIiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUAyYmVkY2FlZjMzNjMiLCJpc3N1ZWRfYXQiOiIyMDI2LTA1LTAzVDIzOjM2OjIyLjUxM1oifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "wNWNFYL91xCcb02bCfQvbVnHfmLAUCyHeX-oVLv2Wy4", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai", BUILD_DEFAULT_PROFILE = "";
+var BUILD_RELEASE_VERSION = "6.6.0-preview.0", BUILD_GIT_SHA = "e74cd1481aa5", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi42LjAtcHJldmlldy4wIiwiZ2l0X3NoYSI6ImU3NGNkMTQ4MWFhNSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAZTc0Y2QxNDgxYWE1IiwiaXNzdWVkX2F0IjoiMjAyNi0wNS0wNFQwNTo1MTo1Mi4yNTNaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "_hrkIsHrYUpg_q-N3P2xraayyLNVbgdyB8-8cRupt6U", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai", BUILD_DEFAULT_PROFILE = "";
 
 // ../../src/version.ts
 import { createHash } from "crypto";
@@ -6609,10 +6609,25 @@ function printHelp() {
 async function cmdStatus(flags) {
   const healthy = await fetch(`${BASE_URL}/health`, { signal: AbortSignal.timeout(2000) }).then((r) => r.ok).catch(() => false);
   const versionInfo = checkServerVersion(BASE_URL, import.meta.url);
+  let activeSessions = [];
+  let sessionCount = 0;
+  if (healthy) {
+    try {
+      const res = await fetch(`${BASE_URL}/v1/browse/sessions`, { signal: AbortSignal.timeout(2000) });
+      if (res.ok) {
+        const data = await res.json();
+        activeSessions = data.sessions ?? [];
+        sessionCount = data.count ?? activeSessions.length;
+      }
+    } catch {}
+  }
   output({
     server: healthy ? "running" : "stopped",
     url: BASE_URL,
-    ...versionInfo ?? {}
+    ...versionInfo ?? {},
+    active_browse_sessions: sessionCount,
+    sessions: activeSessions,
+    chrome_debug_url: process.env.CHROME_DEBUG_URL ?? null
   }, !!flags.pretty);
 }
 async function cmdRestart(flags) {

package/dist/mcp.js

@@ -226,11 +226,11 @@ import { dirname, join, parse } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "6.5.2";
-var BUILD_GIT_SHA = "2bedcaef3363";
+var BUILD_RELEASE_VERSION = "6.6.0-preview.0";
+var BUILD_GIT_SHA = "e74cd1481aa5";
 var BUILD_CODE_HASH = "5d9ebf619c61";
-var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi41LjIiLCJnaXRfc2hhIjoiMmJlZGNhZWYzMzYzIiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUAyYmVkY2FlZjMzNjMiLCJpc3N1ZWRfYXQiOiIyMDI2LTA1LTAzVDIzOjM2OjIyLjUxM1oifQ";
-var BUILD_RELEASE_MANIFEST_SIGNATURE = "wNWNFYL91xCcb02bCfQvbVnHfmLAUCyHeX-oVLv2Wy4";
+var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi42LjAtcHJldmlldy4wIiwiZ2l0X3NoYSI6ImU3NGNkMTQ4MWFhNSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAZTc0Y2QxNDgxYWE1IiwiaXNzdWVkX2F0IjoiMjAyNi0wNS0wNFQwNTo1MTo1Mi4yNTNaIn0";
+var BUILD_RELEASE_MANIFEST_SIGNATURE = "_hrkIsHrYUpg_q-N3P2xraayyLNVbgdyB8-8cRupt6U";
 var BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 var BUILD_DEFAULT_PROFILE = "";
 

package/dist/server.js

@@ -211,13 +211,11 @@ function resolveKuriLaunchConfig(env = process.env) {
     headless = true;
   }
   const cleanRoom = envFlag(env.UNBROWSE_LOCAL_ONLY) || envFlag(env.KURI_CLEAN_ROOM);
-  const browserCookieOptOut = falseyEnv(env.UNBROWSE_IMPORT_BROWSER_COOKIES);
-  const explicitAttach = envFlag(env.KURI_ATTACH_EXISTING_CHROME ?? env.UNBROWSE_ATTACH_EXISTING_CHROME);
   const disableCdpAttach = envFlag(env.KURI_DISABLE_CDP_ATTACH);
-  const canAttachToExistingChrome = !headless && !disableCdpAttach && !cleanRoom;
+  const attachToExistingChrome = !disableCdpAttach && !cleanRoom;
   return {
     headless,
-    attachToExistingChrome: canAttachToExistingChrome && (explicitAttach || browserCookieOptOut)
+    attachToExistingChrome
   };
 }
 function kuriBinaryName() {
@@ -7344,7 +7342,7 @@ var init_capture = __esm(async () => {
 });
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "6.5.2", BUILD_GIT_SHA = "2bedcaef3363", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi41LjIiLCJnaXRfc2hhIjoiMmJlZGNhZWYzMzYzIiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUAyYmVkY2FlZjMzNjMiLCJpc3N1ZWRfYXQiOiIyMDI2LTA1LTAzVDIzOjM2OjIyLjUxM1oifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "wNWNFYL91xCcb02bCfQvbVnHfmLAUCyHeX-oVLv2Wy4", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai", BUILD_DEFAULT_PROFILE = "";
+var BUILD_RELEASE_VERSION = "6.6.0-preview.0", BUILD_GIT_SHA = "e74cd1481aa5", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiNi42LjAtcHJldmlldy4wIiwiZ2l0X3NoYSI6ImU3NGNkMTQ4MWFhNSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAZTc0Y2QxNDgxYWE1IiwiaXNzdWVkX2F0IjoiMjAyNi0wNS0wNFQwNTo1MTo1Mi4yNTNaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "_hrkIsHrYUpg_q-N3P2xraayyLNVbgdyB8-8cRupt6U", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai", BUILD_DEFAULT_PROFILE = "";
 
 // ../../src/version.ts
 import { createHash as createHash2 } from "crypto";
@@ -28155,6 +28153,74 @@ async function registerRoutes(app) {
       next_step: settings.auto_publish_checkpoints ? "Auto-publish after sync/close is enabled unless a domain rule blocks it." : "Auto-publish after sync/close is disabled. Use index for local recompute and publish only when you explicitly want remote share."
     });
   });
+  app.get("/v1/browse/sessions", async (_req, reply) => {
+    const sessions = Array.from(browseSessions.values()).map((s) => ({
+      session_id: s.sessionId,
+      tab_id: s.tabId,
+      url: s.url,
+      domain: s.domain,
+      har_active: s.harActive,
+      broker_port: s.brokerPort ?? null,
+      streaming_publish_active: streamingWatchers.has(s.sessionId)
+    }));
+    return reply.send({ sessions, count: sessions.length });
+  });
+  app.get("/v1/browse/sessions/:id/buffer", async (req, reply) => {
+    const { id } = req.params;
+    let session;
+    for (const s of browseSessions.values()) {
+      if (s.sessionId === id || s.tabId === id) {
+        session = s;
+        break;
+      }
+    }
+    if (!session)
+      return reply.code(404).send({ error: "session_not_found", id });
+    let intercepted = [];
+    let interceptError = null;
+    try {
+      intercepted = await collectInterceptedRequests(session.tabId);
+    } catch (err) {
+      interceptError = err instanceof Error ? err.message : String(err);
+    }
+    let harEntries = [];
+    let harError = null;
+    if (session.harActive) {
+      try {
+        const broker = brokerForSession(session);
+        const stopResult = await broker.harStop(session.tabId);
+        harEntries = stopResult.entries ?? [];
+        try {
+          await broker.harStart(session.tabId);
+        } catch {}
+      } catch (err) {
+        harError = err instanceof Error ? err.message : String(err);
+      }
+    }
+    return reply.send({
+      session: {
+        session_id: session.sessionId,
+        tab_id: session.tabId,
+        url: session.url,
+        domain: session.domain,
+        har_active: session.harActive,
+        streaming_publish_active: streamingWatchers.has(session.sessionId)
+      },
+      intercepted_requests: intercepted,
+      intercepted_count: Array.isArray(intercepted) ? intercepted.length : 0,
+      intercept_error: interceptError,
+      har_entries: harEntries.map((e) => ({
+        url: e?.request?.url,
+        method: e?.request?.method,
+        status: e?.response?.status,
+        mime_type: e?.response?.content?.mimeType,
+        size: e?.response?.bodySize
+      })),
+      har_count: harEntries.length,
+      har_error: harError,
+      total_captured: (Array.isArray(intercepted) ? intercepted.length : 0) + harEntries.length
+    });
+  });
   app.get("/v1/trace/:trace_id", async (req, reply) => {
     const { trace_id } = req.params;
     const traceDir = path6.join(process.env.UNBROWSE_TRACE_DIR ?? path6.join(os4.homedir(), ".unbrowse", "traces"));
@@ -28181,6 +28247,25 @@ async function registerRoutes(app) {
     const { intent, params, context, projection, confirm_unsafe, confirm_third_party_terms, dry_run, force_capture, skip_robots_check, visual_context, budget_ms } = req.body;
     if (!intent)
       return reply.code(400).send({ error: "intent required" });
+    let inflightFlushResult = null;
+    if (context?.url && !force_capture) {
+      try {
+        const activeSession = findActiveSessionForDomain(context.url);
+        if (activeSession) {
+          const flushed = await lightFlushBrowseCapture(activeSession);
+          if (flushed.request_count > 0) {
+            inflightFlushResult = {
+              skill_id: flushed.skill_id,
+              request_count: flushed.request_count,
+              endpoint_count: flushed.endpoint_count
+            };
+            console.log(`[in-flight-flush] session=${activeSession.sessionId.slice(0, 8)} domain=${flushed.domain} requests=${flushed.request_count} endpoints=${flushed.endpoint_count} skill=${flushed.skill_id?.slice(0, 15) ?? "none"}`);
+          }
+        }
+      } catch (err) {
+        console.warn(`[in-flight-flush] failed: ${err.message}`);
+      }
+    }
     try {
       const result = await resolveAndExecute(intent, params ?? {}, context, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, force_capture, skip_robots_check, client_scope: clientScope, budget_ms });
       const res = attachAgentOutcomeHints({ ...result }, {
@@ -28209,6 +28294,9 @@ async function registerRoutes(app) {
           }
         } catch {}
       }
+      if (inflightFlushResult) {
+        res.inflight_flush = inflightFlushResult;
+      }
       return reply.send(res);
     } catch (err) {
       return reply.code(500).send({ error: err.message });
@@ -28853,6 +28941,141 @@ async function registerRoutes(app) {
     session.harActive = true;
     await injectInterceptor(session.tabId).catch(() => {});
   }
+  async function lightFlushBrowseCapture(session) {
+    let harEntries = [];
+    if (session.harActive) {
+      try {
+        const broker = brokerForSession(session);
+        const stopResult = await broker.harStop(session.tabId);
+        harEntries = stopResult.entries ?? [];
+        try {
+          await broker.harStart(session.tabId);
+        } catch {}
+      } catch {}
+    }
+    const allRequests = await enrichPassiveCaptureRequests({
+      tabId: session.tabId,
+      captureUrl: session.url,
+      harEntries,
+      intent: `browse ${session.domain || profileName(session.url)}`
+    });
+    if (allRequests.length === 0) {
+      return { skill_id: null, domain: session.domain, request_count: 0, endpoint_count: 0 };
+    }
+    const jsBundles = new Map;
+    try {
+      const intercepted = await collectInterceptedRequests(session.tabId).catch(() => []);
+      for (const entry of intercepted) {
+        if (entry.is_js && entry.response_body && jsBundles.size < 20) {
+          jsBundles.set(entry.url, entry.response_body);
+        }
+      }
+    } catch {}
+    const syncResult = await cacheBrowseRequests({
+      sessionUrl: session.url,
+      sessionDomain: session.domain,
+      requests: allRequests,
+      getPageHtml: () => brokerForSession(session).getPageHtml(session.tabId),
+      jsBundles: jsBundles.size > 0 ? jsBundles : undefined,
+      intent: `browse ${session.domain || profileName(session.url)}`
+    });
+    if (syncResult.skill && syncResult.domain) {
+      const domainKey = getDomainReuseKey(session.url || syncResult.domain);
+      if (domainKey) {
+        const cacheKey2 = scopedCacheKey("local", `${domainKey}:${syncResult.skill.skill_id}`);
+        writeSkillSnapshot(cacheKey2, syncResult.skill);
+        domainSkillCache.set(domainKey, {
+          skillId: syncResult.skill.skill_id,
+          localSkillPath: snapshotPathForCacheKey(cacheKey2),
+          ts: Date.now()
+        });
+        persistDomainCache();
+      }
+    }
+    return {
+      skill_id: syncResult.skill?.skill_id ?? null,
+      domain: syncResult.domain,
+      request_count: allRequests.length,
+      endpoint_count: syncResult.skill?.endpoints.length ?? 0
+    };
+  }
+  function findActiveSessionForDomain(targetUrl) {
+    const targetDomain = getDomainReuseKey(targetUrl);
+    if (!targetDomain)
+      return null;
+    let match = null;
+    for (const session of browseSessions.values()) {
+      if (!session.harActive)
+        continue;
+      const sessionDomain = getDomainReuseKey(session.url);
+      if (sessionDomain === targetDomain)
+        match = session;
+    }
+    return match;
+  }
+  const streamingWatchers = new Map;
+  const STREAMING_INTERVAL_MS = parseInt(process.env.UNBROWSE_STREAMING_INTERVAL_MS ?? "10000", 10);
+  const STREAMING_ENABLED = process.env.UNBROWSE_STREAMING_PUBLISH !== "0";
+  const streamingState = new Map;
+  function startStreamingWatcher(session) {
+    if (!STREAMING_ENABLED)
+      return;
+    if (streamingWatchers.has(session.sessionId))
+      return;
+    streamingState.set(session.sessionId, { lastEndpointCount: 0, lastSkillId: null });
+    const tick = async () => {
+      if (!browseSessions.has(session.sessionId)) {
+        stopStreamingWatcher(session.sessionId);
+        return;
+      }
+      const live = browseSessions.get(session.sessionId);
+      if (!live || !live.harActive)
+        return;
+      try {
+        const flushed = await lightFlushBrowseCapture(live);
+        const state = streamingState.get(session.sessionId);
+        if (!state)
+          return;
+        const grew = flushed.endpoint_count > state.lastEndpointCount || flushed.skill_id && flushed.skill_id !== state.lastSkillId;
+        if (grew && flushed.skill_id) {
+          state.lastEndpointCount = flushed.endpoint_count;
+          state.lastSkillId = flushed.skill_id;
+          const domainKey = getDomainReuseKey(live.url || flushed.domain);
+          const cacheEntry = domainKey ? domainSkillCache.get(domainKey) : null;
+          if (cacheEntry?.localSkillPath) {
+            const skill = readSkillSnapshot(cacheEntry.localSkillPath);
+            if (skill) {
+              const decision = decideCheckpointPublish(flushed.domain);
+              queueBackgroundIndex({
+                skill: { ...skill },
+                domain: flushed.domain,
+                intent: skill.intent_signature || `browse ${flushed.domain}`,
+                contextUrl: live.url,
+                cacheKey: `streaming:${flushed.domain}:${Date.now()}`,
+                publishAfterIndex: decision.publishQueued
+              });
+              console.log(`[streaming-publish] session=${session.sessionId.slice(0, 8)} domain=${flushed.domain} endpoints=${flushed.endpoint_count} publish=${decision.publishQueued}`);
+            }
+          }
+        }
+      } catch (err) {
+        console.warn(`[streaming-publish] tick failed: ${err.message}`);
+      }
+    };
+    const timer = setInterval(() => {
+      tick();
+    }, STREAMING_INTERVAL_MS);
+    streamingWatchers.set(session.sessionId, timer);
+    console.log(`[streaming-publish] watcher started session=${session.sessionId.slice(0, 8)} interval=${STREAMING_INTERVAL_MS}ms`);
+  }
+  function stopStreamingWatcher(sessionId) {
+    const timer = streamingWatchers.get(sessionId);
+    if (timer) {
+      clearInterval(timer);
+      streamingWatchers.delete(sessionId);
+    }
+    streamingState.delete(sessionId);
+  }
   async function flushBrowseCapture(session, options = {}) {
     let harEntries = [];
     if (session.harActive) {
@@ -29061,6 +29284,7 @@ async function registerRoutes(app) {
           }
         }
       } catch {}
+      startStreamingWatcher(session);
       return reply.send({
         ok: true,
         session_id: session.sessionId,
@@ -29068,7 +29292,20 @@ async function registerRoutes(app) {
         tab_id: session.tabId,
         auth_profile: session.domain,
         ...result.cookiesInjected > 0 ? { cookies_injected: result.cookiesInjected } : {},
-        ...authRequired ? { auth_required: true, auth_hint: authHint } : {}
+        ...authRequired ? { auth_required: true, auth_hint: authHint } : {},
+        autonomy: (() => {
+          const publishDecision = decideCheckpointPublish(session.domain);
+          return {
+            har_active: session.harActive,
+            streaming_publish_active: streamingWatchers.has(session.sessionId),
+            attached_existing_chrome: result.attachedExistingChrome ?? false,
+            chrome_debug_url: process.env.CHROME_DEBUG_URL ?? null,
+            inspect_buffer: `GET ${process.env.UNBROWSE_API_BASE ?? "http://127.0.0.1:6969"}/v1/browse/sessions/${session.sessionId}/buffer`,
+            marketplace_publish_enabled: publishDecision.publishQueued,
+            marketplace_publish_mode: publishDecision.mode,
+            marketplace_publish_reason: publishDecision.reason
+          };
+        })()
       });
     } catch (error) {
       return sendBrowseSessionError(reply, error);
@@ -29110,6 +29347,7 @@ async function registerRoutes(app) {
       activeSession.domain = profileName(activeSession.url);
       const stillLive = await isBrowseSessionLive(activeSession, browseClient).catch(() => false);
       if (!stillLive) {
+        stopStreamingWatcher(activeSession.sessionId);
         removeBrowseSession(browseSessions, activeSession.sessionId);
         throw new BrowseSessionError("session_expired");
       }
@@ -29337,6 +29575,7 @@ async function registerRoutes(app) {
           await saveAuthProfileBestEffort(session2.tabId, session2.domain, "browse_close");
         }
         const syncResult2 = await flushBrowseCapture(session2, { queueIndex: true, queuePublish: true });
+        stopStreamingWatcher(session2.sessionId);
         await broker.closeTab(session2.tabId).catch(() => {});
         removeBrowseSession(browseSessions, session2.sessionId);
         return syncResult2;
@@ -29459,6 +29698,16 @@ async function startUnbrowseServer(options = {}) {
     process.env.UNBROWSE_SKIP_TOS_CHECK = "1";
   }
   startBackgroundRegistration();
+  const UNBROWSE_CDP_PORT = Number(process.env.UNBROWSE_CDP_PORT ?? 9222);
+  if (!process.env.CHROME_DEBUG_URL) {
+    process.env.CHROME_DEBUG_URL = `http://127.0.0.1:${UNBROWSE_CDP_PORT}`;
+  }
+  if (!process.env.PUPPETEER_BROWSER_WS_ENDPOINT) {
+    process.env.PUPPETEER_BROWSER_WS_ENDPOINT = `ws://127.0.0.1:${UNBROWSE_CDP_PORT}`;
+  }
+  if (!process.env.PLAYWRIGHT_CHROMIUM_REMOTE_DEBUGGING_URL) {
+    process.env.PLAYWRIGHT_CHROMIUM_REMOTE_DEBUGGING_URL = `http://127.0.0.1:${UNBROWSE_CDP_PORT}`;
+  }
   const app = Fastify({ logger: options.logger ?? true });
   await app.register(cors, { origin: true });
   await registerRateLimiter(app);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "6.5.2",
+  "version": "6.6.0-preview.0",
   "description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
   "type": "module",
   "bin": {