unbrowse 3.2.0 → 3.2.2-experiments.128ff09

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/server.js CHANGED
@@ -76,6 +76,88 @@ function getPackageRoot(metaUrl) {
76
76
  var init_paths = () => {};
77
77
 
78
78
  // ../../src/kuri/client.ts
79
+ var exports_client = {};
80
+ __export(exports_client, {
81
+ waitForSelector: () => waitForSelector,
82
+ waitForLoad: () => waitForLoad,
83
+ waitForCloudflare: () => waitForCloudflare,
84
+ stop: () => stop,
85
+ start: () => start,
86
+ snapshot: () => snapshot,
87
+ shouldReuseManagedChrome: () => shouldReuseManagedChrome,
88
+ setViewport: () => setViewport,
89
+ setUserAgent: () => setUserAgent,
90
+ setHeaders: () => setHeaders,
91
+ setCredentials: () => setCredentials,
92
+ setCookies: () => setCookies,
93
+ setCookie: () => setCookie,
94
+ setCdpPortForTests: () => setCdpPortForTests,
95
+ sessionSave: () => sessionSave,
96
+ sessionLoad: () => sessionLoad,
97
+ sessionList: () => sessionList,
98
+ select: () => select,
99
+ scrollIntoView: () => scrollIntoView,
100
+ scroll: () => scroll,
101
+ scriptInject: () => scriptInject,
102
+ screenshot: () => screenshot,
103
+ reuseHealthyBrokerIfPossible: () => reuseHealthyBrokerIfPossible,
104
+ resolveKuriPort: () => resolveKuriPort,
105
+ resolveKuriLaunchConfig: () => resolveKuriLaunchConfig,
106
+ reload: () => reload,
107
+ press: () => press,
108
+ newTab: () => newTab,
109
+ networkEnable: () => networkEnable,
110
+ navigate: () => navigate,
111
+ keyboardType: () => keyboardType,
112
+ keyboardInsertText: () => keyboardInsertText,
113
+ keyUp: () => keyUp,
114
+ keyDown: () => keyDown,
115
+ isReady: () => isReady,
116
+ interceptStart: () => interceptStart,
117
+ health: () => health,
118
+ hasCloudflareChallenge: () => hasCloudflareChallenge,
119
+ harStop: () => harStop,
120
+ harStart: () => harStart,
121
+ goForward: () => goForward,
122
+ goBack: () => goBack,
123
+ getText: () => getText,
124
+ getPort: () => getPort,
125
+ getPerfLcp: () => getPerfLcp,
126
+ getPageHtml: () => getPageHtml,
127
+ getNetworkEvents: () => getNetworkEvents,
128
+ getMarkdown: () => getMarkdown,
129
+ getLinks: () => getLinks,
130
+ getKuriSourceCandidates: () => getKuriSourceCandidates,
131
+ getKuriErrorMessage: () => getKuriErrorMessage,
132
+ getKuriClient: () => getKuriClient,
133
+ getKuriBinaryCandidates: () => getKuriBinaryCandidates,
134
+ getErrors: () => getErrors,
135
+ getDefaultTab: () => getDefaultTab,
136
+ getCurrentUrl: () => getCurrentUrl,
137
+ getCookies: () => getCookies,
138
+ getConsole: () => getConsole,
139
+ getCdpPort: () => getCdpPort,
140
+ findText: () => findText,
141
+ findKuriBinary: () => findKuriBinary,
142
+ fill: () => fill,
143
+ extractLoadPluginsFromHtml: () => extractLoadPluginsFromHtml,
144
+ extractLoadPlugins: () => extractLoadPlugins,
145
+ executeInPageFetch: () => executeInPageFetch,
146
+ evaluate: () => evaluate,
147
+ drag: () => drag,
148
+ domQuery: () => domQuery,
149
+ domHtml: () => domHtml,
150
+ domAttributes: () => domAttributes,
151
+ discoverTabs: () => discoverTabs,
152
+ closeTab: () => closeTab,
153
+ click: () => click,
154
+ bestEffortRehydratePlugins: () => bestEffortRehydratePlugins,
155
+ authProfileSave: () => authProfileSave,
156
+ authProfileLoad: () => authProfileLoad,
157
+ authProfileList: () => authProfileList,
158
+ authProfileDelete: () => authProfileDelete,
159
+ action: () => action
160
+ });
79
161
  import { execFileSync, spawn } from "node:child_process";
80
162
  import { existsSync as existsSync2 } from "node:fs";
81
163
  import net from "node:net";
@@ -977,6 +1059,19 @@ async function getPageHtml(tabId, state = defaultBrokerState) {
977
1059
  const result = await evaluate(tabId, "document.documentElement.outerHTML", state);
978
1060
  return String(result ?? "");
979
1061
  }
1062
+ function extractLoadPlugins(value) {
1063
+ if (typeof value !== "string")
1064
+ return [];
1065
+ return Array.from(new Set(value.split(/[\s,;]+/).map((part) => part.trim()).filter(Boolean)));
1066
+ }
1067
+ function extractLoadPluginsFromHtml(html) {
1068
+ const modules = [];
1069
+ const pattern = /data-load-plugins=(["'])(.*?)\1/gi;
1070
+ for (const match of html.matchAll(pattern)) {
1071
+ modules.push(...extractLoadPlugins(match[2]));
1072
+ }
1073
+ return Array.from(new Set(modules));
1074
+ }
980
1075
  async function bestEffortRehydratePlugins(tabId, state = defaultBrokerState) {
981
1076
  const result = await evaluate(tabId, `(async function() {
982
1077
  function splitPlugins(value) {
@@ -1118,6 +1213,9 @@ function getPort(state = defaultBrokerState) {
1118
1213
  function getCdpPort() {
1119
1214
  return kuriCdpPort;
1120
1215
  }
1216
+ function setCdpPortForTests(port) {
1217
+ kuriCdpPort = port;
1218
+ }
1121
1219
  function isReady(state = defaultBrokerState) {
1122
1220
  return state.ready;
1123
1221
  }
@@ -4112,7 +4210,7 @@ function extractEndpoints(requests, wsMessages, context) {
4112
4210
  return "";
4113
4211
  }
4114
4212
  })();
4115
- const isApiUrl = /\/(api|graphql)\b/i.test(urlPath) || /\.(json)(\?|$)/.test(req.url);
4213
+ const isApiUrl = /\/(api|graphql|youtubei|__ssr_data__)\b/i.test(urlPath) || /\.(json)(\?|$)/.test(req.url);
4116
4214
  let graphqlOpName;
4117
4215
  if (/graphql/i.test(req.url)) {
4118
4216
  if (req.request_body) {
@@ -4201,7 +4299,12 @@ function extractEndpoints(requests, wsMessages, context) {
4201
4299
  const sanitizedQParams = isGet ? sanitizeQueryParams(extractQueryParams(req.url)) : undefined;
4202
4300
  let pathTemplate = sanitizeUrlTemplate(normalized);
4203
4301
  const qBindings = sanitizedQParams ? buildQueryBindingMap(Object.keys(sanitizedQParams)) : {};
4204
- const qTemplateStr = sanitizedQParams && Object.keys(sanitizedQParams).length > 0 ? Object.keys(sanitizedQParams).map((k) => `${encodeURIComponent(k)}={${qBindings[k] ?? k}}`).join("&") : null;
4302
+ const existingQKeys = new Set;
4303
+ try {
4304
+ for (const k of new URL(pathTemplate).searchParams.keys())
4305
+ existingQKeys.add(k.toLowerCase());
4306
+ } catch {}
4307
+ const qTemplateStr = sanitizedQParams && Object.keys(sanitizedQParams).length > 0 ? Object.keys(sanitizedQParams).filter((k) => !existingQKeys.has(k.toLowerCase())).map((k) => `${encodeURIComponent(k)}={${qBindings[k] ?? k}}`).join("&") || null : null;
4205
4308
  const { url: templatizedPath, pathParams, pathBindingCandidates } = templatizePathSegments(pathTemplate, req.url, context);
4206
4309
  pathTemplate = templatizedPath;
4207
4310
  const parsedRequestBody = !isGet && req.request_body ? tryParseBody(req.request_body) : undefined;
@@ -4217,7 +4320,7 @@ function extractEndpoints(requests, wsMessages, context) {
4217
4320
  let endpoint = {
4218
4321
  endpoint_id: nanoid2(),
4219
4322
  method: req.method,
4220
- url_template: qTemplateStr ? `${pathTemplate}?${qTemplateStr}` : pathTemplate,
4323
+ url_template: qTemplateStr ? `${pathTemplate}${pathTemplate.includes("?") ? "&" : "?"}${qTemplateStr}` : pathTemplate,
4221
4324
  description: buildEndpointDescription(req, sampleRequest, sampleResponse),
4222
4325
  headers_template: sanitizeHeaders(req.request_headers),
4223
4326
  query: sanitizedQParams,
@@ -4848,7 +4951,7 @@ var init_reverse_engineer = __esm(() => {
4848
4951
  SKIP_HOSTS = /(cloudflare\.com|google-analytics\.com|doubleclick\.net|gstatic\.com|accounts\.google\.com|login\.microsoftonline\.com|auth0\.com|cognito-idp\.|appleid\.apple\.com|github\.com\/login|facebook\.com\/login|protechts\.net|demdex\.net|litms|platform-telemetry|datadoghq\.com|fullstory\.com|launchdarkly\.com|intercom\.io|privy\.io|mypinata\.cloud|sentry\.io|segment\.io|amplitude\.com|mixpanel\.com|hotjar\.com|clarity\.ms|googletagmanager\.com|walletconnect\.com|imagedelivery\.net|cloudflareinsights\.com)/i;
4849
4952
  SKIP_TELEMETRY_HOSTS = /(waa-pa\.|signaler-pa\.|appsgrowthpromo-pa\.|ogads-pa\.|peoplestackwebexperiments-pa\.)/i;
4850
4953
  SKIP_TELEMETRY_PATHS = /\/(log|logging|telemetry|analytics|beacon|ping|heartbeat|metrics)(\/|$)/i;
4851
- RPC_HINTS = /(\/$rpc\/|\/rpc\/|graphql|trending|search|feed|results|batchexecute|\/api\/)/i;
4954
+ RPC_HINTS = /(\/$rpc\/|\/rpc\/|graphql|trending|search|feed|results|batchexecute|\/api\/|youtubei|__ssr_data__)/i;
4852
4955
  ALLOWED_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE"]);
4853
4956
  STRIP_HEADERS = new Set([
4854
4957
  "cookie",
@@ -4926,7 +5029,180 @@ var init_reverse_engineer = __esm(() => {
4926
5029
  "adsize",
4927
5030
  "lineitemid"
4928
5031
  ]);
4929
- ON_DOMAIN_NOISE = /\/(recaptcha|captcha|update-recaptcha|csrf|consent|data-protection|badge|drawer|header-action|geolocation|onboarding|wana\/bids|prebid|bids\/request|ads\/|pixel|beacon|collect|impression|click-tracking|heartbeat|webConfig|config\.json|manifest\.json|service-worker|sw\.js|favicon|robots\.txt|sitemap|opensearch|partial\/[a-zA-Z]+\/mod-|logging|csp-report|gen_204|generate_204|sodar|__|devvit-|user-drawer|action-item)/i;
5032
+ ON_DOMAIN_NOISE = /\/(recaptcha|captcha|update-recaptcha|csrf|consent|data-protection|badge|drawer|header-action|geolocation|onboarding|wana\/bids|prebid|bids\/request|ads\/|pixel|beacon|collect|impression|click-tracking|heartbeat|webConfig|config\.json|manifest\.json|service-worker|sw\.js|favicon|robots\.txt|sitemap|opensearch|partial\/[a-zA-Z]+\/mod-|logging|csp-report|gen_204|generate_204|sodar|__webpack|__next|devvit-|user-drawer|action-item)/i;
5033
+ });
5034
+
5035
+ // ../../src/vault/index.ts
5036
+ var exports_vault = {};
5037
+ __export(exports_vault, {
5038
+ storeCredential: () => storeCredential,
5039
+ setKeytarClientForTests: () => setKeytarClientForTests,
5040
+ resetKeytarClientForTests: () => resetKeytarClientForTests,
5041
+ normalizeKeytarModule: () => normalizeKeytarModule,
5042
+ getCredential: () => getCredential,
5043
+ deleteCredential: () => deleteCredential
5044
+ });
5045
+ import { createCipheriv, createDecipheriv, randomBytes } from "crypto";
5046
+ import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync, writeFileSync as writeFileSync2 } from "fs";
5047
+ import { join as join2 } from "path";
5048
+ import { homedir } from "os";
5049
+ function normalizeKeytarModule(mod) {
5050
+ let candidate = mod;
5051
+ for (let depth = 0;depth < 3; depth++) {
5052
+ if (!candidate || typeof candidate !== "object" || !("default" in candidate))
5053
+ break;
5054
+ candidate = candidate.default;
5055
+ }
5056
+ if (!candidate)
5057
+ return null;
5058
+ if (typeof candidate.setPassword === "function" && typeof candidate.getPassword === "function" && typeof candidate.deletePassword === "function") {
5059
+ return candidate;
5060
+ }
5061
+ return null;
5062
+ }
5063
+ function isKeytarBindingError(error) {
5064
+ const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
5065
+ return KEYTAR_BINDING_ERROR_RE.test(message);
5066
+ }
5067
+ function disableKeytar(error) {
5068
+ keytar = null;
5069
+ if (keytarFallbackLogged)
5070
+ return;
5071
+ const summary = error instanceof Error ? error.message : String(error);
5072
+ log("vault", `keytar runtime unavailable (${summary}); using encrypted file fallback`);
5073
+ keytarFallbackLogged = true;
5074
+ }
5075
+ async function callKeytar(op) {
5076
+ if (!keytar)
5077
+ return KEYTAR_UNAVAILABLE;
5078
+ try {
5079
+ return await op(keytar);
5080
+ } catch (error) {
5081
+ if (!isKeytarBindingError(error))
5082
+ throw error;
5083
+ disableKeytar(error);
5084
+ return KEYTAR_UNAVAILABLE;
5085
+ }
5086
+ }
5087
+ function setKeytarClientForTests(client) {
5088
+ keytar = client;
5089
+ keytarFallbackLogged = false;
5090
+ }
5091
+ function resetKeytarClientForTests() {
5092
+ keytar = importedKeytar;
5093
+ keytarFallbackLogged = false;
5094
+ }
5095
+ function getOrCreateKey() {
5096
+ if (!existsSync3(VAULT_DIR))
5097
+ mkdirSync3(VAULT_DIR, { recursive: true, mode: 448 });
5098
+ if (existsSync3(KEY_FILE))
5099
+ return readFileSync(KEY_FILE);
5100
+ const key = randomBytes(32);
5101
+ writeFileSync2(KEY_FILE, key, { mode: 384 });
5102
+ return key;
5103
+ }
5104
+ function withVaultLock(fn) {
5105
+ const prev = vaultLock;
5106
+ let release;
5107
+ vaultLock = new Promise((r) => {
5108
+ release = r;
5109
+ });
5110
+ return prev.then(fn).finally(() => release());
5111
+ }
5112
+ function readVaultFile() {
5113
+ if (!existsSync3(VAULT_FILE))
5114
+ return {};
5115
+ try {
5116
+ const key = getOrCreateKey();
5117
+ const raw = readFileSync(VAULT_FILE);
5118
+ const iv = raw.subarray(0, 16);
5119
+ const enc = raw.subarray(16);
5120
+ const decipher = createDecipheriv("aes-256-cbc", key, iv);
5121
+ const dec = Buffer.concat([decipher.update(enc), decipher.final()]);
5122
+ return JSON.parse(dec.toString("utf8"));
5123
+ } catch {
5124
+ return {};
5125
+ }
5126
+ }
5127
+ function writeVaultFile(data) {
5128
+ const key = getOrCreateKey();
5129
+ const iv = randomBytes(16);
5130
+ const cipher = createCipheriv("aes-256-cbc", key, iv);
5131
+ const enc = Buffer.concat([cipher.update(JSON.stringify(data), "utf8"), cipher.final()]);
5132
+ writeFileSync2(VAULT_FILE, Buffer.concat([iv, enc]), { mode: 384 });
5133
+ }
5134
+ async function storeCredential(account, value, opts) {
5135
+ const wrapped = {
5136
+ value,
5137
+ stored_at: new Date().toISOString(),
5138
+ expires_at: opts?.expires_at,
5139
+ max_age_ms: opts?.max_age_ms
5140
+ };
5141
+ const serialized = JSON.stringify(wrapped);
5142
+ const keytarResult = await callKeytar((client) => client.setPassword(SERVICE, account, serialized));
5143
+ if (keytarResult !== KEYTAR_UNAVAILABLE)
5144
+ return;
5145
+ await withVaultLock(() => {
5146
+ const data = readVaultFile();
5147
+ data[account] = serialized;
5148
+ writeVaultFile(data);
5149
+ });
5150
+ }
5151
+ function isExpired(cred) {
5152
+ if (cred.expires_at) {
5153
+ return new Date(cred.expires_at).getTime() <= Date.now();
5154
+ }
5155
+ if (cred.max_age_ms) {
5156
+ return new Date(cred.stored_at).getTime() + cred.max_age_ms <= Date.now();
5157
+ }
5158
+ return false;
5159
+ }
5160
+ async function getCredential(account) {
5161
+ let raw;
5162
+ const keytarResult = await callKeytar((client) => client.getPassword(SERVICE, account));
5163
+ if (keytarResult !== KEYTAR_UNAVAILABLE) {
5164
+ raw = keytarResult;
5165
+ } else {
5166
+ const data = readVaultFile();
5167
+ raw = data[account] ?? null;
5168
+ }
5169
+ if (!raw)
5170
+ return null;
5171
+ try {
5172
+ const parsed = JSON.parse(raw);
5173
+ if (parsed.value && parsed.stored_at) {
5174
+ if (isExpired(parsed)) {
5175
+ await deleteCredential(account);
5176
+ return null;
5177
+ }
5178
+ return parsed.value;
5179
+ }
5180
+ } catch {}
5181
+ return raw;
5182
+ }
5183
+ async function deleteCredential(account) {
5184
+ const keytarResult = await callKeytar((client) => client.deletePassword(SERVICE, account));
5185
+ if (keytarResult !== KEYTAR_UNAVAILABLE)
5186
+ return;
5187
+ await withVaultLock(() => {
5188
+ const data = readVaultFile();
5189
+ delete data[account];
5190
+ writeVaultFile(data);
5191
+ });
5192
+ }
5193
+ var KEYTAR_UNAVAILABLE, KEYTAR_BINDING_ERROR_RE, keytar = null, importedKeytar, keytarFallbackLogged = false, SERVICE = "unbrowse", VAULT_DIR, VAULT_FILE, KEY_FILE, vaultLock;
5194
+ var init_vault = __esm(async () => {
5195
+ init_logger();
5196
+ KEYTAR_UNAVAILABLE = Symbol("KEYTAR_UNAVAILABLE");
5197
+ KEYTAR_BINDING_ERROR_RE = /(keytar(?:\.node)?|native bindings?|bindings file|no native build was found|could not locate the bindings file|module did not self-register|err_dlopen_failed|dlopen\(|compiled against a different node\.js version|cannot find module .*keytar|wasm is not supported on this platform|(set|get|delete)password is not a function)/i;
5198
+ try {
5199
+ keytar = normalizeKeytarModule(await import("keytar"));
5200
+ } catch {}
5201
+ importedKeytar = keytar;
5202
+ VAULT_DIR = join2(homedir(), ".unbrowse", "vault");
5203
+ VAULT_FILE = join2(VAULT_DIR, "credentials.enc");
5204
+ KEY_FILE = join2(VAULT_DIR, ".key");
5205
+ vaultLock = Promise.resolve();
4930
5206
  });
4931
5207
 
4932
5208
  // ../../src/runtime/browser-access.ts
@@ -5379,7 +5655,12 @@ function normalizeCapturedUrl(url, baseUrl) {
5379
5655
  if (!url)
5380
5656
  return url;
5381
5657
  try {
5382
- return new URL(url).toString();
5658
+ const parsed = new URL(url);
5659
+ if (baseUrl && (parsed.hostname === "127.0.0.1" || parsed.hostname === "localhost")) {
5660
+ const pageOrigin = new URL(baseUrl).origin;
5661
+ return `${pageOrigin}${parsed.pathname}${parsed.search}`;
5662
+ }
5663
+ return parsed.toString();
5383
5664
  } catch {
5384
5665
  if (!baseUrl)
5385
5666
  return url;
@@ -5780,7 +6061,76 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
5780
6061
  log("capture", "scriptInject unavailable — falling back to evaluate injection");
5781
6062
  }
5782
6063
  }
5783
- await phase("harStart", () => harStart(tabId));
6064
+ try {
6065
+ await phase("harStart", () => harStart(tabId));
6066
+ } catch {}
6067
+ const cdpNetworkEntries = [];
6068
+ const cdpRequestMap = new Map;
6069
+ const cdpResponseMeta = new Map;
6070
+ const cdpFinishedRequests = new Set;
6071
+ let cdpWs = null;
6072
+ let cdpMsgId = 10;
6073
+ const cdpPendingBodies = new Map;
6074
+ const cdpResolvedBodies = new Map;
6075
+ const API_URL_PATTERN = /\/api\/|\/graphql|voyager|youtubei|\/v\d+\//i;
6076
+ try {
6077
+ const cdpPort = getCdpPort();
6078
+ if (cdpPort) {
6079
+ const tabsResp = await fetch(`http://127.0.0.1:${cdpPort}/json`);
6080
+ const tabs = await tabsResp.json();
6081
+ const cdpTab = tabs.find((t) => t.id === tabId) ?? tabs.find((t) => t.type === "page");
6082
+ if (cdpTab?.webSocketDebuggerUrl) {
6083
+ cdpWs = new WebSocket(cdpTab.webSocketDebuggerUrl);
6084
+ await new Promise((resolve, reject) => {
6085
+ cdpWs.onopen = () => resolve();
6086
+ cdpWs.onerror = () => reject(new Error("CDP WS failed"));
6087
+ setTimeout(() => reject(new Error("CDP WS timeout")), 3000);
6088
+ });
6089
+ cdpWs.onmessage = (ev) => {
6090
+ try {
6091
+ const msg = JSON.parse(String(ev.data));
6092
+ if (msg.method === "Network.requestWillBeSent") {
6093
+ const p = msg.params;
6094
+ const reqHeaders = Object.entries(p.request?.headers ?? {}).map(([name, value]) => ({ name, value: String(value) }));
6095
+ cdpRequestMap.set(p.requestId, {
6096
+ method: p.request?.method ?? "GET",
6097
+ url: p.request?.url ?? "",
6098
+ headers: reqHeaders,
6099
+ postData: p.request?.postData
6100
+ });
6101
+ } else if (msg.method === "Network.responseReceived") {
6102
+ const p = msg.params;
6103
+ const respHeaders = Object.entries(p.response?.headers ?? {}).map(([name, value]) => ({ name, value: String(value) }));
6104
+ cdpResponseMeta.set(p.requestId, {
6105
+ status: p.response?.status ?? 0,
6106
+ headers: respHeaders,
6107
+ mimeType: p.response?.mimeType ?? ""
6108
+ });
6109
+ } else if (msg.method === "Network.loadingFinished") {
6110
+ const requestId = msg.params?.requestId;
6111
+ cdpFinishedRequests.add(requestId);
6112
+ const req = cdpRequestMap.get(requestId);
6113
+ const resp = cdpResponseMeta.get(requestId);
6114
+ if (req && resp && API_URL_PATTERN.test(req.url) && (resp.mimeType.includes("json") || resp.mimeType.includes("text"))) {
6115
+ const id = ++cdpMsgId;
6116
+ cdpPendingBodies.set(id, req.url);
6117
+ cdpWs.send(JSON.stringify({ id, method: "Network.getResponseBody", params: { requestId } }));
6118
+ }
6119
+ } else if (msg.id && cdpPendingBodies.has(msg.id)) {
6120
+ const reqUrl = cdpPendingBodies.get(msg.id);
6121
+ cdpPendingBodies.delete(msg.id);
6122
+ if (msg.result?.body) {
6123
+ cdpResolvedBodies.set(reqUrl, msg.result.body);
6124
+ }
6125
+ }
6126
+ } catch {}
6127
+ };
6128
+ cdpWs.send(JSON.stringify({ id: 1, method: "Network.enable", params: {} }));
6129
+ await new Promise((r) => setTimeout(r, 200));
6130
+ log("capture", "CDP network capture enabled (direct websocket)");
6131
+ }
6132
+ }
6133
+ } catch {}
5784
6134
  let pageDomain;
5785
6135
  try {
5786
6136
  pageDomain = getRegistrableDomain(new URL(url).hostname);
@@ -5834,11 +6184,85 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
5834
6184
  const perfEntries = await phase("evaluate:perf", () => collectPerformanceResourceEntries(tabId));
5835
6185
  performanceUrls = await phase("replay-fetch", () => replayPerformanceApiResponses(tabId, perfEntries, responseBodies, url, intent));
5836
6186
  } catch {}
6187
+ if (cdpRequestMap.size > 0) {
6188
+ try {
6189
+ const cdpRawReqs = [...cdpRequestMap.values()].map((r) => ({
6190
+ url: r.url,
6191
+ method: r.method,
6192
+ request_headers: Object.fromEntries(r.headers.map((h) => [h.name.toLowerCase(), h.value])),
6193
+ response_status: 200,
6194
+ response_headers: {},
6195
+ response_body: undefined,
6196
+ timestamp: ""
6197
+ }));
6198
+ const authHeaders2 = extractAuthHeaders(cdpRawReqs);
6199
+ if (Object.keys(authHeaders2).length > 0) {
6200
+ const rawKey = `${domain}-session`;
6201
+ const { getCredential: getCredential2 } = await init_vault().then(() => exports_vault);
6202
+ let existing = {};
6203
+ try {
6204
+ const stored = await getCredential2(rawKey);
6205
+ if (stored)
6206
+ existing = JSON.parse(stored);
6207
+ } catch {}
6208
+ if (!existing.cookies || existing.cookies.length === 0) {
6209
+ try {
6210
+ const loginKey = `auth:${getRegistrableDomain(domain)}`;
6211
+ const loginStored = await getCredential2(loginKey);
6212
+ if (loginStored) {
6213
+ const loginData = JSON.parse(loginStored);
6214
+ if (loginData.cookies?.length)
6215
+ existing.cookies = loginData.cookies;
6216
+ }
6217
+ } catch {}
6218
+ }
6219
+ await storeCredential(rawKey, JSON.stringify({
6220
+ ...existing,
6221
+ headers: { ...existing.headers ?? {}, ...authHeaders2 }
6222
+ }));
6223
+ log("capture", `early auth store: ${Object.keys(authHeaders2).join(", ")} for ${domain}`);
6224
+ }
6225
+ } catch (e) {
6226
+ log("capture", `early auth store failed: ${e instanceof Error ? e.message : String(e)}`);
6227
+ }
6228
+ }
5837
6229
  let harEntries = [];
5838
6230
  try {
5839
- const harResult = await phase("harStop", () => harStop(tabId));
5840
- harEntries = harResult.entries;
6231
+ const harPromise = harStop(tabId);
6232
+ const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve({ entries: [] }), 5000));
6233
+ const harResult = await Promise.race([harPromise, timeoutPromise]);
6234
+ harEntries = harResult.entries ?? [];
5841
6235
  } catch {}
6236
+ if (cdpPendingBodies.size > 0) {
6237
+ await new Promise((r) => setTimeout(r, 1500));
6238
+ }
6239
+ if (cdpWs) {
6240
+ try {
6241
+ cdpWs.close();
6242
+ } catch {}
6243
+ }
6244
+ const harUrls = new Set(harEntries.map((e) => e.request?.url).filter(Boolean));
6245
+ let cdpAdded = 0;
6246
+ for (const [requestId, req] of cdpRequestMap) {
6247
+ if (harUrls.has(req.url))
6248
+ continue;
6249
+ const resp = cdpResponseMeta.get(requestId);
6250
+ if (!resp)
6251
+ continue;
6252
+ const body = cdpResolvedBodies.get(req.url);
6253
+ harEntries.push({
6254
+ startedDateTime: new Date().toISOString(),
6255
+ request: { method: req.method, url: req.url, headers: req.headers, postData: req.postData ? { text: req.postData } : undefined },
6256
+ response: { status: resp.status, headers: resp.headers, content: body ? { text: body, mimeType: resp.mimeType } : {} }
6257
+ });
6258
+ if (body && body.length > 0 && !responseBodies.has(req.url)) {
6259
+ responseBodies.set(req.url, body);
6260
+ }
6261
+ cdpAdded++;
6262
+ }
6263
+ if (cdpAdded > 0) {
6264
+ log("capture", `CDP direct capture added ${cdpAdded} entries (${cdpResolvedBodies.size} with bodies, ${harEntries.length} total HAR)`);
6265
+ }
5842
6266
  const HAR_REPLAY_CT2 = /application\/json|text\/plain|\+json/i;
5843
6267
  let harReplayCount = 0;
5844
6268
  for (const entry of harEntries) {
@@ -5854,7 +6278,7 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
5854
6278
  if (status < 200 || status >= 400)
5855
6279
  continue;
5856
6280
  const ct = (entry.response?.headers ?? []).find((h) => h.name.toLowerCase() === "content-type")?.value ?? "";
5857
- if (!HAR_REPLAY_CT2.test(ct) && !harUrl.includes("/api/") && !harUrl.includes("_search") && !harUrl.includes("graphql"))
6281
+ if (!HAR_REPLAY_CT2.test(ct) && !harUrl.includes("/api/") && !harUrl.includes("_search") && !harUrl.includes("graphql") && !harUrl.includes("youtubei"))
5858
6282
  continue;
5859
6283
  if (harReplayCount >= 20)
5860
6284
  break;
@@ -5862,7 +6286,7 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
5862
6286
  const postData = method !== "GET" ? entry.request?.postData?.text : undefined;
5863
6287
  const replayScript = method === "GET" || !postData ? `(function(){var x=new XMLHttpRequest();x.open('GET','${harUrl.replace(/'/g, "\\'")}',false);x.send();return x.status>=200&&x.status<400?x.responseText:''})()` : `(function(){var x=new XMLHttpRequest();x.open('${method}','${harUrl.replace(/'/g, "\\'")}',false);x.setRequestHeader('Content-Type','application/json');x.send(${JSON.stringify(postData)});return x.status>=200&&x.status<400?x.responseText:''})()`;
5864
6288
  const body = await phase("har-replay", () => evaluate(tabId, replayScript));
5865
- if (typeof body === "string" && body.length > 0 && body.length < 512 * 1024) {
6289
+ if (typeof body === "string" && body.length > 0 && body.length < 524288) {
5866
6290
  responseBodies.set(harUrl, body);
5867
6291
  harReplayCount++;
5868
6292
  log("capture", `har-replay-fetched ${harUrl.substring(0, 80)} (${body.length}B)`);
@@ -5889,6 +6313,77 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
5889
6313
  }
5890
6314
  html = await phase("getPageHtml", () => getPageHtml(tabId));
5891
6315
  } catch {}
6316
+ const SSR_DATA_EXTRACTORS = [
6317
+ { name: "ytmusic", script: `(function(){try{var d=ytcfg.get('YTMUSIC_INITIAL_DATA');if(!d||!d.length)return null;var out={};d.forEach(function(x){if(x.path&&x.data)out[x.path]=x.data});out.__apiKey=ytcfg.get('INNERTUBE_API_KEY')||'';out.__context=ytcfg.get('INNERTUBE_CONTEXT')||null;return JSON.stringify(out)}catch(e){return null}})()` },
6318
+ { name: "youtube", script: `(function(){try{return typeof ytInitialData!=='undefined'?JSON.stringify(ytInitialData):null}catch(e){return null}})()` },
6319
+ { name: "nextjs", script: `(function(){try{return window.__NEXT_DATA__?JSON.stringify(window.__NEXT_DATA__):null}catch(e){return null}})()` },
6320
+ { name: "nuxt", script: `(function(){try{return window.__NUXT__?JSON.stringify(window.__NUXT__):null}catch(e){return null}})()` }
6321
+ ];
6322
+ let embeddedDataCount = 0;
6323
+ for (const extractor of SSR_DATA_EXTRACTORS) {
6324
+ try {
6325
+ const raw = await phase(`ssr:${extractor.name}`, () => evaluate(tabId, extractor.script));
6326
+ if (typeof raw !== "string" || !raw || raw === "null")
6327
+ continue;
6328
+ if (extractor.name === "ytmusic") {
6329
+ const parsed = JSON.parse(raw);
6330
+ const apiKey = parsed.__apiKey || "";
6331
+ const innertubeContext = parsed.__context;
6332
+ delete parsed.__apiKey;
6333
+ delete parsed.__context;
6334
+ for (const [path4, data] of Object.entries(parsed)) {
6335
+ if (!data || typeof data !== "object")
6336
+ continue;
6337
+ const cleaned = { ...data };
6338
+ delete cleaned.responseContext;
6339
+ delete cleaned.trackingParams;
6340
+ delete cleaned.header;
6341
+ delete cleaned.background;
6342
+ let bodyStr = JSON.stringify(cleaned);
6343
+ if (bodyStr.length > 1e4) {
6344
+ bodyStr = bodyStr.substring(0, 1e4) + '"}]}';
6345
+ try {
6346
+ JSON.parse(bodyStr);
6347
+ } catch {
6348
+ bodyStr = JSON.stringify(cleaned).substring(0, 1e4);
6349
+ }
6350
+ }
6351
+ if (bodyStr.length < 100)
6352
+ continue;
6353
+ const origin = new URL(url).origin;
6354
+ const contextParams = new URL(url).searchParams;
6355
+ const keyParam = apiKey ? `key=${apiKey}&` : "";
6356
+ const queryStr = path4.includes("search") && contextParams.toString() ? `?${keyParam}${contextParams.toString()}&prettyPrint=false` : `?${keyParam}prettyPrint=false`;
6357
+ const syntheticUrl = `${origin}/youtubei/v1${path4}${queryStr}`;
6358
+ responseBodies.set(syntheticUrl, bodyStr);
6359
+ const queryValue = contextParams.get("q") ?? "";
6360
+ const postBody = path4.includes("search") && innertubeContext ? JSON.stringify({ context: innertubeContext, query: queryValue }) : path4.includes("browse") && innertubeContext ? JSON.stringify({ context: innertubeContext, browseId: "FEmusic_liked_videos" }) : JSON.stringify({ context: innertubeContext ?? {} });
6361
+ harEntries.push({
6362
+ startedDateTime: new Date().toISOString(),
6363
+ request: { method: "POST", url: syntheticUrl, headers: [{ name: "content-type", value: "application/json" }], postData: { text: postBody } },
6364
+ response: { status: 200, headers: [{ name: "content-type", value: "application/json" }], content: { text: bodyStr, mimeType: "application/json" } }
6365
+ });
6366
+ embeddedDataCount++;
6367
+ log("capture", `ssr:${extractor.name} extracted ${path4} (${bodyStr.length}B)`);
6368
+ }
6369
+ } else {
6370
+ if (raw.length < 100)
6371
+ continue;
6372
+ const syntheticUrl = `${new URL(url).origin}/__ssr_data__/${extractor.name}`;
6373
+ responseBodies.set(syntheticUrl, raw);
6374
+ harEntries.push({
6375
+ startedDateTime: new Date().toISOString(),
6376
+ request: { method: "GET", url: syntheticUrl, headers: [] },
6377
+ response: { status: 200, headers: [{ name: "content-type", value: "application/json" }], content: { text: raw, mimeType: "application/json" } }
6378
+ });
6379
+ embeddedDataCount++;
6380
+ log("capture", `ssr:${extractor.name} extracted (${raw.length}B)`);
6381
+ }
6382
+ } catch {}
6383
+ }
6384
+ if (embeddedDataCount > 0) {
6385
+ log("capture", `embedded SSR data: ${embeddedDataCount} synthetic endpoints injected`);
6386
+ }
5892
6387
  const requests = mergePassiveCaptureData(intercepted, harEntries, extensionEntries, responseBodies, performanceUrls);
5893
6388
  log("capture", `tracked ${harEntries.length} HAR, ${intercepted.length} intercepted, ${extensionEntries.length} extension, ${responseBodies.size} bodies → ${requests.length} merged`);
5894
6389
  const rawCookies = await phase("extractCookies", () => extractCookiesFromPage(tabId, url));
@@ -6287,7 +6782,8 @@ var MAX_CONCURRENT_TABS = 3, activeTabs = 0, waitQueue, activeTabRegistry, inter
6287
6782
  var isData = ct.indexOf('json') !== -1 || ct.indexOf('application/x-protobuf') !== -1 ||
6288
6783
  ct.indexOf('text/plain') !== -1 ||
6289
6784
  url.indexOf('batchexecute') !== -1 || url.indexOf('/api/') !== -1 ||
6290
- url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1;
6785
+ url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1 ||
6786
+ url.indexOf('youtubei') !== -1;
6291
6787
  if (!isJs && !isData) return response;
6292
6788
  if (/\\.(css|woff2?|png|jpg|svg|ico)(\\?|$)/.test(url)) return response;
6293
6789
  var clone = response.clone();
@@ -6337,7 +6833,8 @@ var MAX_CONCURRENT_TABS = 3, activeTabs = 0, waitQueue, activeTabRegistry, inter
6337
6833
  var isData = ct.indexOf('json') !== -1 || ct.indexOf('application/x-protobuf') !== -1 ||
6338
6834
  ct.indexOf('text/plain') !== -1 ||
6339
6835
  url.indexOf('batchexecute') !== -1 || url.indexOf('/api/') !== -1 ||
6340
- url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1;
6836
+ url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1 ||
6837
+ url.indexOf('youtubei') !== -1;
6341
6838
  if (!isJs && !isData) return;
6342
6839
  if (/\\.(css|woff2?|png|jpg|svg|ico)(\\?|$)/.test(url)) return;
6343
6840
  var respBody = xhr.responseText || '';
@@ -6359,11 +6856,13 @@ var MAX_CONCURRENT_TABS = 3, activeTabs = 0, waitQueue, activeTabRegistry, inter
6359
6856
  return origSend.apply(this, arguments);
6360
6857
  };
6361
6858
  })()`, REPLAY_SKIP, HAR_REPLAY_CT;
6362
- var init_capture = __esm(() => {
6859
+ var init_capture = __esm(async () => {
6363
6860
  init_client();
6364
6861
  init_domain();
6365
6862
  init_logger();
6863
+ init_reverse_engineer();
6366
6864
  init_browser_access();
6865
+ await init_vault();
6367
6866
  waitQueue = [];
6368
6867
  activeTabRegistry = new Set;
6369
6868
  interceptorInjectedTabs = new Set;
@@ -6380,17 +6879,17 @@ var init_capture = __esm(() => {
6380
6879
  });
6381
6880
 
6382
6881
  // ../../src/build-info.generated.ts
6383
- var BUILD_RELEASE_VERSION = "3.2.0", BUILD_GIT_SHA = "c3fc3f822751", BUILD_CODE_HASH = "1488fc1d92b7", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4yLjAiLCJnaXRfc2hhIjoiYzNmYzNmODIyNzUxIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0BjM2ZjM2Y4MjI3NTEiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDA1OjA2OjAxLjIwMFoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "xCBEHEB2UsniVYLfzuTpoXlcomZHL2pXht-7Ii1e7mM", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
6882
+ var BUILD_RELEASE_VERSION = "3.2.2-experiments.128ff09", BUILD_GIT_SHA = "128ff09f51b3", BUILD_CODE_HASH = "1488fc1d92b7", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4yLjItZXhwZXJpbWVudHMuMTI4ZmYwOSIsImdpdF9zaGEiOiIxMjhmZjA5ZjUxYjMiLCJjb2RlX2hhc2giOiIxNDg4ZmMxZDkyYjciLCJ0cmFjZV92ZXJzaW9uIjoiMTQ4OGZjMWQ5MmI3QDEyOGZmMDlmNTFiMyIsImlzc3VlZF9hdCI6IjIwMjYtMDQtMDZUMTE6NTk6NDYuOTQyWiJ9", BUILD_RELEASE_MANIFEST_SIGNATURE = "M80x5zV9aWn-teZrpEFDyzwSU53Hwfj6rf19dl9m-jM", BUILD_DEFAULT_BACKEND_URL = "https://unbrowse-backend-experiments.lewis-6d8.workers.dev";
6384
6883
 
6385
6884
  // ../../src/version.ts
6386
6885
  import { createHash } from "crypto";
6387
- import { existsSync as existsSync3, readFileSync, readdirSync } from "fs";
6388
- import { dirname, join as join2, parse } from "path";
6886
+ import { existsSync as existsSync4, readFileSync as readFileSync2, readdirSync } from "fs";
6887
+ import { dirname, join as join3, parse } from "path";
6389
6888
  import { fileURLToPath as fileURLToPath2 } from "url";
6390
6889
  function collectTsFiles(dir) {
6391
6890
  const results = [];
6392
6891
  for (const entry of readdirSync(dir, { withFileTypes: true })) {
6393
- const full = join2(dir, entry.name);
6892
+ const full = join3(dir, entry.name);
6394
6893
  if (entry.isDirectory() && entry.name !== "node_modules") {
6395
6894
  results.push(...collectTsFiles(full));
6396
6895
  } else if (entry.name.endsWith(".ts")) {
@@ -6403,21 +6902,21 @@ function hashFiles(srcDir, files) {
6403
6902
  const hash = createHash("sha256");
6404
6903
  for (const file of files) {
6405
6904
  hash.update(file.slice(srcDir.length));
6406
- hash.update(readFileSync(file, "utf-8"));
6905
+ hash.update(readFileSync2(file, "utf-8"));
6407
6906
  }
6408
6907
  return hash.digest("hex").slice(0, 12);
6409
6908
  }
6410
6909
  function resolveCodeHashSourceDir(moduleDir) {
6411
6910
  const candidates = [
6412
6911
  moduleDir,
6413
- join2(moduleDir, "runtime-src"),
6414
- join2(moduleDir, "..", "runtime-src"),
6415
- join2(moduleDir, "src"),
6416
- join2(moduleDir, "..", "src")
6912
+ join3(moduleDir, "runtime-src"),
6913
+ join3(moduleDir, "..", "runtime-src"),
6914
+ join3(moduleDir, "src"),
6915
+ join3(moduleDir, "..", "src")
6417
6916
  ];
6418
6917
  for (const candidate of candidates) {
6419
6918
  try {
6420
- if (!existsSync3(candidate))
6919
+ if (!existsSync4(candidate))
6421
6920
  continue;
6422
6921
  const files = collectTsFiles(candidate);
6423
6922
  if (files.length > 0)
@@ -6467,7 +6966,7 @@ function getPackageVersionForModuleDir(moduleDir) {
6467
6966
  const root = parse(dir).root;
6468
6967
  while (true) {
6469
6968
  try {
6470
- const pkg = JSON.parse(readFileSync(join2(dir, "package.json"), "utf-8"));
6969
+ const pkg = JSON.parse(readFileSync2(join3(dir, "package.json"), "utf-8"));
6471
6970
  return typeof pkg.version === "string" ? pkg.version : "unknown";
6472
6971
  } catch {}
6473
6972
  if (dir === root)
@@ -6571,18 +7070,18 @@ async function ensureCascadeSplitForSkill(skill, deps = {}) {
6571
7070
  var init_cascade = () => {};
6572
7071
 
6573
7072
  // ../../src/payments/wallet.ts
6574
- import { existsSync as existsSync4, readFileSync as readFileSync2 } from "node:fs";
6575
- import { homedir } from "node:os";
6576
- import { join as join3 } from "node:path";
7073
+ import { existsSync as existsSync5, readFileSync as readFileSync3 } from "node:fs";
7074
+ import { homedir as homedir2 } from "node:os";
7075
+ import { join as join4 } from "node:path";
6577
7076
  function asNonEmptyString(value) {
6578
7077
  return typeof value === "string" && value.trim() ? value.trim() : undefined;
6579
7078
  }
6580
7079
  function getLobsterWalletFromLocalConfig() {
6581
- const agentsPath = join3(process.env.HOME || homedir(), ".lobster", "agents.json");
6582
- if (!existsSync4(agentsPath))
7080
+ const agentsPath = join4(process.env.HOME || homedir2(), ".lobster", "agents.json");
7081
+ if (!existsSync5(agentsPath))
6583
7082
  return;
6584
7083
  try {
6585
- const raw = JSON.parse(readFileSync2(agentsPath, "utf8"));
7084
+ const raw = JSON.parse(readFileSync3(agentsPath, "utf8"));
6586
7085
  const activeAgentId = asNonEmptyString(raw.activeAgentId);
6587
7086
  const activeAgent = Array.isArray(raw.agents) ? raw.agents.find((agent) => asNonEmptyString(agent.id) === activeAgentId) : activeAgentId ? raw.agents?.[activeAgentId] : undefined;
6588
7087
  return asNonEmptyString(activeAgent?.authorizedWallets?.solana) ?? asNonEmptyString(activeAgent?.walletAddress) ?? asNonEmptyString(activeAgent?.wallet_address);
@@ -6733,9 +7232,9 @@ __export(exports_lobster_pay, {
6733
7232
  isLobsterAvailable: () => isLobsterAvailable
6734
7233
  });
6735
7234
  import { execFile, execFileSync as execFileSync2 } from "node:child_process";
6736
- import { existsSync as existsSync5 } from "node:fs";
6737
- import { homedir as homedir2 } from "node:os";
6738
- import { join as join4 } from "node:path";
7235
+ import { existsSync as existsSync6 } from "node:fs";
7236
+ import { homedir as homedir3 } from "node:os";
7237
+ import { join as join5 } from "node:path";
6739
7238
  function getLobsterCommand() {
6740
7239
  try {
6741
7240
  execFileSync2("lobstercash", ["--version"], { stdio: "ignore", timeout: 3000 });
@@ -6743,8 +7242,8 @@ function getLobsterCommand() {
6743
7242
  } catch (_e) {}
6744
7243
  try {
6745
7244
  const npmPrefix = execFileSync2("npm", ["config", "get", "prefix"], { encoding: "utf8", timeout: 5000 }).trim();
6746
- const lobsterPath = join4(npmPrefix, "bin", "lobstercash");
6747
- if (existsSync5(lobsterPath)) {
7245
+ const lobsterPath = join5(npmPrefix, "bin", "lobstercash");
7246
+ if (existsSync6(lobsterPath)) {
6748
7247
  execFileSync2(lobsterPath, ["--version"], { stdio: "ignore", timeout: 3000 });
6749
7248
  return { cmd: lobsterPath, prefix: [] };
6750
7249
  }
@@ -6757,8 +7256,8 @@ function lobsterCmd() {
6757
7256
  return cachedCommand;
6758
7257
  }
6759
7258
  function isLobsterAvailable() {
6760
- const agentsPath = join4(process.env.HOME || homedir2(), ".lobster", "agents.json");
6761
- return existsSync5(agentsPath);
7259
+ const agentsPath = join5(process.env.HOME || homedir3(), ".lobster", "agents.json");
7260
+ return existsSync6(agentsPath);
6762
7261
  }
6763
7262
  function lobsterX402Fetch(url, options) {
6764
7263
  return new Promise((resolve) => {
@@ -6890,10 +7389,10 @@ __export(exports_client2, {
6890
7389
  buildDefaultAgentName: () => buildDefaultAgentName,
6891
7390
  autoFileIssue: () => autoFileIssue
6892
7391
  });
6893
- import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, existsSync as existsSync6, mkdirSync as mkdirSync3, readdirSync as readdirSync2 } from "fs";
6894
- import { join as join5 } from "path";
6895
- import { homedir as homedir3, hostname, release as osRelease } from "os";
6896
- import { randomBytes, createHash as createHash3 } from "crypto";
7392
+ import { readFileSync as readFileSync4, writeFileSync as writeFileSync3, existsSync as existsSync7, mkdirSync as mkdirSync4, readdirSync as readdirSync2 } from "fs";
7393
+ import { join as join6 } from "path";
7394
+ import { homedir as homedir4, hostname, release as osRelease } from "os";
7395
+ import { randomBytes as randomBytes2, createHash as createHash3 } from "crypto";
6897
7396
  import { createInterface } from "readline";
6898
7397
  function buildReleaseAttestationHeaders(manifestBase64, signature) {
6899
7398
  const manifest = manifestBase64.trim();
@@ -6927,18 +7426,18 @@ function scopedSkillKey(skillId, scopeId) {
6927
7426
  return scopeId ? `${scopeId}:${skillId}` : skillId;
6928
7427
  }
6929
7428
  function getSkillCacheDir() {
6930
- return process.env.UNBROWSE_SKILL_CACHE_DIR || join5(getConfigDir(), "skill-cache");
7429
+ return process.env.UNBROWSE_SKILL_CACHE_DIR || join6(getConfigDir(), "skill-cache");
6931
7430
  }
6932
7431
  function getConfigDir() {
6933
7432
  if (process.env.UNBROWSE_CONFIG_DIR)
6934
7433
  return process.env.UNBROWSE_CONFIG_DIR;
6935
- return PROFILE_NAME ? join5(homedir3(), ".unbrowse", "profiles", PROFILE_NAME) : join5(homedir3(), ".unbrowse");
7434
+ return PROFILE_NAME ? join6(homedir4(), ".unbrowse", "profiles", PROFILE_NAME) : join6(homedir4(), ".unbrowse");
6936
7435
  }
6937
7436
  function getConfigPath() {
6938
- return join5(getConfigDir(), "config.json");
7437
+ return join6(getConfigDir(), "config.json");
6939
7438
  }
6940
7439
  function getInstallTelemetryPath() {
6941
- return join5(getConfigDir(), "install-state.json");
7440
+ return join6(getConfigDir(), "install-state.json");
6942
7441
  }
6943
7442
  function getLandingToken() {
6944
7443
  const token = process.env.UNBROWSE_LANDING_TOKEN?.trim();
@@ -6956,8 +7455,8 @@ function isLocalOnlyMode() {
6956
7455
  function loadConfig() {
6957
7456
  try {
6958
7457
  const configPath = getConfigPath();
6959
- if (existsSync6(configPath)) {
6960
- return JSON.parse(readFileSync3(configPath, "utf-8"));
7458
+ if (existsSync7(configPath)) {
7459
+ return JSON.parse(readFileSync4(configPath, "utf-8"));
6961
7460
  }
6962
7461
  } catch {}
6963
7462
  return null;
@@ -6965,15 +7464,15 @@ function loadConfig() {
6965
7464
  function saveConfig(config) {
6966
7465
  const configDir = getConfigDir();
6967
7466
  const configPath = getConfigPath();
6968
- if (!existsSync6(configDir))
6969
- mkdirSync3(configDir, { recursive: true });
6970
- writeFileSync2(configPath, JSON.stringify(config, null, 2), { mode: 384 });
7467
+ if (!existsSync7(configDir))
7468
+ mkdirSync4(configDir, { recursive: true });
7469
+ writeFileSync3(configPath, JSON.stringify(config, null, 2), { mode: 384 });
6971
7470
  }
6972
7471
  function loadInstallTelemetryState() {
6973
7472
  try {
6974
7473
  const statePath = getInstallTelemetryPath();
6975
- if (existsSync6(statePath)) {
6976
- return JSON.parse(readFileSync3(statePath, "utf-8"));
7474
+ if (existsSync7(statePath)) {
7475
+ return JSON.parse(readFileSync4(statePath, "utf-8"));
6977
7476
  }
6978
7477
  } catch {}
6979
7478
  return null;
@@ -6981,13 +7480,13 @@ function loadInstallTelemetryState() {
6981
7480
  function saveInstallTelemetryState(state) {
6982
7481
  const configDir = getConfigDir();
6983
7482
  const statePath = getInstallTelemetryPath();
6984
- if (!existsSync6(configDir))
6985
- mkdirSync3(configDir, { recursive: true });
6986
- writeFileSync2(statePath, JSON.stringify(state, null, 2), { mode: 384 });
7483
+ if (!existsSync7(configDir))
7484
+ mkdirSync4(configDir, { recursive: true });
7485
+ writeFileSync3(statePath, JSON.stringify(state, null, 2), { mode: 384 });
6987
7486
  }
6988
7487
  function createInstallTelemetryState() {
6989
7488
  return {
6990
- install_id: `install_${randomBytes(8).toString("hex")}`,
7489
+ install_id: `install_${randomBytes2(8).toString("hex")}`,
6991
7490
  first_seen_at: new Date().toISOString()
6992
7491
  };
6993
7492
  }
@@ -7127,7 +7626,7 @@ function isValidAgentEmail(value) {
7127
7626
  return EMAIL_RE.test(normalizeAgentEmail(value));
7128
7627
  }
7129
7628
  function buildDefaultAgentName() {
7130
- return `${hostname()}-${randomBytes(3).toString("hex")}`;
7629
+ return `${hostname()}-${randomBytes2(3).toString("hex")}`;
7131
7630
  }
7132
7631
  function resolveAgentName(preferredEmail, fallbackName) {
7133
7632
  const normalized = normalizeAgentEmail(preferredEmail ?? "");
@@ -7482,11 +7981,11 @@ async function waitForBackgroundRegistration(timeoutMs = 0) {
7482
7981
  ]);
7483
7982
  }
7484
7983
  function skillCachePath(skillId) {
7485
- return join5(getSkillCacheDir(), `${skillId}.json`);
7984
+ return join6(getSkillCacheDir(), `${skillId}.json`);
7486
7985
  }
7487
7986
  function readSkillCache(skillId) {
7488
7987
  try {
7489
- const raw = readFileSync3(skillCachePath(skillId), "utf-8");
7988
+ const raw = readFileSync4(skillCachePath(skillId), "utf-8");
7490
7989
  return JSON.parse(raw);
7491
7990
  } catch {
7492
7991
  return null;
@@ -7496,8 +7995,8 @@ function writeSkillCache(skill, scopeId) {
7496
7995
  try {
7497
7996
  recentLocalSkills.set(scopedSkillKey(skill.skill_id, scopeId), skill);
7498
7997
  const skillCacheDir = getSkillCacheDir();
7499
- if (!existsSync6(skillCacheDir))
7500
- mkdirSync3(skillCacheDir, { recursive: true });
7998
+ if (!existsSync7(skillCacheDir))
7999
+ mkdirSync4(skillCacheDir, { recursive: true });
7501
8000
  const existing = readSkillCache(skill.skill_id);
7502
8001
  if (existing) {
7503
8002
  for (const ep of skill.endpoints) {
@@ -7513,7 +8012,7 @@ function writeSkillCache(skill, scopeId) {
7513
8012
  const hasStrategy = skill.endpoints.some((e) => e.exec_strategy);
7514
8013
  if (hasStrategy)
7515
8014
  console.log(`[cache] writing skill ${skill.skill_id} with exec_strategy`);
7516
- writeFileSync2(skillCachePath(skill.skill_id), JSON.stringify(skill), "utf-8");
8015
+ writeFileSync3(skillCachePath(skill.skill_id), JSON.stringify(skill), "utf-8");
7517
8016
  } catch {}
7518
8017
  }
7519
8018
  function cachePublishedSkill(skill, scopeId) {
@@ -7548,7 +8047,7 @@ function isIntentCompatible(lhs, rhs) {
7548
8047
  function findExistingSkillForDomain(domain, intent) {
7549
8048
  try {
7550
8049
  const skillCacheDir = getSkillCacheDir();
7551
- if (!existsSync6(skillCacheDir))
8050
+ if (!existsSync7(skillCacheDir))
7552
8051
  return null;
7553
8052
  const files = readdirSync2(skillCacheDir);
7554
8053
  let compatible = null;
@@ -7557,7 +8056,7 @@ function findExistingSkillForDomain(domain, intent) {
7557
8056
  if (!f.endsWith(".json") || f === "browser-capture.json")
7558
8057
  continue;
7559
8058
  try {
7560
- const raw = readFileSync3(join5(skillCacheDir, f), "utf-8");
8059
+ const raw = readFileSync4(join6(skillCacheDir, f), "utf-8");
7561
8060
  const skill = JSON.parse(raw);
7562
8061
  if (skill.domain === domain && skill.execution_type === "http") {
7563
8062
  if (!fallback)
@@ -7607,11 +8106,11 @@ async function getSkillChunk2(skillId, opts) {
7607
8106
  async function listSkills() {
7608
8107
  if (LOCAL_ONLY) {
7609
8108
  try {
7610
- if (!existsSync6(SKILL_CACHE_DIR))
8109
+ if (!existsSync7(SKILL_CACHE_DIR))
7611
8110
  return [];
7612
8111
  return readdirSync2(SKILL_CACHE_DIR).filter((file) => file.endsWith(".json")).map((file) => {
7613
8112
  try {
7614
- return JSON.parse(readFileSync3(join5(SKILL_CACHE_DIR, file), "utf-8"));
8113
+ return JSON.parse(readFileSync4(join6(SKILL_CACHE_DIR, file), "utf-8"));
7615
8114
  } catch {
7616
8115
  return null;
7617
8116
  }
@@ -8303,10 +8802,10 @@ var init_publish_admission = __esm(() => {
8303
8802
 
8304
8803
  // ../../src/telemetry.ts
8305
8804
  import { createHash as createHash4 } from "node:crypto";
8306
- import { existsSync as existsSync7, mkdirSync as mkdirSync4, writeFileSync as writeFileSync3 } from "node:fs";
8307
- import { join as join6 } from "node:path";
8805
+ import { existsSync as existsSync8, mkdirSync as mkdirSync5, writeFileSync as writeFileSync4 } from "node:fs";
8806
+ import { join as join7 } from "node:path";
8308
8807
  function getTraceDir() {
8309
- return process.env.UNBROWSE_TRACES_DIR ?? join6(process.env.HOME ?? "/tmp", ".unbrowse", "traces");
8808
+ return process.env.UNBROWSE_TRACES_DIR ?? join7(process.env.HOME ?? "/tmp", ".unbrowse", "traces");
8310
8809
  }
8311
8810
  function isTracingEnabled() {
8312
8811
  return process.env.UNBROWSE_DISABLE_TRACES !== "1";
@@ -8380,11 +8879,11 @@ function emitRouteTrace(params) {
8380
8879
  };
8381
8880
  try {
8382
8881
  const traceDir = getTraceDir();
8383
- if (!existsSync7(traceDir))
8384
- mkdirSync4(traceDir, { recursive: true });
8882
+ if (!existsSync8(traceDir))
8883
+ mkdirSync5(traceDir, { recursive: true });
8385
8884
  const stamp = params.started_at.replace(/[:.]/g, "-");
8386
- const file = join6(traceDir, `${stamp}-${params.outcome}-${params.trace_id}.json`);
8387
- writeFileSync3(file, JSON.stringify(artifact, null, 2), "utf-8");
8885
+ const file = join7(traceDir, `${stamp}-${params.outcome}-${params.trace_id}.json`);
8886
+ writeFileSync4(file, JSON.stringify(artifact, null, 2), "utf-8");
8388
8887
  return file;
8389
8888
  } catch {
8390
8889
  return null;
@@ -8818,276 +9317,206 @@ async function resolveAuthTokens(endpoint, cookies, existingAuthHeaders) {
8818
9317
  const triggerUrl = endpoint.trigger_url;
8819
9318
  if (!triggerUrl)
8820
9319
  return {};
8821
- const headerBindings = bindings.filter((b) => b.param_location === "header");
9320
+ const headerBindings = bindings.filter((b) => b.param_location === "header" && !existingAuthHeaders[b.param_name]);
8822
9321
  if (headerBindings.length === 0)
8823
9322
  return {};
8824
9323
  const resolved = {};
8825
- try {
8826
- const tabId = await openResolverTab(triggerUrl, cookies);
8827
- if (!tabId)
8828
- return {};
8829
- try {
8830
- await waitForLoad2(tabId);
8831
- const html = await getPageHtml(tabId).catch(() => "");
8832
- if (typeof html !== "string" || !html.startsWith("<")) {
8833
- return {};
8834
- }
8835
- for (const binding of headerBindings) {
8836
- let value = await resolveBinding(binding, html, cookies);
8837
- if (!value && binding.sources.some((s) => s.kind === "js-bundle")) {
8838
- value = await scanAllScriptResources(tabId, binding);
8839
- }
8840
- if (value) {
8841
- resolved[binding.param_name] = binding.param_name.toLowerCase() === "authorization" ? value.startsWith("Bearer ") ? value : `Bearer ${value}` : value;
8842
- }
8843
- }
8844
- } finally {
8845
- await closeTab(tabId).catch(() => {});
8846
- }
8847
- } catch {}
8848
- return resolved;
8849
- }
8850
- async function resolveBinding(binding, html, cookies) {
8851
- for (const source of binding.sources) {
8852
- let value;
8853
- if (source.kind === "cookie" && source.cookie_names?.length) {
8854
- for (const name of source.cookie_names) {
9324
+ for (const binding of headerBindings) {
9325
+ const cookieSources = binding.sources.filter((s) => s.kind === "cookie");
9326
+ if (cookieSources.length === 0)
9327
+ continue;
9328
+ for (const source of cookieSources) {
9329
+ const names = source.cookie_names ?? [];
9330
+ for (const name of names) {
8855
9331
  const cookie = cookies.find((c) => c.name === name);
8856
- if (cookie?.value) {
8857
- value = cookie.value;
9332
+ if (cookie?.value && cookie.value.length >= 8) {
9333
+ resolved[binding.param_name] = cookie.value;
8858
9334
  break;
8859
9335
  }
8860
9336
  }
8861
- } else if (source.kind === "html-meta" || source.kind === "html-inline-script") {
8862
- value = extractTokenFromHtml(source, html);
8863
- } else if (source.kind === "js-bundle" && source.bundle_url_pattern) {
8864
- try {
8865
- const resp = await fetch(source.bundle_url_pattern);
8866
- if (resp.ok) {
8867
- const body = await resp.text();
8868
- value = extractTokenFromBundle(source, body);
8869
- }
8870
- } catch {}
9337
+ if (resolved[binding.param_name])
9338
+ break;
8871
9339
  }
8872
- if (value && value.length >= 8)
8873
- return value;
8874
9340
  }
8875
- return;
8876
- }
8877
- async function openResolverTab(url, cookies) {
8878
- try {
8879
- const tab = await newTab(url);
8880
- const tabId = typeof tab === "string" ? tab : tab?.tab_id;
8881
- if (!tabId)
8882
- return;
8883
- if (cookies.length > 0) {
8884
- for (const c of cookies) {
8885
- await setCookie(tabId, c.name, c.value, c.domain).catch(() => {});
9341
+ const remaining = headerBindings.filter((b) => !resolved[b.param_name]);
9342
+ if (remaining.length === 0)
9343
+ return formatHeaders(resolved);
9344
+ if (!hasResolvableSources(remaining))
9345
+ return formatHeaders(resolved);
9346
+ const html = await fetchHtml(triggerUrl, cookies);
9347
+ if (html) {
9348
+ const fromHtml = await resolveFromHtml(remaining, html);
9349
+ Object.assign(resolved, fromHtml);
9350
+ const stillMissing = remaining.filter((b) => !resolved[b.param_name]);
9351
+ if (stillMissing.length === 0)
9352
+ return formatHeaders(resolved);
9353
+ if (Object.keys(fromHtml).length > 0 && hasHtmlResolvableSources(stillMissing)) {
9354
+ const browserHtml = await fetchHtmlViaBrowser(triggerUrl, cookies);
9355
+ if (browserHtml) {
9356
+ Object.assign(resolved, await resolveFromHtml(stillMissing, browserHtml));
8886
9357
  }
8887
- await navigate(tabId, url).catch(() => {});
8888
9358
  }
8889
- return tabId;
8890
- } catch {
8891
- return;
9359
+ return formatHeaders(resolved);
8892
9360
  }
8893
- }
8894
- async function waitForLoad2(tabId) {
8895
- const start2 = Date.now();
8896
- while (Date.now() - start2 < RESOLVE_TIMEOUT_MS) {
8897
- try {
8898
- const state = await evaluate(tabId, "document.readyState");
8899
- if (state === "complete" || state === "interactive")
8900
- return;
8901
- } catch {}
8902
- await new Promise((r) => setTimeout(r, 500));
9361
+ if (hasHtmlResolvableSources(remaining)) {
9362
+ const browserHtml = await fetchHtmlViaBrowser(triggerUrl, cookies);
9363
+ if (browserHtml) {
9364
+ Object.assign(resolved, await resolveFromHtml(remaining, browserHtml));
9365
+ }
8903
9366
  }
9367
+ return formatHeaders(resolved);
8904
9368
  }
8905
- async function scanAllScriptResources(tabId, binding) {
8906
- try {
8907
- const raw = await evaluate(tabId, `
8908
- JSON.stringify(
8909
- performance.getEntriesByType('resource')
8910
- .filter(function(e) { return e.initiatorType === 'script'; })
8911
- .map(function(e) { return e.name; })
8912
- )
8913
- `);
8914
- if (typeof raw !== "string" || !raw.startsWith("["))
8915
- return;
8916
- const urls = JSON.parse(raw);
8917
- const tokenPattern = /AAAAAAAAAAAAAAAAAAA[A-Za-z0-9+/=_%-]{20,}/;
8918
- for (const url of urls) {
8919
- try {
8920
- const resp = await fetch(url);
8921
- if (!resp.ok)
8922
- continue;
8923
- const body = await resp.text();
8924
- const m = body.match(tokenPattern);
8925
- if (m && m[0].length >= 20)
8926
- return m[0];
8927
- } catch {}
9369
+ function hasResolvableSources(bindings) {
9370
+ for (const b of bindings) {
9371
+ for (const s of b.sources) {
9372
+ if (s.kind === "html-meta" || s.kind === "html-inline-script")
9373
+ return true;
9374
+ if (s.kind === "js-bundle" && s.bundle_url_pattern && s.bundle_regex)
9375
+ return true;
8928
9376
  }
8929
- } catch {}
8930
- return;
8931
- }
8932
- var RESOLVE_TIMEOUT_MS = 12000;
8933
- var init_token_resolver = __esm(() => {
8934
- init_token_sources();
8935
- init_client();
8936
- });
8937
-
8938
- // ../../src/vault/index.ts
8939
- import { createCipheriv, createDecipheriv, randomBytes as randomBytes2 } from "crypto";
8940
- import { existsSync as existsSync8, mkdirSync as mkdirSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync4 } from "fs";
8941
- import { join as join7 } from "path";
8942
- import { homedir as homedir4 } from "os";
8943
- function normalizeKeytarModule(mod) {
8944
- let candidate = mod;
8945
- for (let depth = 0;depth < 3; depth++) {
8946
- if (!candidate || typeof candidate !== "object" || !("default" in candidate))
8947
- break;
8948
- candidate = candidate.default;
8949
9377
  }
8950
- if (!candidate)
8951
- return null;
8952
- if (typeof candidate.setPassword === "function" && typeof candidate.getPassword === "function" && typeof candidate.deletePassword === "function") {
8953
- return candidate;
8954
- }
8955
- return null;
9378
+ return false;
8956
9379
  }
8957
- function isKeytarBindingError(error) {
8958
- const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
8959
- return KEYTAR_BINDING_ERROR_RE.test(message);
9380
+ function hasHtmlResolvableSources(bindings) {
9381
+ for (const b of bindings) {
9382
+ for (const s of b.sources) {
9383
+ if (s.kind === "html-meta" || s.kind === "html-inline-script")
9384
+ return true;
9385
+ }
9386
+ }
9387
+ return false;
8960
9388
  }
8961
- function disableKeytar(error) {
8962
- keytar = null;
8963
- if (keytarFallbackLogged)
8964
- return;
8965
- const summary = error instanceof Error ? error.message : String(error);
8966
- log("vault", `keytar runtime unavailable (${summary}); using encrypted file fallback`);
8967
- keytarFallbackLogged = true;
9389
+ function formatHeaders(raw) {
9390
+ const out = {};
9391
+ for (const [k, v] of Object.entries(raw)) {
9392
+ if (k.toLowerCase() === "authorization" && !v.startsWith("Bearer ")) {
9393
+ out[k] = `Bearer ${v}`;
9394
+ } else {
9395
+ out[k] = v;
9396
+ }
9397
+ }
9398
+ return out;
8968
9399
  }
8969
- async function callKeytar(op) {
8970
- if (!keytar)
8971
- return KEYTAR_UNAVAILABLE;
8972
- try {
8973
- return await op(keytar);
8974
- } catch (error) {
8975
- if (!isKeytarBindingError(error))
8976
- throw error;
8977
- disableKeytar(error);
8978
- return KEYTAR_UNAVAILABLE;
9400
+ async function resolveFromHtml(bindings, html) {
9401
+ const resolved = {};
9402
+ for (const binding of bindings) {
9403
+ const value = await resolveBinding(binding, html);
9404
+ if (value)
9405
+ resolved[binding.param_name] = value;
8979
9406
  }
9407
+ return resolved;
8980
9408
  }
8981
- function getOrCreateKey() {
8982
- if (!existsSync8(VAULT_DIR))
8983
- mkdirSync5(VAULT_DIR, { recursive: true, mode: 448 });
8984
- if (existsSync8(KEY_FILE))
8985
- return readFileSync4(KEY_FILE);
8986
- const key = randomBytes2(32);
8987
- writeFileSync4(KEY_FILE, key, { mode: 384 });
8988
- return key;
9409
+ async function resolveBinding(binding, html) {
9410
+ for (const source of binding.sources) {
9411
+ let value;
9412
+ if (source.kind === "html-meta" || source.kind === "html-inline-script") {
9413
+ value = extractTokenFromHtml(source, html);
9414
+ } else if (source.kind === "js-bundle" && source.bundle_url_pattern && source.bundle_regex) {
9415
+ value = await resolveJsBundle(source, html);
9416
+ }
9417
+ if (value && value.length >= 8)
9418
+ return value;
9419
+ }
9420
+ return;
8989
9421
  }
8990
- function withVaultLock(fn) {
8991
- const prev = vaultLock;
8992
- let release;
8993
- vaultLock = new Promise((r) => {
8994
- release = r;
8995
- });
8996
- return prev.then(fn).finally(() => release());
9422
+ async function resolveJsBundle(source, html) {
9423
+ const pattern = source.bundle_url_pattern;
9424
+ const scriptSrcRe = /<script[^>]+src=["']([^"']+)["']/gi;
9425
+ let match;
9426
+ while ((match = scriptSrcRe.exec(html)) !== null) {
9427
+ const src = match[1];
9428
+ if (!src.includes(pattern))
9429
+ continue;
9430
+ try {
9431
+ const url = src.startsWith("http") ? src : `https:${src}`;
9432
+ const controller = new AbortController;
9433
+ const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
9434
+ const res = await fetch(url, {
9435
+ headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" },
9436
+ signal: controller.signal
9437
+ });
9438
+ clearTimeout(timeout);
9439
+ if (!res.ok)
9440
+ continue;
9441
+ const bundleContent = await res.text();
9442
+ const extracted = extractTokenFromBundle(source, bundleContent);
9443
+ if (extracted && extracted.length >= 8)
9444
+ return extracted;
9445
+ } catch {}
9446
+ }
9447
+ return;
8997
9448
  }
8998
- function readVaultFile() {
8999
- if (!existsSync8(VAULT_FILE))
9000
- return {};
9449
+ async function fetchHtml(url, cookies) {
9001
9450
  try {
9002
- const key = getOrCreateKey();
9003
- const raw = readFileSync4(VAULT_FILE);
9004
- const iv = raw.subarray(0, 16);
9005
- const enc = raw.subarray(16);
9006
- const decipher = createDecipheriv("aes-256-cbc", key, iv);
9007
- const dec = Buffer.concat([decipher.update(enc), decipher.final()]);
9008
- return JSON.parse(dec.toString("utf8"));
9451
+ const cookieHeader = cookies.map((c) => `${c.name}=${c.value}`).join("; ");
9452
+ const controller = new AbortController;
9453
+ const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
9454
+ const res = await fetch(url, {
9455
+ headers: {
9456
+ Cookie: cookieHeader,
9457
+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
9458
+ Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
9459
+ "Accept-Language": "en-US,en;q=0.9"
9460
+ },
9461
+ redirect: "follow",
9462
+ signal: controller.signal
9463
+ });
9464
+ clearTimeout(timeout);
9465
+ if (!res.ok)
9466
+ return null;
9467
+ const ct = res.headers.get("content-type") ?? "";
9468
+ if (!ct.includes("text/html") && !ct.includes("text/plain") && !ct.includes("application/xhtml"))
9469
+ return null;
9470
+ const html = await res.text();
9471
+ if (!html || html.length < 200 || !html.includes("<"))
9472
+ return null;
9473
+ return html;
9009
9474
  } catch {
9010
- return {};
9011
- }
9012
- }
9013
- function writeVaultFile(data) {
9014
- const key = getOrCreateKey();
9015
- const iv = randomBytes2(16);
9016
- const cipher = createCipheriv("aes-256-cbc", key, iv);
9017
- const enc = Buffer.concat([cipher.update(JSON.stringify(data), "utf8"), cipher.final()]);
9018
- writeFileSync4(VAULT_FILE, Buffer.concat([iv, enc]), { mode: 384 });
9019
- }
9020
- async function storeCredential(account, value, opts) {
9021
- const wrapped = {
9022
- value,
9023
- stored_at: new Date().toISOString(),
9024
- expires_at: opts?.expires_at,
9025
- max_age_ms: opts?.max_age_ms
9026
- };
9027
- const serialized = JSON.stringify(wrapped);
9028
- const keytarResult = await callKeytar((client) => client.setPassword(SERVICE, account, serialized));
9029
- if (keytarResult !== KEYTAR_UNAVAILABLE)
9030
- return;
9031
- await withVaultLock(() => {
9032
- const data = readVaultFile();
9033
- data[account] = serialized;
9034
- writeVaultFile(data);
9035
- });
9036
- }
9037
- function isExpired(cred) {
9038
- if (cred.expires_at) {
9039
- return new Date(cred.expires_at).getTime() <= Date.now();
9040
- }
9041
- if (cred.max_age_ms) {
9042
- return new Date(cred.stored_at).getTime() + cred.max_age_ms <= Date.now();
9475
+ return null;
9043
9476
  }
9044
- return false;
9045
9477
  }
9046
- async function getCredential(account) {
9047
- let raw;
9048
- const keytarResult = await callKeytar((client) => client.getPassword(SERVICE, account));
9049
- if (keytarResult !== KEYTAR_UNAVAILABLE) {
9050
- raw = keytarResult;
9051
- } else {
9052
- const data = readVaultFile();
9053
- raw = data[account] ?? null;
9054
- }
9055
- if (!raw)
9478
+ async function fetchHtmlViaBrowser(url, cookies) {
9479
+ let kuri;
9480
+ try {
9481
+ kuri = await Promise.resolve().then(() => (init_client(), exports_client));
9482
+ } catch {
9056
9483
  return null;
9484
+ }
9485
+ let tabId;
9057
9486
  try {
9058
- const parsed = JSON.parse(raw);
9059
- if (parsed.value && parsed.stored_at) {
9060
- if (isExpired(parsed)) {
9061
- await deleteCredential(account);
9062
- return null;
9487
+ const tab = await kuri.newTab(url);
9488
+ tabId = typeof tab === "string" ? tab : tab?.tab_id;
9489
+ if (!tabId)
9490
+ return null;
9491
+ if (cookies.length > 0) {
9492
+ for (const c of cookies) {
9493
+ await kuri.setCookie(tabId, c.name, c.value, c.domain).catch(() => {});
9063
9494
  }
9064
- return parsed.value;
9495
+ await kuri.navigate(tabId, url).catch(() => {});
9065
9496
  }
9066
- } catch {}
9067
- return raw;
9068
- }
9069
- async function deleteCredential(account) {
9070
- const keytarResult = await callKeytar((client) => client.deletePassword(SERVICE, account));
9071
- if (keytarResult !== KEYTAR_UNAVAILABLE)
9072
- return;
9073
- await withVaultLock(() => {
9074
- const data = readVaultFile();
9075
- delete data[account];
9076
- writeVaultFile(data);
9077
- });
9497
+ const start2 = Date.now();
9498
+ while (Date.now() - start2 < 12000) {
9499
+ try {
9500
+ const state = await kuri.evaluate(tabId, "document.readyState");
9501
+ if (state === "complete" || state === "interactive")
9502
+ break;
9503
+ } catch {}
9504
+ await new Promise((r) => setTimeout(r, 500));
9505
+ }
9506
+ const html = await kuri.getPageHtml(tabId).catch(() => "");
9507
+ if (typeof html !== "string" || !html.startsWith("<"))
9508
+ return null;
9509
+ return html;
9510
+ } catch {
9511
+ return null;
9512
+ } finally {
9513
+ if (tabId)
9514
+ await kuri.closeTab(tabId).catch(() => {});
9515
+ }
9078
9516
  }
9079
- var KEYTAR_UNAVAILABLE, KEYTAR_BINDING_ERROR_RE, keytar = null, keytarFallbackLogged = false, SERVICE = "unbrowse", VAULT_DIR, VAULT_FILE, KEY_FILE, vaultLock;
9080
- var init_vault = __esm(async () => {
9081
- init_logger();
9082
- KEYTAR_UNAVAILABLE = Symbol("KEYTAR_UNAVAILABLE");
9083
- KEYTAR_BINDING_ERROR_RE = /(keytar(?:\.node)?|native bindings?|bindings file|no native build was found|could not locate the bindings file|module did not self-register|err_dlopen_failed|dlopen\(|compiled against a different node\.js version|cannot find module .*keytar|wasm is not supported on this platform|(set|get|delete)password is not a function)/i;
9084
- try {
9085
- keytar = normalizeKeytarModule(await import("keytar"));
9086
- } catch {}
9087
- VAULT_DIR = join7(homedir4(), ".unbrowse", "vault");
9088
- VAULT_FILE = join7(VAULT_DIR, "credentials.enc");
9089
- KEY_FILE = join7(VAULT_DIR, ".key");
9090
- vaultLock = Promise.resolve();
9517
+ var FETCH_TIMEOUT_MS = 8000;
9518
+ var init_token_resolver = __esm(() => {
9519
+ init_token_sources();
9091
9520
  });
9092
9521
 
9093
9522
  // ../../src/runtime/supervisor.ts
@@ -13761,6 +14190,10 @@ function validateWorkflowReplayParams(recipe, params) {
13761
14190
  continue;
13762
14191
  const value = valueForSpec(spec, params);
13763
14192
  if (spec.required && (value == null || value === "")) {
14193
+ if (spec.default_value != null && spec.default_value !== "")
14194
+ continue;
14195
+ if (spec.example_value != null && spec.example_value !== "")
14196
+ continue;
13764
14197
  errors.push({ name: spec.name, reason: "required" });
13765
14198
  continue;
13766
14199
  }
@@ -14055,18 +14488,27 @@ async function reloadExecutionAuthState(skill, epDomain, authHeaders, cookies) {
14055
14488
  cookies.push(...resolved);
14056
14489
  } catch {}
14057
14490
  }
14058
- if (Object.keys(authHeaders).length === 0) {
14059
- try {
14060
- const sessionKey = `${getRegistrableDomain(epDomain)}-session`;
14061
- const sessionData = await getCredential(sessionKey);
14062
- if (sessionData) {
14063
- const parsed = JSON.parse(sessionData);
14064
- if (parsed.headers)
14065
- Object.assign(authHeaders, parsed.headers);
14066
- if (parsed.cookies && cookies.length === 0)
14067
- cookies.push(...parsed.cookies);
14068
- }
14069
- } catch {}
14491
+ {
14492
+ for (const sessionKey of [`${epDomain}-session`, `${getRegistrableDomain(epDomain)}-session`]) {
14493
+ try {
14494
+ const sessionData = await getCredential(sessionKey);
14495
+ if (sessionData) {
14496
+ const parsed = JSON.parse(sessionData);
14497
+ if (parsed.headers)
14498
+ Object.assign(authHeaders, parsed.headers);
14499
+ if (parsed.cookies && cookies.length === 0)
14500
+ cookies.push(...parsed.cookies);
14501
+ if (Object.keys(authHeaders).length > 0)
14502
+ break;
14503
+ }
14504
+ } catch {}
14505
+ }
14506
+ }
14507
+ if (cookies.length > 0 && authHeaders["csrf-token"]) {
14508
+ const jsessionId = cookies.find((c) => c.name === "JSESSIONID");
14509
+ if (jsessionId) {
14510
+ authHeaders["csrf-token"] = jsessionId.value.replace(/"/g, "");
14511
+ }
14070
14512
  }
14071
14513
  }
14072
14514
  function persistWorkflowArtifactForCapture(artifactSkill, captured, capturedAuthHeaders) {
@@ -15087,10 +15529,13 @@ async function executeBrowserCapture(skill, params, options) {
15087
15529
  }));
15088
15530
  }
15089
15531
  if (!auth_profile_ref) {
15090
- const vaultKey = `auth:${targetDomain}`;
15091
- const hasStoredAuth = await getCredential(vaultKey) != null;
15092
- if (hasStoredAuth)
15093
- auth_profile_ref = vaultKey;
15532
+ for (const vaultKey of [`auth:${targetDomain}`, `${domain}-session`, `${targetDomain}-session`]) {
15533
+ const hasStoredAuth = await getCredential(vaultKey) != null;
15534
+ if (hasStoredAuth) {
15535
+ auth_profile_ref = vaultKey;
15536
+ break;
15537
+ }
15538
+ }
15094
15539
  }
15095
15540
  const authBackedCapture = usedStoredAuth || !!auth_profile_ref;
15096
15541
  if (authBackedCapture) {
@@ -15671,7 +16116,7 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
15671
16116
  u.searchParams.set(k, String(v));
15672
16117
  }
15673
16118
  }
15674
- urlTemplate = restoreTemplatePlaceholderEncoding(u.toString());
16119
+ urlTemplate = restoreTemplatePlaceholderEncoding(u.toString()).replace(/%28/gi, "(").replace(/%29/gi, ")").replace(/%2C/gi, ",").replace(/%3A/gi, ":");
15675
16120
  } catch {}
15676
16121
  }
15677
16122
  let url = interpolate(urlTemplate, mergedParams);
@@ -15807,7 +16252,7 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
15807
16252
  let last = { data: null, status: 0 };
15808
16253
  for (const replayUrl of replayUrls) {
15809
16254
  const replayHeaders = buildStructuredReplayHeaders(url, replayUrl, headers);
15810
- log("exec", `server-fetch: ${endpoint.method} ${replayUrl.substring(0, 80)} auth=${(replayHeaders["authorization"] || "none").substring(0, 50)} csrf=${replayHeaders["x-csrf-token"]?.substring(0, 10)}... cookies=${replayHeaders["cookie"]?.length ?? 0}chars`);
16255
+ log("exec", `server-fetch: ${endpoint.method} ${replayUrl.substring(0, 200)} csrf-token=${(replayHeaders["csrf-token"] || "none").substring(0, 20)}... hdrs=${Object.keys(replayHeaders).length} cookies=${replayHeaders["cookie"]?.length ?? 0}chars`);
15811
16256
  const res = await fetch(replayUrl, {
15812
16257
  method: endpoint.method,
15813
16258
  headers: replayHeaders,
@@ -16211,7 +16656,12 @@ function interpolate(template, params) {
16211
16656
  const base = template.substring(0, qIdx);
16212
16657
  const query = template.substring(qIdx + 1);
16213
16658
  const interpolatedBase = base.replace(/\{(\w+)\}/g, (_, k) => params[k] != null ? String(params[k]) : `{${k}}`);
16214
- const interpolatedQuery = query.replace(/\{(\w+)\}/g, (_, k) => params[k] != null ? encodeURIComponent(String(params[k])) : `{${k}}`);
16659
+ const interpolatedQuery = query.replace(/\{(\w+)\}/g, (_, k) => {
16660
+ if (params[k] == null)
16661
+ return `{${k}}`;
16662
+ const val = String(params[k]);
16663
+ return val.replace(/[#&=\s]/g, (ch) => encodeURIComponent(ch));
16664
+ });
16215
16665
  return `${interpolatedBase}?${interpolatedQuery}`;
16216
16666
  }
16217
16667
  function interpolateObj(obj, params) {
@@ -16526,6 +16976,17 @@ function rankEndpoints(endpoints, intent, skillDomain, contextUrl) {
16526
16976
  }
16527
16977
  score += matches * 100;
16528
16978
  }
16979
+ if (rawTokens.length > 0 && pathname) {
16980
+ const pathLower = pathname.toLowerCase();
16981
+ const pathSegs = pathLower.split("/").filter(Boolean);
16982
+ for (const token of rawTokens) {
16983
+ const stemmed = stem(token);
16984
+ if (pathSegs.some((seg) => seg === stemmed || seg === token || seg.includes(token))) {
16985
+ score += 150;
16986
+ break;
16987
+ }
16988
+ }
16989
+ }
16529
16990
  if (ep.dom_extraction)
16530
16991
  score += 25;
16531
16992
  if (descriptionMeta.needs_review && ep.dom_extraction)
@@ -16760,8 +17221,6 @@ function isSpaShell(html) {
16760
17221
  }
16761
17222
  var DEFAULT_BROWSER_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", VALID_VERIFICATION_STATUSES, BM25_K1 = 1.2, BM25_B = 0.75, STOPWORDS, SYNONYMS;
16762
17223
  var init_execution = __esm(async () => {
16763
- init_capture();
16764
- init_capture();
16765
17224
  init_reverse_engineer();
16766
17225
  init_bundle_scanner();
16767
17226
  init_token_resolver();
@@ -16785,6 +17244,8 @@ var init_execution = __esm(async () => {
16785
17244
  init_compile();
16786
17245
  init_publish();
16787
17246
  await __promiseAll([
17247
+ init_capture(),
17248
+ init_capture(),
16788
17249
  init_vault(),
16789
17250
  init_auth(),
16790
17251
  init_indexer(),
@@ -18385,6 +18846,13 @@ function isCachedSkillRelevantForIntent(skill, intent, contextUrl) {
18385
18846
  const hasStructuredSearchEndpoint = candidateSkill.endpoints.some((endpoint) => endpointHasSearchBindings(endpoint) && (!!endpoint.dom_extraction || !!endpoint.response_schema) && endpointMatchesContextOrigin(endpoint, contextUrl) && endpointMatchesExplicitSearchContext(endpoint, contextUrl));
18386
18847
  if (hasStructuredSearchEndpoint)
18387
18848
  return true;
18849
+ if (top && top.score >= 0) {
18850
+ try {
18851
+ const topPath = new URL(top.endpoint.url_template).pathname.toLowerCase();
18852
+ if (/\/(search|find|query|browse|explore)\b/.test(topPath))
18853
+ return true;
18854
+ } catch {}
18855
+ }
18388
18856
  if (collectExplicitSearchContextBindingKeys(contextUrl).size > 0)
18389
18857
  return false;
18390
18858
  }
@@ -19427,7 +19895,7 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
19427
19895
  for (const cookie of cookies)
19428
19896
  await setCookie(handoffTabId, cookie).catch(() => {});
19429
19897
  } catch {}
19430
- await evaluate(handoffTabId, (await Promise.resolve().then(() => (init_capture(), exports_capture))).INTERCEPTOR_SCRIPT).catch(() => {});
19898
+ await evaluate(handoffTabId, (await init_capture().then(() => exports_capture)).INTERCEPTOR_SCRIPT).catch(() => {});
19431
19899
  await harStart(handoffTabId).catch(() => {});
19432
19900
  try {
19433
19901
  const routesModule = await init_routes().then(() => exports_routes);
@@ -20592,7 +21060,7 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
20592
21060
  }
20593
21061
  } catch {}
20594
21062
  }
20595
- if (context?.url && !forceCapture) {
21063
+ if (process.env.UNBROWSE_LOCAL_ONLY === "1" && !forceCapture) {
20596
21064
  return buildNoCachedMatch();
20597
21065
  }
20598
21066
  if (!context?.url) {
@@ -23928,13 +24396,13 @@ function schedulePeriodicVerification() {
23928
24396
  }, VERIFICATION_INTERVAL_MS);
23929
24397
  }
23930
24398
  var VERIFICATION_INTERVAL_MS, VERIFY_ENDPOINT_BATCH_SIZE;
23931
- var init_verification = __esm(() => {
23932
- init_capture();
24399
+ var init_verification = __esm(async () => {
23933
24400
  init_marketplace();
23934
24401
  init_marketplace();
23935
24402
  init_drift();
23936
24403
  init_matrix();
23937
24404
  init_candidates();
24405
+ await init_capture();
23938
24406
  VERIFICATION_INTERVAL_MS = 6 * 60 * 60 * 1000;
23939
24407
  VERIFY_ENDPOINT_BATCH_SIZE = Math.max(1, Number(process.env.UNBROWSE_VERIFY_ENDPOINT_BATCH_SIZE ?? 3));
23940
24408
  });
@@ -24041,9 +24509,11 @@ function schedulePeriodicStaleCleanup() {
24041
24509
  var VERIFY_TIMEOUT_MS, CLEANUP_INTERVAL_MS, CLEANUP_BATCH_SIZE, CLEANUP_VERIFY_ENDPOINT_LIMIT;
24042
24510
  var init_stale_cleanup_runner = __esm(async () => {
24043
24511
  init_marketplace();
24044
- init_verification();
24045
24512
  init_candidates();
24046
- await init_orchestrator();
24513
+ await __promiseAll([
24514
+ init_verification(),
24515
+ init_orchestrator()
24516
+ ]);
24047
24517
  VERIFY_TIMEOUT_MS = Math.max(5000, Number(process.env.UNBROWSE_STALE_VERIFY_TIMEOUT_MS ?? 15000));
24048
24518
  CLEANUP_INTERVAL_MS = Math.max(30 * 60 * 1000, Number(process.env.UNBROWSE_STALE_CLEANUP_INTERVAL_MS ?? 6 * 60 * 60 * 1000));
24049
24519
  CLEANUP_BATCH_SIZE = Math.max(1, Number(process.env.UNBROWSE_STALE_CLEANUP_BATCH_SIZE ?? 25));
@@ -25085,7 +25555,7 @@ async function registerRoutes(app) {
25085
25555
  if (!skill)
25086
25556
  return reply.code(404).send({ error: "Skill not found" });
25087
25557
  try {
25088
- const { verifySkill: verifySkill2 } = await Promise.resolve().then(() => (init_verification(), exports_verification));
25558
+ const { verifySkill: verifySkill2 } = await init_verification().then(() => exports_verification);
25089
25559
  const results = await verifySkill2(skill);
25090
25560
  return reply.send({ skill_id, verification: results });
25091
25561
  } catch (err) {
@@ -25678,7 +26148,6 @@ var BETA_API_URL, TRACES_DIR, BROWSE_BROKER_MAX, BROWSE_BROKER_BASE_PORT, browse
25678
26148
  var init_routes = __esm(async () => {
25679
26149
  init_client();
25680
26150
  init_reverse_engineer();
25681
- init_capture();
25682
26151
  init_marketplace();
25683
26152
  init_graph();
25684
26153
  init_client2();
@@ -25697,6 +26166,7 @@ var init_routes = __esm(async () => {
25697
26166
  init_schema_review();
25698
26167
  init_artifact();
25699
26168
  await __promiseAll([
26169
+ init_capture(),
25700
26170
  init_indexer(),
25701
26171
  init_vault(),
25702
26172
  init_orchestrator(),
@@ -25719,12 +26189,12 @@ import { config as loadEnv } from "dotenv";
25719
26189
 
25720
26190
  // ../../src/server.ts
25721
26191
  init_ratelimit();
25722
- init_verification();
25723
26192
  init_client2();
25724
- init_capture();
25725
26193
  init_version();
25726
26194
  await __promiseAll([
25727
26195
  init_routes(),
26196
+ init_verification(),
26197
+ init_capture(),
25728
26198
  init_stale_cleanup_runner()
25729
26199
  ]);
25730
26200
  import { execSync as execSync2 } from "node:child_process";