unbrowse 3.2.0 → 3.2.2-experiments.128ff09
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +37 -36
- package/dist/mcp.js +5 -5
- package/dist/server.js +813 -343
- package/package.json +1 -1
- package/vendor/kuri/darwin-arm64/kuri +0 -0
- package/vendor/kuri/darwin-x64/kuri +0 -0
- package/vendor/kuri/linux-arm64/kuri +0 -0
- package/vendor/kuri/linux-x64/kuri +0 -0
- package/vendor/kuri/manifest.json +9 -6
- package/vendor/kuri/win-x64/kuri.exe +0 -0
package/dist/server.js
CHANGED
|
@@ -76,6 +76,88 @@ function getPackageRoot(metaUrl) {
|
|
|
76
76
|
var init_paths = () => {};
|
|
77
77
|
|
|
78
78
|
// ../../src/kuri/client.ts
|
|
79
|
+
var exports_client = {};
|
|
80
|
+
__export(exports_client, {
|
|
81
|
+
waitForSelector: () => waitForSelector,
|
|
82
|
+
waitForLoad: () => waitForLoad,
|
|
83
|
+
waitForCloudflare: () => waitForCloudflare,
|
|
84
|
+
stop: () => stop,
|
|
85
|
+
start: () => start,
|
|
86
|
+
snapshot: () => snapshot,
|
|
87
|
+
shouldReuseManagedChrome: () => shouldReuseManagedChrome,
|
|
88
|
+
setViewport: () => setViewport,
|
|
89
|
+
setUserAgent: () => setUserAgent,
|
|
90
|
+
setHeaders: () => setHeaders,
|
|
91
|
+
setCredentials: () => setCredentials,
|
|
92
|
+
setCookies: () => setCookies,
|
|
93
|
+
setCookie: () => setCookie,
|
|
94
|
+
setCdpPortForTests: () => setCdpPortForTests,
|
|
95
|
+
sessionSave: () => sessionSave,
|
|
96
|
+
sessionLoad: () => sessionLoad,
|
|
97
|
+
sessionList: () => sessionList,
|
|
98
|
+
select: () => select,
|
|
99
|
+
scrollIntoView: () => scrollIntoView,
|
|
100
|
+
scroll: () => scroll,
|
|
101
|
+
scriptInject: () => scriptInject,
|
|
102
|
+
screenshot: () => screenshot,
|
|
103
|
+
reuseHealthyBrokerIfPossible: () => reuseHealthyBrokerIfPossible,
|
|
104
|
+
resolveKuriPort: () => resolveKuriPort,
|
|
105
|
+
resolveKuriLaunchConfig: () => resolveKuriLaunchConfig,
|
|
106
|
+
reload: () => reload,
|
|
107
|
+
press: () => press,
|
|
108
|
+
newTab: () => newTab,
|
|
109
|
+
networkEnable: () => networkEnable,
|
|
110
|
+
navigate: () => navigate,
|
|
111
|
+
keyboardType: () => keyboardType,
|
|
112
|
+
keyboardInsertText: () => keyboardInsertText,
|
|
113
|
+
keyUp: () => keyUp,
|
|
114
|
+
keyDown: () => keyDown,
|
|
115
|
+
isReady: () => isReady,
|
|
116
|
+
interceptStart: () => interceptStart,
|
|
117
|
+
health: () => health,
|
|
118
|
+
hasCloudflareChallenge: () => hasCloudflareChallenge,
|
|
119
|
+
harStop: () => harStop,
|
|
120
|
+
harStart: () => harStart,
|
|
121
|
+
goForward: () => goForward,
|
|
122
|
+
goBack: () => goBack,
|
|
123
|
+
getText: () => getText,
|
|
124
|
+
getPort: () => getPort,
|
|
125
|
+
getPerfLcp: () => getPerfLcp,
|
|
126
|
+
getPageHtml: () => getPageHtml,
|
|
127
|
+
getNetworkEvents: () => getNetworkEvents,
|
|
128
|
+
getMarkdown: () => getMarkdown,
|
|
129
|
+
getLinks: () => getLinks,
|
|
130
|
+
getKuriSourceCandidates: () => getKuriSourceCandidates,
|
|
131
|
+
getKuriErrorMessage: () => getKuriErrorMessage,
|
|
132
|
+
getKuriClient: () => getKuriClient,
|
|
133
|
+
getKuriBinaryCandidates: () => getKuriBinaryCandidates,
|
|
134
|
+
getErrors: () => getErrors,
|
|
135
|
+
getDefaultTab: () => getDefaultTab,
|
|
136
|
+
getCurrentUrl: () => getCurrentUrl,
|
|
137
|
+
getCookies: () => getCookies,
|
|
138
|
+
getConsole: () => getConsole,
|
|
139
|
+
getCdpPort: () => getCdpPort,
|
|
140
|
+
findText: () => findText,
|
|
141
|
+
findKuriBinary: () => findKuriBinary,
|
|
142
|
+
fill: () => fill,
|
|
143
|
+
extractLoadPluginsFromHtml: () => extractLoadPluginsFromHtml,
|
|
144
|
+
extractLoadPlugins: () => extractLoadPlugins,
|
|
145
|
+
executeInPageFetch: () => executeInPageFetch,
|
|
146
|
+
evaluate: () => evaluate,
|
|
147
|
+
drag: () => drag,
|
|
148
|
+
domQuery: () => domQuery,
|
|
149
|
+
domHtml: () => domHtml,
|
|
150
|
+
domAttributes: () => domAttributes,
|
|
151
|
+
discoverTabs: () => discoverTabs,
|
|
152
|
+
closeTab: () => closeTab,
|
|
153
|
+
click: () => click,
|
|
154
|
+
bestEffortRehydratePlugins: () => bestEffortRehydratePlugins,
|
|
155
|
+
authProfileSave: () => authProfileSave,
|
|
156
|
+
authProfileLoad: () => authProfileLoad,
|
|
157
|
+
authProfileList: () => authProfileList,
|
|
158
|
+
authProfileDelete: () => authProfileDelete,
|
|
159
|
+
action: () => action
|
|
160
|
+
});
|
|
79
161
|
import { execFileSync, spawn } from "node:child_process";
|
|
80
162
|
import { existsSync as existsSync2 } from "node:fs";
|
|
81
163
|
import net from "node:net";
|
|
@@ -977,6 +1059,19 @@ async function getPageHtml(tabId, state = defaultBrokerState) {
|
|
|
977
1059
|
const result = await evaluate(tabId, "document.documentElement.outerHTML", state);
|
|
978
1060
|
return String(result ?? "");
|
|
979
1061
|
}
|
|
1062
|
+
function extractLoadPlugins(value) {
|
|
1063
|
+
if (typeof value !== "string")
|
|
1064
|
+
return [];
|
|
1065
|
+
return Array.from(new Set(value.split(/[\s,;]+/).map((part) => part.trim()).filter(Boolean)));
|
|
1066
|
+
}
|
|
1067
|
+
function extractLoadPluginsFromHtml(html) {
|
|
1068
|
+
const modules = [];
|
|
1069
|
+
const pattern = /data-load-plugins=(["'])(.*?)\1/gi;
|
|
1070
|
+
for (const match of html.matchAll(pattern)) {
|
|
1071
|
+
modules.push(...extractLoadPlugins(match[2]));
|
|
1072
|
+
}
|
|
1073
|
+
return Array.from(new Set(modules));
|
|
1074
|
+
}
|
|
980
1075
|
async function bestEffortRehydratePlugins(tabId, state = defaultBrokerState) {
|
|
981
1076
|
const result = await evaluate(tabId, `(async function() {
|
|
982
1077
|
function splitPlugins(value) {
|
|
@@ -1118,6 +1213,9 @@ function getPort(state = defaultBrokerState) {
|
|
|
1118
1213
|
function getCdpPort() {
|
|
1119
1214
|
return kuriCdpPort;
|
|
1120
1215
|
}
|
|
1216
|
+
function setCdpPortForTests(port) {
|
|
1217
|
+
kuriCdpPort = port;
|
|
1218
|
+
}
|
|
1121
1219
|
function isReady(state = defaultBrokerState) {
|
|
1122
1220
|
return state.ready;
|
|
1123
1221
|
}
|
|
@@ -4112,7 +4210,7 @@ function extractEndpoints(requests, wsMessages, context) {
|
|
|
4112
4210
|
return "";
|
|
4113
4211
|
}
|
|
4114
4212
|
})();
|
|
4115
|
-
const isApiUrl = /\/(api|graphql)\b/i.test(urlPath) || /\.(json)(\?|$)/.test(req.url);
|
|
4213
|
+
const isApiUrl = /\/(api|graphql|youtubei|__ssr_data__)\b/i.test(urlPath) || /\.(json)(\?|$)/.test(req.url);
|
|
4116
4214
|
let graphqlOpName;
|
|
4117
4215
|
if (/graphql/i.test(req.url)) {
|
|
4118
4216
|
if (req.request_body) {
|
|
@@ -4201,7 +4299,12 @@ function extractEndpoints(requests, wsMessages, context) {
|
|
|
4201
4299
|
const sanitizedQParams = isGet ? sanitizeQueryParams(extractQueryParams(req.url)) : undefined;
|
|
4202
4300
|
let pathTemplate = sanitizeUrlTemplate(normalized);
|
|
4203
4301
|
const qBindings = sanitizedQParams ? buildQueryBindingMap(Object.keys(sanitizedQParams)) : {};
|
|
4204
|
-
const
|
|
4302
|
+
const existingQKeys = new Set;
|
|
4303
|
+
try {
|
|
4304
|
+
for (const k of new URL(pathTemplate).searchParams.keys())
|
|
4305
|
+
existingQKeys.add(k.toLowerCase());
|
|
4306
|
+
} catch {}
|
|
4307
|
+
const qTemplateStr = sanitizedQParams && Object.keys(sanitizedQParams).length > 0 ? Object.keys(sanitizedQParams).filter((k) => !existingQKeys.has(k.toLowerCase())).map((k) => `${encodeURIComponent(k)}={${qBindings[k] ?? k}}`).join("&") || null : null;
|
|
4205
4308
|
const { url: templatizedPath, pathParams, pathBindingCandidates } = templatizePathSegments(pathTemplate, req.url, context);
|
|
4206
4309
|
pathTemplate = templatizedPath;
|
|
4207
4310
|
const parsedRequestBody = !isGet && req.request_body ? tryParseBody(req.request_body) : undefined;
|
|
@@ -4217,7 +4320,7 @@ function extractEndpoints(requests, wsMessages, context) {
|
|
|
4217
4320
|
let endpoint = {
|
|
4218
4321
|
endpoint_id: nanoid2(),
|
|
4219
4322
|
method: req.method,
|
|
4220
|
-
url_template: qTemplateStr ? `${pathTemplate}
|
|
4323
|
+
url_template: qTemplateStr ? `${pathTemplate}${pathTemplate.includes("?") ? "&" : "?"}${qTemplateStr}` : pathTemplate,
|
|
4221
4324
|
description: buildEndpointDescription(req, sampleRequest, sampleResponse),
|
|
4222
4325
|
headers_template: sanitizeHeaders(req.request_headers),
|
|
4223
4326
|
query: sanitizedQParams,
|
|
@@ -4848,7 +4951,7 @@ var init_reverse_engineer = __esm(() => {
|
|
|
4848
4951
|
SKIP_HOSTS = /(cloudflare\.com|google-analytics\.com|doubleclick\.net|gstatic\.com|accounts\.google\.com|login\.microsoftonline\.com|auth0\.com|cognito-idp\.|appleid\.apple\.com|github\.com\/login|facebook\.com\/login|protechts\.net|demdex\.net|litms|platform-telemetry|datadoghq\.com|fullstory\.com|launchdarkly\.com|intercom\.io|privy\.io|mypinata\.cloud|sentry\.io|segment\.io|amplitude\.com|mixpanel\.com|hotjar\.com|clarity\.ms|googletagmanager\.com|walletconnect\.com|imagedelivery\.net|cloudflareinsights\.com)/i;
|
|
4849
4952
|
SKIP_TELEMETRY_HOSTS = /(waa-pa\.|signaler-pa\.|appsgrowthpromo-pa\.|ogads-pa\.|peoplestackwebexperiments-pa\.)/i;
|
|
4850
4953
|
SKIP_TELEMETRY_PATHS = /\/(log|logging|telemetry|analytics|beacon|ping|heartbeat|metrics)(\/|$)/i;
|
|
4851
|
-
RPC_HINTS = /(\/$rpc\/|\/rpc\/|graphql|trending|search|feed|results|batchexecute|\/api
|
|
4954
|
+
RPC_HINTS = /(\/$rpc\/|\/rpc\/|graphql|trending|search|feed|results|batchexecute|\/api\/|youtubei|__ssr_data__)/i;
|
|
4852
4955
|
ALLOWED_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE"]);
|
|
4853
4956
|
STRIP_HEADERS = new Set([
|
|
4854
4957
|
"cookie",
|
|
@@ -4926,7 +5029,180 @@ var init_reverse_engineer = __esm(() => {
|
|
|
4926
5029
|
"adsize",
|
|
4927
5030
|
"lineitemid"
|
|
4928
5031
|
]);
|
|
4929
|
-
ON_DOMAIN_NOISE = /\/(recaptcha|captcha|update-recaptcha|csrf|consent|data-protection|badge|drawer|header-action|geolocation|onboarding|wana\/bids|prebid|bids\/request|ads\/|pixel|beacon|collect|impression|click-tracking|heartbeat|webConfig|config\.json|manifest\.json|service-worker|sw\.js|favicon|robots\.txt|sitemap|opensearch|partial\/[a-zA-Z]+\/mod-|logging|csp-report|gen_204|generate_204|sodar|
|
|
5032
|
+
ON_DOMAIN_NOISE = /\/(recaptcha|captcha|update-recaptcha|csrf|consent|data-protection|badge|drawer|header-action|geolocation|onboarding|wana\/bids|prebid|bids\/request|ads\/|pixel|beacon|collect|impression|click-tracking|heartbeat|webConfig|config\.json|manifest\.json|service-worker|sw\.js|favicon|robots\.txt|sitemap|opensearch|partial\/[a-zA-Z]+\/mod-|logging|csp-report|gen_204|generate_204|sodar|__webpack|__next|devvit-|user-drawer|action-item)/i;
|
|
5033
|
+
});
|
|
5034
|
+
|
|
5035
|
+
// ../../src/vault/index.ts
|
|
5036
|
+
var exports_vault = {};
|
|
5037
|
+
__export(exports_vault, {
|
|
5038
|
+
storeCredential: () => storeCredential,
|
|
5039
|
+
setKeytarClientForTests: () => setKeytarClientForTests,
|
|
5040
|
+
resetKeytarClientForTests: () => resetKeytarClientForTests,
|
|
5041
|
+
normalizeKeytarModule: () => normalizeKeytarModule,
|
|
5042
|
+
getCredential: () => getCredential,
|
|
5043
|
+
deleteCredential: () => deleteCredential
|
|
5044
|
+
});
|
|
5045
|
+
import { createCipheriv, createDecipheriv, randomBytes } from "crypto";
|
|
5046
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync, writeFileSync as writeFileSync2 } from "fs";
|
|
5047
|
+
import { join as join2 } from "path";
|
|
5048
|
+
import { homedir } from "os";
|
|
5049
|
+
function normalizeKeytarModule(mod) {
|
|
5050
|
+
let candidate = mod;
|
|
5051
|
+
for (let depth = 0;depth < 3; depth++) {
|
|
5052
|
+
if (!candidate || typeof candidate !== "object" || !("default" in candidate))
|
|
5053
|
+
break;
|
|
5054
|
+
candidate = candidate.default;
|
|
5055
|
+
}
|
|
5056
|
+
if (!candidate)
|
|
5057
|
+
return null;
|
|
5058
|
+
if (typeof candidate.setPassword === "function" && typeof candidate.getPassword === "function" && typeof candidate.deletePassword === "function") {
|
|
5059
|
+
return candidate;
|
|
5060
|
+
}
|
|
5061
|
+
return null;
|
|
5062
|
+
}
|
|
5063
|
+
function isKeytarBindingError(error) {
|
|
5064
|
+
const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
5065
|
+
return KEYTAR_BINDING_ERROR_RE.test(message);
|
|
5066
|
+
}
|
|
5067
|
+
function disableKeytar(error) {
|
|
5068
|
+
keytar = null;
|
|
5069
|
+
if (keytarFallbackLogged)
|
|
5070
|
+
return;
|
|
5071
|
+
const summary = error instanceof Error ? error.message : String(error);
|
|
5072
|
+
log("vault", `keytar runtime unavailable (${summary}); using encrypted file fallback`);
|
|
5073
|
+
keytarFallbackLogged = true;
|
|
5074
|
+
}
|
|
5075
|
+
async function callKeytar(op) {
|
|
5076
|
+
if (!keytar)
|
|
5077
|
+
return KEYTAR_UNAVAILABLE;
|
|
5078
|
+
try {
|
|
5079
|
+
return await op(keytar);
|
|
5080
|
+
} catch (error) {
|
|
5081
|
+
if (!isKeytarBindingError(error))
|
|
5082
|
+
throw error;
|
|
5083
|
+
disableKeytar(error);
|
|
5084
|
+
return KEYTAR_UNAVAILABLE;
|
|
5085
|
+
}
|
|
5086
|
+
}
|
|
5087
|
+
function setKeytarClientForTests(client) {
|
|
5088
|
+
keytar = client;
|
|
5089
|
+
keytarFallbackLogged = false;
|
|
5090
|
+
}
|
|
5091
|
+
function resetKeytarClientForTests() {
|
|
5092
|
+
keytar = importedKeytar;
|
|
5093
|
+
keytarFallbackLogged = false;
|
|
5094
|
+
}
|
|
5095
|
+
function getOrCreateKey() {
|
|
5096
|
+
if (!existsSync3(VAULT_DIR))
|
|
5097
|
+
mkdirSync3(VAULT_DIR, { recursive: true, mode: 448 });
|
|
5098
|
+
if (existsSync3(KEY_FILE))
|
|
5099
|
+
return readFileSync(KEY_FILE);
|
|
5100
|
+
const key = randomBytes(32);
|
|
5101
|
+
writeFileSync2(KEY_FILE, key, { mode: 384 });
|
|
5102
|
+
return key;
|
|
5103
|
+
}
|
|
5104
|
+
function withVaultLock(fn) {
|
|
5105
|
+
const prev = vaultLock;
|
|
5106
|
+
let release;
|
|
5107
|
+
vaultLock = new Promise((r) => {
|
|
5108
|
+
release = r;
|
|
5109
|
+
});
|
|
5110
|
+
return prev.then(fn).finally(() => release());
|
|
5111
|
+
}
|
|
5112
|
+
function readVaultFile() {
|
|
5113
|
+
if (!existsSync3(VAULT_FILE))
|
|
5114
|
+
return {};
|
|
5115
|
+
try {
|
|
5116
|
+
const key = getOrCreateKey();
|
|
5117
|
+
const raw = readFileSync(VAULT_FILE);
|
|
5118
|
+
const iv = raw.subarray(0, 16);
|
|
5119
|
+
const enc = raw.subarray(16);
|
|
5120
|
+
const decipher = createDecipheriv("aes-256-cbc", key, iv);
|
|
5121
|
+
const dec = Buffer.concat([decipher.update(enc), decipher.final()]);
|
|
5122
|
+
return JSON.parse(dec.toString("utf8"));
|
|
5123
|
+
} catch {
|
|
5124
|
+
return {};
|
|
5125
|
+
}
|
|
5126
|
+
}
|
|
5127
|
+
function writeVaultFile(data) {
|
|
5128
|
+
const key = getOrCreateKey();
|
|
5129
|
+
const iv = randomBytes(16);
|
|
5130
|
+
const cipher = createCipheriv("aes-256-cbc", key, iv);
|
|
5131
|
+
const enc = Buffer.concat([cipher.update(JSON.stringify(data), "utf8"), cipher.final()]);
|
|
5132
|
+
writeFileSync2(VAULT_FILE, Buffer.concat([iv, enc]), { mode: 384 });
|
|
5133
|
+
}
|
|
5134
|
+
async function storeCredential(account, value, opts) {
|
|
5135
|
+
const wrapped = {
|
|
5136
|
+
value,
|
|
5137
|
+
stored_at: new Date().toISOString(),
|
|
5138
|
+
expires_at: opts?.expires_at,
|
|
5139
|
+
max_age_ms: opts?.max_age_ms
|
|
5140
|
+
};
|
|
5141
|
+
const serialized = JSON.stringify(wrapped);
|
|
5142
|
+
const keytarResult = await callKeytar((client) => client.setPassword(SERVICE, account, serialized));
|
|
5143
|
+
if (keytarResult !== KEYTAR_UNAVAILABLE)
|
|
5144
|
+
return;
|
|
5145
|
+
await withVaultLock(() => {
|
|
5146
|
+
const data = readVaultFile();
|
|
5147
|
+
data[account] = serialized;
|
|
5148
|
+
writeVaultFile(data);
|
|
5149
|
+
});
|
|
5150
|
+
}
|
|
5151
|
+
function isExpired(cred) {
|
|
5152
|
+
if (cred.expires_at) {
|
|
5153
|
+
return new Date(cred.expires_at).getTime() <= Date.now();
|
|
5154
|
+
}
|
|
5155
|
+
if (cred.max_age_ms) {
|
|
5156
|
+
return new Date(cred.stored_at).getTime() + cred.max_age_ms <= Date.now();
|
|
5157
|
+
}
|
|
5158
|
+
return false;
|
|
5159
|
+
}
|
|
5160
|
+
async function getCredential(account) {
|
|
5161
|
+
let raw;
|
|
5162
|
+
const keytarResult = await callKeytar((client) => client.getPassword(SERVICE, account));
|
|
5163
|
+
if (keytarResult !== KEYTAR_UNAVAILABLE) {
|
|
5164
|
+
raw = keytarResult;
|
|
5165
|
+
} else {
|
|
5166
|
+
const data = readVaultFile();
|
|
5167
|
+
raw = data[account] ?? null;
|
|
5168
|
+
}
|
|
5169
|
+
if (!raw)
|
|
5170
|
+
return null;
|
|
5171
|
+
try {
|
|
5172
|
+
const parsed = JSON.parse(raw);
|
|
5173
|
+
if (parsed.value && parsed.stored_at) {
|
|
5174
|
+
if (isExpired(parsed)) {
|
|
5175
|
+
await deleteCredential(account);
|
|
5176
|
+
return null;
|
|
5177
|
+
}
|
|
5178
|
+
return parsed.value;
|
|
5179
|
+
}
|
|
5180
|
+
} catch {}
|
|
5181
|
+
return raw;
|
|
5182
|
+
}
|
|
5183
|
+
async function deleteCredential(account) {
|
|
5184
|
+
const keytarResult = await callKeytar((client) => client.deletePassword(SERVICE, account));
|
|
5185
|
+
if (keytarResult !== KEYTAR_UNAVAILABLE)
|
|
5186
|
+
return;
|
|
5187
|
+
await withVaultLock(() => {
|
|
5188
|
+
const data = readVaultFile();
|
|
5189
|
+
delete data[account];
|
|
5190
|
+
writeVaultFile(data);
|
|
5191
|
+
});
|
|
5192
|
+
}
|
|
5193
|
+
var KEYTAR_UNAVAILABLE, KEYTAR_BINDING_ERROR_RE, keytar = null, importedKeytar, keytarFallbackLogged = false, SERVICE = "unbrowse", VAULT_DIR, VAULT_FILE, KEY_FILE, vaultLock;
|
|
5194
|
+
var init_vault = __esm(async () => {
|
|
5195
|
+
init_logger();
|
|
5196
|
+
KEYTAR_UNAVAILABLE = Symbol("KEYTAR_UNAVAILABLE");
|
|
5197
|
+
KEYTAR_BINDING_ERROR_RE = /(keytar(?:\.node)?|native bindings?|bindings file|no native build was found|could not locate the bindings file|module did not self-register|err_dlopen_failed|dlopen\(|compiled against a different node\.js version|cannot find module .*keytar|wasm is not supported on this platform|(set|get|delete)password is not a function)/i;
|
|
5198
|
+
try {
|
|
5199
|
+
keytar = normalizeKeytarModule(await import("keytar"));
|
|
5200
|
+
} catch {}
|
|
5201
|
+
importedKeytar = keytar;
|
|
5202
|
+
VAULT_DIR = join2(homedir(), ".unbrowse", "vault");
|
|
5203
|
+
VAULT_FILE = join2(VAULT_DIR, "credentials.enc");
|
|
5204
|
+
KEY_FILE = join2(VAULT_DIR, ".key");
|
|
5205
|
+
vaultLock = Promise.resolve();
|
|
4930
5206
|
});
|
|
4931
5207
|
|
|
4932
5208
|
// ../../src/runtime/browser-access.ts
|
|
@@ -5379,7 +5655,12 @@ function normalizeCapturedUrl(url, baseUrl) {
|
|
|
5379
5655
|
if (!url)
|
|
5380
5656
|
return url;
|
|
5381
5657
|
try {
|
|
5382
|
-
|
|
5658
|
+
const parsed = new URL(url);
|
|
5659
|
+
if (baseUrl && (parsed.hostname === "127.0.0.1" || parsed.hostname === "localhost")) {
|
|
5660
|
+
const pageOrigin = new URL(baseUrl).origin;
|
|
5661
|
+
return `${pageOrigin}${parsed.pathname}${parsed.search}`;
|
|
5662
|
+
}
|
|
5663
|
+
return parsed.toString();
|
|
5383
5664
|
} catch {
|
|
5384
5665
|
if (!baseUrl)
|
|
5385
5666
|
return url;
|
|
@@ -5780,7 +6061,76 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
|
|
|
5780
6061
|
log("capture", "scriptInject unavailable — falling back to evaluate injection");
|
|
5781
6062
|
}
|
|
5782
6063
|
}
|
|
5783
|
-
|
|
6064
|
+
try {
|
|
6065
|
+
await phase("harStart", () => harStart(tabId));
|
|
6066
|
+
} catch {}
|
|
6067
|
+
const cdpNetworkEntries = [];
|
|
6068
|
+
const cdpRequestMap = new Map;
|
|
6069
|
+
const cdpResponseMeta = new Map;
|
|
6070
|
+
const cdpFinishedRequests = new Set;
|
|
6071
|
+
let cdpWs = null;
|
|
6072
|
+
let cdpMsgId = 10;
|
|
6073
|
+
const cdpPendingBodies = new Map;
|
|
6074
|
+
const cdpResolvedBodies = new Map;
|
|
6075
|
+
const API_URL_PATTERN = /\/api\/|\/graphql|voyager|youtubei|\/v\d+\//i;
|
|
6076
|
+
try {
|
|
6077
|
+
const cdpPort = getCdpPort();
|
|
6078
|
+
if (cdpPort) {
|
|
6079
|
+
const tabsResp = await fetch(`http://127.0.0.1:${cdpPort}/json`);
|
|
6080
|
+
const tabs = await tabsResp.json();
|
|
6081
|
+
const cdpTab = tabs.find((t) => t.id === tabId) ?? tabs.find((t) => t.type === "page");
|
|
6082
|
+
if (cdpTab?.webSocketDebuggerUrl) {
|
|
6083
|
+
cdpWs = new WebSocket(cdpTab.webSocketDebuggerUrl);
|
|
6084
|
+
await new Promise((resolve, reject) => {
|
|
6085
|
+
cdpWs.onopen = () => resolve();
|
|
6086
|
+
cdpWs.onerror = () => reject(new Error("CDP WS failed"));
|
|
6087
|
+
setTimeout(() => reject(new Error("CDP WS timeout")), 3000);
|
|
6088
|
+
});
|
|
6089
|
+
cdpWs.onmessage = (ev) => {
|
|
6090
|
+
try {
|
|
6091
|
+
const msg = JSON.parse(String(ev.data));
|
|
6092
|
+
if (msg.method === "Network.requestWillBeSent") {
|
|
6093
|
+
const p = msg.params;
|
|
6094
|
+
const reqHeaders = Object.entries(p.request?.headers ?? {}).map(([name, value]) => ({ name, value: String(value) }));
|
|
6095
|
+
cdpRequestMap.set(p.requestId, {
|
|
6096
|
+
method: p.request?.method ?? "GET",
|
|
6097
|
+
url: p.request?.url ?? "",
|
|
6098
|
+
headers: reqHeaders,
|
|
6099
|
+
postData: p.request?.postData
|
|
6100
|
+
});
|
|
6101
|
+
} else if (msg.method === "Network.responseReceived") {
|
|
6102
|
+
const p = msg.params;
|
|
6103
|
+
const respHeaders = Object.entries(p.response?.headers ?? {}).map(([name, value]) => ({ name, value: String(value) }));
|
|
6104
|
+
cdpResponseMeta.set(p.requestId, {
|
|
6105
|
+
status: p.response?.status ?? 0,
|
|
6106
|
+
headers: respHeaders,
|
|
6107
|
+
mimeType: p.response?.mimeType ?? ""
|
|
6108
|
+
});
|
|
6109
|
+
} else if (msg.method === "Network.loadingFinished") {
|
|
6110
|
+
const requestId = msg.params?.requestId;
|
|
6111
|
+
cdpFinishedRequests.add(requestId);
|
|
6112
|
+
const req = cdpRequestMap.get(requestId);
|
|
6113
|
+
const resp = cdpResponseMeta.get(requestId);
|
|
6114
|
+
if (req && resp && API_URL_PATTERN.test(req.url) && (resp.mimeType.includes("json") || resp.mimeType.includes("text"))) {
|
|
6115
|
+
const id = ++cdpMsgId;
|
|
6116
|
+
cdpPendingBodies.set(id, req.url);
|
|
6117
|
+
cdpWs.send(JSON.stringify({ id, method: "Network.getResponseBody", params: { requestId } }));
|
|
6118
|
+
}
|
|
6119
|
+
} else if (msg.id && cdpPendingBodies.has(msg.id)) {
|
|
6120
|
+
const reqUrl = cdpPendingBodies.get(msg.id);
|
|
6121
|
+
cdpPendingBodies.delete(msg.id);
|
|
6122
|
+
if (msg.result?.body) {
|
|
6123
|
+
cdpResolvedBodies.set(reqUrl, msg.result.body);
|
|
6124
|
+
}
|
|
6125
|
+
}
|
|
6126
|
+
} catch {}
|
|
6127
|
+
};
|
|
6128
|
+
cdpWs.send(JSON.stringify({ id: 1, method: "Network.enable", params: {} }));
|
|
6129
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
6130
|
+
log("capture", "CDP network capture enabled (direct websocket)");
|
|
6131
|
+
}
|
|
6132
|
+
}
|
|
6133
|
+
} catch {}
|
|
5784
6134
|
let pageDomain;
|
|
5785
6135
|
try {
|
|
5786
6136
|
pageDomain = getRegistrableDomain(new URL(url).hostname);
|
|
@@ -5834,11 +6184,85 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
|
|
|
5834
6184
|
const perfEntries = await phase("evaluate:perf", () => collectPerformanceResourceEntries(tabId));
|
|
5835
6185
|
performanceUrls = await phase("replay-fetch", () => replayPerformanceApiResponses(tabId, perfEntries, responseBodies, url, intent));
|
|
5836
6186
|
} catch {}
|
|
6187
|
+
if (cdpRequestMap.size > 0) {
|
|
6188
|
+
try {
|
|
6189
|
+
const cdpRawReqs = [...cdpRequestMap.values()].map((r) => ({
|
|
6190
|
+
url: r.url,
|
|
6191
|
+
method: r.method,
|
|
6192
|
+
request_headers: Object.fromEntries(r.headers.map((h) => [h.name.toLowerCase(), h.value])),
|
|
6193
|
+
response_status: 200,
|
|
6194
|
+
response_headers: {},
|
|
6195
|
+
response_body: undefined,
|
|
6196
|
+
timestamp: ""
|
|
6197
|
+
}));
|
|
6198
|
+
const authHeaders2 = extractAuthHeaders(cdpRawReqs);
|
|
6199
|
+
if (Object.keys(authHeaders2).length > 0) {
|
|
6200
|
+
const rawKey = `${domain}-session`;
|
|
6201
|
+
const { getCredential: getCredential2 } = await init_vault().then(() => exports_vault);
|
|
6202
|
+
let existing = {};
|
|
6203
|
+
try {
|
|
6204
|
+
const stored = await getCredential2(rawKey);
|
|
6205
|
+
if (stored)
|
|
6206
|
+
existing = JSON.parse(stored);
|
|
6207
|
+
} catch {}
|
|
6208
|
+
if (!existing.cookies || existing.cookies.length === 0) {
|
|
6209
|
+
try {
|
|
6210
|
+
const loginKey = `auth:${getRegistrableDomain(domain)}`;
|
|
6211
|
+
const loginStored = await getCredential2(loginKey);
|
|
6212
|
+
if (loginStored) {
|
|
6213
|
+
const loginData = JSON.parse(loginStored);
|
|
6214
|
+
if (loginData.cookies?.length)
|
|
6215
|
+
existing.cookies = loginData.cookies;
|
|
6216
|
+
}
|
|
6217
|
+
} catch {}
|
|
6218
|
+
}
|
|
6219
|
+
await storeCredential(rawKey, JSON.stringify({
|
|
6220
|
+
...existing,
|
|
6221
|
+
headers: { ...existing.headers ?? {}, ...authHeaders2 }
|
|
6222
|
+
}));
|
|
6223
|
+
log("capture", `early auth store: ${Object.keys(authHeaders2).join(", ")} for ${domain}`);
|
|
6224
|
+
}
|
|
6225
|
+
} catch (e) {
|
|
6226
|
+
log("capture", `early auth store failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
6227
|
+
}
|
|
6228
|
+
}
|
|
5837
6229
|
let harEntries = [];
|
|
5838
6230
|
try {
|
|
5839
|
-
const
|
|
5840
|
-
|
|
6231
|
+
const harPromise = harStop(tabId);
|
|
6232
|
+
const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve({ entries: [] }), 5000));
|
|
6233
|
+
const harResult = await Promise.race([harPromise, timeoutPromise]);
|
|
6234
|
+
harEntries = harResult.entries ?? [];
|
|
5841
6235
|
} catch {}
|
|
6236
|
+
if (cdpPendingBodies.size > 0) {
|
|
6237
|
+
await new Promise((r) => setTimeout(r, 1500));
|
|
6238
|
+
}
|
|
6239
|
+
if (cdpWs) {
|
|
6240
|
+
try {
|
|
6241
|
+
cdpWs.close();
|
|
6242
|
+
} catch {}
|
|
6243
|
+
}
|
|
6244
|
+
const harUrls = new Set(harEntries.map((e) => e.request?.url).filter(Boolean));
|
|
6245
|
+
let cdpAdded = 0;
|
|
6246
|
+
for (const [requestId, req] of cdpRequestMap) {
|
|
6247
|
+
if (harUrls.has(req.url))
|
|
6248
|
+
continue;
|
|
6249
|
+
const resp = cdpResponseMeta.get(requestId);
|
|
6250
|
+
if (!resp)
|
|
6251
|
+
continue;
|
|
6252
|
+
const body = cdpResolvedBodies.get(req.url);
|
|
6253
|
+
harEntries.push({
|
|
6254
|
+
startedDateTime: new Date().toISOString(),
|
|
6255
|
+
request: { method: req.method, url: req.url, headers: req.headers, postData: req.postData ? { text: req.postData } : undefined },
|
|
6256
|
+
response: { status: resp.status, headers: resp.headers, content: body ? { text: body, mimeType: resp.mimeType } : {} }
|
|
6257
|
+
});
|
|
6258
|
+
if (body && body.length > 0 && !responseBodies.has(req.url)) {
|
|
6259
|
+
responseBodies.set(req.url, body);
|
|
6260
|
+
}
|
|
6261
|
+
cdpAdded++;
|
|
6262
|
+
}
|
|
6263
|
+
if (cdpAdded > 0) {
|
|
6264
|
+
log("capture", `CDP direct capture added ${cdpAdded} entries (${cdpResolvedBodies.size} with bodies, ${harEntries.length} total HAR)`);
|
|
6265
|
+
}
|
|
5842
6266
|
const HAR_REPLAY_CT2 = /application\/json|text\/plain|\+json/i;
|
|
5843
6267
|
let harReplayCount = 0;
|
|
5844
6268
|
for (const entry of harEntries) {
|
|
@@ -5854,7 +6278,7 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
|
|
|
5854
6278
|
if (status < 200 || status >= 400)
|
|
5855
6279
|
continue;
|
|
5856
6280
|
const ct = (entry.response?.headers ?? []).find((h) => h.name.toLowerCase() === "content-type")?.value ?? "";
|
|
5857
|
-
if (!HAR_REPLAY_CT2.test(ct) && !harUrl.includes("/api/") && !harUrl.includes("_search") && !harUrl.includes("graphql"))
|
|
6281
|
+
if (!HAR_REPLAY_CT2.test(ct) && !harUrl.includes("/api/") && !harUrl.includes("_search") && !harUrl.includes("graphql") && !harUrl.includes("youtubei"))
|
|
5858
6282
|
continue;
|
|
5859
6283
|
if (harReplayCount >= 20)
|
|
5860
6284
|
break;
|
|
@@ -5862,7 +6286,7 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
|
|
|
5862
6286
|
const postData = method !== "GET" ? entry.request?.postData?.text : undefined;
|
|
5863
6287
|
const replayScript = method === "GET" || !postData ? `(function(){var x=new XMLHttpRequest();x.open('GET','${harUrl.replace(/'/g, "\\'")}',false);x.send();return x.status>=200&&x.status<400?x.responseText:''})()` : `(function(){var x=new XMLHttpRequest();x.open('${method}','${harUrl.replace(/'/g, "\\'")}',false);x.setRequestHeader('Content-Type','application/json');x.send(${JSON.stringify(postData)});return x.status>=200&&x.status<400?x.responseText:''})()`;
|
|
5864
6288
|
const body = await phase("har-replay", () => evaluate(tabId, replayScript));
|
|
5865
|
-
if (typeof body === "string" && body.length > 0 && body.length <
|
|
6289
|
+
if (typeof body === "string" && body.length > 0 && body.length < 524288) {
|
|
5866
6290
|
responseBodies.set(harUrl, body);
|
|
5867
6291
|
harReplayCount++;
|
|
5868
6292
|
log("capture", `har-replay-fetched ${harUrl.substring(0, 80)} (${body.length}B)`);
|
|
@@ -5889,6 +6313,77 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
|
|
|
5889
6313
|
}
|
|
5890
6314
|
html = await phase("getPageHtml", () => getPageHtml(tabId));
|
|
5891
6315
|
} catch {}
|
|
6316
|
+
const SSR_DATA_EXTRACTORS = [
|
|
6317
|
+
{ name: "ytmusic", script: `(function(){try{var d=ytcfg.get('YTMUSIC_INITIAL_DATA');if(!d||!d.length)return null;var out={};d.forEach(function(x){if(x.path&&x.data)out[x.path]=x.data});out.__apiKey=ytcfg.get('INNERTUBE_API_KEY')||'';out.__context=ytcfg.get('INNERTUBE_CONTEXT')||null;return JSON.stringify(out)}catch(e){return null}})()` },
|
|
6318
|
+
{ name: "youtube", script: `(function(){try{return typeof ytInitialData!=='undefined'?JSON.stringify(ytInitialData):null}catch(e){return null}})()` },
|
|
6319
|
+
{ name: "nextjs", script: `(function(){try{return window.__NEXT_DATA__?JSON.stringify(window.__NEXT_DATA__):null}catch(e){return null}})()` },
|
|
6320
|
+
{ name: "nuxt", script: `(function(){try{return window.__NUXT__?JSON.stringify(window.__NUXT__):null}catch(e){return null}})()` }
|
|
6321
|
+
];
|
|
6322
|
+
let embeddedDataCount = 0;
|
|
6323
|
+
for (const extractor of SSR_DATA_EXTRACTORS) {
|
|
6324
|
+
try {
|
|
6325
|
+
const raw = await phase(`ssr:${extractor.name}`, () => evaluate(tabId, extractor.script));
|
|
6326
|
+
if (typeof raw !== "string" || !raw || raw === "null")
|
|
6327
|
+
continue;
|
|
6328
|
+
if (extractor.name === "ytmusic") {
|
|
6329
|
+
const parsed = JSON.parse(raw);
|
|
6330
|
+
const apiKey = parsed.__apiKey || "";
|
|
6331
|
+
const innertubeContext = parsed.__context;
|
|
6332
|
+
delete parsed.__apiKey;
|
|
6333
|
+
delete parsed.__context;
|
|
6334
|
+
for (const [path4, data] of Object.entries(parsed)) {
|
|
6335
|
+
if (!data || typeof data !== "object")
|
|
6336
|
+
continue;
|
|
6337
|
+
const cleaned = { ...data };
|
|
6338
|
+
delete cleaned.responseContext;
|
|
6339
|
+
delete cleaned.trackingParams;
|
|
6340
|
+
delete cleaned.header;
|
|
6341
|
+
delete cleaned.background;
|
|
6342
|
+
let bodyStr = JSON.stringify(cleaned);
|
|
6343
|
+
if (bodyStr.length > 1e4) {
|
|
6344
|
+
bodyStr = bodyStr.substring(0, 1e4) + '"}]}';
|
|
6345
|
+
try {
|
|
6346
|
+
JSON.parse(bodyStr);
|
|
6347
|
+
} catch {
|
|
6348
|
+
bodyStr = JSON.stringify(cleaned).substring(0, 1e4);
|
|
6349
|
+
}
|
|
6350
|
+
}
|
|
6351
|
+
if (bodyStr.length < 100)
|
|
6352
|
+
continue;
|
|
6353
|
+
const origin = new URL(url).origin;
|
|
6354
|
+
const contextParams = new URL(url).searchParams;
|
|
6355
|
+
const keyParam = apiKey ? `key=${apiKey}&` : "";
|
|
6356
|
+
const queryStr = path4.includes("search") && contextParams.toString() ? `?${keyParam}${contextParams.toString()}&prettyPrint=false` : `?${keyParam}prettyPrint=false`;
|
|
6357
|
+
const syntheticUrl = `${origin}/youtubei/v1${path4}${queryStr}`;
|
|
6358
|
+
responseBodies.set(syntheticUrl, bodyStr);
|
|
6359
|
+
const queryValue = contextParams.get("q") ?? "";
|
|
6360
|
+
const postBody = path4.includes("search") && innertubeContext ? JSON.stringify({ context: innertubeContext, query: queryValue }) : path4.includes("browse") && innertubeContext ? JSON.stringify({ context: innertubeContext, browseId: "FEmusic_liked_videos" }) : JSON.stringify({ context: innertubeContext ?? {} });
|
|
6361
|
+
harEntries.push({
|
|
6362
|
+
startedDateTime: new Date().toISOString(),
|
|
6363
|
+
request: { method: "POST", url: syntheticUrl, headers: [{ name: "content-type", value: "application/json" }], postData: { text: postBody } },
|
|
6364
|
+
response: { status: 200, headers: [{ name: "content-type", value: "application/json" }], content: { text: bodyStr, mimeType: "application/json" } }
|
|
6365
|
+
});
|
|
6366
|
+
embeddedDataCount++;
|
|
6367
|
+
log("capture", `ssr:${extractor.name} extracted ${path4} (${bodyStr.length}B)`);
|
|
6368
|
+
}
|
|
6369
|
+
} else {
|
|
6370
|
+
if (raw.length < 100)
|
|
6371
|
+
continue;
|
|
6372
|
+
const syntheticUrl = `${new URL(url).origin}/__ssr_data__/${extractor.name}`;
|
|
6373
|
+
responseBodies.set(syntheticUrl, raw);
|
|
6374
|
+
harEntries.push({
|
|
6375
|
+
startedDateTime: new Date().toISOString(),
|
|
6376
|
+
request: { method: "GET", url: syntheticUrl, headers: [] },
|
|
6377
|
+
response: { status: 200, headers: [{ name: "content-type", value: "application/json" }], content: { text: raw, mimeType: "application/json" } }
|
|
6378
|
+
});
|
|
6379
|
+
embeddedDataCount++;
|
|
6380
|
+
log("capture", `ssr:${extractor.name} extracted (${raw.length}B)`);
|
|
6381
|
+
}
|
|
6382
|
+
} catch {}
|
|
6383
|
+
}
|
|
6384
|
+
if (embeddedDataCount > 0) {
|
|
6385
|
+
log("capture", `embedded SSR data: ${embeddedDataCount} synthetic endpoints injected`);
|
|
6386
|
+
}
|
|
5892
6387
|
const requests = mergePassiveCaptureData(intercepted, harEntries, extensionEntries, responseBodies, performanceUrls);
|
|
5893
6388
|
log("capture", `tracked ${harEntries.length} HAR, ${intercepted.length} intercepted, ${extensionEntries.length} extension, ${responseBodies.size} bodies → ${requests.length} merged`);
|
|
5894
6389
|
const rawCookies = await phase("extractCookies", () => extractCookiesFromPage(tabId, url));
|
|
@@ -6287,7 +6782,8 @@ var MAX_CONCURRENT_TABS = 3, activeTabs = 0, waitQueue, activeTabRegistry, inter
|
|
|
6287
6782
|
var isData = ct.indexOf('json') !== -1 || ct.indexOf('application/x-protobuf') !== -1 ||
|
|
6288
6783
|
ct.indexOf('text/plain') !== -1 ||
|
|
6289
6784
|
url.indexOf('batchexecute') !== -1 || url.indexOf('/api/') !== -1 ||
|
|
6290
|
-
url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1
|
|
6785
|
+
url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1 ||
|
|
6786
|
+
url.indexOf('youtubei') !== -1;
|
|
6291
6787
|
if (!isJs && !isData) return response;
|
|
6292
6788
|
if (/\\.(css|woff2?|png|jpg|svg|ico)(\\?|$)/.test(url)) return response;
|
|
6293
6789
|
var clone = response.clone();
|
|
@@ -6337,7 +6833,8 @@ var MAX_CONCURRENT_TABS = 3, activeTabs = 0, waitQueue, activeTabRegistry, inter
|
|
|
6337
6833
|
var isData = ct.indexOf('json') !== -1 || ct.indexOf('application/x-protobuf') !== -1 ||
|
|
6338
6834
|
ct.indexOf('text/plain') !== -1 ||
|
|
6339
6835
|
url.indexOf('batchexecute') !== -1 || url.indexOf('/api/') !== -1 ||
|
|
6340
|
-
url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1
|
|
6836
|
+
url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1 ||
|
|
6837
|
+
url.indexOf('youtubei') !== -1;
|
|
6341
6838
|
if (!isJs && !isData) return;
|
|
6342
6839
|
if (/\\.(css|woff2?|png|jpg|svg|ico)(\\?|$)/.test(url)) return;
|
|
6343
6840
|
var respBody = xhr.responseText || '';
|
|
@@ -6359,11 +6856,13 @@ var MAX_CONCURRENT_TABS = 3, activeTabs = 0, waitQueue, activeTabRegistry, inter
|
|
|
6359
6856
|
return origSend.apply(this, arguments);
|
|
6360
6857
|
};
|
|
6361
6858
|
})()`, REPLAY_SKIP, HAR_REPLAY_CT;
|
|
6362
|
-
var init_capture = __esm(() => {
|
|
6859
|
+
var init_capture = __esm(async () => {
|
|
6363
6860
|
init_client();
|
|
6364
6861
|
init_domain();
|
|
6365
6862
|
init_logger();
|
|
6863
|
+
init_reverse_engineer();
|
|
6366
6864
|
init_browser_access();
|
|
6865
|
+
await init_vault();
|
|
6367
6866
|
waitQueue = [];
|
|
6368
6867
|
activeTabRegistry = new Set;
|
|
6369
6868
|
interceptorInjectedTabs = new Set;
|
|
@@ -6380,17 +6879,17 @@ var init_capture = __esm(() => {
|
|
|
6380
6879
|
});
|
|
6381
6880
|
|
|
6382
6881
|
// ../../src/build-info.generated.ts
|
|
6383
|
-
var BUILD_RELEASE_VERSION = "3.2.
|
|
6882
|
+
var BUILD_RELEASE_VERSION = "3.2.2-experiments.128ff09", BUILD_GIT_SHA = "128ff09f51b3", BUILD_CODE_HASH = "1488fc1d92b7", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4yLjItZXhwZXJpbWVudHMuMTI4ZmYwOSIsImdpdF9zaGEiOiIxMjhmZjA5ZjUxYjMiLCJjb2RlX2hhc2giOiIxNDg4ZmMxZDkyYjciLCJ0cmFjZV92ZXJzaW9uIjoiMTQ4OGZjMWQ5MmI3QDEyOGZmMDlmNTFiMyIsImlzc3VlZF9hdCI6IjIwMjYtMDQtMDZUMTE6NTk6NDYuOTQyWiJ9", BUILD_RELEASE_MANIFEST_SIGNATURE = "M80x5zV9aWn-teZrpEFDyzwSU53Hwfj6rf19dl9m-jM", BUILD_DEFAULT_BACKEND_URL = "https://unbrowse-backend-experiments.lewis-6d8.workers.dev";
|
|
6384
6883
|
|
|
6385
6884
|
// ../../src/version.ts
|
|
6386
6885
|
import { createHash } from "crypto";
|
|
6387
|
-
import { existsSync as
|
|
6388
|
-
import { dirname, join as
|
|
6886
|
+
import { existsSync as existsSync4, readFileSync as readFileSync2, readdirSync } from "fs";
|
|
6887
|
+
import { dirname, join as join3, parse } from "path";
|
|
6389
6888
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
6390
6889
|
function collectTsFiles(dir) {
|
|
6391
6890
|
const results = [];
|
|
6392
6891
|
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
6393
|
-
const full =
|
|
6892
|
+
const full = join3(dir, entry.name);
|
|
6394
6893
|
if (entry.isDirectory() && entry.name !== "node_modules") {
|
|
6395
6894
|
results.push(...collectTsFiles(full));
|
|
6396
6895
|
} else if (entry.name.endsWith(".ts")) {
|
|
@@ -6403,21 +6902,21 @@ function hashFiles(srcDir, files) {
|
|
|
6403
6902
|
const hash = createHash("sha256");
|
|
6404
6903
|
for (const file of files) {
|
|
6405
6904
|
hash.update(file.slice(srcDir.length));
|
|
6406
|
-
hash.update(
|
|
6905
|
+
hash.update(readFileSync2(file, "utf-8"));
|
|
6407
6906
|
}
|
|
6408
6907
|
return hash.digest("hex").slice(0, 12);
|
|
6409
6908
|
}
|
|
6410
6909
|
function resolveCodeHashSourceDir(moduleDir) {
|
|
6411
6910
|
const candidates = [
|
|
6412
6911
|
moduleDir,
|
|
6413
|
-
|
|
6414
|
-
|
|
6415
|
-
|
|
6416
|
-
|
|
6912
|
+
join3(moduleDir, "runtime-src"),
|
|
6913
|
+
join3(moduleDir, "..", "runtime-src"),
|
|
6914
|
+
join3(moduleDir, "src"),
|
|
6915
|
+
join3(moduleDir, "..", "src")
|
|
6417
6916
|
];
|
|
6418
6917
|
for (const candidate of candidates) {
|
|
6419
6918
|
try {
|
|
6420
|
-
if (!
|
|
6919
|
+
if (!existsSync4(candidate))
|
|
6421
6920
|
continue;
|
|
6422
6921
|
const files = collectTsFiles(candidate);
|
|
6423
6922
|
if (files.length > 0)
|
|
@@ -6467,7 +6966,7 @@ function getPackageVersionForModuleDir(moduleDir) {
|
|
|
6467
6966
|
const root = parse(dir).root;
|
|
6468
6967
|
while (true) {
|
|
6469
6968
|
try {
|
|
6470
|
-
const pkg = JSON.parse(
|
|
6969
|
+
const pkg = JSON.parse(readFileSync2(join3(dir, "package.json"), "utf-8"));
|
|
6471
6970
|
return typeof pkg.version === "string" ? pkg.version : "unknown";
|
|
6472
6971
|
} catch {}
|
|
6473
6972
|
if (dir === root)
|
|
@@ -6571,18 +7070,18 @@ async function ensureCascadeSplitForSkill(skill, deps = {}) {
|
|
|
6571
7070
|
var init_cascade = () => {};
|
|
6572
7071
|
|
|
6573
7072
|
// ../../src/payments/wallet.ts
|
|
6574
|
-
import { existsSync as
|
|
6575
|
-
import { homedir } from "node:os";
|
|
6576
|
-
import { join as
|
|
7073
|
+
import { existsSync as existsSync5, readFileSync as readFileSync3 } from "node:fs";
|
|
7074
|
+
import { homedir as homedir2 } from "node:os";
|
|
7075
|
+
import { join as join4 } from "node:path";
|
|
6577
7076
|
function asNonEmptyString(value) {
|
|
6578
7077
|
return typeof value === "string" && value.trim() ? value.trim() : undefined;
|
|
6579
7078
|
}
|
|
6580
7079
|
function getLobsterWalletFromLocalConfig() {
|
|
6581
|
-
const agentsPath =
|
|
6582
|
-
if (!
|
|
7080
|
+
const agentsPath = join4(process.env.HOME || homedir2(), ".lobster", "agents.json");
|
|
7081
|
+
if (!existsSync5(agentsPath))
|
|
6583
7082
|
return;
|
|
6584
7083
|
try {
|
|
6585
|
-
const raw = JSON.parse(
|
|
7084
|
+
const raw = JSON.parse(readFileSync3(agentsPath, "utf8"));
|
|
6586
7085
|
const activeAgentId = asNonEmptyString(raw.activeAgentId);
|
|
6587
7086
|
const activeAgent = Array.isArray(raw.agents) ? raw.agents.find((agent) => asNonEmptyString(agent.id) === activeAgentId) : activeAgentId ? raw.agents?.[activeAgentId] : undefined;
|
|
6588
7087
|
return asNonEmptyString(activeAgent?.authorizedWallets?.solana) ?? asNonEmptyString(activeAgent?.walletAddress) ?? asNonEmptyString(activeAgent?.wallet_address);
|
|
@@ -6733,9 +7232,9 @@ __export(exports_lobster_pay, {
|
|
|
6733
7232
|
isLobsterAvailable: () => isLobsterAvailable
|
|
6734
7233
|
});
|
|
6735
7234
|
import { execFile, execFileSync as execFileSync2 } from "node:child_process";
|
|
6736
|
-
import { existsSync as
|
|
6737
|
-
import { homedir as
|
|
6738
|
-
import { join as
|
|
7235
|
+
import { existsSync as existsSync6 } from "node:fs";
|
|
7236
|
+
import { homedir as homedir3 } from "node:os";
|
|
7237
|
+
import { join as join5 } from "node:path";
|
|
6739
7238
|
function getLobsterCommand() {
|
|
6740
7239
|
try {
|
|
6741
7240
|
execFileSync2("lobstercash", ["--version"], { stdio: "ignore", timeout: 3000 });
|
|
@@ -6743,8 +7242,8 @@ function getLobsterCommand() {
|
|
|
6743
7242
|
} catch (_e) {}
|
|
6744
7243
|
try {
|
|
6745
7244
|
const npmPrefix = execFileSync2("npm", ["config", "get", "prefix"], { encoding: "utf8", timeout: 5000 }).trim();
|
|
6746
|
-
const lobsterPath =
|
|
6747
|
-
if (
|
|
7245
|
+
const lobsterPath = join5(npmPrefix, "bin", "lobstercash");
|
|
7246
|
+
if (existsSync6(lobsterPath)) {
|
|
6748
7247
|
execFileSync2(lobsterPath, ["--version"], { stdio: "ignore", timeout: 3000 });
|
|
6749
7248
|
return { cmd: lobsterPath, prefix: [] };
|
|
6750
7249
|
}
|
|
@@ -6757,8 +7256,8 @@ function lobsterCmd() {
|
|
|
6757
7256
|
return cachedCommand;
|
|
6758
7257
|
}
|
|
6759
7258
|
function isLobsterAvailable() {
|
|
6760
|
-
const agentsPath =
|
|
6761
|
-
return
|
|
7259
|
+
const agentsPath = join5(process.env.HOME || homedir3(), ".lobster", "agents.json");
|
|
7260
|
+
return existsSync6(agentsPath);
|
|
6762
7261
|
}
|
|
6763
7262
|
function lobsterX402Fetch(url, options) {
|
|
6764
7263
|
return new Promise((resolve) => {
|
|
@@ -6890,10 +7389,10 @@ __export(exports_client2, {
|
|
|
6890
7389
|
buildDefaultAgentName: () => buildDefaultAgentName,
|
|
6891
7390
|
autoFileIssue: () => autoFileIssue
|
|
6892
7391
|
});
|
|
6893
|
-
import { readFileSync as
|
|
6894
|
-
import { join as
|
|
6895
|
-
import { homedir as
|
|
6896
|
-
import { randomBytes, createHash as createHash3 } from "crypto";
|
|
7392
|
+
import { readFileSync as readFileSync4, writeFileSync as writeFileSync3, existsSync as existsSync7, mkdirSync as mkdirSync4, readdirSync as readdirSync2 } from "fs";
|
|
7393
|
+
import { join as join6 } from "path";
|
|
7394
|
+
import { homedir as homedir4, hostname, release as osRelease } from "os";
|
|
7395
|
+
import { randomBytes as randomBytes2, createHash as createHash3 } from "crypto";
|
|
6897
7396
|
import { createInterface } from "readline";
|
|
6898
7397
|
function buildReleaseAttestationHeaders(manifestBase64, signature) {
|
|
6899
7398
|
const manifest = manifestBase64.trim();
|
|
@@ -6927,18 +7426,18 @@ function scopedSkillKey(skillId, scopeId) {
|
|
|
6927
7426
|
return scopeId ? `${scopeId}:${skillId}` : skillId;
|
|
6928
7427
|
}
|
|
6929
7428
|
function getSkillCacheDir() {
|
|
6930
|
-
return process.env.UNBROWSE_SKILL_CACHE_DIR ||
|
|
7429
|
+
return process.env.UNBROWSE_SKILL_CACHE_DIR || join6(getConfigDir(), "skill-cache");
|
|
6931
7430
|
}
|
|
6932
7431
|
function getConfigDir() {
|
|
6933
7432
|
if (process.env.UNBROWSE_CONFIG_DIR)
|
|
6934
7433
|
return process.env.UNBROWSE_CONFIG_DIR;
|
|
6935
|
-
return PROFILE_NAME ?
|
|
7434
|
+
return PROFILE_NAME ? join6(homedir4(), ".unbrowse", "profiles", PROFILE_NAME) : join6(homedir4(), ".unbrowse");
|
|
6936
7435
|
}
|
|
6937
7436
|
function getConfigPath() {
|
|
6938
|
-
return
|
|
7437
|
+
return join6(getConfigDir(), "config.json");
|
|
6939
7438
|
}
|
|
6940
7439
|
function getInstallTelemetryPath() {
|
|
6941
|
-
return
|
|
7440
|
+
return join6(getConfigDir(), "install-state.json");
|
|
6942
7441
|
}
|
|
6943
7442
|
function getLandingToken() {
|
|
6944
7443
|
const token = process.env.UNBROWSE_LANDING_TOKEN?.trim();
|
|
@@ -6956,8 +7455,8 @@ function isLocalOnlyMode() {
|
|
|
6956
7455
|
function loadConfig() {
|
|
6957
7456
|
try {
|
|
6958
7457
|
const configPath = getConfigPath();
|
|
6959
|
-
if (
|
|
6960
|
-
return JSON.parse(
|
|
7458
|
+
if (existsSync7(configPath)) {
|
|
7459
|
+
return JSON.parse(readFileSync4(configPath, "utf-8"));
|
|
6961
7460
|
}
|
|
6962
7461
|
} catch {}
|
|
6963
7462
|
return null;
|
|
@@ -6965,15 +7464,15 @@ function loadConfig() {
|
|
|
6965
7464
|
function saveConfig(config) {
|
|
6966
7465
|
const configDir = getConfigDir();
|
|
6967
7466
|
const configPath = getConfigPath();
|
|
6968
|
-
if (!
|
|
6969
|
-
|
|
6970
|
-
|
|
7467
|
+
if (!existsSync7(configDir))
|
|
7468
|
+
mkdirSync4(configDir, { recursive: true });
|
|
7469
|
+
writeFileSync3(configPath, JSON.stringify(config, null, 2), { mode: 384 });
|
|
6971
7470
|
}
|
|
6972
7471
|
function loadInstallTelemetryState() {
|
|
6973
7472
|
try {
|
|
6974
7473
|
const statePath = getInstallTelemetryPath();
|
|
6975
|
-
if (
|
|
6976
|
-
return JSON.parse(
|
|
7474
|
+
if (existsSync7(statePath)) {
|
|
7475
|
+
return JSON.parse(readFileSync4(statePath, "utf-8"));
|
|
6977
7476
|
}
|
|
6978
7477
|
} catch {}
|
|
6979
7478
|
return null;
|
|
@@ -6981,13 +7480,13 @@ function loadInstallTelemetryState() {
|
|
|
6981
7480
|
function saveInstallTelemetryState(state) {
|
|
6982
7481
|
const configDir = getConfigDir();
|
|
6983
7482
|
const statePath = getInstallTelemetryPath();
|
|
6984
|
-
if (!
|
|
6985
|
-
|
|
6986
|
-
|
|
7483
|
+
if (!existsSync7(configDir))
|
|
7484
|
+
mkdirSync4(configDir, { recursive: true });
|
|
7485
|
+
writeFileSync3(statePath, JSON.stringify(state, null, 2), { mode: 384 });
|
|
6987
7486
|
}
|
|
6988
7487
|
function createInstallTelemetryState() {
|
|
6989
7488
|
return {
|
|
6990
|
-
install_id: `install_${
|
|
7489
|
+
install_id: `install_${randomBytes2(8).toString("hex")}`,
|
|
6991
7490
|
first_seen_at: new Date().toISOString()
|
|
6992
7491
|
};
|
|
6993
7492
|
}
|
|
@@ -7127,7 +7626,7 @@ function isValidAgentEmail(value) {
|
|
|
7127
7626
|
return EMAIL_RE.test(normalizeAgentEmail(value));
|
|
7128
7627
|
}
|
|
7129
7628
|
function buildDefaultAgentName() {
|
|
7130
|
-
return `${hostname()}-${
|
|
7629
|
+
return `${hostname()}-${randomBytes2(3).toString("hex")}`;
|
|
7131
7630
|
}
|
|
7132
7631
|
function resolveAgentName(preferredEmail, fallbackName) {
|
|
7133
7632
|
const normalized = normalizeAgentEmail(preferredEmail ?? "");
|
|
@@ -7482,11 +7981,11 @@ async function waitForBackgroundRegistration(timeoutMs = 0) {
|
|
|
7482
7981
|
]);
|
|
7483
7982
|
}
|
|
7484
7983
|
function skillCachePath(skillId) {
|
|
7485
|
-
return
|
|
7984
|
+
return join6(getSkillCacheDir(), `${skillId}.json`);
|
|
7486
7985
|
}
|
|
7487
7986
|
function readSkillCache(skillId) {
|
|
7488
7987
|
try {
|
|
7489
|
-
const raw =
|
|
7988
|
+
const raw = readFileSync4(skillCachePath(skillId), "utf-8");
|
|
7490
7989
|
return JSON.parse(raw);
|
|
7491
7990
|
} catch {
|
|
7492
7991
|
return null;
|
|
@@ -7496,8 +7995,8 @@ function writeSkillCache(skill, scopeId) {
|
|
|
7496
7995
|
try {
|
|
7497
7996
|
recentLocalSkills.set(scopedSkillKey(skill.skill_id, scopeId), skill);
|
|
7498
7997
|
const skillCacheDir = getSkillCacheDir();
|
|
7499
|
-
if (!
|
|
7500
|
-
|
|
7998
|
+
if (!existsSync7(skillCacheDir))
|
|
7999
|
+
mkdirSync4(skillCacheDir, { recursive: true });
|
|
7501
8000
|
const existing = readSkillCache(skill.skill_id);
|
|
7502
8001
|
if (existing) {
|
|
7503
8002
|
for (const ep of skill.endpoints) {
|
|
@@ -7513,7 +8012,7 @@ function writeSkillCache(skill, scopeId) {
|
|
|
7513
8012
|
const hasStrategy = skill.endpoints.some((e) => e.exec_strategy);
|
|
7514
8013
|
if (hasStrategy)
|
|
7515
8014
|
console.log(`[cache] writing skill ${skill.skill_id} with exec_strategy`);
|
|
7516
|
-
|
|
8015
|
+
writeFileSync3(skillCachePath(skill.skill_id), JSON.stringify(skill), "utf-8");
|
|
7517
8016
|
} catch {}
|
|
7518
8017
|
}
|
|
7519
8018
|
function cachePublishedSkill(skill, scopeId) {
|
|
@@ -7548,7 +8047,7 @@ function isIntentCompatible(lhs, rhs) {
|
|
|
7548
8047
|
function findExistingSkillForDomain(domain, intent) {
|
|
7549
8048
|
try {
|
|
7550
8049
|
const skillCacheDir = getSkillCacheDir();
|
|
7551
|
-
if (!
|
|
8050
|
+
if (!existsSync7(skillCacheDir))
|
|
7552
8051
|
return null;
|
|
7553
8052
|
const files = readdirSync2(skillCacheDir);
|
|
7554
8053
|
let compatible = null;
|
|
@@ -7557,7 +8056,7 @@ function findExistingSkillForDomain(domain, intent) {
|
|
|
7557
8056
|
if (!f.endsWith(".json") || f === "browser-capture.json")
|
|
7558
8057
|
continue;
|
|
7559
8058
|
try {
|
|
7560
|
-
const raw =
|
|
8059
|
+
const raw = readFileSync4(join6(skillCacheDir, f), "utf-8");
|
|
7561
8060
|
const skill = JSON.parse(raw);
|
|
7562
8061
|
if (skill.domain === domain && skill.execution_type === "http") {
|
|
7563
8062
|
if (!fallback)
|
|
@@ -7607,11 +8106,11 @@ async function getSkillChunk2(skillId, opts) {
|
|
|
7607
8106
|
async function listSkills() {
|
|
7608
8107
|
if (LOCAL_ONLY) {
|
|
7609
8108
|
try {
|
|
7610
|
-
if (!
|
|
8109
|
+
if (!existsSync7(SKILL_CACHE_DIR))
|
|
7611
8110
|
return [];
|
|
7612
8111
|
return readdirSync2(SKILL_CACHE_DIR).filter((file) => file.endsWith(".json")).map((file) => {
|
|
7613
8112
|
try {
|
|
7614
|
-
return JSON.parse(
|
|
8113
|
+
return JSON.parse(readFileSync4(join6(SKILL_CACHE_DIR, file), "utf-8"));
|
|
7615
8114
|
} catch {
|
|
7616
8115
|
return null;
|
|
7617
8116
|
}
|
|
@@ -8303,10 +8802,10 @@ var init_publish_admission = __esm(() => {
|
|
|
8303
8802
|
|
|
8304
8803
|
// ../../src/telemetry.ts
|
|
8305
8804
|
import { createHash as createHash4 } from "node:crypto";
|
|
8306
|
-
import { existsSync as
|
|
8307
|
-
import { join as
|
|
8805
|
+
import { existsSync as existsSync8, mkdirSync as mkdirSync5, writeFileSync as writeFileSync4 } from "node:fs";
|
|
8806
|
+
import { join as join7 } from "node:path";
|
|
8308
8807
|
function getTraceDir() {
|
|
8309
|
-
return process.env.UNBROWSE_TRACES_DIR ??
|
|
8808
|
+
return process.env.UNBROWSE_TRACES_DIR ?? join7(process.env.HOME ?? "/tmp", ".unbrowse", "traces");
|
|
8310
8809
|
}
|
|
8311
8810
|
function isTracingEnabled() {
|
|
8312
8811
|
return process.env.UNBROWSE_DISABLE_TRACES !== "1";
|
|
@@ -8380,11 +8879,11 @@ function emitRouteTrace(params) {
|
|
|
8380
8879
|
};
|
|
8381
8880
|
try {
|
|
8382
8881
|
const traceDir = getTraceDir();
|
|
8383
|
-
if (!
|
|
8384
|
-
|
|
8882
|
+
if (!existsSync8(traceDir))
|
|
8883
|
+
mkdirSync5(traceDir, { recursive: true });
|
|
8385
8884
|
const stamp = params.started_at.replace(/[:.]/g, "-");
|
|
8386
|
-
const file =
|
|
8387
|
-
|
|
8885
|
+
const file = join7(traceDir, `${stamp}-${params.outcome}-${params.trace_id}.json`);
|
|
8886
|
+
writeFileSync4(file, JSON.stringify(artifact, null, 2), "utf-8");
|
|
8388
8887
|
return file;
|
|
8389
8888
|
} catch {
|
|
8390
8889
|
return null;
|
|
@@ -8818,276 +9317,206 @@ async function resolveAuthTokens(endpoint, cookies, existingAuthHeaders) {
|
|
|
8818
9317
|
const triggerUrl = endpoint.trigger_url;
|
|
8819
9318
|
if (!triggerUrl)
|
|
8820
9319
|
return {};
|
|
8821
|
-
const headerBindings = bindings.filter((b) => b.param_location === "header");
|
|
9320
|
+
const headerBindings = bindings.filter((b) => b.param_location === "header" && !existingAuthHeaders[b.param_name]);
|
|
8822
9321
|
if (headerBindings.length === 0)
|
|
8823
9322
|
return {};
|
|
8824
9323
|
const resolved = {};
|
|
8825
|
-
|
|
8826
|
-
const
|
|
8827
|
-
if (
|
|
8828
|
-
|
|
8829
|
-
|
|
8830
|
-
|
|
8831
|
-
const
|
|
8832
|
-
if (typeof html !== "string" || !html.startsWith("<")) {
|
|
8833
|
-
return {};
|
|
8834
|
-
}
|
|
8835
|
-
for (const binding of headerBindings) {
|
|
8836
|
-
let value = await resolveBinding(binding, html, cookies);
|
|
8837
|
-
if (!value && binding.sources.some((s) => s.kind === "js-bundle")) {
|
|
8838
|
-
value = await scanAllScriptResources(tabId, binding);
|
|
8839
|
-
}
|
|
8840
|
-
if (value) {
|
|
8841
|
-
resolved[binding.param_name] = binding.param_name.toLowerCase() === "authorization" ? value.startsWith("Bearer ") ? value : `Bearer ${value}` : value;
|
|
8842
|
-
}
|
|
8843
|
-
}
|
|
8844
|
-
} finally {
|
|
8845
|
-
await closeTab(tabId).catch(() => {});
|
|
8846
|
-
}
|
|
8847
|
-
} catch {}
|
|
8848
|
-
return resolved;
|
|
8849
|
-
}
|
|
8850
|
-
async function resolveBinding(binding, html, cookies) {
|
|
8851
|
-
for (const source of binding.sources) {
|
|
8852
|
-
let value;
|
|
8853
|
-
if (source.kind === "cookie" && source.cookie_names?.length) {
|
|
8854
|
-
for (const name of source.cookie_names) {
|
|
9324
|
+
for (const binding of headerBindings) {
|
|
9325
|
+
const cookieSources = binding.sources.filter((s) => s.kind === "cookie");
|
|
9326
|
+
if (cookieSources.length === 0)
|
|
9327
|
+
continue;
|
|
9328
|
+
for (const source of cookieSources) {
|
|
9329
|
+
const names = source.cookie_names ?? [];
|
|
9330
|
+
for (const name of names) {
|
|
8855
9331
|
const cookie = cookies.find((c) => c.name === name);
|
|
8856
|
-
if (cookie?.value) {
|
|
8857
|
-
|
|
9332
|
+
if (cookie?.value && cookie.value.length >= 8) {
|
|
9333
|
+
resolved[binding.param_name] = cookie.value;
|
|
8858
9334
|
break;
|
|
8859
9335
|
}
|
|
8860
9336
|
}
|
|
8861
|
-
|
|
8862
|
-
|
|
8863
|
-
} else if (source.kind === "js-bundle" && source.bundle_url_pattern) {
|
|
8864
|
-
try {
|
|
8865
|
-
const resp = await fetch(source.bundle_url_pattern);
|
|
8866
|
-
if (resp.ok) {
|
|
8867
|
-
const body = await resp.text();
|
|
8868
|
-
value = extractTokenFromBundle(source, body);
|
|
8869
|
-
}
|
|
8870
|
-
} catch {}
|
|
9337
|
+
if (resolved[binding.param_name])
|
|
9338
|
+
break;
|
|
8871
9339
|
}
|
|
8872
|
-
if (value && value.length >= 8)
|
|
8873
|
-
return value;
|
|
8874
9340
|
}
|
|
8875
|
-
|
|
8876
|
-
|
|
8877
|
-
|
|
8878
|
-
|
|
8879
|
-
|
|
8880
|
-
|
|
8881
|
-
|
|
8882
|
-
|
|
8883
|
-
|
|
8884
|
-
|
|
8885
|
-
|
|
9341
|
+
const remaining = headerBindings.filter((b) => !resolved[b.param_name]);
|
|
9342
|
+
if (remaining.length === 0)
|
|
9343
|
+
return formatHeaders(resolved);
|
|
9344
|
+
if (!hasResolvableSources(remaining))
|
|
9345
|
+
return formatHeaders(resolved);
|
|
9346
|
+
const html = await fetchHtml(triggerUrl, cookies);
|
|
9347
|
+
if (html) {
|
|
9348
|
+
const fromHtml = await resolveFromHtml(remaining, html);
|
|
9349
|
+
Object.assign(resolved, fromHtml);
|
|
9350
|
+
const stillMissing = remaining.filter((b) => !resolved[b.param_name]);
|
|
9351
|
+
if (stillMissing.length === 0)
|
|
9352
|
+
return formatHeaders(resolved);
|
|
9353
|
+
if (Object.keys(fromHtml).length > 0 && hasHtmlResolvableSources(stillMissing)) {
|
|
9354
|
+
const browserHtml = await fetchHtmlViaBrowser(triggerUrl, cookies);
|
|
9355
|
+
if (browserHtml) {
|
|
9356
|
+
Object.assign(resolved, await resolveFromHtml(stillMissing, browserHtml));
|
|
8886
9357
|
}
|
|
8887
|
-
await navigate(tabId, url).catch(() => {});
|
|
8888
9358
|
}
|
|
8889
|
-
return
|
|
8890
|
-
} catch {
|
|
8891
|
-
return;
|
|
9359
|
+
return formatHeaders(resolved);
|
|
8892
9360
|
}
|
|
8893
|
-
|
|
8894
|
-
|
|
8895
|
-
|
|
8896
|
-
|
|
8897
|
-
|
|
8898
|
-
const state = await evaluate(tabId, "document.readyState");
|
|
8899
|
-
if (state === "complete" || state === "interactive")
|
|
8900
|
-
return;
|
|
8901
|
-
} catch {}
|
|
8902
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
9361
|
+
if (hasHtmlResolvableSources(remaining)) {
|
|
9362
|
+
const browserHtml = await fetchHtmlViaBrowser(triggerUrl, cookies);
|
|
9363
|
+
if (browserHtml) {
|
|
9364
|
+
Object.assign(resolved, await resolveFromHtml(remaining, browserHtml));
|
|
9365
|
+
}
|
|
8903
9366
|
}
|
|
9367
|
+
return formatHeaders(resolved);
|
|
8904
9368
|
}
|
|
8905
|
-
|
|
8906
|
-
|
|
8907
|
-
const
|
|
8908
|
-
|
|
8909
|
-
|
|
8910
|
-
|
|
8911
|
-
|
|
8912
|
-
)
|
|
8913
|
-
`);
|
|
8914
|
-
if (typeof raw !== "string" || !raw.startsWith("["))
|
|
8915
|
-
return;
|
|
8916
|
-
const urls = JSON.parse(raw);
|
|
8917
|
-
const tokenPattern = /AAAAAAAAAAAAAAAAAAA[A-Za-z0-9+/=_%-]{20,}/;
|
|
8918
|
-
for (const url of urls) {
|
|
8919
|
-
try {
|
|
8920
|
-
const resp = await fetch(url);
|
|
8921
|
-
if (!resp.ok)
|
|
8922
|
-
continue;
|
|
8923
|
-
const body = await resp.text();
|
|
8924
|
-
const m = body.match(tokenPattern);
|
|
8925
|
-
if (m && m[0].length >= 20)
|
|
8926
|
-
return m[0];
|
|
8927
|
-
} catch {}
|
|
9369
|
+
function hasResolvableSources(bindings) {
|
|
9370
|
+
for (const b of bindings) {
|
|
9371
|
+
for (const s of b.sources) {
|
|
9372
|
+
if (s.kind === "html-meta" || s.kind === "html-inline-script")
|
|
9373
|
+
return true;
|
|
9374
|
+
if (s.kind === "js-bundle" && s.bundle_url_pattern && s.bundle_regex)
|
|
9375
|
+
return true;
|
|
8928
9376
|
}
|
|
8929
|
-
} catch {}
|
|
8930
|
-
return;
|
|
8931
|
-
}
|
|
8932
|
-
var RESOLVE_TIMEOUT_MS = 12000;
|
|
8933
|
-
var init_token_resolver = __esm(() => {
|
|
8934
|
-
init_token_sources();
|
|
8935
|
-
init_client();
|
|
8936
|
-
});
|
|
8937
|
-
|
|
8938
|
-
// ../../src/vault/index.ts
|
|
8939
|
-
import { createCipheriv, createDecipheriv, randomBytes as randomBytes2 } from "crypto";
|
|
8940
|
-
import { existsSync as existsSync8, mkdirSync as mkdirSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync4 } from "fs";
|
|
8941
|
-
import { join as join7 } from "path";
|
|
8942
|
-
import { homedir as homedir4 } from "os";
|
|
8943
|
-
function normalizeKeytarModule(mod) {
|
|
8944
|
-
let candidate = mod;
|
|
8945
|
-
for (let depth = 0;depth < 3; depth++) {
|
|
8946
|
-
if (!candidate || typeof candidate !== "object" || !("default" in candidate))
|
|
8947
|
-
break;
|
|
8948
|
-
candidate = candidate.default;
|
|
8949
9377
|
}
|
|
8950
|
-
|
|
8951
|
-
return null;
|
|
8952
|
-
if (typeof candidate.setPassword === "function" && typeof candidate.getPassword === "function" && typeof candidate.deletePassword === "function") {
|
|
8953
|
-
return candidate;
|
|
8954
|
-
}
|
|
8955
|
-
return null;
|
|
9378
|
+
return false;
|
|
8956
9379
|
}
|
|
8957
|
-
function
|
|
8958
|
-
const
|
|
8959
|
-
|
|
9380
|
+
function hasHtmlResolvableSources(bindings) {
|
|
9381
|
+
for (const b of bindings) {
|
|
9382
|
+
for (const s of b.sources) {
|
|
9383
|
+
if (s.kind === "html-meta" || s.kind === "html-inline-script")
|
|
9384
|
+
return true;
|
|
9385
|
+
}
|
|
9386
|
+
}
|
|
9387
|
+
return false;
|
|
8960
9388
|
}
|
|
8961
|
-
function
|
|
8962
|
-
|
|
8963
|
-
|
|
8964
|
-
|
|
8965
|
-
|
|
8966
|
-
|
|
8967
|
-
|
|
9389
|
+
function formatHeaders(raw) {
|
|
9390
|
+
const out = {};
|
|
9391
|
+
for (const [k, v] of Object.entries(raw)) {
|
|
9392
|
+
if (k.toLowerCase() === "authorization" && !v.startsWith("Bearer ")) {
|
|
9393
|
+
out[k] = `Bearer ${v}`;
|
|
9394
|
+
} else {
|
|
9395
|
+
out[k] = v;
|
|
9396
|
+
}
|
|
9397
|
+
}
|
|
9398
|
+
return out;
|
|
8968
9399
|
}
|
|
8969
|
-
async function
|
|
8970
|
-
|
|
8971
|
-
|
|
8972
|
-
|
|
8973
|
-
|
|
8974
|
-
|
|
8975
|
-
if (!isKeytarBindingError(error))
|
|
8976
|
-
throw error;
|
|
8977
|
-
disableKeytar(error);
|
|
8978
|
-
return KEYTAR_UNAVAILABLE;
|
|
9400
|
+
async function resolveFromHtml(bindings, html) {
|
|
9401
|
+
const resolved = {};
|
|
9402
|
+
for (const binding of bindings) {
|
|
9403
|
+
const value = await resolveBinding(binding, html);
|
|
9404
|
+
if (value)
|
|
9405
|
+
resolved[binding.param_name] = value;
|
|
8979
9406
|
}
|
|
9407
|
+
return resolved;
|
|
8980
9408
|
}
|
|
8981
|
-
function
|
|
8982
|
-
|
|
8983
|
-
|
|
8984
|
-
|
|
8985
|
-
|
|
8986
|
-
|
|
8987
|
-
|
|
8988
|
-
|
|
9409
|
+
async function resolveBinding(binding, html) {
|
|
9410
|
+
for (const source of binding.sources) {
|
|
9411
|
+
let value;
|
|
9412
|
+
if (source.kind === "html-meta" || source.kind === "html-inline-script") {
|
|
9413
|
+
value = extractTokenFromHtml(source, html);
|
|
9414
|
+
} else if (source.kind === "js-bundle" && source.bundle_url_pattern && source.bundle_regex) {
|
|
9415
|
+
value = await resolveJsBundle(source, html);
|
|
9416
|
+
}
|
|
9417
|
+
if (value && value.length >= 8)
|
|
9418
|
+
return value;
|
|
9419
|
+
}
|
|
9420
|
+
return;
|
|
8989
9421
|
}
|
|
8990
|
-
function
|
|
8991
|
-
const
|
|
8992
|
-
|
|
8993
|
-
|
|
8994
|
-
|
|
8995
|
-
|
|
8996
|
-
|
|
9422
|
+
async function resolveJsBundle(source, html) {
|
|
9423
|
+
const pattern = source.bundle_url_pattern;
|
|
9424
|
+
const scriptSrcRe = /<script[^>]+src=["']([^"']+)["']/gi;
|
|
9425
|
+
let match;
|
|
9426
|
+
while ((match = scriptSrcRe.exec(html)) !== null) {
|
|
9427
|
+
const src = match[1];
|
|
9428
|
+
if (!src.includes(pattern))
|
|
9429
|
+
continue;
|
|
9430
|
+
try {
|
|
9431
|
+
const url = src.startsWith("http") ? src : `https:${src}`;
|
|
9432
|
+
const controller = new AbortController;
|
|
9433
|
+
const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
|
9434
|
+
const res = await fetch(url, {
|
|
9435
|
+
headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" },
|
|
9436
|
+
signal: controller.signal
|
|
9437
|
+
});
|
|
9438
|
+
clearTimeout(timeout);
|
|
9439
|
+
if (!res.ok)
|
|
9440
|
+
continue;
|
|
9441
|
+
const bundleContent = await res.text();
|
|
9442
|
+
const extracted = extractTokenFromBundle(source, bundleContent);
|
|
9443
|
+
if (extracted && extracted.length >= 8)
|
|
9444
|
+
return extracted;
|
|
9445
|
+
} catch {}
|
|
9446
|
+
}
|
|
9447
|
+
return;
|
|
8997
9448
|
}
|
|
8998
|
-
function
|
|
8999
|
-
if (!existsSync8(VAULT_FILE))
|
|
9000
|
-
return {};
|
|
9449
|
+
async function fetchHtml(url, cookies) {
|
|
9001
9450
|
try {
|
|
9002
|
-
const
|
|
9003
|
-
const
|
|
9004
|
-
const
|
|
9005
|
-
const
|
|
9006
|
-
|
|
9007
|
-
|
|
9008
|
-
|
|
9451
|
+
const cookieHeader = cookies.map((c) => `${c.name}=${c.value}`).join("; ");
|
|
9452
|
+
const controller = new AbortController;
|
|
9453
|
+
const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
|
9454
|
+
const res = await fetch(url, {
|
|
9455
|
+
headers: {
|
|
9456
|
+
Cookie: cookieHeader,
|
|
9457
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
9458
|
+
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
9459
|
+
"Accept-Language": "en-US,en;q=0.9"
|
|
9460
|
+
},
|
|
9461
|
+
redirect: "follow",
|
|
9462
|
+
signal: controller.signal
|
|
9463
|
+
});
|
|
9464
|
+
clearTimeout(timeout);
|
|
9465
|
+
if (!res.ok)
|
|
9466
|
+
return null;
|
|
9467
|
+
const ct = res.headers.get("content-type") ?? "";
|
|
9468
|
+
if (!ct.includes("text/html") && !ct.includes("text/plain") && !ct.includes("application/xhtml"))
|
|
9469
|
+
return null;
|
|
9470
|
+
const html = await res.text();
|
|
9471
|
+
if (!html || html.length < 200 || !html.includes("<"))
|
|
9472
|
+
return null;
|
|
9473
|
+
return html;
|
|
9009
9474
|
} catch {
|
|
9010
|
-
return
|
|
9011
|
-
}
|
|
9012
|
-
}
|
|
9013
|
-
function writeVaultFile(data) {
|
|
9014
|
-
const key = getOrCreateKey();
|
|
9015
|
-
const iv = randomBytes2(16);
|
|
9016
|
-
const cipher = createCipheriv("aes-256-cbc", key, iv);
|
|
9017
|
-
const enc = Buffer.concat([cipher.update(JSON.stringify(data), "utf8"), cipher.final()]);
|
|
9018
|
-
writeFileSync4(VAULT_FILE, Buffer.concat([iv, enc]), { mode: 384 });
|
|
9019
|
-
}
|
|
9020
|
-
async function storeCredential(account, value, opts) {
|
|
9021
|
-
const wrapped = {
|
|
9022
|
-
value,
|
|
9023
|
-
stored_at: new Date().toISOString(),
|
|
9024
|
-
expires_at: opts?.expires_at,
|
|
9025
|
-
max_age_ms: opts?.max_age_ms
|
|
9026
|
-
};
|
|
9027
|
-
const serialized = JSON.stringify(wrapped);
|
|
9028
|
-
const keytarResult = await callKeytar((client) => client.setPassword(SERVICE, account, serialized));
|
|
9029
|
-
if (keytarResult !== KEYTAR_UNAVAILABLE)
|
|
9030
|
-
return;
|
|
9031
|
-
await withVaultLock(() => {
|
|
9032
|
-
const data = readVaultFile();
|
|
9033
|
-
data[account] = serialized;
|
|
9034
|
-
writeVaultFile(data);
|
|
9035
|
-
});
|
|
9036
|
-
}
|
|
9037
|
-
function isExpired(cred) {
|
|
9038
|
-
if (cred.expires_at) {
|
|
9039
|
-
return new Date(cred.expires_at).getTime() <= Date.now();
|
|
9040
|
-
}
|
|
9041
|
-
if (cred.max_age_ms) {
|
|
9042
|
-
return new Date(cred.stored_at).getTime() + cred.max_age_ms <= Date.now();
|
|
9475
|
+
return null;
|
|
9043
9476
|
}
|
|
9044
|
-
return false;
|
|
9045
9477
|
}
|
|
9046
|
-
async function
|
|
9047
|
-
let
|
|
9048
|
-
|
|
9049
|
-
|
|
9050
|
-
|
|
9051
|
-
} else {
|
|
9052
|
-
const data = readVaultFile();
|
|
9053
|
-
raw = data[account] ?? null;
|
|
9054
|
-
}
|
|
9055
|
-
if (!raw)
|
|
9478
|
+
async function fetchHtmlViaBrowser(url, cookies) {
|
|
9479
|
+
let kuri;
|
|
9480
|
+
try {
|
|
9481
|
+
kuri = await Promise.resolve().then(() => (init_client(), exports_client));
|
|
9482
|
+
} catch {
|
|
9056
9483
|
return null;
|
|
9484
|
+
}
|
|
9485
|
+
let tabId;
|
|
9057
9486
|
try {
|
|
9058
|
-
const
|
|
9059
|
-
|
|
9060
|
-
|
|
9061
|
-
|
|
9062
|
-
|
|
9487
|
+
const tab = await kuri.newTab(url);
|
|
9488
|
+
tabId = typeof tab === "string" ? tab : tab?.tab_id;
|
|
9489
|
+
if (!tabId)
|
|
9490
|
+
return null;
|
|
9491
|
+
if (cookies.length > 0) {
|
|
9492
|
+
for (const c of cookies) {
|
|
9493
|
+
await kuri.setCookie(tabId, c.name, c.value, c.domain).catch(() => {});
|
|
9063
9494
|
}
|
|
9064
|
-
|
|
9495
|
+
await kuri.navigate(tabId, url).catch(() => {});
|
|
9065
9496
|
}
|
|
9066
|
-
|
|
9067
|
-
|
|
9068
|
-
|
|
9069
|
-
|
|
9070
|
-
|
|
9071
|
-
|
|
9072
|
-
|
|
9073
|
-
|
|
9074
|
-
|
|
9075
|
-
|
|
9076
|
-
|
|
9077
|
-
|
|
9497
|
+
const start2 = Date.now();
|
|
9498
|
+
while (Date.now() - start2 < 12000) {
|
|
9499
|
+
try {
|
|
9500
|
+
const state = await kuri.evaluate(tabId, "document.readyState");
|
|
9501
|
+
if (state === "complete" || state === "interactive")
|
|
9502
|
+
break;
|
|
9503
|
+
} catch {}
|
|
9504
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
9505
|
+
}
|
|
9506
|
+
const html = await kuri.getPageHtml(tabId).catch(() => "");
|
|
9507
|
+
if (typeof html !== "string" || !html.startsWith("<"))
|
|
9508
|
+
return null;
|
|
9509
|
+
return html;
|
|
9510
|
+
} catch {
|
|
9511
|
+
return null;
|
|
9512
|
+
} finally {
|
|
9513
|
+
if (tabId)
|
|
9514
|
+
await kuri.closeTab(tabId).catch(() => {});
|
|
9515
|
+
}
|
|
9078
9516
|
}
|
|
9079
|
-
var
|
|
9080
|
-
var
|
|
9081
|
-
|
|
9082
|
-
KEYTAR_UNAVAILABLE = Symbol("KEYTAR_UNAVAILABLE");
|
|
9083
|
-
KEYTAR_BINDING_ERROR_RE = /(keytar(?:\.node)?|native bindings?|bindings file|no native build was found|could not locate the bindings file|module did not self-register|err_dlopen_failed|dlopen\(|compiled against a different node\.js version|cannot find module .*keytar|wasm is not supported on this platform|(set|get|delete)password is not a function)/i;
|
|
9084
|
-
try {
|
|
9085
|
-
keytar = normalizeKeytarModule(await import("keytar"));
|
|
9086
|
-
} catch {}
|
|
9087
|
-
VAULT_DIR = join7(homedir4(), ".unbrowse", "vault");
|
|
9088
|
-
VAULT_FILE = join7(VAULT_DIR, "credentials.enc");
|
|
9089
|
-
KEY_FILE = join7(VAULT_DIR, ".key");
|
|
9090
|
-
vaultLock = Promise.resolve();
|
|
9517
|
+
var FETCH_TIMEOUT_MS = 8000;
|
|
9518
|
+
var init_token_resolver = __esm(() => {
|
|
9519
|
+
init_token_sources();
|
|
9091
9520
|
});
|
|
9092
9521
|
|
|
9093
9522
|
// ../../src/runtime/supervisor.ts
|
|
@@ -13761,6 +14190,10 @@ function validateWorkflowReplayParams(recipe, params) {
|
|
|
13761
14190
|
continue;
|
|
13762
14191
|
const value = valueForSpec(spec, params);
|
|
13763
14192
|
if (spec.required && (value == null || value === "")) {
|
|
14193
|
+
if (spec.default_value != null && spec.default_value !== "")
|
|
14194
|
+
continue;
|
|
14195
|
+
if (spec.example_value != null && spec.example_value !== "")
|
|
14196
|
+
continue;
|
|
13764
14197
|
errors.push({ name: spec.name, reason: "required" });
|
|
13765
14198
|
continue;
|
|
13766
14199
|
}
|
|
@@ -14055,18 +14488,27 @@ async function reloadExecutionAuthState(skill, epDomain, authHeaders, cookies) {
|
|
|
14055
14488
|
cookies.push(...resolved);
|
|
14056
14489
|
} catch {}
|
|
14057
14490
|
}
|
|
14058
|
-
|
|
14059
|
-
|
|
14060
|
-
|
|
14061
|
-
|
|
14062
|
-
|
|
14063
|
-
|
|
14064
|
-
|
|
14065
|
-
|
|
14066
|
-
|
|
14067
|
-
|
|
14068
|
-
|
|
14069
|
-
|
|
14491
|
+
{
|
|
14492
|
+
for (const sessionKey of [`${epDomain}-session`, `${getRegistrableDomain(epDomain)}-session`]) {
|
|
14493
|
+
try {
|
|
14494
|
+
const sessionData = await getCredential(sessionKey);
|
|
14495
|
+
if (sessionData) {
|
|
14496
|
+
const parsed = JSON.parse(sessionData);
|
|
14497
|
+
if (parsed.headers)
|
|
14498
|
+
Object.assign(authHeaders, parsed.headers);
|
|
14499
|
+
if (parsed.cookies && cookies.length === 0)
|
|
14500
|
+
cookies.push(...parsed.cookies);
|
|
14501
|
+
if (Object.keys(authHeaders).length > 0)
|
|
14502
|
+
break;
|
|
14503
|
+
}
|
|
14504
|
+
} catch {}
|
|
14505
|
+
}
|
|
14506
|
+
}
|
|
14507
|
+
if (cookies.length > 0 && authHeaders["csrf-token"]) {
|
|
14508
|
+
const jsessionId = cookies.find((c) => c.name === "JSESSIONID");
|
|
14509
|
+
if (jsessionId) {
|
|
14510
|
+
authHeaders["csrf-token"] = jsessionId.value.replace(/"/g, "");
|
|
14511
|
+
}
|
|
14070
14512
|
}
|
|
14071
14513
|
}
|
|
14072
14514
|
function persistWorkflowArtifactForCapture(artifactSkill, captured, capturedAuthHeaders) {
|
|
@@ -15087,10 +15529,13 @@ async function executeBrowserCapture(skill, params, options) {
|
|
|
15087
15529
|
}));
|
|
15088
15530
|
}
|
|
15089
15531
|
if (!auth_profile_ref) {
|
|
15090
|
-
const vaultKey
|
|
15091
|
-
|
|
15092
|
-
|
|
15093
|
-
|
|
15532
|
+
for (const vaultKey of [`auth:${targetDomain}`, `${domain}-session`, `${targetDomain}-session`]) {
|
|
15533
|
+
const hasStoredAuth = await getCredential(vaultKey) != null;
|
|
15534
|
+
if (hasStoredAuth) {
|
|
15535
|
+
auth_profile_ref = vaultKey;
|
|
15536
|
+
break;
|
|
15537
|
+
}
|
|
15538
|
+
}
|
|
15094
15539
|
}
|
|
15095
15540
|
const authBackedCapture = usedStoredAuth || !!auth_profile_ref;
|
|
15096
15541
|
if (authBackedCapture) {
|
|
@@ -15671,7 +16116,7 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
|
|
|
15671
16116
|
u.searchParams.set(k, String(v));
|
|
15672
16117
|
}
|
|
15673
16118
|
}
|
|
15674
|
-
urlTemplate = restoreTemplatePlaceholderEncoding(u.toString());
|
|
16119
|
+
urlTemplate = restoreTemplatePlaceholderEncoding(u.toString()).replace(/%28/gi, "(").replace(/%29/gi, ")").replace(/%2C/gi, ",").replace(/%3A/gi, ":");
|
|
15675
16120
|
} catch {}
|
|
15676
16121
|
}
|
|
15677
16122
|
let url = interpolate(urlTemplate, mergedParams);
|
|
@@ -15807,7 +16252,7 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
|
|
|
15807
16252
|
let last = { data: null, status: 0 };
|
|
15808
16253
|
for (const replayUrl of replayUrls) {
|
|
15809
16254
|
const replayHeaders = buildStructuredReplayHeaders(url, replayUrl, headers);
|
|
15810
|
-
log("exec", `server-fetch: ${endpoint.method} ${replayUrl.substring(0,
|
|
16255
|
+
log("exec", `server-fetch: ${endpoint.method} ${replayUrl.substring(0, 200)} csrf-token=${(replayHeaders["csrf-token"] || "none").substring(0, 20)}... hdrs=${Object.keys(replayHeaders).length} cookies=${replayHeaders["cookie"]?.length ?? 0}chars`);
|
|
15811
16256
|
const res = await fetch(replayUrl, {
|
|
15812
16257
|
method: endpoint.method,
|
|
15813
16258
|
headers: replayHeaders,
|
|
@@ -16211,7 +16656,12 @@ function interpolate(template, params) {
|
|
|
16211
16656
|
const base = template.substring(0, qIdx);
|
|
16212
16657
|
const query = template.substring(qIdx + 1);
|
|
16213
16658
|
const interpolatedBase = base.replace(/\{(\w+)\}/g, (_, k) => params[k] != null ? String(params[k]) : `{${k}}`);
|
|
16214
|
-
const interpolatedQuery = query.replace(/\{(\w+)\}/g, (_, k) =>
|
|
16659
|
+
const interpolatedQuery = query.replace(/\{(\w+)\}/g, (_, k) => {
|
|
16660
|
+
if (params[k] == null)
|
|
16661
|
+
return `{${k}}`;
|
|
16662
|
+
const val = String(params[k]);
|
|
16663
|
+
return val.replace(/[#&=\s]/g, (ch) => encodeURIComponent(ch));
|
|
16664
|
+
});
|
|
16215
16665
|
return `${interpolatedBase}?${interpolatedQuery}`;
|
|
16216
16666
|
}
|
|
16217
16667
|
function interpolateObj(obj, params) {
|
|
@@ -16526,6 +16976,17 @@ function rankEndpoints(endpoints, intent, skillDomain, contextUrl) {
|
|
|
16526
16976
|
}
|
|
16527
16977
|
score += matches * 100;
|
|
16528
16978
|
}
|
|
16979
|
+
if (rawTokens.length > 0 && pathname) {
|
|
16980
|
+
const pathLower = pathname.toLowerCase();
|
|
16981
|
+
const pathSegs = pathLower.split("/").filter(Boolean);
|
|
16982
|
+
for (const token of rawTokens) {
|
|
16983
|
+
const stemmed = stem(token);
|
|
16984
|
+
if (pathSegs.some((seg) => seg === stemmed || seg === token || seg.includes(token))) {
|
|
16985
|
+
score += 150;
|
|
16986
|
+
break;
|
|
16987
|
+
}
|
|
16988
|
+
}
|
|
16989
|
+
}
|
|
16529
16990
|
if (ep.dom_extraction)
|
|
16530
16991
|
score += 25;
|
|
16531
16992
|
if (descriptionMeta.needs_review && ep.dom_extraction)
|
|
@@ -16760,8 +17221,6 @@ function isSpaShell(html) {
|
|
|
16760
17221
|
}
|
|
16761
17222
|
var DEFAULT_BROWSER_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", VALID_VERIFICATION_STATUSES, BM25_K1 = 1.2, BM25_B = 0.75, STOPWORDS, SYNONYMS;
|
|
16762
17223
|
var init_execution = __esm(async () => {
|
|
16763
|
-
init_capture();
|
|
16764
|
-
init_capture();
|
|
16765
17224
|
init_reverse_engineer();
|
|
16766
17225
|
init_bundle_scanner();
|
|
16767
17226
|
init_token_resolver();
|
|
@@ -16785,6 +17244,8 @@ var init_execution = __esm(async () => {
|
|
|
16785
17244
|
init_compile();
|
|
16786
17245
|
init_publish();
|
|
16787
17246
|
await __promiseAll([
|
|
17247
|
+
init_capture(),
|
|
17248
|
+
init_capture(),
|
|
16788
17249
|
init_vault(),
|
|
16789
17250
|
init_auth(),
|
|
16790
17251
|
init_indexer(),
|
|
@@ -18385,6 +18846,13 @@ function isCachedSkillRelevantForIntent(skill, intent, contextUrl) {
|
|
|
18385
18846
|
const hasStructuredSearchEndpoint = candidateSkill.endpoints.some((endpoint) => endpointHasSearchBindings(endpoint) && (!!endpoint.dom_extraction || !!endpoint.response_schema) && endpointMatchesContextOrigin(endpoint, contextUrl) && endpointMatchesExplicitSearchContext(endpoint, contextUrl));
|
|
18386
18847
|
if (hasStructuredSearchEndpoint)
|
|
18387
18848
|
return true;
|
|
18849
|
+
if (top && top.score >= 0) {
|
|
18850
|
+
try {
|
|
18851
|
+
const topPath = new URL(top.endpoint.url_template).pathname.toLowerCase();
|
|
18852
|
+
if (/\/(search|find|query|browse|explore)\b/.test(topPath))
|
|
18853
|
+
return true;
|
|
18854
|
+
} catch {}
|
|
18855
|
+
}
|
|
18388
18856
|
if (collectExplicitSearchContextBindingKeys(contextUrl).size > 0)
|
|
18389
18857
|
return false;
|
|
18390
18858
|
}
|
|
@@ -19427,7 +19895,7 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
19427
19895
|
for (const cookie of cookies)
|
|
19428
19896
|
await setCookie(handoffTabId, cookie).catch(() => {});
|
|
19429
19897
|
} catch {}
|
|
19430
|
-
await evaluate(handoffTabId, (await
|
|
19898
|
+
await evaluate(handoffTabId, (await init_capture().then(() => exports_capture)).INTERCEPTOR_SCRIPT).catch(() => {});
|
|
19431
19899
|
await harStart(handoffTabId).catch(() => {});
|
|
19432
19900
|
try {
|
|
19433
19901
|
const routesModule = await init_routes().then(() => exports_routes);
|
|
@@ -20592,7 +21060,7 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
20592
21060
|
}
|
|
20593
21061
|
} catch {}
|
|
20594
21062
|
}
|
|
20595
|
-
if (
|
|
21063
|
+
if (process.env.UNBROWSE_LOCAL_ONLY === "1" && !forceCapture) {
|
|
20596
21064
|
return buildNoCachedMatch();
|
|
20597
21065
|
}
|
|
20598
21066
|
if (!context?.url) {
|
|
@@ -23928,13 +24396,13 @@ function schedulePeriodicVerification() {
|
|
|
23928
24396
|
}, VERIFICATION_INTERVAL_MS);
|
|
23929
24397
|
}
|
|
23930
24398
|
var VERIFICATION_INTERVAL_MS, VERIFY_ENDPOINT_BATCH_SIZE;
|
|
23931
|
-
var init_verification = __esm(() => {
|
|
23932
|
-
init_capture();
|
|
24399
|
+
var init_verification = __esm(async () => {
|
|
23933
24400
|
init_marketplace();
|
|
23934
24401
|
init_marketplace();
|
|
23935
24402
|
init_drift();
|
|
23936
24403
|
init_matrix();
|
|
23937
24404
|
init_candidates();
|
|
24405
|
+
await init_capture();
|
|
23938
24406
|
VERIFICATION_INTERVAL_MS = 6 * 60 * 60 * 1000;
|
|
23939
24407
|
VERIFY_ENDPOINT_BATCH_SIZE = Math.max(1, Number(process.env.UNBROWSE_VERIFY_ENDPOINT_BATCH_SIZE ?? 3));
|
|
23940
24408
|
});
|
|
@@ -24041,9 +24509,11 @@ function schedulePeriodicStaleCleanup() {
|
|
|
24041
24509
|
var VERIFY_TIMEOUT_MS, CLEANUP_INTERVAL_MS, CLEANUP_BATCH_SIZE, CLEANUP_VERIFY_ENDPOINT_LIMIT;
|
|
24042
24510
|
var init_stale_cleanup_runner = __esm(async () => {
|
|
24043
24511
|
init_marketplace();
|
|
24044
|
-
init_verification();
|
|
24045
24512
|
init_candidates();
|
|
24046
|
-
await
|
|
24513
|
+
await __promiseAll([
|
|
24514
|
+
init_verification(),
|
|
24515
|
+
init_orchestrator()
|
|
24516
|
+
]);
|
|
24047
24517
|
VERIFY_TIMEOUT_MS = Math.max(5000, Number(process.env.UNBROWSE_STALE_VERIFY_TIMEOUT_MS ?? 15000));
|
|
24048
24518
|
CLEANUP_INTERVAL_MS = Math.max(30 * 60 * 1000, Number(process.env.UNBROWSE_STALE_CLEANUP_INTERVAL_MS ?? 6 * 60 * 60 * 1000));
|
|
24049
24519
|
CLEANUP_BATCH_SIZE = Math.max(1, Number(process.env.UNBROWSE_STALE_CLEANUP_BATCH_SIZE ?? 25));
|
|
@@ -25085,7 +25555,7 @@ async function registerRoutes(app) {
|
|
|
25085
25555
|
if (!skill)
|
|
25086
25556
|
return reply.code(404).send({ error: "Skill not found" });
|
|
25087
25557
|
try {
|
|
25088
|
-
const { verifySkill: verifySkill2 } = await
|
|
25558
|
+
const { verifySkill: verifySkill2 } = await init_verification().then(() => exports_verification);
|
|
25089
25559
|
const results = await verifySkill2(skill);
|
|
25090
25560
|
return reply.send({ skill_id, verification: results });
|
|
25091
25561
|
} catch (err) {
|
|
@@ -25678,7 +26148,6 @@ var BETA_API_URL, TRACES_DIR, BROWSE_BROKER_MAX, BROWSE_BROKER_BASE_PORT, browse
|
|
|
25678
26148
|
var init_routes = __esm(async () => {
|
|
25679
26149
|
init_client();
|
|
25680
26150
|
init_reverse_engineer();
|
|
25681
|
-
init_capture();
|
|
25682
26151
|
init_marketplace();
|
|
25683
26152
|
init_graph();
|
|
25684
26153
|
init_client2();
|
|
@@ -25697,6 +26166,7 @@ var init_routes = __esm(async () => {
|
|
|
25697
26166
|
init_schema_review();
|
|
25698
26167
|
init_artifact();
|
|
25699
26168
|
await __promiseAll([
|
|
26169
|
+
init_capture(),
|
|
25700
26170
|
init_indexer(),
|
|
25701
26171
|
init_vault(),
|
|
25702
26172
|
init_orchestrator(),
|
|
@@ -25719,12 +26189,12 @@ import { config as loadEnv } from "dotenv";
|
|
|
25719
26189
|
|
|
25720
26190
|
// ../../src/server.ts
|
|
25721
26191
|
init_ratelimit();
|
|
25722
|
-
init_verification();
|
|
25723
26192
|
init_client2();
|
|
25724
|
-
init_capture();
|
|
25725
26193
|
init_version();
|
|
25726
26194
|
await __promiseAll([
|
|
25727
26195
|
init_routes(),
|
|
26196
|
+
init_verification(),
|
|
26197
|
+
init_capture(),
|
|
25728
26198
|
init_stale_cleanup_runner()
|
|
25729
26199
|
]);
|
|
25730
26200
|
import { execSync as execSync2 } from "node:child_process";
|