unbrowse 10.1.3 → 10.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/cli.js +318 -8
- package/runtime/mcp.js +402 -11
package/package.json
CHANGED
package/runtime/cli.js
CHANGED
|
@@ -64314,6 +64314,83 @@ function resolutionHostMatches(url, data) {
|
|
|
64314
64314
|
return true;
|
|
64315
64315
|
return hosts.some((h) => h === reqHost);
|
|
64316
64316
|
}
|
|
64317
|
+
function isConcreteResourceLeaf(leaf) {
|
|
64318
|
+
return !!leaf && !/^(search|explore|trending|tabs|home|for-you|foryou|latest|live|people|posts|videos)$/.test(leaf);
|
|
64319
|
+
}
|
|
64320
|
+
function urlHasConcreteResourcePath(url) {
|
|
64321
|
+
if (!url)
|
|
64322
|
+
return false;
|
|
64323
|
+
try {
|
|
64324
|
+
const segments = new URL(url).pathname.split("/").filter(Boolean);
|
|
64325
|
+
if (segments.length === 0)
|
|
64326
|
+
return false;
|
|
64327
|
+
const leaf = decodeURIComponent(segments[segments.length - 1] ?? "").toLowerCase();
|
|
64328
|
+
return isConcreteResourceLeaf(leaf);
|
|
64329
|
+
} catch {
|
|
64330
|
+
return false;
|
|
64331
|
+
}
|
|
64332
|
+
}
|
|
64333
|
+
function pathCoherent(candidate, contextPath) {
|
|
64334
|
+
const toPath = (s) => {
|
|
64335
|
+
try {
|
|
64336
|
+
return new URL(s).pathname;
|
|
64337
|
+
} catch {
|
|
64338
|
+
return s;
|
|
64339
|
+
}
|
|
64340
|
+
};
|
|
64341
|
+
const decodeSeg = (s) => {
|
|
64342
|
+
try {
|
|
64343
|
+
return decodeURIComponent(s);
|
|
64344
|
+
} catch {
|
|
64345
|
+
return s;
|
|
64346
|
+
}
|
|
64347
|
+
};
|
|
64348
|
+
const candSegments = toPath(candidate).split("/").filter(Boolean).map(decodeSeg);
|
|
64349
|
+
const ctxSegments = toPath(contextPath).split("/").filter(Boolean).map(decodeSeg);
|
|
64350
|
+
if (candSegments.length !== ctxSegments.length)
|
|
64351
|
+
return false;
|
|
64352
|
+
for (let i = 0;i < candSegments.length; i++) {
|
|
64353
|
+
const cand = candSegments[i] ?? "";
|
|
64354
|
+
const ctx = ctxSegments[i] ?? "";
|
|
64355
|
+
const isHole = /^\{[^}]*\}$/.test(cand);
|
|
64356
|
+
if (!isHole && cand !== ctx)
|
|
64357
|
+
return false;
|
|
64358
|
+
}
|
|
64359
|
+
return true;
|
|
64360
|
+
}
|
|
64361
|
+
function collectRecordUrls(data) {
|
|
64362
|
+
const out = [];
|
|
64363
|
+
const push = (rec) => {
|
|
64364
|
+
if (!rec || typeof rec !== "object")
|
|
64365
|
+
return;
|
|
64366
|
+
const r = rec;
|
|
64367
|
+
for (const k of ["url", "link", "href", "source_url", "url_template"]) {
|
|
64368
|
+
const v = r[k];
|
|
64369
|
+
if (typeof v === "string" && v)
|
|
64370
|
+
out.push(v);
|
|
64371
|
+
}
|
|
64372
|
+
};
|
|
64373
|
+
if (Array.isArray(data))
|
|
64374
|
+
for (const rec of data.slice(0, 20))
|
|
64375
|
+
push(rec);
|
|
64376
|
+
else
|
|
64377
|
+
push(data);
|
|
64378
|
+
return out;
|
|
64379
|
+
}
|
|
64380
|
+
function resolutionPathMatches(url, data) {
|
|
64381
|
+
if (!urlHasConcreteResourcePath(url))
|
|
64382
|
+
return true;
|
|
64383
|
+
let contextPath = "";
|
|
64384
|
+
try {
|
|
64385
|
+
contextPath = new URL(url).pathname;
|
|
64386
|
+
} catch {
|
|
64387
|
+
return true;
|
|
64388
|
+
}
|
|
64389
|
+
const urls = collectRecordUrls(data);
|
|
64390
|
+
if (urls.length === 0)
|
|
64391
|
+
return true;
|
|
64392
|
+
return urls.some((u) => pathCoherent(u, contextPath));
|
|
64393
|
+
}
|
|
64317
64394
|
var LIST_INTENT_RE, ITEM_SCHEMA_TYPES, COLLECTION_KEYS;
|
|
64318
64395
|
var init_cardinality = __esm(() => {
|
|
64319
64396
|
LIST_INTENT_RE = /\b(search|find|lookup|browse|discover|list(?:ings?)?|feed|catalog(?:ue)?)\b/i;
|
|
@@ -64846,7 +64923,7 @@ var init_telemetry = __esm(() => {
|
|
|
64846
64923
|
});
|
|
64847
64924
|
|
|
64848
64925
|
// .tmp-runtime-src/build-info.generated.ts
|
|
64849
|
-
var BUILD_RELEASE_VERSION = "10.1.
|
|
64926
|
+
var BUILD_RELEASE_VERSION = "10.1.5", BUILD_GIT_SHA = "8eceb88fcef6", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMTAuMS41IiwiZ2l0X3NoYSI6IjhlY2ViODhmY2VmNiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAOGVjZWI4OGZjZWY2IiwiaXNzdWVkX2F0IjoiMjAyNi0wNi0yM1QxMDo1MTo0OC45MThaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "O20RJ_2xTauDd3yWJFVqNsYakrh0Cfd67uJvkSxypBQ", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai", BUILD_DEFAULT_PROFILE = "";
|
|
64850
64927
|
|
|
64851
64928
|
// .tmp-runtime-src/version.ts
|
|
64852
64929
|
import { createHash as createHash8 } from "crypto";
|
|
@@ -113921,6 +113998,50 @@ function cardinalityMatches2(intent, subject, opts) {
|
|
|
113921
113998
|
return !routeLooksLikeSingleItem2(subject.route);
|
|
113922
113999
|
}
|
|
113923
114000
|
}
|
|
114001
|
+
function isConcreteResourceLeaf2(leaf) {
|
|
114002
|
+
return !!leaf && !/^(search|explore|trending|tabs|home|for-you|foryou|latest|live|people|posts|videos)$/.test(leaf);
|
|
114003
|
+
}
|
|
114004
|
+
function urlHasConcreteResourcePath2(url) {
|
|
114005
|
+
if (!url)
|
|
114006
|
+
return false;
|
|
114007
|
+
try {
|
|
114008
|
+
const segments = new URL(url).pathname.split("/").filter(Boolean);
|
|
114009
|
+
if (segments.length === 0)
|
|
114010
|
+
return false;
|
|
114011
|
+
const leaf = decodeURIComponent(segments[segments.length - 1] ?? "").toLowerCase();
|
|
114012
|
+
return isConcreteResourceLeaf2(leaf);
|
|
114013
|
+
} catch {
|
|
114014
|
+
return false;
|
|
114015
|
+
}
|
|
114016
|
+
}
|
|
114017
|
+
function pathCoherent2(candidate, contextPath) {
|
|
114018
|
+
const toPath = (s) => {
|
|
114019
|
+
try {
|
|
114020
|
+
return new URL(s).pathname;
|
|
114021
|
+
} catch {
|
|
114022
|
+
return s;
|
|
114023
|
+
}
|
|
114024
|
+
};
|
|
114025
|
+
const decodeSeg = (s) => {
|
|
114026
|
+
try {
|
|
114027
|
+
return decodeURIComponent(s);
|
|
114028
|
+
} catch {
|
|
114029
|
+
return s;
|
|
114030
|
+
}
|
|
114031
|
+
};
|
|
114032
|
+
const candSegments = toPath(candidate).split("/").filter(Boolean).map(decodeSeg);
|
|
114033
|
+
const ctxSegments = toPath(contextPath).split("/").filter(Boolean).map(decodeSeg);
|
|
114034
|
+
if (candSegments.length !== ctxSegments.length)
|
|
114035
|
+
return false;
|
|
114036
|
+
for (let i = 0;i < candSegments.length; i++) {
|
|
114037
|
+
const cand = candSegments[i] ?? "";
|
|
114038
|
+
const ctx = ctxSegments[i] ?? "";
|
|
114039
|
+
const isHole = /^\{[^}]*\}$/.test(cand);
|
|
114040
|
+
if (!isHole && cand !== ctx)
|
|
114041
|
+
return false;
|
|
114042
|
+
}
|
|
114043
|
+
return true;
|
|
114044
|
+
}
|
|
113924
114045
|
var LIST_INTENT_RE2, ITEM_SCHEMA_TYPES2, COLLECTION_KEYS2;
|
|
113925
114046
|
var init_cardinality2 = __esm(() => {
|
|
113926
114047
|
LIST_INTENT_RE2 = /\b(search|find|lookup|browse|discover|list(?:ings?)?|feed|catalog(?:ue)?)\b/i;
|
|
@@ -197284,7 +197405,7 @@ function isConcreteEntityDetailIntent(intent, contextUrl) {
|
|
|
197284
197405
|
return false;
|
|
197285
197406
|
try {
|
|
197286
197407
|
const leaf = decodeURIComponent(new URL(contextUrl).pathname.split("/").filter(Boolean).pop() ?? "").toLowerCase();
|
|
197287
|
-
return
|
|
197408
|
+
return isConcreteResourceLeaf2(leaf);
|
|
197288
197409
|
} catch {
|
|
197289
197410
|
return false;
|
|
197290
197411
|
}
|
|
@@ -197296,7 +197417,8 @@ function marketplaceSkillMatchesContext(skill, intent, contextUrl) {
|
|
|
197296
197417
|
if (isFeedTimelineIntent(intent, contextUrl)) {
|
|
197297
197418
|
return skill.endpoints.some((endpoint) => endpointMatchesFeedTimelineContext(endpoint, contextUrl));
|
|
197298
197419
|
}
|
|
197299
|
-
|
|
197420
|
+
const concretePath = contextUrl && urlHasConcreteResourcePath2(contextUrl);
|
|
197421
|
+
if (!contextUrl || !isConcreteEntityDetailIntent(intent, contextUrl) && !concretePath)
|
|
197300
197422
|
return true;
|
|
197301
197423
|
let contextPath = "";
|
|
197302
197424
|
try {
|
|
@@ -197306,6 +197428,17 @@ function marketplaceSkillMatchesContext(skill, intent, contextUrl) {
|
|
|
197306
197428
|
}
|
|
197307
197429
|
if (!contextPath)
|
|
197308
197430
|
return true;
|
|
197431
|
+
if (concretePath) {
|
|
197432
|
+
for (const endpoint of skill.endpoints ?? []) {
|
|
197433
|
+
let triggerPath = "";
|
|
197434
|
+
try {
|
|
197435
|
+
triggerPath = endpoint.trigger_url ? new URL(endpoint.trigger_url).pathname : "";
|
|
197436
|
+
} catch {}
|
|
197437
|
+
if (pathCoherent2(endpoint.url_template, contextPath) || triggerPath === contextPath)
|
|
197438
|
+
return true;
|
|
197439
|
+
}
|
|
197440
|
+
return false;
|
|
197441
|
+
}
|
|
197309
197442
|
let hasApiLikeEndpoint = false;
|
|
197310
197443
|
for (const endpoint of skill.endpoints ?? []) {
|
|
197311
197444
|
let path10 = "";
|
|
@@ -197324,6 +197457,55 @@ function marketplaceSkillMatchesContext(skill, intent, contextUrl) {
|
|
|
197324
197457
|
}
|
|
197325
197458
|
return hasApiLikeEndpoint;
|
|
197326
197459
|
}
|
|
197460
|
+
function skillPathCoherentForConcreteUrl(skill, contextUrl) {
|
|
197461
|
+
if (!urlHasConcreteResourcePath2(contextUrl))
|
|
197462
|
+
return true;
|
|
197463
|
+
let contextPath = "";
|
|
197464
|
+
try {
|
|
197465
|
+
contextPath = new URL(contextUrl).pathname;
|
|
197466
|
+
} catch {
|
|
197467
|
+
return true;
|
|
197468
|
+
}
|
|
197469
|
+
return (skill.endpoints ?? []).some((ep) => {
|
|
197470
|
+
if (pathCoherent2(ep.url_template, contextPath))
|
|
197471
|
+
return true;
|
|
197472
|
+
try {
|
|
197473
|
+
return ep.trigger_url ? new URL(ep.trigger_url).pathname === contextPath : false;
|
|
197474
|
+
} catch {
|
|
197475
|
+
return false;
|
|
197476
|
+
}
|
|
197477
|
+
});
|
|
197478
|
+
}
|
|
197479
|
+
async function fetchLiteralConcreteBody(url) {
|
|
197480
|
+
try {
|
|
197481
|
+
const res = await fetch(url, {
|
|
197482
|
+
headers: { Accept: "*/*", "User-Agent": "unbrowse/1.0" },
|
|
197483
|
+
signal: AbortSignal.timeout(15000),
|
|
197484
|
+
redirect: "follow"
|
|
197485
|
+
});
|
|
197486
|
+
if (!res.ok)
|
|
197487
|
+
return null;
|
|
197488
|
+
const ct = res.headers.get("content-type") ?? "";
|
|
197489
|
+
const body = await res.text();
|
|
197490
|
+
if (!(body.length > 0 && body.length < 2000000))
|
|
197491
|
+
return null;
|
|
197492
|
+
const trimmed = body.trimStart();
|
|
197493
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
197494
|
+
try {
|
|
197495
|
+
const parsed = JSON.parse(body);
|
|
197496
|
+
if (parsed !== null && typeof parsed === "object")
|
|
197497
|
+
return { result: parsed, content_type: ct };
|
|
197498
|
+
} catch {}
|
|
197499
|
+
}
|
|
197500
|
+
return {
|
|
197501
|
+
result: { url, content_type: ct, text: body, source_url: url, extraction: { source: "direct-fetch", rejected: false } },
|
|
197502
|
+
content_type: ct
|
|
197503
|
+
};
|
|
197504
|
+
} catch (e) {
|
|
197505
|
+
console.log(`[direct-fetch] ${url} literal concrete fetch failed: ${e instanceof Error ? e.message : String(e)} — falling through to skill`);
|
|
197506
|
+
return null;
|
|
197507
|
+
}
|
|
197508
|
+
}
|
|
197327
197509
|
function prioritizeIntentMatchedApis(ranked, intent, contextUrl) {
|
|
197328
197510
|
const preferred = inferPreferredEntityTokens(intent);
|
|
197329
197511
|
if (preferred.length === 0)
|
|
@@ -198886,6 +199068,27 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
198886
199068
|
timing: finalize("marketplace", w.data, w.skill.skill_id, w.skill, recipeTrace)
|
|
198887
199069
|
};
|
|
198888
199070
|
}
|
|
199071
|
+
if ((w.kind === "marketplace" || w.kind === "local-skill") && urlHasConcreteResourcePath2(raceContextUrl) && !skillPathCoherentForConcreteUrl(w.skill, raceContextUrl)) {
|
|
199072
|
+
const literal = await fetchLiteralConcreteBody(raceContextUrl);
|
|
199073
|
+
if (literal) {
|
|
199074
|
+
const trace2 = {
|
|
199075
|
+
trace_id: nanoid(),
|
|
199076
|
+
skill_id: "direct-fetch",
|
|
199077
|
+
endpoint_id: "direct-fetch",
|
|
199078
|
+
started_at: new Date(t0).toISOString(),
|
|
199079
|
+
completed_at: new Date().toISOString(),
|
|
199080
|
+
success: true
|
|
199081
|
+
};
|
|
199082
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal URL preferred over non-path-coherent skill ${w.skill.skill_id} (ct=${literal.content_type})`);
|
|
199083
|
+
return {
|
|
199084
|
+
result: literal.result,
|
|
199085
|
+
trace: trace2,
|
|
199086
|
+
source: "direct-fetch",
|
|
199087
|
+
skill: undefined,
|
|
199088
|
+
timing: finalize("direct-fetch", literal.result, "direct-fetch", undefined, trace2)
|
|
199089
|
+
};
|
|
199090
|
+
}
|
|
199091
|
+
}
|
|
198889
199092
|
if (w.kind === "marketplace") {
|
|
198890
199093
|
return buildDeferral(w.skill, "marketplace", { decision_trace: decisionTrace });
|
|
198891
199094
|
}
|
|
@@ -198914,6 +199117,46 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
198914
199117
|
}
|
|
198915
199118
|
console.log(`[direct-fetch] ${raceContextUrl} probe said JSON but direct fetch yielded no usable JSON — falling through to exa`);
|
|
198916
199119
|
}
|
|
199120
|
+
if (w.kind === "probe" && w.status >= 200 && w.status < 400 && !probeLooksLikeDirectJsonApi(w) && !probeLooksLikeFetchableHtmlDocument(w) && urlHasConcreteResourcePath2(raceContextUrl)) {
|
|
199121
|
+
try {
|
|
199122
|
+
const litRes = await fetch(raceContextUrl, {
|
|
199123
|
+
headers: { Accept: "*/*", "User-Agent": "unbrowse/1.0" },
|
|
199124
|
+
signal: AbortSignal.timeout(15000),
|
|
199125
|
+
redirect: "follow"
|
|
199126
|
+
});
|
|
199127
|
+
if (litRes.ok) {
|
|
199128
|
+
const ct = litRes.headers.get("content-type") ?? "";
|
|
199129
|
+
const body = await litRes.text();
|
|
199130
|
+
if (body.length > 0 && body.length < 2000000) {
|
|
199131
|
+
const trace2 = {
|
|
199132
|
+
trace_id: nanoid(),
|
|
199133
|
+
skill_id: "direct-fetch",
|
|
199134
|
+
endpoint_id: "direct-fetch",
|
|
199135
|
+
started_at: new Date(t0).toISOString(),
|
|
199136
|
+
completed_at: new Date().toISOString(),
|
|
199137
|
+
success: true
|
|
199138
|
+
};
|
|
199139
|
+
const result2 = {
|
|
199140
|
+
url: raceContextUrl,
|
|
199141
|
+
content_type: ct,
|
|
199142
|
+
text: body,
|
|
199143
|
+
source_url: raceContextUrl,
|
|
199144
|
+
extraction: { source: "direct-fetch", rejected: false }
|
|
199145
|
+
};
|
|
199146
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal URL — direct fetch over exa (ct=${ct}, ${body.length}B)`);
|
|
199147
|
+
return {
|
|
199148
|
+
result: result2,
|
|
199149
|
+
trace: trace2,
|
|
199150
|
+
source: "direct-fetch",
|
|
199151
|
+
skill: undefined,
|
|
199152
|
+
timing: finalize("direct-fetch", result2, "direct-fetch", undefined, trace2)
|
|
199153
|
+
};
|
|
199154
|
+
}
|
|
199155
|
+
}
|
|
199156
|
+
} catch (e) {
|
|
199157
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal fetch failed: ${e instanceof Error ? e.message : String(e)} — falling through to exa`);
|
|
199158
|
+
}
|
|
199159
|
+
}
|
|
198917
199160
|
if (w.kind === "probe" && probeLooksLikeFetchableHtmlDocument(w)) {
|
|
198918
199161
|
const directDoc = await fetchDirectDocument(raceContextUrl);
|
|
198919
199162
|
if (directDoc && !directDoc.rejected) {
|
|
@@ -199612,6 +199855,7 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
199612
199855
|
return null;
|
|
199613
199856
|
};
|
|
199614
199857
|
let directFetchCookieHeader = "";
|
|
199858
|
+
let directFetchCookiesAttached = 0;
|
|
199615
199859
|
try {
|
|
199616
199860
|
const dfHost = new URL(ctxUrl).hostname.replace(/^www\./, "");
|
|
199617
199861
|
const { extractBrowserCookies: extractBrowserCookies2 } = await Promise.resolve().then(() => (init_browser_cookies(), exports_browser_cookies));
|
|
@@ -199622,6 +199866,7 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
199622
199866
|
return `${c.name}=${v}`;
|
|
199623
199867
|
}).filter((pair) => !pair.endsWith("=")).join("; ");
|
|
199624
199868
|
if (directFetchCookieHeader) {
|
|
199869
|
+
directFetchCookiesAttached = directFetchCookieHeader.split("; ").filter(Boolean).length;
|
|
199625
199870
|
console.log(`[direct-fetch] ${ctxUrl} attaching ${dfCookies.length} browser cookie(s) for ${dfHost} on native fetch`);
|
|
199626
199871
|
}
|
|
199627
199872
|
}
|
|
@@ -199743,7 +199988,8 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
199743
199988
|
trace: trace2,
|
|
199744
199989
|
source: "direct-fetch",
|
|
199745
199990
|
skill: undefined,
|
|
199746
|
-
timing: t
|
|
199991
|
+
timing: t,
|
|
199992
|
+
...directFetchCookiesAttached > 0 ? { cookies_injected: directFetchCookiesAttached } : {}
|
|
199747
199993
|
};
|
|
199748
199994
|
}
|
|
199749
199995
|
} else if (directRes.status === 403 || directRes.status === 429 || directRes.status === 503) {
|
|
@@ -293749,6 +293995,7 @@ async function handler14(parsed, opts) {
|
|
|
293749
293995
|
const conn = wsEndpoint ? await attach(wsEndpoint) : await spawnChrome({ headless: true, perContextProxy: false, persist: true });
|
|
293750
293996
|
const target = await createTarget(conn, url, {});
|
|
293751
293997
|
let cookiesInjected = 0;
|
|
293998
|
+
let injectedCookies = [];
|
|
293752
293999
|
try {
|
|
293753
294000
|
const { shouldImportBrowserCookies: shouldImportBrowserCookies2 } = await init_auth().then(() => exports_auth);
|
|
293754
294001
|
if (shouldImportBrowserCookies2()) {
|
|
@@ -293756,6 +294003,7 @@ async function handler14(parsed, opts) {
|
|
|
293756
294003
|
const { findBestBrowserSession: findBestBrowserSession3, extractBrowserCookies: extractBrowserCookies3 } = await Promise.resolve().then(() => (init_browser_cookies(), exports_browser_cookies));
|
|
293757
294004
|
const best = findBestBrowserSession3(host);
|
|
293758
294005
|
const cookies = best?.cookies ?? extractBrowserCookies3(host).cookies;
|
|
294006
|
+
injectedCookies = cookies.map((c) => ({ name: c.name, value: c.value }));
|
|
293759
294007
|
if (cookies.length > 0) {
|
|
293760
294008
|
const toCdpSameSite = (s) => {
|
|
293761
294009
|
const v = (s ?? "").toLowerCase();
|
|
@@ -293809,6 +294057,44 @@ async function handler14(parsed, opts) {
|
|
|
293809
294057
|
process.stderr.write(`[page] act go: body read skipped: ${readErr instanceof Error ? readErr.message : String(readErr)}
|
|
293810
294058
|
`);
|
|
293811
294059
|
}
|
|
294060
|
+
const decodesToJsonRecord = (s) => {
|
|
294061
|
+
if (!s)
|
|
294062
|
+
return false;
|
|
294063
|
+
const t = s.trim();
|
|
294064
|
+
if (!(t.startsWith("{") || t.startsWith("[")))
|
|
294065
|
+
return false;
|
|
294066
|
+
try {
|
|
294067
|
+
const v = JSON.parse(t);
|
|
294068
|
+
return v !== null && typeof v === "object";
|
|
294069
|
+
} catch {
|
|
294070
|
+
return false;
|
|
294071
|
+
}
|
|
294072
|
+
};
|
|
294073
|
+
if (!decodesToJsonRecord(pageText) && injectedCookies.length > 0) {
|
|
294074
|
+
try {
|
|
294075
|
+
const cookieHeader = injectedCookies.map((c) => {
|
|
294076
|
+
const v = c.value.startsWith('"') && c.value.endsWith('"') ? c.value.slice(1, -1) : c.value;
|
|
294077
|
+
return `${c.name}=${v}`;
|
|
294078
|
+
}).join("; ");
|
|
294079
|
+
const ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
|
294080
|
+
const res = await fetch(url, {
|
|
294081
|
+
headers: { Cookie: cookieHeader, "User-Agent": ua, Accept: "application/json" },
|
|
294082
|
+
signal: AbortSignal.timeout(1e4)
|
|
294083
|
+
});
|
|
294084
|
+
const directBody = await res.text();
|
|
294085
|
+
if (decodesToJsonRecord(directBody)) {
|
|
294086
|
+
pageText = directBody.slice(0, 200000);
|
|
294087
|
+
process.stderr.write(`[auth] act go: navigated body unstructured/walled; supplemented page.text from authenticated direct fetch (status ${res.status}, ${directBody.length} bytes)
|
|
294088
|
+
`);
|
|
294089
|
+
} else {
|
|
294090
|
+
process.stderr.write(`[auth] act go: direct-fetch supplement skipped — body not a JSON record (status ${res.status})
|
|
294091
|
+
`);
|
|
294092
|
+
}
|
|
294093
|
+
} catch (directErr) {
|
|
294094
|
+
process.stderr.write(`[auth] act go: direct-fetch supplement failed: ${directErr instanceof Error ? directErr.message : String(directErr)}
|
|
294095
|
+
`);
|
|
294096
|
+
}
|
|
294097
|
+
}
|
|
293812
294098
|
let captcha;
|
|
293813
294099
|
if (parsed.flags.solve === true || process.env.UNBROWSE_AUTO_SOLVE === "1") {
|
|
293814
294100
|
const { clearCaptchaInRender: clearCaptchaInRender2 } = await Promise.resolve().then(() => (init_captcha_render(), exports_captcha_render));
|
|
@@ -304031,7 +304317,7 @@ async function cmdResolve(flags) {
|
|
|
304031
304317
|
if (resolveCacheSafe(flags)) {
|
|
304032
304318
|
const cachedHit = peekResolution(resolveCacheKeyFor(flags, intent), resolveCacheTtlMs());
|
|
304033
304319
|
const cachedData = cachedHit ? cachedHit.result ?? cachedHit.data : undefined;
|
|
304034
|
-
if (cachedHit && resolutionCardinalityMatches(intent, cachedData) && resolutionHostMatches(typeof flags.url === "string" ? flags.url : undefined, cachedData)) {
|
|
304320
|
+
if (cachedHit && resolutionCardinalityMatches(intent, cachedData) && resolutionHostMatches(typeof flags.url === "string" ? flags.url : undefined, cachedData) && resolutionPathMatches(typeof flags.url === "string" ? flags.url : undefined, cachedData)) {
|
|
304035
304321
|
const replay = markResolveCacheReplay(cachedHit);
|
|
304036
304322
|
const hostType2 = detectTelemetryHostType2();
|
|
304037
304323
|
if (process.env.UNBROWSE_LANDING_TOKEN || process.env.UNBROWSE_ATTRIBUTION_B64) {
|
|
@@ -304616,6 +304902,30 @@ async function cmdRun(args, flags, verb = "run") {
|
|
|
304616
304902
|
run_plan: runPlan
|
|
304617
304903
|
};
|
|
304618
304904
|
}
|
|
304905
|
+
const isActGoNavigateVariant = (() => {
|
|
304906
|
+
const i = (intent ?? "").trim().toLowerCase();
|
|
304907
|
+
return i === "act go" || i === "go" || i === "act read" || i === "read";
|
|
304908
|
+
})();
|
|
304909
|
+
function mirrorNavigateBodyToPage(result) {
|
|
304910
|
+
if (!isActGoNavigateVariant)
|
|
304911
|
+
return result;
|
|
304912
|
+
if ("page" in result)
|
|
304913
|
+
return result;
|
|
304914
|
+
if (result.source !== "direct-fetch")
|
|
304915
|
+
return result;
|
|
304916
|
+
const body = result.result;
|
|
304917
|
+
if (body === null || typeof body !== "object")
|
|
304918
|
+
return result;
|
|
304919
|
+
let text3;
|
|
304920
|
+
try {
|
|
304921
|
+
text3 = JSON.stringify(body);
|
|
304922
|
+
} catch {
|
|
304923
|
+
return result;
|
|
304924
|
+
}
|
|
304925
|
+
if (!text3)
|
|
304926
|
+
return result;
|
|
304927
|
+
return { ...result, page: { text: text3.slice(0, 200000) } };
|
|
304928
|
+
}
|
|
304619
304929
|
function endpointsDiscovered(result) {
|
|
304620
304930
|
if (typeof result.endpoints_discovered === "number")
|
|
304621
304931
|
return result.endpoints_discovered;
|
|
@@ -304726,11 +305036,11 @@ async function cmdRun(args, flags, verb = "run") {
|
|
|
304726
305036
|
return;
|
|
304727
305037
|
}
|
|
304728
305038
|
if (isResolveSuccessResult(result)) {
|
|
304729
|
-
output(decorate(result), !!flags.pretty);
|
|
305039
|
+
output(decorate(mirrorNavigateBodyToPage(result)), !!flags.pretty);
|
|
304730
305040
|
return;
|
|
304731
305041
|
}
|
|
304732
305042
|
if (!shouldIndexFallback(result)) {
|
|
304733
|
-
output(decorate(result), !!flags.pretty);
|
|
305043
|
+
output(decorate(mirrorNavigateBodyToPage(result)), !!flags.pretty);
|
|
304734
305044
|
return;
|
|
304735
305045
|
}
|
|
304736
305046
|
if (flags["no-index"]) {
|
|
@@ -304756,7 +305066,7 @@ async function cmdRun(args, flags, verb = "run") {
|
|
|
304756
305066
|
if (!captureLooksThin(capture)) {
|
|
304757
305067
|
result = await resolveStep("after_index");
|
|
304758
305068
|
if (isResolveSuccessResult(result)) {
|
|
304759
|
-
output(decorate(result), !!flags.pretty);
|
|
305069
|
+
output(decorate(mirrorNavigateBodyToPage(result)), !!flags.pretty);
|
|
304760
305070
|
return;
|
|
304761
305071
|
}
|
|
304762
305072
|
if (resolveResultError(result) === "auth_required") {
|
package/runtime/mcp.js
CHANGED
|
@@ -101443,7 +101443,7 @@ async function mergedAuthHeaders(key, signer) {
|
|
|
101443
101443
|
var AUTH_DOMAIN = "unbrowse-auth:v1";
|
|
101444
101444
|
|
|
101445
101445
|
// .tmp-runtime-src/build-info.generated.ts
|
|
101446
|
-
var BUILD_RELEASE_VERSION = "10.1.
|
|
101446
|
+
var BUILD_RELEASE_VERSION = "10.1.5", BUILD_GIT_SHA = "8eceb88fcef6", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMTAuMS41IiwiZ2l0X3NoYSI6IjhlY2ViODhmY2VmNiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAOGVjZWI4OGZjZWY2IiwiaXNzdWVkX2F0IjoiMjAyNi0wNi0yM1QxMDo1MTo0OC45MThaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "O20RJ_2xTauDd3yWJFVqNsYakrh0Cfd67uJvkSxypBQ", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai", BUILD_DEFAULT_PROFILE = "";
|
|
101447
101447
|
|
|
101448
101448
|
// .tmp-runtime-src/version.ts
|
|
101449
101449
|
import { createHash as createHash6 } from "crypto";
|
|
@@ -104975,6 +104975,83 @@ function resolutionHostMatches(url, data) {
|
|
|
104975
104975
|
return true;
|
|
104976
104976
|
return hosts.some((h) => h === reqHost);
|
|
104977
104977
|
}
|
|
104978
|
+
function isConcreteResourceLeaf(leaf) {
|
|
104979
|
+
return !!leaf && !/^(search|explore|trending|tabs|home|for-you|foryou|latest|live|people|posts|videos)$/.test(leaf);
|
|
104980
|
+
}
|
|
104981
|
+
function urlHasConcreteResourcePath(url) {
|
|
104982
|
+
if (!url)
|
|
104983
|
+
return false;
|
|
104984
|
+
try {
|
|
104985
|
+
const segments = new URL(url).pathname.split("/").filter(Boolean);
|
|
104986
|
+
if (segments.length === 0)
|
|
104987
|
+
return false;
|
|
104988
|
+
const leaf = decodeURIComponent(segments[segments.length - 1] ?? "").toLowerCase();
|
|
104989
|
+
return isConcreteResourceLeaf(leaf);
|
|
104990
|
+
} catch {
|
|
104991
|
+
return false;
|
|
104992
|
+
}
|
|
104993
|
+
}
|
|
104994
|
+
function pathCoherent(candidate, contextPath) {
|
|
104995
|
+
const toPath = (s) => {
|
|
104996
|
+
try {
|
|
104997
|
+
return new URL(s).pathname;
|
|
104998
|
+
} catch {
|
|
104999
|
+
return s;
|
|
105000
|
+
}
|
|
105001
|
+
};
|
|
105002
|
+
const decodeSeg = (s) => {
|
|
105003
|
+
try {
|
|
105004
|
+
return decodeURIComponent(s);
|
|
105005
|
+
} catch {
|
|
105006
|
+
return s;
|
|
105007
|
+
}
|
|
105008
|
+
};
|
|
105009
|
+
const candSegments = toPath(candidate).split("/").filter(Boolean).map(decodeSeg);
|
|
105010
|
+
const ctxSegments = toPath(contextPath).split("/").filter(Boolean).map(decodeSeg);
|
|
105011
|
+
if (candSegments.length !== ctxSegments.length)
|
|
105012
|
+
return false;
|
|
105013
|
+
for (let i = 0;i < candSegments.length; i++) {
|
|
105014
|
+
const cand = candSegments[i] ?? "";
|
|
105015
|
+
const ctx = ctxSegments[i] ?? "";
|
|
105016
|
+
const isHole = /^\{[^}]*\}$/.test(cand);
|
|
105017
|
+
if (!isHole && cand !== ctx)
|
|
105018
|
+
return false;
|
|
105019
|
+
}
|
|
105020
|
+
return true;
|
|
105021
|
+
}
|
|
105022
|
+
function collectRecordUrls(data) {
|
|
105023
|
+
const out = [];
|
|
105024
|
+
const push = (rec) => {
|
|
105025
|
+
if (!rec || typeof rec !== "object")
|
|
105026
|
+
return;
|
|
105027
|
+
const r = rec;
|
|
105028
|
+
for (const k of ["url", "link", "href", "source_url", "url_template"]) {
|
|
105029
|
+
const v = r[k];
|
|
105030
|
+
if (typeof v === "string" && v)
|
|
105031
|
+
out.push(v);
|
|
105032
|
+
}
|
|
105033
|
+
};
|
|
105034
|
+
if (Array.isArray(data))
|
|
105035
|
+
for (const rec of data.slice(0, 20))
|
|
105036
|
+
push(rec);
|
|
105037
|
+
else
|
|
105038
|
+
push(data);
|
|
105039
|
+
return out;
|
|
105040
|
+
}
|
|
105041
|
+
function resolutionPathMatches(url, data) {
|
|
105042
|
+
if (!urlHasConcreteResourcePath(url))
|
|
105043
|
+
return true;
|
|
105044
|
+
let contextPath = "";
|
|
105045
|
+
try {
|
|
105046
|
+
contextPath = new URL(url).pathname;
|
|
105047
|
+
} catch {
|
|
105048
|
+
return true;
|
|
105049
|
+
}
|
|
105050
|
+
const urls = collectRecordUrls(data);
|
|
105051
|
+
if (urls.length === 0)
|
|
105052
|
+
return true;
|
|
105053
|
+
return urls.some((u) => pathCoherent(u, contextPath));
|
|
105054
|
+
}
|
|
104978
105055
|
var LIST_INTENT_RE, ITEM_SCHEMA_TYPES, COLLECTION_KEYS;
|
|
104979
105056
|
var init_cardinality = __esm(() => {
|
|
104980
105057
|
LIST_INTENT_RE = /\b(search|find|lookup|browse|discover|list(?:ings?)?|feed|catalog(?:ue)?)\b/i;
|
|
@@ -194465,7 +194542,7 @@ function isConcreteEntityDetailIntent(intent, contextUrl) {
|
|
|
194465
194542
|
return false;
|
|
194466
194543
|
try {
|
|
194467
194544
|
const leaf = decodeURIComponent(new URL(contextUrl).pathname.split("/").filter(Boolean).pop() ?? "").toLowerCase();
|
|
194468
|
-
return
|
|
194545
|
+
return isConcreteResourceLeaf(leaf);
|
|
194469
194546
|
} catch {
|
|
194470
194547
|
return false;
|
|
194471
194548
|
}
|
|
@@ -194477,7 +194554,8 @@ function marketplaceSkillMatchesContext(skill, intent, contextUrl) {
|
|
|
194477
194554
|
if (isFeedTimelineIntent(intent, contextUrl)) {
|
|
194478
194555
|
return skill.endpoints.some((endpoint) => endpointMatchesFeedTimelineContext(endpoint, contextUrl));
|
|
194479
194556
|
}
|
|
194480
|
-
|
|
194557
|
+
const concretePath = contextUrl && urlHasConcreteResourcePath(contextUrl);
|
|
194558
|
+
if (!contextUrl || !isConcreteEntityDetailIntent(intent, contextUrl) && !concretePath)
|
|
194481
194559
|
return true;
|
|
194482
194560
|
let contextPath = "";
|
|
194483
194561
|
try {
|
|
@@ -194487,6 +194565,17 @@ function marketplaceSkillMatchesContext(skill, intent, contextUrl) {
|
|
|
194487
194565
|
}
|
|
194488
194566
|
if (!contextPath)
|
|
194489
194567
|
return true;
|
|
194568
|
+
if (concretePath) {
|
|
194569
|
+
for (const endpoint of skill.endpoints ?? []) {
|
|
194570
|
+
let triggerPath = "";
|
|
194571
|
+
try {
|
|
194572
|
+
triggerPath = endpoint.trigger_url ? new URL(endpoint.trigger_url).pathname : "";
|
|
194573
|
+
} catch {}
|
|
194574
|
+
if (pathCoherent(endpoint.url_template, contextPath) || triggerPath === contextPath)
|
|
194575
|
+
return true;
|
|
194576
|
+
}
|
|
194577
|
+
return false;
|
|
194578
|
+
}
|
|
194490
194579
|
let hasApiLikeEndpoint = false;
|
|
194491
194580
|
for (const endpoint of skill.endpoints ?? []) {
|
|
194492
194581
|
let path8 = "";
|
|
@@ -194505,6 +194594,55 @@ function marketplaceSkillMatchesContext(skill, intent, contextUrl) {
|
|
|
194505
194594
|
}
|
|
194506
194595
|
return hasApiLikeEndpoint;
|
|
194507
194596
|
}
|
|
194597
|
+
function skillPathCoherentForConcreteUrl(skill, contextUrl) {
|
|
194598
|
+
if (!urlHasConcreteResourcePath(contextUrl))
|
|
194599
|
+
return true;
|
|
194600
|
+
let contextPath = "";
|
|
194601
|
+
try {
|
|
194602
|
+
contextPath = new URL(contextUrl).pathname;
|
|
194603
|
+
} catch {
|
|
194604
|
+
return true;
|
|
194605
|
+
}
|
|
194606
|
+
return (skill.endpoints ?? []).some((ep) => {
|
|
194607
|
+
if (pathCoherent(ep.url_template, contextPath))
|
|
194608
|
+
return true;
|
|
194609
|
+
try {
|
|
194610
|
+
return ep.trigger_url ? new URL(ep.trigger_url).pathname === contextPath : false;
|
|
194611
|
+
} catch {
|
|
194612
|
+
return false;
|
|
194613
|
+
}
|
|
194614
|
+
});
|
|
194615
|
+
}
|
|
194616
|
+
async function fetchLiteralConcreteBody(url) {
|
|
194617
|
+
try {
|
|
194618
|
+
const res = await fetch(url, {
|
|
194619
|
+
headers: { Accept: "*/*", "User-Agent": "unbrowse/1.0" },
|
|
194620
|
+
signal: AbortSignal.timeout(15000),
|
|
194621
|
+
redirect: "follow"
|
|
194622
|
+
});
|
|
194623
|
+
if (!res.ok)
|
|
194624
|
+
return null;
|
|
194625
|
+
const ct = res.headers.get("content-type") ?? "";
|
|
194626
|
+
const body = await res.text();
|
|
194627
|
+
if (!(body.length > 0 && body.length < 2000000))
|
|
194628
|
+
return null;
|
|
194629
|
+
const trimmed = body.trimStart();
|
|
194630
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
194631
|
+
try {
|
|
194632
|
+
const parsed = JSON.parse(body);
|
|
194633
|
+
if (parsed !== null && typeof parsed === "object")
|
|
194634
|
+
return { result: parsed, content_type: ct };
|
|
194635
|
+
} catch {}
|
|
194636
|
+
}
|
|
194637
|
+
return {
|
|
194638
|
+
result: { url, content_type: ct, text: body, source_url: url, extraction: { source: "direct-fetch", rejected: false } },
|
|
194639
|
+
content_type: ct
|
|
194640
|
+
};
|
|
194641
|
+
} catch (e) {
|
|
194642
|
+
console.log(`[direct-fetch] ${url} literal concrete fetch failed: ${e instanceof Error ? e.message : String(e)} — falling through to skill`);
|
|
194643
|
+
return null;
|
|
194644
|
+
}
|
|
194645
|
+
}
|
|
194508
194646
|
function prioritizeIntentMatchedApis(ranked, intent, contextUrl) {
|
|
194509
194647
|
const preferred = inferPreferredEntityTokens(intent);
|
|
194510
194648
|
if (preferred.length === 0)
|
|
@@ -196067,6 +196205,27 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
196067
196205
|
timing: finalize("marketplace", w.data, w.skill.skill_id, w.skill, recipeTrace)
|
|
196068
196206
|
};
|
|
196069
196207
|
}
|
|
196208
|
+
if ((w.kind === "marketplace" || w.kind === "local-skill") && urlHasConcreteResourcePath(raceContextUrl) && !skillPathCoherentForConcreteUrl(w.skill, raceContextUrl)) {
|
|
196209
|
+
const literal = await fetchLiteralConcreteBody(raceContextUrl);
|
|
196210
|
+
if (literal) {
|
|
196211
|
+
const trace2 = {
|
|
196212
|
+
trace_id: nanoid(),
|
|
196213
|
+
skill_id: "direct-fetch",
|
|
196214
|
+
endpoint_id: "direct-fetch",
|
|
196215
|
+
started_at: new Date(t0).toISOString(),
|
|
196216
|
+
completed_at: new Date().toISOString(),
|
|
196217
|
+
success: true
|
|
196218
|
+
};
|
|
196219
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal URL preferred over non-path-coherent skill ${w.skill.skill_id} (ct=${literal.content_type})`);
|
|
196220
|
+
return {
|
|
196221
|
+
result: literal.result,
|
|
196222
|
+
trace: trace2,
|
|
196223
|
+
source: "direct-fetch",
|
|
196224
|
+
skill: undefined,
|
|
196225
|
+
timing: finalize("direct-fetch", literal.result, "direct-fetch", undefined, trace2)
|
|
196226
|
+
};
|
|
196227
|
+
}
|
|
196228
|
+
}
|
|
196070
196229
|
if (w.kind === "marketplace") {
|
|
196071
196230
|
return buildDeferral(w.skill, "marketplace", { decision_trace: decisionTrace });
|
|
196072
196231
|
}
|
|
@@ -196095,6 +196254,46 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
196095
196254
|
}
|
|
196096
196255
|
console.log(`[direct-fetch] ${raceContextUrl} probe said JSON but direct fetch yielded no usable JSON — falling through to exa`);
|
|
196097
196256
|
}
|
|
196257
|
+
if (w.kind === "probe" && w.status >= 200 && w.status < 400 && !probeLooksLikeDirectJsonApi(w) && !probeLooksLikeFetchableHtmlDocument(w) && urlHasConcreteResourcePath(raceContextUrl)) {
|
|
196258
|
+
try {
|
|
196259
|
+
const litRes = await fetch(raceContextUrl, {
|
|
196260
|
+
headers: { Accept: "*/*", "User-Agent": "unbrowse/1.0" },
|
|
196261
|
+
signal: AbortSignal.timeout(15000),
|
|
196262
|
+
redirect: "follow"
|
|
196263
|
+
});
|
|
196264
|
+
if (litRes.ok) {
|
|
196265
|
+
const ct = litRes.headers.get("content-type") ?? "";
|
|
196266
|
+
const body = await litRes.text();
|
|
196267
|
+
if (body.length > 0 && body.length < 2000000) {
|
|
196268
|
+
const trace2 = {
|
|
196269
|
+
trace_id: nanoid(),
|
|
196270
|
+
skill_id: "direct-fetch",
|
|
196271
|
+
endpoint_id: "direct-fetch",
|
|
196272
|
+
started_at: new Date(t0).toISOString(),
|
|
196273
|
+
completed_at: new Date().toISOString(),
|
|
196274
|
+
success: true
|
|
196275
|
+
};
|
|
196276
|
+
const result2 = {
|
|
196277
|
+
url: raceContextUrl,
|
|
196278
|
+
content_type: ct,
|
|
196279
|
+
text: body,
|
|
196280
|
+
source_url: raceContextUrl,
|
|
196281
|
+
extraction: { source: "direct-fetch", rejected: false }
|
|
196282
|
+
};
|
|
196283
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal URL — direct fetch over exa (ct=${ct}, ${body.length}B)`);
|
|
196284
|
+
return {
|
|
196285
|
+
result: result2,
|
|
196286
|
+
trace: trace2,
|
|
196287
|
+
source: "direct-fetch",
|
|
196288
|
+
skill: undefined,
|
|
196289
|
+
timing: finalize("direct-fetch", result2, "direct-fetch", undefined, trace2)
|
|
196290
|
+
};
|
|
196291
|
+
}
|
|
196292
|
+
}
|
|
196293
|
+
} catch (e) {
|
|
196294
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal fetch failed: ${e instanceof Error ? e.message : String(e)} — falling through to exa`);
|
|
196295
|
+
}
|
|
196296
|
+
}
|
|
196098
196297
|
if (w.kind === "probe" && probeLooksLikeFetchableHtmlDocument(w)) {
|
|
196099
196298
|
const directDoc = await fetchDirectDocument(raceContextUrl);
|
|
196100
196299
|
if (directDoc && !directDoc.rejected) {
|
|
@@ -196793,6 +196992,7 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
196793
196992
|
return null;
|
|
196794
196993
|
};
|
|
196795
196994
|
let directFetchCookieHeader = "";
|
|
196995
|
+
let directFetchCookiesAttached = 0;
|
|
196796
196996
|
try {
|
|
196797
196997
|
const dfHost = new URL(ctxUrl).hostname.replace(/^www\./, "");
|
|
196798
196998
|
const { extractBrowserCookies: extractBrowserCookies2 } = await Promise.resolve().then(() => (init_browser_cookies(), exports_browser_cookies));
|
|
@@ -196803,6 +197003,7 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
196803
197003
|
return `${c.name}=${v}`;
|
|
196804
197004
|
}).filter((pair) => !pair.endsWith("=")).join("; ");
|
|
196805
197005
|
if (directFetchCookieHeader) {
|
|
197006
|
+
directFetchCookiesAttached = directFetchCookieHeader.split("; ").filter(Boolean).length;
|
|
196806
197007
|
console.log(`[direct-fetch] ${ctxUrl} attaching ${dfCookies.length} browser cookie(s) for ${dfHost} on native fetch`);
|
|
196807
197008
|
}
|
|
196808
197009
|
}
|
|
@@ -196924,7 +197125,8 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
196924
197125
|
trace: trace2,
|
|
196925
197126
|
source: "direct-fetch",
|
|
196926
197127
|
skill: undefined,
|
|
196927
|
-
timing: t
|
|
197128
|
+
timing: t,
|
|
197129
|
+
...directFetchCookiesAttached > 0 ? { cookies_injected: directFetchCookiesAttached } : {}
|
|
196928
197130
|
};
|
|
196929
197131
|
}
|
|
196930
197132
|
} else if (directRes.status === 403 || directRes.status === 429 || directRes.status === 503) {
|
|
@@ -285821,6 +286023,7 @@ async function handler14(parsed, opts) {
|
|
|
285821
286023
|
const conn = wsEndpoint ? await attach(wsEndpoint) : await spawnChrome({ headless: true, perContextProxy: false, persist: true });
|
|
285822
286024
|
const target = await createTarget(conn, url, {});
|
|
285823
286025
|
let cookiesInjected = 0;
|
|
286026
|
+
let injectedCookies = [];
|
|
285824
286027
|
try {
|
|
285825
286028
|
const { shouldImportBrowserCookies: shouldImportBrowserCookies2 } = await init_auth().then(() => exports_auth);
|
|
285826
286029
|
if (shouldImportBrowserCookies2()) {
|
|
@@ -285828,6 +286031,7 @@ async function handler14(parsed, opts) {
|
|
|
285828
286031
|
const { findBestBrowserSession: findBestBrowserSession2, extractBrowserCookies: extractBrowserCookies2 } = await Promise.resolve().then(() => (init_browser_cookies(), exports_browser_cookies));
|
|
285829
286032
|
const best = findBestBrowserSession2(host);
|
|
285830
286033
|
const cookies = best?.cookies ?? extractBrowserCookies2(host).cookies;
|
|
286034
|
+
injectedCookies = cookies.map((c) => ({ name: c.name, value: c.value }));
|
|
285831
286035
|
if (cookies.length > 0) {
|
|
285832
286036
|
const toCdpSameSite = (s) => {
|
|
285833
286037
|
const v = (s ?? "").toLowerCase();
|
|
@@ -285881,6 +286085,44 @@ async function handler14(parsed, opts) {
|
|
|
285881
286085
|
process.stderr.write(`[page] act go: body read skipped: ${readErr instanceof Error ? readErr.message : String(readErr)}
|
|
285882
286086
|
`);
|
|
285883
286087
|
}
|
|
286088
|
+
const decodesToJsonRecord = (s) => {
|
|
286089
|
+
if (!s)
|
|
286090
|
+
return false;
|
|
286091
|
+
const t = s.trim();
|
|
286092
|
+
if (!(t.startsWith("{") || t.startsWith("[")))
|
|
286093
|
+
return false;
|
|
286094
|
+
try {
|
|
286095
|
+
const v = JSON.parse(t);
|
|
286096
|
+
return v !== null && typeof v === "object";
|
|
286097
|
+
} catch {
|
|
286098
|
+
return false;
|
|
286099
|
+
}
|
|
286100
|
+
};
|
|
286101
|
+
if (!decodesToJsonRecord(pageText) && injectedCookies.length > 0) {
|
|
286102
|
+
try {
|
|
286103
|
+
const cookieHeader = injectedCookies.map((c) => {
|
|
286104
|
+
const v = c.value.startsWith('"') && c.value.endsWith('"') ? c.value.slice(1, -1) : c.value;
|
|
286105
|
+
return `${c.name}=${v}`;
|
|
286106
|
+
}).join("; ");
|
|
286107
|
+
const ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
|
286108
|
+
const res = await fetch(url, {
|
|
286109
|
+
headers: { Cookie: cookieHeader, "User-Agent": ua, Accept: "application/json" },
|
|
286110
|
+
signal: AbortSignal.timeout(1e4)
|
|
286111
|
+
});
|
|
286112
|
+
const directBody = await res.text();
|
|
286113
|
+
if (decodesToJsonRecord(directBody)) {
|
|
286114
|
+
pageText = directBody.slice(0, 200000);
|
|
286115
|
+
process.stderr.write(`[auth] act go: navigated body unstructured/walled; supplemented page.text from authenticated direct fetch (status ${res.status}, ${directBody.length} bytes)
|
|
286116
|
+
`);
|
|
286117
|
+
} else {
|
|
286118
|
+
process.stderr.write(`[auth] act go: direct-fetch supplement skipped — body not a JSON record (status ${res.status})
|
|
286119
|
+
`);
|
|
286120
|
+
}
|
|
286121
|
+
} catch (directErr) {
|
|
286122
|
+
process.stderr.write(`[auth] act go: direct-fetch supplement failed: ${directErr instanceof Error ? directErr.message : String(directErr)}
|
|
286123
|
+
`);
|
|
286124
|
+
}
|
|
286125
|
+
}
|
|
285884
286126
|
let captcha;
|
|
285885
286127
|
if (parsed.flags.solve === true || process.env.UNBROWSE_AUTO_SOLVE === "1") {
|
|
285886
286128
|
const { clearCaptchaInRender: clearCaptchaInRender2 } = await Promise.resolve().then(() => (init_captcha_render(), exports_captcha_render));
|
|
@@ -296280,7 +296522,7 @@ async function cmdResolve(flags) {
|
|
|
296280
296522
|
if (resolveCacheSafe(flags)) {
|
|
296281
296523
|
const cachedHit = peekResolution(resolveCacheKeyFor(flags, intent), resolveCacheTtlMs());
|
|
296282
296524
|
const cachedData = cachedHit ? cachedHit.result ?? cachedHit.data : undefined;
|
|
296283
|
-
if (cachedHit && resolutionCardinalityMatches(intent, cachedData) && resolutionHostMatches(typeof flags.url === "string" ? flags.url : undefined, cachedData)) {
|
|
296525
|
+
if (cachedHit && resolutionCardinalityMatches(intent, cachedData) && resolutionHostMatches(typeof flags.url === "string" ? flags.url : undefined, cachedData) && resolutionPathMatches(typeof flags.url === "string" ? flags.url : undefined, cachedData)) {
|
|
296284
296526
|
const replay = markResolveCacheReplay(cachedHit);
|
|
296285
296527
|
const hostType2 = detectTelemetryHostType();
|
|
296286
296528
|
if (process.env.UNBROWSE_LANDING_TOKEN || process.env.UNBROWSE_ATTRIBUTION_B64) {
|
|
@@ -296851,6 +297093,30 @@ async function cmdRun(args, flags, verb = "run") {
|
|
|
296851
297093
|
run_plan: runPlan
|
|
296852
297094
|
};
|
|
296853
297095
|
}
|
|
297096
|
+
const isActGoNavigateVariant = (() => {
|
|
297097
|
+
const i = (intent ?? "").trim().toLowerCase();
|
|
297098
|
+
return i === "act go" || i === "go" || i === "act read" || i === "read";
|
|
297099
|
+
})();
|
|
297100
|
+
function mirrorNavigateBodyToPage(result) {
|
|
297101
|
+
if (!isActGoNavigateVariant)
|
|
297102
|
+
return result;
|
|
297103
|
+
if ("page" in result)
|
|
297104
|
+
return result;
|
|
297105
|
+
if (result.source !== "direct-fetch")
|
|
297106
|
+
return result;
|
|
297107
|
+
const body = result.result;
|
|
297108
|
+
if (body === null || typeof body !== "object")
|
|
297109
|
+
return result;
|
|
297110
|
+
let text3;
|
|
297111
|
+
try {
|
|
297112
|
+
text3 = JSON.stringify(body);
|
|
297113
|
+
} catch {
|
|
297114
|
+
return result;
|
|
297115
|
+
}
|
|
297116
|
+
if (!text3)
|
|
297117
|
+
return result;
|
|
297118
|
+
return { ...result, page: { text: text3.slice(0, 200000) } };
|
|
297119
|
+
}
|
|
296854
297120
|
function endpointsDiscovered(result) {
|
|
296855
297121
|
if (typeof result.endpoints_discovered === "number")
|
|
296856
297122
|
return result.endpoints_discovered;
|
|
@@ -296961,11 +297227,11 @@ async function cmdRun(args, flags, verb = "run") {
|
|
|
296961
297227
|
return;
|
|
296962
297228
|
}
|
|
296963
297229
|
if (isResolveSuccessResult(result)) {
|
|
296964
|
-
output(decorate(result), !!flags.pretty);
|
|
297230
|
+
output(decorate(mirrorNavigateBodyToPage(result)), !!flags.pretty);
|
|
296965
297231
|
return;
|
|
296966
297232
|
}
|
|
296967
297233
|
if (!shouldIndexFallback(result)) {
|
|
296968
|
-
output(decorate(result), !!flags.pretty);
|
|
297234
|
+
output(decorate(mirrorNavigateBodyToPage(result)), !!flags.pretty);
|
|
296969
297235
|
return;
|
|
296970
297236
|
}
|
|
296971
297237
|
if (flags["no-index"]) {
|
|
@@ -296991,7 +297257,7 @@ async function cmdRun(args, flags, verb = "run") {
|
|
|
296991
297257
|
if (!captureLooksThin(capture)) {
|
|
296992
297258
|
result = await resolveStep("after_index");
|
|
296993
297259
|
if (isResolveSuccessResult(result)) {
|
|
296994
|
-
output(decorate(result), !!flags.pretty);
|
|
297260
|
+
output(decorate(mirrorNavigateBodyToPage(result)), !!flags.pretty);
|
|
296995
297261
|
return;
|
|
296996
297262
|
}
|
|
296997
297263
|
if (resolveResultError(result) === "auth_required") {
|
|
@@ -301630,7 +301896,7 @@ function isConcreteEntityDetailIntent2(intent, contextUrl) {
|
|
|
301630
301896
|
return false;
|
|
301631
301897
|
try {
|
|
301632
301898
|
const leaf = decodeURIComponent(new URL(contextUrl).pathname.split("/").filter(Boolean).pop() ?? "").toLowerCase();
|
|
301633
|
-
return
|
|
301899
|
+
return isConcreteResourceLeaf(leaf);
|
|
301634
301900
|
} catch {
|
|
301635
301901
|
return false;
|
|
301636
301902
|
}
|
|
@@ -301642,7 +301908,8 @@ function marketplaceSkillMatchesContext2(skill, intent, contextUrl) {
|
|
|
301642
301908
|
if (isFeedTimelineIntent2(intent, contextUrl)) {
|
|
301643
301909
|
return skill.endpoints.some((endpoint) => endpointMatchesFeedTimelineContext2(endpoint, contextUrl));
|
|
301644
301910
|
}
|
|
301645
|
-
|
|
301911
|
+
const concretePath = contextUrl && urlHasConcreteResourcePath(contextUrl);
|
|
301912
|
+
if (!contextUrl || !isConcreteEntityDetailIntent2(intent, contextUrl) && !concretePath)
|
|
301646
301913
|
return true;
|
|
301647
301914
|
let contextPath = "";
|
|
301648
301915
|
try {
|
|
@@ -301652,6 +301919,17 @@ function marketplaceSkillMatchesContext2(skill, intent, contextUrl) {
|
|
|
301652
301919
|
}
|
|
301653
301920
|
if (!contextPath)
|
|
301654
301921
|
return true;
|
|
301922
|
+
if (concretePath) {
|
|
301923
|
+
for (const endpoint of skill.endpoints ?? []) {
|
|
301924
|
+
let triggerPath = "";
|
|
301925
|
+
try {
|
|
301926
|
+
triggerPath = endpoint.trigger_url ? new URL(endpoint.trigger_url).pathname : "";
|
|
301927
|
+
} catch {}
|
|
301928
|
+
if (pathCoherent(endpoint.url_template, contextPath) || triggerPath === contextPath)
|
|
301929
|
+
return true;
|
|
301930
|
+
}
|
|
301931
|
+
return false;
|
|
301932
|
+
}
|
|
301655
301933
|
let hasApiLikeEndpoint = false;
|
|
301656
301934
|
for (const endpoint of skill.endpoints ?? []) {
|
|
301657
301935
|
let path27 = "";
|
|
@@ -301670,6 +301948,55 @@ function marketplaceSkillMatchesContext2(skill, intent, contextUrl) {
|
|
|
301670
301948
|
}
|
|
301671
301949
|
return hasApiLikeEndpoint;
|
|
301672
301950
|
}
|
|
301951
|
+
function skillPathCoherentForConcreteUrl2(skill, contextUrl) {
|
|
301952
|
+
if (!urlHasConcreteResourcePath(contextUrl))
|
|
301953
|
+
return true;
|
|
301954
|
+
let contextPath = "";
|
|
301955
|
+
try {
|
|
301956
|
+
contextPath = new URL(contextUrl).pathname;
|
|
301957
|
+
} catch {
|
|
301958
|
+
return true;
|
|
301959
|
+
}
|
|
301960
|
+
return (skill.endpoints ?? []).some((ep) => {
|
|
301961
|
+
if (pathCoherent(ep.url_template, contextPath))
|
|
301962
|
+
return true;
|
|
301963
|
+
try {
|
|
301964
|
+
return ep.trigger_url ? new URL(ep.trigger_url).pathname === contextPath : false;
|
|
301965
|
+
} catch {
|
|
301966
|
+
return false;
|
|
301967
|
+
}
|
|
301968
|
+
});
|
|
301969
|
+
}
|
|
301970
|
+
async function fetchLiteralConcreteBody2(url) {
|
|
301971
|
+
try {
|
|
301972
|
+
const res = await fetch(url, {
|
|
301973
|
+
headers: { Accept: "*/*", "User-Agent": "unbrowse/1.0" },
|
|
301974
|
+
signal: AbortSignal.timeout(15000),
|
|
301975
|
+
redirect: "follow"
|
|
301976
|
+
});
|
|
301977
|
+
if (!res.ok)
|
|
301978
|
+
return null;
|
|
301979
|
+
const ct = res.headers.get("content-type") ?? "";
|
|
301980
|
+
const body = await res.text();
|
|
301981
|
+
if (!(body.length > 0 && body.length < 2000000))
|
|
301982
|
+
return null;
|
|
301983
|
+
const trimmed = body.trimStart();
|
|
301984
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
301985
|
+
try {
|
|
301986
|
+
const parsed = JSON.parse(body);
|
|
301987
|
+
if (parsed !== null && typeof parsed === "object")
|
|
301988
|
+
return { result: parsed, content_type: ct };
|
|
301989
|
+
} catch {}
|
|
301990
|
+
}
|
|
301991
|
+
return {
|
|
301992
|
+
result: { url, content_type: ct, text: body, source_url: url, extraction: { source: "direct-fetch", rejected: false } },
|
|
301993
|
+
content_type: ct
|
|
301994
|
+
};
|
|
301995
|
+
} catch (e) {
|
|
301996
|
+
console.log(`[direct-fetch] ${url} literal concrete fetch failed: ${e instanceof Error ? e.message : String(e)} — falling through to skill`);
|
|
301997
|
+
return null;
|
|
301998
|
+
}
|
|
301999
|
+
}
|
|
301673
302000
|
function prioritizeIntentMatchedApis2(ranked, intent, contextUrl) {
|
|
301674
302001
|
const preferred = inferPreferredEntityTokens2(intent);
|
|
301675
302002
|
if (preferred.length === 0)
|
|
@@ -303232,6 +303559,27 @@ async function resolveAndExecute2(intent, params = {}, context, projection, opti
|
|
|
303232
303559
|
timing: finalize("marketplace", w.data, w.skill.skill_id, w.skill, recipeTrace)
|
|
303233
303560
|
};
|
|
303234
303561
|
}
|
|
303562
|
+
if ((w.kind === "marketplace" || w.kind === "local-skill") && urlHasConcreteResourcePath(raceContextUrl) && !skillPathCoherentForConcreteUrl2(w.skill, raceContextUrl)) {
|
|
303563
|
+
const literal = await fetchLiteralConcreteBody2(raceContextUrl);
|
|
303564
|
+
if (literal) {
|
|
303565
|
+
const trace2 = {
|
|
303566
|
+
trace_id: nanoid(),
|
|
303567
|
+
skill_id: "direct-fetch",
|
|
303568
|
+
endpoint_id: "direct-fetch",
|
|
303569
|
+
started_at: new Date(t0).toISOString(),
|
|
303570
|
+
completed_at: new Date().toISOString(),
|
|
303571
|
+
success: true
|
|
303572
|
+
};
|
|
303573
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal URL preferred over non-path-coherent skill ${w.skill.skill_id} (ct=${literal.content_type})`);
|
|
303574
|
+
return {
|
|
303575
|
+
result: literal.result,
|
|
303576
|
+
trace: trace2,
|
|
303577
|
+
source: "direct-fetch",
|
|
303578
|
+
skill: undefined,
|
|
303579
|
+
timing: finalize("direct-fetch", literal.result, "direct-fetch", undefined, trace2)
|
|
303580
|
+
};
|
|
303581
|
+
}
|
|
303582
|
+
}
|
|
303235
303583
|
if (w.kind === "marketplace") {
|
|
303236
303584
|
return buildDeferral(w.skill, "marketplace", { decision_trace: decisionTrace });
|
|
303237
303585
|
}
|
|
@@ -303260,6 +303608,46 @@ async function resolveAndExecute2(intent, params = {}, context, projection, opti
|
|
|
303260
303608
|
}
|
|
303261
303609
|
console.log(`[direct-fetch] ${raceContextUrl} probe said JSON but direct fetch yielded no usable JSON — falling through to exa`);
|
|
303262
303610
|
}
|
|
303611
|
+
if (w.kind === "probe" && w.status >= 200 && w.status < 400 && !probeLooksLikeDirectJsonApi2(w) && !probeLooksLikeFetchableHtmlDocument2(w) && urlHasConcreteResourcePath(raceContextUrl)) {
|
|
303612
|
+
try {
|
|
303613
|
+
const litRes = await fetch(raceContextUrl, {
|
|
303614
|
+
headers: { Accept: "*/*", "User-Agent": "unbrowse/1.0" },
|
|
303615
|
+
signal: AbortSignal.timeout(15000),
|
|
303616
|
+
redirect: "follow"
|
|
303617
|
+
});
|
|
303618
|
+
if (litRes.ok) {
|
|
303619
|
+
const ct = litRes.headers.get("content-type") ?? "";
|
|
303620
|
+
const body = await litRes.text();
|
|
303621
|
+
if (body.length > 0 && body.length < 2000000) {
|
|
303622
|
+
const trace2 = {
|
|
303623
|
+
trace_id: nanoid(),
|
|
303624
|
+
skill_id: "direct-fetch",
|
|
303625
|
+
endpoint_id: "direct-fetch",
|
|
303626
|
+
started_at: new Date(t0).toISOString(),
|
|
303627
|
+
completed_at: new Date().toISOString(),
|
|
303628
|
+
success: true
|
|
303629
|
+
};
|
|
303630
|
+
const result2 = {
|
|
303631
|
+
url: raceContextUrl,
|
|
303632
|
+
content_type: ct,
|
|
303633
|
+
text: body,
|
|
303634
|
+
source_url: raceContextUrl,
|
|
303635
|
+
extraction: { source: "direct-fetch", rejected: false }
|
|
303636
|
+
};
|
|
303637
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal URL — direct fetch over exa (ct=${ct}, ${body.length}B)`);
|
|
303638
|
+
return {
|
|
303639
|
+
result: result2,
|
|
303640
|
+
trace: trace2,
|
|
303641
|
+
source: "direct-fetch",
|
|
303642
|
+
skill: undefined,
|
|
303643
|
+
timing: finalize("direct-fetch", result2, "direct-fetch", undefined, trace2)
|
|
303644
|
+
};
|
|
303645
|
+
}
|
|
303646
|
+
}
|
|
303647
|
+
} catch (e) {
|
|
303648
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal fetch failed: ${e instanceof Error ? e.message : String(e)} — falling through to exa`);
|
|
303649
|
+
}
|
|
303650
|
+
}
|
|
303263
303651
|
if (w.kind === "probe" && probeLooksLikeFetchableHtmlDocument2(w)) {
|
|
303264
303652
|
const directDoc = await fetchDirectDocument(raceContextUrl);
|
|
303265
303653
|
if (directDoc && !directDoc.rejected) {
|
|
@@ -303958,6 +304346,7 @@ async function resolveAndExecute2(intent, params = {}, context, projection, opti
|
|
|
303958
304346
|
return null;
|
|
303959
304347
|
};
|
|
303960
304348
|
let directFetchCookieHeader = "";
|
|
304349
|
+
let directFetchCookiesAttached = 0;
|
|
303961
304350
|
try {
|
|
303962
304351
|
const dfHost = new URL(ctxUrl).hostname.replace(/^www\./, "");
|
|
303963
304352
|
const { extractBrowserCookies: extractBrowserCookies3 } = await Promise.resolve().then(() => (init_browser_cookies(), exports_browser_cookies));
|
|
@@ -303968,6 +304357,7 @@ async function resolveAndExecute2(intent, params = {}, context, projection, opti
|
|
|
303968
304357
|
return `${c.name}=${v}`;
|
|
303969
304358
|
}).filter((pair) => !pair.endsWith("=")).join("; ");
|
|
303970
304359
|
if (directFetchCookieHeader) {
|
|
304360
|
+
directFetchCookiesAttached = directFetchCookieHeader.split("; ").filter(Boolean).length;
|
|
303971
304361
|
console.log(`[direct-fetch] ${ctxUrl} attaching ${dfCookies.length} browser cookie(s) for ${dfHost} on native fetch`);
|
|
303972
304362
|
}
|
|
303973
304363
|
}
|
|
@@ -304089,7 +304479,8 @@ async function resolveAndExecute2(intent, params = {}, context, projection, opti
|
|
|
304089
304479
|
trace: trace2,
|
|
304090
304480
|
source: "direct-fetch",
|
|
304091
304481
|
skill: undefined,
|
|
304092
|
-
timing: t
|
|
304482
|
+
timing: t,
|
|
304483
|
+
...directFetchCookiesAttached > 0 ? { cookies_injected: directFetchCookiesAttached } : {}
|
|
304093
304484
|
};
|
|
304094
304485
|
}
|
|
304095
304486
|
} else if (directRes.status === 403 || directRes.status === 429 || directRes.status === 503) {
|