unbrowse 10.1.2 → 10.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/cli.js +349 -6
- package/runtime/mcp.js +455 -10
package/package.json
CHANGED
package/runtime/cli.js
CHANGED
|
@@ -64268,6 +64268,129 @@ function cardinalityMatches(intent, subject, opts) {
|
|
|
64268
64268
|
function resolutionCardinalityMatches(intent, data) {
|
|
64269
64269
|
return cardinalityMatches(intent, { kind: "value", value: data });
|
|
64270
64270
|
}
|
|
64271
|
+
function registrableHostOf(input) {
|
|
64272
|
+
if (!input)
|
|
64273
|
+
return null;
|
|
64274
|
+
const host = (() => {
|
|
64275
|
+
try {
|
|
64276
|
+
return new URL(input).hostname;
|
|
64277
|
+
} catch {}
|
|
64278
|
+
try {
|
|
64279
|
+
return new URL(`https://${input}`).hostname;
|
|
64280
|
+
} catch {
|
|
64281
|
+
return null;
|
|
64282
|
+
}
|
|
64283
|
+
})();
|
|
64284
|
+
if (!host)
|
|
64285
|
+
return null;
|
|
64286
|
+
return host.replace(/^www\./, "").split(".").slice(-2).join(".") || null;
|
|
64287
|
+
}
|
|
64288
|
+
function collectRecordHosts(data) {
|
|
64289
|
+
const out = [];
|
|
64290
|
+
const push = (rec) => {
|
|
64291
|
+
if (!rec || typeof rec !== "object")
|
|
64292
|
+
return;
|
|
64293
|
+
const r = rec;
|
|
64294
|
+
for (const k of ["url", "link", "href", "source_url"]) {
|
|
64295
|
+
const v = r[k];
|
|
64296
|
+
const h = typeof v === "string" ? registrableHostOf(v) : null;
|
|
64297
|
+
if (h)
|
|
64298
|
+
out.push(h);
|
|
64299
|
+
}
|
|
64300
|
+
};
|
|
64301
|
+
if (Array.isArray(data))
|
|
64302
|
+
for (const rec of data.slice(0, 20))
|
|
64303
|
+
push(rec);
|
|
64304
|
+
else
|
|
64305
|
+
push(data);
|
|
64306
|
+
return out;
|
|
64307
|
+
}
|
|
64308
|
+
function resolutionHostMatches(url, data) {
|
|
64309
|
+
const reqHost = registrableHostOf(url);
|
|
64310
|
+
if (!reqHost)
|
|
64311
|
+
return true;
|
|
64312
|
+
const hosts = collectRecordHosts(data);
|
|
64313
|
+
if (hosts.length === 0)
|
|
64314
|
+
return true;
|
|
64315
|
+
return hosts.some((h) => h === reqHost);
|
|
64316
|
+
}
|
|
64317
|
+
function isConcreteResourceLeaf(leaf) {
|
|
64318
|
+
return !!leaf && !/^(search|explore|trending|tabs|home|for-you|foryou|latest|live|people|posts|videos)$/.test(leaf);
|
|
64319
|
+
}
|
|
64320
|
+
function urlHasConcreteResourcePath(url) {
|
|
64321
|
+
if (!url)
|
|
64322
|
+
return false;
|
|
64323
|
+
try {
|
|
64324
|
+
const segments = new URL(url).pathname.split("/").filter(Boolean);
|
|
64325
|
+
if (segments.length === 0)
|
|
64326
|
+
return false;
|
|
64327
|
+
const leaf = decodeURIComponent(segments[segments.length - 1] ?? "").toLowerCase();
|
|
64328
|
+
return isConcreteResourceLeaf(leaf);
|
|
64329
|
+
} catch {
|
|
64330
|
+
return false;
|
|
64331
|
+
}
|
|
64332
|
+
}
|
|
64333
|
+
function pathCoherent(candidate, contextPath) {
|
|
64334
|
+
const toPath = (s) => {
|
|
64335
|
+
try {
|
|
64336
|
+
return new URL(s).pathname;
|
|
64337
|
+
} catch {
|
|
64338
|
+
return s;
|
|
64339
|
+
}
|
|
64340
|
+
};
|
|
64341
|
+
const decodeSeg = (s) => {
|
|
64342
|
+
try {
|
|
64343
|
+
return decodeURIComponent(s);
|
|
64344
|
+
} catch {
|
|
64345
|
+
return s;
|
|
64346
|
+
}
|
|
64347
|
+
};
|
|
64348
|
+
const candSegments = toPath(candidate).split("/").filter(Boolean).map(decodeSeg);
|
|
64349
|
+
const ctxSegments = toPath(contextPath).split("/").filter(Boolean).map(decodeSeg);
|
|
64350
|
+
if (candSegments.length !== ctxSegments.length)
|
|
64351
|
+
return false;
|
|
64352
|
+
for (let i = 0;i < candSegments.length; i++) {
|
|
64353
|
+
const cand = candSegments[i] ?? "";
|
|
64354
|
+
const ctx = ctxSegments[i] ?? "";
|
|
64355
|
+
const isHole = /^\{[^}]*\}$/.test(cand);
|
|
64356
|
+
if (!isHole && cand !== ctx)
|
|
64357
|
+
return false;
|
|
64358
|
+
}
|
|
64359
|
+
return true;
|
|
64360
|
+
}
|
|
64361
|
+
function collectRecordUrls(data) {
|
|
64362
|
+
const out = [];
|
|
64363
|
+
const push = (rec) => {
|
|
64364
|
+
if (!rec || typeof rec !== "object")
|
|
64365
|
+
return;
|
|
64366
|
+
const r = rec;
|
|
64367
|
+
for (const k of ["url", "link", "href", "source_url", "url_template"]) {
|
|
64368
|
+
const v = r[k];
|
|
64369
|
+
if (typeof v === "string" && v)
|
|
64370
|
+
out.push(v);
|
|
64371
|
+
}
|
|
64372
|
+
};
|
|
64373
|
+
if (Array.isArray(data))
|
|
64374
|
+
for (const rec of data.slice(0, 20))
|
|
64375
|
+
push(rec);
|
|
64376
|
+
else
|
|
64377
|
+
push(data);
|
|
64378
|
+
return out;
|
|
64379
|
+
}
|
|
64380
|
+
function resolutionPathMatches(url, data) {
|
|
64381
|
+
if (!urlHasConcreteResourcePath(url))
|
|
64382
|
+
return true;
|
|
64383
|
+
let contextPath = "";
|
|
64384
|
+
try {
|
|
64385
|
+
contextPath = new URL(url).pathname;
|
|
64386
|
+
} catch {
|
|
64387
|
+
return true;
|
|
64388
|
+
}
|
|
64389
|
+
const urls = collectRecordUrls(data);
|
|
64390
|
+
if (urls.length === 0)
|
|
64391
|
+
return true;
|
|
64392
|
+
return urls.some((u) => pathCoherent(u, contextPath));
|
|
64393
|
+
}
|
|
64271
64394
|
var LIST_INTENT_RE, ITEM_SCHEMA_TYPES, COLLECTION_KEYS;
|
|
64272
64395
|
var init_cardinality = __esm(() => {
|
|
64273
64396
|
LIST_INTENT_RE = /\b(search|find|lookup|browse|discover|list(?:ings?)?|feed|catalog(?:ue)?)\b/i;
|
|
@@ -64800,7 +64923,7 @@ var init_telemetry = __esm(() => {
|
|
|
64800
64923
|
});
|
|
64801
64924
|
|
|
64802
64925
|
// .tmp-runtime-src/build-info.generated.ts
|
|
64803
|
-
var BUILD_RELEASE_VERSION = "10.1.
|
|
64926
|
+
var BUILD_RELEASE_VERSION = "10.1.4", BUILD_GIT_SHA = "089d49f75f26", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMTAuMS40IiwiZ2l0X3NoYSI6IjA4OWQ0OWY3NWYyNiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAMDg5ZDQ5Zjc1ZjI2IiwiaXNzdWVkX2F0IjoiMjAyNi0wNi0yM1QxMDowNjoyMy45MjNaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "cDbcbJgh0OyGCynwPC9q8d8xvPCtfzsY9H0lAtidjN4", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai", BUILD_DEFAULT_PROFILE = "";
|
|
64804
64927
|
|
|
64805
64928
|
// .tmp-runtime-src/version.ts
|
|
64806
64929
|
import { createHash as createHash8 } from "crypto";
|
|
@@ -113875,6 +113998,50 @@ function cardinalityMatches2(intent, subject, opts) {
|
|
|
113875
113998
|
return !routeLooksLikeSingleItem2(subject.route);
|
|
113876
113999
|
}
|
|
113877
114000
|
}
|
|
114001
|
+
function isConcreteResourceLeaf2(leaf) {
|
|
114002
|
+
return !!leaf && !/^(search|explore|trending|tabs|home|for-you|foryou|latest|live|people|posts|videos)$/.test(leaf);
|
|
114003
|
+
}
|
|
114004
|
+
function urlHasConcreteResourcePath2(url) {
|
|
114005
|
+
if (!url)
|
|
114006
|
+
return false;
|
|
114007
|
+
try {
|
|
114008
|
+
const segments = new URL(url).pathname.split("/").filter(Boolean);
|
|
114009
|
+
if (segments.length === 0)
|
|
114010
|
+
return false;
|
|
114011
|
+
const leaf = decodeURIComponent(segments[segments.length - 1] ?? "").toLowerCase();
|
|
114012
|
+
return isConcreteResourceLeaf2(leaf);
|
|
114013
|
+
} catch {
|
|
114014
|
+
return false;
|
|
114015
|
+
}
|
|
114016
|
+
}
|
|
114017
|
+
function pathCoherent2(candidate, contextPath) {
|
|
114018
|
+
const toPath = (s) => {
|
|
114019
|
+
try {
|
|
114020
|
+
return new URL(s).pathname;
|
|
114021
|
+
} catch {
|
|
114022
|
+
return s;
|
|
114023
|
+
}
|
|
114024
|
+
};
|
|
114025
|
+
const decodeSeg = (s) => {
|
|
114026
|
+
try {
|
|
114027
|
+
return decodeURIComponent(s);
|
|
114028
|
+
} catch {
|
|
114029
|
+
return s;
|
|
114030
|
+
}
|
|
114031
|
+
};
|
|
114032
|
+
const candSegments = toPath(candidate).split("/").filter(Boolean).map(decodeSeg);
|
|
114033
|
+
const ctxSegments = toPath(contextPath).split("/").filter(Boolean).map(decodeSeg);
|
|
114034
|
+
if (candSegments.length !== ctxSegments.length)
|
|
114035
|
+
return false;
|
|
114036
|
+
for (let i = 0;i < candSegments.length; i++) {
|
|
114037
|
+
const cand = candSegments[i] ?? "";
|
|
114038
|
+
const ctx = ctxSegments[i] ?? "";
|
|
114039
|
+
const isHole = /^\{[^}]*\}$/.test(cand);
|
|
114040
|
+
if (!isHole && cand !== ctx)
|
|
114041
|
+
return false;
|
|
114042
|
+
}
|
|
114043
|
+
return true;
|
|
114044
|
+
}
|
|
113878
114045
|
var LIST_INTENT_RE2, ITEM_SCHEMA_TYPES2, COLLECTION_KEYS2;
|
|
113879
114046
|
var init_cardinality2 = __esm(() => {
|
|
113880
114047
|
LIST_INTENT_RE2 = /\b(search|find|lookup|browse|discover|list(?:ings?)?|feed|catalog(?:ue)?)\b/i;
|
|
@@ -195657,7 +195824,9 @@ function shouldAutoWalk(requestedUrl, topUrl, topScore, minScore = 0.8) {
|
|
|
195657
195824
|
return false;
|
|
195658
195825
|
const reqReg = registrableHost(requestedUrl);
|
|
195659
195826
|
const topReg = registrableHost(topUrl);
|
|
195660
|
-
|
|
195827
|
+
if (reqReg)
|
|
195828
|
+
return reqReg === topReg;
|
|
195829
|
+
return (topScore ?? 0) >= minScore;
|
|
195661
195830
|
}
|
|
195662
195831
|
function pickWalkTarget(requestedUrl, ranked, minScore = 0.8) {
|
|
195663
195832
|
const eligible = ranked.filter((c) => c?.url && shouldAutoWalk(requestedUrl, c.url, c.score, minScore));
|
|
@@ -196114,6 +196283,17 @@ function endpointMatchesContextOrigin(endpoint, contextUrl) {
|
|
|
196114
196283
|
return true;
|
|
196115
196284
|
}
|
|
196116
196285
|
}
|
|
196286
|
+
function cachedSkillHostMatchesContext(skillDomain, contextUrl) {
|
|
196287
|
+
if (!contextUrl)
|
|
196288
|
+
return true;
|
|
196289
|
+
const ctxReg = registrableHost(contextUrl);
|
|
196290
|
+
if (!ctxReg)
|
|
196291
|
+
return true;
|
|
196292
|
+
const skillReg = registrableHost(skillDomain) ?? registrableHost(`https://${skillDomain ?? ""}`);
|
|
196293
|
+
if (!skillReg)
|
|
196294
|
+
return true;
|
|
196295
|
+
return skillReg === ctxReg;
|
|
196296
|
+
}
|
|
196117
196297
|
function endpointTargetsMismatchedLocalReplayHost(endpoint, contextUrl) {
|
|
196118
196298
|
if (!contextUrl)
|
|
196119
196299
|
return false;
|
|
@@ -197225,7 +197405,7 @@ function isConcreteEntityDetailIntent(intent, contextUrl) {
|
|
|
197225
197405
|
return false;
|
|
197226
197406
|
try {
|
|
197227
197407
|
const leaf = decodeURIComponent(new URL(contextUrl).pathname.split("/").filter(Boolean).pop() ?? "").toLowerCase();
|
|
197228
|
-
return
|
|
197408
|
+
return isConcreteResourceLeaf2(leaf);
|
|
197229
197409
|
} catch {
|
|
197230
197410
|
return false;
|
|
197231
197411
|
}
|
|
@@ -197237,7 +197417,8 @@ function marketplaceSkillMatchesContext(skill, intent, contextUrl) {
|
|
|
197237
197417
|
if (isFeedTimelineIntent(intent, contextUrl)) {
|
|
197238
197418
|
return skill.endpoints.some((endpoint) => endpointMatchesFeedTimelineContext(endpoint, contextUrl));
|
|
197239
197419
|
}
|
|
197240
|
-
|
|
197420
|
+
const concretePath = contextUrl && urlHasConcreteResourcePath2(contextUrl);
|
|
197421
|
+
if (!contextUrl || !isConcreteEntityDetailIntent(intent, contextUrl) && !concretePath)
|
|
197241
197422
|
return true;
|
|
197242
197423
|
let contextPath = "";
|
|
197243
197424
|
try {
|
|
@@ -197247,6 +197428,17 @@ function marketplaceSkillMatchesContext(skill, intent, contextUrl) {
|
|
|
197247
197428
|
}
|
|
197248
197429
|
if (!contextPath)
|
|
197249
197430
|
return true;
|
|
197431
|
+
if (concretePath) {
|
|
197432
|
+
for (const endpoint of skill.endpoints ?? []) {
|
|
197433
|
+
let triggerPath = "";
|
|
197434
|
+
try {
|
|
197435
|
+
triggerPath = endpoint.trigger_url ? new URL(endpoint.trigger_url).pathname : "";
|
|
197436
|
+
} catch {}
|
|
197437
|
+
if (pathCoherent2(endpoint.url_template, contextPath) || triggerPath === contextPath)
|
|
197438
|
+
return true;
|
|
197439
|
+
}
|
|
197440
|
+
return false;
|
|
197441
|
+
}
|
|
197250
197442
|
let hasApiLikeEndpoint = false;
|
|
197251
197443
|
for (const endpoint of skill.endpoints ?? []) {
|
|
197252
197444
|
let path10 = "";
|
|
@@ -197265,6 +197457,55 @@ function marketplaceSkillMatchesContext(skill, intent, contextUrl) {
|
|
|
197265
197457
|
}
|
|
197266
197458
|
return hasApiLikeEndpoint;
|
|
197267
197459
|
}
|
|
197460
|
+
function skillPathCoherentForConcreteUrl(skill, contextUrl) {
|
|
197461
|
+
if (!urlHasConcreteResourcePath2(contextUrl))
|
|
197462
|
+
return true;
|
|
197463
|
+
let contextPath = "";
|
|
197464
|
+
try {
|
|
197465
|
+
contextPath = new URL(contextUrl).pathname;
|
|
197466
|
+
} catch {
|
|
197467
|
+
return true;
|
|
197468
|
+
}
|
|
197469
|
+
return (skill.endpoints ?? []).some((ep) => {
|
|
197470
|
+
if (pathCoherent2(ep.url_template, contextPath))
|
|
197471
|
+
return true;
|
|
197472
|
+
try {
|
|
197473
|
+
return ep.trigger_url ? new URL(ep.trigger_url).pathname === contextPath : false;
|
|
197474
|
+
} catch {
|
|
197475
|
+
return false;
|
|
197476
|
+
}
|
|
197477
|
+
});
|
|
197478
|
+
}
|
|
197479
|
+
async function fetchLiteralConcreteBody(url) {
|
|
197480
|
+
try {
|
|
197481
|
+
const res = await fetch(url, {
|
|
197482
|
+
headers: { Accept: "*/*", "User-Agent": "unbrowse/1.0" },
|
|
197483
|
+
signal: AbortSignal.timeout(15000),
|
|
197484
|
+
redirect: "follow"
|
|
197485
|
+
});
|
|
197486
|
+
if (!res.ok)
|
|
197487
|
+
return null;
|
|
197488
|
+
const ct = res.headers.get("content-type") ?? "";
|
|
197489
|
+
const body = await res.text();
|
|
197490
|
+
if (!(body.length > 0 && body.length < 2000000))
|
|
197491
|
+
return null;
|
|
197492
|
+
const trimmed = body.trimStart();
|
|
197493
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
197494
|
+
try {
|
|
197495
|
+
const parsed = JSON.parse(body);
|
|
197496
|
+
if (parsed !== null && typeof parsed === "object")
|
|
197497
|
+
return { result: parsed, content_type: ct };
|
|
197498
|
+
} catch {}
|
|
197499
|
+
}
|
|
197500
|
+
return {
|
|
197501
|
+
result: { url, content_type: ct, text: body, source_url: url, extraction: { source: "direct-fetch", rejected: false } },
|
|
197502
|
+
content_type: ct
|
|
197503
|
+
};
|
|
197504
|
+
} catch (e) {
|
|
197505
|
+
console.log(`[direct-fetch] ${url} literal concrete fetch failed: ${e instanceof Error ? e.message : String(e)} — falling through to skill`);
|
|
197506
|
+
return null;
|
|
197507
|
+
}
|
|
197508
|
+
}
|
|
197268
197509
|
function prioritizeIntentMatchedApis(ranked, intent, contextUrl) {
|
|
197269
197510
|
const preferred = inferPreferredEntityTokens(intent);
|
|
197270
197511
|
if (preferred.length === 0)
|
|
@@ -198827,6 +199068,27 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
198827
199068
|
timing: finalize("marketplace", w.data, w.skill.skill_id, w.skill, recipeTrace)
|
|
198828
199069
|
};
|
|
198829
199070
|
}
|
|
199071
|
+
if ((w.kind === "marketplace" || w.kind === "local-skill") && urlHasConcreteResourcePath2(raceContextUrl) && !skillPathCoherentForConcreteUrl(w.skill, raceContextUrl)) {
|
|
199072
|
+
const literal = await fetchLiteralConcreteBody(raceContextUrl);
|
|
199073
|
+
if (literal) {
|
|
199074
|
+
const trace2 = {
|
|
199075
|
+
trace_id: nanoid(),
|
|
199076
|
+
skill_id: "direct-fetch",
|
|
199077
|
+
endpoint_id: "direct-fetch",
|
|
199078
|
+
started_at: new Date(t0).toISOString(),
|
|
199079
|
+
completed_at: new Date().toISOString(),
|
|
199080
|
+
success: true
|
|
199081
|
+
};
|
|
199082
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal URL preferred over non-path-coherent skill ${w.skill.skill_id} (ct=${literal.content_type})`);
|
|
199083
|
+
return {
|
|
199084
|
+
result: literal.result,
|
|
199085
|
+
trace: trace2,
|
|
199086
|
+
source: "direct-fetch",
|
|
199087
|
+
skill: undefined,
|
|
199088
|
+
timing: finalize("direct-fetch", literal.result, "direct-fetch", undefined, trace2)
|
|
199089
|
+
};
|
|
199090
|
+
}
|
|
199091
|
+
}
|
|
198830
199092
|
if (w.kind === "marketplace") {
|
|
198831
199093
|
return buildDeferral(w.skill, "marketplace", { decision_trace: decisionTrace });
|
|
198832
199094
|
}
|
|
@@ -198855,6 +199117,46 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
198855
199117
|
}
|
|
198856
199118
|
console.log(`[direct-fetch] ${raceContextUrl} probe said JSON but direct fetch yielded no usable JSON — falling through to exa`);
|
|
198857
199119
|
}
|
|
199120
|
+
if (w.kind === "probe" && w.status >= 200 && w.status < 400 && !probeLooksLikeDirectJsonApi(w) && !probeLooksLikeFetchableHtmlDocument(w) && urlHasConcreteResourcePath2(raceContextUrl)) {
|
|
199121
|
+
try {
|
|
199122
|
+
const litRes = await fetch(raceContextUrl, {
|
|
199123
|
+
headers: { Accept: "*/*", "User-Agent": "unbrowse/1.0" },
|
|
199124
|
+
signal: AbortSignal.timeout(15000),
|
|
199125
|
+
redirect: "follow"
|
|
199126
|
+
});
|
|
199127
|
+
if (litRes.ok) {
|
|
199128
|
+
const ct = litRes.headers.get("content-type") ?? "";
|
|
199129
|
+
const body = await litRes.text();
|
|
199130
|
+
if (body.length > 0 && body.length < 2000000) {
|
|
199131
|
+
const trace2 = {
|
|
199132
|
+
trace_id: nanoid(),
|
|
199133
|
+
skill_id: "direct-fetch",
|
|
199134
|
+
endpoint_id: "direct-fetch",
|
|
199135
|
+
started_at: new Date(t0).toISOString(),
|
|
199136
|
+
completed_at: new Date().toISOString(),
|
|
199137
|
+
success: true
|
|
199138
|
+
};
|
|
199139
|
+
const result2 = {
|
|
199140
|
+
url: raceContextUrl,
|
|
199141
|
+
content_type: ct,
|
|
199142
|
+
text: body,
|
|
199143
|
+
source_url: raceContextUrl,
|
|
199144
|
+
extraction: { source: "direct-fetch", rejected: false }
|
|
199145
|
+
};
|
|
199146
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal URL — direct fetch over exa (ct=${ct}, ${body.length}B)`);
|
|
199147
|
+
return {
|
|
199148
|
+
result: result2,
|
|
199149
|
+
trace: trace2,
|
|
199150
|
+
source: "direct-fetch",
|
|
199151
|
+
skill: undefined,
|
|
199152
|
+
timing: finalize("direct-fetch", result2, "direct-fetch", undefined, trace2)
|
|
199153
|
+
};
|
|
199154
|
+
}
|
|
199155
|
+
}
|
|
199156
|
+
} catch (e) {
|
|
199157
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal fetch failed: ${e instanceof Error ? e.message : String(e)} — falling through to exa`);
|
|
199158
|
+
}
|
|
199159
|
+
}
|
|
198858
199160
|
if (w.kind === "probe" && probeLooksLikeFetchableHtmlDocument(w)) {
|
|
198859
199161
|
const directDoc = await fetchDirectDocument(raceContextUrl);
|
|
198860
199162
|
if (directDoc && !directDoc.rejected) {
|
|
@@ -199091,7 +199393,7 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
199091
199393
|
if (!forceCapture && !agentChoseEndpoint) {
|
|
199092
199394
|
const cachedResult = routeResultCache.get(cacheKey2);
|
|
199093
199395
|
if (cachedResult) {
|
|
199094
|
-
if (!shouldReuseRouteResultSnapshot(cachedResult, queryIntent, context?.url)) {
|
|
199396
|
+
if (!cachedSkillHostMatchesContext(cachedResult.skill.domain, context?.url) || !shouldReuseRouteResultSnapshot(cachedResult, queryIntent, context?.url)) {
|
|
199095
199397
|
routeResultCache.delete(cacheKey2);
|
|
199096
199398
|
} else {
|
|
199097
199399
|
timing.cache_hit = true;
|
|
@@ -199128,6 +199430,11 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
199128
199430
|
persistRouteCache();
|
|
199129
199431
|
continue;
|
|
199130
199432
|
}
|
|
199433
|
+
if (!cachedSkillHostMatchesContext(cached5.domain, context?.url)) {
|
|
199434
|
+
skillRouteCache.delete(scopedKey);
|
|
199435
|
+
persistRouteCache();
|
|
199436
|
+
continue;
|
|
199437
|
+
}
|
|
199131
199438
|
const skill = readSkillSnapshot(cached5.localSkillPath) ?? await getSkillWithTimeout(cached5.skillId, clientScope);
|
|
199132
199439
|
if (!skill || !isCachedSkillRelevantForIntent(skill, queryIntent, context?.url)) {
|
|
199133
199440
|
skillRouteCache.delete(scopedKey);
|
|
@@ -199230,6 +199537,11 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
199230
199537
|
persistRouteCache();
|
|
199231
199538
|
continue;
|
|
199232
199539
|
}
|
|
199540
|
+
if (!cachedSkillHostMatchesContext(cached6.domain, context?.url)) {
|
|
199541
|
+
skillRouteCache.delete(scopedKey);
|
|
199542
|
+
persistRouteCache();
|
|
199543
|
+
continue;
|
|
199544
|
+
}
|
|
199233
199545
|
const skill = readSkillSnapshot(cached6.localSkillPath) ?? await getSkillWithTimeout(cached6.skillId, clientScope);
|
|
199234
199546
|
if (!skill)
|
|
199235
199547
|
continue;
|
|
@@ -293679,6 +293991,7 @@ async function handler14(parsed, opts) {
|
|
|
293679
293991
|
const wsEndpoint = typeof parsed.flags.ws === "string" ? parsed.flags.ws : undefined;
|
|
293680
293992
|
const conn = wsEndpoint ? await attach(wsEndpoint) : await spawnChrome({ headless: true, perContextProxy: false, persist: true });
|
|
293681
293993
|
const target = await createTarget(conn, url, {});
|
|
293994
|
+
let cookiesInjected = 0;
|
|
293682
293995
|
try {
|
|
293683
293996
|
const { shouldImportBrowserCookies: shouldImportBrowserCookies2 } = await init_auth().then(() => exports_auth);
|
|
293684
293997
|
if (shouldImportBrowserCookies2()) {
|
|
@@ -293709,6 +294022,7 @@ async function handler14(parsed, opts) {
|
|
|
293709
294022
|
});
|
|
293710
294023
|
await conn.call("Network.setCookies", { cookies: cdpCookies }, target.sessionId);
|
|
293711
294024
|
await conn.call("Page.navigate", { url }, target.sessionId);
|
|
294025
|
+
cookiesInjected = cdpCookies.length;
|
|
293712
294026
|
process.stderr.write(`[auth] act go: injected ${cdpCookies.length} cookie(s) for ${host}, re-navigated authenticated
|
|
293713
294027
|
`);
|
|
293714
294028
|
} else {
|
|
@@ -293718,6 +294032,24 @@ async function handler14(parsed, opts) {
|
|
|
293718
294032
|
}
|
|
293719
294033
|
} catch (cookieErr) {
|
|
293720
294034
|
process.stderr.write(`[auth] act go: cookie injection skipped: ${cookieErr instanceof Error ? cookieErr.message : String(cookieErr)}
|
|
294035
|
+
`);
|
|
294036
|
+
}
|
|
294037
|
+
let pageText = null;
|
|
294038
|
+
try {
|
|
294039
|
+
await conn.call("Runtime.enable", {}, target.sessionId);
|
|
294040
|
+
const deadline = Date.now() + 6000;
|
|
294041
|
+
while (Date.now() < deadline) {
|
|
294042
|
+
const rs = await conn.call("Runtime.evaluate", { expression: "document.readyState", returnByValue: true }, target.sessionId);
|
|
294043
|
+
if (rs?.result?.value === "complete")
|
|
294044
|
+
break;
|
|
294045
|
+
await new Promise((res) => setTimeout(res, 200));
|
|
294046
|
+
}
|
|
294047
|
+
const body = await conn.call("Runtime.evaluate", { expression: "document.body ? document.body.innerText : ''", returnByValue: true }, target.sessionId);
|
|
294048
|
+
const v = body?.result?.value;
|
|
294049
|
+
if (typeof v === "string" && v.length > 0)
|
|
294050
|
+
pageText = v.slice(0, 200000);
|
|
294051
|
+
} catch (readErr) {
|
|
294052
|
+
process.stderr.write(`[page] act go: body read skipped: ${readErr instanceof Error ? readErr.message : String(readErr)}
|
|
293721
294053
|
`);
|
|
293722
294054
|
}
|
|
293723
294055
|
let captcha;
|
|
@@ -293750,6 +294082,8 @@ async function handler14(parsed, opts) {
|
|
|
293750
294082
|
context_id: "",
|
|
293751
294083
|
chrome_ws_url: conn.endpoint,
|
|
293752
294084
|
url,
|
|
294085
|
+
cookies_injected: cookiesInjected,
|
|
294086
|
+
...pageText !== null ? { page: { text: pageText } } : {},
|
|
293753
294087
|
...captcha ? { captcha } : {},
|
|
293754
294088
|
audit: {
|
|
293755
294089
|
ok: navAudit.ok,
|
|
@@ -303939,7 +304273,8 @@ async function cmdResolve(flags) {
|
|
|
303939
304273
|
}
|
|
303940
304274
|
if (resolveCacheSafe(flags)) {
|
|
303941
304275
|
const cachedHit = peekResolution(resolveCacheKeyFor(flags, intent), resolveCacheTtlMs());
|
|
303942
|
-
|
|
304276
|
+
const cachedData = cachedHit ? cachedHit.result ?? cachedHit.data : undefined;
|
|
304277
|
+
if (cachedHit && resolutionCardinalityMatches(intent, cachedData) && resolutionHostMatches(typeof flags.url === "string" ? flags.url : undefined, cachedData) && resolutionPathMatches(typeof flags.url === "string" ? flags.url : undefined, cachedData)) {
|
|
303943
304278
|
const replay = markResolveCacheReplay(cachedHit);
|
|
303944
304279
|
const hostType2 = detectTelemetryHostType2();
|
|
303945
304280
|
if (process.env.UNBROWSE_LANDING_TOKEN || process.env.UNBROWSE_ATTRIBUTION_B64) {
|
|
@@ -304236,6 +304571,14 @@ function parseCmdHoleIntentArgs(args, flags, verb) {
|
|
|
304236
304571
|
return { error: `usage: unbrowse ${verb} <url> "task"` };
|
|
304237
304572
|
return { url: args[0], intent: intent2 };
|
|
304238
304573
|
}
|
|
304574
|
+
const urlArgs = args.filter(looksLikeUrl);
|
|
304575
|
+
if (urlArgs.length === 1) {
|
|
304576
|
+
const rest = args.filter((a) => a !== urlArgs[0]);
|
|
304577
|
+
const intent2 = flagIntent ?? (rest.length > 0 ? rest.join(" ") : undefined);
|
|
304578
|
+
if (!intent2)
|
|
304579
|
+
return { error: `usage: unbrowse ${verb} <url> "task"` };
|
|
304580
|
+
return { url: urlArgs[0], intent: intent2 };
|
|
304581
|
+
}
|
|
304239
304582
|
const intent = flagIntent ?? (args.length > 0 ? args.join(" ") : undefined);
|
|
304240
304583
|
if (!intent)
|
|
304241
304584
|
return { error: `usage: unbrowse ${verb} "task" [--url <url>]` };
|
package/runtime/mcp.js
CHANGED
|
@@ -101443,7 +101443,7 @@ async function mergedAuthHeaders(key, signer) {
|
|
|
101443
101443
|
var AUTH_DOMAIN = "unbrowse-auth:v1";
|
|
101444
101444
|
|
|
101445
101445
|
// .tmp-runtime-src/build-info.generated.ts
|
|
101446
|
-
var BUILD_RELEASE_VERSION = "10.1.
|
|
101446
|
+
var BUILD_RELEASE_VERSION = "10.1.4", BUILD_GIT_SHA = "089d49f75f26", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMTAuMS40IiwiZ2l0X3NoYSI6IjA4OWQ0OWY3NWYyNiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAMDg5ZDQ5Zjc1ZjI2IiwiaXNzdWVkX2F0IjoiMjAyNi0wNi0yM1QxMDowNjoyMy45MjNaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "cDbcbJgh0OyGCynwPC9q8d8xvPCtfzsY9H0lAtidjN4", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai", BUILD_DEFAULT_PROFILE = "";
|
|
101447
101447
|
|
|
101448
101448
|
// .tmp-runtime-src/version.ts
|
|
101449
101449
|
import { createHash as createHash6 } from "crypto";
|
|
@@ -104929,6 +104929,129 @@ function cardinalityMatches(intent, subject, opts) {
|
|
|
104929
104929
|
function resolutionCardinalityMatches(intent, data) {
|
|
104930
104930
|
return cardinalityMatches(intent, { kind: "value", value: data });
|
|
104931
104931
|
}
|
|
104932
|
+
function registrableHostOf(input) {
|
|
104933
|
+
if (!input)
|
|
104934
|
+
return null;
|
|
104935
|
+
const host = (() => {
|
|
104936
|
+
try {
|
|
104937
|
+
return new URL(input).hostname;
|
|
104938
|
+
} catch {}
|
|
104939
|
+
try {
|
|
104940
|
+
return new URL(`https://${input}`).hostname;
|
|
104941
|
+
} catch {
|
|
104942
|
+
return null;
|
|
104943
|
+
}
|
|
104944
|
+
})();
|
|
104945
|
+
if (!host)
|
|
104946
|
+
return null;
|
|
104947
|
+
return host.replace(/^www\./, "").split(".").slice(-2).join(".") || null;
|
|
104948
|
+
}
|
|
104949
|
+
function collectRecordHosts(data) {
|
|
104950
|
+
const out = [];
|
|
104951
|
+
const push = (rec) => {
|
|
104952
|
+
if (!rec || typeof rec !== "object")
|
|
104953
|
+
return;
|
|
104954
|
+
const r = rec;
|
|
104955
|
+
for (const k of ["url", "link", "href", "source_url"]) {
|
|
104956
|
+
const v = r[k];
|
|
104957
|
+
const h = typeof v === "string" ? registrableHostOf(v) : null;
|
|
104958
|
+
if (h)
|
|
104959
|
+
out.push(h);
|
|
104960
|
+
}
|
|
104961
|
+
};
|
|
104962
|
+
if (Array.isArray(data))
|
|
104963
|
+
for (const rec of data.slice(0, 20))
|
|
104964
|
+
push(rec);
|
|
104965
|
+
else
|
|
104966
|
+
push(data);
|
|
104967
|
+
return out;
|
|
104968
|
+
}
|
|
104969
|
+
function resolutionHostMatches(url, data) {
|
|
104970
|
+
const reqHost = registrableHostOf(url);
|
|
104971
|
+
if (!reqHost)
|
|
104972
|
+
return true;
|
|
104973
|
+
const hosts = collectRecordHosts(data);
|
|
104974
|
+
if (hosts.length === 0)
|
|
104975
|
+
return true;
|
|
104976
|
+
return hosts.some((h) => h === reqHost);
|
|
104977
|
+
}
|
|
104978
|
+
function isConcreteResourceLeaf(leaf) {
|
|
104979
|
+
return !!leaf && !/^(search|explore|trending|tabs|home|for-you|foryou|latest|live|people|posts|videos)$/.test(leaf);
|
|
104980
|
+
}
|
|
104981
|
+
function urlHasConcreteResourcePath(url) {
|
|
104982
|
+
if (!url)
|
|
104983
|
+
return false;
|
|
104984
|
+
try {
|
|
104985
|
+
const segments = new URL(url).pathname.split("/").filter(Boolean);
|
|
104986
|
+
if (segments.length === 0)
|
|
104987
|
+
return false;
|
|
104988
|
+
const leaf = decodeURIComponent(segments[segments.length - 1] ?? "").toLowerCase();
|
|
104989
|
+
return isConcreteResourceLeaf(leaf);
|
|
104990
|
+
} catch {
|
|
104991
|
+
return false;
|
|
104992
|
+
}
|
|
104993
|
+
}
|
|
104994
|
+
function pathCoherent(candidate, contextPath) {
|
|
104995
|
+
const toPath = (s) => {
|
|
104996
|
+
try {
|
|
104997
|
+
return new URL(s).pathname;
|
|
104998
|
+
} catch {
|
|
104999
|
+
return s;
|
|
105000
|
+
}
|
|
105001
|
+
};
|
|
105002
|
+
const decodeSeg = (s) => {
|
|
105003
|
+
try {
|
|
105004
|
+
return decodeURIComponent(s);
|
|
105005
|
+
} catch {
|
|
105006
|
+
return s;
|
|
105007
|
+
}
|
|
105008
|
+
};
|
|
105009
|
+
const candSegments = toPath(candidate).split("/").filter(Boolean).map(decodeSeg);
|
|
105010
|
+
const ctxSegments = toPath(contextPath).split("/").filter(Boolean).map(decodeSeg);
|
|
105011
|
+
if (candSegments.length !== ctxSegments.length)
|
|
105012
|
+
return false;
|
|
105013
|
+
for (let i = 0;i < candSegments.length; i++) {
|
|
105014
|
+
const cand = candSegments[i] ?? "";
|
|
105015
|
+
const ctx = ctxSegments[i] ?? "";
|
|
105016
|
+
const isHole = /^\{[^}]*\}$/.test(cand);
|
|
105017
|
+
if (!isHole && cand !== ctx)
|
|
105018
|
+
return false;
|
|
105019
|
+
}
|
|
105020
|
+
return true;
|
|
105021
|
+
}
|
|
105022
|
+
function collectRecordUrls(data) {
|
|
105023
|
+
const out = [];
|
|
105024
|
+
const push = (rec) => {
|
|
105025
|
+
if (!rec || typeof rec !== "object")
|
|
105026
|
+
return;
|
|
105027
|
+
const r = rec;
|
|
105028
|
+
for (const k of ["url", "link", "href", "source_url", "url_template"]) {
|
|
105029
|
+
const v = r[k];
|
|
105030
|
+
if (typeof v === "string" && v)
|
|
105031
|
+
out.push(v);
|
|
105032
|
+
}
|
|
105033
|
+
};
|
|
105034
|
+
if (Array.isArray(data))
|
|
105035
|
+
for (const rec of data.slice(0, 20))
|
|
105036
|
+
push(rec);
|
|
105037
|
+
else
|
|
105038
|
+
push(data);
|
|
105039
|
+
return out;
|
|
105040
|
+
}
|
|
105041
|
+
function resolutionPathMatches(url, data) {
|
|
105042
|
+
if (!urlHasConcreteResourcePath(url))
|
|
105043
|
+
return true;
|
|
105044
|
+
let contextPath = "";
|
|
105045
|
+
try {
|
|
105046
|
+
contextPath = new URL(url).pathname;
|
|
105047
|
+
} catch {
|
|
105048
|
+
return true;
|
|
105049
|
+
}
|
|
105050
|
+
const urls = collectRecordUrls(data);
|
|
105051
|
+
if (urls.length === 0)
|
|
105052
|
+
return true;
|
|
105053
|
+
return urls.some((u) => pathCoherent(u, contextPath));
|
|
105054
|
+
}
|
|
104932
105055
|
var LIST_INTENT_RE, ITEM_SCHEMA_TYPES, COLLECTION_KEYS;
|
|
104933
105056
|
var init_cardinality = __esm(() => {
|
|
104934
105057
|
LIST_INTENT_RE = /\b(search|find|lookup|browse|discover|list(?:ings?)?|feed|catalog(?:ue)?)\b/i;
|
|
@@ -192838,7 +192961,9 @@ function shouldAutoWalk(requestedUrl, topUrl, topScore, minScore = 0.8) {
|
|
|
192838
192961
|
return false;
|
|
192839
192962
|
const reqReg = registrableHost(requestedUrl);
|
|
192840
192963
|
const topReg = registrableHost(topUrl);
|
|
192841
|
-
|
|
192964
|
+
if (reqReg)
|
|
192965
|
+
return reqReg === topReg;
|
|
192966
|
+
return (topScore ?? 0) >= minScore;
|
|
192842
192967
|
}
|
|
192843
192968
|
function pickWalkTarget(requestedUrl, ranked, minScore = 0.8) {
|
|
192844
192969
|
const eligible = ranked.filter((c) => c?.url && shouldAutoWalk(requestedUrl, c.url, c.score, minScore));
|
|
@@ -193295,6 +193420,17 @@ function endpointMatchesContextOrigin(endpoint, contextUrl) {
|
|
|
193295
193420
|
return true;
|
|
193296
193421
|
}
|
|
193297
193422
|
}
|
|
193423
|
+
function cachedSkillHostMatchesContext(skillDomain, contextUrl) {
|
|
193424
|
+
if (!contextUrl)
|
|
193425
|
+
return true;
|
|
193426
|
+
const ctxReg = registrableHost(contextUrl);
|
|
193427
|
+
if (!ctxReg)
|
|
193428
|
+
return true;
|
|
193429
|
+
const skillReg = registrableHost(skillDomain) ?? registrableHost(`https://${skillDomain ?? ""}`);
|
|
193430
|
+
if (!skillReg)
|
|
193431
|
+
return true;
|
|
193432
|
+
return skillReg === ctxReg;
|
|
193433
|
+
}
|
|
193298
193434
|
function endpointTargetsMismatchedLocalReplayHost(endpoint, contextUrl) {
|
|
193299
193435
|
if (!contextUrl)
|
|
193300
193436
|
return false;
|
|
@@ -194406,7 +194542,7 @@ function isConcreteEntityDetailIntent(intent, contextUrl) {
|
|
|
194406
194542
|
return false;
|
|
194407
194543
|
try {
|
|
194408
194544
|
const leaf = decodeURIComponent(new URL(contextUrl).pathname.split("/").filter(Boolean).pop() ?? "").toLowerCase();
|
|
194409
|
-
return
|
|
194545
|
+
return isConcreteResourceLeaf(leaf);
|
|
194410
194546
|
} catch {
|
|
194411
194547
|
return false;
|
|
194412
194548
|
}
|
|
@@ -194418,7 +194554,8 @@ function marketplaceSkillMatchesContext(skill, intent, contextUrl) {
|
|
|
194418
194554
|
if (isFeedTimelineIntent(intent, contextUrl)) {
|
|
194419
194555
|
return skill.endpoints.some((endpoint) => endpointMatchesFeedTimelineContext(endpoint, contextUrl));
|
|
194420
194556
|
}
|
|
194421
|
-
|
|
194557
|
+
const concretePath = contextUrl && urlHasConcreteResourcePath(contextUrl);
|
|
194558
|
+
if (!contextUrl || !isConcreteEntityDetailIntent(intent, contextUrl) && !concretePath)
|
|
194422
194559
|
return true;
|
|
194423
194560
|
let contextPath = "";
|
|
194424
194561
|
try {
|
|
@@ -194428,6 +194565,17 @@ function marketplaceSkillMatchesContext(skill, intent, contextUrl) {
|
|
|
194428
194565
|
}
|
|
194429
194566
|
if (!contextPath)
|
|
194430
194567
|
return true;
|
|
194568
|
+
if (concretePath) {
|
|
194569
|
+
for (const endpoint of skill.endpoints ?? []) {
|
|
194570
|
+
let triggerPath = "";
|
|
194571
|
+
try {
|
|
194572
|
+
triggerPath = endpoint.trigger_url ? new URL(endpoint.trigger_url).pathname : "";
|
|
194573
|
+
} catch {}
|
|
194574
|
+
if (pathCoherent(endpoint.url_template, contextPath) || triggerPath === contextPath)
|
|
194575
|
+
return true;
|
|
194576
|
+
}
|
|
194577
|
+
return false;
|
|
194578
|
+
}
|
|
194431
194579
|
let hasApiLikeEndpoint = false;
|
|
194432
194580
|
for (const endpoint of skill.endpoints ?? []) {
|
|
194433
194581
|
let path8 = "";
|
|
@@ -194446,6 +194594,55 @@ function marketplaceSkillMatchesContext(skill, intent, contextUrl) {
|
|
|
194446
194594
|
}
|
|
194447
194595
|
return hasApiLikeEndpoint;
|
|
194448
194596
|
}
|
|
194597
|
+
function skillPathCoherentForConcreteUrl(skill, contextUrl) {
|
|
194598
|
+
if (!urlHasConcreteResourcePath(contextUrl))
|
|
194599
|
+
return true;
|
|
194600
|
+
let contextPath = "";
|
|
194601
|
+
try {
|
|
194602
|
+
contextPath = new URL(contextUrl).pathname;
|
|
194603
|
+
} catch {
|
|
194604
|
+
return true;
|
|
194605
|
+
}
|
|
194606
|
+
return (skill.endpoints ?? []).some((ep) => {
|
|
194607
|
+
if (pathCoherent(ep.url_template, contextPath))
|
|
194608
|
+
return true;
|
|
194609
|
+
try {
|
|
194610
|
+
return ep.trigger_url ? new URL(ep.trigger_url).pathname === contextPath : false;
|
|
194611
|
+
} catch {
|
|
194612
|
+
return false;
|
|
194613
|
+
}
|
|
194614
|
+
});
|
|
194615
|
+
}
|
|
194616
|
+
async function fetchLiteralConcreteBody(url) {
|
|
194617
|
+
try {
|
|
194618
|
+
const res = await fetch(url, {
|
|
194619
|
+
headers: { Accept: "*/*", "User-Agent": "unbrowse/1.0" },
|
|
194620
|
+
signal: AbortSignal.timeout(15000),
|
|
194621
|
+
redirect: "follow"
|
|
194622
|
+
});
|
|
194623
|
+
if (!res.ok)
|
|
194624
|
+
return null;
|
|
194625
|
+
const ct = res.headers.get("content-type") ?? "";
|
|
194626
|
+
const body = await res.text();
|
|
194627
|
+
if (!(body.length > 0 && body.length < 2000000))
|
|
194628
|
+
return null;
|
|
194629
|
+
const trimmed = body.trimStart();
|
|
194630
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
194631
|
+
try {
|
|
194632
|
+
const parsed = JSON.parse(body);
|
|
194633
|
+
if (parsed !== null && typeof parsed === "object")
|
|
194634
|
+
return { result: parsed, content_type: ct };
|
|
194635
|
+
} catch {}
|
|
194636
|
+
}
|
|
194637
|
+
return {
|
|
194638
|
+
result: { url, content_type: ct, text: body, source_url: url, extraction: { source: "direct-fetch", rejected: false } },
|
|
194639
|
+
content_type: ct
|
|
194640
|
+
};
|
|
194641
|
+
} catch (e) {
|
|
194642
|
+
console.log(`[direct-fetch] ${url} literal concrete fetch failed: ${e instanceof Error ? e.message : String(e)} — falling through to skill`);
|
|
194643
|
+
return null;
|
|
194644
|
+
}
|
|
194645
|
+
}
|
|
194449
194646
|
function prioritizeIntentMatchedApis(ranked, intent, contextUrl) {
|
|
194450
194647
|
const preferred = inferPreferredEntityTokens(intent);
|
|
194451
194648
|
if (preferred.length === 0)
|
|
@@ -196008,6 +196205,27 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
196008
196205
|
timing: finalize("marketplace", w.data, w.skill.skill_id, w.skill, recipeTrace)
|
|
196009
196206
|
};
|
|
196010
196207
|
}
|
|
196208
|
+
if ((w.kind === "marketplace" || w.kind === "local-skill") && urlHasConcreteResourcePath(raceContextUrl) && !skillPathCoherentForConcreteUrl(w.skill, raceContextUrl)) {
|
|
196209
|
+
const literal = await fetchLiteralConcreteBody(raceContextUrl);
|
|
196210
|
+
if (literal) {
|
|
196211
|
+
const trace2 = {
|
|
196212
|
+
trace_id: nanoid(),
|
|
196213
|
+
skill_id: "direct-fetch",
|
|
196214
|
+
endpoint_id: "direct-fetch",
|
|
196215
|
+
started_at: new Date(t0).toISOString(),
|
|
196216
|
+
completed_at: new Date().toISOString(),
|
|
196217
|
+
success: true
|
|
196218
|
+
};
|
|
196219
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal URL preferred over non-path-coherent skill ${w.skill.skill_id} (ct=${literal.content_type})`);
|
|
196220
|
+
return {
|
|
196221
|
+
result: literal.result,
|
|
196222
|
+
trace: trace2,
|
|
196223
|
+
source: "direct-fetch",
|
|
196224
|
+
skill: undefined,
|
|
196225
|
+
timing: finalize("direct-fetch", literal.result, "direct-fetch", undefined, trace2)
|
|
196226
|
+
};
|
|
196227
|
+
}
|
|
196228
|
+
}
|
|
196011
196229
|
if (w.kind === "marketplace") {
|
|
196012
196230
|
return buildDeferral(w.skill, "marketplace", { decision_trace: decisionTrace });
|
|
196013
196231
|
}
|
|
@@ -196036,6 +196254,46 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
196036
196254
|
}
|
|
196037
196255
|
console.log(`[direct-fetch] ${raceContextUrl} probe said JSON but direct fetch yielded no usable JSON — falling through to exa`);
|
|
196038
196256
|
}
|
|
196257
|
+
if (w.kind === "probe" && w.status >= 200 && w.status < 400 && !probeLooksLikeDirectJsonApi(w) && !probeLooksLikeFetchableHtmlDocument(w) && urlHasConcreteResourcePath(raceContextUrl)) {
|
|
196258
|
+
try {
|
|
196259
|
+
const litRes = await fetch(raceContextUrl, {
|
|
196260
|
+
headers: { Accept: "*/*", "User-Agent": "unbrowse/1.0" },
|
|
196261
|
+
signal: AbortSignal.timeout(15000),
|
|
196262
|
+
redirect: "follow"
|
|
196263
|
+
});
|
|
196264
|
+
if (litRes.ok) {
|
|
196265
|
+
const ct = litRes.headers.get("content-type") ?? "";
|
|
196266
|
+
const body = await litRes.text();
|
|
196267
|
+
if (body.length > 0 && body.length < 2000000) {
|
|
196268
|
+
const trace2 = {
|
|
196269
|
+
trace_id: nanoid(),
|
|
196270
|
+
skill_id: "direct-fetch",
|
|
196271
|
+
endpoint_id: "direct-fetch",
|
|
196272
|
+
started_at: new Date(t0).toISOString(),
|
|
196273
|
+
completed_at: new Date().toISOString(),
|
|
196274
|
+
success: true
|
|
196275
|
+
};
|
|
196276
|
+
const result2 = {
|
|
196277
|
+
url: raceContextUrl,
|
|
196278
|
+
content_type: ct,
|
|
196279
|
+
text: body,
|
|
196280
|
+
source_url: raceContextUrl,
|
|
196281
|
+
extraction: { source: "direct-fetch", rejected: false }
|
|
196282
|
+
};
|
|
196283
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal URL — direct fetch over exa (ct=${ct}, ${body.length}B)`);
|
|
196284
|
+
return {
|
|
196285
|
+
result: result2,
|
|
196286
|
+
trace: trace2,
|
|
196287
|
+
source: "direct-fetch",
|
|
196288
|
+
skill: undefined,
|
|
196289
|
+
timing: finalize("direct-fetch", result2, "direct-fetch", undefined, trace2)
|
|
196290
|
+
};
|
|
196291
|
+
}
|
|
196292
|
+
}
|
|
196293
|
+
} catch (e) {
|
|
196294
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal fetch failed: ${e instanceof Error ? e.message : String(e)} — falling through to exa`);
|
|
196295
|
+
}
|
|
196296
|
+
}
|
|
196039
196297
|
if (w.kind === "probe" && probeLooksLikeFetchableHtmlDocument(w)) {
|
|
196040
196298
|
const directDoc = await fetchDirectDocument(raceContextUrl);
|
|
196041
196299
|
if (directDoc && !directDoc.rejected) {
|
|
@@ -196272,7 +196530,7 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
196272
196530
|
if (!forceCapture && !agentChoseEndpoint) {
|
|
196273
196531
|
const cachedResult = routeResultCache.get(cacheKey2);
|
|
196274
196532
|
if (cachedResult) {
|
|
196275
|
-
if (!shouldReuseRouteResultSnapshot(cachedResult, queryIntent, context?.url)) {
|
|
196533
|
+
if (!cachedSkillHostMatchesContext(cachedResult.skill.domain, context?.url) || !shouldReuseRouteResultSnapshot(cachedResult, queryIntent, context?.url)) {
|
|
196276
196534
|
routeResultCache.delete(cacheKey2);
|
|
196277
196535
|
} else {
|
|
196278
196536
|
timing.cache_hit = true;
|
|
@@ -196309,6 +196567,11 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
196309
196567
|
persistRouteCache();
|
|
196310
196568
|
continue;
|
|
196311
196569
|
}
|
|
196570
|
+
if (!cachedSkillHostMatchesContext(cached5.domain, context?.url)) {
|
|
196571
|
+
skillRouteCache.delete(scopedKey);
|
|
196572
|
+
persistRouteCache();
|
|
196573
|
+
continue;
|
|
196574
|
+
}
|
|
196312
196575
|
const skill = readSkillSnapshot(cached5.localSkillPath) ?? await getSkillWithTimeout(cached5.skillId, clientScope);
|
|
196313
196576
|
if (!skill || !isCachedSkillRelevantForIntent(skill, queryIntent, context?.url)) {
|
|
196314
196577
|
skillRouteCache.delete(scopedKey);
|
|
@@ -196411,6 +196674,11 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
196411
196674
|
persistRouteCache();
|
|
196412
196675
|
continue;
|
|
196413
196676
|
}
|
|
196677
|
+
if (!cachedSkillHostMatchesContext(cached6.domain, context?.url)) {
|
|
196678
|
+
skillRouteCache.delete(scopedKey);
|
|
196679
|
+
persistRouteCache();
|
|
196680
|
+
continue;
|
|
196681
|
+
}
|
|
196414
196682
|
const skill = readSkillSnapshot(cached6.localSkillPath) ?? await getSkillWithTimeout(cached6.skillId, clientScope);
|
|
196415
196683
|
if (!skill)
|
|
196416
196684
|
continue;
|
|
@@ -285751,6 +286019,7 @@ async function handler14(parsed, opts) {
|
|
|
285751
286019
|
const wsEndpoint = typeof parsed.flags.ws === "string" ? parsed.flags.ws : undefined;
|
|
285752
286020
|
const conn = wsEndpoint ? await attach(wsEndpoint) : await spawnChrome({ headless: true, perContextProxy: false, persist: true });
|
|
285753
286021
|
const target = await createTarget(conn, url, {});
|
|
286022
|
+
let cookiesInjected = 0;
|
|
285754
286023
|
try {
|
|
285755
286024
|
const { shouldImportBrowserCookies: shouldImportBrowserCookies2 } = await init_auth().then(() => exports_auth);
|
|
285756
286025
|
if (shouldImportBrowserCookies2()) {
|
|
@@ -285781,6 +286050,7 @@ async function handler14(parsed, opts) {
|
|
|
285781
286050
|
});
|
|
285782
286051
|
await conn.call("Network.setCookies", { cookies: cdpCookies }, target.sessionId);
|
|
285783
286052
|
await conn.call("Page.navigate", { url }, target.sessionId);
|
|
286053
|
+
cookiesInjected = cdpCookies.length;
|
|
285784
286054
|
process.stderr.write(`[auth] act go: injected ${cdpCookies.length} cookie(s) for ${host}, re-navigated authenticated
|
|
285785
286055
|
`);
|
|
285786
286056
|
} else {
|
|
@@ -285790,6 +286060,24 @@ async function handler14(parsed, opts) {
|
|
|
285790
286060
|
}
|
|
285791
286061
|
} catch (cookieErr) {
|
|
285792
286062
|
process.stderr.write(`[auth] act go: cookie injection skipped: ${cookieErr instanceof Error ? cookieErr.message : String(cookieErr)}
|
|
286063
|
+
`);
|
|
286064
|
+
}
|
|
286065
|
+
let pageText = null;
|
|
286066
|
+
try {
|
|
286067
|
+
await conn.call("Runtime.enable", {}, target.sessionId);
|
|
286068
|
+
const deadline = Date.now() + 6000;
|
|
286069
|
+
while (Date.now() < deadline) {
|
|
286070
|
+
const rs = await conn.call("Runtime.evaluate", { expression: "document.readyState", returnByValue: true }, target.sessionId);
|
|
286071
|
+
if (rs?.result?.value === "complete")
|
|
286072
|
+
break;
|
|
286073
|
+
await new Promise((res) => setTimeout(res, 200));
|
|
286074
|
+
}
|
|
286075
|
+
const body = await conn.call("Runtime.evaluate", { expression: "document.body ? document.body.innerText : ''", returnByValue: true }, target.sessionId);
|
|
286076
|
+
const v = body?.result?.value;
|
|
286077
|
+
if (typeof v === "string" && v.length > 0)
|
|
286078
|
+
pageText = v.slice(0, 200000);
|
|
286079
|
+
} catch (readErr) {
|
|
286080
|
+
process.stderr.write(`[page] act go: body read skipped: ${readErr instanceof Error ? readErr.message : String(readErr)}
|
|
285793
286081
|
`);
|
|
285794
286082
|
}
|
|
285795
286083
|
let captcha;
|
|
@@ -285822,6 +286110,8 @@ async function handler14(parsed, opts) {
|
|
|
285822
286110
|
context_id: "",
|
|
285823
286111
|
chrome_ws_url: conn.endpoint,
|
|
285824
286112
|
url,
|
|
286113
|
+
cookies_injected: cookiesInjected,
|
|
286114
|
+
...pageText !== null ? { page: { text: pageText } } : {},
|
|
285825
286115
|
...captcha ? { captcha } : {},
|
|
285826
286116
|
audit: {
|
|
285827
286117
|
ok: navAudit.ok,
|
|
@@ -296188,7 +296478,8 @@ async function cmdResolve(flags) {
|
|
|
296188
296478
|
}
|
|
296189
296479
|
if (resolveCacheSafe(flags)) {
|
|
296190
296480
|
const cachedHit = peekResolution(resolveCacheKeyFor(flags, intent), resolveCacheTtlMs());
|
|
296191
|
-
|
|
296481
|
+
const cachedData = cachedHit ? cachedHit.result ?? cachedHit.data : undefined;
|
|
296482
|
+
if (cachedHit && resolutionCardinalityMatches(intent, cachedData) && resolutionHostMatches(typeof flags.url === "string" ? flags.url : undefined, cachedData) && resolutionPathMatches(typeof flags.url === "string" ? flags.url : undefined, cachedData)) {
|
|
296192
296483
|
const replay = markResolveCacheReplay(cachedHit);
|
|
296193
296484
|
const hostType2 = detectTelemetryHostType();
|
|
296194
296485
|
if (process.env.UNBROWSE_LANDING_TOKEN || process.env.UNBROWSE_ATTRIBUTION_B64) {
|
|
@@ -296474,6 +296765,14 @@ function parseCmdHoleIntentArgs(args, flags, verb) {
|
|
|
296474
296765
|
return { error: `usage: unbrowse ${verb} <url> "task"` };
|
|
296475
296766
|
return { url: args[0], intent: intent2 };
|
|
296476
296767
|
}
|
|
296768
|
+
const urlArgs = args.filter(looksLikeUrl);
|
|
296769
|
+
if (urlArgs.length === 1) {
|
|
296770
|
+
const rest = args.filter((a) => a !== urlArgs[0]);
|
|
296771
|
+
const intent2 = flagIntent ?? (rest.length > 0 ? rest.join(" ") : undefined);
|
|
296772
|
+
if (!intent2)
|
|
296773
|
+
return { error: `usage: unbrowse ${verb} <url> "task"` };
|
|
296774
|
+
return { url: urlArgs[0], intent: intent2 };
|
|
296775
|
+
}
|
|
296477
296776
|
const intent = flagIntent ?? (args.length > 0 ? args.join(" ") : undefined);
|
|
296478
296777
|
if (!intent)
|
|
296479
296778
|
return { error: `usage: unbrowse ${verb} "task" [--url <url>]` };
|
|
@@ -299908,6 +300207,7 @@ __export(exports_orchestrator, {
|
|
|
299908
300207
|
compositeContentId: () => compositeContentId2,
|
|
299909
300208
|
compositeAddress: () => compositeAddress2,
|
|
299910
300209
|
chooseBestRouteCacheCandidate: () => chooseBestRouteCacheCandidate2,
|
|
300210
|
+
cachedSkillHostMatchesContext: () => cachedSkillHostMatchesContext2,
|
|
299911
300211
|
buildResolveCacheKey: () => buildResolveCacheKey2,
|
|
299912
300212
|
buildCompositeEdges: () => buildCompositeEdges2,
|
|
299913
300213
|
attachCompositeToSkill: () => attachCompositeToSkill2,
|
|
@@ -299940,7 +300240,9 @@ function shouldAutoWalk2(requestedUrl, topUrl, topScore, minScore = 0.8) {
|
|
|
299940
300240
|
return false;
|
|
299941
300241
|
const reqReg = registrableHost2(requestedUrl);
|
|
299942
300242
|
const topReg = registrableHost2(topUrl);
|
|
299943
|
-
|
|
300243
|
+
if (reqReg)
|
|
300244
|
+
return reqReg === topReg;
|
|
300245
|
+
return (topScore ?? 0) >= minScore;
|
|
299944
300246
|
}
|
|
299945
300247
|
function pickWalkTarget2(requestedUrl, ranked, minScore = 0.8) {
|
|
299946
300248
|
const eligible = ranked.filter((c) => c?.url && shouldAutoWalk2(requestedUrl, c.url, c.score, minScore));
|
|
@@ -300397,6 +300699,17 @@ function endpointMatchesContextOrigin2(endpoint, contextUrl) {
|
|
|
300397
300699
|
return true;
|
|
300398
300700
|
}
|
|
300399
300701
|
}
|
|
300702
|
+
function cachedSkillHostMatchesContext2(skillDomain, contextUrl) {
|
|
300703
|
+
if (!contextUrl)
|
|
300704
|
+
return true;
|
|
300705
|
+
const ctxReg = registrableHost2(contextUrl);
|
|
300706
|
+
if (!ctxReg)
|
|
300707
|
+
return true;
|
|
300708
|
+
const skillReg = registrableHost2(skillDomain) ?? registrableHost2(`https://${skillDomain ?? ""}`);
|
|
300709
|
+
if (!skillReg)
|
|
300710
|
+
return true;
|
|
300711
|
+
return skillReg === ctxReg;
|
|
300712
|
+
}
|
|
300400
300713
|
function endpointTargetsMismatchedLocalReplayHost2(endpoint, contextUrl) {
|
|
300401
300714
|
if (!contextUrl)
|
|
300402
300715
|
return false;
|
|
@@ -301516,7 +301829,7 @@ function isConcreteEntityDetailIntent2(intent, contextUrl) {
|
|
|
301516
301829
|
return false;
|
|
301517
301830
|
try {
|
|
301518
301831
|
const leaf = decodeURIComponent(new URL(contextUrl).pathname.split("/").filter(Boolean).pop() ?? "").toLowerCase();
|
|
301519
|
-
return
|
|
301832
|
+
return isConcreteResourceLeaf(leaf);
|
|
301520
301833
|
} catch {
|
|
301521
301834
|
return false;
|
|
301522
301835
|
}
|
|
@@ -301528,7 +301841,8 @@ function marketplaceSkillMatchesContext2(skill, intent, contextUrl) {
|
|
|
301528
301841
|
if (isFeedTimelineIntent2(intent, contextUrl)) {
|
|
301529
301842
|
return skill.endpoints.some((endpoint) => endpointMatchesFeedTimelineContext2(endpoint, contextUrl));
|
|
301530
301843
|
}
|
|
301531
|
-
|
|
301844
|
+
const concretePath = contextUrl && urlHasConcreteResourcePath(contextUrl);
|
|
301845
|
+
if (!contextUrl || !isConcreteEntityDetailIntent2(intent, contextUrl) && !concretePath)
|
|
301532
301846
|
return true;
|
|
301533
301847
|
let contextPath = "";
|
|
301534
301848
|
try {
|
|
@@ -301538,6 +301852,17 @@ function marketplaceSkillMatchesContext2(skill, intent, contextUrl) {
|
|
|
301538
301852
|
}
|
|
301539
301853
|
if (!contextPath)
|
|
301540
301854
|
return true;
|
|
301855
|
+
if (concretePath) {
|
|
301856
|
+
for (const endpoint of skill.endpoints ?? []) {
|
|
301857
|
+
let triggerPath = "";
|
|
301858
|
+
try {
|
|
301859
|
+
triggerPath = endpoint.trigger_url ? new URL(endpoint.trigger_url).pathname : "";
|
|
301860
|
+
} catch {}
|
|
301861
|
+
if (pathCoherent(endpoint.url_template, contextPath) || triggerPath === contextPath)
|
|
301862
|
+
return true;
|
|
301863
|
+
}
|
|
301864
|
+
return false;
|
|
301865
|
+
}
|
|
301541
301866
|
let hasApiLikeEndpoint = false;
|
|
301542
301867
|
for (const endpoint of skill.endpoints ?? []) {
|
|
301543
301868
|
let path27 = "";
|
|
@@ -301556,6 +301881,55 @@ function marketplaceSkillMatchesContext2(skill, intent, contextUrl) {
|
|
|
301556
301881
|
}
|
|
301557
301882
|
return hasApiLikeEndpoint;
|
|
301558
301883
|
}
|
|
301884
|
+
function skillPathCoherentForConcreteUrl2(skill, contextUrl) {
|
|
301885
|
+
if (!urlHasConcreteResourcePath(contextUrl))
|
|
301886
|
+
return true;
|
|
301887
|
+
let contextPath = "";
|
|
301888
|
+
try {
|
|
301889
|
+
contextPath = new URL(contextUrl).pathname;
|
|
301890
|
+
} catch {
|
|
301891
|
+
return true;
|
|
301892
|
+
}
|
|
301893
|
+
return (skill.endpoints ?? []).some((ep) => {
|
|
301894
|
+
if (pathCoherent(ep.url_template, contextPath))
|
|
301895
|
+
return true;
|
|
301896
|
+
try {
|
|
301897
|
+
return ep.trigger_url ? new URL(ep.trigger_url).pathname === contextPath : false;
|
|
301898
|
+
} catch {
|
|
301899
|
+
return false;
|
|
301900
|
+
}
|
|
301901
|
+
});
|
|
301902
|
+
}
|
|
301903
|
+
async function fetchLiteralConcreteBody2(url) {
|
|
301904
|
+
try {
|
|
301905
|
+
const res = await fetch(url, {
|
|
301906
|
+
headers: { Accept: "*/*", "User-Agent": "unbrowse/1.0" },
|
|
301907
|
+
signal: AbortSignal.timeout(15000),
|
|
301908
|
+
redirect: "follow"
|
|
301909
|
+
});
|
|
301910
|
+
if (!res.ok)
|
|
301911
|
+
return null;
|
|
301912
|
+
const ct = res.headers.get("content-type") ?? "";
|
|
301913
|
+
const body = await res.text();
|
|
301914
|
+
if (!(body.length > 0 && body.length < 2000000))
|
|
301915
|
+
return null;
|
|
301916
|
+
const trimmed = body.trimStart();
|
|
301917
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
301918
|
+
try {
|
|
301919
|
+
const parsed = JSON.parse(body);
|
|
301920
|
+
if (parsed !== null && typeof parsed === "object")
|
|
301921
|
+
return { result: parsed, content_type: ct };
|
|
301922
|
+
} catch {}
|
|
301923
|
+
}
|
|
301924
|
+
return {
|
|
301925
|
+
result: { url, content_type: ct, text: body, source_url: url, extraction: { source: "direct-fetch", rejected: false } },
|
|
301926
|
+
content_type: ct
|
|
301927
|
+
};
|
|
301928
|
+
} catch (e) {
|
|
301929
|
+
console.log(`[direct-fetch] ${url} literal concrete fetch failed: ${e instanceof Error ? e.message : String(e)} — falling through to skill`);
|
|
301930
|
+
return null;
|
|
301931
|
+
}
|
|
301932
|
+
}
|
|
301559
301933
|
function prioritizeIntentMatchedApis2(ranked, intent, contextUrl) {
|
|
301560
301934
|
const preferred = inferPreferredEntityTokens2(intent);
|
|
301561
301935
|
if (preferred.length === 0)
|
|
@@ -303118,6 +303492,27 @@ async function resolveAndExecute2(intent, params = {}, context, projection, opti
|
|
|
303118
303492
|
timing: finalize("marketplace", w.data, w.skill.skill_id, w.skill, recipeTrace)
|
|
303119
303493
|
};
|
|
303120
303494
|
}
|
|
303495
|
+
if ((w.kind === "marketplace" || w.kind === "local-skill") && urlHasConcreteResourcePath(raceContextUrl) && !skillPathCoherentForConcreteUrl2(w.skill, raceContextUrl)) {
|
|
303496
|
+
const literal = await fetchLiteralConcreteBody2(raceContextUrl);
|
|
303497
|
+
if (literal) {
|
|
303498
|
+
const trace2 = {
|
|
303499
|
+
trace_id: nanoid(),
|
|
303500
|
+
skill_id: "direct-fetch",
|
|
303501
|
+
endpoint_id: "direct-fetch",
|
|
303502
|
+
started_at: new Date(t0).toISOString(),
|
|
303503
|
+
completed_at: new Date().toISOString(),
|
|
303504
|
+
success: true
|
|
303505
|
+
};
|
|
303506
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal URL preferred over non-path-coherent skill ${w.skill.skill_id} (ct=${literal.content_type})`);
|
|
303507
|
+
return {
|
|
303508
|
+
result: literal.result,
|
|
303509
|
+
trace: trace2,
|
|
303510
|
+
source: "direct-fetch",
|
|
303511
|
+
skill: undefined,
|
|
303512
|
+
timing: finalize("direct-fetch", literal.result, "direct-fetch", undefined, trace2)
|
|
303513
|
+
};
|
|
303514
|
+
}
|
|
303515
|
+
}
|
|
303121
303516
|
if (w.kind === "marketplace") {
|
|
303122
303517
|
return buildDeferral(w.skill, "marketplace", { decision_trace: decisionTrace });
|
|
303123
303518
|
}
|
|
@@ -303146,6 +303541,46 @@ async function resolveAndExecute2(intent, params = {}, context, projection, opti
|
|
|
303146
303541
|
}
|
|
303147
303542
|
console.log(`[direct-fetch] ${raceContextUrl} probe said JSON but direct fetch yielded no usable JSON — falling through to exa`);
|
|
303148
303543
|
}
|
|
303544
|
+
if (w.kind === "probe" && w.status >= 200 && w.status < 400 && !probeLooksLikeDirectJsonApi2(w) && !probeLooksLikeFetchableHtmlDocument2(w) && urlHasConcreteResourcePath(raceContextUrl)) {
|
|
303545
|
+
try {
|
|
303546
|
+
const litRes = await fetch(raceContextUrl, {
|
|
303547
|
+
headers: { Accept: "*/*", "User-Agent": "unbrowse/1.0" },
|
|
303548
|
+
signal: AbortSignal.timeout(15000),
|
|
303549
|
+
redirect: "follow"
|
|
303550
|
+
});
|
|
303551
|
+
if (litRes.ok) {
|
|
303552
|
+
const ct = litRes.headers.get("content-type") ?? "";
|
|
303553
|
+
const body = await litRes.text();
|
|
303554
|
+
if (body.length > 0 && body.length < 2000000) {
|
|
303555
|
+
const trace2 = {
|
|
303556
|
+
trace_id: nanoid(),
|
|
303557
|
+
skill_id: "direct-fetch",
|
|
303558
|
+
endpoint_id: "direct-fetch",
|
|
303559
|
+
started_at: new Date(t0).toISOString(),
|
|
303560
|
+
completed_at: new Date().toISOString(),
|
|
303561
|
+
success: true
|
|
303562
|
+
};
|
|
303563
|
+
const result2 = {
|
|
303564
|
+
url: raceContextUrl,
|
|
303565
|
+
content_type: ct,
|
|
303566
|
+
text: body,
|
|
303567
|
+
source_url: raceContextUrl,
|
|
303568
|
+
extraction: { source: "direct-fetch", rejected: false }
|
|
303569
|
+
};
|
|
303570
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal URL — direct fetch over exa (ct=${ct}, ${body.length}B)`);
|
|
303571
|
+
return {
|
|
303572
|
+
result: result2,
|
|
303573
|
+
trace: trace2,
|
|
303574
|
+
source: "direct-fetch",
|
|
303575
|
+
skill: undefined,
|
|
303576
|
+
timing: finalize("direct-fetch", result2, "direct-fetch", undefined, trace2)
|
|
303577
|
+
};
|
|
303578
|
+
}
|
|
303579
|
+
}
|
|
303580
|
+
} catch (e) {
|
|
303581
|
+
console.log(`[direct-fetch] ${raceContextUrl} concrete-resource literal fetch failed: ${e instanceof Error ? e.message : String(e)} — falling through to exa`);
|
|
303582
|
+
}
|
|
303583
|
+
}
|
|
303149
303584
|
if (w.kind === "probe" && probeLooksLikeFetchableHtmlDocument2(w)) {
|
|
303150
303585
|
const directDoc = await fetchDirectDocument(raceContextUrl);
|
|
303151
303586
|
if (directDoc && !directDoc.rejected) {
|
|
@@ -303382,7 +303817,7 @@ async function resolveAndExecute2(intent, params = {}, context, projection, opti
|
|
|
303382
303817
|
if (!forceCapture && !agentChoseEndpoint) {
|
|
303383
303818
|
const cachedResult = routeResultCache2.get(cacheKey2);
|
|
303384
303819
|
if (cachedResult) {
|
|
303385
|
-
if (!shouldReuseRouteResultSnapshot2(cachedResult, queryIntent, context?.url)) {
|
|
303820
|
+
if (!cachedSkillHostMatchesContext2(cachedResult.skill.domain, context?.url) || !shouldReuseRouteResultSnapshot2(cachedResult, queryIntent, context?.url)) {
|
|
303386
303821
|
routeResultCache2.delete(cacheKey2);
|
|
303387
303822
|
} else {
|
|
303388
303823
|
timing.cache_hit = true;
|
|
@@ -303419,6 +303854,11 @@ async function resolveAndExecute2(intent, params = {}, context, projection, opti
|
|
|
303419
303854
|
persistRouteCache2();
|
|
303420
303855
|
continue;
|
|
303421
303856
|
}
|
|
303857
|
+
if (!cachedSkillHostMatchesContext2(cached6.domain, context?.url)) {
|
|
303858
|
+
skillRouteCache2.delete(scopedKey2);
|
|
303859
|
+
persistRouteCache2();
|
|
303860
|
+
continue;
|
|
303861
|
+
}
|
|
303422
303862
|
const skill = readSkillSnapshot2(cached6.localSkillPath) ?? await getSkillWithTimeout2(cached6.skillId, clientScope);
|
|
303423
303863
|
if (!skill || !isCachedSkillRelevantForIntent2(skill, queryIntent, context?.url)) {
|
|
303424
303864
|
skillRouteCache2.delete(scopedKey2);
|
|
@@ -303521,6 +303961,11 @@ async function resolveAndExecute2(intent, params = {}, context, projection, opti
|
|
|
303521
303961
|
persistRouteCache2();
|
|
303522
303962
|
continue;
|
|
303523
303963
|
}
|
|
303964
|
+
if (!cachedSkillHostMatchesContext2(cached7.domain, context?.url)) {
|
|
303965
|
+
skillRouteCache2.delete(scopedKey2);
|
|
303966
|
+
persistRouteCache2();
|
|
303967
|
+
continue;
|
|
303968
|
+
}
|
|
303524
303969
|
const skill = readSkillSnapshot2(cached7.localSkillPath) ?? await getSkillWithTimeout2(cached7.skillId, clientScope);
|
|
303525
303970
|
if (!skill)
|
|
303526
303971
|
continue;
|