unbrowse 2.10.2 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -7
- package/bin/unbrowse-wrapper.mjs +14 -30
- package/dist/cli.js +1802 -464
- package/package.json +5 -2
- package/runtime-src/api/browse-index.ts +254 -0
- package/runtime-src/api/browse-session.ts +177 -0
- package/runtime-src/api/browse-submit.ts +433 -0
- package/runtime-src/api/routes.ts +263 -242
- package/runtime-src/auth/index.ts +68 -23
- package/runtime-src/cli.ts +123 -24
- package/runtime-src/client/index.ts +1 -1
- package/runtime-src/execution/index.ts +55 -10
- package/runtime-src/execution/robots.ts +167 -0
- package/runtime-src/kuri/client.ts +281 -89
- package/runtime-src/orchestrator/index.ts +65 -38
- package/runtime-src/orchestrator/timing-economics.ts +80 -0
- package/runtime-src/reverse-engineer/index.ts +25 -3
- package/runtime-src/runtime/setup.ts +3 -3
- package/runtime-src/types/skill.ts +2 -0
package/dist/cli.js
CHANGED
|
@@ -241,8 +241,85 @@ var init_logger = __esm(() => {
|
|
|
241
241
|
});
|
|
242
242
|
|
|
243
243
|
// ../../src/kuri/client.ts
|
|
244
|
+
var exports_client = {};
|
|
245
|
+
__export(exports_client, {
|
|
246
|
+
waitForSelector: () => waitForSelector,
|
|
247
|
+
waitForLoad: () => waitForLoad,
|
|
248
|
+
waitForCloudflare: () => waitForCloudflare,
|
|
249
|
+
stop: () => stop,
|
|
250
|
+
start: () => start,
|
|
251
|
+
snapshot: () => snapshot,
|
|
252
|
+
setViewport: () => setViewport,
|
|
253
|
+
setUserAgent: () => setUserAgent,
|
|
254
|
+
setHeaders: () => setHeaders,
|
|
255
|
+
setCredentials: () => setCredentials,
|
|
256
|
+
setCookies: () => setCookies,
|
|
257
|
+
setCookie: () => setCookie,
|
|
258
|
+
sessionSave: () => sessionSave,
|
|
259
|
+
sessionLoad: () => sessionLoad,
|
|
260
|
+
sessionList: () => sessionList,
|
|
261
|
+
select: () => select,
|
|
262
|
+
scrollIntoView: () => scrollIntoView,
|
|
263
|
+
scroll: () => scroll,
|
|
264
|
+
scriptInject: () => scriptInject,
|
|
265
|
+
screenshot: () => screenshot,
|
|
266
|
+
resolveKuriPort: () => resolveKuriPort,
|
|
267
|
+
reload: () => reload,
|
|
268
|
+
press: () => press,
|
|
269
|
+
newTab: () => newTab,
|
|
270
|
+
networkEnable: () => networkEnable,
|
|
271
|
+
navigate: () => navigate,
|
|
272
|
+
keyboardType: () => keyboardType,
|
|
273
|
+
keyboardInsertText: () => keyboardInsertText,
|
|
274
|
+
keyUp: () => keyUp,
|
|
275
|
+
keyDown: () => keyDown,
|
|
276
|
+
isReady: () => isReady,
|
|
277
|
+
interceptStart: () => interceptStart,
|
|
278
|
+
health: () => health,
|
|
279
|
+
hasCloudflareChallenge: () => hasCloudflareChallenge,
|
|
280
|
+
harStop: () => harStop,
|
|
281
|
+
harStart: () => harStart,
|
|
282
|
+
goForward: () => goForward,
|
|
283
|
+
goBack: () => goBack,
|
|
284
|
+
getText: () => getText,
|
|
285
|
+
getPort: () => getPort,
|
|
286
|
+
getPerfLcp: () => getPerfLcp,
|
|
287
|
+
getPageHtml: () => getPageHtml,
|
|
288
|
+
getNetworkEvents: () => getNetworkEvents,
|
|
289
|
+
getMarkdown: () => getMarkdown,
|
|
290
|
+
getLinks: () => getLinks,
|
|
291
|
+
getKuriSourceCandidates: () => getKuriSourceCandidates,
|
|
292
|
+
getKuriErrorMessage: () => getKuriErrorMessage,
|
|
293
|
+
getKuriBinaryCandidates: () => getKuriBinaryCandidates,
|
|
294
|
+
getErrors: () => getErrors,
|
|
295
|
+
getDefaultTab: () => getDefaultTab,
|
|
296
|
+
getCurrentUrl: () => getCurrentUrl,
|
|
297
|
+
getCookies: () => getCookies,
|
|
298
|
+
getConsole: () => getConsole,
|
|
299
|
+
findText: () => findText,
|
|
300
|
+
findKuriBinary: () => findKuriBinary,
|
|
301
|
+
fill: () => fill,
|
|
302
|
+
extractLoadPluginsFromHtml: () => extractLoadPluginsFromHtml,
|
|
303
|
+
extractLoadPlugins: () => extractLoadPlugins,
|
|
304
|
+
executeInPageFetch: () => executeInPageFetch,
|
|
305
|
+
evaluate: () => evaluate,
|
|
306
|
+
drag: () => drag,
|
|
307
|
+
domQuery: () => domQuery,
|
|
308
|
+
domHtml: () => domHtml,
|
|
309
|
+
domAttributes: () => domAttributes,
|
|
310
|
+
discoverTabs: () => discoverTabs,
|
|
311
|
+
closeTab: () => closeTab,
|
|
312
|
+
click: () => click,
|
|
313
|
+
bestEffortRehydratePlugins: () => bestEffortRehydratePlugins,
|
|
314
|
+
authProfileSave: () => authProfileSave,
|
|
315
|
+
authProfileLoad: () => authProfileLoad,
|
|
316
|
+
authProfileList: () => authProfileList,
|
|
317
|
+
authProfileDelete: () => authProfileDelete,
|
|
318
|
+
action: () => action
|
|
319
|
+
});
|
|
244
320
|
import { execFileSync, spawn } from "node:child_process";
|
|
245
321
|
import { existsSync as existsSync3 } from "node:fs";
|
|
322
|
+
import net from "node:net";
|
|
246
323
|
import path3 from "node:path";
|
|
247
324
|
function kuriBinaryName() {
|
|
248
325
|
return process.platform === "win32" ? "kuri.exe" : "kuri";
|
|
@@ -316,6 +393,46 @@ async function discoverCdpPort() {
|
|
|
316
393
|
}
|
|
317
394
|
log("kuri", "could not discover CDP port — tab discovery may fail");
|
|
318
395
|
}
|
|
396
|
+
async function isKuriHealthyOnPort(port) {
|
|
397
|
+
try {
|
|
398
|
+
const health = await fetch(`http://127.0.0.1:${port}/health`, {
|
|
399
|
+
signal: AbortSignal.timeout(1000)
|
|
400
|
+
});
|
|
401
|
+
return health.ok;
|
|
402
|
+
} catch {
|
|
403
|
+
return false;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
async function isTcpPortOpen(port, timeoutMs = 400) {
|
|
407
|
+
return await new Promise((resolve) => {
|
|
408
|
+
const socket = net.createConnection({ host: "127.0.0.1", port });
|
|
409
|
+
const finish = (open) => {
|
|
410
|
+
socket.removeAllListeners();
|
|
411
|
+
socket.destroy();
|
|
412
|
+
resolve(open);
|
|
413
|
+
};
|
|
414
|
+
socket.setTimeout(timeoutMs);
|
|
415
|
+
socket.once("connect", () => finish(true));
|
|
416
|
+
socket.once("timeout", () => finish(false));
|
|
417
|
+
socket.once("error", () => finish(false));
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
async function resolveKuriPort(preferredPort, deps = {}) {
|
|
421
|
+
const isHealthyPort = deps.isHealthyPort ?? isKuriHealthyOnPort;
|
|
422
|
+
const isPortOpen = deps.isPortOpen ?? isTcpPortOpen;
|
|
423
|
+
const searchLimit = deps.searchLimit ?? KURI_PORT_SEARCH_LIMIT;
|
|
424
|
+
if (await isHealthyPort(preferredPort))
|
|
425
|
+
return preferredPort;
|
|
426
|
+
if (!await isPortOpen(preferredPort))
|
|
427
|
+
return preferredPort;
|
|
428
|
+
for (let candidate = preferredPort + 1;candidate <= preferredPort + searchLimit; candidate++) {
|
|
429
|
+
if (await isHealthyPort(candidate))
|
|
430
|
+
return candidate;
|
|
431
|
+
if (!await isPortOpen(candidate))
|
|
432
|
+
return candidate;
|
|
433
|
+
}
|
|
434
|
+
return preferredPort;
|
|
435
|
+
}
|
|
319
436
|
function kuriUrl(path4, params) {
|
|
320
437
|
const base = `http://127.0.0.1:${kuriPort}${path4}`;
|
|
321
438
|
if (!params || Object.keys(params).length === 0)
|
|
@@ -382,105 +499,118 @@ async function waitForChildExit(child, timeoutMs = 2000) {
|
|
|
382
499
|
async function start(port) {
|
|
383
500
|
if (kuriReady)
|
|
384
501
|
return;
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
if (
|
|
502
|
+
if (kuriStartPromise)
|
|
503
|
+
return kuriStartPromise;
|
|
504
|
+
const startPromise = (async () => {
|
|
505
|
+
const requestedPort = port ?? Number(process.env.KURI_PORT || KURI_DEFAULT_PORT);
|
|
506
|
+
kuriPort = await resolveKuriPort(requestedPort);
|
|
507
|
+
if (kuriPort !== requestedPort) {
|
|
508
|
+
log("kuri", `preferred port ${requestedPort} is occupied but unhealthy; falling back to ${kuriPort}`);
|
|
509
|
+
}
|
|
510
|
+
if (await isKuriHealthyOnPort(kuriPort)) {
|
|
391
511
|
log("kuri", `already running on port ${kuriPort}`);
|
|
392
512
|
kuriReady = true;
|
|
393
513
|
await discoverCdpPort();
|
|
394
514
|
await ensureTabsDiscovered();
|
|
395
515
|
return;
|
|
396
516
|
}
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
await new Promise((r) => setTimeout(r, KURI_SPAWN_RETRY_DELAY_MS));
|
|
421
|
-
}
|
|
422
|
-
let exitedBeforeReady = false;
|
|
423
|
-
kuriProcess = spawn(binary, [], {
|
|
424
|
-
env,
|
|
425
|
-
stdio: ["ignore", "pipe", "pipe"]
|
|
426
|
-
});
|
|
427
|
-
kuriProcess.stderr?.on("data", (chunk) => {
|
|
428
|
-
const line = chunk.toString().trim();
|
|
429
|
-
if (line)
|
|
430
|
-
log("kuri", `[stderr] ${line}`);
|
|
431
|
-
const cdpMatch = line.match(/CDP port:\s*(\d+)/);
|
|
432
|
-
if (cdpMatch) {
|
|
433
|
-
kuriCdpPort = parseInt(cdpMatch[1], 10);
|
|
434
|
-
log("kuri", `discovered CDP port: ${kuriCdpPort}`);
|
|
517
|
+
const binary = findKuriBinary();
|
|
518
|
+
log("kuri", `starting: ${binary} on port ${kuriPort}`);
|
|
519
|
+
if (!existsSync3(binary)) {
|
|
520
|
+
throw new Error(`Kuri binary not found at ${binary}`);
|
|
521
|
+
}
|
|
522
|
+
await discoverCdpPort();
|
|
523
|
+
const env = {
|
|
524
|
+
...process.env,
|
|
525
|
+
PORT: String(kuriPort),
|
|
526
|
+
HOST: "127.0.0.1",
|
|
527
|
+
HEADLESS: "false"
|
|
528
|
+
};
|
|
529
|
+
if (kuriCdpPort) {
|
|
530
|
+
env.CDP_URL = `ws://127.0.0.1:${kuriCdpPort}`;
|
|
531
|
+
log("kuri", `connecting to existing Chrome on port ${kuriCdpPort}`);
|
|
532
|
+
} else {
|
|
533
|
+
log("kuri", "no existing Chrome found — Kuri will launch managed Chrome");
|
|
534
|
+
}
|
|
535
|
+
const maxAttempts = KURI_SPAWN_RETRIES + 1;
|
|
536
|
+
for (let attempt = 1;attempt <= maxAttempts; attempt++) {
|
|
537
|
+
if (attempt > 1) {
|
|
538
|
+
log("kuri", `spawn retry ${attempt}/${maxAttempts} after ${KURI_SPAWN_RETRY_DELAY_MS}ms`);
|
|
539
|
+
await new Promise((r) => setTimeout(r, KURI_SPAWN_RETRY_DELAY_MS));
|
|
435
540
|
}
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
const res = await fetch(`http://127.0.0.1:${kuriPort}/health`, {
|
|
450
|
-
signal: AbortSignal.timeout(500)
|
|
451
|
-
});
|
|
452
|
-
if (res.ok) {
|
|
453
|
-
kuriReady = true;
|
|
454
|
-
log("kuri", `ready on port ${kuriPort}`);
|
|
455
|
-
await new Promise((r) => setTimeout(r, 300));
|
|
456
|
-
if (!kuriCdpPort)
|
|
457
|
-
await discoverCdpPort();
|
|
458
|
-
await ensureTabsDiscovered();
|
|
459
|
-
return;
|
|
541
|
+
let exitedBeforeReady = false;
|
|
542
|
+
kuriProcess = spawn(binary, [], {
|
|
543
|
+
env,
|
|
544
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
545
|
+
});
|
|
546
|
+
kuriProcess.stderr?.on("data", (chunk) => {
|
|
547
|
+
const line = chunk.toString().trim();
|
|
548
|
+
if (line)
|
|
549
|
+
log("kuri", `[stderr] ${line}`);
|
|
550
|
+
const cdpMatch = line.match(/CDP port:\s*(\d+)/);
|
|
551
|
+
if (cdpMatch) {
|
|
552
|
+
kuriCdpPort = parseInt(cdpMatch[1], 10);
|
|
553
|
+
log("kuri", `discovered CDP port: ${kuriCdpPort}`);
|
|
460
554
|
}
|
|
555
|
+
});
|
|
556
|
+
kuriProcess.on("exit", (code) => {
|
|
557
|
+
if (!kuriReady)
|
|
558
|
+
exitedBeforeReady = true;
|
|
559
|
+
log("kuri", `process exited with code ${code}`);
|
|
560
|
+
kuriReady = false;
|
|
561
|
+
kuriProcess = null;
|
|
562
|
+
});
|
|
563
|
+
const deadline = Date.now() + KURI_STARTUP_TIMEOUT_MS;
|
|
564
|
+
while (Date.now() < deadline) {
|
|
565
|
+
if (exitedBeforeReady)
|
|
566
|
+
break;
|
|
567
|
+
try {
|
|
568
|
+
const res = await fetch(`http://127.0.0.1:${kuriPort}/health`, {
|
|
569
|
+
signal: AbortSignal.timeout(500)
|
|
570
|
+
});
|
|
571
|
+
if (res.ok) {
|
|
572
|
+
kuriReady = true;
|
|
573
|
+
log("kuri", `ready on port ${kuriPort}`);
|
|
574
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
575
|
+
if (!kuriCdpPort)
|
|
576
|
+
await discoverCdpPort();
|
|
577
|
+
await ensureTabsDiscovered();
|
|
578
|
+
return;
|
|
579
|
+
}
|
|
580
|
+
} catch {}
|
|
581
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
582
|
+
}
|
|
583
|
+
if (kuriReady)
|
|
584
|
+
return;
|
|
585
|
+
if (kuriProcess) {
|
|
586
|
+
kuriProcess.kill();
|
|
587
|
+
await waitForChildExit(kuriProcess);
|
|
588
|
+
}
|
|
589
|
+
try {
|
|
590
|
+
execFileSync("pkill", ["-f", `remote-debugging-port=${kuriCdpPort ?? 9222}`], { stdio: "ignore" });
|
|
591
|
+
await new Promise((r) => setTimeout(r, 1000));
|
|
461
592
|
} catch {}
|
|
462
|
-
await new Promise((r) => setTimeout(r, 200));
|
|
463
593
|
}
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
594
|
+
throw new Error(`Kuri failed to start after ${maxAttempts} attempts`);
|
|
595
|
+
})();
|
|
596
|
+
kuriStartPromise = startPromise.finally(() => {
|
|
597
|
+
if (kuriStartPromise === startPromise) {
|
|
598
|
+
kuriStartPromise = null;
|
|
469
599
|
}
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
await new Promise((r) => setTimeout(r, 1000));
|
|
473
|
-
} catch {}
|
|
474
|
-
}
|
|
475
|
-
throw new Error(`Kuri failed to start after ${maxAttempts} attempts`);
|
|
600
|
+
});
|
|
601
|
+
return kuriStartPromise;
|
|
476
602
|
}
|
|
477
603
|
async function stop() {
|
|
604
|
+
if (kuriStartPromise) {
|
|
605
|
+
await kuriStartPromise.catch(() => {});
|
|
606
|
+
}
|
|
478
607
|
if (kuriProcess) {
|
|
479
608
|
kuriProcess.kill("SIGTERM");
|
|
480
609
|
kuriProcess = null;
|
|
481
610
|
}
|
|
482
611
|
kuriReady = false;
|
|
483
612
|
kuriCdpPort = null;
|
|
613
|
+
kuriStartPromise = null;
|
|
484
614
|
}
|
|
485
615
|
async function discoverTabs() {
|
|
486
616
|
await ensureTabsDiscovered();
|
|
@@ -574,6 +704,25 @@ async function evaluate(tabId, expression) {
|
|
|
574
704
|
return inner.value;
|
|
575
705
|
return inner.description ?? raw;
|
|
576
706
|
}
|
|
707
|
+
function getKuriErrorMessage(value) {
|
|
708
|
+
if (typeof value === "string")
|
|
709
|
+
return null;
|
|
710
|
+
if (!value || typeof value !== "object")
|
|
711
|
+
return null;
|
|
712
|
+
const record = value;
|
|
713
|
+
if (typeof record.error === "string")
|
|
714
|
+
return record.error;
|
|
715
|
+
if (typeof record.message === "string")
|
|
716
|
+
return record.message;
|
|
717
|
+
if (record.result && typeof record.result === "object") {
|
|
718
|
+
const nested = record.result;
|
|
719
|
+
if (typeof nested.error === "string")
|
|
720
|
+
return nested.error;
|
|
721
|
+
if (typeof nested.message === "string")
|
|
722
|
+
return nested.message;
|
|
723
|
+
}
|
|
724
|
+
return null;
|
|
725
|
+
}
|
|
577
726
|
async function getCookies(tabId) {
|
|
578
727
|
const raw = await kuriGet("/cookies", { tab_id: tabId });
|
|
579
728
|
return raw?.result?.cookies ?? [];
|
|
@@ -672,6 +821,9 @@ async function harStop(tabId) {
|
|
|
672
821
|
async function networkEnable(tabId) {
|
|
673
822
|
await kuriGet("/network", { tab_id: tabId, mode: "enable" });
|
|
674
823
|
}
|
|
824
|
+
async function interceptStart(tabId) {
|
|
825
|
+
await kuriGet("/intercept/start", { tab_id: tabId });
|
|
826
|
+
}
|
|
675
827
|
async function getText(tabId) {
|
|
676
828
|
const result = await kuriGet("/text", { tab_id: tabId });
|
|
677
829
|
return result?.text ?? "";
|
|
@@ -713,12 +865,106 @@ async function newTab(url) {
|
|
|
713
865
|
}
|
|
714
866
|
async function getCurrentUrl(tabId) {
|
|
715
867
|
const result = await evaluate(tabId, "window.location.href");
|
|
716
|
-
return
|
|
868
|
+
return typeof result === "string" ? result : "";
|
|
717
869
|
}
|
|
718
870
|
async function getPageHtml(tabId) {
|
|
719
871
|
const result = await evaluate(tabId, "document.documentElement.outerHTML");
|
|
720
872
|
return String(result ?? "");
|
|
721
873
|
}
|
|
874
|
+
function extractLoadPlugins(value) {
|
|
875
|
+
if (typeof value !== "string")
|
|
876
|
+
return [];
|
|
877
|
+
return Array.from(new Set(value.split(/[\s,;]+/).map((part) => part.trim()).filter(Boolean)));
|
|
878
|
+
}
|
|
879
|
+
function extractLoadPluginsFromHtml(html) {
|
|
880
|
+
const modules = [];
|
|
881
|
+
const pattern = /data-load-plugins=(["'])(.*?)\1/gi;
|
|
882
|
+
for (const match of html.matchAll(pattern)) {
|
|
883
|
+
modules.push(...extractLoadPlugins(match[2]));
|
|
884
|
+
}
|
|
885
|
+
return Array.from(new Set(modules));
|
|
886
|
+
}
|
|
887
|
+
async function bestEffortRehydratePlugins(tabId) {
|
|
888
|
+
const result = await evaluate(tabId, `(async function() {
|
|
889
|
+
function splitPlugins(value) {
|
|
890
|
+
return String(value || "")
|
|
891
|
+
.split(/[\\s,;]+/)
|
|
892
|
+
.map(function(part) { return part.trim(); })
|
|
893
|
+
.filter(Boolean);
|
|
894
|
+
}
|
|
895
|
+
function pluginPath(name) {
|
|
896
|
+
if (/^https?:\\/\\//i.test(name) || name.startsWith("/")) return name;
|
|
897
|
+
return "/etc/designs/wrs/footLibs/js/plugins/" + (name.endsWith(".js") ? name : name + ".js");
|
|
898
|
+
}
|
|
899
|
+
var modules = Array.from(new Set(
|
|
900
|
+
Array.from(document.querySelectorAll("[data-load-plugins]"))
|
|
901
|
+
.flatMap(function(node) { return splitPlugins(node.getAttribute("data-load-plugins")); })
|
|
902
|
+
));
|
|
903
|
+
if (modules.length === 0) {
|
|
904
|
+
return JSON.stringify({ attempted: false, loaded: false, nooped: true, reason: "no_plugins", modules: [] });
|
|
905
|
+
}
|
|
906
|
+
if (!window.WRS || typeof window.WRS.require !== "function") {
|
|
907
|
+
return JSON.stringify({ attempted: false, loaded: false, nooped: true, reason: "missing_wrs_require", modules: modules });
|
|
908
|
+
}
|
|
909
|
+
var requireWrs = window.WRS.require.bind(window.WRS);
|
|
910
|
+
async function loadModules(paths) {
|
|
911
|
+
return await new Promise(function(resolve) {
|
|
912
|
+
var done = false;
|
|
913
|
+
var timer = setTimeout(function() {
|
|
914
|
+
if (done) return;
|
|
915
|
+
done = true;
|
|
916
|
+
resolve({ ok: false, reason: "timeout" });
|
|
917
|
+
}, 1500);
|
|
918
|
+
try {
|
|
919
|
+
requireWrs(paths, function() {
|
|
920
|
+
if (done) return;
|
|
921
|
+
done = true;
|
|
922
|
+
clearTimeout(timer);
|
|
923
|
+
resolve({ ok: true });
|
|
924
|
+
});
|
|
925
|
+
} catch (error) {
|
|
926
|
+
if (done) return;
|
|
927
|
+
done = true;
|
|
928
|
+
clearTimeout(timer);
|
|
929
|
+
resolve({ ok: false, reason: error && error.message ? error.message : String(error) });
|
|
930
|
+
}
|
|
931
|
+
});
|
|
932
|
+
}
|
|
933
|
+
var configResult = await loadModules(["/etc/designs/wrs/footLibs/js/config.js"]);
|
|
934
|
+
var pluginResult = await loadModules(modules.map(pluginPath));
|
|
935
|
+
for (var i = 0; i < 6; i++) {
|
|
936
|
+
await new Promise(function(resolve) { return setTimeout(resolve, 100); });
|
|
937
|
+
}
|
|
938
|
+
return JSON.stringify({
|
|
939
|
+
attempted: true,
|
|
940
|
+
loaded: !!pluginResult.ok,
|
|
941
|
+
nooped: false,
|
|
942
|
+
reason: pluginResult.ok ? undefined : pluginResult.reason,
|
|
943
|
+
config_loaded: !!configResult.ok,
|
|
944
|
+
modules: modules,
|
|
945
|
+
});
|
|
946
|
+
})()`);
|
|
947
|
+
if (typeof result !== "string") {
|
|
948
|
+
return {
|
|
949
|
+
attempted: false,
|
|
950
|
+
loaded: false,
|
|
951
|
+
nooped: true,
|
|
952
|
+
reason: "invalid_rehydrate_result",
|
|
953
|
+
modules: []
|
|
954
|
+
};
|
|
955
|
+
}
|
|
956
|
+
try {
|
|
957
|
+
return JSON.parse(result);
|
|
958
|
+
} catch {
|
|
959
|
+
return {
|
|
960
|
+
attempted: false,
|
|
961
|
+
loaded: false,
|
|
962
|
+
nooped: true,
|
|
963
|
+
reason: "invalid_rehydrate_result",
|
|
964
|
+
modules: []
|
|
965
|
+
};
|
|
966
|
+
}
|
|
967
|
+
}
|
|
722
968
|
async function hasCloudflareChallenge(tabId) {
|
|
723
969
|
const result = await evaluate(tabId, `(function() {
|
|
724
970
|
var html = document.documentElement.innerHTML;
|
|
@@ -765,6 +1011,20 @@ async function executeInPageFetch(tabId, url, method, headers, body) {
|
|
|
765
1011
|
return { status: 0, data: result };
|
|
766
1012
|
}
|
|
767
1013
|
}
|
|
1014
|
+
async function health() {
|
|
1015
|
+
try {
|
|
1016
|
+
const result = await kuriGet("/health");
|
|
1017
|
+
return { ok: result?.ok === true || result?.status === "ok", tabs: result?.tabs };
|
|
1018
|
+
} catch {
|
|
1019
|
+
return { ok: false };
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
function getPort() {
|
|
1023
|
+
return kuriPort;
|
|
1024
|
+
}
|
|
1025
|
+
function isReady() {
|
|
1026
|
+
return kuriReady;
|
|
1027
|
+
}
|
|
768
1028
|
async function action(tabId, actionType, ref, value) {
|
|
769
1029
|
const params = { tab_id: tabId, action: actionType, ref };
|
|
770
1030
|
if (value !== undefined)
|
|
@@ -836,25 +1096,99 @@ async function waitForLoad(tabId, timeoutMs) {
|
|
|
836
1096
|
async function keyboardType(tabId, text) {
|
|
837
1097
|
return kuriGet("/keyboard/type", { tab_id: tabId, text });
|
|
838
1098
|
}
|
|
1099
|
+
async function keyboardInsertText(tabId, text) {
|
|
1100
|
+
return kuriGet("/keyboard/inserttext", { tab_id: tabId, text });
|
|
1101
|
+
}
|
|
1102
|
+
async function keyDown(tabId, key) {
|
|
1103
|
+
return kuriGet("/keydown", { tab_id: tabId, key });
|
|
1104
|
+
}
|
|
1105
|
+
async function keyUp(tabId, key) {
|
|
1106
|
+
return kuriGet("/keyup", { tab_id: tabId, key });
|
|
1107
|
+
}
|
|
839
1108
|
async function scrollIntoView(tabId, ref) {
|
|
840
1109
|
return kuriGet("/scrollintoview", { tab_id: tabId, ref });
|
|
841
1110
|
}
|
|
1111
|
+
async function drag(tabId, sourceRef, targetRef) {
|
|
1112
|
+
return kuriGet("/drag", { tab_id: tabId, source: sourceRef, target: targetRef });
|
|
1113
|
+
}
|
|
1114
|
+
async function domQuery(tabId, selector, all = false) {
|
|
1115
|
+
const params = { tab_id: tabId, selector };
|
|
1116
|
+
if (all)
|
|
1117
|
+
params.all = "true";
|
|
1118
|
+
return await kuriGet("/dom/query", params);
|
|
1119
|
+
}
|
|
1120
|
+
async function domHtml(tabId, nodeId) {
|
|
1121
|
+
return kuriGet("/dom/html", { tab_id: tabId, node_id: String(nodeId) });
|
|
1122
|
+
}
|
|
1123
|
+
async function domAttributes(tabId, opts) {
|
|
1124
|
+
const params = { tab_id: tabId };
|
|
1125
|
+
if (opts.ref)
|
|
1126
|
+
params.ref = opts.ref;
|
|
1127
|
+
if (opts.selector)
|
|
1128
|
+
params.selector = opts.selector;
|
|
1129
|
+
return kuriGet("/dom/attributes", params);
|
|
1130
|
+
}
|
|
842
1131
|
async function scriptInject(tabId, source) {
|
|
843
1132
|
return kuriPost("/script/inject", { tab_id: tabId }, { source });
|
|
844
1133
|
}
|
|
1134
|
+
async function setCredentials(tabId, username, password) {
|
|
1135
|
+
return kuriGet("/set/credentials", { tab_id: tabId, username, password });
|
|
1136
|
+
}
|
|
1137
|
+
async function setViewport(tabId, width, height) {
|
|
1138
|
+
return kuriGet("/set/viewport", { tab_id: tabId, width: String(width), height: String(height) });
|
|
1139
|
+
}
|
|
1140
|
+
async function setUserAgent(tabId, ua) {
|
|
1141
|
+
return kuriGet("/set/useragent", { tab_id: tabId, ua });
|
|
1142
|
+
}
|
|
1143
|
+
async function sessionSave() {
|
|
1144
|
+
return kuriGet("/session/save");
|
|
1145
|
+
}
|
|
1146
|
+
async function sessionLoad(state) {
|
|
1147
|
+
return kuriPost("/session/load", {}, state);
|
|
1148
|
+
}
|
|
1149
|
+
async function sessionList() {
|
|
1150
|
+
return kuriGet("/session/list");
|
|
1151
|
+
}
|
|
845
1152
|
async function goBack(tabId) {
|
|
846
1153
|
return kuriGet("/back", { tab_id: tabId });
|
|
847
1154
|
}
|
|
848
1155
|
async function goForward(tabId) {
|
|
849
1156
|
return kuriGet("/forward", { tab_id: tabId });
|
|
850
1157
|
}
|
|
1158
|
+
async function reload(tabId) {
|
|
1159
|
+
return kuriGet("/reload", { tab_id: tabId });
|
|
1160
|
+
}
|
|
1161
|
+
async function getNetworkEvents(tabId) {
|
|
1162
|
+
return kuriGet("/network", { tab_id: tabId });
|
|
1163
|
+
}
|
|
1164
|
+
async function getPerfLcp(tabId) {
|
|
1165
|
+
return kuriGet("/perf/lcp", { tab_id: tabId });
|
|
1166
|
+
}
|
|
1167
|
+
async function findText(tabId, query) {
|
|
1168
|
+
return kuriGet("/find", { tab_id: tabId, query });
|
|
1169
|
+
}
|
|
1170
|
+
async function getLinks(tabId) {
|
|
1171
|
+
return kuriGet("/links", { tab_id: tabId });
|
|
1172
|
+
}
|
|
1173
|
+
async function getConsole(tabId) {
|
|
1174
|
+
return kuriGet("/console", { tab_id: tabId });
|
|
1175
|
+
}
|
|
1176
|
+
async function getErrors(tabId) {
|
|
1177
|
+
return kuriGet("/errors", { tab_id: tabId });
|
|
1178
|
+
}
|
|
851
1179
|
async function authProfileSave(tabId, name) {
|
|
852
1180
|
return kuriGet("/auth/profile/save", { tab_id: tabId, name });
|
|
853
1181
|
}
|
|
854
1182
|
async function authProfileLoad(tabId, name) {
|
|
855
1183
|
return kuriGet("/auth/profile/load", { tab_id: tabId, name });
|
|
856
1184
|
}
|
|
857
|
-
|
|
1185
|
+
async function authProfileList(tabId) {
|
|
1186
|
+
return kuriGet("/auth/profile/list", { tab_id: tabId });
|
|
1187
|
+
}
|
|
1188
|
+
async function authProfileDelete(name) {
|
|
1189
|
+
return kuriGet("/auth/profile/delete", { name });
|
|
1190
|
+
}
|
|
1191
|
+
var KURI_DEFAULT_PORT = 7700, KURI_STARTUP_TIMEOUT_MS = 1e4, KURI_REQUEST_TIMEOUT_MS = 30000, KURI_SPAWN_RETRIES = 3, KURI_SPAWN_RETRY_DELAY_MS = 1000, KURI_PORT_SEARCH_LIMIT = 10, kuriProcess = null, kuriPort, kuriCdpPort = null, kuriReady = false, kuriStartPromise = null;
|
|
858
1192
|
var init_client = __esm(() => {
|
|
859
1193
|
init_logger();
|
|
860
1194
|
init_paths();
|
|
@@ -1277,20 +1611,6 @@ function mergeContextTemplateParams(params, urlTemplate, contextUrl) {
|
|
|
1277
1611
|
}
|
|
1278
1612
|
|
|
1279
1613
|
// ../../src/graph/index.ts
|
|
1280
|
-
var exports_graph = {};
|
|
1281
|
-
__export(exports_graph, {
|
|
1282
|
-
toAgentSkillChunkView: () => toAgentSkillChunkView,
|
|
1283
|
-
resolveEndpointSemantic: () => resolveEndpointSemantic,
|
|
1284
|
-
operationSoftPenalty: () => operationSoftPenalty,
|
|
1285
|
-
knownBindingsFromInputs: () => knownBindingsFromInputs,
|
|
1286
|
-
isRunnable: () => isRunnable,
|
|
1287
|
-
isOperationHardExcluded: () => isOperationHardExcluded,
|
|
1288
|
-
inferEndpointSemantic: () => inferEndpointSemantic,
|
|
1289
|
-
getSkillChunk: () => getSkillChunk,
|
|
1290
|
-
ensureSkillOperationGraph: () => ensureSkillOperationGraph,
|
|
1291
|
-
computeReachableEndpoints: () => computeReachableEndpoints,
|
|
1292
|
-
buildSkillOperationGraph: () => buildSkillOperationGraph
|
|
1293
|
-
});
|
|
1294
1614
|
function normalizeTokenText(text) {
|
|
1295
1615
|
return text.replace(/([a-z0-9])([A-Z])/g, "$1 $2").replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2").replace(/([a-zA-Z])(\d)/g, "$1 $2").replace(/(\d)([a-zA-Z])/g, "$1 $2");
|
|
1296
1616
|
}
|
|
@@ -2654,8 +2974,8 @@ function templatizeBodyObject(value, context, path4 = "", bodyParams = {}) {
|
|
|
2654
2974
|
function inferCsrfPlan(req, parsedBody) {
|
|
2655
2975
|
const headers = Object.fromEntries(Object.entries(req.request_headers).map(([key, value]) => [key.toLowerCase(), value]));
|
|
2656
2976
|
const cookies = parseCookieHeader(headers["cookie"]);
|
|
2657
|
-
const csrfCookieNames = Object.keys(cookies).filter((name) => /^(ct0|csrf_token|_csrf|csrftoken|xsrf-token|_xsrf)$/i.test(name));
|
|
2658
|
-
const headerName = ["x-csrf-token", "x-xsrf-token", "x-csrftoken"].find((name) => typeof headers[name] === "string" && headers[name].length > 0);
|
|
2977
|
+
const csrfCookieNames = Object.keys(cookies).filter((name) => /^(ct0|csrf_token|_csrf|csrftoken|xsrf-token|_xsrf|jsessionid)$/i.test(name));
|
|
2978
|
+
const headerName = ["x-csrf-token", "x-xsrf-token", "x-csrftoken", "csrf-token"].find((name) => typeof headers[name] === "string" && headers[name].length > 0);
|
|
2659
2979
|
if (headerName && csrfCookieNames.length > 0) {
|
|
2660
2980
|
return {
|
|
2661
2981
|
source: "cookie",
|
|
@@ -3165,6 +3485,15 @@ function isSensitiveHeader(name) {
|
|
|
3165
3485
|
return true;
|
|
3166
3486
|
return false;
|
|
3167
3487
|
}
|
|
3488
|
+
function isReplayCriticalHeader(name, value) {
|
|
3489
|
+
const lower = name.toLowerCase();
|
|
3490
|
+
if (REPLAY_HEADER_EXACT.has(lower)) {
|
|
3491
|
+
if (lower !== "accept")
|
|
3492
|
+
return true;
|
|
3493
|
+
return /application\/vnd\./i.test(value);
|
|
3494
|
+
}
|
|
3495
|
+
return REPLAY_HEADER_PREFIXES.some((prefix) => lower.startsWith(prefix));
|
|
3496
|
+
}
|
|
3168
3497
|
function sanitizeHeaders(headers) {
|
|
3169
3498
|
return Object.fromEntries(Object.entries(headers ?? {}).filter(([k]) => {
|
|
3170
3499
|
const lower = k.toLowerCase();
|
|
@@ -3180,7 +3509,7 @@ function extractAuthHeaders(requests) {
|
|
|
3180
3509
|
const lower = k.toLowerCase();
|
|
3181
3510
|
if (lower === "cookie" || lower === "content-length" || lower === "host")
|
|
3182
3511
|
continue;
|
|
3183
|
-
if (isSensitiveHeader(k) && !authHeaders[lower]) {
|
|
3512
|
+
if ((isSensitiveHeader(k) || isReplayCriticalHeader(k, v)) && !authHeaders[lower]) {
|
|
3184
3513
|
authHeaders[lower] = v;
|
|
3185
3514
|
}
|
|
3186
3515
|
}
|
|
@@ -3481,7 +3810,7 @@ function isCloudflareChallenge(responseBody) {
|
|
|
3481
3810
|
const bodyLower = responseBody.toLowerCase();
|
|
3482
3811
|
return CF_MARKERS.some((marker) => bodyLower.includes(marker.toLowerCase()));
|
|
3483
3812
|
}
|
|
3484
|
-
var SKIP_EXTENSIONS, SKIP_JS_BUNDLES, SKIP_PATHS, SKIP_HOSTS, SKIP_TELEMETRY_HOSTS, SKIP_TELEMETRY_PATHS, RPC_HINTS, ALLOWED_METHODS, STRIP_HEADERS, STRIP_HEADER_PREFIXES, SAFE_HEADERS, SENSITIVE_HEADER_PATTERN, SENSITIVE_QUERY_PARAMS, FRAMEWORK_QUERY_PARAMS, AD_HOSTS, AD_SCHEMA_KEYS, AD_SCHEMA_THRESHOLD = 3, ON_DOMAIN_NOISE;
|
|
3813
|
+
var SKIP_EXTENSIONS, SKIP_JS_BUNDLES, SKIP_PATHS, SKIP_HOSTS, SKIP_TELEMETRY_HOSTS, SKIP_TELEMETRY_PATHS, RPC_HINTS, ALLOWED_METHODS, STRIP_HEADERS, STRIP_HEADER_PREFIXES, REPLAY_HEADER_PREFIXES, REPLAY_HEADER_EXACT, SAFE_HEADERS, SENSITIVE_HEADER_PATTERN, SENSITIVE_QUERY_PARAMS, FRAMEWORK_QUERY_PARAMS, AD_HOSTS, AD_SCHEMA_KEYS, AD_SCHEMA_THRESHOLD = 3, ON_DOMAIN_NOISE;
|
|
3485
3814
|
var init_reverse_engineer = __esm(() => {
|
|
3486
3815
|
init_transform();
|
|
3487
3816
|
init_domain();
|
|
@@ -3520,6 +3849,16 @@ var init_reverse_engineer = __esm(() => {
|
|
|
3520
3849
|
"x-stripe-",
|
|
3521
3850
|
"x-firebase-"
|
|
3522
3851
|
];
|
|
3852
|
+
REPLAY_HEADER_PREFIXES = [
|
|
3853
|
+
"x-li-"
|
|
3854
|
+
];
|
|
3855
|
+
REPLAY_HEADER_EXACT = new Set([
|
|
3856
|
+
"accept",
|
|
3857
|
+
"csrf-token",
|
|
3858
|
+
"origin",
|
|
3859
|
+
"x-requested-with",
|
|
3860
|
+
"x-restli-protocol-version"
|
|
3861
|
+
]);
|
|
3523
3862
|
SAFE_HEADERS = new Set([
|
|
3524
3863
|
"accept",
|
|
3525
3864
|
"accept-encoding",
|
|
@@ -4749,6 +5088,7 @@ __export(exports_client2, {
|
|
|
4749
5088
|
getSkill: () => getSkill,
|
|
4750
5089
|
getRecentLocalSkill: () => getRecentLocalSkill,
|
|
4751
5090
|
getMyProfile: () => getMyProfile2,
|
|
5091
|
+
getLocalWalletContext: () => getLocalWalletContext2,
|
|
4752
5092
|
getEndpointSchema: () => getEndpointSchema,
|
|
4753
5093
|
getCreatorEarnings: () => getCreatorEarnings,
|
|
4754
5094
|
getApiKey: () => getApiKey2,
|
|
@@ -6338,6 +6678,32 @@ import fs2 from "node:fs";
|
|
|
6338
6678
|
function getProfilePath(domain) {
|
|
6339
6679
|
return path4.join(os3.homedir(), ".unbrowse", "profiles", getRegistrableDomain(domain));
|
|
6340
6680
|
}
|
|
6681
|
+
function assessInteractiveLoginState(input) {
|
|
6682
|
+
let parsed;
|
|
6683
|
+
try {
|
|
6684
|
+
parsed = new URL(input.currentUrl);
|
|
6685
|
+
} catch {
|
|
6686
|
+
return { status: "pending", reason: "invalid_url" };
|
|
6687
|
+
}
|
|
6688
|
+
const currentDomain = parsed.hostname.toLowerCase();
|
|
6689
|
+
const targetNorm = input.targetDomain.toLowerCase();
|
|
6690
|
+
const isOnTarget = currentDomain === targetNorm || currentDomain.endsWith(`.${targetNorm}`);
|
|
6691
|
+
if (!isOnTarget)
|
|
6692
|
+
return { status: "pending", reason: "off_target_domain" };
|
|
6693
|
+
if (input.hasCloudflareChallenge)
|
|
6694
|
+
return { status: "blocked", reason: "cloudflare_challenge" };
|
|
6695
|
+
if (input.pageText && CLOUDFLARE_TEXT.test(input.pageText))
|
|
6696
|
+
return { status: "blocked", reason: "cloudflare_text" };
|
|
6697
|
+
if (LOGIN_PATHS.test(parsed.pathname))
|
|
6698
|
+
return { status: "pending", reason: "still_on_login_path" };
|
|
6699
|
+
if (input.currentCookieCount > input.initialCookieCount) {
|
|
6700
|
+
return { status: "authenticated", reason: "new_cookies_on_target" };
|
|
6701
|
+
}
|
|
6702
|
+
if (input.currentCookieCount > 0) {
|
|
6703
|
+
return { status: "authenticated", reason: "cookies_present_on_target" };
|
|
6704
|
+
}
|
|
6705
|
+
return { status: "pending", reason: "no_session_cookies" };
|
|
6706
|
+
}
|
|
6341
6707
|
async function interactiveLogin(url, domain) {
|
|
6342
6708
|
const targetDomain = domain ?? new URL(url).hostname;
|
|
6343
6709
|
const profileDir = getProfilePath(targetDomain);
|
|
@@ -6360,6 +6726,7 @@ async function interactiveLogin(url, domain) {
|
|
|
6360
6726
|
const initialCookieCount = initialCookies.filter((c) => isDomainMatch(c.domain, targetDomain)).length;
|
|
6361
6727
|
log("auth", `initial cookies for ${targetDomain}: ${initialCookieCount}`);
|
|
6362
6728
|
let loggedIn = false;
|
|
6729
|
+
let blockedReason = null;
|
|
6363
6730
|
let lastLoggedUrl = "";
|
|
6364
6731
|
const effectiveTimeout = loginConfig.interactive ? LOGIN_TIMEOUT_MS : loginConfig.timeout_ms;
|
|
6365
6732
|
while (Date.now() - startTime < effectiveTimeout) {
|
|
@@ -6367,37 +6734,40 @@ async function interactiveLogin(url, domain) {
|
|
|
6367
6734
|
const elapsed = Date.now() - startTime;
|
|
6368
6735
|
try {
|
|
6369
6736
|
const currentUrl = await getCurrentUrl(tabId);
|
|
6370
|
-
const currentDomain = new URL(currentUrl).hostname.toLowerCase();
|
|
6371
|
-
const targetNorm = targetDomain.toLowerCase();
|
|
6372
6737
|
if (currentUrl !== lastLoggedUrl) {
|
|
6373
6738
|
log("auth", `navigated to: ${currentUrl}`);
|
|
6374
6739
|
lastLoggedUrl = currentUrl;
|
|
6375
6740
|
}
|
|
6376
6741
|
if (elapsed < MIN_WAIT_MS)
|
|
6377
6742
|
continue;
|
|
6378
|
-
const
|
|
6379
|
-
|
|
6380
|
-
|
|
6381
|
-
|
|
6382
|
-
|
|
6383
|
-
|
|
6384
|
-
|
|
6385
|
-
|
|
6386
|
-
|
|
6387
|
-
|
|
6388
|
-
|
|
6389
|
-
|
|
6390
|
-
|
|
6391
|
-
|
|
6392
|
-
|
|
6393
|
-
|
|
6743
|
+
const currentCookies = await getCookies(tabId);
|
|
6744
|
+
const currentCookieCount = currentCookies.filter((c) => isDomainMatch(c.domain, targetDomain)).length;
|
|
6745
|
+
const hasCloudflareChallenge2 = await hasCloudflareChallenge(tabId).catch(() => false);
|
|
6746
|
+
const pageText = hasCloudflareChallenge2 ? await getText(tabId).catch(() => "") : "";
|
|
6747
|
+
const assessment = assessInteractiveLoginState({
|
|
6748
|
+
currentUrl,
|
|
6749
|
+
targetDomain,
|
|
6750
|
+
initialCookieCount,
|
|
6751
|
+
currentCookieCount,
|
|
6752
|
+
hasCloudflareChallenge: hasCloudflareChallenge2,
|
|
6753
|
+
pageText
|
|
6754
|
+
});
|
|
6755
|
+
if (assessment.status === "authenticated") {
|
|
6756
|
+
loggedIn = true;
|
|
6757
|
+
log("auth", `login complete — ${currentUrl} (cookies: ${initialCookieCount} → ${currentCookieCount}; ${assessment.reason})`);
|
|
6758
|
+
break;
|
|
6759
|
+
}
|
|
6760
|
+
if (assessment.status === "blocked") {
|
|
6761
|
+
blockedReason = assessment.reason;
|
|
6762
|
+
log("auth", `login blocked — ${currentUrl} (${assessment.reason})`);
|
|
6394
6763
|
}
|
|
6395
6764
|
} catch {}
|
|
6396
6765
|
}
|
|
6397
6766
|
if (!loggedIn) {
|
|
6398
6767
|
log("auth", `login wait ended after ${Math.round((Date.now() - startTime) / 1000)}s — fallback: ${loginConfig.fallback_strategy}`);
|
|
6399
6768
|
if (loginConfig.fallback_strategy === "fail") {
|
|
6400
|
-
|
|
6769
|
+
const error = blockedReason ? `Login blocked (${blockedReason})` : "Login timed out (fallback: fail)";
|
|
6770
|
+
return { success: false, domain: targetDomain, cookies_stored: 0, error };
|
|
6401
6771
|
}
|
|
6402
6772
|
if (loginConfig.fallback_strategy === "skip") {
|
|
6403
6773
|
log("auth", `skipping cookie capture per fallback_strategy`);
|
|
@@ -6533,13 +6903,15 @@ async function refreshAuthFromBrowser(domain) {
|
|
|
6533
6903
|
}
|
|
6534
6904
|
return false;
|
|
6535
6905
|
}
|
|
6536
|
-
var LOGIN_TIMEOUT_MS = 300000, POLL_INTERVAL_MS = 2000, MIN_WAIT_MS = 15000;
|
|
6906
|
+
var LOGIN_TIMEOUT_MS = 300000, POLL_INTERVAL_MS = 2000, MIN_WAIT_MS = 15000, LOGIN_PATHS, CLOUDFLARE_TEXT;
|
|
6537
6907
|
var init_auth = __esm(async () => {
|
|
6538
6908
|
init_client();
|
|
6539
6909
|
init_domain();
|
|
6540
6910
|
init_logger();
|
|
6541
6911
|
init_supervisor();
|
|
6542
6912
|
await init_vault();
|
|
6913
|
+
LOGIN_PATHS = /\/(login|signin|sign-in|sso|auth|oauth|uas\/login|checkpoint)/i;
|
|
6914
|
+
CLOUDFLARE_TEXT = /just a moment|attention required|verify you are human|cloudflare/i;
|
|
6543
6915
|
});
|
|
6544
6916
|
|
|
6545
6917
|
// ../../src/auth/runtime.ts
|
|
@@ -9708,6 +10080,116 @@ var init_payments = __esm(() => {
|
|
|
9708
10080
|
PRICING_API_URL = process.env.UNBROWSE_BACKEND_URL ?? "https://beta-api.unbrowse.ai";
|
|
9709
10081
|
});
|
|
9710
10082
|
|
|
10083
|
+
// ../../src/execution/robots.ts
|
|
10084
|
+
function parseRobotsTxt(text) {
|
|
10085
|
+
const groups = [];
|
|
10086
|
+
let current = null;
|
|
10087
|
+
for (const rawLine of text.split(/\r?\n/)) {
|
|
10088
|
+
const line = rawLine.replace(/#.*$/, "").trim();
|
|
10089
|
+
if (!line) {
|
|
10090
|
+
current = null;
|
|
10091
|
+
continue;
|
|
10092
|
+
}
|
|
10093
|
+
const colon = line.indexOf(":");
|
|
10094
|
+
if (colon === -1)
|
|
10095
|
+
continue;
|
|
10096
|
+
const field = line.slice(0, colon).trim().toLowerCase();
|
|
10097
|
+
const value = line.slice(colon + 1).trim();
|
|
10098
|
+
if (field === "user-agent") {
|
|
10099
|
+
if (!current) {
|
|
10100
|
+
current = { agents: [], disallow: [], allow: [] };
|
|
10101
|
+
groups.push(current);
|
|
10102
|
+
}
|
|
10103
|
+
current.agents.push(value.toLowerCase());
|
|
10104
|
+
} else if (field === "disallow") {
|
|
10105
|
+
if (current && value)
|
|
10106
|
+
current.disallow.push(value);
|
|
10107
|
+
} else if (field === "allow") {
|
|
10108
|
+
if (current && value)
|
|
10109
|
+
current.allow.push(value);
|
|
10110
|
+
} else if (field === "crawl-delay") {
|
|
10111
|
+
if (current)
|
|
10112
|
+
current.crawlDelay = parseFloat(value);
|
|
10113
|
+
} else {
|
|
10114
|
+
current = null;
|
|
10115
|
+
}
|
|
10116
|
+
}
|
|
10117
|
+
return groups;
|
|
10118
|
+
}
|
|
10119
|
+
function selectRules(groups, agent) {
|
|
10120
|
+
const lower = agent.toLowerCase();
|
|
10121
|
+
const exact = groups.find((g) => g.agents.some((a) => a === lower));
|
|
10122
|
+
if (exact)
|
|
10123
|
+
return exact;
|
|
10124
|
+
return groups.find((g) => g.agents.includes("*")) ?? null;
|
|
10125
|
+
}
|
|
10126
|
+
function pathMatches(path5, prefix) {
|
|
10127
|
+
if (prefix.endsWith("$")) {
|
|
10128
|
+
return path5 === prefix.slice(0, -1);
|
|
10129
|
+
}
|
|
10130
|
+
return path5.startsWith(prefix);
|
|
10131
|
+
}
|
|
10132
|
+
function longestMatch(path5, patterns) {
|
|
10133
|
+
let best = -1;
|
|
10134
|
+
for (const p of patterns) {
|
|
10135
|
+
const base = p.endsWith("$") ? p.slice(0, -1) : p;
|
|
10136
|
+
if (pathMatches(path5, p) && base.length > best)
|
|
10137
|
+
best = base.length;
|
|
10138
|
+
}
|
|
10139
|
+
return best;
|
|
10140
|
+
}
|
|
10141
|
+
async function fetchRules(origin) {
|
|
10142
|
+
const now = Date.now();
|
|
10143
|
+
const cached = cache.get(origin);
|
|
10144
|
+
if (cached && now - cached.fetchedAt < TTL_MS)
|
|
10145
|
+
return cached.rules;
|
|
10146
|
+
try {
|
|
10147
|
+
const res = await fetch(`${origin}/robots.txt`, {
|
|
10148
|
+
headers: { "user-agent": USER_AGENT },
|
|
10149
|
+
redirect: "follow",
|
|
10150
|
+
signal: AbortSignal.timeout(5000)
|
|
10151
|
+
});
|
|
10152
|
+
if (!res.ok) {
|
|
10153
|
+
cache.set(origin, { rules: [], fetchedAt: now });
|
|
10154
|
+
return [];
|
|
10155
|
+
}
|
|
10156
|
+
const text = await res.text();
|
|
10157
|
+
const rules = parseRobotsTxt(text);
|
|
10158
|
+
cache.set(origin, { rules, fetchedAt: now });
|
|
10159
|
+
return rules;
|
|
10160
|
+
} catch {
|
|
10161
|
+
cache.set(origin, { rules: [], fetchedAt: now });
|
|
10162
|
+
return [];
|
|
10163
|
+
}
|
|
10164
|
+
}
|
|
10165
|
+
async function isAllowedByRobots(url) {
|
|
10166
|
+
let origin;
|
|
10167
|
+
let pathname;
|
|
10168
|
+
try {
|
|
10169
|
+
const parsed = new URL(url);
|
|
10170
|
+
origin = parsed.origin;
|
|
10171
|
+
pathname = parsed.pathname || "/";
|
|
10172
|
+
} catch {
|
|
10173
|
+
return true;
|
|
10174
|
+
}
|
|
10175
|
+
const groups = await fetchRules(origin);
|
|
10176
|
+
const rules = selectRules(groups, USER_AGENT);
|
|
10177
|
+
if (!rules)
|
|
10178
|
+
return true;
|
|
10179
|
+
const allowLen = longestMatch(pathname, rules.allow);
|
|
10180
|
+
const disallowLen = longestMatch(pathname, rules.disallow);
|
|
10181
|
+
if (disallowLen < 0)
|
|
10182
|
+
return true;
|
|
10183
|
+
if (allowLen >= disallowLen)
|
|
10184
|
+
return true;
|
|
10185
|
+
return false;
|
|
10186
|
+
}
|
|
10187
|
+
var USER_AGENT = "unbrowse", TTL_MS, cache;
|
|
10188
|
+
var init_robots = __esm(() => {
|
|
10189
|
+
TTL_MS = 24 * 60 * 60 * 1000;
|
|
10190
|
+
cache = new Map;
|
|
10191
|
+
});
|
|
10192
|
+
|
|
9711
10193
|
// ../../src/execution/index.ts
|
|
9712
10194
|
var exports_execution = {};
|
|
9713
10195
|
__export(exports_execution, {
|
|
@@ -10144,6 +10626,20 @@ function buildStructuredReplayHeaders(originalUrl, replayUrl, baseHeaders) {
|
|
|
10144
10626
|
}
|
|
10145
10627
|
return headers;
|
|
10146
10628
|
}
|
|
10629
|
+
function normalizeReplayHeaders(...bags) {
|
|
10630
|
+
const normalized = {};
|
|
10631
|
+
for (const bag of bags) {
|
|
10632
|
+
for (const [key, value] of Object.entries(bag ?? {})) {
|
|
10633
|
+
if (typeof value !== "string")
|
|
10634
|
+
continue;
|
|
10635
|
+
const trimmed = value.trim();
|
|
10636
|
+
if (!trimmed)
|
|
10637
|
+
continue;
|
|
10638
|
+
normalized[key.toLowerCase()] = trimmed;
|
|
10639
|
+
}
|
|
10640
|
+
}
|
|
10641
|
+
return normalized;
|
|
10642
|
+
}
|
|
10147
10643
|
function shouldFallbackToBrowserReplay(data, endpoint, intent, contextUrl) {
|
|
10148
10644
|
const replayUrl = resolveExecutionUrlTemplate(endpoint, contextUrl);
|
|
10149
10645
|
if (!isDocumentLikeUrl(replayUrl))
|
|
@@ -10540,11 +11036,11 @@ async function executeBrowserCapture(skill, params, options) {
|
|
|
10540
11036
|
}
|
|
10541
11037
|
})();
|
|
10542
11038
|
const AUTH_PROVIDERS = /accounts\.google\.com|login\.microsoftonline\.com|auth0\.com|cognito-idp\.|appleid\.apple\.com|github\.com|facebook\.com/i;
|
|
10543
|
-
const
|
|
11039
|
+
const LOGIN_PATHS2 = /\/(login|signin|sign-in|sso|auth|uas\/login|checkpoint|oauth)/i;
|
|
10544
11040
|
const redirectedToAuth = finalDomain !== targetDomain && AUTH_PROVIDERS.test(finalDomain);
|
|
10545
11041
|
const redirectedToLogin = captured.final_url !== url && (() => {
|
|
10546
11042
|
try {
|
|
10547
|
-
return
|
|
11043
|
+
return LOGIN_PATHS2.test(new URL(String(captured.final_url)).pathname);
|
|
10548
11044
|
} catch {
|
|
10549
11045
|
return false;
|
|
10550
11046
|
}
|
|
@@ -10651,7 +11147,7 @@ async function executeBrowserCapture(skill, params, options) {
|
|
|
10651
11147
|
const cleanEndpoints = endpoints.filter((ep) => {
|
|
10652
11148
|
try {
|
|
10653
11149
|
const host = new URL(ep.url_template).hostname;
|
|
10654
|
-
return !AUTH_PROVIDERS.test(host) && !
|
|
11150
|
+
return !AUTH_PROVIDERS.test(host) && !LOGIN_PATHS2.test(new URL(ep.url_template).pathname);
|
|
10655
11151
|
} catch {
|
|
10656
11152
|
return true;
|
|
10657
11153
|
}
|
|
@@ -11067,9 +11563,9 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
|
|
|
11067
11563
|
}
|
|
11068
11564
|
}
|
|
11069
11565
|
if (!skill.skill_id.startsWith("local:") && skill.execution_type === "http" && skill.owner_type !== "agent") {
|
|
11070
|
-
const
|
|
11566
|
+
const wallet = getLocalWalletContext2();
|
|
11071
11567
|
const gate = await checkPaymentRequirement(skill.skill_id, endpoint.endpoint_id, {
|
|
11072
|
-
wallet_configured: !!
|
|
11568
|
+
wallet_configured: !!wallet.wallet_address
|
|
11073
11569
|
});
|
|
11074
11570
|
if (gate.status === "payment_required" || gate.status === "wallet_not_configured" || gate.status === "insufficient_balance") {
|
|
11075
11571
|
const trace2 = stampTrace({
|
|
@@ -11089,7 +11585,8 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
|
|
|
11089
11585
|
price_usd: gate.requirement?.amount,
|
|
11090
11586
|
payment_status: gate.status,
|
|
11091
11587
|
message: gate.message,
|
|
11092
|
-
wallet_provider: "lobster.cash",
|
|
11588
|
+
wallet_provider: wallet.wallet_provider ?? "lobster.cash",
|
|
11589
|
+
wallet_address: wallet.wallet_address,
|
|
11093
11590
|
indexing_fallback_available: true
|
|
11094
11591
|
}
|
|
11095
11592
|
};
|
|
@@ -11258,14 +11755,38 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
|
|
|
11258
11755
|
} catch {}
|
|
11259
11756
|
}
|
|
11260
11757
|
}
|
|
11758
|
+
if (!options?.skip_robots_check) {
|
|
11759
|
+
const allowed = await isAllowedByRobots(url);
|
|
11760
|
+
if (!allowed) {
|
|
11761
|
+
const traceId = nanoid5();
|
|
11762
|
+
log("exec", `robots.txt blocked ${url}`);
|
|
11763
|
+
return {
|
|
11764
|
+
trace: stampTrace({
|
|
11765
|
+
trace_id: traceId,
|
|
11766
|
+
skill_id: skill.skill_id,
|
|
11767
|
+
endpoint_id: endpoint.endpoint_id,
|
|
11768
|
+
started_at: startedAt,
|
|
11769
|
+
completed_at: new Date().toISOString(),
|
|
11770
|
+
success: false,
|
|
11771
|
+
error: "robots_txt_disallowed"
|
|
11772
|
+
}),
|
|
11773
|
+
result: {
|
|
11774
|
+
error: "robots_txt_disallowed",
|
|
11775
|
+
message: `robots.txt disallows access to ${url} for the Unbrowse user-agent.`
|
|
11776
|
+
}
|
|
11777
|
+
};
|
|
11778
|
+
}
|
|
11779
|
+
}
|
|
11261
11780
|
const structuredReplayUrl = isSafe ? deriveStructuredDataReplayUrl(url) : url;
|
|
11262
11781
|
const hasStructuredReplay = structuredReplayUrl !== url;
|
|
11263
11782
|
const serverFetch = async () => {
|
|
11264
|
-
const
|
|
11783
|
+
const endpointHeaders = normalizeReplayHeaders(endpoint.headers_template);
|
|
11784
|
+
const sessionHeaders = normalizeReplayHeaders(authHeaders);
|
|
11785
|
+
const defaultAccept = !endpoint.dom_extraction && !endpointHeaders["accept"] && !sessionHeaders["accept"] ? { accept: "application/json" } : {};
|
|
11265
11786
|
const headers = {
|
|
11266
11787
|
...defaultAccept,
|
|
11267
|
-
...
|
|
11268
|
-
...
|
|
11788
|
+
...endpointHeaders,
|
|
11789
|
+
...sessionHeaders
|
|
11269
11790
|
};
|
|
11270
11791
|
delete headers["sec-ch-ua"];
|
|
11271
11792
|
delete headers["sec-ch-ua-mobile"];
|
|
@@ -11279,7 +11800,7 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
|
|
|
11279
11800
|
headers["cookie"] = cookieStr;
|
|
11280
11801
|
const csrfCookie = cookies.find((c) => /^(ct0|csrf_token|_csrf|csrftoken|XSRF-TOKEN|_xsrf)$/i.test(c.name));
|
|
11281
11802
|
if (csrfCookie) {
|
|
11282
|
-
const v = csrfCookie.value.startsWith(") && csrfCookie.value.endsWith(") ? csrfCookie.value.slice(1, -1) : csrfCookie.value;
|
|
11803
|
+
const v = csrfCookie.value.startsWith('"') && csrfCookie.value.endsWith('"') ? csrfCookie.value.slice(1, -1) : csrfCookie.value;
|
|
11283
11804
|
headers["x-csrf-token"] = v;
|
|
11284
11805
|
headers["x-xsrf-token"] = v;
|
|
11285
11806
|
}
|
|
@@ -11289,7 +11810,7 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
|
|
|
11289
11810
|
if (csrfCookie) {
|
|
11290
11811
|
const v = csrfCookie.value.startsWith('"') && csrfCookie.value.endsWith('"') ? csrfCookie.value.slice(1, -1) : csrfCookie.value;
|
|
11291
11812
|
if (endpoint.csrf_plan.source === "cookie" || endpoint.csrf_plan.source === "header") {
|
|
11292
|
-
headers[endpoint.csrf_plan.param_name.toLowerCase()]
|
|
11813
|
+
headers[endpoint.csrf_plan.param_name.toLowerCase()] = v;
|
|
11293
11814
|
} else if (endpoint.csrf_plan.source === "form" && body && typeof body === "object" && !Array.isArray(body)) {
|
|
11294
11815
|
body[endpoint.csrf_plan.param_name] ??= v;
|
|
11295
11816
|
}
|
|
@@ -11588,6 +12109,7 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
|
|
|
11588
12109
|
}
|
|
11589
12110
|
})();
|
|
11590
12111
|
if (consumerConfig.agent_id) {
|
|
12112
|
+
const wallet = getLocalWalletContext2();
|
|
11591
12113
|
recordTransaction({
|
|
11592
12114
|
transaction_id: trace.trace_id,
|
|
11593
12115
|
consumer_id: consumerConfig.agent_id,
|
|
@@ -11595,7 +12117,7 @@ async function executeEndpoint(skill, endpoint, params = {}, projection, options
|
|
|
11595
12117
|
skill_id: skill.skill_id,
|
|
11596
12118
|
endpoint_id: endpoint.endpoint_id,
|
|
11597
12119
|
price_usd: skill.base_price_usd,
|
|
11598
|
-
payment_proof:
|
|
12120
|
+
payment_proof: wallet.wallet_address ? `wallet:${wallet.wallet_address}` : undefined
|
|
11599
12121
|
}).catch(() => {});
|
|
11600
12122
|
}
|
|
11601
12123
|
}
|
|
@@ -12195,6 +12717,7 @@ var init_execution = __esm(async () => {
|
|
|
12195
12717
|
init_version();
|
|
12196
12718
|
init_search_forms();
|
|
12197
12719
|
init_payments();
|
|
12720
|
+
init_robots();
|
|
12198
12721
|
await __promiseAll([
|
|
12199
12722
|
init_vault(),
|
|
12200
12723
|
init_auth(),
|
|
@@ -13145,6 +13668,54 @@ var init_first_pass_action = __esm(() => {
|
|
|
13145
13668
|
init_client();
|
|
13146
13669
|
});
|
|
13147
13670
|
|
|
13671
|
+
// ../../src/orchestrator/timing-economics.ts
|
|
13672
|
+
function computeTimingEconomics({
|
|
13673
|
+
source,
|
|
13674
|
+
totalMs,
|
|
13675
|
+
result,
|
|
13676
|
+
skill,
|
|
13677
|
+
paidSearchUc = 0,
|
|
13678
|
+
paidExecutionUc = 0
|
|
13679
|
+
}) {
|
|
13680
|
+
const resultStr = typeof result === "string" ? result : JSON.stringify(result ?? "");
|
|
13681
|
+
const responseBytes = resultStr.length;
|
|
13682
|
+
const responseTokens = Math.ceil(responseBytes / CHARS_PER_TOKEN);
|
|
13683
|
+
const actualCostUc = responseTokens * TOKEN_COST_UC + paidSearchUc + paidExecutionUc;
|
|
13684
|
+
const economics = {
|
|
13685
|
+
response_bytes: responseBytes,
|
|
13686
|
+
response_tokens: responseTokens,
|
|
13687
|
+
tokens_saved: 0,
|
|
13688
|
+
tokens_saved_pct: 0,
|
|
13689
|
+
time_saved_pct: 0,
|
|
13690
|
+
actual_cost_uc: actualCostUc
|
|
13691
|
+
};
|
|
13692
|
+
if (!SAVINGS_SOURCES.has(source))
|
|
13693
|
+
return economics;
|
|
13694
|
+
const baselineTokens = skill?.discovery_cost?.capture_tokens ?? DEFAULT_CAPTURE_TOKENS2;
|
|
13695
|
+
const baselineMs = skill?.discovery_cost?.capture_ms ?? DEFAULT_CAPTURE_MS;
|
|
13696
|
+
const baselineCostUc = baselineTokens * TOKEN_COST_UC;
|
|
13697
|
+
economics.tokens_saved = Math.max(0, baselineTokens - responseTokens);
|
|
13698
|
+
economics.tokens_saved_pct = baselineTokens > 0 ? Math.round(economics.tokens_saved / baselineTokens * 100) : 0;
|
|
13699
|
+
economics.time_saved_pct = baselineMs > 0 ? Math.round(Math.max(0, baselineMs - totalMs) / baselineMs * 100) : 0;
|
|
13700
|
+
economics.baseline_total_ms = baselineMs;
|
|
13701
|
+
economics.time_saved_ms = Math.max(0, baselineMs - totalMs);
|
|
13702
|
+
economics.baseline_cost_uc = baselineCostUc;
|
|
13703
|
+
economics.cost_saved_uc = Math.max(0, baselineCostUc - actualCostUc);
|
|
13704
|
+
economics.baseline_source = skill?.discovery_cost ? "real" : "estimated";
|
|
13705
|
+
return economics;
|
|
13706
|
+
}
|
|
13707
|
+
var DEFAULT_CAPTURE_MS = 22000, DEFAULT_CAPTURE_TOKENS2 = 30000, CHARS_PER_TOKEN = 4, TOKEN_COST_PER_MILLION_USD = 3, TOKEN_COST_UC, SAVINGS_SOURCES;
|
|
13708
|
+
var init_timing_economics = __esm(() => {
|
|
13709
|
+
TOKEN_COST_UC = Math.round(TOKEN_COST_PER_MILLION_USD);
|
|
13710
|
+
SAVINGS_SOURCES = new Set([
|
|
13711
|
+
"marketplace",
|
|
13712
|
+
"route-cache",
|
|
13713
|
+
"first-pass",
|
|
13714
|
+
"direct-fetch",
|
|
13715
|
+
"browser-action"
|
|
13716
|
+
]);
|
|
13717
|
+
});
|
|
13718
|
+
|
|
13148
13719
|
// ../../src/payments/wallet.ts
|
|
13149
13720
|
function checkWalletConfigured() {
|
|
13150
13721
|
if (process.env.LOBSTER_WALLET_ADDRESS) {
|
|
@@ -13581,6 +14152,31 @@ function isSearchLikeIntent(intent, contextUrl) {
|
|
|
13581
14152
|
return false;
|
|
13582
14153
|
}
|
|
13583
14154
|
}
|
|
14155
|
+
function buildLocalCanonicalReplaySkill(intent, contextUrl) {
|
|
14156
|
+
const endpoint = buildCanonicalDocumentEndpoint(contextUrl, intent, false);
|
|
14157
|
+
if (!endpoint)
|
|
14158
|
+
return;
|
|
14159
|
+
const domain = new URL(contextUrl).hostname.replace(/^www\./, "");
|
|
14160
|
+
const now = new Date().toISOString();
|
|
14161
|
+
const skill = {
|
|
14162
|
+
skill_id: `canonical-${createHash7("sha1").update(contextUrl).digest("hex").slice(0, 12)}`,
|
|
14163
|
+
version: "1.0.0",
|
|
14164
|
+
schema_version: "1",
|
|
14165
|
+
name: `Canonical replay for ${domain}`,
|
|
14166
|
+
intent_signature: intent,
|
|
14167
|
+
intents: [intent],
|
|
14168
|
+
domain,
|
|
14169
|
+
description: `Deterministic structured replay for ${contextUrl}`,
|
|
14170
|
+
owner_type: "agent",
|
|
14171
|
+
execution_type: "http",
|
|
14172
|
+
endpoints: [endpoint],
|
|
14173
|
+
lifecycle: "active",
|
|
14174
|
+
created_at: now,
|
|
14175
|
+
updated_at: now
|
|
14176
|
+
};
|
|
14177
|
+
cachePublishedSkill(skill);
|
|
14178
|
+
return skill;
|
|
14179
|
+
}
|
|
13584
14180
|
function isCachedSkillRelevantForIntent(skill, intent, contextUrl) {
|
|
13585
14181
|
if (!hasUsableEndpoints(skill))
|
|
13586
14182
|
return false;
|
|
@@ -14392,9 +14988,6 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
14392
14988
|
const queryIntent = selectSearchTermsForExecution(intent) ?? extractSearchTermsFromIntent(intent) ?? intent;
|
|
14393
14989
|
if (queryIntent !== intent)
|
|
14394
14990
|
decisionTrace.query_intent = queryIntent;
|
|
14395
|
-
const DEFAULT_CAPTURE_MS = 22000;
|
|
14396
|
-
const DEFAULT_CAPTURE_TOKENS = 30000;
|
|
14397
|
-
const CHARS_PER_TOKEN = 4;
|
|
14398
14991
|
const agentChoseEndpoint = !!params.endpoint_id;
|
|
14399
14992
|
const forceCapture = !!options?.force_capture;
|
|
14400
14993
|
const clientScope = options?.client_scope ?? "global";
|
|
@@ -14418,32 +15011,33 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
14418
15011
|
timing.actual_total_ms = timing.total_ms;
|
|
14419
15012
|
timing.source = source;
|
|
14420
15013
|
timing.skill_id = skillId;
|
|
14421
|
-
const
|
|
14422
|
-
|
|
14423
|
-
|
|
14424
|
-
|
|
14425
|
-
|
|
14426
|
-
|
|
14427
|
-
|
|
14428
|
-
|
|
14429
|
-
|
|
14430
|
-
|
|
14431
|
-
|
|
14432
|
-
|
|
14433
|
-
|
|
14434
|
-
|
|
14435
|
-
timing.
|
|
14436
|
-
|
|
14437
|
-
|
|
14438
|
-
|
|
14439
|
-
timing.
|
|
14440
|
-
|
|
15014
|
+
const economics = computeTimingEconomics({
|
|
15015
|
+
source,
|
|
15016
|
+
totalMs: timing.total_ms,
|
|
15017
|
+
result: result2,
|
|
15018
|
+
skill,
|
|
15019
|
+
paidSearchUc: timing.paid_search_uc ?? 0,
|
|
15020
|
+
paidExecutionUc: timing.paid_execution_uc ?? 0
|
|
15021
|
+
});
|
|
15022
|
+
timing.response_bytes = economics.response_bytes;
|
|
15023
|
+
timing.tokens_saved = economics.tokens_saved;
|
|
15024
|
+
timing.tokens_saved_pct = economics.tokens_saved_pct;
|
|
15025
|
+
timing.time_saved_pct = economics.time_saved_pct;
|
|
15026
|
+
timing.actual_cost_uc = economics.actual_cost_uc;
|
|
15027
|
+
if (economics.baseline_total_ms != null)
|
|
15028
|
+
timing.baseline_total_ms = economics.baseline_total_ms;
|
|
15029
|
+
if (economics.time_saved_ms != null)
|
|
15030
|
+
timing.time_saved_ms = economics.time_saved_ms;
|
|
15031
|
+
if (economics.baseline_cost_uc != null)
|
|
15032
|
+
timing.baseline_cost_uc = economics.baseline_cost_uc;
|
|
15033
|
+
if (economics.cost_saved_uc != null)
|
|
15034
|
+
timing.cost_saved_uc = economics.cost_saved_uc;
|
|
14441
15035
|
if (trace2) {
|
|
14442
|
-
trace2.tokens_used =
|
|
15036
|
+
trace2.tokens_used = economics.response_tokens;
|
|
14443
15037
|
trace2.tokens_saved = timing.tokens_saved;
|
|
14444
15038
|
trace2.tokens_saved_pct = timing.tokens_saved_pct;
|
|
14445
15039
|
}
|
|
14446
|
-
console.log(`[perf] ${source}: ${timing.total_ms}ms (time_saved=${timing.time_saved_pct}% tokens_saved=${timing.tokens_saved_pct}%${
|
|
15040
|
+
console.log(`[perf] ${source}: ${timing.total_ms}ms (time_saved=${timing.time_saved_pct}% tokens_saved=${timing.tokens_saved_pct}%${economics.baseline_source === "real" ? " [real baseline]" : economics.baseline_source === "estimated" ? " [estimated]" : ""})`);
|
|
14447
15041
|
const lifecycleSource = source === "marketplace" ? "marketplace" : source === "route-cache" ? "cache" : "live-capture";
|
|
14448
15042
|
const skillIdForLifecycle = skillId ?? "unknown";
|
|
14449
15043
|
const now = new Date().toISOString();
|
|
@@ -14953,6 +15547,7 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
14953
15547
|
if (source === "marketplace" && skill.base_price_usd && skill.base_price_usd > 0) {
|
|
14954
15548
|
try {
|
|
14955
15549
|
const walletCheck = checkWalletConfigured();
|
|
15550
|
+
const wallet = getLocalWalletContext2();
|
|
14956
15551
|
const paymentResult = await checkPaymentRequirement(skill.skill_id, candidate.endpoint.endpoint_id, {
|
|
14957
15552
|
price_usd: String(skill.base_price_usd),
|
|
14958
15553
|
wallet_configured: walletCheck.configured
|
|
@@ -14970,7 +15565,8 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
14970
15565
|
payment_status: paymentResult.status,
|
|
14971
15566
|
message: paymentResult.message,
|
|
14972
15567
|
next_step: paymentResult.next_step,
|
|
14973
|
-
wallet_provider: "lobster.cash",
|
|
15568
|
+
wallet_provider: wallet.wallet_provider ?? "lobster.cash",
|
|
15569
|
+
wallet_address: wallet.wallet_address,
|
|
14974
15570
|
indexing_fallback_available: true
|
|
14975
15571
|
},
|
|
14976
15572
|
trace: execOut.trace,
|
|
@@ -15308,6 +15904,14 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
15308
15904
|
]);
|
|
15309
15905
|
} catch (err) {
|
|
15310
15906
|
if (isX402Error(err)) {
|
|
15907
|
+
const localCanonicalSkill = context?.url && !isSearchLikeIntent(queryIntent, context.url) ? buildLocalCanonicalReplaySkill(queryIntent, context.url) : undefined;
|
|
15908
|
+
if (localCanonicalSkill) {
|
|
15909
|
+
const deferred2 = await buildDeferralWithAutoExec(localCanonicalSkill, "marketplace", {
|
|
15910
|
+
local_canonical_replay: true,
|
|
15911
|
+
payment_bypass: "canonical-detail-page"
|
|
15912
|
+
});
|
|
15913
|
+
return deferred2.orchestratorResult;
|
|
15914
|
+
}
|
|
15311
15915
|
const trace2 = {
|
|
15312
15916
|
trace_id: nanoid7(),
|
|
15313
15917
|
skill_id: "marketplace-search",
|
|
@@ -15322,7 +15926,8 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
|
|
|
15322
15926
|
result: {
|
|
15323
15927
|
error: "payment_required",
|
|
15324
15928
|
payment_status: "payment_required",
|
|
15325
|
-
wallet_provider: "lobster.cash",
|
|
15929
|
+
wallet_provider: getLocalWalletContext2().wallet_provider ?? "lobster.cash",
|
|
15930
|
+
wallet_address: getLocalWalletContext2().wallet_address,
|
|
15326
15931
|
message: "Marketplace search requires payment before shared graph results are returned.",
|
|
15327
15932
|
next_step: "Pay the Tier 3 search fee, or re-run with force capture for free local discovery.",
|
|
15328
15933
|
indexing_fallback_available: true,
|
|
@@ -15964,6 +16569,7 @@ var init_orchestrator = __esm(async () => {
|
|
|
15964
16569
|
init_trace_store();
|
|
15965
16570
|
init_passive_publish();
|
|
15966
16571
|
init_first_pass_action();
|
|
16572
|
+
init_timing_economics();
|
|
15967
16573
|
init_payments();
|
|
15968
16574
|
init_version();
|
|
15969
16575
|
init_runtime();
|
|
@@ -16557,43 +17163,757 @@ var init_session_logs = __esm(() => {
|
|
|
16557
17163
|
init_domain();
|
|
16558
17164
|
});
|
|
16559
17165
|
|
|
16560
|
-
// ../../src/
|
|
16561
|
-
function
|
|
16562
|
-
|
|
16563
|
-
return 0;
|
|
16564
|
-
const tested = matrix.filter((c) => c.status !== "untested").length;
|
|
16565
|
-
return tested / matrix.length;
|
|
17166
|
+
// ../../src/api/browse-session.ts
|
|
17167
|
+
function extractBrowseFailureMessage(value) {
|
|
17168
|
+
return typeof value === "string" ? value : getKuriErrorMessage(value);
|
|
16566
17169
|
}
|
|
16567
|
-
|
|
16568
|
-
|
|
16569
|
-
|
|
16570
|
-
|
|
16571
|
-
|
|
16572
|
-
|
|
16573
|
-
|
|
16574
|
-
|
|
16575
|
-
|
|
16576
|
-
|
|
16577
|
-
|
|
16578
|
-
|
|
16579
|
-
|
|
16580
|
-
|
|
16581
|
-
|
|
16582
|
-
|
|
16583
|
-
|
|
16584
|
-
|
|
16585
|
-
|
|
16586
|
-
|
|
16587
|
-
|
|
16588
|
-
|
|
16589
|
-
|
|
16590
|
-
|
|
16591
|
-
|
|
16592
|
-
|
|
16593
|
-
|
|
16594
|
-
|
|
16595
|
-
|
|
16596
|
-
|
|
17170
|
+
function isRecoverableBrowseFailure(value) {
|
|
17171
|
+
const message = extractBrowseFailureMessage(value);
|
|
17172
|
+
if (!message)
|
|
17173
|
+
return false;
|
|
17174
|
+
const normalized = message.toLowerCase();
|
|
17175
|
+
return RECOVERABLE_BROWSE_FAILURES.some((needle) => normalized.includes(needle));
|
|
17176
|
+
}
|
|
17177
|
+
async function createBrowseSession(sessions, client, injectInterceptor2) {
|
|
17178
|
+
await client.start().catch(() => {});
|
|
17179
|
+
const tabId = await client.newTab();
|
|
17180
|
+
await client.harStart(tabId).catch(() => {});
|
|
17181
|
+
await injectInterceptor2(tabId);
|
|
17182
|
+
const session = { tabId, url: "about:blank", harActive: true, domain: "" };
|
|
17183
|
+
sessions.set("default", session);
|
|
17184
|
+
return session;
|
|
17185
|
+
}
|
|
17186
|
+
function extractDomain2(url) {
|
|
17187
|
+
if (!url)
|
|
17188
|
+
return "";
|
|
17189
|
+
try {
|
|
17190
|
+
return new URL(url).hostname.replace(/^www\./, "");
|
|
17191
|
+
} catch {
|
|
17192
|
+
return "";
|
|
17193
|
+
}
|
|
17194
|
+
}
|
|
17195
|
+
async function adoptExistingBrowseTab(sessions, client, injectInterceptor2, preferredDomain) {
|
|
17196
|
+
try {
|
|
17197
|
+
const tabs = await client.discoverTabs();
|
|
17198
|
+
const normalizedPreferred = preferredDomain?.replace(/^www\./, "") ?? "";
|
|
17199
|
+
const candidate = tabs.find((tab) => {
|
|
17200
|
+
const domain = extractDomain2(tab.url);
|
|
17201
|
+
return !!domain && !!normalizedPreferred && domain === normalizedPreferred;
|
|
17202
|
+
}) ?? tabs.find((tab) => /^https?:\/\//.test(tab.url ?? ""));
|
|
17203
|
+
if (!candidate?.id)
|
|
17204
|
+
return null;
|
|
17205
|
+
await client.harStart(candidate.id).catch(() => {});
|
|
17206
|
+
await injectInterceptor2(candidate.id);
|
|
17207
|
+
const session = {
|
|
17208
|
+
tabId: candidate.id,
|
|
17209
|
+
url: candidate.url ?? "about:blank",
|
|
17210
|
+
harActive: true,
|
|
17211
|
+
domain: extractDomain2(candidate.url)
|
|
17212
|
+
};
|
|
17213
|
+
sessions.set("default", session);
|
|
17214
|
+
return session;
|
|
17215
|
+
} catch {
|
|
17216
|
+
return null;
|
|
17217
|
+
}
|
|
17218
|
+
}
|
|
17219
|
+
async function dropBrowseSession(sessions, client, session) {
|
|
17220
|
+
if (!session)
|
|
17221
|
+
return;
|
|
17222
|
+
await client.closeTab(session.tabId).catch(() => {});
|
|
17223
|
+
if (sessions.get("default")?.tabId === session.tabId) {
|
|
17224
|
+
sessions.delete("default");
|
|
17225
|
+
}
|
|
17226
|
+
}
|
|
17227
|
+
async function isBrowseSessionLive(session, client) {
|
|
17228
|
+
if (!session.tabId)
|
|
17229
|
+
return false;
|
|
17230
|
+
try {
|
|
17231
|
+
const tabs = await client.discoverTabs();
|
|
17232
|
+
if (!tabs.some((tab) => tab.id === session.tabId))
|
|
17233
|
+
return false;
|
|
17234
|
+
const currentUrl = await client.getCurrentUrl(session.tabId);
|
|
17235
|
+
return typeof currentUrl === "string" && currentUrl.length > 0;
|
|
17236
|
+
} catch {
|
|
17237
|
+
return false;
|
|
17238
|
+
}
|
|
17239
|
+
}
|
|
17240
|
+
async function resetBrowseSession(sessions, client, injectInterceptor2) {
|
|
17241
|
+
const existing = sessions.get("default");
|
|
17242
|
+
const preferredDomain = existing?.domain || extractDomain2(existing?.url);
|
|
17243
|
+
await dropBrowseSession(sessions, client, existing);
|
|
17244
|
+
const adopted = await adoptExistingBrowseTab(sessions, client, injectInterceptor2, preferredDomain);
|
|
17245
|
+
if (adopted)
|
|
17246
|
+
return adopted;
|
|
17247
|
+
return createBrowseSession(sessions, client, injectInterceptor2);
|
|
17248
|
+
}
|
|
17249
|
+
async function getOrCreateBrowseSession(sessions, client, injectInterceptor2) {
|
|
17250
|
+
const existing = sessions.get("default");
|
|
17251
|
+
if (existing && await isBrowseSessionLive(existing, client))
|
|
17252
|
+
return existing;
|
|
17253
|
+
const preferredDomain = existing?.domain || extractDomain2(existing?.url);
|
|
17254
|
+
if (existing)
|
|
17255
|
+
await dropBrowseSession(sessions, client, existing);
|
|
17256
|
+
const adopted = await adoptExistingBrowseTab(sessions, client, injectInterceptor2, preferredDomain);
|
|
17257
|
+
if (adopted)
|
|
17258
|
+
return adopted;
|
|
17259
|
+
return createBrowseSession(sessions, client, injectInterceptor2);
|
|
17260
|
+
}
|
|
17261
|
+
async function withRecoveredBrowseSession(sessions, client, injectInterceptor2, run, shouldReset) {
|
|
17262
|
+
let session = await getOrCreateBrowseSession(sessions, client, injectInterceptor2);
|
|
17263
|
+
try {
|
|
17264
|
+
const result2 = await run(session);
|
|
17265
|
+
if (!shouldReset || !shouldReset(result2)) {
|
|
17266
|
+
return { session, result: result2, recovered: false };
|
|
17267
|
+
}
|
|
17268
|
+
} catch (error) {
|
|
17269
|
+
if (!isRecoverableBrowseFailure(error))
|
|
17270
|
+
throw error;
|
|
17271
|
+
}
|
|
17272
|
+
session = await resetBrowseSession(sessions, client, injectInterceptor2);
|
|
17273
|
+
const result = await run(session);
|
|
17274
|
+
return { session, result, recovered: true };
|
|
17275
|
+
}
|
|
17276
|
+
var RECOVERABLE_BROWSE_FAILURES;
|
|
17277
|
+
var init_browse_session = __esm(() => {
|
|
17278
|
+
init_client();
|
|
17279
|
+
RECOVERABLE_BROWSE_FAILURES = [
|
|
17280
|
+
"cdp command failed",
|
|
17281
|
+
"transport closed",
|
|
17282
|
+
"target closed",
|
|
17283
|
+
"tab not found",
|
|
17284
|
+
"session closed",
|
|
17285
|
+
"execution context was destroyed",
|
|
17286
|
+
"cannot find context with specified id",
|
|
17287
|
+
"no such target"
|
|
17288
|
+
];
|
|
17289
|
+
});
|
|
17290
|
+
|
|
17291
|
+
// ../../src/api/browse-index.ts
|
|
17292
|
+
import { nanoid as nanoid8 } from "nanoid";
|
|
17293
|
+
import { readFileSync as readFileSync9 } from "node:fs";
|
|
17294
|
+
function normalizeBrowseUrl(url, baseUrl) {
|
|
17295
|
+
if (!url)
|
|
17296
|
+
return url;
|
|
17297
|
+
try {
|
|
17298
|
+
return new URL(url).toString();
|
|
17299
|
+
} catch {
|
|
17300
|
+
if (!baseUrl)
|
|
17301
|
+
return url;
|
|
17302
|
+
try {
|
|
17303
|
+
return new URL(url, baseUrl).toString();
|
|
17304
|
+
} catch {
|
|
17305
|
+
return url;
|
|
17306
|
+
}
|
|
17307
|
+
}
|
|
17308
|
+
}
|
|
17309
|
+
function harEntriesToRawRequests(entries, baseUrl) {
|
|
17310
|
+
return entries.filter((entry) => entry.request && entry.response).map((entry) => ({
|
|
17311
|
+
url: normalizeBrowseUrl(entry.request.url, baseUrl),
|
|
17312
|
+
method: entry.request.method,
|
|
17313
|
+
request_headers: Object.fromEntries((entry.request.headers ?? []).map((header) => [header.name.toLowerCase(), header.value])),
|
|
17314
|
+
request_body: entry.request.postData?.text,
|
|
17315
|
+
response_status: entry.response.status,
|
|
17316
|
+
response_headers: Object.fromEntries((entry.response.headers ?? []).map((header) => [header.name.toLowerCase(), header.value])),
|
|
17317
|
+
response_body: entry.response.content?.text,
|
|
17318
|
+
timestamp: entry.startedDateTime ?? new Date().toISOString()
|
|
17319
|
+
}));
|
|
17320
|
+
}
|
|
17321
|
+
function buildBrowseRequestKey(request) {
|
|
17322
|
+
return [
|
|
17323
|
+
request.method,
|
|
17324
|
+
request.url,
|
|
17325
|
+
typeof request.request_body === "string" ? request.request_body : JSON.stringify(request.request_body ?? null)
|
|
17326
|
+
].join(":");
|
|
17327
|
+
}
|
|
17328
|
+
function mergeBrowseRequests(intercepted, harEntries, baseUrl) {
|
|
17329
|
+
const normalizedIntercepted = intercepted.map((request) => ({
|
|
17330
|
+
...request,
|
|
17331
|
+
url: normalizeBrowseUrl(request.url, baseUrl)
|
|
17332
|
+
}));
|
|
17333
|
+
const harRequests = harEntriesToRawRequests(harEntries, baseUrl);
|
|
17334
|
+
const seen = new Set;
|
|
17335
|
+
const allRequests = [];
|
|
17336
|
+
for (const request of normalizedIntercepted) {
|
|
17337
|
+
const key = buildBrowseRequestKey(request);
|
|
17338
|
+
if (!seen.has(key)) {
|
|
17339
|
+
seen.add(key);
|
|
17340
|
+
allRequests.push(request);
|
|
17341
|
+
}
|
|
17342
|
+
}
|
|
17343
|
+
for (const request of harRequests) {
|
|
17344
|
+
const key = buildBrowseRequestKey(request);
|
|
17345
|
+
if (!seen.has(key)) {
|
|
17346
|
+
seen.add(key);
|
|
17347
|
+
allRequests.push(request);
|
|
17348
|
+
}
|
|
17349
|
+
}
|
|
17350
|
+
return allRequests;
|
|
17351
|
+
}
|
|
17352
|
+
async function cacheBrowseRequests(params) {
|
|
17353
|
+
const { sessionUrl, sessionDomain, requests, getPageHtml: getPageHtml2 } = params;
|
|
17354
|
+
let domain;
|
|
17355
|
+
try {
|
|
17356
|
+
domain = new URL(sessionUrl).hostname;
|
|
17357
|
+
} catch {
|
|
17358
|
+
domain = sessionDomain;
|
|
17359
|
+
}
|
|
17360
|
+
const rawEndpoints = extractEndpoints(requests, undefined, { pageUrl: sessionUrl, finalUrl: sessionUrl });
|
|
17361
|
+
if (rawEndpoints.length > 0) {
|
|
17362
|
+
const existingSkill = findExistingSkillForDomain(domain);
|
|
17363
|
+
let allExisting = existingSkill?.endpoints ?? [];
|
|
17364
|
+
const domainKey = getDomainReuseKey(sessionUrl ?? domain);
|
|
17365
|
+
if (domainKey) {
|
|
17366
|
+
const cached = domainSkillCache.get(domainKey);
|
|
17367
|
+
if (cached?.localSkillPath) {
|
|
17368
|
+
try {
|
|
17369
|
+
const snapshot2 = JSON.parse(readFileSync9(cached.localSkillPath, "utf-8"));
|
|
17370
|
+
if (snapshot2?.endpoints?.length > 0) {
|
|
17371
|
+
allExisting = mergeEndpoints(allExisting, snapshot2.endpoints);
|
|
17372
|
+
}
|
|
17373
|
+
} catch {}
|
|
17374
|
+
}
|
|
17375
|
+
}
|
|
17376
|
+
const mergedEndpoints = allExisting.length > 0 ? mergeEndpoints(allExisting, rawEndpoints) : rawEndpoints;
|
|
17377
|
+
if (!existingSkill || mergedEndpoints.length >= existingSkill.endpoints.length) {
|
|
17378
|
+
for (const endpoint of mergedEndpoints) {
|
|
17379
|
+
if (!endpoint.description)
|
|
17380
|
+
endpoint.description = generateLocalDescription(endpoint);
|
|
17381
|
+
}
|
|
17382
|
+
const quickSkill = {
|
|
17383
|
+
skill_id: existingSkill?.skill_id ?? nanoid8(),
|
|
17384
|
+
version: "1.0.0",
|
|
17385
|
+
schema_version: "1",
|
|
17386
|
+
lifecycle: "active",
|
|
17387
|
+
execution_type: "http",
|
|
17388
|
+
created_at: existingSkill?.created_at ?? new Date().toISOString(),
|
|
17389
|
+
updated_at: new Date().toISOString(),
|
|
17390
|
+
name: domain,
|
|
17391
|
+
intent_signature: `browse ${domain}`,
|
|
17392
|
+
domain,
|
|
17393
|
+
description: `API skill for ${domain}`,
|
|
17394
|
+
owner_type: "agent",
|
|
17395
|
+
endpoints: mergedEndpoints,
|
|
17396
|
+
operation_graph: buildSkillOperationGraph(mergedEndpoints),
|
|
17397
|
+
intents: Array.from(new Set([...existingSkill?.intents ?? [], `browse ${domain}`]))
|
|
17398
|
+
};
|
|
17399
|
+
const cacheKey = buildResolveCacheKey(domain, `browse ${domain}`, sessionUrl);
|
|
17400
|
+
const scopedKey = scopedCacheKey("global", cacheKey);
|
|
17401
|
+
writeSkillSnapshot(scopedKey, quickSkill);
|
|
17402
|
+
if (domainKey) {
|
|
17403
|
+
domainSkillCache.set(domainKey, {
|
|
17404
|
+
skillId: quickSkill.skill_id,
|
|
17405
|
+
localSkillPath: snapshotPathForCacheKey(scopedKey),
|
|
17406
|
+
ts: Date.now()
|
|
17407
|
+
});
|
|
17408
|
+
persistDomainCache();
|
|
17409
|
+
}
|
|
17410
|
+
try {
|
|
17411
|
+
cachePublishedSkill(quickSkill);
|
|
17412
|
+
} catch {}
|
|
17413
|
+
upsertDagEdgesFromOperationGraph(quickSkill);
|
|
17414
|
+
invalidateRouteCacheForDomain(domain);
|
|
17415
|
+
return { domain, indexed: true, mode: "http", skill: quickSkill };
|
|
17416
|
+
}
|
|
17417
|
+
return { domain, indexed: false, mode: "http", skill: existingSkill ?? null };
|
|
17418
|
+
}
|
|
17419
|
+
if (!getPageHtml2)
|
|
17420
|
+
return { domain, indexed: false, mode: "none", skill: null };
|
|
17421
|
+
try {
|
|
17422
|
+
const html = await getPageHtml2();
|
|
17423
|
+
if (!html || !html.startsWith("<"))
|
|
17424
|
+
return { domain, indexed: false, mode: "none", skill: null };
|
|
17425
|
+
const { extractFromDOM: extractFromDOM2 } = await Promise.resolve().then(() => (init_extraction(), exports_extraction));
|
|
17426
|
+
const { detectSearchForms: detectSearchForms2, isStructuredSearchForm: isStructuredSearchForm2 } = await Promise.resolve().then(() => (init_search_forms(), exports_search_forms));
|
|
17427
|
+
const { inferSchema: inferSchema2 } = await Promise.resolve().then(() => (init_transform(), exports_transform));
|
|
17428
|
+
const { templatizeQueryParams: templatizeQueryParams2 } = await init_execution().then(() => exports_execution);
|
|
17429
|
+
const extracted = extractFromDOM2(html, `browse ${domain}`);
|
|
17430
|
+
const searchForms = detectSearchForms2(html);
|
|
17431
|
+
const validForm = searchForms.find((form) => isStructuredSearchForm2(form));
|
|
17432
|
+
if (!extracted.data && !validForm)
|
|
17433
|
+
return { domain, indexed: false, mode: "none", skill: null };
|
|
17434
|
+
const urlTemplate = templatizeQueryParams2(sessionUrl);
|
|
17435
|
+
const endpoint = {
|
|
17436
|
+
endpoint_id: nanoid8(),
|
|
17437
|
+
method: "GET",
|
|
17438
|
+
url_template: urlTemplate,
|
|
17439
|
+
idempotency: "safe",
|
|
17440
|
+
verification_status: "verified",
|
|
17441
|
+
reliability_score: extracted.confidence ?? 0.7,
|
|
17442
|
+
description: validForm ? `Search form for ${domain}` : `Page content from ${domain}`,
|
|
17443
|
+
response_schema: extracted.data ? inferSchema2([extracted.data]) : undefined,
|
|
17444
|
+
dom_extraction: {
|
|
17445
|
+
extraction_method: extracted.extraction_method ?? "repeated-elements",
|
|
17446
|
+
confidence: extracted.confidence ?? 0.7,
|
|
17447
|
+
...extracted.selector ? { selector: extracted.selector } : {}
|
|
17448
|
+
},
|
|
17449
|
+
trigger_url: sessionUrl,
|
|
17450
|
+
...validForm ? { search_form: validForm } : {}
|
|
17451
|
+
};
|
|
17452
|
+
endpoint.semantic = inferEndpointSemantic(endpoint, {
|
|
17453
|
+
sampleResponse: extracted.data,
|
|
17454
|
+
observedAt: new Date().toISOString(),
|
|
17455
|
+
sampleRequestUrl: sessionUrl
|
|
17456
|
+
});
|
|
17457
|
+
const existing = findExistingSkillForDomain(domain);
|
|
17458
|
+
const allEndpoints = existing ? mergeEndpoints(existing.endpoints, [endpoint]) : [endpoint];
|
|
17459
|
+
for (const candidate of allEndpoints) {
|
|
17460
|
+
if (!candidate.description)
|
|
17461
|
+
candidate.description = generateLocalDescription(candidate);
|
|
17462
|
+
}
|
|
17463
|
+
const skill = {
|
|
17464
|
+
skill_id: existing?.skill_id ?? nanoid8(),
|
|
17465
|
+
version: "1.0.0",
|
|
17466
|
+
schema_version: "1",
|
|
17467
|
+
lifecycle: "active",
|
|
17468
|
+
execution_type: "http",
|
|
17469
|
+
created_at: existing?.created_at ?? new Date().toISOString(),
|
|
17470
|
+
updated_at: new Date().toISOString(),
|
|
17471
|
+
name: domain,
|
|
17472
|
+
intent_signature: `browse ${domain}`,
|
|
17473
|
+
domain,
|
|
17474
|
+
description: `DOM skill for ${domain}`,
|
|
17475
|
+
owner_type: "agent",
|
|
17476
|
+
endpoints: allEndpoints,
|
|
17477
|
+
operation_graph: buildSkillOperationGraph(allEndpoints),
|
|
17478
|
+
intents: [...new Set([...existing?.intents ?? [], `browse ${domain}`])]
|
|
17479
|
+
};
|
|
17480
|
+
const cacheKey = buildResolveCacheKey(domain, `browse ${domain}`, sessionUrl);
|
|
17481
|
+
const scopedKey = scopedCacheKey("global", cacheKey);
|
|
17482
|
+
writeSkillSnapshot(scopedKey, skill);
|
|
17483
|
+
const domainReuseKey = getDomainReuseKey(sessionUrl ?? domain);
|
|
17484
|
+
if (domainReuseKey) {
|
|
17485
|
+
domainSkillCache.set(domainReuseKey, {
|
|
17486
|
+
skillId: skill.skill_id,
|
|
17487
|
+
localSkillPath: snapshotPathForCacheKey(scopedKey),
|
|
17488
|
+
ts: Date.now()
|
|
17489
|
+
});
|
|
17490
|
+
persistDomainCache();
|
|
17491
|
+
}
|
|
17492
|
+
try {
|
|
17493
|
+
cachePublishedSkill(skill);
|
|
17494
|
+
} catch {}
|
|
17495
|
+
upsertDagEdgesFromOperationGraph(skill);
|
|
17496
|
+
invalidateRouteCacheForDomain(domain);
|
|
17497
|
+
return { domain, indexed: true, mode: "dom", skill };
|
|
17498
|
+
} catch {
|
|
17499
|
+
return { domain, indexed: false, mode: "none", skill: null };
|
|
17500
|
+
}
|
|
17501
|
+
}
|
|
17502
|
+
var init_browse_index = __esm(async () => {
|
|
17503
|
+
init_reverse_engineer();
|
|
17504
|
+
init_graph();
|
|
17505
|
+
init_client2();
|
|
17506
|
+
init_marketplace();
|
|
17507
|
+
init_dag_feedback();
|
|
17508
|
+
await init_orchestrator();
|
|
17509
|
+
});
|
|
17510
|
+
|
|
17511
|
+
// ../../src/api/browse-submit.ts
|
|
17512
|
+
function sleep(ms) {
|
|
17513
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
17514
|
+
}
|
|
17515
|
+
function asRecord(value) {
|
|
17516
|
+
return value && typeof value === "object" ? value : null;
|
|
17517
|
+
}
|
|
17518
|
+
function isUrlWaitHint(value) {
|
|
17519
|
+
if (!value)
|
|
17520
|
+
return false;
|
|
17521
|
+
return /^https?:\/\//i.test(value) || value.startsWith("/");
|
|
17522
|
+
}
|
|
17523
|
+
function hasMeaningfulPageChange(beforeHtml, afterHtml) {
|
|
17524
|
+
const before = beforeHtml.trim();
|
|
17525
|
+
const after = afterHtml.trim();
|
|
17526
|
+
if (!after)
|
|
17527
|
+
return false;
|
|
17528
|
+
if (!before)
|
|
17529
|
+
return after.length > 64;
|
|
17530
|
+
if (before === after)
|
|
17531
|
+
return false;
|
|
17532
|
+
if (Math.abs(before.length - after.length) > 48)
|
|
17533
|
+
return true;
|
|
17534
|
+
const beforeBody = before.match(/<body[\s\S]*?>([\s\S]*?)<\/body>/i)?.[1] ?? before;
|
|
17535
|
+
const afterBody = after.match(/<body[\s\S]*?>([\s\S]*?)<\/body>/i)?.[1] ?? after;
|
|
17536
|
+
return beforeBody.trim() !== afterBody.trim();
|
|
17537
|
+
}
|
|
17538
|
+
function buildDomSubmitExpression(options) {
|
|
17539
|
+
return `(function() {
|
|
17540
|
+
function findForm(selector) {
|
|
17541
|
+
if (selector) return document.querySelector(selector);
|
|
17542
|
+
var active = document.activeElement;
|
|
17543
|
+
if (active && active.closest) {
|
|
17544
|
+
var fromActive = active.closest("form");
|
|
17545
|
+
if (fromActive) return fromActive;
|
|
17546
|
+
}
|
|
17547
|
+
return document.querySelector("form");
|
|
17548
|
+
}
|
|
17549
|
+
function findSubmitter(form, selector) {
|
|
17550
|
+
if (!form) return null;
|
|
17551
|
+
if (selector) return document.querySelector(selector);
|
|
17552
|
+
var active = document.activeElement;
|
|
17553
|
+
if (active && form.contains(active) && /^(submit|image)$/i.test(active.getAttribute("type") || "")) return active;
|
|
17554
|
+
return form.querySelector('button[type="submit"], input[type="submit"], input[type="image"], button:not([type])');
|
|
17555
|
+
}
|
|
17556
|
+
|
|
17557
|
+
var form = findForm(${JSON.stringify(options.formSelector ?? "")});
|
|
17558
|
+
if (!form) {
|
|
17559
|
+
return JSON.stringify({ ok: false, reason: "form_not_found" });
|
|
17560
|
+
}
|
|
17561
|
+
|
|
17562
|
+
var submitter = findSubmitter(form, ${JSON.stringify(options.submitSelector ?? "")});
|
|
17563
|
+
var meta = {
|
|
17564
|
+
ok: true,
|
|
17565
|
+
form_action: form.getAttribute("action") || "",
|
|
17566
|
+
form_method: (form.getAttribute("method") || "GET").toUpperCase(),
|
|
17567
|
+
submitter: submitter ? (submitter.getAttribute("name") || submitter.id || submitter.textContent || submitter.tagName || "").trim() : null,
|
|
17568
|
+
submit_selector_used: ${JSON.stringify(options.submitSelector ?? null)},
|
|
17569
|
+
form_selector_used: ${JSON.stringify(options.formSelector ?? null)},
|
|
17570
|
+
};
|
|
17571
|
+
|
|
17572
|
+
if (submitter && typeof submitter.click === "function") {
|
|
17573
|
+
submitter.click();
|
|
17574
|
+
return JSON.stringify({ ...meta, submit_kind: "click" });
|
|
17575
|
+
}
|
|
17576
|
+
if (typeof form.requestSubmit === "function") {
|
|
17577
|
+
form.requestSubmit();
|
|
17578
|
+
return JSON.stringify({ ...meta, submit_kind: "requestSubmit" });
|
|
17579
|
+
}
|
|
17580
|
+
if (typeof form.submit === "function") {
|
|
17581
|
+
form.submit();
|
|
17582
|
+
return JSON.stringify({ ...meta, submit_kind: "submit" });
|
|
17583
|
+
}
|
|
17584
|
+
return JSON.stringify({ ok: false, reason: "submit_unavailable" });
|
|
17585
|
+
})()`;
|
|
17586
|
+
}
|
|
17587
|
+
function buildSameOriginFetchExpression(options) {
|
|
17588
|
+
return `(async function() {
|
|
17589
|
+
function splitPlugins(value) {
|
|
17590
|
+
return String(value || "")
|
|
17591
|
+
.split(/[\\s,;]+/)
|
|
17592
|
+
.map(function(part) { return part.trim(); })
|
|
17593
|
+
.filter(Boolean);
|
|
17594
|
+
}
|
|
17595
|
+
function pluginPath(name) {
|
|
17596
|
+
if (/^https?:\\/\\//i.test(name) || name.startsWith("/")) return name;
|
|
17597
|
+
return "/etc/designs/wrs/footLibs/js/plugins/" + (name.endsWith(".js") ? name : name + ".js");
|
|
17598
|
+
}
|
|
17599
|
+
async function bestEffortRehydrate() {
|
|
17600
|
+
var modules = Array.from(new Set(
|
|
17601
|
+
Array.from(document.querySelectorAll("[data-load-plugins]"))
|
|
17602
|
+
.flatMap(function(node) { return splitPlugins(node.getAttribute("data-load-plugins")); })
|
|
17603
|
+
));
|
|
17604
|
+
if (modules.length === 0) {
|
|
17605
|
+
return { attempted: false, loaded: false, nooped: true, reason: "no_plugins", modules: [] };
|
|
17606
|
+
}
|
|
17607
|
+
if (!window.WRS || typeof window.WRS.require !== "function") {
|
|
17608
|
+
return { attempted: false, loaded: false, nooped: true, reason: "missing_wrs_require", modules: modules };
|
|
17609
|
+
}
|
|
17610
|
+
var requireWrs = window.WRS.require.bind(window.WRS);
|
|
17611
|
+
async function loadModules(paths) {
|
|
17612
|
+
return await new Promise(function(resolve) {
|
|
17613
|
+
var done = false;
|
|
17614
|
+
var timer = setTimeout(function() {
|
|
17615
|
+
if (done) return;
|
|
17616
|
+
done = true;
|
|
17617
|
+
resolve({ ok: false, reason: "timeout" });
|
|
17618
|
+
}, 1500);
|
|
17619
|
+
try {
|
|
17620
|
+
requireWrs(paths, function() {
|
|
17621
|
+
if (done) return;
|
|
17622
|
+
done = true;
|
|
17623
|
+
clearTimeout(timer);
|
|
17624
|
+
resolve({ ok: true });
|
|
17625
|
+
});
|
|
17626
|
+
} catch (error) {
|
|
17627
|
+
if (done) return;
|
|
17628
|
+
done = true;
|
|
17629
|
+
clearTimeout(timer);
|
|
17630
|
+
resolve({ ok: false, reason: error && error.message ? error.message : String(error) });
|
|
17631
|
+
}
|
|
17632
|
+
});
|
|
17633
|
+
}
|
|
17634
|
+
|
|
17635
|
+
var configResult = await loadModules(["/etc/designs/wrs/footLibs/js/config.js"]);
|
|
17636
|
+
var pluginResult = await loadModules(modules.map(pluginPath));
|
|
17637
|
+
for (var i = 0; i < 6; i++) {
|
|
17638
|
+
await new Promise(function(resolve) { return setTimeout(resolve, 100); });
|
|
17639
|
+
}
|
|
17640
|
+
return {
|
|
17641
|
+
attempted: true,
|
|
17642
|
+
loaded: !!pluginResult.ok,
|
|
17643
|
+
nooped: false,
|
|
17644
|
+
reason: pluginResult.ok ? undefined : pluginResult.reason,
|
|
17645
|
+
config_loaded: !!configResult.ok,
|
|
17646
|
+
modules: modules,
|
|
17647
|
+
};
|
|
17648
|
+
}
|
|
17649
|
+
function findForm(selector) {
|
|
17650
|
+
if (selector) return document.querySelector(selector);
|
|
17651
|
+
var active = document.activeElement;
|
|
17652
|
+
if (active && active.closest) {
|
|
17653
|
+
var fromActive = active.closest("form");
|
|
17654
|
+
if (fromActive) return fromActive;
|
|
17655
|
+
}
|
|
17656
|
+
return document.querySelector("form");
|
|
17657
|
+
}
|
|
17658
|
+
function findSubmitter(form, selector) {
|
|
17659
|
+
if (!form) return null;
|
|
17660
|
+
if (selector) return document.querySelector(selector);
|
|
17661
|
+
var active = document.activeElement;
|
|
17662
|
+
if (active && form.contains(active) && /^(submit|image)$/i.test(active.getAttribute("type") || "")) return active;
|
|
17663
|
+
return form.querySelector('button[type="submit"], input[type="submit"], input[type="image"], button:not([type])');
|
|
17664
|
+
}
|
|
17665
|
+
|
|
17666
|
+
var form = findForm(${JSON.stringify(options.formSelector ?? "")});
|
|
17667
|
+
if (!form) return JSON.stringify({ ok: false, reason: "form_not_found" });
|
|
17668
|
+
|
|
17669
|
+
var submitter = findSubmitter(form, ${JSON.stringify(options.submitSelector ?? "")});
|
|
17670
|
+
var method = (form.getAttribute("method") || "GET").toUpperCase();
|
|
17671
|
+
var action = form.getAttribute("action") || window.location.href;
|
|
17672
|
+
var targetUrl = new URL(action, window.location.href);
|
|
17673
|
+
if (targetUrl.origin !== window.location.origin) {
|
|
17674
|
+
return JSON.stringify({ ok: false, reason: "cross_origin", url: targetUrl.href });
|
|
17675
|
+
}
|
|
17676
|
+
|
|
17677
|
+
var formData = new FormData(form);
|
|
17678
|
+
if (submitter && submitter.name) {
|
|
17679
|
+
var submitValue = submitter.value != null ? submitter.value : "";
|
|
17680
|
+
if (!formData.has(submitter.name)) formData.append(submitter.name, submitValue);
|
|
17681
|
+
}
|
|
17682
|
+
|
|
17683
|
+
var headers = {};
|
|
17684
|
+
var requestUrl = targetUrl.href;
|
|
17685
|
+
var body;
|
|
17686
|
+
if (method === "GET") {
|
|
17687
|
+
var params = new URLSearchParams();
|
|
17688
|
+
Array.from(formData.entries()).forEach(function(entry) {
|
|
17689
|
+
var value = entry[1];
|
|
17690
|
+
if (typeof value === "string") params.append(entry[0], value);
|
|
17691
|
+
});
|
|
17692
|
+
var query = params.toString();
|
|
17693
|
+
if (query) requestUrl += (requestUrl.includes("?") ? "&" : "?") + query;
|
|
17694
|
+
} else if ((form.enctype || "").includes("application/x-www-form-urlencoded")) {
|
|
17695
|
+
var encoded = new URLSearchParams();
|
|
17696
|
+
Array.from(formData.entries()).forEach(function(entry) {
|
|
17697
|
+
var value = entry[1];
|
|
17698
|
+
if (typeof value === "string") encoded.append(entry[0], value);
|
|
17699
|
+
});
|
|
17700
|
+
body = encoded.toString();
|
|
17701
|
+
headers["Content-Type"] = "application/x-www-form-urlencoded; charset=UTF-8";
|
|
17702
|
+
} else {
|
|
17703
|
+
body = formData;
|
|
17704
|
+
}
|
|
17705
|
+
|
|
17706
|
+
try {
|
|
17707
|
+
var response = await fetch(requestUrl, {
|
|
17708
|
+
method: method,
|
|
17709
|
+
body: method === "GET" ? undefined : body,
|
|
17710
|
+
headers: headers,
|
|
17711
|
+
credentials: "include",
|
|
17712
|
+
redirect: "follow",
|
|
17713
|
+
});
|
|
17714
|
+
var contentType = response.headers.get("content-type") || "";
|
|
17715
|
+
var text = await response.text();
|
|
17716
|
+
var finalUrl = response.url || requestUrl;
|
|
17717
|
+
if (!/text\\/html|application\\/xhtml\\+xml/i.test(contentType)) {
|
|
17718
|
+
return JSON.stringify({
|
|
17719
|
+
ok: false,
|
|
17720
|
+
reason: "non_html_response",
|
|
17721
|
+
status: response.status,
|
|
17722
|
+
url: finalUrl,
|
|
17723
|
+
content_type: contentType,
|
|
17724
|
+
});
|
|
17725
|
+
}
|
|
17726
|
+
|
|
17727
|
+
document.open();
|
|
17728
|
+
document.write(text);
|
|
17729
|
+
document.close();
|
|
17730
|
+
if (finalUrl && finalUrl !== window.location.href) {
|
|
17731
|
+
history.replaceState({}, "", finalUrl);
|
|
17732
|
+
}
|
|
17733
|
+
await new Promise(function(resolve) { return setTimeout(resolve, 50); });
|
|
17734
|
+
var rehydrate = await bestEffortRehydrate();
|
|
17735
|
+
return JSON.stringify({
|
|
17736
|
+
ok: true,
|
|
17737
|
+
status: response.status,
|
|
17738
|
+
url: finalUrl,
|
|
17739
|
+
same_origin_html_rehydrated: true,
|
|
17740
|
+
rehydrate: rehydrate,
|
|
17741
|
+
});
|
|
17742
|
+
} catch (error) {
|
|
17743
|
+
return JSON.stringify({
|
|
17744
|
+
ok: false,
|
|
17745
|
+
reason: error && error.message ? error.message : String(error),
|
|
17746
|
+
});
|
|
17747
|
+
}
|
|
17748
|
+
})()`;
|
|
17749
|
+
}
|
|
17750
|
+
function parseJsonString(value) {
|
|
17751
|
+
if (typeof value !== "string")
|
|
17752
|
+
return asRecord(value);
|
|
17753
|
+
try {
|
|
17754
|
+
return asRecord(JSON.parse(value));
|
|
17755
|
+
} catch {
|
|
17756
|
+
return null;
|
|
17757
|
+
}
|
|
17758
|
+
}
|
|
17759
|
+
async function waitForSubmitOutcome(client, tabId, beforeUrl, beforeHtml, options) {
|
|
17760
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_SUBMIT_TIMEOUT_MS;
|
|
17761
|
+
const deadline = Date.now() + timeoutMs;
|
|
17762
|
+
const waitFor = options.waitFor?.trim();
|
|
17763
|
+
if (waitFor && !isUrlWaitHint(waitFor)) {
|
|
17764
|
+
try {
|
|
17765
|
+
const waitResult = await client.waitForSelector(tabId, waitFor, timeoutMs);
|
|
17766
|
+
if (waitResult?.status === "found" || waitResult?.status === "ready") {
|
|
17767
|
+
const url = await client.getCurrentUrl(tabId).catch(() => beforeUrl);
|
|
17768
|
+
const html = await client.getPageHtml(tabId).catch(() => beforeHtml);
|
|
17769
|
+
return { ok: true, url, html };
|
|
17770
|
+
}
|
|
17771
|
+
} catch {}
|
|
17772
|
+
}
|
|
17773
|
+
while (Date.now() < deadline) {
|
|
17774
|
+
const url = await client.getCurrentUrl(tabId).catch(() => "");
|
|
17775
|
+
const html = await client.getPageHtml(tabId).catch(() => "");
|
|
17776
|
+
if (waitFor && isUrlWaitHint(waitFor) && url.includes(waitFor)) {
|
|
17777
|
+
return { ok: true, url, html };
|
|
17778
|
+
}
|
|
17779
|
+
if (url && url !== beforeUrl && !url.startsWith("about:blank")) {
|
|
17780
|
+
return { ok: true, url, html };
|
|
17781
|
+
}
|
|
17782
|
+
if (hasMeaningfulPageChange(beforeHtml, html)) {
|
|
17783
|
+
return { ok: true, url: url || beforeUrl, html };
|
|
17784
|
+
}
|
|
17785
|
+
await sleep(SUBMIT_POLL_INTERVAL_MS);
|
|
17786
|
+
}
|
|
17787
|
+
return { ok: false, url: beforeUrl, html: beforeHtml };
|
|
17788
|
+
}
|
|
17789
|
+
async function submitBrowseForm(deps, options = {}) {
|
|
17790
|
+
const { client, session, restartCapture, rehydratePlugins } = deps;
|
|
17791
|
+
const sameOriginFetchFallback = options.sameOriginFetchFallback !== false;
|
|
17792
|
+
const beforeUrl = await client.getCurrentUrl(session.tabId).catch(() => session.url);
|
|
17793
|
+
const beforeHtml = await client.getPageHtml(session.tabId).catch(() => "");
|
|
17794
|
+
let submitMeta = null;
|
|
17795
|
+
let submitError = null;
|
|
17796
|
+
try {
|
|
17797
|
+
submitMeta = parseJsonString(await client.evaluate(session.tabId, buildDomSubmitExpression(options)));
|
|
17798
|
+
} catch (error) {
|
|
17799
|
+
submitError = error;
|
|
17800
|
+
}
|
|
17801
|
+
if (!submitMeta?.ok && submitMeta?.reason === "form_not_found") {
|
|
17802
|
+
return {
|
|
17803
|
+
ok: false,
|
|
17804
|
+
url: beforeUrl || session.url,
|
|
17805
|
+
mode: "noop",
|
|
17806
|
+
fallback_used: false,
|
|
17807
|
+
same_origin_html_rehydrated: false,
|
|
17808
|
+
recoverable: false,
|
|
17809
|
+
reason: "form_not_found",
|
|
17810
|
+
submit_meta: submitMeta
|
|
17811
|
+
};
|
|
17812
|
+
}
|
|
17813
|
+
const domOutcome = await waitForSubmitOutcome(client, session.tabId, beforeUrl, beforeHtml, options);
|
|
17814
|
+
if (domOutcome.ok) {
|
|
17815
|
+
session.url = domOutcome.url || beforeUrl || session.url;
|
|
17816
|
+
await restartCapture(session);
|
|
17817
|
+
return {
|
|
17818
|
+
ok: true,
|
|
17819
|
+
url: session.url,
|
|
17820
|
+
mode: "dom",
|
|
17821
|
+
fallback_used: false,
|
|
17822
|
+
same_origin_html_rehydrated: false,
|
|
17823
|
+
wait_for: options.waitFor,
|
|
17824
|
+
submit_meta: submitMeta
|
|
17825
|
+
};
|
|
17826
|
+
}
|
|
17827
|
+
if (submitError && !isRecoverableBrowseFailure(submitError) && !sameOriginFetchFallback) {
|
|
17828
|
+
throw submitError;
|
|
17829
|
+
}
|
|
17830
|
+
if (!sameOriginFetchFallback) {
|
|
17831
|
+
return {
|
|
17832
|
+
ok: false,
|
|
17833
|
+
url: beforeUrl || session.url,
|
|
17834
|
+
mode: "noop",
|
|
17835
|
+
fallback_used: false,
|
|
17836
|
+
same_origin_html_rehydrated: false,
|
|
17837
|
+
recoverable: !!submitError && isRecoverableBrowseFailure(submitError),
|
|
17838
|
+
reason: submitError instanceof Error ? submitError.message : "submit_failed",
|
|
17839
|
+
submit_meta: submitMeta
|
|
17840
|
+
};
|
|
17841
|
+
}
|
|
17842
|
+
const fallbackPayload = parseJsonString(await client.evaluate(session.tabId, buildSameOriginFetchExpression(options)));
|
|
17843
|
+
if (!fallbackPayload?.ok) {
|
|
17844
|
+
return {
|
|
17845
|
+
ok: false,
|
|
17846
|
+
url: String(fallbackPayload?.url ?? beforeUrl ?? session.url),
|
|
17847
|
+
mode: "same_origin_fetch",
|
|
17848
|
+
fallback_used: true,
|
|
17849
|
+
same_origin_html_rehydrated: false,
|
|
17850
|
+
recoverable: !!submitError && isRecoverableBrowseFailure(submitError),
|
|
17851
|
+
reason: String(fallbackPayload?.reason ?? "same_origin_fetch_failed"),
|
|
17852
|
+
status: typeof fallbackPayload?.status === "number" ? fallbackPayload.status : undefined,
|
|
17853
|
+
submit_meta: submitMeta
|
|
17854
|
+
};
|
|
17855
|
+
}
|
|
17856
|
+
const finalUrl = String(fallbackPayload.url ?? await client.getCurrentUrl(session.tabId).catch(() => beforeUrl));
|
|
17857
|
+
session.url = finalUrl || beforeUrl || session.url;
|
|
17858
|
+
let rehydrate = fallbackPayload.rehydrate;
|
|
17859
|
+
if (!rehydrate) {
|
|
17860
|
+
rehydrate = await rehydratePlugins(session.tabId).catch(() => null);
|
|
17861
|
+
}
|
|
17862
|
+
await restartCapture(session);
|
|
17863
|
+
return {
|
|
17864
|
+
ok: true,
|
|
17865
|
+
url: session.url,
|
|
17866
|
+
mode: "same_origin_fetch",
|
|
17867
|
+
fallback_used: true,
|
|
17868
|
+
same_origin_html_rehydrated: fallbackPayload.same_origin_html_rehydrated === true,
|
|
17869
|
+
status: typeof fallbackPayload.status === "number" ? fallbackPayload.status : undefined,
|
|
17870
|
+
wait_for: options.waitFor,
|
|
17871
|
+
submit_meta: submitMeta,
|
|
17872
|
+
rehydrate
|
|
17873
|
+
};
|
|
17874
|
+
}
|
|
17875
|
+
var DEFAULT_SUBMIT_TIMEOUT_MS = 8000, SUBMIT_POLL_INTERVAL_MS = 250;
|
|
17876
|
+
var init_browse_submit = __esm(() => {
|
|
17877
|
+
init_browse_session();
|
|
17878
|
+
});
|
|
17879
|
+
|
|
17880
|
+
// ../../src/verification/matrix.ts
|
|
17881
|
+
function computeVerificationCoverage(matrix) {
|
|
17882
|
+
if (matrix.length === 0)
|
|
17883
|
+
return 0;
|
|
17884
|
+
const tested = matrix.filter((c) => c.status !== "untested").length;
|
|
17885
|
+
return tested / matrix.length;
|
|
17886
|
+
}
|
|
17887
|
+
var INITIAL_MATRIX;
|
|
17888
|
+
var init_matrix = __esm(() => {
|
|
17889
|
+
INITIAL_MATRIX = [
|
|
17890
|
+
{ host: "openclaw", capability: "capture", status: "pass", last_verified: "2026-03-31" },
|
|
17891
|
+
{ host: "openclaw", capability: "execute", status: "pass", last_verified: "2026-03-31" },
|
|
17892
|
+
{ host: "openclaw", capability: "search", status: "pass", last_verified: "2026-03-31" },
|
|
17893
|
+
{ host: "mcp", capability: "execute", status: "pass", last_verified: "2026-03-31" },
|
|
17894
|
+
{ host: "mcp", capability: "search", status: "pass", last_verified: "2026-03-31" },
|
|
17895
|
+
{ host: "cli", capability: "capture", status: "pass", last_verified: "2026-03-31" },
|
|
17896
|
+
{ host: "cli", capability: "execute", status: "pass", last_verified: "2026-03-31" },
|
|
17897
|
+
{ host: "cli", capability: "search", status: "pass", last_verified: "2026-03-31" },
|
|
17898
|
+
{ host: "cli", capability: "publish", status: "pass", last_verified: "2026-03-31" },
|
|
17899
|
+
{ host: "hermes", capability: "execute", status: "untested" },
|
|
17900
|
+
{ host: "elizaos", capability: "execute", status: "untested" },
|
|
17901
|
+
{ host: "langchain", capability: "execute", status: "untested" },
|
|
17902
|
+
{ host: "langchain", capability: "search", status: "untested" }
|
|
17903
|
+
];
|
|
17904
|
+
});
|
|
17905
|
+
|
|
17906
|
+
// ../../src/verification/index.ts
|
|
17907
|
+
var exports_verification = {};
|
|
17908
|
+
__export(exports_verification, {
|
|
17909
|
+
verifySkillWithCoverage: () => verifySkillWithCoverage,
|
|
17910
|
+
verifySkill: () => verifySkill,
|
|
17911
|
+
verifyEndpoint: () => verifyEndpoint,
|
|
17912
|
+
schedulePeriodicVerification: () => schedulePeriodicVerification
|
|
17913
|
+
});
|
|
17914
|
+
async function verifyEndpoint(skill, endpoint) {
|
|
17915
|
+
if (endpoint.method !== "GET")
|
|
17916
|
+
return endpoint.verification_status;
|
|
16597
17917
|
try {
|
|
16598
17918
|
const { status, data } = await executeInBrowser(endpoint.url_template, endpoint.method, endpoint.headers_template ?? {}, undefined, undefined, undefined);
|
|
16599
17919
|
if (status < 200 || status >= 300) {
|
|
@@ -16671,7 +17991,7 @@ __export(exports_routes, {
|
|
|
16671
17991
|
registerBrowseSession: () => registerBrowseSession,
|
|
16672
17992
|
buildAnalyticsSessionPayload: () => buildAnalyticsSessionPayload
|
|
16673
17993
|
});
|
|
16674
|
-
import { nanoid as
|
|
17994
|
+
import { nanoid as nanoid9 } from "nanoid";
|
|
16675
17995
|
import { writeFileSync as writeFileSync8, existsSync as existsSync12, mkdirSync as mkdirSync9 } from "fs";
|
|
16676
17996
|
import { join as join12 } from "path";
|
|
16677
17997
|
function buildAnalyticsSessionPayload(result, opts) {
|
|
@@ -16691,18 +18011,6 @@ function buildAnalyticsSessionPayload(result, opts) {
|
|
|
16691
18011
|
browser_mode: opts.browser_mode ?? "unknown"
|
|
16692
18012
|
};
|
|
16693
18013
|
}
|
|
16694
|
-
function harEntriesToRawRequests(entries) {
|
|
16695
|
-
return entries.filter((e) => e.request && e.response).map((e) => ({
|
|
16696
|
-
url: e.request.url,
|
|
16697
|
-
method: e.request.method,
|
|
16698
|
-
request_headers: Object.fromEntries((e.request.headers ?? []).map((h) => [h.name.toLowerCase(), h.value])),
|
|
16699
|
-
request_body: e.request.postData?.text,
|
|
16700
|
-
response_status: e.response.status,
|
|
16701
|
-
response_headers: Object.fromEntries((e.response.headers ?? []).map((h) => [h.name.toLowerCase(), h.value])),
|
|
16702
|
-
response_body: e.response.content?.text,
|
|
16703
|
-
timestamp: e.startedDateTime ?? new Date().toISOString()
|
|
16704
|
-
}));
|
|
16705
|
-
}
|
|
16706
18014
|
function passiveIndexFromRequests(requests, pageUrl) {
|
|
16707
18015
|
if (requests.length === 0)
|
|
16708
18016
|
return;
|
|
@@ -16739,7 +18047,7 @@ function passiveIndexFromRequests(requests, pageUrl) {
|
|
|
16739
18047
|
const enrichedEndpoints = mergedEndpoints;
|
|
16740
18048
|
const operationGraph = buildSkillOperationGraph(enrichedEndpoints);
|
|
16741
18049
|
const skill = {
|
|
16742
|
-
skill_id: existingSkill?.skill_id ??
|
|
18050
|
+
skill_id: existingSkill?.skill_id ?? nanoid9(),
|
|
16743
18051
|
version: "1.0.0",
|
|
16744
18052
|
schema_version: "1",
|
|
16745
18053
|
lifecycle: "active",
|
|
@@ -17280,148 +18588,245 @@ async function registerRoutes(app) {
|
|
|
17280
18588
|
return "unknown";
|
|
17281
18589
|
}
|
|
17282
18590
|
}
|
|
17283
|
-
async function
|
|
17284
|
-
|
|
17285
|
-
|
|
17286
|
-
|
|
17287
|
-
|
|
17288
|
-
|
|
17289
|
-
await harStart(tabId).catch(() => {});
|
|
17290
|
-
await injectInterceptor(tabId);
|
|
17291
|
-
const session = { tabId, url: "about:blank", harActive: true, domain: "" };
|
|
17292
|
-
browseSessions.set("default", session);
|
|
17293
|
-
return session;
|
|
18591
|
+
async function restartBrowseCapture(session) {
|
|
18592
|
+
await networkEnable(session.tabId).catch(() => {});
|
|
18593
|
+
await harStart(session.tabId).catch(() => {});
|
|
18594
|
+
await scriptInject(session.tabId, INTERCEPTOR_SCRIPT).catch(() => {});
|
|
18595
|
+
session.harActive = true;
|
|
18596
|
+
await injectInterceptor(session.tabId).catch(() => {});
|
|
17294
18597
|
}
|
|
17295
18598
|
app.post("/v1/browse/go", async (req, reply) => {
|
|
17296
18599
|
const { url } = req.body;
|
|
17297
18600
|
if (!url)
|
|
17298
18601
|
return reply.code(400).send({ error: "url required" });
|
|
17299
|
-
const session = await
|
|
17300
|
-
|
|
17301
|
-
|
|
17302
|
-
|
|
17303
|
-
|
|
17304
|
-
|
|
17305
|
-
|
|
17306
|
-
|
|
17307
|
-
|
|
17308
|
-
|
|
17309
|
-
|
|
17310
|
-
|
|
17311
|
-
|
|
17312
|
-
|
|
17313
|
-
|
|
17314
|
-
|
|
17315
|
-
|
|
17316
|
-
|
|
17317
|
-
|
|
17318
|
-
|
|
18602
|
+
const { session, result } = await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session2) => {
|
|
18603
|
+
const newDomain = profileName(url);
|
|
18604
|
+
if (session2.harActive && session2.url !== "about:blank") {
|
|
18605
|
+
try {
|
|
18606
|
+
const { entries } = await harStop(session2.tabId);
|
|
18607
|
+
passiveIndexHar(entries, session2.url);
|
|
18608
|
+
} catch {}
|
|
18609
|
+
session2.harActive = false;
|
|
18610
|
+
}
|
|
18611
|
+
if (session2.domain && session2.domain !== newDomain) {
|
|
18612
|
+
await authProfileSave(session2.tabId, session2.domain).catch(() => {});
|
|
18613
|
+
}
|
|
18614
|
+
let cookiesInjected = 0;
|
|
18615
|
+
if (newDomain && newDomain !== session2.domain) {
|
|
18616
|
+
await authProfileLoad(session2.tabId, newDomain).catch(() => {});
|
|
18617
|
+
try {
|
|
18618
|
+
const { cookies: browserCookies } = extractBrowserCookies(newDomain);
|
|
18619
|
+
if (browserCookies.length > 0) {
|
|
18620
|
+
for (const c of browserCookies) {
|
|
18621
|
+
await setCookie(session2.tabId, c).catch(() => {});
|
|
18622
|
+
}
|
|
18623
|
+
cookiesInjected = browserCookies.length;
|
|
17319
18624
|
}
|
|
17320
|
-
|
|
17321
|
-
|
|
17322
|
-
|
|
17323
|
-
|
|
17324
|
-
|
|
17325
|
-
|
|
17326
|
-
|
|
17327
|
-
|
|
17328
|
-
|
|
17329
|
-
|
|
17330
|
-
|
|
18625
|
+
} catch {}
|
|
18626
|
+
}
|
|
18627
|
+
await restartBrowseCapture(session2);
|
|
18628
|
+
await navigate(session2.tabId, url);
|
|
18629
|
+
const finalUrl = await getCurrentUrl(session2.tabId).catch(() => url);
|
|
18630
|
+
session2.url = typeof finalUrl === "string" && finalUrl.startsWith("http") ? finalUrl : url;
|
|
18631
|
+
session2.domain = profileName(session2.url);
|
|
18632
|
+
await injectInterceptor(session2.tabId);
|
|
18633
|
+
return { cookiesInjected };
|
|
18634
|
+
}, (result2) => isRecoverableBrowseFailure(result2));
|
|
18635
|
+
return reply.send({
|
|
18636
|
+
ok: true,
|
|
18637
|
+
url: session.url,
|
|
18638
|
+
tab_id: session.tabId,
|
|
18639
|
+
auth_profile: session.domain,
|
|
18640
|
+
...result.cookiesInjected > 0 ? { cookies_injected: result.cookiesInjected } : {}
|
|
18641
|
+
});
|
|
18642
|
+
});
|
|
18643
|
+
app.post("/v1/browse/submit", async (req, reply) => {
|
|
18644
|
+
const {
|
|
18645
|
+
form_selector: formSelector,
|
|
18646
|
+
submit_selector: submitSelector,
|
|
18647
|
+
wait_for: waitFor,
|
|
18648
|
+
same_origin_fetch_fallback: sameOriginFetchFallback,
|
|
18649
|
+
timeout_ms: timeoutMs
|
|
18650
|
+
} = req.body ?? {};
|
|
18651
|
+
const { session, result, recovered } = await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session2) => submitBrowseForm({
|
|
18652
|
+
client: exports_client,
|
|
18653
|
+
session: session2,
|
|
18654
|
+
restartCapture: restartBrowseCapture,
|
|
18655
|
+
rehydratePlugins: bestEffortRehydratePlugins
|
|
18656
|
+
}, {
|
|
18657
|
+
formSelector,
|
|
18658
|
+
submitSelector,
|
|
18659
|
+
waitFor,
|
|
18660
|
+
sameOriginFetchFallback,
|
|
18661
|
+
timeoutMs
|
|
18662
|
+
}), (result2) => !result2.ok && result2.recoverable === true);
|
|
18663
|
+
session.url = result.url || await getCurrentUrl(session.tabId).catch(() => session.url);
|
|
17331
18664
|
session.domain = profileName(session.url);
|
|
17332
|
-
|
|
17333
|
-
return reply.send({
|
|
18665
|
+
const statusCode = result.ok ? 200 : result.recoverable ? 502 : 400;
|
|
18666
|
+
return reply.code(statusCode).send({
|
|
18667
|
+
...result,
|
|
18668
|
+
recovered,
|
|
18669
|
+
tab_id: session.tabId,
|
|
18670
|
+
url: session.url
|
|
18671
|
+
});
|
|
17334
18672
|
});
|
|
17335
18673
|
app.post("/v1/browse/snap", async (req, reply) => {
|
|
17336
18674
|
const { filter } = req.body ?? {};
|
|
17337
|
-
const session = await
|
|
17338
|
-
const snapshot2 = await snapshot(session.tabId, filter);
|
|
18675
|
+
const { session, result: snapshot2 } = await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session2) => snapshot(session2.tabId, filter), (snapshot3) => typeof snapshot3 !== "string" || snapshot3.trim().length === 0);
|
|
17339
18676
|
return reply.send({ snapshot: snapshot2, tab_id: session.tabId });
|
|
17340
18677
|
});
|
|
17341
18678
|
app.post("/v1/browse/click", async (req, reply) => {
|
|
17342
18679
|
const { ref } = req.body;
|
|
17343
18680
|
if (!ref)
|
|
17344
18681
|
return reply.code(400).send({ error: "ref required" });
|
|
17345
|
-
|
|
17346
|
-
|
|
18682
|
+
await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session) => {
|
|
18683
|
+
await click(session.tabId, ref);
|
|
18684
|
+
return true;
|
|
18685
|
+
});
|
|
17347
18686
|
return reply.send({ ok: true });
|
|
17348
18687
|
});
|
|
17349
18688
|
app.post("/v1/browse/fill", async (req, reply) => {
|
|
17350
18689
|
const { ref, value } = req.body;
|
|
17351
18690
|
if (!ref || value === undefined)
|
|
17352
18691
|
return reply.code(400).send({ error: "ref and value required" });
|
|
17353
|
-
|
|
17354
|
-
|
|
18692
|
+
await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session) => {
|
|
18693
|
+
await fill(session.tabId, ref, value);
|
|
18694
|
+
return true;
|
|
18695
|
+
});
|
|
17355
18696
|
return reply.send({ ok: true });
|
|
17356
18697
|
});
|
|
17357
18698
|
app.post("/v1/browse/type", async (req, reply) => {
|
|
17358
18699
|
const { text } = req.body;
|
|
17359
18700
|
if (!text)
|
|
17360
18701
|
return reply.code(400).send({ error: "text required" });
|
|
17361
|
-
|
|
17362
|
-
|
|
18702
|
+
await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session) => {
|
|
18703
|
+
await keyboardType(session.tabId, text);
|
|
18704
|
+
return true;
|
|
18705
|
+
});
|
|
17363
18706
|
return reply.send({ ok: true });
|
|
17364
18707
|
});
|
|
17365
18708
|
app.post("/v1/browse/press", async (req, reply) => {
|
|
17366
18709
|
const { key } = req.body;
|
|
17367
18710
|
if (!key)
|
|
17368
18711
|
return reply.code(400).send({ error: "key required" });
|
|
17369
|
-
|
|
17370
|
-
|
|
18712
|
+
await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session) => {
|
|
18713
|
+
await press(session.tabId, key);
|
|
18714
|
+
return true;
|
|
18715
|
+
});
|
|
17371
18716
|
return reply.send({ ok: true });
|
|
17372
18717
|
});
|
|
17373
18718
|
app.post("/v1/browse/select", async (req, reply) => {
|
|
17374
18719
|
const { ref, value } = req.body;
|
|
17375
18720
|
if (!ref || value === undefined)
|
|
17376
18721
|
return reply.code(400).send({ error: "ref and value required" });
|
|
17377
|
-
|
|
17378
|
-
|
|
18722
|
+
await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session) => {
|
|
18723
|
+
await select(session.tabId, ref, value);
|
|
18724
|
+
return true;
|
|
18725
|
+
});
|
|
17379
18726
|
return reply.send({ ok: true });
|
|
17380
18727
|
});
|
|
17381
18728
|
app.post("/v1/browse/scroll", async (req, reply) => {
|
|
17382
18729
|
const { direction, amount } = req.body ?? {};
|
|
17383
|
-
|
|
17384
|
-
|
|
18730
|
+
await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session) => {
|
|
18731
|
+
await scroll(session.tabId, direction ?? "down", amount);
|
|
18732
|
+
return true;
|
|
18733
|
+
});
|
|
17385
18734
|
return reply.send({ ok: true });
|
|
17386
18735
|
});
|
|
17387
18736
|
app.get("/v1/browse/screenshot", async (_req, reply) => {
|
|
17388
|
-
const session = await
|
|
17389
|
-
const data = await screenshot(session.tabId);
|
|
18737
|
+
const { session, result: data } = await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session2) => screenshot(session2.tabId), (data2) => typeof data2 !== "string" || data2.trim().length === 0);
|
|
17390
18738
|
return reply.send({ screenshot: data, tab_id: session.tabId });
|
|
17391
18739
|
});
|
|
17392
18740
|
app.get("/v1/browse/text", async (_req, reply) => {
|
|
17393
|
-
const
|
|
17394
|
-
const text = await getText(session.tabId);
|
|
18741
|
+
const { result: text } = await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session) => getText(session.tabId), (text2) => typeof text2 !== "string");
|
|
17395
18742
|
return reply.send({ text });
|
|
17396
18743
|
});
|
|
17397
18744
|
app.get("/v1/browse/markdown", async (_req, reply) => {
|
|
17398
|
-
const
|
|
17399
|
-
const markdown = await getMarkdown(session.tabId);
|
|
18745
|
+
const { result: markdown } = await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session) => getMarkdown(session.tabId), (markdown2) => typeof markdown2 !== "string");
|
|
17400
18746
|
return reply.send({ markdown });
|
|
17401
18747
|
});
|
|
17402
18748
|
app.get("/v1/browse/cookies", async (_req, reply) => {
|
|
17403
|
-
const
|
|
17404
|
-
const cookies = await getCookies(session.tabId);
|
|
18749
|
+
const { result: cookies } = await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session) => getCookies(session.tabId));
|
|
17405
18750
|
return reply.send({ cookies });
|
|
17406
18751
|
});
|
|
17407
18752
|
app.post("/v1/browse/eval", async (req, reply) => {
|
|
17408
18753
|
const { expression } = req.body;
|
|
17409
18754
|
if (!expression)
|
|
17410
18755
|
return reply.code(400).send({ error: "expression required" });
|
|
17411
|
-
const
|
|
17412
|
-
const result = await evaluate(session.tabId, expression);
|
|
18756
|
+
const { result } = await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session) => evaluate(session.tabId, expression), (result2) => isRecoverableBrowseFailure(result2));
|
|
17413
18757
|
return reply.send({ result });
|
|
17414
18758
|
});
|
|
17415
18759
|
app.post("/v1/browse/back", async (_req, reply) => {
|
|
17416
|
-
|
|
17417
|
-
|
|
18760
|
+
await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session) => {
|
|
18761
|
+
await goBack(session.tabId);
|
|
18762
|
+
return true;
|
|
18763
|
+
});
|
|
17418
18764
|
return reply.send({ ok: true });
|
|
17419
18765
|
});
|
|
17420
18766
|
app.post("/v1/browse/forward", async (_req, reply) => {
|
|
17421
|
-
|
|
17422
|
-
|
|
18767
|
+
await withRecoveredBrowseSession(browseSessions, exports_client, injectInterceptor, async (session) => {
|
|
18768
|
+
await goForward(session.tabId);
|
|
18769
|
+
return true;
|
|
18770
|
+
});
|
|
17423
18771
|
return reply.send({ ok: true });
|
|
17424
18772
|
});
|
|
18773
|
+
app.post("/v1/browse/sync", async (_req, reply) => {
|
|
18774
|
+
const session = browseSessions.get("default");
|
|
18775
|
+
if (!session)
|
|
18776
|
+
return reply.send({ ok: false, error: "no active session" });
|
|
18777
|
+
let intercepted = [];
|
|
18778
|
+
try {
|
|
18779
|
+
const raw = await collectInterceptedRequests(session.tabId);
|
|
18780
|
+
intercepted = raw.map((request) => ({
|
|
18781
|
+
url: request.url,
|
|
18782
|
+
method: request.method,
|
|
18783
|
+
request_headers: request.request_headers ?? {},
|
|
18784
|
+
request_body: request.request_body,
|
|
18785
|
+
response_status: request.response_status,
|
|
18786
|
+
response_headers: request.response_headers ?? {},
|
|
18787
|
+
response_body: request.response_body,
|
|
18788
|
+
timestamp: request.timestamp
|
|
18789
|
+
}));
|
|
18790
|
+
} catch {}
|
|
18791
|
+
let harEntries = [];
|
|
18792
|
+
if (session.harActive) {
|
|
18793
|
+
try {
|
|
18794
|
+
const { entries } = await harStop(session.tabId);
|
|
18795
|
+
harEntries = entries;
|
|
18796
|
+
} catch {}
|
|
18797
|
+
}
|
|
18798
|
+
session.harActive = false;
|
|
18799
|
+
const allRequests = mergeBrowseRequests(intercepted, harEntries, session.url);
|
|
18800
|
+
const syncResult = await cacheBrowseRequests({
|
|
18801
|
+
sessionUrl: session.url,
|
|
18802
|
+
sessionDomain: session.domain,
|
|
18803
|
+
requests: allRequests,
|
|
18804
|
+
getPageHtml: () => getPageHtml(session.tabId)
|
|
18805
|
+
});
|
|
18806
|
+
await networkEnable(session.tabId).catch(() => {});
|
|
18807
|
+
await harStart(session.tabId).catch(() => {});
|
|
18808
|
+
await scriptInject(session.tabId, INTERCEPTOR_SCRIPT).catch(() => {});
|
|
18809
|
+
session.harActive = true;
|
|
18810
|
+
await injectInterceptor(session.tabId).catch(() => {});
|
|
18811
|
+
return reply.send({
|
|
18812
|
+
ok: true,
|
|
18813
|
+
tab_id: session.tabId,
|
|
18814
|
+
indexed: syncResult.indexed,
|
|
18815
|
+
mode: syncResult.mode,
|
|
18816
|
+
domain: syncResult.domain,
|
|
18817
|
+
skill_id: syncResult.skill?.skill_id ?? null,
|
|
18818
|
+
endpoint_count: syncResult.skill?.endpoints.length ?? 0,
|
|
18819
|
+
endpoints: (syncResult.skill?.endpoints ?? []).map((endpoint) => ({
|
|
18820
|
+
endpoint_id: endpoint.endpoint_id,
|
|
18821
|
+
method: endpoint.method,
|
|
18822
|
+
url_template: endpoint.url_template,
|
|
18823
|
+
description: endpoint.description,
|
|
18824
|
+
trigger_url: endpoint.trigger_url,
|
|
18825
|
+
action_kind: endpoint.semantic?.action_kind,
|
|
18826
|
+
resource_kind: endpoint.semantic?.resource_kind
|
|
18827
|
+
}))
|
|
18828
|
+
});
|
|
18829
|
+
});
|
|
17425
18830
|
app.post("/v1/browse/close", async (_req, reply) => {
|
|
17426
18831
|
const session = browseSessions.get("default");
|
|
17427
18832
|
if (!session)
|
|
@@ -17450,174 +18855,23 @@ async function registerRoutes(app) {
|
|
|
17450
18855
|
harEntries = entries;
|
|
17451
18856
|
} catch {}
|
|
17452
18857
|
}
|
|
17453
|
-
const
|
|
17454
|
-
const
|
|
17455
|
-
|
|
17456
|
-
|
|
17457
|
-
|
|
17458
|
-
|
|
17459
|
-
|
|
17460
|
-
allRequests.push(r);
|
|
17461
|
-
}
|
|
17462
|
-
}
|
|
17463
|
-
for (const r of harRequests) {
|
|
17464
|
-
const key = `${r.method}:${r.url}`;
|
|
17465
|
-
if (!seen.has(key)) {
|
|
17466
|
-
seen.add(key);
|
|
17467
|
-
allRequests.push(r);
|
|
17468
|
-
}
|
|
17469
|
-
}
|
|
17470
|
-
{
|
|
17471
|
-
let domain;
|
|
17472
|
-
try {
|
|
17473
|
-
domain = new URL(session.url).hostname;
|
|
17474
|
-
} catch {
|
|
17475
|
-
domain = session.domain;
|
|
17476
|
-
}
|
|
17477
|
-
const rawEndpoints = extractEndpoints(allRequests, undefined, { pageUrl: session.url, finalUrl: session.url });
|
|
17478
|
-
if (rawEndpoints.length > 0) {
|
|
17479
|
-
const existingSkill = findExistingSkillForDomain(domain);
|
|
17480
|
-
let allExisting = existingSkill?.endpoints ?? [];
|
|
17481
|
-
const domainKey = getDomainReuseKey(session.url ?? domain);
|
|
17482
|
-
if (domainKey) {
|
|
17483
|
-
const cached = domainSkillCache.get(domainKey);
|
|
17484
|
-
if (cached?.localSkillPath) {
|
|
17485
|
-
try {
|
|
17486
|
-
const snapshot2 = JSON.parse(__require("fs").readFileSync(cached.localSkillPath, "utf-8"));
|
|
17487
|
-
if (snapshot2?.endpoints?.length > 0) {
|
|
17488
|
-
allExisting = mergeEndpoints(allExisting, snapshot2.endpoints);
|
|
17489
|
-
}
|
|
17490
|
-
} catch {}
|
|
17491
|
-
}
|
|
17492
|
-
}
|
|
17493
|
-
const mergedEps = allExisting.length > 0 ? mergeEndpoints(allExisting, rawEndpoints) : rawEndpoints;
|
|
17494
|
-
if (!existingSkill || mergedEps.length >= existingSkill.endpoints.length) {
|
|
17495
|
-
for (const ep of mergedEps) {
|
|
17496
|
-
if (!ep.description)
|
|
17497
|
-
ep.description = generateLocalDescription(ep);
|
|
17498
|
-
}
|
|
17499
|
-
const quickSkill = {
|
|
17500
|
-
skill_id: existingSkill?.skill_id ?? nanoid8(),
|
|
17501
|
-
version: "1.0.0",
|
|
17502
|
-
schema_version: "1",
|
|
17503
|
-
lifecycle: "active",
|
|
17504
|
-
execution_type: "http",
|
|
17505
|
-
created_at: existingSkill?.created_at ?? new Date().toISOString(),
|
|
17506
|
-
updated_at: new Date().toISOString(),
|
|
17507
|
-
name: domain,
|
|
17508
|
-
intent_signature: `browse ${domain}`,
|
|
17509
|
-
domain,
|
|
17510
|
-
description: `API skill for ${domain}`,
|
|
17511
|
-
owner_type: "agent",
|
|
17512
|
-
endpoints: mergedEps,
|
|
17513
|
-
intents: Array.from(new Set([...existingSkill?.intents ?? [], `browse ${domain}`]))
|
|
17514
|
-
};
|
|
17515
|
-
const cacheKey = buildResolveCacheKey(domain, `browse ${domain}`, session.url);
|
|
17516
|
-
const scopedKey = scopedCacheKey("global", cacheKey);
|
|
17517
|
-
writeSkillSnapshot(scopedKey, quickSkill);
|
|
17518
|
-
const domainKey2 = getDomainReuseKey(session.url ?? domain);
|
|
17519
|
-
if (domainKey2) {
|
|
17520
|
-
domainSkillCache.set(domainKey2, {
|
|
17521
|
-
skillId: quickSkill.skill_id,
|
|
17522
|
-
localSkillPath: snapshotPathForCacheKey(scopedKey),
|
|
17523
|
-
ts: Date.now()
|
|
17524
|
-
});
|
|
17525
|
-
persistDomainCache();
|
|
17526
|
-
}
|
|
17527
|
-
try {
|
|
17528
|
-
cachePublishedSkill(quickSkill);
|
|
17529
|
-
} catch {}
|
|
17530
|
-
invalidateRouteCacheForDomain(domain);
|
|
17531
|
-
console.log(`[passive-index] ${domain}: ${mergedEps.length} endpoints cached synchronously`);
|
|
17532
|
-
}
|
|
17533
|
-
} else {
|
|
17534
|
-
let domain2;
|
|
17535
|
-
try {
|
|
17536
|
-
domain2 = new URL(session.url).hostname;
|
|
17537
|
-
} catch {
|
|
17538
|
-
domain2 = session.domain;
|
|
17539
|
-
}
|
|
17540
|
-
try {
|
|
17541
|
-
const html = await getPageHtml(session.tabId);
|
|
17542
|
-
if (html && typeof html === "string" && html.startsWith("<")) {
|
|
17543
|
-
const { extractFromDOM: extractFromDOM2 } = await Promise.resolve().then(() => (init_extraction(), exports_extraction));
|
|
17544
|
-
const { detectSearchForms: detectSearchForms2, isStructuredSearchForm: isStructuredSearchForm2 } = await Promise.resolve().then(() => (init_search_forms(), exports_search_forms));
|
|
17545
|
-
const { inferSchema: inferSchema2 } = await Promise.resolve().then(() => (init_transform(), exports_transform));
|
|
17546
|
-
const { inferEndpointSemantic: inferEndpointSemantic2 } = await Promise.resolve().then(() => (init_graph(), exports_graph));
|
|
17547
|
-
const { templatizeQueryParams: templatizeQueryParams2 } = await init_execution().then(() => exports_execution);
|
|
17548
|
-
const extracted = extractFromDOM2(html, `browse ${domain2}`);
|
|
17549
|
-
const searchForms = detectSearchForms2(html);
|
|
17550
|
-
const validForm = searchForms.find((s) => isStructuredSearchForm2(s));
|
|
17551
|
-
if (extracted.data || validForm) {
|
|
17552
|
-
const urlTemplate = templatizeQueryParams2(session.url);
|
|
17553
|
-
const ep = {
|
|
17554
|
-
endpoint_id: nanoid8(),
|
|
17555
|
-
method: "GET",
|
|
17556
|
-
url_template: urlTemplate,
|
|
17557
|
-
idempotency: "safe",
|
|
17558
|
-
verification_status: "verified",
|
|
17559
|
-
reliability_score: extracted.confidence ?? 0.7,
|
|
17560
|
-
description: validForm ? `Search form for ${domain2}` : `Page content from ${domain2}`,
|
|
17561
|
-
response_schema: extracted.data ? inferSchema2([extracted.data]) : undefined,
|
|
17562
|
-
dom_extraction: {
|
|
17563
|
-
extraction_method: extracted.extraction_method ?? "repeated-elements",
|
|
17564
|
-
confidence: extracted.confidence ?? 0.7,
|
|
17565
|
-
...extracted.selector ? { selector: extracted.selector } : {},
|
|
17566
|
-
...validForm ? { search_form: validForm } : {}
|
|
17567
|
-
},
|
|
17568
|
-
trigger_url: session.url
|
|
17569
|
-
};
|
|
17570
|
-
ep.semantic = inferEndpointSemantic2(ep, {
|
|
17571
|
-
sampleResponse: extracted.data,
|
|
17572
|
-
observedAt: new Date().toISOString(),
|
|
17573
|
-
sampleRequestUrl: session.url
|
|
17574
|
-
});
|
|
17575
|
-
const existing = findExistingSkillForDomain(domain2);
|
|
17576
|
-
const allEps = existing ? mergeEndpoints(existing.endpoints, [ep]) : [ep];
|
|
17577
|
-
for (const e of allEps) {
|
|
17578
|
-
if (!e.description)
|
|
17579
|
-
e.description = generateLocalDescription(e);
|
|
17580
|
-
}
|
|
17581
|
-
const skill = {
|
|
17582
|
-
skill_id: existing?.skill_id ?? nanoid8(),
|
|
17583
|
-
version: "1.0.0",
|
|
17584
|
-
schema_version: "1",
|
|
17585
|
-
lifecycle: "active",
|
|
17586
|
-
execution_type: "http",
|
|
17587
|
-
created_at: existing?.created_at ?? new Date().toISOString(),
|
|
17588
|
-
updated_at: new Date().toISOString(),
|
|
17589
|
-
name: domain2,
|
|
17590
|
-
intent_signature: `browse ${domain2}`,
|
|
17591
|
-
domain: domain2,
|
|
17592
|
-
description: `DOM skill for ${domain2}`,
|
|
17593
|
-
owner_type: "agent",
|
|
17594
|
-
endpoints: allEps,
|
|
17595
|
-
intents: [...new Set([...existing?.intents ?? [], `browse ${domain2}`])]
|
|
17596
|
-
};
|
|
17597
|
-
const ck = buildResolveCacheKey(domain2, `browse ${domain2}`, session.url);
|
|
17598
|
-
const sk = scopedCacheKey("global", ck);
|
|
17599
|
-
writeSkillSnapshot(sk, skill);
|
|
17600
|
-
const dk = getDomainReuseKey(session.url ?? domain2);
|
|
17601
|
-
if (dk) {
|
|
17602
|
-
domainSkillCache.set(dk, { skillId: skill.skill_id, localSkillPath: snapshotPathForCacheKey(sk), ts: Date.now() });
|
|
17603
|
-
persistDomainCache();
|
|
17604
|
-
}
|
|
17605
|
-
try {
|
|
17606
|
-
cachePublishedSkill(skill);
|
|
17607
|
-
} catch {}
|
|
17608
|
-
invalidateRouteCacheForDomain(domain2);
|
|
17609
|
-
console.log(`[close] ${domain2}: DOM endpoint created (form=${!!validForm})`);
|
|
17610
|
-
}
|
|
17611
|
-
}
|
|
17612
|
-
} catch (err) {
|
|
17613
|
-
console.log(`[close] DOM fallback failed: ${err instanceof Error ? err.message : err}`);
|
|
17614
|
-
}
|
|
17615
|
-
}
|
|
17616
|
-
}
|
|
18858
|
+
const allRequests = mergeBrowseRequests(intercepted, harEntries, session.url);
|
|
18859
|
+
const syncResult = await cacheBrowseRequests({
|
|
18860
|
+
sessionUrl: session.url,
|
|
18861
|
+
sessionDomain: session.domain,
|
|
18862
|
+
requests: allRequests,
|
|
18863
|
+
getPageHtml: () => getPageHtml(session.tabId)
|
|
18864
|
+
});
|
|
17617
18865
|
passiveIndexFromRequests(allRequests, session.url);
|
|
17618
18866
|
await closeTab(session.tabId).catch(() => {});
|
|
17619
18867
|
browseSessions.delete("default");
|
|
17620
|
-
return reply.send({
|
|
18868
|
+
return reply.send({
|
|
18869
|
+
ok: true,
|
|
18870
|
+
indexed: syncResult.indexed,
|
|
18871
|
+
mode: syncResult.mode,
|
|
18872
|
+
endpoint_count: syncResult.skill?.endpoints.length ?? 0,
|
|
18873
|
+
auth_saved: session.domain || null
|
|
18874
|
+
});
|
|
17621
18875
|
});
|
|
17622
18876
|
}
|
|
17623
18877
|
function saveTrace(trace) {
|
|
@@ -17642,6 +18896,8 @@ var init_routes = __esm(async () => {
|
|
|
17642
18896
|
init_ratelimit();
|
|
17643
18897
|
init_graph();
|
|
17644
18898
|
init_session_logs();
|
|
18899
|
+
init_browse_session();
|
|
18900
|
+
init_browse_submit();
|
|
17645
18901
|
await __promiseAll([
|
|
17646
18902
|
init_indexer(),
|
|
17647
18903
|
init_vault(),
|
|
@@ -17649,7 +18905,8 @@ var init_routes = __esm(async () => {
|
|
|
17649
18905
|
init_orchestrator(),
|
|
17650
18906
|
init_execution(),
|
|
17651
18907
|
init_auth(),
|
|
17652
|
-
init_indexer()
|
|
18908
|
+
init_indexer(),
|
|
18909
|
+
init_browse_index()
|
|
17653
18910
|
]);
|
|
17654
18911
|
BETA_API_URL = process.env.UNBROWSE_BACKEND_URL || "https://beta-api.unbrowse.ai";
|
|
17655
18912
|
TRACES_DIR = process.env.TRACES_DIR ?? join12(process.cwd(), "traces");
|
|
@@ -18286,7 +19543,7 @@ function buildDepsMetadata(pack, taskName) {
|
|
|
18286
19543
|
// ../../src/runtime/local-server.ts
|
|
18287
19544
|
init_paths();
|
|
18288
19545
|
init_supervisor();
|
|
18289
|
-
import { openSync, readFileSync as
|
|
19546
|
+
import { openSync, readFileSync as readFileSync10, unlinkSync as unlinkSync2, writeFileSync as writeFileSync10 } from "node:fs";
|
|
18290
19547
|
import path6 from "node:path";
|
|
18291
19548
|
import { spawn as spawn2 } from "node:child_process";
|
|
18292
19549
|
async function isServerHealthy(baseUrl, timeoutMs = 2000) {
|
|
@@ -18316,7 +19573,7 @@ function isPidAlive(pid) {
|
|
|
18316
19573
|
}
|
|
18317
19574
|
function readPidState(pidFile) {
|
|
18318
19575
|
try {
|
|
18319
|
-
return JSON.parse(
|
|
19576
|
+
return JSON.parse(readFileSync10(pidFile, "utf-8"));
|
|
18320
19577
|
} catch {
|
|
18321
19578
|
return null;
|
|
18322
19579
|
}
|
|
@@ -18335,7 +19592,7 @@ function deriveListenEnv(baseUrl) {
|
|
|
18335
19592
|
function getVersion(metaUrl) {
|
|
18336
19593
|
try {
|
|
18337
19594
|
const root = getPackageRoot(metaUrl);
|
|
18338
|
-
const pkg = JSON.parse(
|
|
19595
|
+
const pkg = JSON.parse(readFileSync10(path6.join(root, "package.json"), "utf-8"));
|
|
18339
19596
|
return pkg.version ?? "unknown";
|
|
18340
19597
|
} catch {
|
|
18341
19598
|
return "unknown";
|
|
@@ -18682,7 +19939,7 @@ async function runSetup(options) {
|
|
|
18682
19939
|
const wallet = {
|
|
18683
19940
|
...finalWalletCheck,
|
|
18684
19941
|
lobster_installed: lobsterInstalled,
|
|
18685
|
-
message: finalWalletCheck.configured ? `Wallet configured (${finalWalletCheck.provider})
|
|
19942
|
+
message: finalWalletCheck.configured ? `Wallet configured (${finalWalletCheck.provider}). This address is the contributor truth: it is synced onto your agent profile, used for contributor payouts when your routes earn, and used for paid-route spending.` : lobsterInstalled ? "lobster.cash installed but wallet not paired. Pair it now so this wallet address becomes your contributor payout target and your paid-route spending wallet. Run: lobstercash setup" : "No wallet configured. Install/pair a wallet so your contributor payouts have a destination address and premium-route spending can clear automatically. Without it you stay in free indexing mode only.",
|
|
18686
19943
|
install_hint: finalWalletCheck.configured ? undefined : lobsterInstalled ? "lobstercash setup" : "npx skills add https://github.com/Crossmint/lobstercash-cli-skills --global --yes"
|
|
18687
19944
|
};
|
|
18688
19945
|
return {
|
|
@@ -18756,6 +20013,15 @@ function info(msg) {
|
|
|
18756
20013
|
process.stderr.write(`[unbrowse] ${msg}
|
|
18757
20014
|
`);
|
|
18758
20015
|
}
|
|
20016
|
+
function resolveResultError(result) {
|
|
20017
|
+
return result.result?.error ?? result.error;
|
|
20018
|
+
}
|
|
20019
|
+
function resolveLoginUrl(result, fallbackUrl) {
|
|
20020
|
+
return result.result?.login_url ?? fallbackUrl ?? "";
|
|
20021
|
+
}
|
|
20022
|
+
function hasIndexingFallback(result) {
|
|
20023
|
+
return result.result?.indexing_fallback_available === true;
|
|
20024
|
+
}
|
|
18759
20025
|
async function withPendingNotice(promise, message, delayMs = 3000) {
|
|
18760
20026
|
let done = false;
|
|
18761
20027
|
const timer = setTimeout(() => {
|
|
@@ -18838,20 +20104,51 @@ async function cmdResolve(flags) {
|
|
|
18838
20104
|
return result.skill?.skill_id ?? result.skill_id;
|
|
18839
20105
|
}
|
|
18840
20106
|
const startedAt = Date.now();
|
|
18841
|
-
|
|
18842
|
-
|
|
18843
|
-
|
|
18844
|
-
|
|
18845
|
-
|
|
18846
|
-
|
|
18847
|
-
|
|
18848
|
-
|
|
20107
|
+
async function resolveOnce(message = "Still working. First-time capture/indexing for a site can take 20-80s. Waiting is usually better than falling back.") {
|
|
20108
|
+
return withPendingNotice(api3("POST", "/v1/intent/resolve", body), message);
|
|
20109
|
+
}
|
|
20110
|
+
let result = await resolveOnce();
|
|
20111
|
+
let attemptedForceCapture = !!body.force_capture;
|
|
20112
|
+
let attemptedCookieImport = false;
|
|
20113
|
+
let attemptedInteractiveLogin = false;
|
|
20114
|
+
while (true) {
|
|
20115
|
+
const resultError = resolveResultError(result);
|
|
20116
|
+
if (resultError === "payment_required" && hasIndexingFallback(result) && url && !attemptedForceCapture) {
|
|
20117
|
+
attemptedForceCapture = true;
|
|
20118
|
+
body.force_capture = true;
|
|
20119
|
+
info("Marketplace search is paid here. Falling back to free live capture on the exact URL...");
|
|
20120
|
+
result = await resolveOnce("Running free live capture...");
|
|
20121
|
+
continue;
|
|
20122
|
+
}
|
|
20123
|
+
if (resultError === "auth_required") {
|
|
20124
|
+
const loginUrl = resolveLoginUrl(result, url);
|
|
20125
|
+
if (!loginUrl)
|
|
20126
|
+
break;
|
|
20127
|
+
if (!attemptedCookieImport) {
|
|
20128
|
+
attemptedCookieImport = true;
|
|
20129
|
+
info("Site requires authentication. Trying browser cookie import first...");
|
|
20130
|
+
const stealResult = await api3("POST", "/v1/auth/steal", { url: loginUrl });
|
|
20131
|
+
const cookiesStored = typeof stealResult.cookies_stored === "number" ? stealResult.cookies_stored : Number(stealResult.cookies_stored ?? 0);
|
|
20132
|
+
if (stealResult.success === true && cookiesStored > 0) {
|
|
20133
|
+
info(`Imported ${cookiesStored} browser cookies. Retrying...`);
|
|
20134
|
+
result = await resolveOnce("Retrying after browser cookie import...");
|
|
20135
|
+
continue;
|
|
20136
|
+
}
|
|
20137
|
+
}
|
|
20138
|
+
if (!attemptedInteractiveLogin) {
|
|
20139
|
+
attemptedInteractiveLogin = true;
|
|
20140
|
+
info("Site requires authentication. Opening browser for login...");
|
|
20141
|
+
const loginResult = await api3("POST", "/v1/auth/login", { url: loginUrl });
|
|
20142
|
+
if (loginResult.error || loginResult.success === false) {
|
|
20143
|
+
const message = typeof loginResult.error === "string" ? loginResult.error : "interactive login did not produce a reusable session";
|
|
20144
|
+
die(`Login failed: ${message}. Run: unbrowse login --url "${loginUrl}"`);
|
|
20145
|
+
}
|
|
18849
20146
|
info("Login complete. Retrying...");
|
|
18850
|
-
result = await
|
|
18851
|
-
|
|
18852
|
-
die(`Login failed: ${err.message}. Run: unbrowse login --url "${loginUrl}"`);
|
|
20147
|
+
result = await resolveOnce("Retrying after login...");
|
|
20148
|
+
continue;
|
|
18853
20149
|
}
|
|
18854
20150
|
}
|
|
20151
|
+
break;
|
|
18855
20152
|
}
|
|
18856
20153
|
if (explicitEndpointId && result.available_endpoints) {
|
|
18857
20154
|
const skillId = resolveSkillId();
|
|
@@ -18871,10 +20168,19 @@ async function cmdResolve(flags) {
|
|
|
18871
20168
|
const resultObj = result.result;
|
|
18872
20169
|
if (resultObj?.status === "browse_session_open") {
|
|
18873
20170
|
info(`No cached API. Browser session open on ${resultObj.domain ?? resultObj.url}.`);
|
|
20171
|
+
info(`Preferred flow: snap -> click/fill/eval -> submit -> sync -> close.`);
|
|
18874
20172
|
info(`Use these commands to get your data:`);
|
|
18875
|
-
const commands = resultObj.commands ?? [
|
|
20173
|
+
const commands = resultObj.commands ?? [
|
|
20174
|
+
"unbrowse snap --filter interactive",
|
|
20175
|
+
"unbrowse click <ref>",
|
|
20176
|
+
"unbrowse fill <ref> <value>",
|
|
20177
|
+
'unbrowse submit --wait-for "/next-step"',
|
|
20178
|
+
"unbrowse sync",
|
|
20179
|
+
"unbrowse close"
|
|
20180
|
+
];
|
|
18876
20181
|
for (const cmd of commands)
|
|
18877
20182
|
info(` ${cmd}`);
|
|
20183
|
+
info(`For JS-heavy forms: prefer real date/time clicks first, inspect hidden inputs with eval when needed, then submit.`);
|
|
18878
20184
|
info(`All traffic is being passively captured. Run "unbrowse close" when done.`);
|
|
18879
20185
|
output(slimTrace(result), !!flags.pretty);
|
|
18880
20186
|
return;
|
|
@@ -19157,7 +20463,8 @@ var CLI_REFERENCE = {
|
|
|
19157
20463
|
{ name: "skill", usage: "<id>", desc: "Get skill details" },
|
|
19158
20464
|
{ name: "search", usage: '--intent "..." [--domain "..."]', desc: "Search marketplace" },
|
|
19159
20465
|
{ name: "sessions", usage: '--domain "..." [--limit N]', desc: "Debug session logs" },
|
|
19160
|
-
{ name: "go", usage: "<url>", desc: "
|
|
20466
|
+
{ name: "go", usage: "<url>", desc: "Open a live Kuri browser tab for capture-first workflows" },
|
|
20467
|
+
{ name: "submit", usage: "[--form-selector sel] [--submit-selector sel] [--wait-for hint]", desc: "Submit current form with DOM-first + same-origin rehydrate fallback for JS-heavy flows" },
|
|
19161
20468
|
{ name: "snap", usage: "[--filter interactive]", desc: "A11y snapshot with @eN refs" },
|
|
19162
20469
|
{ name: "click", usage: "<ref>", desc: "Click element by ref (e.g. e5)" },
|
|
19163
20470
|
{ name: "fill", usage: "<ref> <value>", desc: "Fill input by ref" },
|
|
@@ -19172,6 +20479,7 @@ var CLI_REFERENCE = {
|
|
|
19172
20479
|
{ name: "eval", usage: "<expression>", desc: "Evaluate JavaScript" },
|
|
19173
20480
|
{ name: "back", usage: "", desc: "Navigate back" },
|
|
19174
20481
|
{ name: "forward", usage: "", desc: "Navigate forward" },
|
|
20482
|
+
{ name: "sync", usage: "", desc: "Flush the current step's captured traffic into route cache without closing tab" },
|
|
19175
20483
|
{ name: "close", usage: "", desc: "Close browse session, flush + index traffic" }
|
|
19176
20484
|
],
|
|
19177
20485
|
globalFlags: [
|
|
@@ -19195,6 +20503,10 @@ var CLI_REFERENCE = {
|
|
|
19195
20503
|
"unbrowse setup",
|
|
19196
20504
|
'unbrowse resolve --intent "top stories" --url "https://news.ycombinator.com" --execute',
|
|
19197
20505
|
'unbrowse resolve --intent "get timeline" --url "https://x.com"',
|
|
20506
|
+
'unbrowse go "https://www.mandai.com/en/ticketing/admission-and-rides/parks-selection.html"',
|
|
20507
|
+
"unbrowse snap --filter interactive",
|
|
20508
|
+
'unbrowse submit --wait-for "/time-selection.html"',
|
|
20509
|
+
"unbrowse sync",
|
|
19198
20510
|
"unbrowse execute --skill abc --endpoint def --pretty",
|
|
19199
20511
|
"unbrowse execute --skill abc --endpoint def --schema --pretty",
|
|
19200
20512
|
'unbrowse execute --skill abc --endpoint def --path "data.items[]" --extract "name,url" --limit 10 --pretty',
|
|
@@ -19227,6 +20539,8 @@ function printHelp() {
|
|
|
19227
20539
|
for (const e of r.examples) {
|
|
19228
20540
|
lines.push(` ${e}`);
|
|
19229
20541
|
}
|
|
20542
|
+
lines.push("", "Browser workflow:", " 1. go -> open the live tab you want to work in", " 2. snap -> inspect refs and confirm the page state", " 3. click/fill/eval -> set real page state", " 4. submit -> prefer DOM submit; auto-falls back to same-origin rehydrate", " 5. sync -> flush captured routes after a successful step", " 6. close -> finish capture + indexing");
|
|
20543
|
+
lines.push("", "JS-heavy forms:", " Prefer real calendar/time clicks before submit.", " If the UI is flaky, inspect hidden inputs/cookies with eval, then submit the real form.");
|
|
19230
20544
|
lines.push("");
|
|
19231
20545
|
process.stderr.write(lines.join(`
|
|
19232
20546
|
`));
|
|
@@ -19383,6 +20697,21 @@ async function cmdGo(args, flags) {
|
|
|
19383
20697
|
die("Usage: unbrowse go <url>");
|
|
19384
20698
|
output(await api3("POST", "/v1/browse/go", { url }), !!flags.pretty);
|
|
19385
20699
|
}
|
|
20700
|
+
async function cmdSubmit(flags) {
|
|
20701
|
+
const body = {};
|
|
20702
|
+
if (typeof flags["form-selector"] === "string")
|
|
20703
|
+
body.form_selector = flags["form-selector"];
|
|
20704
|
+
if (typeof flags["submit-selector"] === "string")
|
|
20705
|
+
body.submit_selector = flags["submit-selector"];
|
|
20706
|
+
if (typeof flags["wait-for"] === "string")
|
|
20707
|
+
body.wait_for = flags["wait-for"];
|
|
20708
|
+
if (typeof flags["timeout-ms"] === "string")
|
|
20709
|
+
body.timeout_ms = Number(flags["timeout-ms"]);
|
|
20710
|
+
if (flags["same-origin-fetch-fallback"] !== undefined) {
|
|
20711
|
+
body.same_origin_fetch_fallback = flags["same-origin-fetch-fallback"] !== "false";
|
|
20712
|
+
}
|
|
20713
|
+
output(await api3("POST", "/v1/browse/submit", body), !!flags.pretty);
|
|
20714
|
+
}
|
|
19386
20715
|
async function cmdSnap(flags) {
|
|
19387
20716
|
const filter = flags.filter;
|
|
19388
20717
|
const result = await api3("POST", "/v1/browse/snap", { filter });
|
|
@@ -19462,6 +20791,9 @@ async function cmdBack() {
|
|
|
19462
20791
|
async function cmdForward() {
|
|
19463
20792
|
output(await api3("POST", "/v1/browse/forward"), false);
|
|
19464
20793
|
}
|
|
20794
|
+
async function cmdSync(flags) {
|
|
20795
|
+
output(await api3("POST", "/v1/browse/sync"), !!flags.pretty);
|
|
20796
|
+
}
|
|
19465
20797
|
async function cmdClose() {
|
|
19466
20798
|
output(await api3("POST", "/v1/browse/close"), false);
|
|
19467
20799
|
}
|
|
@@ -19557,6 +20889,7 @@ async function main() {
|
|
|
19557
20889
|
"upgrade",
|
|
19558
20890
|
"update",
|
|
19559
20891
|
"go",
|
|
20892
|
+
"submit",
|
|
19560
20893
|
"snap",
|
|
19561
20894
|
"click",
|
|
19562
20895
|
"fill",
|
|
@@ -19571,6 +20904,7 @@ async function main() {
|
|
|
19571
20904
|
"eval",
|
|
19572
20905
|
"back",
|
|
19573
20906
|
"forward",
|
|
20907
|
+
"sync",
|
|
19574
20908
|
"close",
|
|
19575
20909
|
"connect-chrome"
|
|
19576
20910
|
]);
|
|
@@ -19622,6 +20956,8 @@ async function main() {
|
|
|
19622
20956
|
return cmdSessions(flags);
|
|
19623
20957
|
case "go":
|
|
19624
20958
|
return cmdGo(args, flags);
|
|
20959
|
+
case "submit":
|
|
20960
|
+
return cmdSubmit(flags);
|
|
19625
20961
|
case "snap":
|
|
19626
20962
|
return cmdSnap(flags);
|
|
19627
20963
|
case "click":
|
|
@@ -19650,6 +20986,8 @@ async function main() {
|
|
|
19650
20986
|
return cmdBack();
|
|
19651
20987
|
case "forward":
|
|
19652
20988
|
return cmdForward();
|
|
20989
|
+
case "sync":
|
|
20990
|
+
return cmdSync(flags);
|
|
19653
20991
|
case "close":
|
|
19654
20992
|
return cmdClose();
|
|
19655
20993
|
case "connect-chrome":
|