unbrowse 2.12.0 → 2.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -56
- package/bin/unbrowse-wrapper.mjs +0 -0
- package/dist/cli.js +232 -471
- package/package.json +1 -1
- package/runtime-src/analytics-session.ts +5 -27
- package/runtime-src/api/browse-session.ts +3 -1
- package/runtime-src/api/browse-submit.ts +27 -34
- package/runtime-src/api/routes.ts +103 -145
- package/runtime-src/cli.ts +8 -175
- package/runtime-src/client/index.ts +44 -15
- package/runtime-src/execution/index.ts +1 -6
- package/runtime-src/indexer/index.ts +2 -4
- package/runtime-src/kuri/client.ts +35 -2
- package/runtime-src/orchestrator/index.ts +2 -2
- package/runtime-src/reverse-engineer/index.ts +0 -8
- package/runtime-src/runtime/setup.ts +7 -7
- package/runtime-src/server.ts +4 -3
- package/dist/mcp.js +0 -20392
- package/runtime-src/agent-outcome.ts +0 -166
- package/runtime-src/mcp.ts +0 -1065
package/runtime-src/cli.ts
CHANGED
|
@@ -8,7 +8,6 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { config as loadEnv } from "dotenv";
|
|
11
|
-
import { spawn } from "node:child_process";
|
|
12
11
|
import {
|
|
13
12
|
detectTelemetryHostType,
|
|
14
13
|
ensureCliInstallTracked,
|
|
@@ -19,7 +18,7 @@ import {
|
|
|
19
18
|
} from "./client/index.js";
|
|
20
19
|
import { findSitePack, findTask, allSitePacks, buildDepsGraph, planExecution, buildDepsMetadata, type SitePack } from "./cli/shortcuts.js";
|
|
21
20
|
import { ensureLocalServer, checkServerVersion, stopServer, restartServer } from "./runtime/local-server.js";
|
|
22
|
-
import { isMainModule
|
|
21
|
+
import { isMainModule } from "./runtime/paths.js";
|
|
23
22
|
import { drainPendingIndexJobs } from "./indexer/index.js";
|
|
24
23
|
import { drainPendingPassivePublishes } from "./orchestrator/passive-publish.js";
|
|
25
24
|
import { runSetup, type SetupReport, type SetupScope } from "./runtime/setup.js";
|
|
@@ -156,68 +155,16 @@ function slimTrace(obj: Record<string, unknown>): Record<string, unknown> {
|
|
|
156
155
|
};
|
|
157
156
|
if ("result" in obj) out.result = obj.result;
|
|
158
157
|
if (obj.available_endpoints) out.available_endpoints = obj.available_endpoints;
|
|
159
|
-
if (obj.impact) out.impact = obj.impact;
|
|
160
|
-
if (obj.next_actions) out.next_actions = obj.next_actions;
|
|
161
|
-
if (obj.next_step) out.next_step = obj.next_step;
|
|
162
158
|
if (obj.source) out.source = obj.source;
|
|
163
159
|
if (obj.skill) out.skill = obj.skill;
|
|
164
160
|
return out;
|
|
165
161
|
}
|
|
166
162
|
|
|
167
|
-
function formatSavedDuration(ms: number): string {
|
|
168
|
-
if (ms >= 60_000) return `${(ms / 60_000).toFixed(1)}m`;
|
|
169
|
-
if (ms >= 10_000) return `${Math.round(ms / 1000)}s`;
|
|
170
|
-
if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`;
|
|
171
|
-
return `${ms}ms`;
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
function emitImpactSummary(result: Record<string, unknown>): void {
|
|
175
|
-
const impact = result.impact as Record<string, unknown> | undefined;
|
|
176
|
-
if (!impact) return;
|
|
177
|
-
|
|
178
|
-
const timeSavedMs = typeof impact.time_saved_ms === "number" ? impact.time_saved_ms : 0;
|
|
179
|
-
const tokensSaved = typeof impact.tokens_saved === "number" ? impact.tokens_saved : 0;
|
|
180
|
-
const timeSavedPct = typeof impact.time_saved_pct === "number" ? impact.time_saved_pct : 0;
|
|
181
|
-
const tokensSavedPct = typeof impact.tokens_saved_pct === "number" ? impact.tokens_saved_pct : 0;
|
|
182
|
-
const browserAvoided = impact.browser_avoided === true;
|
|
183
|
-
if (timeSavedMs <= 0 && tokensSaved <= 0 && !browserAvoided) return;
|
|
184
|
-
|
|
185
|
-
const parts: string[] = [];
|
|
186
|
-
if (timeSavedMs > 0) parts.push(`${formatSavedDuration(timeSavedMs)} saved (${timeSavedPct}% faster)`);
|
|
187
|
-
if (tokensSaved > 0) parts.push(`${tokensSaved.toLocaleString("en-US")} tokens saved (${tokensSavedPct}% less context)`);
|
|
188
|
-
if (browserAvoided) parts.push("browser avoided");
|
|
189
|
-
info(parts.join(" • "));
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
function emitNextActionSummary(result: Record<string, unknown>): void {
|
|
193
|
-
const nextActions = Array.isArray(result.next_actions)
|
|
194
|
-
? result.next_actions as Array<Record<string, unknown>>
|
|
195
|
-
: [];
|
|
196
|
-
if (nextActions.length === 0) return;
|
|
197
|
-
info("Likely next actions:");
|
|
198
|
-
for (const action of nextActions.slice(0, 3)) {
|
|
199
|
-
const command = typeof action.command === "string" ? action.command : "";
|
|
200
|
-
const title = typeof action.title === "string" ? action.title : (action.endpoint_id as string | undefined) ?? "next step";
|
|
201
|
-
const why = typeof action.why === "string" ? action.why : "";
|
|
202
|
-
info(` ${command || title}${why ? ` # ${why}` : ""}`);
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
|
|
206
163
|
|
|
207
164
|
async function cmdHealth(flags: Record<string, string | boolean>): Promise<void> {
|
|
208
165
|
output(await api("GET", "/health"), !!flags.pretty);
|
|
209
166
|
}
|
|
210
167
|
|
|
211
|
-
function telemetryDomainFromInput(domain?: string, url?: string): string | null {
|
|
212
|
-
if (domain?.trim()) return domain.trim().replace(/^www\./, "");
|
|
213
|
-
if (!url?.trim()) return null;
|
|
214
|
-
try {
|
|
215
|
-
return new URL(url).hostname.replace(/^www\./, "");
|
|
216
|
-
} catch {
|
|
217
|
-
return null;
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
|
|
221
168
|
async function cmdResolve(flags: Record<string, string | boolean>): Promise<void> {
|
|
222
169
|
const intent = flags.intent as string;
|
|
223
170
|
if (!intent) die("--intent is required");
|
|
@@ -233,9 +180,6 @@ async function cmdResolve(flags: Record<string, string | boolean>): Promise<void
|
|
|
233
180
|
hostType,
|
|
234
181
|
properties: {
|
|
235
182
|
command: "resolve",
|
|
236
|
-
intent,
|
|
237
|
-
domain: telemetryDomainFromInput(flags.domain as string | undefined, flags.url as string | undefined),
|
|
238
|
-
url: typeof flags.url === "string" ? flags.url : null,
|
|
239
183
|
has_url: typeof flags.url === "string",
|
|
240
184
|
has_domain: typeof flags.domain === "string",
|
|
241
185
|
auto_execute: !!flags.execute,
|
|
@@ -392,9 +336,6 @@ async function cmdResolve(flags: Record<string, string | boolean>): Promise<void
|
|
|
392
336
|
hostType,
|
|
393
337
|
properties: {
|
|
394
338
|
command: "resolve",
|
|
395
|
-
intent,
|
|
396
|
-
domain: telemetryDomainFromInput(domain, url),
|
|
397
|
-
url: url ?? null,
|
|
398
339
|
source: result.source,
|
|
399
340
|
auto_execute: autoExecute,
|
|
400
341
|
explicit_endpoint: explicitEndpointId ?? null,
|
|
@@ -403,8 +344,6 @@ async function cmdResolve(flags: Record<string, string | boolean>): Promise<void
|
|
|
403
344
|
}
|
|
404
345
|
|
|
405
346
|
result = slimTrace(result);
|
|
406
|
-
emitImpactSummary(result);
|
|
407
|
-
emitNextActionSummary(result);
|
|
408
347
|
|
|
409
348
|
const skill = result.skill as Record<string, unknown> | undefined;
|
|
410
349
|
const trace = result.trace as Record<string, unknown> | undefined;
|
|
@@ -420,9 +359,6 @@ async function cmdResolve(flags: Record<string, string | boolean>): Promise<void
|
|
|
420
359
|
hostType,
|
|
421
360
|
properties: {
|
|
422
361
|
command: "resolve",
|
|
423
|
-
intent,
|
|
424
|
-
domain: telemetryDomainFromInput(flags.domain as string | undefined, flags.url as string | undefined),
|
|
425
|
-
url: typeof flags.url === "string" ? flags.url : null,
|
|
426
362
|
failure_stage: "resolve",
|
|
427
363
|
failure_reason: message,
|
|
428
364
|
},
|
|
@@ -533,9 +469,6 @@ async function cmdExecute(flags: Record<string, string | boolean>): Promise<void
|
|
|
533
469
|
hostType,
|
|
534
470
|
properties: {
|
|
535
471
|
command: "execute",
|
|
536
|
-
intent: typeof flags.intent === "string" ? flags.intent : null,
|
|
537
|
-
domain: telemetryDomainFromInput(undefined, flags.url as string | undefined),
|
|
538
|
-
url: typeof flags.url === "string" ? flags.url : null,
|
|
539
472
|
skill_id: skillId,
|
|
540
473
|
endpoint_id: typeof flags.endpoint === "string" ? flags.endpoint : null,
|
|
541
474
|
},
|
|
@@ -569,9 +502,6 @@ async function cmdExecute(flags: Record<string, string | boolean>): Promise<void
|
|
|
569
502
|
hostType,
|
|
570
503
|
properties: {
|
|
571
504
|
command: "execute",
|
|
572
|
-
intent: typeof flags.intent === "string" ? flags.intent : null,
|
|
573
|
-
domain: telemetryDomainFromInput(undefined, flags.url as string | undefined),
|
|
574
|
-
url: typeof flags.url === "string" ? flags.url : null,
|
|
575
505
|
skill_id: skillId,
|
|
576
506
|
endpoint_id: typeof flags.endpoint === "string" ? flags.endpoint : null,
|
|
577
507
|
},
|
|
@@ -580,8 +510,6 @@ async function cmdExecute(flags: Record<string, string | boolean>): Promise<void
|
|
|
580
510
|
|
|
581
511
|
// Strip metadata bloat
|
|
582
512
|
result = slimTrace(result);
|
|
583
|
-
emitImpactSummary(result);
|
|
584
|
-
emitNextActionSummary(result);
|
|
585
513
|
|
|
586
514
|
const pathFlag = flags.path as string | undefined;
|
|
587
515
|
const extractFlag = flags.extract as string | undefined;
|
|
@@ -592,12 +520,7 @@ async function cmdExecute(flags: Record<string, string | boolean>): Promise<void
|
|
|
592
520
|
// --schema: show response structure without data
|
|
593
521
|
if (schemaFlag && !rawFlag) {
|
|
594
522
|
const data = result.result;
|
|
595
|
-
output({
|
|
596
|
-
trace: result.trace,
|
|
597
|
-
schema: schemaOf(data),
|
|
598
|
-
...(result.impact ? { impact: result.impact } : {}),
|
|
599
|
-
...(result.next_actions ? { next_actions: result.next_actions } : {}),
|
|
600
|
-
}, !!flags.pretty);
|
|
523
|
+
output({ trace: result.trace, schema: schemaOf(data) }, !!flags.pretty);
|
|
601
524
|
return;
|
|
602
525
|
}
|
|
603
526
|
|
|
@@ -615,13 +538,7 @@ async function cmdExecute(flags: Record<string, string | boolean>): Promise<void
|
|
|
615
538
|
const limited = limitFlag ? extracted.slice(0, limitFlag) : extracted;
|
|
616
539
|
|
|
617
540
|
const trace = result.trace as Record<string, unknown> | undefined;
|
|
618
|
-
const out: Record<string, unknown> = {
|
|
619
|
-
trace: result.trace,
|
|
620
|
-
data: limited,
|
|
621
|
-
count: limited.length,
|
|
622
|
-
...(result.impact ? { impact: result.impact } : {}),
|
|
623
|
-
...(result.next_actions ? { next_actions: result.next_actions } : {}),
|
|
624
|
-
};
|
|
541
|
+
const out: Record<string, unknown> = { trace: result.trace, data: limited, count: limited.length };
|
|
625
542
|
|
|
626
543
|
// Prompt agent to review when this is likely a first-time execute
|
|
627
544
|
if (trace?.skill_id && trace?.endpoint_id && limited.length > 0) {
|
|
@@ -639,8 +556,6 @@ async function cmdExecute(flags: Record<string, string | boolean>): Promise<void
|
|
|
639
556
|
const schema = schemaOf(result.result);
|
|
640
557
|
output({
|
|
641
558
|
trace: result.trace,
|
|
642
|
-
...(result.impact ? { impact: result.impact } : {}),
|
|
643
|
-
...(result.next_actions ? { next_actions: result.next_actions } : {}),
|
|
644
559
|
extraction_hints: {
|
|
645
560
|
message: "Response is large. Use --path/--extract/--limit to filter, or --schema to see structure, or --raw for full response.",
|
|
646
561
|
schema_tree: schema,
|
|
@@ -659,9 +574,6 @@ async function cmdExecute(flags: Record<string, string | boolean>): Promise<void
|
|
|
659
574
|
hostType,
|
|
660
575
|
properties: {
|
|
661
576
|
command: "execute",
|
|
662
|
-
intent: typeof flags.intent === "string" ? flags.intent : null,
|
|
663
|
-
domain: telemetryDomainFromInput(undefined, flags.url as string | undefined),
|
|
664
|
-
url: typeof flags.url === "string" ? flags.url : null,
|
|
665
577
|
skill_id: skillId,
|
|
666
578
|
failure_stage: "execute",
|
|
667
579
|
failure_reason: message,
|
|
@@ -736,53 +648,7 @@ async function cmdSearch(flags: Record<string, string | boolean>): Promise<void>
|
|
|
736
648
|
const path = domain ? "/v1/search/domain" : "/v1/search";
|
|
737
649
|
const body: Record<string, unknown> = { intent, k: Number(flags.k) || 5 };
|
|
738
650
|
if (domain) body.domain = domain;
|
|
739
|
-
|
|
740
|
-
await ensureCliInstallTracked(hostType);
|
|
741
|
-
await recordFunnelTelemetryEvent("cli_invoked", {
|
|
742
|
-
source: "cli",
|
|
743
|
-
hostType,
|
|
744
|
-
properties: { command: "search" },
|
|
745
|
-
});
|
|
746
|
-
await recordFunnelTelemetryEvent("search_started", {
|
|
747
|
-
source: "cli",
|
|
748
|
-
hostType,
|
|
749
|
-
properties: {
|
|
750
|
-
command: "search",
|
|
751
|
-
intent,
|
|
752
|
-
domain: domain ?? null,
|
|
753
|
-
k: body.k,
|
|
754
|
-
},
|
|
755
|
-
});
|
|
756
|
-
try {
|
|
757
|
-
const result = await api("POST", path, body) as Record<string, unknown>;
|
|
758
|
-
const results = Array.isArray(result.results) ? result.results : [];
|
|
759
|
-
await recordFunnelTelemetryEvent("search_completed", {
|
|
760
|
-
source: "cli",
|
|
761
|
-
hostType,
|
|
762
|
-
properties: {
|
|
763
|
-
command: "search",
|
|
764
|
-
intent,
|
|
765
|
-
domain: domain ?? null,
|
|
766
|
-
k: body.k,
|
|
767
|
-
result_count: results.length,
|
|
768
|
-
},
|
|
769
|
-
});
|
|
770
|
-
output(result, !!flags.pretty);
|
|
771
|
-
} catch (error) {
|
|
772
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
773
|
-
await recordFunnelTelemetryEvent("search_failed", {
|
|
774
|
-
source: "cli",
|
|
775
|
-
hostType,
|
|
776
|
-
properties: {
|
|
777
|
-
command: "search",
|
|
778
|
-
intent,
|
|
779
|
-
domain: domain ?? null,
|
|
780
|
-
failure_stage: "search",
|
|
781
|
-
failure_reason: message,
|
|
782
|
-
},
|
|
783
|
-
});
|
|
784
|
-
throw error;
|
|
785
|
-
}
|
|
651
|
+
output(await api("POST", path, body), !!flags.pretty);
|
|
786
652
|
}
|
|
787
653
|
|
|
788
654
|
async function cmdSessions(flags: Record<string, string | boolean>): Promise<void> {
|
|
@@ -891,7 +757,6 @@ async function cmdSetup(flags: Record<string, string | boolean>): Promise<void>
|
|
|
891
757
|
export const CLI_REFERENCE = {
|
|
892
758
|
commands: [
|
|
893
759
|
{ name: "health", usage: "", desc: "Server health check" },
|
|
894
|
-
{ name: "mcp", usage: "[--no-auto-start]", desc: "Run the stdio MCP server" },
|
|
895
760
|
{ name: "setup", usage: "[--opencode auto|global|project|off] [--no-start]", desc: "Bootstrap browser deps + Open Code command" },
|
|
896
761
|
{ name: "resolve", usage: '--intent "..." --url "..." [opts]', desc: "Resolve intent → search/capture/execute" },
|
|
897
762
|
{ name: "execute", usage: "--skill ID --endpoint ID [opts]", desc: "Execute a specific endpoint" },
|
|
@@ -904,7 +769,7 @@ export const CLI_REFERENCE = {
|
|
|
904
769
|
{ name: "search", usage: '--intent "..." [--domain "..."]', desc: "Search marketplace" },
|
|
905
770
|
{ name: "sessions", usage: '--domain "..." [--limit N]', desc: "Debug session logs" },
|
|
906
771
|
{ name: "go", usage: '<url>', desc: "Open a live Kuri browser tab for capture-first workflows" },
|
|
907
|
-
{ name: "submit", usage: "[--form-selector sel] [--submit-selector sel] [--wait-for hint]", desc: "Submit current form
|
|
772
|
+
{ name: "submit", usage: "[--form-selector sel] [--submit-selector sel] [--wait-for hint]", desc: "Submit current form with DOM-first + same-origin rehydrate fallback for JS-heavy flows" },
|
|
908
773
|
{ name: "snap", usage: "[--filter interactive]", desc: "A11y snapshot with @eN refs" },
|
|
909
774
|
{ name: "click", usage: "<ref>", desc: "Click element by ref (e.g. e5)" },
|
|
910
775
|
{ name: "fill", usage: "<ref> <value>", desc: "Fill input by ref" },
|
|
@@ -941,7 +806,6 @@ export const CLI_REFERENCE = {
|
|
|
941
806
|
],
|
|
942
807
|
examples: [
|
|
943
808
|
"unbrowse setup",
|
|
944
|
-
"unbrowse mcp",
|
|
945
809
|
'unbrowse resolve --intent "top stories" --url "https://news.ycombinator.com" --execute',
|
|
946
810
|
'unbrowse resolve --intent "get timeline" --url "https://x.com"',
|
|
947
811
|
'unbrowse go "https://www.mandai.com/en/ticketing/admission-and-rides/parks-selection.html"',
|
|
@@ -996,8 +860,8 @@ function printHelp(): void {
|
|
|
996
860
|
" 1. go -> open the live tab you want to work in",
|
|
997
861
|
" 2. snap -> inspect refs and confirm the page state",
|
|
998
862
|
" 3. click/fill/eval -> set real page state",
|
|
999
|
-
" 4. submit -> prefer DOM submit; auto-
|
|
1000
|
-
" 5. sync -> flush
|
|
863
|
+
" 4. submit -> prefer DOM submit; auto-falls back to same-origin rehydrate",
|
|
864
|
+
" 5. sync -> flush captured routes after a successful step",
|
|
1001
865
|
" 6. close -> finish capture + indexing",
|
|
1002
866
|
);
|
|
1003
867
|
|
|
@@ -1059,35 +923,6 @@ async function cmdUpgrade(flags: Record<string, string | boolean>): Promise<void
|
|
|
1059
923
|
}
|
|
1060
924
|
}
|
|
1061
925
|
|
|
1062
|
-
async function cmdMcp(flags: Record<string, string | boolean>): Promise<void> {
|
|
1063
|
-
const entrypoint = resolveSiblingEntrypoint(import.meta.url, "mcp");
|
|
1064
|
-
const child = spawn(
|
|
1065
|
-
process.execPath,
|
|
1066
|
-
[...runtimeArgsForEntrypoint(import.meta.url, entrypoint), ...(flags["no-auto-start"] ? ["--no-auto-start"] : [])],
|
|
1067
|
-
{
|
|
1068
|
-
cwd: process.cwd(),
|
|
1069
|
-
stdio: "inherit",
|
|
1070
|
-
env: {
|
|
1071
|
-
...process.env,
|
|
1072
|
-
MCP_SERVER_MODE: "1",
|
|
1073
|
-
},
|
|
1074
|
-
},
|
|
1075
|
-
);
|
|
1076
|
-
|
|
1077
|
-
const code = await new Promise<number>((resolve, reject) => {
|
|
1078
|
-
child.once("error", reject);
|
|
1079
|
-
child.once("exit", (exitCode, signal) => {
|
|
1080
|
-
if (signal) {
|
|
1081
|
-
process.kill(process.pid, signal);
|
|
1082
|
-
return;
|
|
1083
|
-
}
|
|
1084
|
-
resolve(exitCode ?? 1);
|
|
1085
|
-
});
|
|
1086
|
-
});
|
|
1087
|
-
|
|
1088
|
-
if (code !== 0) process.exit(code);
|
|
1089
|
-
}
|
|
1090
|
-
|
|
1091
926
|
// ---------------------------------------------------------------------------
|
|
1092
927
|
// Site/task shortcut commands
|
|
1093
928
|
// ---------------------------------------------------------------------------
|
|
@@ -1415,7 +1250,6 @@ async function main(): Promise<void> {
|
|
|
1415
1250
|
}
|
|
1416
1251
|
|
|
1417
1252
|
// Server lifecycle commands (don't need ensureLocalServer)
|
|
1418
|
-
if (command === "mcp") return cmdMcp(flags);
|
|
1419
1253
|
if (command === "status") return cmdStatus(flags);
|
|
1420
1254
|
if (command === "stop") { cmdStop(flags); return; }
|
|
1421
1255
|
if (command === "restart") return cmdRestart(flags);
|
|
@@ -1424,7 +1258,7 @@ async function main(): Promise<void> {
|
|
|
1424
1258
|
|
|
1425
1259
|
// --- Shortcut resolution: unbrowse <site> [task] [flags] ---
|
|
1426
1260
|
const KNOWN_COMMANDS = new Set([
|
|
1427
|
-
"health", "
|
|
1261
|
+
"health", "setup", "resolve", "execute", "exec",
|
|
1428
1262
|
"feedback", "fb", "review", "publish", "login", "skills", "skill", "search", "sessions",
|
|
1429
1263
|
"status", "stop", "restart", "upgrade", "update",
|
|
1430
1264
|
"go", "submit", "snap", "click", "fill", "type", "press", "select", "scroll",
|
|
@@ -1455,7 +1289,6 @@ async function main(): Promise<void> {
|
|
|
1455
1289
|
|
|
1456
1290
|
switch (command) {
|
|
1457
1291
|
case "health": return cmdHealth(flags);
|
|
1458
|
-
case "mcp": return cmdMcp(flags);
|
|
1459
1292
|
case "setup": return cmdSetup(flags);
|
|
1460
1293
|
case "resolve": return cmdResolve(flags);
|
|
1461
1294
|
case "execute": case "exec": return cmdExecute(flags);
|
|
@@ -518,7 +518,8 @@ async function promptAgentEmail(defaultName: string): Promise<string> {
|
|
|
518
518
|
}
|
|
519
519
|
}
|
|
520
520
|
|
|
521
|
-
async function checkTosStatus(): Promise<
|
|
521
|
+
async function checkTosStatus(options?: { exitOnFailure?: boolean }): Promise<boolean> {
|
|
522
|
+
const exitOnFailure = options?.exitOnFailure ?? true;
|
|
522
523
|
const config = loadConfig();
|
|
523
524
|
|
|
524
525
|
let tosInfo: { version: string; summary: string; url: string };
|
|
@@ -527,11 +528,11 @@ async function checkTosStatus(): Promise<void> {
|
|
|
527
528
|
} catch {
|
|
528
529
|
// Offline — allow usage with whatever ToS was previously accepted.
|
|
529
530
|
// Backend will enforce on next actual API call anyway.
|
|
530
|
-
return;
|
|
531
|
+
return true;
|
|
531
532
|
}
|
|
532
533
|
|
|
533
534
|
if (config?.tos_accepted_version === tosInfo.version) {
|
|
534
|
-
return; // Already accepted current version
|
|
535
|
+
return true; // Already accepted current version
|
|
535
536
|
}
|
|
536
537
|
|
|
537
538
|
// Need re-acceptance
|
|
@@ -539,7 +540,8 @@ async function checkTosStatus(): Promise<void> {
|
|
|
539
540
|
const accepted = await promptTosAcceptance(tosInfo.summary, tosInfo.url);
|
|
540
541
|
if (!accepted) {
|
|
541
542
|
console.log("You must accept the updated Terms of Service to continue using Unbrowse.");
|
|
542
|
-
process.exit(1);
|
|
543
|
+
if (exitOnFailure) process.exit(1);
|
|
544
|
+
return false;
|
|
543
545
|
}
|
|
544
546
|
|
|
545
547
|
// Call accept-tos endpoint
|
|
@@ -557,17 +559,20 @@ async function checkTosStatus(): Promise<void> {
|
|
|
557
559
|
console.warn(`Failed to record ToS acceptance: ${(err as Error).message}`);
|
|
558
560
|
// Don't block — backend will enforce on next call
|
|
559
561
|
}
|
|
562
|
+
return true;
|
|
560
563
|
}
|
|
561
564
|
|
|
562
565
|
/** Auto-register with the backend if no API key is configured. Persists to ~/.unbrowse/config.json. */
|
|
563
|
-
export async function ensureRegistered(options?: { promptForEmail?: boolean }): Promise<void> {
|
|
566
|
+
export async function ensureRegistered(options?: { promptForEmail?: boolean; exitOnFailure?: boolean }): Promise<void> {
|
|
564
567
|
if (LOCAL_ONLY) return;
|
|
568
|
+
const exitOnFailure = options?.exitOnFailure ?? true;
|
|
565
569
|
const usableKey = await findUsableApiKey();
|
|
566
570
|
if (usableKey) {
|
|
567
571
|
if (usableKey.source === "config") {
|
|
568
572
|
console.log("[unbrowse] Restored saved registration.");
|
|
569
573
|
}
|
|
570
|
-
await checkTosStatus();
|
|
574
|
+
const accepted = await checkTosStatus({ exitOnFailure });
|
|
575
|
+
if (!accepted) return;
|
|
571
576
|
try {
|
|
572
577
|
const profile = await getMyProfile();
|
|
573
578
|
const wallet = getLocalWalletContext();
|
|
@@ -592,7 +597,8 @@ export async function ensureRegistered(options?: { promptForEmail?: boolean }):
|
|
|
592
597
|
const accepted = await promptTosAcceptance(tosInfo.summary, tosInfo.url);
|
|
593
598
|
if (!accepted) {
|
|
594
599
|
console.log("You must accept the Terms of Service to use Unbrowse.");
|
|
595
|
-
process.exit(1);
|
|
600
|
+
if (exitOnFailure) process.exit(1);
|
|
601
|
+
return;
|
|
596
602
|
}
|
|
597
603
|
|
|
598
604
|
// Step 3: Register with ToS version
|
|
@@ -628,8 +634,38 @@ export async function ensureRegistered(options?: { promptForEmail?: boolean }):
|
|
|
628
634
|
} catch (err) {
|
|
629
635
|
console.warn(`Registration failed: ${(err as Error).message}`);
|
|
630
636
|
console.warn("Set UNBROWSE_API_KEY manually or try again.");
|
|
631
|
-
process.exit(1);
|
|
637
|
+
if (exitOnFailure) process.exit(1);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
let backgroundRegistrationPromise: Promise<void> | null = null;
|
|
642
|
+
|
|
643
|
+
export function startBackgroundRegistration(options?: { promptForEmail?: boolean }): Promise<void> {
|
|
644
|
+
if (LOCAL_ONLY) return Promise.resolve();
|
|
645
|
+
if (backgroundRegistrationPromise) return backgroundRegistrationPromise;
|
|
646
|
+
backgroundRegistrationPromise = ensureRegistered({
|
|
647
|
+
promptForEmail: options?.promptForEmail,
|
|
648
|
+
exitOnFailure: false,
|
|
649
|
+
})
|
|
650
|
+
.catch((err) => {
|
|
651
|
+
console.warn(`[unbrowse] Background registration failed: ${(err as Error).message}`);
|
|
652
|
+
})
|
|
653
|
+
.finally(() => {
|
|
654
|
+
backgroundRegistrationPromise = null;
|
|
655
|
+
});
|
|
656
|
+
return backgroundRegistrationPromise;
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
export async function waitForBackgroundRegistration(timeoutMs = 0): Promise<void> {
|
|
660
|
+
if (!backgroundRegistrationPromise) return;
|
|
661
|
+
if (timeoutMs <= 0) {
|
|
662
|
+
await backgroundRegistrationPromise;
|
|
663
|
+
return;
|
|
632
664
|
}
|
|
665
|
+
await Promise.race([
|
|
666
|
+
backgroundRegistrationPromise,
|
|
667
|
+
new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
|
|
668
|
+
]);
|
|
633
669
|
}
|
|
634
670
|
|
|
635
671
|
// --- Skill CRUD ---
|
|
@@ -950,13 +986,6 @@ export interface AnalyticsSessionPayload {
|
|
|
950
986
|
cached_skill_calls?: number;
|
|
951
987
|
fresh_index_calls?: number;
|
|
952
988
|
browser_mode?: "default" | "replaced" | "manual" | "unknown";
|
|
953
|
-
success?: boolean;
|
|
954
|
-
source?: string;
|
|
955
|
-
time_saved_ms?: number;
|
|
956
|
-
time_saved_pct?: number;
|
|
957
|
-
tokens_saved?: number;
|
|
958
|
-
tokens_saved_pct?: number;
|
|
959
|
-
cost_saved_uc?: number;
|
|
960
989
|
}
|
|
961
990
|
|
|
962
991
|
/**
|
|
@@ -1899,14 +1899,9 @@ export async function executeEndpoint(
|
|
|
1899
1899
|
}
|
|
1900
1900
|
}
|
|
1901
1901
|
|
|
1902
|
-
const hasAuthContext =
|
|
1903
|
-
cookies.length > 0 ||
|
|
1904
|
-
Object.keys(authHeaders).length > 0 ||
|
|
1905
|
-
!!skill.auth_profile_ref ||
|
|
1906
|
-
endpoint.semantic?.auth_required === true;
|
|
1907
1902
|
|
|
1908
1903
|
// robots.txt compliance gate — block disallowed paths before any network call.
|
|
1909
|
-
if (!options?.skip_robots_check
|
|
1904
|
+
if (!options?.skip_robots_check) {
|
|
1910
1905
|
const allowed = await isAllowedByRobots(url);
|
|
1911
1906
|
if (!allowed) {
|
|
1912
1907
|
const traceId = nanoid();
|
|
@@ -17,6 +17,8 @@ import { join } from "node:path";
|
|
|
17
17
|
import { homedir } from "node:os";
|
|
18
18
|
|
|
19
19
|
const UNBROWSE_CONFIG_PATH = join(homedir(), ".unbrowse", "config.json");
|
|
20
|
+
const SKILL_SNAPSHOT_DIR = process.env.UNBROWSE_SKILL_SNAPSHOT_DIR
|
|
21
|
+
?? join(process.env.HOME ?? "/tmp", ".unbrowse", "skill-snapshots");
|
|
20
22
|
|
|
21
23
|
/** Read agent_id from local config — used for contributor attribution on publish. */
|
|
22
24
|
function getLocalAgentId(): string | undefined {
|
|
@@ -283,10 +285,6 @@ export function mergeAgentReview(
|
|
|
283
285
|
};
|
|
284
286
|
});
|
|
285
287
|
}
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
const SKILL_SNAPSHOT_DIR = join(process.env.HOME ?? "/tmp", ".unbrowse", "skill-snapshots");
|
|
289
|
-
|
|
290
288
|
/**
|
|
291
289
|
* Find existing domain snapshots and merge incoming endpoints into them.
|
|
292
290
|
* Returns a merged skill with all endpoints from both existing snapshots
|
|
@@ -549,6 +549,32 @@ async function waitForTabRegistration(tabId: string, timeoutMs = 2_000): Promise
|
|
|
549
549
|
}
|
|
550
550
|
}
|
|
551
551
|
|
|
552
|
+
async function createTabViaChromeCdp(url = "about:blank"): Promise<string> {
|
|
553
|
+
if (!kuriCdpPort) return "";
|
|
554
|
+
try {
|
|
555
|
+
const res = await fetch(`http://127.0.0.1:${kuriCdpPort}/json/new?${url}`, {
|
|
556
|
+
method: "PUT",
|
|
557
|
+
signal: AbortSignal.timeout(5000),
|
|
558
|
+
});
|
|
559
|
+
const target = await res.json() as { id?: string; targetId?: string };
|
|
560
|
+
return target?.id ?? target?.targetId ?? "";
|
|
561
|
+
} catch (err) {
|
|
562
|
+
log("kuri", `Chrome tab creation failed: ${err instanceof Error ? err.message : err}`);
|
|
563
|
+
return "";
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
async function findReusableIdleTab(): Promise<string> {
|
|
568
|
+
await ensureTabsDiscovered();
|
|
569
|
+
try {
|
|
570
|
+
const tabs = (await kuriGet("/tabs")) as Array<{ id?: string; url?: string }>;
|
|
571
|
+
const candidate = tabs.find((tab) => /^(about:blank|chrome:\/\/newtab\/?)$/i.test(tab?.url ?? ""));
|
|
572
|
+
return candidate?.id ?? "";
|
|
573
|
+
} catch {
|
|
574
|
+
return "";
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
|
|
552
578
|
/** Navigate tab to URL. */
|
|
553
579
|
export async function navigate(tabId: string, url: string): Promise<void> {
|
|
554
580
|
await kuriGet("/navigate", { tab_id: tabId, url });
|
|
@@ -771,8 +797,15 @@ export async function closeTab(tabId: string): Promise<void> {
|
|
|
771
797
|
export async function newTab(url?: string): Promise<string> {
|
|
772
798
|
const params: Record<string, string> = {};
|
|
773
799
|
if (url) params.url = url;
|
|
774
|
-
|
|
775
|
-
|
|
800
|
+
let tabId = "";
|
|
801
|
+
try {
|
|
802
|
+
const result = (await kuriGet("/tab/new", params)) as { tab_id?: string; id?: string; targetId?: string };
|
|
803
|
+
tabId = result?.tab_id ?? result?.id ?? result?.targetId ?? "";
|
|
804
|
+
} catch {
|
|
805
|
+
tabId = "";
|
|
806
|
+
}
|
|
807
|
+
if (!tabId) tabId = await createTabViaChromeCdp(url ?? "about:blank");
|
|
808
|
+
if (!tabId) tabId = await findReusableIdleTab();
|
|
776
809
|
if (tabId) {
|
|
777
810
|
await waitForTabRegistration(tabId).catch(() => {});
|
|
778
811
|
}
|
|
@@ -128,7 +128,8 @@ const skillRouteCache = new Map<
|
|
|
128
128
|
{ skillId: string; domain: string; endpointId?: string; localSkillPath?: string; ts: number }
|
|
129
129
|
>();
|
|
130
130
|
const ROUTE_CACHE_FILE = join(process.env.HOME ?? "/tmp", ".unbrowse", "route-cache.json");
|
|
131
|
-
const SKILL_SNAPSHOT_DIR =
|
|
131
|
+
const SKILL_SNAPSHOT_DIR = process.env.UNBROWSE_SKILL_SNAPSHOT_DIR
|
|
132
|
+
?? join(process.env.HOME ?? "/tmp", ".unbrowse", "skill-snapshots");
|
|
132
133
|
|
|
133
134
|
// Domain-level skill cache: maps domain → best skillId (independent of intent/URL)
|
|
134
135
|
// This enables cross-intent reuse: "find keyboards" seeds cache, "find monitors" reuses it
|
|
@@ -1953,7 +1954,6 @@ export async function resolveAndExecute(
|
|
|
1953
1954
|
result: {
|
|
1954
1955
|
message: `Found ${epRanked.length} endpoint(s). Pick one and call POST /v1/skills/${resolvedSkill.skill_id}/execute with params.endpoint_id.`,
|
|
1955
1956
|
skill_id: resolvedSkill.skill_id,
|
|
1956
|
-
suggested_next_operation_id: chunk.available_operation_ids[0],
|
|
1957
1957
|
available_operations: chunk.operations.map((operation) => ({
|
|
1958
1958
|
operation_id: operation.operation_id,
|
|
1959
1959
|
endpoint_id: operation.endpoint_id,
|
|
@@ -1155,15 +1155,7 @@ function templatizePathSegments(
|
|
|
1155
1155
|
try {
|
|
1156
1156
|
const contextSegments = new URL(context.pageUrl).pathname.split("/");
|
|
1157
1157
|
const contextSeg = contextSegments[i];
|
|
1158
|
-
const prevSeg = tSegments[i - 1] ?? "";
|
|
1159
|
-
const prevContextSeg = contextSegments[i - 1] ?? "";
|
|
1160
|
-
const nextSeg = tSegments[i + 1] ?? "";
|
|
1161
|
-
const nextContextSeg = contextSegments[i + 1] ?? "";
|
|
1162
|
-
const hasStructuralNeighborMatch =
|
|
1163
|
-
(!!prevSeg && !!prevContextSeg && prevSeg === prevContextSeg) ||
|
|
1164
|
-
(!!nextSeg && !!nextContextSeg && nextSeg === nextContextSeg);
|
|
1165
1158
|
if (contextSeg && contextSeg !== tSeg &&
|
|
1166
|
-
hasStructuralNeighborMatch &&
|
|
1167
1159
|
!contextSeg.includes(".") &&
|
|
1168
1160
|
contextSeg.length >= 2 && contextSeg.length <= 40 &&
|
|
1169
1161
|
!/^(api|v\d+|www|en|es|fr|de|latest|search|i)$/i.test(contextSeg)) {
|
|
@@ -197,9 +197,9 @@ export async function runSetup(options?: {
|
|
|
197
197
|
const lobsterInstalled = hasBinary("lobstercash") ||
|
|
198
198
|
existsSync(path.join(os.homedir(), ".agents", "skills", "lobstercash", "SKILL.md"));
|
|
199
199
|
|
|
200
|
-
// Auto-setup
|
|
200
|
+
// Auto-setup lobster.cash wallet if skill is installed but wallet not configured
|
|
201
201
|
if (!skipWalletSetup && !walletCheck.configured && lobsterInstalled) {
|
|
202
|
-
console.log("[unbrowse]
|
|
202
|
+
console.log("[unbrowse] lobster.cash skill detected but wallet not configured — running wallet setup...");
|
|
203
203
|
try {
|
|
204
204
|
execFileSync("npx", ["@crossmint/lobster-cli", "setup"], {
|
|
205
205
|
stdio: "inherit",
|
|
@@ -211,7 +211,7 @@ export async function runSetup(options?: {
|
|
|
211
211
|
console.log(`[unbrowse] wallet configured (${recheck.provider})`);
|
|
212
212
|
}
|
|
213
213
|
} catch {
|
|
214
|
-
console.warn("[unbrowse]
|
|
214
|
+
console.warn("[unbrowse] lobster.cash wallet setup failed or was skipped — continuing without wallet");
|
|
215
215
|
}
|
|
216
216
|
}
|
|
217
217
|
|
|
@@ -223,13 +223,13 @@ export async function runSetup(options?: {
|
|
|
223
223
|
message: finalWalletCheck.configured
|
|
224
224
|
? `Wallet configured (${finalWalletCheck.provider}). This address is the contributor truth: it is synced onto your agent profile, used for contributor payouts when your routes earn, and used for paid-route spending.`
|
|
225
225
|
: lobsterInstalled
|
|
226
|
-
? "
|
|
227
|
-
: "No wallet configured.
|
|
226
|
+
? "lobster.cash installed but wallet not paired. Pair it now so this wallet address becomes your contributor payout target and your paid-route spending wallet. Run: lobstercash setup"
|
|
227
|
+
: "No wallet configured. Install/pair a wallet so your contributor payouts have a destination address and premium-route spending can clear automatically. Without it you stay in free indexing mode only.",
|
|
228
228
|
install_hint: finalWalletCheck.configured
|
|
229
229
|
? undefined
|
|
230
230
|
: lobsterInstalled
|
|
231
|
-
? "
|
|
232
|
-
: "npx
|
|
231
|
+
? "lobstercash setup"
|
|
232
|
+
: "npx skills add https://github.com/Crossmint/lobstercash-cli-skills --global --yes",
|
|
233
233
|
};
|
|
234
234
|
|
|
235
235
|
return {
|
package/runtime-src/server.ts
CHANGED
|
@@ -6,7 +6,7 @@ import cors from "@fastify/cors";
|
|
|
6
6
|
import { registerRoutes } from "./api/routes.js";
|
|
7
7
|
import { registerRateLimiter } from "./ratelimit/index.js";
|
|
8
8
|
import { schedulePeriodicVerification } from "./verification/index.js";
|
|
9
|
-
import {
|
|
9
|
+
import { startBackgroundRegistration } from "./client/index.js";
|
|
10
10
|
import { shutdownAllBrowsers } from "./capture/index.js";
|
|
11
11
|
import * as kuri from "./kuri/client.js";
|
|
12
12
|
|
|
@@ -63,8 +63,9 @@ export async function startUnbrowseServer(options: StartServerOptions = {}): Pro
|
|
|
63
63
|
|
|
64
64
|
// Kuri starts on demand when browse/capture commands need it.
|
|
65
65
|
// No eager start — avoids launching Chrome on every server restart.
|
|
66
|
-
|
|
67
|
-
|
|
66
|
+
// Registration is allowed to finish in the background so /health is not
|
|
67
|
+
// blocked by remote Worker latency during server bootstrap.
|
|
68
|
+
void startBackgroundRegistration();
|
|
68
69
|
|
|
69
70
|
const app = Fastify({ logger: options.logger ?? true });
|
|
70
71
|
await app.register(cors, { origin: true });
|