unbrowse 3.0.2 → 3.1.0-experiments.5e7a7bb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +629 -101
- package/dist/mcp.js +710 -73
- package/package.json +1 -1
- package/runtime-src/api/browse-index.ts +26 -4
- package/runtime-src/api/routes.ts +43 -9
- package/runtime-src/browser/index.ts +2 -1
- package/runtime-src/build-info.generated.ts +5 -5
- package/runtime-src/capture/index.ts +113 -0
- package/runtime-src/cli.ts +190 -2
- package/runtime-src/client/index.ts +28 -12
- package/runtime-src/execution/index.ts +43 -21
- package/runtime-src/execution/token-resolver.ts +122 -0
- package/runtime-src/graph/index.ts +14 -6
- package/runtime-src/impact-log.ts +227 -0
- package/runtime-src/kuri/client.ts +5 -1
- package/runtime-src/marketplace/index.ts +9 -1
- package/runtime-src/mcp.ts +247 -34
- package/runtime-src/orchestrator/browser-agent.ts +2 -1
- package/runtime-src/orchestrator/index.ts +7 -3
- package/runtime-src/payments/lobster-pay.ts +182 -0
- package/runtime-src/reverse-engineer/token-sources.ts +357 -0
- package/runtime-src/types/skill.ts +19 -0
- package/vendor/kuri/darwin-arm64/kuri +0 -0
- package/vendor/kuri/darwin-x64/kuri +0 -0
- package/vendor/kuri/linux-arm64/kuri +0 -0
- package/vendor/kuri/linux-x64/kuri +0 -0
- package/vendor/kuri/manifest.json +10 -6
- package/vendor/kuri/win-x64/kuri.exe +0 -0
package/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { nanoid } from "nanoid";
|
|
2
2
|
import { readFileSync } from "node:fs";
|
|
3
|
-
import { extractEndpoints } from "../reverse-engineer/index.js";
|
|
3
|
+
import { extractEndpoints, extractAuthHeaders } from "../reverse-engineer/index.js";
|
|
4
|
+
import { enrichEndpointsWithTokenSources } from "../reverse-engineer/token-sources.js";
|
|
4
5
|
import { buildSkillOperationGraph, inferEndpointSemantic } from "../graph/index.js";
|
|
5
6
|
import { validateExtractionQuality } from "../execution/index.js";
|
|
6
7
|
import { assessIntentResult } from "../intent-match.js";
|
|
@@ -10,6 +11,8 @@ import type { RawRequest } from "../capture/index.js";
|
|
|
10
11
|
import { cachePublishedSkill, findExistingSkillForDomain } from "../client/index.js";
|
|
11
12
|
import { mergeEndpoints } from "../marketplace/index.js";
|
|
12
13
|
import { upsertDagEdgesFromOperationGraph } from "../orchestrator/dag-feedback.js";
|
|
14
|
+
import { storeCredential } from "../vault/index.js";
|
|
15
|
+
import { getRegistrableDomain } from "../domain.js";
|
|
13
16
|
import {
|
|
14
17
|
buildResolveCacheKey,
|
|
15
18
|
domainSkillCache,
|
|
@@ -151,14 +154,25 @@ export async function cacheBrowseRequests(params: {
|
|
|
151
154
|
sessionDomain: string;
|
|
152
155
|
requests: RawRequest[];
|
|
153
156
|
getPageHtml?: () => Promise<string>;
|
|
157
|
+
jsBundles?: Map<string, string>;
|
|
154
158
|
intent?: string;
|
|
155
159
|
}): Promise<BrowseIndexResult> {
|
|
156
|
-
const { sessionUrl, sessionDomain, requests, getPageHtml } = params;
|
|
160
|
+
const { sessionUrl, sessionDomain, requests, getPageHtml, jsBundles } = params;
|
|
157
161
|
let domain: string;
|
|
158
162
|
try { domain = new URL(sessionUrl).hostname; } catch { domain = sessionDomain; }
|
|
159
163
|
const intent = params.intent ?? `browse ${domain}`;
|
|
160
164
|
|
|
161
165
|
const rawEndpoints = extractEndpoints(requests, undefined, { pageUrl: sessionUrl, finalUrl: sessionUrl });
|
|
166
|
+
|
|
167
|
+
// Extract and persist auth headers (authorization, csrf, bearer tokens)
|
|
168
|
+
// so serverFetch can replay them. Use registrable domain for vault key
|
|
169
|
+
// so ads.x.com and ads-api.x.com share the same session.
|
|
170
|
+
const capturedAuthHeaders = extractAuthHeaders(requests);
|
|
171
|
+
if (Object.keys(capturedAuthHeaders).length > 0) {
|
|
172
|
+
const sessionKey = `${getRegistrableDomain(domain)}-session`;
|
|
173
|
+
await storeCredential(sessionKey, JSON.stringify({ headers: capturedAuthHeaders })).catch(() => {});
|
|
174
|
+
}
|
|
175
|
+
|
|
162
176
|
if (rawEndpoints.length > 0) {
|
|
163
177
|
const existingSkill = findExistingSkillForDomain(domain);
|
|
164
178
|
let allExisting = existingSkill?.endpoints ?? [];
|
|
@@ -183,7 +197,6 @@ export async function cacheBrowseRequests(params: {
|
|
|
183
197
|
for (const endpoint of mergedEndpoints) {
|
|
184
198
|
if (!endpoint.description) endpoint.description = generateLocalDescription(endpoint);
|
|
185
199
|
}
|
|
186
|
-
|
|
187
200
|
const quickSkill: SkillManifest = {
|
|
188
201
|
skill_id: existingSkill?.skill_id ?? nanoid(),
|
|
189
202
|
version: "1.0.0",
|
|
@@ -202,6 +215,16 @@ export async function cacheBrowseRequests(params: {
|
|
|
202
215
|
intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
|
|
203
216
|
};
|
|
204
217
|
|
|
218
|
+
// Token source discovery: scan live HTML for tokens used in captured
|
|
219
|
+
// request headers and attach AuthTokenBinding entries so serverFetch
|
|
220
|
+
// can rescrape fresh tokens on replay.
|
|
221
|
+
try {
|
|
222
|
+
const html = getPageHtml ? await getPageHtml() : undefined;
|
|
223
|
+
if (html && html.startsWith("<")) {
|
|
224
|
+
enrichEndpointsWithTokenSources(quickSkill.endpoints, requests, html, jsBundles);
|
|
225
|
+
}
|
|
226
|
+
} catch { /* best-effort */ }
|
|
227
|
+
|
|
205
228
|
const cacheKey = buildResolveCacheKey(domain, intent, sessionUrl);
|
|
206
229
|
const scopedKey = scopedCacheKey("global", cacheKey);
|
|
207
230
|
writeSkillSnapshot(scopedKey, quickSkill);
|
|
@@ -294,7 +317,6 @@ export async function cacheBrowseRequests(params: {
|
|
|
294
317
|
operation_graph: buildSkillOperationGraph(allEndpoints),
|
|
295
318
|
intents: [...new Set([...(existing?.intents ?? []), intent])],
|
|
296
319
|
};
|
|
297
|
-
|
|
298
320
|
const cacheKey = buildResolveCacheKey(domain, intent, sessionUrl);
|
|
299
321
|
const scopedKey = scopedCacheKey("global", cacheKey);
|
|
300
322
|
writeSkillSnapshot(scopedKey, skill);
|
|
@@ -2,7 +2,7 @@ import type { FastifyInstance } from "fastify";
|
|
|
2
2
|
import * as kuri from "../kuri/client.js";
|
|
3
3
|
import type { KuriHarEntry } from "../kuri/client.js";
|
|
4
4
|
import { extractEndpoints, extractAuthHeaders } from "../reverse-engineer/index.js";
|
|
5
|
-
import { INTERCEPTOR_SCRIPT, enrichPassiveCaptureRequests, injectInterceptor } from "../capture/index.js";
|
|
5
|
+
import { INTERCEPTOR_SCRIPT, enrichPassiveCaptureRequests, injectInterceptor, collectInterceptedRequests } from "../capture/index.js";
|
|
6
6
|
import { indexSkillLocally, mergeAgentReview, publishIndexedSkill, queueBackgroundIndex } from "../indexer/index.js";
|
|
7
7
|
import { nanoid } from "nanoid";
|
|
8
8
|
import type { ExecutionTrace, OrchestrationTiming, ProjectionOptions, SkillManifest } from "../types/index.js";
|
|
@@ -11,6 +11,7 @@ import { buildSkillOperationGraph, getEndpointDescriptionMetadata, getSkillChunk
|
|
|
11
11
|
import { augmentEndpointsWithAgent } from "../graph/agent-augment.js";
|
|
12
12
|
import { findExistingSkillForDomain, cachePublishedSkill } from "../client/index.js";
|
|
13
13
|
import { storeCredential } from "../vault/index.js";
|
|
14
|
+
import { getRegistrableDomain } from "../domain.js";
|
|
14
15
|
import { generateLocalDescription, writeSkillSnapshot, buildResolveCacheKey, getDomainReuseKey, domainSkillCache, persistDomainCache, scopedCacheKey, snapshotPathForCacheKey, invalidateRouteCacheForDomain, summarizeSchema, extractSampleValues } from "../orchestrator/index.js";
|
|
15
16
|
import { TRACE_VERSION, CODE_HASH, DEFAULT_BACKEND_URL, GIT_SHA, PACKAGE_VERSION } from "../version.js";
|
|
16
17
|
import { promoteExplicitExecution, resolveAndExecute, type OrchestratorResult } from "../orchestrator/index.js";
|
|
@@ -135,7 +136,7 @@ function passiveIndexFromRequests(
|
|
|
135
136
|
// 2. Extract and store auth credentials (cookies + sensitive headers)
|
|
136
137
|
const capturedAuthHeaders = extractAuthHeaders(requests);
|
|
137
138
|
if (Object.keys(capturedAuthHeaders).length > 0) {
|
|
138
|
-
const authKey = `${domain}-session`;
|
|
139
|
+
const authKey = `${getRegistrableDomain(domain)}-session`;
|
|
139
140
|
await storeCredential(authKey, JSON.stringify({ headers: capturedAuthHeaders }));
|
|
140
141
|
}
|
|
141
142
|
|
|
@@ -1178,11 +1179,9 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
1178
1179
|
|
|
1179
1180
|
async function restartBrowseCapture(session: BrowseSession): Promise<void> {
|
|
1180
1181
|
const broker = brokerForSession(session);
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
return;
|
|
1185
|
-
}
|
|
1182
|
+
// Start HAR + interceptor regardless of page load state — the page will load
|
|
1183
|
+
// and HAR records all traffic from the moment it starts.
|
|
1184
|
+
await broker.waitForLoad(session.tabId, 2_000).catch(() => null);
|
|
1186
1185
|
await broker.networkEnable(session.tabId).catch(() => {});
|
|
1187
1186
|
await broker.harStart(session.tabId).catch(() => {});
|
|
1188
1187
|
await broker.scriptInject(session.tabId, INTERCEPTOR_SCRIPT).catch(() => {});
|
|
@@ -1234,11 +1233,23 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
1234
1233
|
harEntries,
|
|
1235
1234
|
intent: `browse ${session.domain || profileName(session.url)}`,
|
|
1236
1235
|
});
|
|
1236
|
+
|
|
1237
|
+
// Collect JS bundle bodies for token source scanning
|
|
1238
|
+
const jsBundles = new Map<string, string>();
|
|
1239
|
+
try {
|
|
1240
|
+
const intercepted = await collectInterceptedRequests(session.tabId).catch(() => []);
|
|
1241
|
+
for (const entry of intercepted) {
|
|
1242
|
+
if (entry.is_js && entry.response_body && jsBundles.size < 20) {
|
|
1243
|
+
jsBundles.set(entry.url, entry.response_body);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
} catch { /* best-effort */ }
|
|
1237
1247
|
const syncResult = await cacheBrowseRequests({
|
|
1238
1248
|
sessionUrl: session.url,
|
|
1239
1249
|
sessionDomain: session.domain,
|
|
1240
1250
|
requests: allRequests,
|
|
1241
1251
|
getPageHtml: () => brokerForSession(session).getPageHtml(session.tabId),
|
|
1252
|
+
jsBundles: jsBundles.size > 0 ? jsBundles : undefined,
|
|
1242
1253
|
intent: `browse ${session.domain || profileName(session.url)}`,
|
|
1243
1254
|
});
|
|
1244
1255
|
|
|
@@ -1325,12 +1336,35 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
1325
1336
|
|
|
1326
1337
|
let cookiesInjected = 0;
|
|
1327
1338
|
if (newDomain && newDomain !== session.domain) {
|
|
1328
|
-
|
|
1329
|
-
|
|
1339
|
+
// Check if the browser already has fresh session cookies for this domain.
|
|
1340
|
+
// If so, skip vault/profile cookie injection — browser cookies are fresher
|
|
1341
|
+
// and injecting stale vault cookies (e.g. JSESSIONID) causes HTTP 400 on
|
|
1342
|
+
// sites like LinkedIn that validate CSRF alignment.
|
|
1343
|
+
const browserHasFreshSession = await (async () => {
|
|
1344
|
+
try {
|
|
1345
|
+
const { extractBrowserCookies } = await import("../auth/browser-cookies.js");
|
|
1346
|
+
const { cookies } = extractBrowserCookies(newDomain);
|
|
1347
|
+
// Consider the session fresh if we have session-like cookies that aren't expired
|
|
1348
|
+
const now = Date.now() / 1000;
|
|
1349
|
+
return cookies.some((c) =>
|
|
1350
|
+
(c.httpOnly || c.secure) && (!c.expires || c.expires > now),
|
|
1351
|
+
);
|
|
1352
|
+
} catch { return false; }
|
|
1353
|
+
})();
|
|
1354
|
+
|
|
1355
|
+
if (browserHasFreshSession) {
|
|
1356
|
+
// Import browser cookies via CDP (they're fresh from Chrome's jar)
|
|
1357
|
+
cookiesInjected = await importBrowserCookiesIntoTab(session.tabId, newDomain);
|
|
1358
|
+
} else {
|
|
1359
|
+
// No fresh browser cookies — load from vault/auth profile
|
|
1360
|
+
cookiesInjected = await importBrowserCookiesIntoTab(session.tabId, newDomain);
|
|
1361
|
+
await loadAuthProfileBestEffort(session.tabId, newDomain, "browse_go");
|
|
1362
|
+
}
|
|
1330
1363
|
}
|
|
1331
1364
|
|
|
1332
1365
|
await restartBrowseCapture(session);
|
|
1333
1366
|
|
|
1367
|
+
await broker.navigate(session.tabId, url);
|
|
1334
1368
|
await broker.navigate(session.tabId, url);
|
|
1335
1369
|
const finalUrl = await broker.getCurrentUrl(session.tabId).catch(() => url);
|
|
1336
1370
|
session.url = typeof finalUrl === "string" && finalUrl.startsWith("http") ? finalUrl : url;
|
|
@@ -10,6 +10,7 @@ import { buildSkillOperationGraph } from "../graph/index.js";
|
|
|
10
10
|
import { augmentEndpointsWithAgent } from "../graph/agent-augment.js";
|
|
11
11
|
import { findExistingSkillForDomain, cachePublishedSkill } from "../client/index.js";
|
|
12
12
|
import { storeCredential } from "../vault/index.js";
|
|
13
|
+
import { getRegistrableDomain } from "../domain.js";
|
|
13
14
|
import {
|
|
14
15
|
importBrowserCookiesIntoTab,
|
|
15
16
|
loadAuthProfileBestEffort,
|
|
@@ -83,7 +84,7 @@ function passiveIndexHar(entries: KuriHarEntry[], pageUrl: string): void {
|
|
|
83
84
|
// Store auth credentials
|
|
84
85
|
const capturedAuthHeaders = extractAuthHeaders(requests);
|
|
85
86
|
if (Object.keys(capturedAuthHeaders).length > 0) {
|
|
86
|
-
await storeCredential(`${domain}-session`, JSON.stringify({ headers: capturedAuthHeaders }));
|
|
87
|
+
await storeCredential(`${getRegistrableDomain(domain)}-session`, JSON.stringify({ headers: capturedAuthHeaders }));
|
|
87
88
|
}
|
|
88
89
|
|
|
89
90
|
// Merge with existing skill (never reduce endpoint count)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export const BUILD_RELEASE_VERSION = "3.0.
|
|
2
|
-
export const BUILD_GIT_SHA = "
|
|
1
|
+
export const BUILD_RELEASE_VERSION = "3.1.0-experiments.5e7a7bb";
|
|
2
|
+
export const BUILD_GIT_SHA = "5e7a7bb949c1";
|
|
3
3
|
export const BUILD_CODE_HASH = "1488fc1d92b7";
|
|
4
|
-
export const BUILD_RELEASE_MANIFEST_BASE64 = "
|
|
5
|
-
export const BUILD_RELEASE_MANIFEST_SIGNATURE = "
|
|
6
|
-
export const BUILD_DEFAULT_BACKEND_URL = "https://
|
|
4
|
+
export const BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4xLjAtZXhwZXJpbWVudHMuNWU3YTdiYiIsImdpdF9zaGEiOiI1ZTdhN2JiOTQ5YzEiLCJjb2RlX2hhc2giOiIxNDg4ZmMxZDkyYjciLCJ0cmFjZV92ZXJzaW9uIjoiMTQ4OGZjMWQ5MmI3QDVlN2E3YmI5NDljMSIsImlzc3VlZF9hdCI6IjIwMjYtMDQtMDVUMTQ6NTY6MjkuNjY2WiJ9";
|
|
5
|
+
export const BUILD_RELEASE_MANIFEST_SIGNATURE = "OuZD9NeemoStAyT3-MgMS3V3eeatbRMKkVY_J4_6nsM";
|
|
6
|
+
export const BUILD_DEFAULT_BACKEND_URL = "https://unbrowse-backend-experiments.lewis-6d8.workers.dev";
|
|
@@ -30,6 +30,114 @@ const activeTabRegistry = new Set<string>();
|
|
|
30
30
|
// Tracks tabs where scriptInject has been registered (persistent across navigations)
|
|
31
31
|
const interceptorInjectedTabs = new Set<string>();
|
|
32
32
|
|
|
33
|
+
// Tracks tabs where CDP-level document-start injection has been registered
|
|
34
|
+
const cdpDocStartTabs = new Set<string>();
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Register a script via Chrome's Page.addScriptToEvaluateOnNewDocument CDP method directly.
|
|
38
|
+
* This runs BEFORE any page JS on every navigation — critical for catching early fetch() calls.
|
|
39
|
+
* Falls back silently if CDP is unavailable.
|
|
40
|
+
*/
|
|
41
|
+
export async function registerDocumentStartScript(tabId: string, source: string): Promise<boolean> {
|
|
42
|
+
if (cdpDocStartTabs.has(tabId)) return true;
|
|
43
|
+
const cdpPort = kuri.getCdpPort();
|
|
44
|
+
if (!cdpPort) return false;
|
|
45
|
+
|
|
46
|
+
try {
|
|
47
|
+
// Get the WebSocket URL for this tab
|
|
48
|
+
const resp = await fetch(`http://127.0.0.1:${cdpPort}/json`);
|
|
49
|
+
const targets = await resp.json() as Array<{ id: string; webSocketDebuggerUrl?: string }>;
|
|
50
|
+
const target = targets.find((t) => t.id === tabId);
|
|
51
|
+
if (!target?.webSocketDebuggerUrl) return false;
|
|
52
|
+
|
|
53
|
+
// Connect via WebSocket and send CDP command
|
|
54
|
+
const ws = new WebSocket(target.webSocketDebuggerUrl);
|
|
55
|
+
const result = await new Promise<boolean>((resolve) => {
|
|
56
|
+
const timeout = setTimeout(() => { ws.close(); resolve(false); }, 3000);
|
|
57
|
+
ws.onopen = () => {
|
|
58
|
+
ws.send(JSON.stringify({
|
|
59
|
+
id: 1,
|
|
60
|
+
method: "Page.addScriptToEvaluateOnNewDocument",
|
|
61
|
+
params: { source },
|
|
62
|
+
}));
|
|
63
|
+
};
|
|
64
|
+
ws.onmessage = (event) => {
|
|
65
|
+
try {
|
|
66
|
+
const msg = JSON.parse(String(event.data));
|
|
67
|
+
if (msg.id === 1) {
|
|
68
|
+
clearTimeout(timeout);
|
|
69
|
+
ws.close();
|
|
70
|
+
resolve(!msg.error);
|
|
71
|
+
}
|
|
72
|
+
} catch { /* ignore parse errors */ }
|
|
73
|
+
};
|
|
74
|
+
ws.onerror = () => { clearTimeout(timeout); ws.close(); resolve(false); };
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
if (result) {
|
|
78
|
+
cdpDocStartTabs.add(tabId);
|
|
79
|
+
log("capture", `document-start interceptor registered via direct CDP for tab ${tabId}`);
|
|
80
|
+
}
|
|
81
|
+
return result;
|
|
82
|
+
} catch {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Tracks captured request headers from CDP Network events
|
|
88
|
+
const cdpCapturedHeaders = new Map<string, Map<string, Record<string, string>>>();
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Enable CDP Network.requestWillBeSent listener to capture full request headers
|
|
92
|
+
* including auth headers that HAR/interceptor miss. Stores headers indexed by URL.
|
|
93
|
+
*/
|
|
94
|
+
export async function enableNetworkHeaderCapture(tabId: string): Promise<void> {
|
|
95
|
+
const cdpPort = kuri.getCdpPort();
|
|
96
|
+
if (!cdpPort) return;
|
|
97
|
+
|
|
98
|
+
try {
|
|
99
|
+
const resp = await fetch(`http://127.0.0.1:${cdpPort}/json`);
|
|
100
|
+
const targets = await resp.json() as Array<{ id: string; webSocketDebuggerUrl?: string }>;
|
|
101
|
+
const target = targets.find((t) => t.id === tabId);
|
|
102
|
+
if (!target?.webSocketDebuggerUrl) return;
|
|
103
|
+
|
|
104
|
+
const headers = new Map<string, Record<string, string>>();
|
|
105
|
+
cdpCapturedHeaders.set(tabId, headers);
|
|
106
|
+
|
|
107
|
+
const ws = new WebSocket(target.webSocketDebuggerUrl);
|
|
108
|
+
ws.onopen = () => {
|
|
109
|
+
// Enable network domain (may already be enabled, that's OK)
|
|
110
|
+
ws.send(JSON.stringify({ id: 1, method: "Network.enable", params: {} }));
|
|
111
|
+
};
|
|
112
|
+
ws.onmessage = (event) => {
|
|
113
|
+
try {
|
|
114
|
+
const msg = JSON.parse(String(event.data));
|
|
115
|
+
if (msg.method === "Network.requestWillBeSent") {
|
|
116
|
+
const req = msg.params?.request;
|
|
117
|
+
if (req?.url && req?.headers) {
|
|
118
|
+
// Only capture headers for API-like URLs
|
|
119
|
+
if (/\/(api|graphql|v\d+)\b/i.test(req.url) || /ads-api|voyager/i.test(req.url)) {
|
|
120
|
+
headers.set(req.url, req.headers);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
} catch { /* ignore */ }
|
|
125
|
+
};
|
|
126
|
+
// Keep ws alive — it will be cleaned up when tab closes
|
|
127
|
+
ws.onerror = () => { cdpCapturedHeaders.delete(tabId); };
|
|
128
|
+
|
|
129
|
+
log("capture", `CDP network header capture enabled for tab ${tabId}`);
|
|
130
|
+
} catch { /* best-effort */ }
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Get captured request headers for a tab. Returns a map of URL → headers.
|
|
135
|
+
* These come from CDP Network.requestWillBeSent which includes auth headers
|
|
136
|
+
* that HAR and the JS interceptor miss.
|
|
137
|
+
*/
|
|
138
|
+
export function getCapturedNetworkHeaders(tabId: string): Map<string, Record<string, string>> {
|
|
139
|
+
return cdpCapturedHeaders.get(tabId) ?? new Map();
|
|
140
|
+
}
|
|
33
141
|
// Hard timeout per capture: 90s prevents stuck tabs from holding slots forever
|
|
34
142
|
const CAPTURE_TIMEOUT_MS = 90_000;
|
|
35
143
|
const CAPTURE_NAV_TIMEOUT_MS = 20_000;
|
|
@@ -376,6 +484,11 @@ export const INTERCEPTOR_SCRIPT = `(function() {
|
|
|
376
484
|
var method = (opts.method || 'GET').toUpperCase();
|
|
377
485
|
var reqBody = opts.body ? String(opts.body).substring(0, MAX_BODY) : undefined;
|
|
378
486
|
var reqHeaders = {};
|
|
487
|
+
// Extract headers from Request object (first arg)
|
|
488
|
+
if (args[0] && typeof args[0] === 'object' && args[0].headers && typeof args[0].headers.forEach === 'function') {
|
|
489
|
+
args[0].headers.forEach(function(v, k) { reqHeaders[k] = v; });
|
|
490
|
+
}
|
|
491
|
+
// Override/merge with explicit opts.headers (second arg)
|
|
379
492
|
if (opts.headers) {
|
|
380
493
|
if (typeof opts.headers.forEach === 'function') {
|
|
381
494
|
opts.headers.forEach(function(v, k) { reqHeaders[k] = v; });
|
package/runtime-src/cli.ts
CHANGED
|
@@ -13,10 +13,15 @@ import {
|
|
|
13
13
|
detectTelemetryHostType,
|
|
14
14
|
ensureCliInstallTracked,
|
|
15
15
|
ensureRegistered,
|
|
16
|
+
getAgentId,
|
|
16
17
|
getApiKey,
|
|
18
|
+
getCreatorEarnings,
|
|
19
|
+
getMyProfile,
|
|
20
|
+
getTransactionHistory,
|
|
17
21
|
recordFunnelTelemetryEvent,
|
|
18
22
|
recordInstallTelemetryEvent,
|
|
19
23
|
} from "./client/index.js";
|
|
24
|
+
import { appendImpact, getImpactLogPath, impactFromResult, readImpactSummary } from "./impact-log.js";
|
|
20
25
|
import { findSitePack, findTask, allSitePacks, buildDepsGraph, planExecution, buildDepsMetadata, type SitePack } from "./cli/shortcuts.js";
|
|
21
26
|
import { ensureLocalServer, checkServerVersion, stopServer, restartServer } from "./runtime/local-server.js";
|
|
22
27
|
import { isBundledVirtualEntrypoint, isMainModule, resolveSiblingEntrypoint, runtimeArgsForEntrypoint } from "./runtime/paths.js";
|
|
@@ -30,6 +35,7 @@ loadEnv({ path: ".env.runtime", quiet: true });
|
|
|
30
35
|
|
|
31
36
|
const BASE_URL = process.env.UNBROWSE_URL || "http://localhost:6969";
|
|
32
37
|
const CLI_CLIENT_ID = process.env.UNBROWSE_CLIENT_ID || `cli-${process.ppid || process.pid}`;
|
|
38
|
+
let walletNudgeShown = false;
|
|
33
39
|
|
|
34
40
|
// ---------------------------------------------------------------------------
|
|
35
41
|
// Arg parser
|
|
@@ -184,6 +190,14 @@ function formatSavedDuration(ms: number): string {
|
|
|
184
190
|
return `${ms}ms`;
|
|
185
191
|
}
|
|
186
192
|
|
|
193
|
+
function formatCostUsd(uc: number): string {
|
|
194
|
+
// uc = micro-USD (1e-6 USD). 1M uc = $1.
|
|
195
|
+
const usd = uc / 1_000_000;
|
|
196
|
+
if (usd >= 1) return `$${usd.toFixed(2)}`;
|
|
197
|
+
if (usd >= 0.01) return `$${usd.toFixed(3)}`;
|
|
198
|
+
return `$${usd.toFixed(4)}`;
|
|
199
|
+
}
|
|
200
|
+
|
|
187
201
|
function emitImpactSummary(result: Record<string, unknown>): void {
|
|
188
202
|
const impact = result.impact as Record<string, unknown> | undefined;
|
|
189
203
|
if (!impact) return;
|
|
@@ -192,12 +206,14 @@ function emitImpactSummary(result: Record<string, unknown>): void {
|
|
|
192
206
|
const tokensSaved = typeof impact.tokens_saved === "number" ? impact.tokens_saved : 0;
|
|
193
207
|
const timeSavedPct = typeof impact.time_saved_pct === "number" ? impact.time_saved_pct : 0;
|
|
194
208
|
const tokensSavedPct = typeof impact.tokens_saved_pct === "number" ? impact.tokens_saved_pct : 0;
|
|
209
|
+
const costSavedUc = typeof impact.cost_saved_uc === "number" ? impact.cost_saved_uc : 0;
|
|
195
210
|
const browserAvoided = impact.browser_avoided === true;
|
|
196
|
-
if (timeSavedMs <= 0 && tokensSaved <= 0 && !browserAvoided) return;
|
|
211
|
+
if (timeSavedMs <= 0 && tokensSaved <= 0 && costSavedUc <= 0 && !browserAvoided) return;
|
|
197
212
|
|
|
198
213
|
const parts: string[] = [];
|
|
199
214
|
if (timeSavedMs > 0) parts.push(`${formatSavedDuration(timeSavedMs)} saved (${timeSavedPct}% faster)`);
|
|
200
215
|
if (tokensSaved > 0) parts.push(`${tokensSaved.toLocaleString("en-US")} tokens saved (${tokensSavedPct}% less context)`);
|
|
216
|
+
if (costSavedUc > 0) parts.push(`${formatCostUsd(costSavedUc)} saved`);
|
|
201
217
|
if (browserAvoided) parts.push("browser avoided");
|
|
202
218
|
info(parts.join(" • "));
|
|
203
219
|
}
|
|
@@ -370,10 +386,27 @@ async function cmdResolve(flags: Record<string, string | boolean>): Promise<void
|
|
|
370
386
|
|
|
371
387
|
result = slimTrace(result);
|
|
372
388
|
emitImpactSummary(result);
|
|
389
|
+
{
|
|
390
|
+
const entry = impactFromResult("resolve", result, { intent, domain, });
|
|
391
|
+
if (entry) appendImpact(entry);
|
|
392
|
+
}
|
|
373
393
|
emitNextActionSummary(result);
|
|
374
394
|
|
|
375
395
|
const skill = result.skill as Record<string, unknown> | undefined;
|
|
376
396
|
const trace = result.trace as Record<string, unknown> | undefined;
|
|
397
|
+
|
|
398
|
+
// Nudge wallet setup after successful resolve that indexed routes
|
|
399
|
+
if (trace?.success && !walletNudgeShown) {
|
|
400
|
+
try {
|
|
401
|
+
const { checkWalletConfigured } = await import("./payments/wallet.js");
|
|
402
|
+
const wallet = checkWalletConfigured();
|
|
403
|
+
if (!wallet.configured) {
|
|
404
|
+
info("You're indexing routes but have no payout wallet. Run: npx @crossmint/lobster-cli setup");
|
|
405
|
+
walletNudgeShown = true;
|
|
406
|
+
}
|
|
407
|
+
} catch (_e) { /* non-fatal */ }
|
|
408
|
+
}
|
|
409
|
+
|
|
377
410
|
if (skill?.skill_id && trace) {
|
|
378
411
|
(result as Record<string, unknown>)._feedback = `unbrowse feedback --skill ${skill.skill_id} --endpoint ${trace.endpoint_id || "?"} --rating <1-5>`;
|
|
379
412
|
}
|
|
@@ -548,6 +581,13 @@ async function cmdExecute(flags: Record<string, string | boolean>): Promise<void
|
|
|
548
581
|
// Strip metadata bloat
|
|
549
582
|
result = slimTrace(result);
|
|
550
583
|
emitImpactSummary(result);
|
|
584
|
+
{
|
|
585
|
+
const entry = impactFromResult("execute", result, {
|
|
586
|
+
skill_id: skillId,
|
|
587
|
+
endpoint_id: typeof flags.endpoint === "string" ? flags.endpoint : undefined,
|
|
588
|
+
});
|
|
589
|
+
if (entry) appendImpact(entry);
|
|
590
|
+
}
|
|
551
591
|
emitNextActionSummary(result);
|
|
552
592
|
|
|
553
593
|
const pathFlag = flags.path as string | undefined;
|
|
@@ -812,6 +852,18 @@ async function cmdSetup(flags: Record<string, string | boolean>): Promise<void>
|
|
|
812
852
|
}
|
|
813
853
|
}
|
|
814
854
|
|
|
855
|
+
// Wallet status — tell the user if they're missing payout config
|
|
856
|
+
if (report.wallet.configured) {
|
|
857
|
+
info(`Wallet configured (${report.wallet.provider}): ${(report.wallet as Record<string, unknown>).wallet_address ?? "linked"}`);
|
|
858
|
+
} else if ((report.wallet as Record<string, unknown>).lobster_installed) {
|
|
859
|
+
info("Wallet not paired — your indexed routes won't earn payouts.");
|
|
860
|
+
info("Run: npx @crossmint/lobster-cli setup");
|
|
861
|
+
} else {
|
|
862
|
+
info("No wallet configured — you're indexing routes for free.");
|
|
863
|
+
info("Set up a wallet so you earn when agents use your routes:");
|
|
864
|
+
info(" npx @crossmint/lobster-cli setup");
|
|
865
|
+
}
|
|
866
|
+
|
|
815
867
|
await recordInstallTelemetryEvent("setup", {
|
|
816
868
|
hostType,
|
|
817
869
|
status: report.browser_engine.action === "failed" ? "failed" : "installed",
|
|
@@ -913,6 +965,7 @@ export const CLI_REFERENCE = {
|
|
|
913
965
|
{ name: "forward", usage: "[--session id]", desc: "Navigate forward" },
|
|
914
966
|
{ name: "sync", usage: "[--session id]", desc: "Checkpoint current capture, keep tab open, queue background index + publish, then inspect via skill/publish review" },
|
|
915
967
|
{ name: "close", usage: "[--session id]", desc: "Checkpoint capture, queue background index + publish, close browse session, then inspect via skill/publish review" },
|
|
968
|
+
{ name: "stats", usage: "[--json] [--pretty]", desc: "Show lifetime time/tokens/cost saved and marketplace earnings/spending" },
|
|
916
969
|
],
|
|
917
970
|
globalFlags: [
|
|
918
971
|
{ flag: "--pretty", desc: "Indented JSON output" },
|
|
@@ -954,6 +1007,139 @@ export const CLI_REFERENCE = {
|
|
|
954
1007
|
],
|
|
955
1008
|
};
|
|
956
1009
|
|
|
1010
|
+
|
|
1011
|
+
// ---------------------------------------------------------------------------
|
|
1012
|
+
// stats — show lifetime impact (savings + earnings) for the current agent
|
|
1013
|
+
// ---------------------------------------------------------------------------
|
|
1014
|
+
|
|
1015
|
+
function formatTotalDuration(ms: number): string {
|
|
1016
|
+
if (ms >= 3_600_000) return `${(ms / 3_600_000).toFixed(1)}h`;
|
|
1017
|
+
if (ms >= 60_000) return `${(ms / 60_000).toFixed(1)}m`;
|
|
1018
|
+
if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`;
|
|
1019
|
+
return `${ms}ms`;
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
async function cmdStats(flags: Record<string, string | boolean>): Promise<void> {
|
|
1023
|
+
const pretty = !!flags.pretty;
|
|
1024
|
+
const jsonOnly = !!flags.json;
|
|
1025
|
+
const local = readImpactSummary();
|
|
1026
|
+
const agentId = getAgentId();
|
|
1027
|
+
|
|
1028
|
+
type EarningsLedger = { total_earned_uc: number; total_earned_usd: number; transaction_count: number; last_transaction_at?: string } | null;
|
|
1029
|
+
type SpendingLedger = { total_spent_uc: number; total_spent_usd: number; transaction_count: number; last_transaction_at?: string } | null;
|
|
1030
|
+
|
|
1031
|
+
let profile: Awaited<ReturnType<typeof getMyProfile>> | null = null;
|
|
1032
|
+
let earnings: { ledger: EarningsLedger; transactions: unknown[] } | null = null;
|
|
1033
|
+
let spending: { ledger: SpendingLedger; transactions: unknown[] } | null = null;
|
|
1034
|
+
const remoteErrors: Record<string, string> = {};
|
|
1035
|
+
|
|
1036
|
+
if (agentId) {
|
|
1037
|
+
const results = await Promise.allSettled([
|
|
1038
|
+
getMyProfile(),
|
|
1039
|
+
getCreatorEarnings(agentId),
|
|
1040
|
+
getTransactionHistory(agentId),
|
|
1041
|
+
]);
|
|
1042
|
+
if (results[0].status === "fulfilled") profile = results[0].value;
|
|
1043
|
+
else remoteErrors.profile = (results[0].reason as Error)?.message ?? String(results[0].reason);
|
|
1044
|
+
if (results[1].status === "fulfilled") earnings = results[1].value as { ledger: EarningsLedger; transactions: unknown[] };
|
|
1045
|
+
else remoteErrors.earnings = (results[1].reason as Error)?.message ?? String(results[1].reason);
|
|
1046
|
+
if (results[2].status === "fulfilled") spending = results[2].value as { ledger: SpendingLedger; transactions: unknown[] };
|
|
1047
|
+
else remoteErrors.spending = (results[2].reason as Error)?.message ?? String(results[2].reason);
|
|
1048
|
+
} else {
|
|
1049
|
+
remoteErrors.profile = "No agent_id in local config. Run `unbrowse setup` to register.";
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
const earnedUsd = earnings?.ledger?.total_earned_usd ?? 0;
|
|
1053
|
+
const spentUsd = spending?.ledger?.total_spent_usd ?? 0;
|
|
1054
|
+
const netUsd = earnedUsd - spentUsd;
|
|
1055
|
+
const savedUsd = local.total_cost_saved_uc / 1_000_000;
|
|
1056
|
+
|
|
1057
|
+
const payload = {
|
|
1058
|
+
agent_id: agentId,
|
|
1059
|
+
profile,
|
|
1060
|
+
impact: {
|
|
1061
|
+
total_runs: local.total_runs,
|
|
1062
|
+
successful_runs: local.successful_runs,
|
|
1063
|
+
browser_avoided_runs: local.browser_avoided_runs,
|
|
1064
|
+
total_time_saved_ms: local.total_time_saved_ms,
|
|
1065
|
+
total_time_saved_human: formatTotalDuration(local.total_time_saved_ms),
|
|
1066
|
+
total_tokens_saved: local.total_tokens_saved,
|
|
1067
|
+
total_cost_saved_usd: Number(savedUsd.toFixed(6)),
|
|
1068
|
+
avg_time_saved_pct: local.avg_time_saved_pct,
|
|
1069
|
+
avg_tokens_saved_pct: local.avg_tokens_saved_pct,
|
|
1070
|
+
by_source: local.by_source,
|
|
1071
|
+
first_entry_at: local.first_entry_at,
|
|
1072
|
+
last_entry_at: local.last_entry_at,
|
|
1073
|
+
log_path: getImpactLogPath(),
|
|
1074
|
+
},
|
|
1075
|
+
earnings: {
|
|
1076
|
+
total_earned_usd: earnedUsd,
|
|
1077
|
+
total_earned_uc: earnings?.ledger?.total_earned_uc ?? 0,
|
|
1078
|
+
transaction_count: earnings?.ledger?.transaction_count ?? 0,
|
|
1079
|
+
last_transaction_at: earnings?.ledger?.last_transaction_at ?? null,
|
|
1080
|
+
},
|
|
1081
|
+
spending: {
|
|
1082
|
+
total_spent_usd: spentUsd,
|
|
1083
|
+
total_spent_uc: spending?.ledger?.total_spent_uc ?? 0,
|
|
1084
|
+
transaction_count: spending?.ledger?.transaction_count ?? 0,
|
|
1085
|
+
last_transaction_at: spending?.ledger?.last_transaction_at ?? null,
|
|
1086
|
+
},
|
|
1087
|
+
net_usd: netUsd,
|
|
1088
|
+
...(Object.keys(remoteErrors).length > 0 ? { remote_errors: remoteErrors } : {}),
|
|
1089
|
+
};
|
|
1090
|
+
|
|
1091
|
+
if (jsonOnly) {
|
|
1092
|
+
output(payload, pretty);
|
|
1093
|
+
return;
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
// Human-readable view (to stderr like other info) + JSON to stdout for piping.
|
|
1097
|
+
const lines: string[] = [];
|
|
1098
|
+
lines.push("Unbrowse stats");
|
|
1099
|
+
lines.push(` agent_id: ${agentId ?? "(not registered — run `unbrowse setup`)"}`);
|
|
1100
|
+
if (profile?.name) lines.push(` name: ${profile.name}`);
|
|
1101
|
+
lines.push("");
|
|
1102
|
+
lines.push("Impact (local, this machine):");
|
|
1103
|
+
if (local.total_runs === 0) {
|
|
1104
|
+
lines.push(" No resolve/execute runs recorded yet.");
|
|
1105
|
+
lines.push(` Log file: ${getImpactLogPath()}`);
|
|
1106
|
+
} else {
|
|
1107
|
+
lines.push(` Runs: ${local.total_runs} (${local.successful_runs} successful, ${local.browser_avoided_runs} browser-avoided)`);
|
|
1108
|
+
if (local.total_time_saved_ms > 0) {
|
|
1109
|
+
lines.push(` Time saved: ${formatTotalDuration(local.total_time_saved_ms)} (avg ${local.avg_time_saved_pct}% faster)`);
|
|
1110
|
+
}
|
|
1111
|
+
if (local.total_tokens_saved > 0) {
|
|
1112
|
+
lines.push(` Tokens saved: ${local.total_tokens_saved.toLocaleString("en-US")} (avg ${local.avg_tokens_saved_pct}% less context)`);
|
|
1113
|
+
}
|
|
1114
|
+
if (savedUsd > 0) {
|
|
1115
|
+
lines.push(` Cost saved: ${formatCostUsd(local.total_cost_saved_uc)}`);
|
|
1116
|
+
}
|
|
1117
|
+
if (Object.keys(local.by_source).length > 0) {
|
|
1118
|
+
const topSources = Object.entries(local.by_source)
|
|
1119
|
+
.sort((a, b) => b[1] - a[1])
|
|
1120
|
+
.slice(0, 5)
|
|
1121
|
+
.map(([k, v]) => `${k}=${v}`)
|
|
1122
|
+
.join(", ");
|
|
1123
|
+
lines.push(` By source: ${topSources}`);
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
lines.push("");
|
|
1127
|
+
lines.push("Money (backend ledger):");
|
|
1128
|
+
if (!agentId) {
|
|
1129
|
+
lines.push(" (not registered — earnings/spending unavailable)");
|
|
1130
|
+
} else if (remoteErrors.earnings || remoteErrors.spending) {
|
|
1131
|
+
if (remoteErrors.earnings) lines.push(` earnings: error — ${remoteErrors.earnings}`);
|
|
1132
|
+
if (remoteErrors.spending) lines.push(` spending: error — ${remoteErrors.spending}`);
|
|
1133
|
+
} else {
|
|
1134
|
+
lines.push(` Earned: $${earnedUsd.toFixed(4)} (${earnings?.ledger?.transaction_count ?? 0} payouts)`);
|
|
1135
|
+
lines.push(` Spent: $${spentUsd.toFixed(4)} (${spending?.ledger?.transaction_count ?? 0} payments)`);
|
|
1136
|
+
lines.push(` Net: ${netUsd >= 0 ? "+" : ""}$${netUsd.toFixed(4)}`);
|
|
1137
|
+
}
|
|
1138
|
+
lines.push("");
|
|
1139
|
+
info(lines.join("\n"));
|
|
1140
|
+
output(payload, pretty);
|
|
1141
|
+
}
|
|
1142
|
+
|
|
957
1143
|
function printHelp(): void {
|
|
958
1144
|
const r = CLI_REFERENCE;
|
|
959
1145
|
const lines: string[] = ["unbrowse \u2014 shell-safe CLI for the local API", ""];
|
|
@@ -1463,6 +1649,7 @@ async function main(): Promise<void> {
|
|
|
1463
1649
|
if (command === "restart") return cmdRestart(flags);
|
|
1464
1650
|
if (command === "upgrade" || command === "update") return cmdUpgrade(flags);
|
|
1465
1651
|
if (command === "connect-chrome") return cmdConnectChrome();
|
|
1652
|
+
if (command === "stats") return cmdStats(flags);
|
|
1466
1653
|
|
|
1467
1654
|
// --- Shortcut resolution: unbrowse <site> [task] [flags] ---
|
|
1468
1655
|
const KNOWN_COMMANDS = new Set([
|
|
@@ -1471,7 +1658,7 @@ async function main(): Promise<void> {
|
|
|
1471
1658
|
"status", "stop", "restart", "upgrade", "update",
|
|
1472
1659
|
"go", "submit", "snap", "click", "fill", "type", "press", "select", "scroll",
|
|
1473
1660
|
"screenshot", "text", "markdown", "cookies", "eval", "back", "forward", "sync", "close",
|
|
1474
|
-
"connect-chrome",
|
|
1661
|
+
"connect-chrome", "stats",
|
|
1475
1662
|
]);
|
|
1476
1663
|
|
|
1477
1664
|
if (!KNOWN_COMMANDS.has(command)) {
|
|
@@ -1533,6 +1720,7 @@ async function main(): Promise<void> {
|
|
|
1533
1720
|
case "sync": return cmdSync(flags);
|
|
1534
1721
|
case "close": return cmdClose(flags);
|
|
1535
1722
|
case "connect-chrome": return cmdConnectChrome();
|
|
1723
|
+
case "stats": return cmdStats(flags);
|
|
1536
1724
|
default: info(`Unknown command: ${command}`); printHelp(); process.exit(1);
|
|
1537
1725
|
}
|
|
1538
1726
|
}
|