unbrowse 2.12.0 → 2.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -56
- package/bin/unbrowse-wrapper.mjs +0 -0
- package/dist/cli.js +232 -471
- package/package.json +1 -1
- package/runtime-src/analytics-session.ts +5 -27
- package/runtime-src/api/browse-session.ts +3 -1
- package/runtime-src/api/browse-submit.ts +27 -34
- package/runtime-src/api/routes.ts +103 -145
- package/runtime-src/cli.ts +8 -175
- package/runtime-src/client/index.ts +44 -15
- package/runtime-src/execution/index.ts +1 -6
- package/runtime-src/indexer/index.ts +2 -4
- package/runtime-src/kuri/client.ts +35 -2
- package/runtime-src/orchestrator/index.ts +2 -2
- package/runtime-src/reverse-engineer/index.ts +0 -8
- package/runtime-src/runtime/setup.ts +7 -7
- package/runtime-src/server.ts +4 -3
- package/dist/mcp.js +0 -20392
- package/runtime-src/agent-outcome.ts +0 -166
- package/runtime-src/mcp.ts +0 -1065
package/package.json
CHANGED
|
@@ -10,30 +10,15 @@ export interface AnalyticsSessionPayload {
|
|
|
10
10
|
cached_skill_calls: number;
|
|
11
11
|
fresh_index_calls: number;
|
|
12
12
|
browser_mode: "default" | "replaced" | "manual" | "unknown";
|
|
13
|
-
success?: boolean;
|
|
14
|
-
source?: string;
|
|
15
|
-
time_saved_ms?: number;
|
|
16
|
-
time_saved_pct?: number;
|
|
17
|
-
tokens_saved?: number;
|
|
18
|
-
tokens_saved_pct?: number;
|
|
19
|
-
cost_saved_uc?: number;
|
|
20
13
|
}
|
|
21
14
|
|
|
22
15
|
export function buildAnalyticsSessionPayload(
|
|
23
16
|
sessionId: string,
|
|
24
17
|
startedAt: string,
|
|
25
|
-
source: OrchestrationTiming["source"]
|
|
26
|
-
trace: Pick<ExecutionTrace, "completed_at" | "
|
|
27
|
-
network_events?: unknown[];
|
|
28
|
-
},
|
|
29
|
-
timing?: Pick<OrchestrationTiming, "time_saved_ms" | "time_saved_pct" | "cost_saved_uc">,
|
|
18
|
+
source: OrchestrationTiming["source"],
|
|
19
|
+
trace: Pick<ExecutionTrace, "completed_at" | "network_events" | "trace_version">,
|
|
30
20
|
): AnalyticsSessionPayload {
|
|
31
|
-
const cacheLike = source === "marketplace" || source === "route-cache";
|
|
32
|
-
const browserMode = source === "live-capture" || source === "browser-action"
|
|
33
|
-
? "default"
|
|
34
|
-
: source === "first-pass"
|
|
35
|
-
? "default"
|
|
36
|
-
: "replaced";
|
|
21
|
+
const cacheLike = source === "marketplace" || source === "route-cache" || source === "first-pass";
|
|
37
22
|
return {
|
|
38
23
|
session_id: sessionId,
|
|
39
24
|
started_at: startedAt,
|
|
@@ -42,14 +27,7 @@ export function buildAnalyticsSessionPayload(
|
|
|
42
27
|
api_calls: Math.max(1, trace.network_events?.length ?? 0),
|
|
43
28
|
discovery_queries: cacheLike ? 1 : 0,
|
|
44
29
|
cached_skill_calls: cacheLike ? 1 : 0,
|
|
45
|
-
fresh_index_calls: source === "live-capture"
|
|
46
|
-
browser_mode:
|
|
47
|
-
success: trace.success ?? true,
|
|
48
|
-
source,
|
|
49
|
-
time_saved_ms: timing?.time_saved_ms,
|
|
50
|
-
time_saved_pct: timing?.time_saved_pct,
|
|
51
|
-
tokens_saved: trace.tokens_saved,
|
|
52
|
-
tokens_saved_pct: trace.tokens_saved_pct,
|
|
53
|
-
cost_saved_uc: timing?.cost_saved_uc,
|
|
30
|
+
fresh_index_calls: source === "live-capture" ? 1 : 0,
|
|
31
|
+
browser_mode: "unknown",
|
|
54
32
|
};
|
|
55
33
|
}
|
|
@@ -50,6 +50,7 @@ async function createBrowseSession(
|
|
|
50
50
|
): Promise<BrowseSession> {
|
|
51
51
|
await client.start().catch(() => {});
|
|
52
52
|
const tabId = await client.newTab();
|
|
53
|
+
if (!tabId) throw new Error("Failed to create browser tab");
|
|
53
54
|
await client.harStart(tabId).catch(() => {});
|
|
54
55
|
await injectInterceptor(tabId);
|
|
55
56
|
const session: BrowseSession = { tabId, url: "about:blank", harActive: true, domain: "" };
|
|
@@ -80,7 +81,8 @@ async function adoptExistingBrowseTab(
|
|
|
80
81
|
const domain = extractDomain(tab.url);
|
|
81
82
|
return !!domain && !!normalizedPreferred && domain === normalizedPreferred;
|
|
82
83
|
}) ??
|
|
83
|
-
tabs.find((tab) => /^
|
|
84
|
+
tabs.find((tab) => /^(about:blank|chrome:\/\/newtab\/?)$/i.test(tab.url ?? "")) ??
|
|
85
|
+
(!normalizedPreferred ? tabs.find((tab) => /^https?:\/\//.test(tab.url ?? "")) : undefined);
|
|
84
86
|
|
|
85
87
|
if (!candidate?.id) return null;
|
|
86
88
|
await client.harStart(candidate.id).catch(() => {});
|
|
@@ -19,20 +19,10 @@ export interface BrowseSubmitClient {
|
|
|
19
19
|
export interface BrowseSubmitDeps {
|
|
20
20
|
client: BrowseSubmitClient;
|
|
21
21
|
session: BrowseSession;
|
|
22
|
-
flushCapture?: (session: BrowseSession) => Promise<BrowseSubmitCaptureSyncResult | null>;
|
|
23
22
|
restartCapture: (session: BrowseSession) => Promise<void>;
|
|
24
23
|
rehydratePlugins: (tabId: string) => Promise<unknown>;
|
|
25
24
|
}
|
|
26
25
|
|
|
27
|
-
export interface BrowseSubmitCaptureSyncResult {
|
|
28
|
-
indexed: boolean;
|
|
29
|
-
mode: "http" | "dom" | "none";
|
|
30
|
-
skill_id?: string | null;
|
|
31
|
-
endpoint_count: number;
|
|
32
|
-
request_count?: number;
|
|
33
|
-
background_publish_queued?: boolean;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
26
|
export interface BrowseSubmitResult {
|
|
37
27
|
ok: boolean;
|
|
38
28
|
url: string;
|
|
@@ -44,12 +34,12 @@ export interface BrowseSubmitResult {
|
|
|
44
34
|
status?: number;
|
|
45
35
|
wait_for?: string;
|
|
46
36
|
submit_meta?: Record<string, unknown> | null;
|
|
47
|
-
capture_sync?: BrowseSubmitCaptureSyncResult | null;
|
|
48
37
|
rehydrate?: unknown;
|
|
49
38
|
}
|
|
50
39
|
|
|
51
40
|
const DEFAULT_SUBMIT_TIMEOUT_MS = 8_000;
|
|
52
41
|
const SUBMIT_POLL_INTERVAL_MS = 250;
|
|
42
|
+
const SUBMIT_SETTLE_WINDOW_MS = 1_000;
|
|
53
43
|
|
|
54
44
|
function sleep(ms: number): Promise<void> {
|
|
55
45
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
@@ -300,6 +290,27 @@ function parseJsonString(value: unknown): Record<string, unknown> | null {
|
|
|
300
290
|
}
|
|
301
291
|
}
|
|
302
292
|
|
|
293
|
+
async function settleSubmitDestination(
|
|
294
|
+
client: BrowseSubmitClient,
|
|
295
|
+
tabId: string,
|
|
296
|
+
url: string,
|
|
297
|
+
html: string,
|
|
298
|
+
): Promise<{ url: string; html: string }> {
|
|
299
|
+
let settledUrl = url;
|
|
300
|
+
let settledHtml = html;
|
|
301
|
+
const deadline = Date.now() + SUBMIT_SETTLE_WINDOW_MS;
|
|
302
|
+
|
|
303
|
+
while (Date.now() < deadline) {
|
|
304
|
+
await sleep(Math.min(SUBMIT_POLL_INTERVAL_MS, Math.max(50, deadline - Date.now())));
|
|
305
|
+
const nextUrl = await client.getCurrentUrl(tabId).catch(() => "");
|
|
306
|
+
const nextHtml = await client.getPageHtml(tabId).catch(() => "");
|
|
307
|
+
if (nextUrl && !nextUrl.startsWith("about:blank")) settledUrl = nextUrl;
|
|
308
|
+
if (nextHtml) settledHtml = nextHtml;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
return { url: settledUrl, html: settledHtml };
|
|
312
|
+
}
|
|
313
|
+
|
|
303
314
|
async function waitForSubmitOutcome(
|
|
304
315
|
client: BrowseSubmitClient,
|
|
305
316
|
tabId: string,
|
|
@@ -317,7 +328,7 @@ async function waitForSubmitOutcome(
|
|
|
317
328
|
if (waitResult?.status === "found" || waitResult?.status === "ready") {
|
|
318
329
|
const url = await client.getCurrentUrl(tabId).catch(() => beforeUrl);
|
|
319
330
|
const html = await client.getPageHtml(tabId).catch(() => beforeHtml);
|
|
320
|
-
return { ok: true, url, html };
|
|
331
|
+
return { ok: true, ...await settleSubmitDestination(client, tabId, url, html) };
|
|
321
332
|
}
|
|
322
333
|
} catch {
|
|
323
334
|
// fall through to polling
|
|
@@ -329,13 +340,13 @@ async function waitForSubmitOutcome(
|
|
|
329
340
|
const html = await client.getPageHtml(tabId).catch(() => "");
|
|
330
341
|
|
|
331
342
|
if (waitFor && isUrlWaitHint(waitFor) && url.includes(waitFor)) {
|
|
332
|
-
return { ok: true, url, html };
|
|
343
|
+
return { ok: true, ...await settleSubmitDestination(client, tabId, url, html) };
|
|
333
344
|
}
|
|
334
345
|
if (url && url !== beforeUrl && !url.startsWith("about:blank")) {
|
|
335
|
-
return { ok: true, url, html };
|
|
346
|
+
return { ok: true, ...await settleSubmitDestination(client, tabId, url, html) };
|
|
336
347
|
}
|
|
337
348
|
if (hasMeaningfulPageChange(beforeHtml, html)) {
|
|
338
|
-
return { ok: true,
|
|
349
|
+
return { ok: true, ...await settleSubmitDestination(client, tabId, url || beforeUrl, html) };
|
|
339
350
|
}
|
|
340
351
|
|
|
341
352
|
await sleep(SUBMIT_POLL_INTERVAL_MS);
|
|
@@ -348,7 +359,7 @@ export async function submitBrowseForm(
|
|
|
348
359
|
deps: BrowseSubmitDeps,
|
|
349
360
|
options: BrowseSubmitOptions = {},
|
|
350
361
|
): Promise<BrowseSubmitResult> {
|
|
351
|
-
const { client, session,
|
|
362
|
+
const { client, session, restartCapture, rehydratePlugins } = deps;
|
|
352
363
|
const sameOriginFetchFallback = options.sameOriginFetchFallback !== false;
|
|
353
364
|
const beforeUrl = await client.getCurrentUrl(session.tabId).catch(() => session.url);
|
|
354
365
|
const beforeHtml = await client.getPageHtml(session.tabId).catch(() => "");
|
|
@@ -377,14 +388,6 @@ export async function submitBrowseForm(
|
|
|
377
388
|
const domOutcome = await waitForSubmitOutcome(client, session.tabId, beforeUrl, beforeHtml, options);
|
|
378
389
|
if (domOutcome.ok) {
|
|
379
390
|
session.url = domOutcome.url || beforeUrl || session.url;
|
|
380
|
-
let captureSync: BrowseSubmitCaptureSyncResult | null = null;
|
|
381
|
-
if (flushCapture) {
|
|
382
|
-
try {
|
|
383
|
-
captureSync = await flushCapture(session);
|
|
384
|
-
} catch {
|
|
385
|
-
captureSync = null;
|
|
386
|
-
}
|
|
387
|
-
}
|
|
388
391
|
await restartCapture(session);
|
|
389
392
|
return {
|
|
390
393
|
ok: true,
|
|
@@ -394,7 +397,6 @@ export async function submitBrowseForm(
|
|
|
394
397
|
same_origin_html_rehydrated: false,
|
|
395
398
|
wait_for: options.waitFor,
|
|
396
399
|
submit_meta: submitMeta,
|
|
397
|
-
capture_sync: captureSync,
|
|
398
400
|
};
|
|
399
401
|
}
|
|
400
402
|
|
|
@@ -438,14 +440,6 @@ export async function submitBrowseForm(
|
|
|
438
440
|
rehydrate = await rehydratePlugins(session.tabId).catch(() => null);
|
|
439
441
|
}
|
|
440
442
|
|
|
441
|
-
let captureSync: BrowseSubmitCaptureSyncResult | null = null;
|
|
442
|
-
if (flushCapture) {
|
|
443
|
-
try {
|
|
444
|
-
captureSync = await flushCapture(session);
|
|
445
|
-
} catch {
|
|
446
|
-
captureSync = null;
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
443
|
await restartCapture(session);
|
|
450
444
|
return {
|
|
451
445
|
ok: true,
|
|
@@ -456,7 +450,6 @@ export async function submitBrowseForm(
|
|
|
456
450
|
status: typeof fallbackPayload.status === "number" ? fallbackPayload.status as number : undefined,
|
|
457
451
|
wait_for: options.waitFor,
|
|
458
452
|
submit_meta: submitMeta,
|
|
459
|
-
capture_sync: captureSync,
|
|
460
453
|
rehydrate,
|
|
461
454
|
};
|
|
462
455
|
}
|
|
@@ -19,12 +19,11 @@ import { getSkill } from "../marketplace/index.js";
|
|
|
19
19
|
import { executeSkill, rankEndpoints } from "../execution/index.js";
|
|
20
20
|
import { interactiveLogin, extractBrowserAuth } from "../auth/index.js";
|
|
21
21
|
import { publishSkill } from "../marketplace/index.js";
|
|
22
|
-
import { recordFeedback, recordDiagnostics, recordExecution, getApiKey, getRecentLocalSkill, recordAnalyticsSession, type AnalyticsSessionPayload } from "../client/index.js";
|
|
22
|
+
import { recordFeedback, recordDiagnostics, recordExecution, getApiKey, getRecentLocalSkill, recordAnalyticsSession, waitForBackgroundRegistration, type AnalyticsSessionPayload } from "../client/index.js";
|
|
23
23
|
import { ROUTE_LIMITS } from "../ratelimit/index.js";
|
|
24
24
|
import { getSkillChunk, toAgentSkillChunkView } from "../graph/index.js";
|
|
25
25
|
import { listRecentSessionsForDomain } from "../session-logs.js";
|
|
26
26
|
import { mergeAgentReview } from "../indexer/index.js";
|
|
27
|
-
import { attachAgentOutcomeHints } from "../agent-outcome.js";
|
|
28
27
|
import { writeFileSync, existsSync, mkdirSync } from "fs";
|
|
29
28
|
import { join } from "path";
|
|
30
29
|
import { type BrowseSession, getOrCreateBrowseSession, isRecoverableBrowseFailure, withRecoveredBrowseSession } from "./browse-session.js";
|
|
@@ -36,15 +35,15 @@ const BETA_API_URL = process.env.UNBROWSE_BACKEND_URL || "https://beta-api.unbro
|
|
|
36
35
|
const TRACES_DIR = process.env.TRACES_DIR ?? join(process.cwd(), "traces");
|
|
37
36
|
|
|
38
37
|
type AnalyticsSessionResult = {
|
|
39
|
-
trace: Pick<ExecutionTrace, "trace_id" | "started_at" | "completed_at" | "endpoint_id" | "trace_version"
|
|
40
|
-
timing?: Pick<OrchestrationTiming, "source"
|
|
38
|
+
trace: Pick<ExecutionTrace, "trace_id" | "started_at" | "completed_at" | "endpoint_id" | "trace_version">;
|
|
39
|
+
timing?: Pick<OrchestrationTiming, "source">;
|
|
41
40
|
source?: OrchestratorResult["source"];
|
|
42
41
|
};
|
|
43
42
|
|
|
44
43
|
export function buildAnalyticsSessionPayload(
|
|
45
44
|
result: AnalyticsSessionResult,
|
|
46
45
|
opts: {
|
|
47
|
-
browser_mode
|
|
46
|
+
browser_mode: AnalyticsSessionPayload["browser_mode"];
|
|
48
47
|
discovery_queries: number;
|
|
49
48
|
cached_skill_calls?: number;
|
|
50
49
|
fresh_index_calls?: number;
|
|
@@ -52,11 +51,6 @@ export function buildAnalyticsSessionPayload(
|
|
|
52
51
|
): AnalyticsSessionPayload {
|
|
53
52
|
const source = result.timing?.source ?? result.source;
|
|
54
53
|
const apiCalls = result.trace.endpoint_id ? 1 : 0;
|
|
55
|
-
const browserMode = opts.browser_mode ?? (
|
|
56
|
-
source === "live-capture" || source === "first-pass" || source === "browser-action"
|
|
57
|
-
? "default"
|
|
58
|
-
: "replaced"
|
|
59
|
-
);
|
|
60
54
|
const cachedSkillCalls = opts.cached_skill_calls ?? (
|
|
61
55
|
apiCalls > 0 && source !== "live-capture" && source !== "first-pass" ? 1 : 0
|
|
62
56
|
);
|
|
@@ -73,14 +67,7 @@ export function buildAnalyticsSessionPayload(
|
|
|
73
67
|
discovery_queries: opts.discovery_queries,
|
|
74
68
|
cached_skill_calls: cachedSkillCalls,
|
|
75
69
|
fresh_index_calls: freshIndexCalls,
|
|
76
|
-
browser_mode:
|
|
77
|
-
success: result.trace.success ?? true,
|
|
78
|
-
source,
|
|
79
|
-
time_saved_ms: result.timing?.time_saved_ms,
|
|
80
|
-
time_saved_pct: result.timing?.time_saved_pct,
|
|
81
|
-
tokens_saved: result.trace.tokens_saved ?? result.timing?.tokens_saved,
|
|
82
|
-
tokens_saved_pct: result.trace.tokens_saved_pct ?? result.timing?.tokens_saved_pct,
|
|
83
|
-
cost_saved_uc: result.timing?.cost_saved_uc,
|
|
70
|
+
browser_mode: opts.browser_mode ?? "unknown",
|
|
84
71
|
};
|
|
85
72
|
}
|
|
86
73
|
|
|
@@ -328,7 +315,11 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
328
315
|
app.addHook("onRequest", async (req, reply) => {
|
|
329
316
|
if (req.url === "/health" || req.url === "/v1/stats") return;
|
|
330
317
|
|
|
331
|
-
|
|
318
|
+
let key = getApiKey();
|
|
319
|
+
if (!key) {
|
|
320
|
+
await waitForBackgroundRegistration(15_000);
|
|
321
|
+
key = getApiKey();
|
|
322
|
+
}
|
|
332
323
|
if (!key) {
|
|
333
324
|
return reply.code(401).send({
|
|
334
325
|
error: "api_key_required",
|
|
@@ -355,11 +346,7 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
355
346
|
const result = await resolveAndExecute(intent, params ?? {}, context, projection, { confirm_unsafe, dry_run, force_capture, client_scope: clientScope });
|
|
356
347
|
|
|
357
348
|
// Surface timing breakdown
|
|
358
|
-
const res =
|
|
359
|
-
skill: result.skill,
|
|
360
|
-
endpointId: result.trace.endpoint_id,
|
|
361
|
-
timing: result.timing,
|
|
362
|
-
});
|
|
349
|
+
const res = result as unknown as Record<string, unknown>;
|
|
363
350
|
if (result.timing) {
|
|
364
351
|
res.timing = result.timing;
|
|
365
352
|
}
|
|
@@ -372,10 +359,11 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
372
359
|
}
|
|
373
360
|
|
|
374
361
|
await recordAnalyticsSession(buildAnalyticsSessionPayload(result, {
|
|
362
|
+
browser_mode: "replaced",
|
|
375
363
|
discovery_queries: 1,
|
|
376
364
|
})).catch(() => {});
|
|
377
365
|
|
|
378
|
-
return reply.send(
|
|
366
|
+
return reply.send(result);
|
|
379
367
|
} catch (err) {
|
|
380
368
|
return reply.code(500).send({ error: (err as Error).message });
|
|
381
369
|
}
|
|
@@ -646,22 +634,16 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
646
634
|
recordExecution(freshResult.trace.skill_id, freshResult.trace.endpoint_id, freshResult.trace, skill).catch(() => {});
|
|
647
635
|
}
|
|
648
636
|
await recordAnalyticsSession(buildAnalyticsSessionPayload(freshResult, {
|
|
637
|
+
browser_mode: "manual",
|
|
649
638
|
discovery_queries: 1,
|
|
650
639
|
})).catch(() => {});
|
|
651
|
-
|
|
640
|
+
return reply.send({
|
|
652
641
|
...freshResult,
|
|
653
642
|
_recovery: {
|
|
654
643
|
reason: "stale_endpoint_404",
|
|
655
644
|
original_skill_id: skill_id,
|
|
656
645
|
message: "Original endpoint returned 404. Auto-recovered with fresh capture.",
|
|
657
646
|
},
|
|
658
|
-
} as Record<string, unknown>, {
|
|
659
|
-
skill: freshResult.skill ?? skill,
|
|
660
|
-
endpointId: freshResult.trace.endpoint_id,
|
|
661
|
-
timing: freshResult.timing,
|
|
662
|
-
});
|
|
663
|
-
return reply.send({
|
|
664
|
-
...recovered,
|
|
665
647
|
});
|
|
666
648
|
} catch {
|
|
667
649
|
// Recovery failed — return original 404 with guidance
|
|
@@ -669,14 +651,13 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
669
651
|
}
|
|
670
652
|
|
|
671
653
|
await recordAnalyticsSession(buildAnalyticsSessionPayload(execResult, {
|
|
654
|
+
browser_mode: "manual",
|
|
672
655
|
discovery_queries: 0,
|
|
656
|
+
cached_skill_calls: execResult.trace.endpoint_id ? 1 : 0,
|
|
657
|
+
fresh_index_calls: 0,
|
|
673
658
|
})).catch(() => {});
|
|
674
659
|
|
|
675
|
-
|
|
676
|
-
skill,
|
|
677
|
-
endpointId: execResult.trace.endpoint_id,
|
|
678
|
-
});
|
|
679
|
-
return reply.send(response);
|
|
660
|
+
return reply.send(execResult);
|
|
680
661
|
} catch (err) {
|
|
681
662
|
return reply.code(500).send({ error: (err as Error).message });
|
|
682
663
|
}
|
|
@@ -880,96 +861,6 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
880
861
|
await injectInterceptor(session.tabId).catch(() => {});
|
|
881
862
|
}
|
|
882
863
|
|
|
883
|
-
async function flushBrowseCapture(
|
|
884
|
-
session: BrowseSession,
|
|
885
|
-
options: { queueBackgroundPublish?: boolean } = {},
|
|
886
|
-
): Promise<{
|
|
887
|
-
indexed: boolean;
|
|
888
|
-
mode: "http" | "dom" | "none";
|
|
889
|
-
domain: string;
|
|
890
|
-
skill_id: string | null;
|
|
891
|
-
endpoint_count: number;
|
|
892
|
-
endpoints: Array<{
|
|
893
|
-
endpoint_id: string;
|
|
894
|
-
method: string;
|
|
895
|
-
url_template: string;
|
|
896
|
-
description?: string;
|
|
897
|
-
trigger_url?: string;
|
|
898
|
-
action_kind?: string;
|
|
899
|
-
resource_kind?: string;
|
|
900
|
-
}>;
|
|
901
|
-
request_count: number;
|
|
902
|
-
background_publish_queued: boolean;
|
|
903
|
-
}> {
|
|
904
|
-
let intercepted: RawRequest[] = [];
|
|
905
|
-
try {
|
|
906
|
-
const raw = await collectInterceptedRequests(session.tabId);
|
|
907
|
-
intercepted = raw.map((request) => ({
|
|
908
|
-
url: request.url,
|
|
909
|
-
method: request.method,
|
|
910
|
-
request_headers: request.request_headers ?? {},
|
|
911
|
-
request_body: request.request_body,
|
|
912
|
-
response_status: request.response_status,
|
|
913
|
-
response_headers: request.response_headers ?? {},
|
|
914
|
-
response_body: request.response_body,
|
|
915
|
-
timestamp: request.timestamp,
|
|
916
|
-
}));
|
|
917
|
-
} catch { /* non-fatal */ }
|
|
918
|
-
|
|
919
|
-
let harEntries: KuriHarEntry[] = [];
|
|
920
|
-
if (session.harActive) {
|
|
921
|
-
try {
|
|
922
|
-
const { entries } = await kuri.harStop(session.tabId);
|
|
923
|
-
harEntries = entries;
|
|
924
|
-
} catch { /* non-fatal */ }
|
|
925
|
-
}
|
|
926
|
-
session.harActive = false;
|
|
927
|
-
|
|
928
|
-
const allRequests = mergeBrowseRequests(intercepted, harEntries, session.url);
|
|
929
|
-
const syncResult = await cacheBrowseRequests({
|
|
930
|
-
sessionUrl: session.url,
|
|
931
|
-
sessionDomain: session.domain,
|
|
932
|
-
requests: allRequests,
|
|
933
|
-
getPageHtml: () => kuri.getPageHtml(session.tabId),
|
|
934
|
-
});
|
|
935
|
-
|
|
936
|
-
let backgroundPublishQueued = false;
|
|
937
|
-
if (options.queueBackgroundPublish) {
|
|
938
|
-
if (allRequests.length > 0) {
|
|
939
|
-
passiveIndexFromRequests(allRequests, session.url);
|
|
940
|
-
backgroundPublishQueued = true;
|
|
941
|
-
} else if (syncResult.skill) {
|
|
942
|
-
queueBackgroundIndex({
|
|
943
|
-
skill: { ...syncResult.skill },
|
|
944
|
-
domain: syncResult.domain,
|
|
945
|
-
intent: syncResult.skill.intent_signature || `browse ${syncResult.domain}`,
|
|
946
|
-
contextUrl: session.url,
|
|
947
|
-
cacheKey: `browse-submit:${syncResult.domain}:${Date.now()}`,
|
|
948
|
-
});
|
|
949
|
-
backgroundPublishQueued = true;
|
|
950
|
-
}
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
return {
|
|
954
|
-
indexed: syncResult.indexed,
|
|
955
|
-
mode: syncResult.mode,
|
|
956
|
-
domain: syncResult.domain,
|
|
957
|
-
skill_id: syncResult.skill?.skill_id ?? null,
|
|
958
|
-
endpoint_count: syncResult.skill?.endpoints.length ?? 0,
|
|
959
|
-
endpoints: (syncResult.skill?.endpoints ?? []).map((endpoint) => ({
|
|
960
|
-
endpoint_id: endpoint.endpoint_id,
|
|
961
|
-
method: endpoint.method,
|
|
962
|
-
url_template: endpoint.url_template,
|
|
963
|
-
description: endpoint.description,
|
|
964
|
-
trigger_url: endpoint.trigger_url,
|
|
965
|
-
action_kind: endpoint.semantic?.action_kind,
|
|
966
|
-
resource_kind: endpoint.semantic?.resource_kind,
|
|
967
|
-
})),
|
|
968
|
-
request_count: allRequests.length,
|
|
969
|
-
background_publish_queued: backgroundPublishQueued,
|
|
970
|
-
};
|
|
971
|
-
}
|
|
972
|
-
|
|
973
864
|
// POST /v1/browse/go — navigate to URL
|
|
974
865
|
app.post("/v1/browse/go", async (req, reply) => {
|
|
975
866
|
const { url } = req.body as { url: string };
|
|
@@ -1060,7 +951,6 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
1060
951
|
{
|
|
1061
952
|
client: kuri,
|
|
1062
953
|
session,
|
|
1063
|
-
flushCapture: async (session) => await flushBrowseCapture(session, { queueBackgroundPublish: true }),
|
|
1064
954
|
restartCapture: restartBrowseCapture,
|
|
1065
955
|
rehydratePlugins: kuri.bestEffortRehydratePlugins,
|
|
1066
956
|
},
|
|
@@ -1079,14 +969,8 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
1079
969
|
session.domain = profileName(session.url);
|
|
1080
970
|
|
|
1081
971
|
const statusCode = result.ok ? 200 : (result.recoverable ? 502 : 400);
|
|
1082
|
-
const nextStep = result.ok
|
|
1083
|
-
? (result.capture_sync?.background_publish_queued
|
|
1084
|
-
? "Background publish queued for this step. Continue the flow, then run `unbrowse close` when you're done to save auth and finalize any remaining capture."
|
|
1085
|
-
: "If more UI steps remain, continue the flow. Run `unbrowse close` when you're done to save auth and finalize capture.")
|
|
1086
|
-
: "Inspect the page state with `unbrowse snap --filter interactive`, then retry submit with selectors or a wait hint if needed.";
|
|
1087
972
|
return reply.code(statusCode).send({
|
|
1088
973
|
...result,
|
|
1089
|
-
next_step: nextStep,
|
|
1090
974
|
recovered,
|
|
1091
975
|
tab_id: session.tabId,
|
|
1092
976
|
url: session.url,
|
|
@@ -1254,9 +1138,44 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
1254
1138
|
app.post("/v1/browse/sync", async (_req, reply) => {
|
|
1255
1139
|
const session = browseSessions.get("default");
|
|
1256
1140
|
if (!session) return reply.send({ ok: false, error: "no active session" });
|
|
1257
|
-
const syncResult = await flushBrowseCapture(session);
|
|
1258
1141
|
|
|
1259
|
-
|
|
1142
|
+
let intercepted: RawRequest[] = [];
|
|
1143
|
+
try {
|
|
1144
|
+
const raw = await collectInterceptedRequests(session.tabId);
|
|
1145
|
+
intercepted = raw.map((request) => ({
|
|
1146
|
+
url: request.url,
|
|
1147
|
+
method: request.method,
|
|
1148
|
+
request_headers: request.request_headers ?? {},
|
|
1149
|
+
request_body: request.request_body,
|
|
1150
|
+
response_status: request.response_status,
|
|
1151
|
+
response_headers: request.response_headers ?? {},
|
|
1152
|
+
response_body: request.response_body,
|
|
1153
|
+
timestamp: request.timestamp,
|
|
1154
|
+
}));
|
|
1155
|
+
} catch { /* non-fatal */ }
|
|
1156
|
+
|
|
1157
|
+
let harEntries: KuriHarEntry[] = [];
|
|
1158
|
+
if (session.harActive) {
|
|
1159
|
+
try {
|
|
1160
|
+
const { entries } = await kuri.harStop(session.tabId);
|
|
1161
|
+
harEntries = entries;
|
|
1162
|
+
} catch { /* non-fatal */ }
|
|
1163
|
+
}
|
|
1164
|
+
session.harActive = false;
|
|
1165
|
+
|
|
1166
|
+
const allRequests = mergeBrowseRequests(intercepted, harEntries, session.url);
|
|
1167
|
+
const syncResult = await cacheBrowseRequests({
|
|
1168
|
+
sessionUrl: session.url,
|
|
1169
|
+
sessionDomain: session.domain,
|
|
1170
|
+
requests: allRequests,
|
|
1171
|
+
getPageHtml: () => kuri.getPageHtml(session.tabId),
|
|
1172
|
+
});
|
|
1173
|
+
|
|
1174
|
+
await kuri.networkEnable(session.tabId).catch(() => {});
|
|
1175
|
+
await kuri.harStart(session.tabId).catch(() => {});
|
|
1176
|
+
await kuri.scriptInject(session.tabId, INTERCEPTOR_SCRIPT).catch(() => {});
|
|
1177
|
+
session.harActive = true;
|
|
1178
|
+
await injectInterceptor(session.tabId).catch(() => {});
|
|
1260
1179
|
|
|
1261
1180
|
return reply.send({
|
|
1262
1181
|
ok: true,
|
|
@@ -1264,10 +1183,17 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
1264
1183
|
indexed: syncResult.indexed,
|
|
1265
1184
|
mode: syncResult.mode,
|
|
1266
1185
|
domain: syncResult.domain,
|
|
1267
|
-
skill_id: syncResult.skill_id,
|
|
1268
|
-
endpoint_count: syncResult.
|
|
1269
|
-
endpoints: syncResult.endpoints
|
|
1270
|
-
|
|
1186
|
+
skill_id: syncResult.skill?.skill_id ?? null,
|
|
1187
|
+
endpoint_count: syncResult.skill?.endpoints.length ?? 0,
|
|
1188
|
+
endpoints: (syncResult.skill?.endpoints ?? []).map((endpoint) => ({
|
|
1189
|
+
endpoint_id: endpoint.endpoint_id,
|
|
1190
|
+
method: endpoint.method,
|
|
1191
|
+
url_template: endpoint.url_template,
|
|
1192
|
+
description: endpoint.description,
|
|
1193
|
+
trigger_url: endpoint.trigger_url,
|
|
1194
|
+
action_kind: endpoint.semantic?.action_kind,
|
|
1195
|
+
resource_kind: endpoint.semantic?.resource_kind,
|
|
1196
|
+
})),
|
|
1271
1197
|
});
|
|
1272
1198
|
});
|
|
1273
1199
|
|
|
@@ -1281,16 +1207,48 @@ export async function registerRoutes(app: FastifyInstance) {
|
|
|
1281
1207
|
await kuri.authProfileSave(session.tabId, session.domain).catch(() => {});
|
|
1282
1208
|
}
|
|
1283
1209
|
|
|
1284
|
-
|
|
1210
|
+
// Collect intercepted fetch/XHR requests (has response bodies HAR misses)
|
|
1211
|
+
let intercepted: RawRequest[] = [];
|
|
1212
|
+
try {
|
|
1213
|
+
const raw = await collectInterceptedRequests(session.tabId);
|
|
1214
|
+
intercepted = raw.map(r => ({
|
|
1215
|
+
url: r.url,
|
|
1216
|
+
method: r.method,
|
|
1217
|
+
request_headers: r.request_headers ?? {},
|
|
1218
|
+
request_body: r.request_body,
|
|
1219
|
+
response_status: r.response_status,
|
|
1220
|
+
response_headers: r.response_headers ?? {},
|
|
1221
|
+
response_body: r.response_body,
|
|
1222
|
+
timestamp: r.timestamp,
|
|
1223
|
+
}));
|
|
1224
|
+
} catch { /* non-fatal */ }
|
|
1225
|
+
|
|
1226
|
+
// Also collect HAR entries
|
|
1227
|
+
let harEntries: KuriHarEntry[] = [];
|
|
1228
|
+
if (session.harActive) {
|
|
1229
|
+
try {
|
|
1230
|
+
const { entries } = await kuri.harStop(session.tabId);
|
|
1231
|
+
harEntries = entries;
|
|
1232
|
+
} catch { /* non-fatal */ }
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
const allRequests = mergeBrowseRequests(intercepted, harEntries, session.url);
|
|
1236
|
+
const syncResult = await cacheBrowseRequests({
|
|
1237
|
+
sessionUrl: session.url,
|
|
1238
|
+
sessionDomain: session.domain,
|
|
1239
|
+
requests: allRequests,
|
|
1240
|
+
getPageHtml: () => kuri.getPageHtml(session.tabId),
|
|
1241
|
+
});
|
|
1242
|
+
|
|
1243
|
+
// Run full async enrichment pipeline (agent augmentation, graph, marketplace publish)
|
|
1244
|
+
passiveIndexFromRequests(allRequests, session.url);
|
|
1285
1245
|
await kuri.closeTab(session.tabId).catch(() => {});
|
|
1286
1246
|
browseSessions.delete("default");
|
|
1287
1247
|
return reply.send({
|
|
1288
1248
|
ok: true,
|
|
1289
1249
|
indexed: syncResult.indexed,
|
|
1290
1250
|
mode: syncResult.mode,
|
|
1291
|
-
endpoint_count: syncResult.
|
|
1292
|
-
request_count: syncResult.request_count,
|
|
1293
|
-
background_publish_queued: syncResult.background_publish_queued,
|
|
1251
|
+
endpoint_count: syncResult.skill?.endpoints.length ?? 0,
|
|
1294
1252
|
auth_saved: session.domain || null,
|
|
1295
1253
|
});
|
|
1296
1254
|
});
|