unbrowse 3.1.0 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/cli.js +455 -96
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +695 -46
  4. package/dist/server.js +25811 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/darwin-arm64/kuri +0 -0
  7. package/vendor/kuri/darwin-x64/kuri +0 -0
  8. package/vendor/kuri/linux-arm64/kuri +0 -0
  9. package/vendor/kuri/linux-x64/kuri +0 -0
  10. package/vendor/kuri/manifest.json +7 -10
  11. package/runtime-src/agent-outcome.ts +0 -166
  12. package/runtime-src/analytics-session.ts +0 -55
  13. package/runtime-src/api/browse-index.ts +0 -317
  14. package/runtime-src/api/browse-session.ts +0 -572
  15. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  16. package/runtime-src/api/browse-submit.ts +0 -1184
  17. package/runtime-src/api/routes.ts +0 -1823
  18. package/runtime-src/auth/browser-cookies.ts +0 -423
  19. package/runtime-src/auth/index.ts +0 -535
  20. package/runtime-src/auth/runtime.ts +0 -116
  21. package/runtime-src/browser/index.ts +0 -659
  22. package/runtime-src/browser/types.ts +0 -41
  23. package/runtime-src/build-info.generated.ts +0 -6
  24. package/runtime-src/capture/index.ts +0 -1794
  25. package/runtime-src/capture/prefetch.ts +0 -95
  26. package/runtime-src/capture/rsc.ts +0 -45
  27. package/runtime-src/cli/shortcuts.ts +0 -273
  28. package/runtime-src/cli.ts +0 -1572
  29. package/runtime-src/client/graph-client.ts +0 -100
  30. package/runtime-src/client/index.ts +0 -1425
  31. package/runtime-src/debug-trace.ts +0 -18
  32. package/runtime-src/domain.ts +0 -38
  33. package/runtime-src/execution/index.ts +0 -3397
  34. package/runtime-src/execution/retry.ts +0 -46
  35. package/runtime-src/execution/robots.ts +0 -167
  36. package/runtime-src/execution/search-forms.ts +0 -188
  37. package/runtime-src/extraction/index.ts +0 -1507
  38. package/runtime-src/foundry/publish-bundle.ts +0 -392
  39. package/runtime-src/graph/agent-augment.ts +0 -315
  40. package/runtime-src/graph/index.ts +0 -1524
  41. package/runtime-src/graph/local-fixtures.ts +0 -393
  42. package/runtime-src/graph/local-harness.ts +0 -646
  43. package/runtime-src/graph/planner.ts +0 -411
  44. package/runtime-src/graph/session.ts +0 -294
  45. package/runtime-src/graph/trace-store.ts +0 -136
  46. package/runtime-src/index.ts +0 -24
  47. package/runtime-src/indexer/index.ts +0 -465
  48. package/runtime-src/intent-match.ts +0 -1515
  49. package/runtime-src/kuri/client.ts +0 -1839
  50. package/runtime-src/logger.ts +0 -30
  51. package/runtime-src/marketplace/index.ts +0 -103
  52. package/runtime-src/mcp.ts +0 -1747
  53. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  54. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  55. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  56. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  57. package/runtime-src/orchestrator/index.ts +0 -4480
  58. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  59. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  60. package/runtime-src/payments/cascade.ts +0 -137
  61. package/runtime-src/payments/index.ts +0 -270
  62. package/runtime-src/payments/lobster-pay.ts +0 -182
  63. package/runtime-src/payments/wallet.ts +0 -98
  64. package/runtime-src/publish/review-context.ts +0 -93
  65. package/runtime-src/publish/sanitize.ts +0 -197
  66. package/runtime-src/publish/schema-review.ts +0 -192
  67. package/runtime-src/publish-admission.ts +0 -388
  68. package/runtime-src/ratelimit/index.ts +0 -23
  69. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  70. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  71. package/runtime-src/reverse-engineer/index.ts +0 -1551
  72. package/runtime-src/router.ts +0 -17
  73. package/runtime-src/routing-telemetry.ts +0 -395
  74. package/runtime-src/runtime/browser-access.ts +0 -11
  75. package/runtime-src/runtime/browser-auth.ts +0 -12
  76. package/runtime-src/runtime/browser-host.ts +0 -48
  77. package/runtime-src/runtime/lifecycle.ts +0 -17
  78. package/runtime-src/runtime/local-server.ts +0 -311
  79. package/runtime-src/runtime/paths.ts +0 -99
  80. package/runtime-src/runtime/setup.ts +0 -251
  81. package/runtime-src/runtime/supervisor.ts +0 -69
  82. package/runtime-src/runtime/update-hints.ts +0 -351
  83. package/runtime-src/server.ts +0 -100
  84. package/runtime-src/session-logs.ts +0 -142
  85. package/runtime-src/settings.ts +0 -221
  86. package/runtime-src/single-binary.ts +0 -143
  87. package/runtime-src/site-policy.ts +0 -54
  88. package/runtime-src/stale-cleanup-runner.ts +0 -144
  89. package/runtime-src/stale-cleanup.ts +0 -133
  90. package/runtime-src/telemetry-attribution.ts +0 -120
  91. package/runtime-src/telemetry.ts +0 -253
  92. package/runtime-src/template-params.ts +0 -141
  93. package/runtime-src/transform/drift.ts +0 -60
  94. package/runtime-src/transform/index.ts +0 -277
  95. package/runtime-src/types/index.ts +0 -1
  96. package/runtime-src/types/skill.ts +0 -912
  97. package/runtime-src/vault/index.ts +0 -196
  98. package/runtime-src/verification/auth-gate.ts +0 -8
  99. package/runtime-src/verification/candidates.ts +0 -27
  100. package/runtime-src/verification/index.ts +0 -120
  101. package/runtime-src/verification/matrix.ts +0 -30
  102. package/runtime-src/version.ts +0 -148
  103. package/runtime-src/workflow/artifact.ts +0 -161
  104. package/runtime-src/workflow/compile.ts +0 -808
  105. package/runtime-src/workflow/publish.ts +0 -225
  106. package/runtime-src/workflow/runtime.ts +0 -213
  107. package/vendor/kuri/win-x64/kuri.exe +0 -0
@@ -1,1823 +0,0 @@
1
- import type { FastifyInstance } from "fastify";
2
- import * as kuri from "../kuri/client.js";
3
- import type { KuriHarEntry } from "../kuri/client.js";
4
- import { extractEndpoints, extractAuthHeaders } from "../reverse-engineer/index.js";
5
- import { INTERCEPTOR_SCRIPT, enrichPassiveCaptureRequests, injectInterceptor } from "../capture/index.js";
6
- import { indexSkillLocally, mergeAgentReview, publishIndexedSkill, queueBackgroundIndex } from "../indexer/index.js";
7
- import { nanoid } from "nanoid";
8
- import type { ExecutionTrace, OrchestrationTiming, ProjectionOptions, SkillManifest } from "../types/index.js";
9
- import { mergeEndpoints } from "../marketplace/index.js";
10
- import { buildSkillOperationGraph, getEndpointDescriptionMetadata, getSkillChunk, toAgentSkillChunkView } from "../graph/index.js";
11
- import { augmentEndpointsWithAgent } from "../graph/agent-augment.js";
12
- import { findExistingSkillForDomain, cachePublishedSkill } from "../client/index.js";
13
- import { storeCredential } from "../vault/index.js";
14
- import { generateLocalDescription, writeSkillSnapshot, buildResolveCacheKey, getDomainReuseKey, domainSkillCache, persistDomainCache, scopedCacheKey, snapshotPathForCacheKey, invalidateRouteCacheForDomain, summarizeSchema, extractSampleValues } from "../orchestrator/index.js";
15
- import { TRACE_VERSION, CODE_HASH, DEFAULT_BACKEND_URL, GIT_SHA, PACKAGE_VERSION } from "../version.js";
16
- import { promoteExplicitExecution, resolveAndExecute, type OrchestratorResult } from "../orchestrator/index.js";
17
- import { getSkill } from "../marketplace/index.js";
18
- import { executeSkill, rankEndpoints } from "../execution/index.js";
19
- import {
20
- extractBrowserAuth,
21
- importBrowserCookiesIntoTab,
22
- loginWithBrowserFallback,
23
- loadAuthProfileBestEffort,
24
- saveAuthProfileBestEffort,
25
- } from "../auth/index.js";
26
- import { recordFeedback, recordDiagnostics, recordExecution, getApiKey, getRecentLocalSkill, recordAnalyticsSession, type AnalyticsSessionPayload } from "../client/index.js";
27
- import { ROUTE_LIMITS } from "../ratelimit/index.js";
28
- import { listRecentSessionsForDomain } from "../session-logs.js";
29
- import { attachAgentOutcomeHints } from "../agent-outcome.js";
30
- import { writeFileSync, existsSync, mkdirSync } from "fs";
31
- import { join } from "path";
32
- import {
33
- BrowseSessionError,
34
- createRegisteredBrowseSession,
35
- extractBrowseFailureMessage,
36
- getOrCreateNavigateBrowseSession,
37
- isBrowseSessionLive,
38
- isRecoverableBrowseFailure,
39
- type BrowseSession,
40
- withSerializedStrictBrowseSession,
41
- removeBrowseSession,
42
- } from "./browse-session.js";
43
- import { cacheBrowseRequests, harEntriesToRawRequests } from "./browse-index.js";
44
- import { isUrlWaitHint, resolveSubmitWaitHint, submitBrowseForm } from "./browse-submit.js";
45
- import { cleanupStaleSkills } from "../stale-cleanup-runner.js";
46
- import {
47
- decideCheckpointPublish,
48
- decideExplicitPublish,
49
- getCapturePipelineSettings,
50
- updateCapturePipelineSettings,
51
- } from "../settings.js";
52
- import { publishFoundryBundle } from "../foundry/publish-bundle.js";
53
- import { buildEndpointReviewContext } from "../publish/review-context.js";
54
- import { applyWorkflowSchemaReviews } from "../publish/schema-review.js";
55
- import { readWorkflowArtifact, writeWorkflowArtifact } from "../workflow/artifact.js";
56
-
57
- const BETA_API_URL = process.env.UNBROWSE_BACKEND_URL || DEFAULT_BACKEND_URL;
58
-
59
- const TRACES_DIR = process.env.TRACES_DIR ?? join(process.cwd(), "traces");
60
- const BROWSE_BROKER_MAX = Math.max(1, Number(process.env.KURI_MULTI_BROKER_MAX ?? "2"));
61
- const BROWSE_BROKER_BASE_PORT = Number(process.env.KURI_PORT ?? "7700");
62
-
63
- type AnalyticsSessionResult = {
64
- trace: Pick<ExecutionTrace, "trace_id" | "started_at" | "completed_at" | "endpoint_id" | "trace_version" | "success" | "tokens_saved" | "tokens_saved_pct" | "api_call_count">;
65
- timing?: Pick<OrchestrationTiming, "source" | "time_saved_ms" | "time_saved_pct" | "cost_saved_uc" | "tokens_saved" | "tokens_saved_pct">;
66
- source?: OrchestratorResult["source"];
67
- };
68
-
69
- export function buildAnalyticsSessionPayload(
70
- result: AnalyticsSessionResult,
71
- opts: {
72
- browser_mode?: AnalyticsSessionPayload["browser_mode"];
73
- discovery_queries: number;
74
- cached_skill_calls?: number;
75
- fresh_index_calls?: number;
76
- },
77
- ): AnalyticsSessionPayload {
78
- const source = result.timing?.source ?? result.source;
79
- const apiCalls = result.trace.api_call_count ?? (result.trace.endpoint_id ? 1 : 0);
80
- const browserMode = opts.browser_mode ?? (
81
- source === "live-capture" || source === "first-pass" || source === "browser-action"
82
- ? "default"
83
- : "replaced"
84
- );
85
- const cachedSkillCalls = opts.cached_skill_calls ?? (
86
- apiCalls > 0 && source !== "live-capture" && source !== "first-pass" ? 1 : 0
87
- );
88
- const freshIndexCalls = opts.fresh_index_calls ?? (
89
- apiCalls > 0 && (source === "live-capture" || source === "first-pass") ? 1 : 0
90
- );
91
-
92
- return {
93
- session_id: result.trace.trace_id,
94
- started_at: result.trace.started_at,
95
- completed_at: result.trace.completed_at,
96
- trace_version: result.trace.trace_version ?? TRACE_VERSION,
97
- api_calls: apiCalls,
98
- discovery_queries: opts.discovery_queries,
99
- cached_skill_calls: cachedSkillCalls,
100
- fresh_index_calls: freshIndexCalls,
101
- browser_mode: browserMode,
102
- success: result.trace.success ?? true,
103
- source,
104
- time_saved_ms: result.timing?.time_saved_ms,
105
- time_saved_pct: result.timing?.time_saved_pct,
106
- tokens_saved: result.trace.tokens_saved ?? result.timing?.tokens_saved,
107
- tokens_saved_pct: result.trace.tokens_saved_pct ?? result.timing?.tokens_saved_pct,
108
- cost_saved_uc: result.timing?.cost_saved_uc,
109
- };
110
- }
111
-
112
-
113
- /** Process HAR entries into routes and queue local index, with remote share opt-in only. */
114
- function passiveIndexFromRequests(
115
- requests: RawRequest[],
116
- pageUrl: string,
117
- options: { publishAfterIndex?: boolean } = {},
118
- ): void {
119
- if (requests.length === 0) return;
120
-
121
- let domain: string;
122
- try { domain = new URL(pageUrl).hostname; } catch { return; }
123
- const intent = `browse ${domain}`;
124
-
125
- // Fire-and-forget — full pipeline runs async
126
- void (async () => {
127
- try {
128
- // 1. Extract endpoints from captured traffic
129
- const rawEndpoints = extractEndpoints(requests, undefined, { pageUrl, finalUrl: pageUrl });
130
- if (rawEndpoints.length === 0) {
131
- console.error(`[passive-index] ${domain}: 0 endpoints from ${requests.length} requests`);
132
- return;
133
- }
134
-
135
- // 2. Extract and store auth credentials (cookies + sensitive headers)
136
- const capturedAuthHeaders = extractAuthHeaders(requests);
137
- if (Object.keys(capturedAuthHeaders).length > 0) {
138
- const authKey = `${domain}-session`;
139
- await storeCredential(authKey, JSON.stringify({ headers: capturedAuthHeaders }));
140
- }
141
-
142
- // 3. Merge with existing skill for this domain (never reduce endpoint count)
143
- const existingSkill = findExistingSkillForDomain(domain, intent);
144
- const mergedEndpoints = existingSkill
145
- ? mergeEndpoints(existingSkill.endpoints, rawEndpoints)
146
- : rawEndpoints;
147
- // Guard: if passive capture found fewer endpoints than what exists, keep the richer set
148
- if (existingSkill && mergedEndpoints.length < existingSkill.endpoints.length) {
149
- console.error(`[passive-index] ${domain}: skipping — would reduce ${existingSkill.endpoints.length} → ${mergedEndpoints.length} endpoints`);
150
- return;
151
- }
152
-
153
- // 4. Generate descriptions for endpoints without them (enables BM25 ranking)
154
- for (const ep of mergedEndpoints) {
155
- if (!ep.description) {
156
- ep.description = generateLocalDescription(ep);
157
- }
158
- }
159
-
160
- // 5. Skip LLM-based augmentation — the calling agent IS the LLM.
161
- // Endpoint descriptions come from generateLocalDescription (heuristic).
162
- // The agent reviews endpoints in the deferral response and picks the right one.
163
- const enrichedEndpoints = mergedEndpoints;
164
-
165
- // 6. Build operation dependency graph
166
- const operationGraph = buildSkillOperationGraph(enrichedEndpoints);
167
-
168
- // 7. Assemble full skill manifest
169
- const skill: SkillManifest = {
170
- skill_id: existingSkill?.skill_id ?? nanoid(),
171
- version: "1.0.0",
172
- schema_version: "1",
173
- lifecycle: "active" as const,
174
- execution_type: "http" as const,
175
- created_at: existingSkill?.created_at ?? new Date().toISOString(),
176
- updated_at: new Date().toISOString(),
177
- name: domain,
178
- intent_signature: intent,
179
- domain,
180
- description: `API skill for ${domain}`,
181
- owner_type: "agent" as const,
182
- endpoints: enrichedEndpoints,
183
- operation_graph: operationGraph,
184
- intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
185
- };
186
-
187
- // 8. Cache locally for immediate reuse — write to BOTH the published skill cache
188
- // AND the domain skill snapshot so resolveAndExecute finds it on next call
189
- try { cachePublishedSkill(skill); } catch { /* best-effort */ }
190
-
191
- // Write domain skill snapshot (keyed by resolve cache key)
192
- const bgCacheKey = buildResolveCacheKey(domain, intent, pageUrl);
193
- const bgScopedKey = scopedCacheKey("global", bgCacheKey);
194
- writeSkillSnapshot(bgScopedKey, skill);
195
-
196
- // Update domain-level reuse cache
197
- const bgDomainKey = getDomainReuseKey(pageUrl ?? domain);
198
- if (bgDomainKey) {
199
- domainSkillCache.set(bgDomainKey, {
200
- skillId: skill.skill_id,
201
- localSkillPath: snapshotPathForCacheKey(bgScopedKey),
202
- ts: Date.now(),
203
- });
204
- persistDomainCache();
205
- }
206
-
207
- // 9. Queue local index, and only remote-share when the caller explicitly asked for it.
208
- const cacheKey = `passive:${domain}:${Date.now()}`;
209
- queueBackgroundIndex({
210
- skill,
211
- domain,
212
- intent,
213
- contextUrl: pageUrl,
214
- cacheKey,
215
- publishAfterIndex: options.publishAfterIndex === true,
216
- });
217
-
218
- console.error(`[passive-index] ${domain}: ${enrichedEndpoints.length} endpoints indexed from ${requests.length} requests`);
219
- } catch (err) {
220
- console.error(`[passive-index] ${domain} failed: ${err instanceof Error ? err.message : err}`);
221
- }
222
- })();
223
- }
224
-
225
- /** Convenience wrapper: convert HAR entries and run passive indexing */
226
- function passiveIndexHar(
227
- entries: KuriHarEntry[],
228
- pageUrl: string,
229
- options: { publishAfterIndex?: boolean } = {},
230
- ): void {
231
- passiveIndexFromRequests(harEntriesToRawRequests(entries), pageUrl, options);
232
- }
233
- // ── Browse session state (module-level so orchestrator can register sessions) ──
234
- const browseSessions = new Map<string, BrowseSession>();
235
-
236
- function browseBrokerPorts(): number[] {
237
- return Array.from({ length: BROWSE_BROKER_MAX }, (_, index) => BROWSE_BROKER_BASE_PORT + index);
238
- }
239
-
240
- function brokerForSession(session: BrowseSession | undefined): kuri.KuriClient {
241
- if (session?.client) return session.client as kuri.KuriClient;
242
- if (session?.brokerPort !== undefined) return kuri.getKuriClient(session.brokerPort);
243
- return kuri.getKuriClient();
244
- }
245
-
246
- function selectBrowseBrokerClient(requestedSessionId?: string): kuri.KuriClient {
247
- if (requestedSessionId) {
248
- const existing = browseSessions.get(requestedSessionId);
249
- if (existing?.client) return existing.client as kuri.KuriClient;
250
- if (existing) return brokerForSession(existing);
251
- }
252
-
253
- const loads = new Map<number, number>(browseBrokerPorts().map((port) => [port, 0]));
254
- for (const session of browseSessions.values()) {
255
- const port = session.brokerPort ?? BROWSE_BROKER_BASE_PORT;
256
- loads.set(port, (loads.get(port) ?? 0) + 1);
257
- }
258
- const [selectedPort] = [...loads.entries()].sort((a, b) => a[1] - b[1] || a[0] - b[0])[0] ?? [BROWSE_BROKER_BASE_PORT, 0];
259
- return kuri.getKuriClient(selectedPort);
260
- }
261
-
262
- async function loadSkillForMutation(skillId: string, clientScope?: string): Promise<SkillManifest | null> {
263
- let skill = getRecentLocalSkill(skillId, clientScope);
264
- if (!skill) {
265
- for (const [, entry] of domainSkillCache) {
266
- if (entry.skillId === skillId && entry.localSkillPath) {
267
- try { skill = JSON.parse(require("fs").readFileSync(entry.localSkillPath, "utf-8")); } catch {}
268
- break;
269
- }
270
- }
271
- }
272
- if (!skill) skill = await getSkill(skillId, clientScope);
273
- return skill;
274
- }
275
-
276
- function buildSkillIndexJob(skill: SkillManifest, clientScope?: string): {
277
- skill: SkillManifest;
278
- domain: string;
279
- intent: string;
280
- clientScope?: string;
281
- cacheKey: string;
282
- } {
283
- const intent = skill.intent_signature || `browse ${skill.domain}`;
284
- return {
285
- skill,
286
- domain: skill.domain,
287
- intent,
288
- clientScope,
289
- cacheKey: buildResolveCacheKey(skill.domain, intent, undefined),
290
- };
291
- }
292
-
293
- /** Register a browse session from the orchestrator (Phase 4 handoff) */
294
- export function registerBrowseSession(tabId: string, url: string, domain: string): BrowseSession {
295
- const client = kuri.getKuriClient();
296
- return createRegisteredBrowseSession(browseSessions, {
297
- tabId,
298
- url,
299
- harActive: true,
300
- domain,
301
- brokerPort: client.getPort(),
302
- client,
303
- });
304
- }
305
-
306
- // ── /v1/stats cache ──────────────────────────────────────────────────
307
- let statsCache: { data: unknown; ts: number } | null = null;
308
- const STATS_CACHE_TTL = 5 * 60 * 1000; // 5 minutes
309
-
310
- async function fetchStats() {
311
- if (statsCache && Date.now() - statsCache.ts < STATS_CACHE_TTL) {
312
- return statsCache.data;
313
- }
314
-
315
- const npmPoint = (pkg: string, range: string) =>
316
- fetch(`https://api.npmjs.org/downloads/point/${range}/${pkg}`)
317
- .then(r => r.json() as Promise<{ downloads?: number }>);
318
-
319
- const npmRange = (pkg: string) =>
320
- fetch(`https://api.npmjs.org/downloads/range/last-month/${pkg}`)
321
- .then(r => r.json() as Promise<{ downloads?: Array<{ day: string; downloads: number }> }>);
322
-
323
- const externalCalls: Promise<unknown>[] = [
324
- npmPoint("unbrowse", "last-month"),
325
- npmPoint("unbrowse-openclaw", "last-month"),
326
- npmPoint("unbrowse", "1970-01-01:2099-12-31"),
327
- npmPoint("unbrowse-openclaw", "1970-01-01:2099-12-31"),
328
- npmRange("unbrowse"),
329
- npmRange("unbrowse-openclaw"),
330
- fetch("https://api.github.com/repos/anthropic-ai/unbrowse", {
331
- headers: { "User-Agent": "unbrowse-stats" },
332
- }).then(r => r.json() as Promise<Record<string, unknown>>),
333
- ];
334
-
335
- // Only call Unkey analytics if the key is available as an env var
336
- const unkeyAnalyticsKey = process.env.UNKEY_ANALYTICS_KEY;
337
- if (unkeyAnalyticsKey) {
338
- externalCalls.push(
339
- fetch("https://api.unkey.com/v2/analytics.getVerifications", {
340
- method: "POST",
341
- headers: {
342
- Authorization: `Bearer ${unkeyAnalyticsKey}`,
343
- "Content-Type": "application/json",
344
- },
345
- body: JSON.stringify({ apiId: "api_2bUScBc8U6JNsXLrhfHwfqzXHJDi" }),
346
- }).then(r => r.json() as Promise<unknown>),
347
- );
348
- }
349
-
350
- const [
351
- unbrowse30d, plugin30d,
352
- unbrowseAll, pluginAll,
353
- unbrowseDaily, pluginDaily,
354
- github,
355
- ...rest
356
- ] = await Promise.allSettled(externalCalls);
357
- const unkey = rest[0]; // may be undefined if no key
358
-
359
- const val = <T>(r: PromiseSettledResult<T> | undefined): T | null =>
360
- r?.status === "fulfilled" ? r.value : null;
361
-
362
- // npm numbers
363
- const u30 = val(unbrowse30d)?.downloads ?? null;
364
- const p30 = val(plugin30d)?.downloads ?? null;
365
- const uAll = val(unbrowseAll)?.downloads ?? null;
366
- const pAll = val(pluginAll)?.downloads ?? null;
367
-
368
- // daily breakdown — merge the two packages by day
369
- const uDays = val(unbrowseDaily)?.downloads ?? [];
370
- const pDays = val(pluginDaily)?.downloads ?? [];
371
- const dayMap = new Map<string, { unbrowse: number; plugin: number }>();
372
- for (const d of uDays) dayMap.set(d.day, { unbrowse: d.downloads, plugin: 0 });
373
- for (const d of pDays) {
374
- const entry = dayMap.get(d.day);
375
- if (entry) entry.plugin = d.downloads;
376
- else dayMap.set(d.day, { unbrowse: 0, plugin: d.downloads });
377
- }
378
- const daily = [...dayMap.entries()]
379
- .sort(([a], [b]) => a.localeCompare(b))
380
- .map(([day, v]) => ({ day, unbrowse: v.unbrowse, plugin: v.plugin, total: v.unbrowse + v.plugin }));
381
-
382
- // github
383
- const gh = val(github);
384
- const githubData = gh && typeof gh === "object"
385
- ? {
386
- stars: (gh as Record<string, number>).stargazers_count ?? null,
387
- forks: (gh as Record<string, number>).forks_count ?? null,
388
- open_issues: (gh as Record<string, number>).open_issues_count ?? null,
389
- watchers: (gh as Record<string, number>).watchers_count ?? null,
390
- }
391
- : { stars: null, forks: null, open_issues: null, watchers: null };
392
-
393
- // unkey
394
- let agentsData: { total_api_calls_30d: number | null; note?: string } = {
395
- total_api_calls_30d: null,
396
- note: "unkey analytics unavailable",
397
- };
398
- const uk = val(unkey);
399
- if (uk && Array.isArray(uk)) {
400
- const total = (uk as Array<{ total?: number }>).reduce((s, v) => s + (v.total ?? 0), 0);
401
- agentsData = { total_api_calls_30d: total };
402
- } else if (uk && typeof uk === "object" && (uk as Record<string, unknown>).total != null) {
403
- agentsData = { total_api_calls_30d: (uk as Record<string, number>).total };
404
- }
405
-
406
- const data = {
407
- npm: {
408
- unbrowse: { last_30d: u30, all_time: uAll },
409
- openclaw_plugin: { last_30d: p30, all_time: pAll },
410
- combined: {
411
- last_30d: u30 != null && p30 != null ? u30 + p30 : (u30 ?? p30),
412
- all_time: uAll != null && pAll != null ? uAll + pAll : (uAll ?? pAll),
413
- },
414
- daily,
415
- },
416
- github: githubData,
417
- agents: agentsData,
418
- fetched_at: new Date().toISOString(),
419
- };
420
-
421
- statsCache = { data, ts: Date.now() };
422
- return data;
423
- }
424
-
425
- export async function registerRoutes(app: FastifyInstance) {
426
- const clientScopeFor = (req: { headers: Record<string, unknown>; id: string }) =>
427
- (typeof req.headers["x-unbrowse-client-id"] === "string" && req.headers["x-unbrowse-client-id"].trim())
428
- ? req.headers["x-unbrowse-client-id"].trim()
429
- : req.id;
430
-
431
- function checkpointPublishCommand(skillId: string | null, confirmPublish = false): string {
432
- return skillId
433
- ? `unbrowse publish --skill ${skillId}${confirmPublish ? " --confirm-publish" : ""}`
434
- : `unbrowse publish --skill <skill_id>${confirmPublish ? " --confirm-publish" : ""}`;
435
- }
436
-
437
- function buildPublishFailureNextStep(skillId: string, validationErrors?: string[]): string {
438
- const reviewErrors = (validationErrors ?? []).filter((error) => error.startsWith("review_required:"));
439
- if (reviewErrors.length > 0) {
440
- return `Remote share blocked: ${reviewErrors.length} endpoint(s) still need review. Re-run ${checkpointPublishCommand(skillId)} to inspect review_context, then publish again with reviewed endpoints.`;
441
- }
442
- return `Remote share did not complete. Inspect validation_errors, adjust the contract locally, then retry ${checkpointPublishCommand(skillId, true)}.`;
443
- }
444
-
445
- function buildCheckpointNextStep(
446
- action: "sync" | "close",
447
- result: {
448
- skill_id: string | null;
449
- pipeline: {
450
- index_queued: boolean;
451
- publish_queued: boolean;
452
- };
453
- publish_policy: {
454
- mode: "auto" | "disabled" | "blacklisted" | "prompt";
455
- reason: string;
456
- matched_domain?: string;
457
- };
458
- },
459
- sessionId?: string,
460
- ): string {
461
- const sessionHint = sessionId ? ` --session ${sessionId}` : "";
462
- const publishCommand = checkpointPublishCommand(
463
- result.skill_id,
464
- result.publish_policy.mode === "blacklisted" || result.publish_policy.mode === "prompt",
465
- );
466
-
467
- if (!result.pipeline.index_queued) {
468
- return action === "sync"
469
- ? `Checkpoint recorded, but no new capture was available to index. Continue browsing, then run \`unbrowse close${sessionHint}\` for the final checkpoint.`
470
- : "Final checkpoint recorded, but no new capture was available to index or publish.";
471
- }
472
-
473
- if (result.publish_policy.mode === "auto") {
474
- return action === "sync"
475
- ? `Checkpoint saved. Background index + publish queued. Continue browsing, then run \`unbrowse close${sessionHint}\` for the final checkpoint.`
476
- : "Final checkpoint saved. Background index + publish queued. Inspect the indexed contract or wait for publish to complete.";
477
- }
478
-
479
- const base = result.publish_policy.mode === "disabled"
480
- ? "Checkpoint saved. Local index queued, but auto-publish is disabled in settings."
481
- : `Checkpoint saved. Local index queued, but auto-publish did not run: ${result.publish_policy.reason}`;
482
-
483
- const suffix = result.skill_id
484
- ? ` Review the indexed contract, then run \`${publishCommand}\` only if you own this index.`
485
- : "";
486
-
487
- if (action === "sync") {
488
- return `${base} Continue browsing, then run \`unbrowse close${sessionHint}\` when done.${suffix}`;
489
- }
490
- return `${base}${suffix}`;
491
- }
492
-
493
- // Auth gate: block all routes except /health when no API key is configured
494
- app.addHook("onRequest", async (req, reply) => {
495
- if (req.url === "/health" || req.url === "/v1/stats" || req.url.startsWith("/v1/settings")) return;
496
-
497
- const key = getApiKey();
498
- if (!key) {
499
- return reply.code(401).send({
500
- error: "api_key_required",
501
- message: "No API key configured. Restart the server to auto-register, or run: bash scripts/setup.sh",
502
- docs_url: "https://unbrowse.ai",
503
- });
504
- }
505
- });
506
-
507
- app.get("/v1/settings", async (_req, reply) => {
508
- return reply.send({
509
- capture_pipeline: getCapturePipelineSettings(),
510
- });
511
- });
512
-
513
- app.post("/v1/settings", async (req, reply) => {
514
- const body = (req.body ?? {}) as {
515
- auto_publish_checkpoints?: boolean;
516
- publish_domain_blacklist?: string[];
517
- publish_domain_promptlist?: string[];
518
- clear_publish_domain_blacklist?: boolean;
519
- clear_publish_domain_promptlist?: boolean;
520
- };
521
-
522
- const settings = updateCapturePipelineSettings({
523
- auto_publish_checkpoints: typeof body.auto_publish_checkpoints === "boolean"
524
- ? body.auto_publish_checkpoints
525
- : undefined,
526
- publish_domain_blacklist: Array.isArray(body.publish_domain_blacklist)
527
- ? body.publish_domain_blacklist
528
- : undefined,
529
- publish_domain_promptlist: Array.isArray(body.publish_domain_promptlist)
530
- ? body.publish_domain_promptlist
531
- : undefined,
532
- clear_publish_domain_blacklist: body.clear_publish_domain_blacklist === true,
533
- clear_publish_domain_promptlist: body.clear_publish_domain_promptlist === true,
534
- });
535
-
536
- return reply.send({
537
- ok: true,
538
- capture_pipeline: settings,
539
- next_step: settings.auto_publish_checkpoints
540
- ? "Auto-publish after sync/close is enabled unless a domain rule blocks it."
541
- : "Auto-publish after sync/close is disabled. Use index for local recompute and publish only when you explicitly want remote share.",
542
- });
543
- });
544
-
545
- // POST /v1/intent/resolve
546
- app.post("/v1/intent/resolve", { config: { rateLimit: ROUTE_LIMITS["/v1/intent/resolve"] } }, async (req, reply) => {
547
- const clientScope = clientScopeFor(req);
548
- const { intent, params, context, projection, confirm_unsafe, confirm_third_party_terms, dry_run, force_capture } = req.body as {
549
- intent: string;
550
- params?: Record<string, unknown>;
551
- context?: { url?: string; domain?: string };
552
- projection?: ProjectionOptions;
553
- confirm_unsafe?: boolean;
554
- confirm_third_party_terms?: boolean;
555
- dry_run?: boolean;
556
- force_capture?: boolean;
557
- };
558
- if (!intent) return reply.code(400).send({ error: "intent required" });
559
- try {
560
- const result = await resolveAndExecute(intent, params ?? {}, context, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, force_capture, client_scope: clientScope });
561
-
562
- // Surface timing breakdown
563
- const res = attachAgentOutcomeHints({ ...result } as Record<string, unknown>, {
564
- skill: result.skill,
565
- endpointId: result.trace.endpoint_id,
566
- timing: result.timing,
567
- });
568
- if (result.timing) {
569
- res.timing = result.timing;
570
- }
571
-
572
- // If the orchestrator already included available_endpoints in result (deferral),
573
- // also append them at the top level for backward compatibility.
574
- const innerResult = result.result as Record<string, unknown> | null;
575
- if (innerResult?.available_endpoints && !res.available_endpoints) {
576
- res.available_endpoints = innerResult.available_endpoints;
577
- }
578
-
579
- await recordAnalyticsSession(buildAnalyticsSessionPayload(result, {
580
- discovery_queries: 1,
581
- })).catch(() => {});
582
-
583
- return reply.send(res);
584
- } catch (err) {
585
- return reply.code(500).send({ error: (err as Error).message });
586
- }
587
- });
588
-
589
- // GET /v1/skills/:skill_id — local route so skill lookups hit disk cache before proxying to backend
590
- app.get("/v1/skills/:skill_id", async (req, reply) => {
591
- const clientScope = clientScopeFor(req);
592
- const { skill_id } = req.params as { skill_id: string };
593
- // Check local caches: recent skills → domain snapshots → marketplace
594
- let skill = getRecentLocalSkill(skill_id, clientScope);
595
- if (!skill) {
596
- for (const [, entry] of domainSkillCache) {
597
- if (entry.skillId === skill_id && entry.localSkillPath) {
598
- try { skill = JSON.parse(require("fs").readFileSync(entry.localSkillPath, "utf-8")); } catch {}
599
- break;
600
- }
601
- }
602
- }
603
- if (!skill) skill = await getSkill(skill_id, clientScope);
604
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
605
- return reply.send(skill);
606
- });
607
-
608
- // POST /v1/skills/:skill_id/review — agent submits reviewed descriptions + synthetic examples
609
- app.post("/v1/skills/:skill_id/review", async (req, reply) => {
610
- const clientScope = clientScopeFor(req);
611
- const { skill_id } = req.params as { skill_id: string };
612
- const { endpoints: reviews } = req.body as {
613
- endpoints: import("../types/index.js").EndpointReviewPayload[];
614
- };
615
- if (!reviews?.length) return reply.code(400).send({ error: "endpoints[] required" });
616
-
617
- let skill = await loadSkillForMutation(skill_id, clientScope);
618
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
619
-
620
- const updated = mergeAgentReview(skill.endpoints, reviews);
621
- skill.endpoints = updated;
622
- skill.updated_at = new Date().toISOString();
623
- const updatedWorkflowArtifact = applyWorkflowSchemaReviews(readWorkflowArtifact(skill.skill_id), reviews);
624
- if (updatedWorkflowArtifact) writeWorkflowArtifact(updatedWorkflowArtifact);
625
-
626
- const indexed = await indexSkillLocally(buildSkillIndexJob(skill, clientScope));
627
- return reply.send({
628
- ok: true,
629
- skill_id: indexed.skill.skill_id,
630
- endpoints_updated: reviews.length,
631
- indexed: true,
632
- publish_status: "indexed",
633
- endpoint_count: indexed.skill.endpoints.length,
634
- });
635
- });
636
-
637
- // POST /v1/skills/:skill_id/index — local-only graph/export recompute from cached state
638
- app.post("/v1/skills/:skill_id/index", async (req, reply) => {
639
- const clientScope = clientScopeFor(req);
640
- const { skill_id } = req.params as { skill_id: string };
641
- const skill = await loadSkillForMutation(skill_id, clientScope);
642
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
643
-
644
- const indexed = await indexSkillLocally(buildSkillIndexJob(skill, clientScope));
645
- return reply.send({
646
- ok: true,
647
- skill_id: indexed.skill.skill_id,
648
- indexed: true,
649
- publish_status: "indexed",
650
- endpoint_count: indexed.skill.endpoints.length,
651
- domain: indexed.domain,
652
- next_step: `Local contracts re-indexed. Review them, then run ${checkpointPublishCommand(indexed.skill.skill_id)} when you explicitly want remote share.`,
653
- });
654
- });
655
-
656
- // POST /v1/skills/:skill_id/publish — two-phase agent-driven publish
657
- // Phase 1 (no endpoints body): re-index locally, then return endpoints needing descriptions
658
- // Phase 2 (with endpoints): merge descriptions, re-index locally, then publish remotely
659
- app.post("/v1/skills/:skill_id/publish", async (req, reply) => {
660
- const clientScope = clientScopeFor(req);
661
- const { skill_id } = req.params as { skill_id: string };
662
- const { endpoints: reviews, confirm_publish } = (req.body as {
663
- endpoints?: import("../types/index.js").EndpointReviewPayload[];
664
- confirm_publish?: boolean;
665
- }) ?? {};
666
-
667
- let skill = await loadSkillForMutation(skill_id, clientScope);
668
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
669
-
670
- if (reviews?.length) {
671
- const publishDecision = decideExplicitPublish(skill.domain, confirm_publish === true);
672
- if (!publishDecision.allowed) {
673
- return reply.code(409).send({
674
- error: "publish_confirmation_required",
675
- domain: skill.domain,
676
- publish_policy: {
677
- mode: publishDecision.mode,
678
- reason: publishDecision.reason,
679
- matched_domain: publishDecision.matchedDomain,
680
- },
681
- next_step: `Re-run ${checkpointPublishCommand(skill.skill_id, true)} only if you own this index.`,
682
- });
683
- }
684
- const updated = mergeAgentReview(skill.endpoints, reviews);
685
- skill.endpoints = updated;
686
- skill.updated_at = new Date().toISOString();
687
- const updatedWorkflowArtifact = applyWorkflowSchemaReviews(readWorkflowArtifact(skill.skill_id), reviews);
688
- if (updatedWorkflowArtifact) writeWorkflowArtifact(updatedWorkflowArtifact);
689
- const indexed = await indexSkillLocally(buildSkillIndexJob(skill, clientScope));
690
- const publishResult = await publishIndexedSkill(indexed);
691
- return reply.send({
692
- ok: true,
693
- skill_id: skill.skill_id,
694
- endpoints_updated: reviews.length,
695
- indexed: true,
696
- published: publishResult.published,
697
- publish_status: publishResult.publishStatus,
698
- ...(publishResult.publishedAt ? { published_at: publishResult.publishedAt } : {}),
699
- ...(publishResult.validationErrors ? { validation_errors: publishResult.validationErrors } : {}),
700
- next_step: publishResult.publishStatus === "published"
701
- ? "Remote share completed. Re-run resolve/skill inspection to use the published contract."
702
- : buildPublishFailureNextStep(skill.skill_id, publishResult.validationErrors),
703
- });
704
- }
705
-
706
- const indexed = await indexSkillLocally(buildSkillIndexJob(skill, clientScope));
707
- const ranked = rankEndpoints(indexed.skill.endpoints, indexed.skill.intent_signature, indexed.skill.domain);
708
- const endpoints_to_describe = ranked.map((r) => {
709
- const descriptionMeta = getEndpointDescriptionMetadata(r.endpoint);
710
- return {
711
- endpoint_id: r.endpoint.endpoint_id,
712
- method: r.endpoint.method,
713
- url: r.endpoint.url_template.length > 120
714
- ? r.endpoint.url_template.slice(0, 120) + "..."
715
- : r.endpoint.url_template,
716
- current_description: descriptionMeta.display,
717
- description_source: descriptionMeta.source,
718
- description_needs_review: descriptionMeta.needs_review,
719
- ...(descriptionMeta.warning ? { description_warning: descriptionMeta.warning } : {}),
720
- schema_summary: r.endpoint.response_schema
721
- ? summarizeSchema(r.endpoint.response_schema)
722
- : null,
723
- sample_values: extractSampleValues(r.endpoint.semantic?.example_response_compact),
724
- input_params: r.endpoint.semantic?.requires?.map((b) => ({
725
- key: b.key,
726
- type: b.type ?? b.semantic_type,
727
- required: b.required ?? false,
728
- example: b.example_value,
729
- })) ?? [],
730
- dom_extraction: !!r.endpoint.dom_extraction,
731
- review_context: buildEndpointReviewContext(indexed.skill, r.endpoint.endpoint_id),
732
- _fill_description:
733
- "DESCRIBE THIS ENDPOINT — what it returns, key params, action type, and any audience/eligibility/pricing/validity constraints",
734
- };
735
- });
736
-
737
- return reply.send({
738
- skill_id: indexed.skill.skill_id,
739
- domain: indexed.skill.domain,
740
- indexed: true,
741
- publish_status: "indexed",
742
- endpoint_count: indexed.skill.endpoints.length,
743
- endpoints_to_describe,
744
- next_step:
745
- `Fill each endpoint's description using review_context (deps, request_schema, response_schema, provenance, trigger page) plus any audience/eligibility/pricing/validity caveats, then call: unbrowse publish --skill ${indexed.skill.skill_id} --endpoints '[{endpoint_id, description, action_kind, resource_kind}]'`,
746
- _next_step:
747
- `Fill each endpoint's description using review_context (deps, request_schema, response_schema, provenance, trigger page) plus any audience/eligibility/pricing/validity caveats, then call: unbrowse publish --skill ${indexed.skill.skill_id} --endpoints '[{endpoint_id, description, action_kind, resource_kind}]'`,
748
- });
749
- });
750
-
751
- // POST /v1/foundry/publish-bundle — derive bundle/share/host artifacts from one preset
752
- app.post("/v1/foundry/publish-bundle", async (req, reply) => {
753
- const { preset_path, site_url, hosts } = (req.body as {
754
- preset_path?: string;
755
- site_url?: string;
756
- hosts?: string[];
757
- }) ?? {};
758
-
759
- if (!preset_path?.trim()) {
760
- return reply.code(400).send({ error: "preset_path is required" });
761
- }
762
-
763
- try {
764
- const result = publishFoundryBundle({
765
- presetPath: preset_path,
766
- ...(site_url?.trim() ? { siteUrl: site_url } : {}),
767
- ...(Array.isArray(hosts) && hosts.length > 0 ? {
768
- hosts: hosts.filter((host): host is "codex" | "claude" | "openclaw" =>
769
- host === "codex" || host === "claude" || host === "openclaw",
770
- ),
771
- } : {}),
772
- });
773
- return reply.send(result);
774
- } catch (error) {
775
- return reply.code(400).send({
776
- error: error instanceof Error ? error.message : String(error),
777
- });
778
- }
779
- });
780
- // POST /v1/skills/:skill_id/chunk — dynamic subgraph load for the current intent/bindings
781
- app.post("/v1/skills/:skill_id/chunk", async (req, reply) => {
782
- const clientScope = clientScopeFor(req);
783
- const { skill_id } = req.params as { skill_id: string };
784
- const { intent, operation_id, known_bindings, max_operations } = req.body as {
785
- intent?: string;
786
- operation_id?: string;
787
- known_bindings?: Record<string, unknown>;
788
- max_operations?: number;
789
- };
790
- const skill = getRecentLocalSkill(skill_id, clientScope) ?? await getSkill(skill_id, clientScope);
791
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
792
- return reply.send(toAgentSkillChunkView(getSkillChunk(skill, {
793
- intent,
794
- seed_operation_id: operation_id,
795
- known_bindings,
796
- max_operations,
797
- })));
798
- });
799
-
800
- // POST /v1/skills/:skill_id/execute
801
- app.post("/v1/skills/:skill_id/execute", { config: { rateLimit: ROUTE_LIMITS["/v1/skills/:skill_id/execute"] } }, async (req, reply) => {
802
- const clientScope = clientScopeFor(req);
803
- const { skill_id } = req.params as { skill_id: string };
804
- const { params, projection, confirm_unsafe, confirm_third_party_terms, dry_run, intent, context_url } = req.body as {
805
- params?: Record<string, unknown>;
806
- projection?: ProjectionOptions;
807
- confirm_unsafe?: boolean;
808
- confirm_third_party_terms?: boolean;
809
- dry_run?: boolean;
810
- intent?: string;
811
- context_url?: string;
812
- };
813
- // Check local caches first: recent skills → domain snapshots → marketplace
814
- let skill = getRecentLocalSkill(skill_id, clientScope);
815
- if (!skill) {
816
- // Check domain snapshot cache — passively indexed skills live here
817
- const { findExistingSkillForDomain: findLocal } = await import("../client/index.js");
818
- for (const [, entry] of domainSkillCache) {
819
- if (entry.skillId === skill_id && entry.localSkillPath) {
820
- try {
821
- skill = JSON.parse(require("fs").readFileSync(entry.localSkillPath, "utf-8"));
822
- } catch { /* snapshot read failed */ }
823
- break;
824
- }
825
- }
826
- }
827
- if (!skill) skill = await getSkill(skill_id, clientScope);
828
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
829
- const execParams = {
830
- ...(params ?? {}),
831
- ...(context_url && typeof params?.url !== "string" ? { url: context_url } : {}),
832
- };
833
- try {
834
- const execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, intent, contextUrl: context_url, client_scope: clientScope });
835
- saveTrace(execResult.trace);
836
- if (execResult.trace.endpoint_id) {
837
- recordExecution(skill.skill_id, execResult.trace.endpoint_id, execResult.trace, skill).catch(() => {});
838
- }
839
- if (execResult.trace.success) {
840
- promoteExplicitExecution(
841
- clientScope,
842
- intent || skill.intent_signature,
843
- context_url || (typeof execParams.url === "string" ? execParams.url : undefined),
844
- skill,
845
- execResult.trace.endpoint_id,
846
- execResult.result,
847
- );
848
- }
849
-
850
- // Auto-recovery: if endpoint returned 404 (stale), re-capture via orchestrator
851
- if (
852
- execResult.trace.status_code === 404 &&
853
- skill.domain &&
854
- skill.intent_signature &&
855
- skill.execution_type !== "browser-capture"
856
- ) {
857
- try {
858
- const recoveryUrl =
859
- context_url ||
860
- (typeof execParams.url === "string" && execParams.url) ||
861
- skill.endpoints.find((endpoint) => typeof endpoint.trigger_url === "string" && endpoint.trigger_url)?.trigger_url ||
862
- `https://${skill.domain}`;
863
- const freshResult = await resolveAndExecute(
864
- intent || skill.intent_signature,
865
- { ...execParams, url: recoveryUrl },
866
- { url: recoveryUrl },
867
- projection,
868
- { confirm_unsafe, confirm_third_party_terms, dry_run, intent: intent || skill.intent_signature, client_scope: clientScope }
869
- );
870
- saveTrace(freshResult.trace);
871
- if (freshResult.trace?.skill_id && freshResult.trace?.endpoint_id) {
872
- recordExecution(freshResult.trace.skill_id, freshResult.trace.endpoint_id, freshResult.trace, skill).catch(() => {});
873
- }
874
- await recordAnalyticsSession(buildAnalyticsSessionPayload(freshResult, {
875
- discovery_queries: 1,
876
- })).catch(() => {});
877
- const recovered = attachAgentOutcomeHints({
878
- ...freshResult,
879
- _recovery: {
880
- reason: "stale_endpoint_404",
881
- original_skill_id: skill_id,
882
- message: "Original endpoint returned 404. Auto-recovered with fresh capture.",
883
- },
884
- } as Record<string, unknown>, {
885
- skill: freshResult.skill ?? skill,
886
- endpointId: freshResult.trace.endpoint_id,
887
- timing: freshResult.timing,
888
- });
889
- return reply.send({
890
- ...recovered,
891
- });
892
- } catch {
893
- // Recovery failed — return original 404 with guidance
894
- }
895
- }
896
-
897
- await recordAnalyticsSession(buildAnalyticsSessionPayload(execResult, {
898
- discovery_queries: 0,
899
- })).catch(() => {});
900
-
901
- const response = attachAgentOutcomeHints({ ...execResult } as Record<string, unknown>, {
902
- skill,
903
- endpointId: execResult.trace.endpoint_id,
904
- });
905
- return reply.send(response);
906
- } catch (err) {
907
- return reply.code(500).send({ error: (err as Error).message });
908
- }
909
- });
910
-
911
- // POST /v1/skills/:skill_id/auth -- store credentials (cookies/headers) for a skill
912
- app.post("/v1/skills/:skill_id/auth", async (req, reply) => {
913
- const { skill_id } = req.params as { skill_id: string };
914
- const skill = await getSkill(skill_id);
915
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
916
-
917
- const body = req.body as {
918
- cookies?: Array<{ name: string; value: string; domain: string; path?: string }>;
919
- headers?: Record<string, string>;
920
- };
921
- if (!body.cookies && !body.headers) {
922
- return reply.code(400).send({ error: "Provide cookies or headers" });
923
- }
924
-
925
- const ref = `${skill.domain}-session`;
926
- await storeCredential(ref, JSON.stringify({ cookies: body.cookies ?? [], headers: body.headers ?? {} }));
927
-
928
- // Patch the skill manifest to reference the stored credentials
929
- if (!skill.auth_profile_ref) {
930
- await publishSkill({ ...skill, auth_profile_ref: ref });
931
- }
932
-
933
- return reply.send({ ok: true, auth_profile_ref: ref });
934
- });
935
-
936
- // POST /v1/auth/login — interactive OAuth flow or direct browser cookie extraction
937
- app.post("/v1/auth/login", { config: { rateLimit: ROUTE_LIMITS["/v1/auth/login"] } }, async (req, reply) => {
938
- const {
939
- url,
940
- browser,
941
- chrome_profile,
942
- firefox_profile,
943
- chromium_profile,
944
- chromium_user_data_dir,
945
- chromium_cookie_db_path,
946
- safe_storage_service,
947
- browser_name,
948
- } = req.body as {
949
- url: string;
950
- browser?: "auto" | "firefox" | "chrome" | "chromium";
951
- chrome_profile?: string;
952
- firefox_profile?: string;
953
- chromium_profile?: string;
954
- chromium_user_data_dir?: string;
955
- chromium_cookie_db_path?: string;
956
- safe_storage_service?: string;
957
- browser_name?: string;
958
- };
959
- if (!url) return reply.code(400).send({ error: "url required" });
960
- try {
961
- const result = await loginWithBrowserFallback(url, {
962
- browser,
963
- chromeProfile: chrome_profile,
964
- firefoxProfile: firefox_profile,
965
- chromium: {
966
- profile: chromium_profile,
967
- userDataDir: chromium_user_data_dir,
968
- cookieDbPath: chromium_cookie_db_path,
969
- safeStorageService: safe_storage_service,
970
- browserName: browser_name,
971
- },
972
- });
973
- return reply.send(result);
974
- } catch (err) {
975
- return reply.code(500).send({ error: (err as Error).message });
976
- }
977
- });
978
-
979
- // POST /v1/auth/steal — extract cookies from Firefox/Chrome/custom Chromium-family SQLite DBs.
980
- // No browser launch, Chrome can stay open. Higher rate limit since it's instant.
981
- app.post("/v1/auth/steal", { config: { rateLimit: { max: 30, timeWindow: "1 minute" } } }, async (req, reply) => {
982
- const {
983
- url,
984
- browser,
985
- chrome_profile,
986
- firefox_profile,
987
- chromium_profile,
988
- chromium_user_data_dir,
989
- chromium_cookie_db_path,
990
- safe_storage_service,
991
- browser_name,
992
- } = req.body as {
993
- url: string;
994
- browser?: "auto" | "firefox" | "chrome" | "chromium";
995
- chrome_profile?: string;
996
- firefox_profile?: string;
997
- chromium_profile?: string;
998
- chromium_user_data_dir?: string;
999
- chromium_cookie_db_path?: string;
1000
- safe_storage_service?: string;
1001
- browser_name?: string;
1002
- };
1003
- if (!url) return reply.code(400).send({ error: "url required" });
1004
- try {
1005
- const domain = new URL(url).hostname;
1006
- const result = await extractBrowserAuth(domain, {
1007
- browser,
1008
- chromeProfile: chrome_profile,
1009
- firefoxProfile: firefox_profile,
1010
- chromium: {
1011
- profile: chromium_profile,
1012
- userDataDir: chromium_user_data_dir,
1013
- cookieDbPath: chromium_cookie_db_path,
1014
- safeStorageService: safe_storage_service,
1015
- browserName: browser_name,
1016
- },
1017
- });
1018
- return reply.send(result);
1019
- } catch (err) {
1020
- return reply.code(500).send({ error: (err as Error).message });
1021
- }
1022
- });
1023
-
1024
- // POST /v1/skills/:skill_id/verify — trigger verification
1025
- app.post("/v1/skills/:skill_id/verify", async (req, reply) => {
1026
- const { skill_id } = req.params as { skill_id: string };
1027
- const skill = await getSkill(skill_id);
1028
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
1029
- try {
1030
- const { verifySkill } = await import("../verification/index.js");
1031
- const results = await verifySkill(skill);
1032
- return reply.send({ skill_id, verification: results });
1033
- } catch (err) {
1034
- return reply.code(500).send({ error: (err as Error).message });
1035
- }
1036
- });
1037
-
1038
- // POST /v1/stale/cleanup — verify active skills, mark dead endpoints stale, drop local cache entries
1039
- app.post("/v1/stale/cleanup", async (req, reply) => {
1040
- const body = (req.body ?? {}) as {
1041
- skill_id?: string;
1042
- domain?: string;
1043
- limit?: number;
1044
- };
1045
- try {
1046
- const result = await cleanupStaleSkills({
1047
- skill_id: typeof body.skill_id === "string" ? body.skill_id : undefined,
1048
- domain: typeof body.domain === "string" ? body.domain : undefined,
1049
- limit: typeof body.limit === "number" ? body.limit : undefined,
1050
- });
1051
- return reply.send(result);
1052
- } catch (err) {
1053
- return reply.code(500).send({ error: (err as Error).message });
1054
- }
1055
- });
1056
-
1057
- // POST /v1/feedback — submit execution feedback with optional diagnostics
1058
- app.post("/v1/feedback", async (req, reply) => {
1059
- const { skill_id, target_id, endpoint_id, rating, outcome, diagnostics } = req.body as {
1060
- skill_id?: string;
1061
- target_id?: string;
1062
- endpoint_id?: string;
1063
- rating?: number;
1064
- outcome?: string;
1065
- diagnostics?: {
1066
- total_ms?: number;
1067
- bottleneck?: string;
1068
- wrong_endpoint?: boolean;
1069
- expected_data?: string;
1070
- got_data?: string;
1071
- trace_version?: string;
1072
- };
1073
- };
1074
- const resolvedSkillId = skill_id ?? target_id;
1075
- if (!resolvedSkillId || !endpoint_id || rating == null) {
1076
- return reply.code(400).send({ error: "skill_id, endpoint_id, and rating required" });
1077
- }
1078
- try {
1079
- const avg_rating = await recordFeedback(resolvedSkillId, endpoint_id, rating);
1080
- // Forward diagnostics to backend for version-grouped analysis
1081
- if (diagnostics) {
1082
- recordDiagnostics(resolvedSkillId, endpoint_id, diagnostics).catch(() => {});
1083
- }
1084
- return reply.send({ ok: true, avg_rating });
1085
- } catch (err) {
1086
- return reply.code(500).send({ error: (err as Error).message });
1087
- }
1088
- });
1089
-
1090
- // GET /v1/stats — public, no auth required
1091
- app.get("/v1/stats", async (_req, reply) => {
1092
- try {
1093
- const data = await fetchStats();
1094
- return reply.send(data);
1095
- } catch (err) {
1096
- return reply.code(500).send({ error: (err as Error).message });
1097
- }
1098
- });
1099
-
1100
- // GET /health
1101
- app.get("/health", async (_req, reply) => reply.send({
1102
- status: "ok",
1103
- package_version: PACKAGE_VERSION,
1104
- trace_version: TRACE_VERSION,
1105
- code_hash: CODE_HASH,
1106
- git_sha: GIT_SHA,
1107
- }));
1108
-
1109
- // GET /v1/sessions/:domain — read local trace/debug files instead of proxying to backend
1110
- app.get("/v1/sessions/:domain", async (req, reply) => {
1111
- const { domain } = req.params as { domain: string };
1112
- const query = req.query as { limit?: string | number };
1113
- const limitRaw = typeof query.limit === "number" ? query.limit : Number(query.limit ?? 10);
1114
- const limit = Number.isFinite(limitRaw) ? Math.min(Math.max(limitRaw, 1), 50) : 10;
1115
- return reply.send({
1116
- domain,
1117
- sessions: listRecentSessionsForDomain(TRACES_DIR, domain, limit),
1118
- });
1119
- });
1120
-
1121
- // Catch-all proxy: forward unmatched /v1/* routes to the configured backend origin
1122
- app.all("/v1/*", async (req, reply) => {
1123
- const key = getApiKey();
1124
- const upstream = `${BETA_API_URL}${req.url}`;
1125
- const headers: Record<string, string> = { "Content-Type": "application/json" };
1126
- if (key) headers["Authorization"] = `Bearer ${key}`;
1127
-
1128
- try {
1129
- const res = await fetch(upstream, {
1130
- method: req.method,
1131
- headers,
1132
- body: req.method !== "GET" && req.method !== "HEAD" ? JSON.stringify(req.body) : undefined,
1133
- });
1134
- const text = await res.text();
1135
- try {
1136
- return reply.code(res.status).send(JSON.parse(text));
1137
- } catch {
1138
- return reply.code(res.status).send({ error: text || `Upstream returned ${res.status}` });
1139
- }
1140
- } catch (err) {
1141
- return reply.code(502).send({ error: `Proxy to beta-api failed: ${(err as Error).message}` });
1142
- }
1143
- });
1144
-
1145
- // ── Browse session management ─────────────────────────────────────────
1146
- // Kuri browser actions with passive HAR indexing. The server manages a
1147
- // per-session tab + HAR state so every action the agent takes through
1148
- // the CLI is passively captured and indexed.
1149
-
1150
- // browseSessions is module-level (shared with orchestrator via registerBrowseSession)
1151
-
1152
- function requestedSessionId(req: { body?: unknown; query?: unknown }): string | undefined {
1153
- const body = req.body && typeof req.body === "object" ? req.body as Record<string, unknown> : null;
1154
- if (typeof body?.session_id === "string" && body.session_id.trim()) return body.session_id;
1155
- const query = req.query && typeof req.query === "object" ? req.query as Record<string, unknown> : null;
1156
- if (typeof query?.session_id === "string" && query.session_id.trim()) return query.session_id;
1157
- return undefined;
1158
- }
1159
-
1160
- function sendBrowseSessionError(reply: { code: (statusCode: number) => { send: (body: unknown) => unknown } }, error: unknown): unknown {
1161
- if (error instanceof BrowseSessionError) {
1162
- return reply.code(error.statusCode).send({ error: error.code });
1163
- }
1164
- if (isRecoverableBrowseFailure(error)) {
1165
- return reply.code(502).send({
1166
- error: "recoverable_browse_failure",
1167
- message: extractBrowseFailureMessage(error) ?? "recoverable_browse_failure",
1168
- recoverable: true,
1169
- });
1170
- }
1171
- throw error;
1172
- }
1173
-
1174
- /** Extract registrable domain for auth profile naming */
1175
- function profileName(url: string): string {
1176
- try { return new URL(url).hostname.replace(/^www\./, ""); } catch { return "unknown"; }
1177
- }
1178
-
1179
- async function restartBrowseCapture(session: BrowseSession): Promise<void> {
1180
- const broker = brokerForSession(session);
1181
- const load = await broker.waitForLoad(session.tabId, 2_000).catch(() => null);
1182
- if (load && load.status === "timeout") {
1183
- session.harActive = false;
1184
- return;
1185
- }
1186
- await broker.networkEnable(session.tabId).catch(() => {});
1187
- await broker.harStart(session.tabId).catch(() => {});
1188
- await broker.scriptInject(session.tabId, INTERCEPTOR_SCRIPT).catch(() => {});
1189
- session.harActive = true;
1190
- await injectInterceptor(session.tabId).catch(() => {});
1191
- }
1192
- async function flushBrowseCapture(
1193
- session: BrowseSession,
1194
- options: { queueIndex?: boolean; queuePublish?: boolean } = {},
1195
- ): Promise<{
1196
- indexed: boolean;
1197
- mode: "http" | "dom" | "none";
1198
- domain: string;
1199
- skill_id: string | null;
1200
- endpoint_count: number;
1201
- endpoints: Array<{
1202
- endpoint_id: string;
1203
- method: string;
1204
- url_template: string;
1205
- description?: string;
1206
- trigger_url?: string;
1207
- action_kind?: string;
1208
- resource_kind?: string;
1209
- }>;
1210
- request_count: number;
1211
- pipeline: {
1212
- index_queued: boolean;
1213
- publish_queued: boolean;
1214
- };
1215
- publish_policy: {
1216
- mode: "auto" | "disabled" | "blacklisted" | "prompt";
1217
- reason: string;
1218
- matched_domain?: string;
1219
- };
1220
- background_publish_queued: boolean;
1221
- }> {
1222
- let harEntries: KuriHarEntry[] = [];
1223
- if (session.harActive) {
1224
- try {
1225
- const { entries } = await brokerForSession(session).harStop(session.tabId);
1226
- harEntries = entries;
1227
- } catch { /* non-fatal */ }
1228
- }
1229
- session.harActive = false;
1230
-
1231
- const allRequests = await enrichPassiveCaptureRequests({
1232
- tabId: session.tabId,
1233
- captureUrl: session.url,
1234
- harEntries,
1235
- intent: `browse ${session.domain || profileName(session.url)}`,
1236
- });
1237
- const syncResult = await cacheBrowseRequests({
1238
- sessionUrl: session.url,
1239
- sessionDomain: session.domain,
1240
- requests: allRequests,
1241
- getPageHtml: () => brokerForSession(session).getPageHtml(session.tabId),
1242
- intent: `browse ${session.domain || profileName(session.url)}`,
1243
- });
1244
-
1245
- let indexQueued = false;
1246
- let publishQueued = false;
1247
- const publishDecision = options.queuePublish
1248
- ? decideCheckpointPublish(syncResult.domain)
1249
- : {
1250
- publishQueued: false,
1251
- mode: "disabled" as const,
1252
- reason: "Remote publish not requested for this checkpoint.",
1253
- };
1254
- if (options.queueIndex) {
1255
- if (syncResult.skill) {
1256
- queueBackgroundIndex({
1257
- skill: { ...syncResult.skill },
1258
- domain: syncResult.domain,
1259
- intent: syncResult.skill.intent_signature || `browse ${syncResult.domain}`,
1260
- contextUrl: session.url,
1261
- cacheKey: `browse-submit:${syncResult.domain}:${Date.now()}`,
1262
- publishAfterIndex: publishDecision.publishQueued,
1263
- });
1264
- indexQueued = true;
1265
- publishQueued = publishDecision.publishQueued;
1266
- } else if (allRequests.length > 0) {
1267
- passiveIndexFromRequests(allRequests, session.url, {
1268
- publishAfterIndex: publishDecision.publishQueued,
1269
- });
1270
- indexQueued = true;
1271
- publishQueued = publishDecision.publishQueued;
1272
- }
1273
- }
1274
- return {
1275
- indexed: syncResult.indexed,
1276
- mode: syncResult.mode,
1277
- domain: syncResult.domain,
1278
- skill_id: syncResult.skill?.skill_id ?? null,
1279
- endpoint_count: syncResult.skill?.endpoints.length ?? 0,
1280
- endpoints: (syncResult.skill?.endpoints ?? []).map((endpoint) => ({
1281
- endpoint_id: endpoint.endpoint_id,
1282
- method: endpoint.method,
1283
- url_template: endpoint.url_template,
1284
- description: endpoint.description,
1285
- trigger_url: endpoint.trigger_url,
1286
- action_kind: endpoint.semantic?.action_kind,
1287
- resource_kind: endpoint.semantic?.resource_kind,
1288
- })),
1289
- request_count: allRequests.length,
1290
- pipeline: {
1291
- index_queued: indexQueued,
1292
- publish_queued: publishQueued,
1293
- },
1294
- publish_policy: {
1295
- mode: publishDecision.mode,
1296
- reason: publishDecision.reason,
1297
- ...(publishDecision.matchedDomain ? { matched_domain: publishDecision.matchedDomain } : {}),
1298
- },
1299
- background_publish_queued: publishQueued,
1300
- };
1301
- }
1302
-
1303
- // POST /v1/browse/go — navigate to URL
1304
- app.post("/v1/browse/go", async (req, reply) => {
1305
- const { url } = req.body as { url: string };
1306
- if (!url) return reply.code(400).send({ error: "url required" });
1307
- try {
1308
- const sessionId = requestedSessionId(req);
1309
- const browseClient = selectBrowseBrokerClient(sessionId);
1310
- const navigateSession = async (session: BrowseSession) => {
1311
- const broker = brokerForSession(session);
1312
- const newDomain = profileName(url);
1313
-
1314
- if (session.harActive && session.url !== "about:blank") {
1315
- try {
1316
- const { entries } = await broker.harStop(session.tabId);
1317
- passiveIndexHar(entries, session.url, { publishAfterIndex: false });
1318
- } catch { /* non-fatal */ }
1319
- session.harActive = false;
1320
- }
1321
-
1322
- if (session.domain && session.domain !== newDomain) {
1323
- await saveAuthProfileBestEffort(session.tabId, session.domain, "browse_go");
1324
- }
1325
-
1326
- let cookiesInjected = 0;
1327
- if (newDomain && newDomain !== session.domain) {
1328
- // Check if the browser already has fresh session cookies for this domain.
1329
- // If so, skip vault/profile cookie injection — browser cookies are fresher
1330
- // and injecting stale vault cookies (e.g. JSESSIONID) causes HTTP 400 on
1331
- // sites like LinkedIn that validate CSRF alignment.
1332
- const browserHasFreshSession = await (async () => {
1333
- try {
1334
- const { extractBrowserCookies } = await import("../auth/browser-cookies.js");
1335
- const { cookies } = extractBrowserCookies(newDomain);
1336
- // Consider the session fresh if we have session-like cookies that aren't expired
1337
- const now = Date.now() / 1000;
1338
- return cookies.some((c) =>
1339
- (c.httpOnly || c.secure) && (!c.expires || c.expires > now),
1340
- );
1341
- } catch { return false; }
1342
- })();
1343
-
1344
- if (browserHasFreshSession) {
1345
- // Import browser cookies via CDP (they're fresh from Chrome's jar)
1346
- cookiesInjected = await importBrowserCookiesIntoTab(session.tabId, newDomain);
1347
- } else {
1348
- // No fresh browser cookies — load from vault/auth profile
1349
- cookiesInjected = await importBrowserCookiesIntoTab(session.tabId, newDomain);
1350
- await loadAuthProfileBestEffort(session.tabId, newDomain, "browse_go");
1351
- }
1352
- }
1353
-
1354
- await restartBrowseCapture(session);
1355
-
1356
- await broker.navigate(session.tabId, url);
1357
- await broker.navigate(session.tabId, url);
1358
- const finalUrl = await broker.getCurrentUrl(session.tabId).catch(() => url);
1359
- session.url = typeof finalUrl === "string" && finalUrl.startsWith("http") ? finalUrl : url;
1360
- session.domain = profileName(session.url);
1361
- await injectInterceptor(session.tabId);
1362
- const stillLive = await isBrowseSessionLive(session, browseClient).catch(() => false);
1363
- if (!stillLive) throw { error: "CDP command failed" };
1364
-
1365
- return { cookiesInjected };
1366
- };
1367
-
1368
- let session: BrowseSession;
1369
- let result: { cookiesInjected: number };
1370
- if (sessionId) {
1371
- const navigated = await withSerializedStrictBrowseSession(
1372
- browseSessions,
1373
- browseClient,
1374
- sessionId,
1375
- navigateSession,
1376
- );
1377
- session = navigated.session;
1378
- result = navigated.result;
1379
- } else {
1380
- session = await getOrCreateNavigateBrowseSession(
1381
- browseSessions,
1382
- browseClient,
1383
- injectInterceptor,
1384
- );
1385
- result = await navigateSession(session);
1386
- }
1387
-
1388
- return reply.send({
1389
- ok: true,
1390
- session_id: session.sessionId,
1391
- url: session.url,
1392
- tab_id: session.tabId,
1393
- auth_profile: session.domain,
1394
- ...(result.cookiesInjected > 0 ? { cookies_injected: result.cookiesInjected } : {}),
1395
- });
1396
- } catch (error) {
1397
- return sendBrowseSessionError(reply, error);
1398
- }
1399
- });
1400
-
1401
- // POST /v1/browse/submit — submit active form; same-origin fetch+rehydrate is explicit opt-in
1402
- app.post("/v1/browse/submit", async (req, reply) => {
1403
- const {
1404
- form_selector: formSelector,
1405
- submit_selector: submitSelector,
1406
- wait_for: waitFor,
1407
- same_origin_fetch_fallback: sameOriginFetchFallback,
1408
- timeout_ms: timeoutMs,
1409
- assist_site_state: assistSiteState,
1410
- } = (req.body as {
1411
- form_selector?: string;
1412
- submit_selector?: string;
1413
- wait_for?: string;
1414
- same_origin_fetch_fallback?: boolean;
1415
- timeout_ms?: number;
1416
- assist_site_state?: boolean;
1417
- }) ?? {};
1418
-
1419
- try {
1420
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1421
- const { session, result } = await withSerializedStrictBrowseSession(
1422
- browseSessions,
1423
- browseClient,
1424
- requestedSessionId(req),
1425
- async (session) => submitBrowseForm(
1426
- {
1427
- client: brokerForSession(session),
1428
- session,
1429
- flushCapture: async (session) => await flushBrowseCapture(session, { queueBackgroundPublish: false }),
1430
- flushCapture: async (session) => await flushBrowseCapture(session, { queueBackgroundPublish: false }),
1431
- restartCapture: restartBrowseCapture,
1432
- rehydratePlugins: (tabId) => brokerForSession(session).bestEffortRehydratePlugins(tabId),
1433
- },
1434
- {
1435
- formSelector,
1436
- submitSelector,
1437
- waitFor,
1438
- sameOriginFetchFallback,
1439
- timeoutMs,
1440
- assistSiteState,
1441
- },
1442
- ),
1443
- (result) => !result.ok && result.recoverable === true,
1444
- );
1445
-
1446
- let activeSession = session;
1447
- const hintedDestination = result.wait_for && isUrlWaitHint(result.wait_for)
1448
- ? resolveSubmitWaitHint(activeSession.url || "about:blank", result.wait_for)
1449
- : null;
1450
- const rawResultUrl = typeof result.url === "string" ? result.url : "";
1451
- activeSession.url = rawResultUrl || await brokerForSession(activeSession).getCurrentUrl(activeSession.tabId).catch(() => activeSession.url);
1452
- if (result.ok && hintedDestination && (!rawResultUrl || !rawResultUrl.includes(result.wait_for ?? ""))) {
1453
- activeSession.url = hintedDestination;
1454
- }
1455
- activeSession.domain = profileName(activeSession.url);
1456
- const stillLive = await isBrowseSessionLive(activeSession, browseClient).catch(() => false);
1457
- if (!stillLive) {
1458
- removeBrowseSession(browseSessions, activeSession.sessionId);
1459
- throw new BrowseSessionError("session_expired");
1460
- }
1461
-
1462
- const statusCode = result.ok ? 200 : (result.recoverable ? 502 : 400);
1463
- const sessionHint = `--session ${activeSession.sessionId}`;
1464
- const nextStep = result.ok
1465
- ? `If more UI steps remain, continue the flow. Run \`unbrowse sync ${sessionHint}\` after meaningful transitions, then \`unbrowse close ${sessionHint}\` when you're done to checkpoint the final capture, save auth, and queue the background pipeline.`
1466
- : `Inspect the page state with \`unbrowse snap ${sessionHint} --filter interactive\`, then retry submit with selectors or a wait hint if needed.`;
1467
- return reply.code(statusCode).send({
1468
- ...result,
1469
- session_id: activeSession.sessionId,
1470
- next_step: nextStep,
1471
- recovered: false,
1472
- tab_id: activeSession.tabId,
1473
- url: activeSession.url,
1474
- });
1475
- } catch (error) {
1476
- return sendBrowseSessionError(reply, error);
1477
- }
1478
- });
1479
-
1480
- // POST /v1/browse/snap — a11y snapshot
1481
- app.post("/v1/browse/snap", async (req, reply) => {
1482
- const { filter } = (req.body as { filter?: string; session_id?: string }) ?? {};
1483
- try {
1484
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1485
- const { session, result: snapshot } = await withSerializedStrictBrowseSession(
1486
- browseSessions,
1487
- browseClient,
1488
- requestedSessionId(req),
1489
- async (session) => brokerForSession(session).snapshot(session.tabId, filter),
1490
- );
1491
- return reply.send({ snapshot, session_id: session.sessionId, tab_id: session.tabId });
1492
- } catch (error) {
1493
- return sendBrowseSessionError(reply, error);
1494
- }
1495
- });
1496
-
1497
- // POST /v1/browse/click — click by ref
1498
- app.post("/v1/browse/click", async (req, reply) => {
1499
- const { ref } = req.body as { ref: string; session_id?: string };
1500
- if (!ref) return reply.code(400).send({ error: "ref required" });
1501
- try {
1502
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1503
- const { session } = await withSerializedStrictBrowseSession(
1504
- browseSessions,
1505
- browseClient,
1506
- requestedSessionId(req),
1507
- async (session) => {
1508
- await brokerForSession(session).click(session.tabId, ref);
1509
- return true;
1510
- },
1511
- );
1512
- await isBrowseSessionLive(session, browseClient).catch(() => false);
1513
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId, url: session.url });
1514
- } catch (error) {
1515
- return sendBrowseSessionError(reply, error);
1516
- }
1517
- });
1518
-
1519
- // POST /v1/browse/fill — fill input by ref
1520
- app.post("/v1/browse/fill", async (req, reply) => {
1521
- const { ref, value } = req.body as { ref: string; value: string; session_id?: string };
1522
- if (!ref || value === undefined) return reply.code(400).send({ error: "ref and value required" });
1523
- try {
1524
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1525
- const { session } = await withSerializedStrictBrowseSession(
1526
- browseSessions,
1527
- browseClient,
1528
- requestedSessionId(req),
1529
- async (session) => {
1530
- await brokerForSession(session).fill(session.tabId, ref, value);
1531
- return true;
1532
- },
1533
- );
1534
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1535
- } catch (error) {
1536
- return sendBrowseSessionError(reply, error);
1537
- }
1538
- });
1539
-
1540
- // POST /v1/browse/type — keyboard type
1541
- app.post("/v1/browse/type", async (req, reply) => {
1542
- const { text } = req.body as { text: string; session_id?: string };
1543
- if (!text) return reply.code(400).send({ error: "text required" });
1544
- try {
1545
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1546
- const { session } = await withSerializedStrictBrowseSession(
1547
- browseSessions,
1548
- browseClient,
1549
- requestedSessionId(req),
1550
- async (session) => {
1551
- await brokerForSession(session).keyboardType(session.tabId, text);
1552
- return true;
1553
- },
1554
- );
1555
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1556
- } catch (error) {
1557
- return sendBrowseSessionError(reply, error);
1558
- }
1559
- });
1560
-
1561
- // POST /v1/browse/press — press key
1562
- app.post("/v1/browse/press", async (req, reply) => {
1563
- const { key } = req.body as { key: string; session_id?: string };
1564
- if (!key) return reply.code(400).send({ error: "key required" });
1565
- try {
1566
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1567
- const { session } = await withSerializedStrictBrowseSession(
1568
- browseSessions,
1569
- browseClient,
1570
- requestedSessionId(req),
1571
- async (session) => {
1572
- await brokerForSession(session).press(session.tabId, key);
1573
- return true;
1574
- },
1575
- );
1576
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1577
- } catch (error) {
1578
- return sendBrowseSessionError(reply, error);
1579
- }
1580
- });
1581
-
1582
- // POST /v1/browse/select — select option by ref
1583
- app.post("/v1/browse/select", async (req, reply) => {
1584
- const { ref, value } = req.body as { ref: string; value: string; session_id?: string };
1585
- if (!ref || value === undefined) return reply.code(400).send({ error: "ref and value required" });
1586
- try {
1587
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1588
- const { session } = await withSerializedStrictBrowseSession(
1589
- browseSessions,
1590
- browseClient,
1591
- requestedSessionId(req),
1592
- async (session) => {
1593
- await brokerForSession(session).select(session.tabId, ref, value);
1594
- return true;
1595
- },
1596
- );
1597
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1598
- } catch (error) {
1599
- return sendBrowseSessionError(reply, error);
1600
- }
1601
- });
1602
-
1603
- // POST /v1/browse/scroll — scroll
1604
- app.post("/v1/browse/scroll", async (req, reply) => {
1605
- const { direction, amount } = (req.body as { direction?: string; amount?: number; session_id?: string }) ?? {};
1606
- try {
1607
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1608
- const { session } = await withSerializedStrictBrowseSession(
1609
- browseSessions,
1610
- browseClient,
1611
- requestedSessionId(req),
1612
- async (session) => {
1613
- await brokerForSession(session).scroll(session.tabId, (direction as any) ?? "down", amount);
1614
- return true;
1615
- },
1616
- );
1617
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1618
- } catch (error) {
1619
- return sendBrowseSessionError(reply, error);
1620
- }
1621
- });
1622
-
1623
- // GET /v1/browse/screenshot — capture screenshot
1624
- app.get("/v1/browse/screenshot", async (req, reply) => {
1625
- try {
1626
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1627
- const { session, result: data } = await withSerializedStrictBrowseSession(
1628
- browseSessions,
1629
- browseClient,
1630
- requestedSessionId(req),
1631
- async (session) => brokerForSession(session).screenshot(session.tabId),
1632
- );
1633
- return reply.send({ screenshot: data, session_id: session.sessionId, tab_id: session.tabId });
1634
- } catch (error) {
1635
- return sendBrowseSessionError(reply, error);
1636
- }
1637
- });
1638
-
1639
- // GET /v1/browse/text — page text
1640
- app.get("/v1/browse/text", async (req, reply) => {
1641
- try {
1642
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1643
- const { session, result: text } = await withSerializedStrictBrowseSession(
1644
- browseSessions,
1645
- browseClient,
1646
- requestedSessionId(req),
1647
- async (session) => brokerForSession(session).getText(session.tabId),
1648
- );
1649
- return reply.send({ text, session_id: session.sessionId, tab_id: session.tabId });
1650
- } catch (error) {
1651
- return sendBrowseSessionError(reply, error);
1652
- }
1653
- });
1654
-
1655
- // GET /v1/browse/markdown — page as markdown
1656
- app.get("/v1/browse/markdown", async (req, reply) => {
1657
- try {
1658
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1659
- const { session, result: markdown } = await withSerializedStrictBrowseSession(
1660
- browseSessions,
1661
- browseClient,
1662
- requestedSessionId(req),
1663
- async (session) => brokerForSession(session).getMarkdown(session.tabId),
1664
- );
1665
- return reply.send({ markdown, session_id: session.sessionId, tab_id: session.tabId });
1666
- } catch (error) {
1667
- return sendBrowseSessionError(reply, error);
1668
- }
1669
- });
1670
-
1671
- // GET /v1/browse/cookies — page cookies
1672
- app.get("/v1/browse/cookies", async (req, reply) => {
1673
- try {
1674
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1675
- const { session, result: cookies } = await withSerializedStrictBrowseSession(
1676
- browseSessions,
1677
- browseClient,
1678
- requestedSessionId(req),
1679
- async (session) => brokerForSession(session).getCookies(session.tabId),
1680
- );
1681
- return reply.send({ cookies, session_id: session.sessionId, tab_id: session.tabId });
1682
- } catch (error) {
1683
- return sendBrowseSessionError(reply, error);
1684
- }
1685
- });
1686
-
1687
- // POST /v1/browse/eval — evaluate JS
1688
- app.post("/v1/browse/eval", async (req, reply) => {
1689
- const { expression } = req.body as { expression: string; session_id?: string };
1690
- if (!expression) return reply.code(400).send({ error: "expression required" });
1691
- try {
1692
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1693
- const { session, result } = await withSerializedStrictBrowseSession(
1694
- browseSessions,
1695
- browseClient,
1696
- requestedSessionId(req),
1697
- async (session) => brokerForSession(session).evaluate(session.tabId, expression),
1698
- );
1699
- return reply.send({ result, session_id: session.sessionId, tab_id: session.tabId });
1700
- } catch (error) {
1701
- return sendBrowseSessionError(reply, error);
1702
- }
1703
- });
1704
-
1705
- // POST /v1/browse/back — navigate back
1706
- app.post("/v1/browse/back", async (req, reply) => {
1707
- try {
1708
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1709
- const { session } = await withSerializedStrictBrowseSession(
1710
- browseSessions,
1711
- browseClient,
1712
- requestedSessionId(req),
1713
- async (session) => {
1714
- await brokerForSession(session).goBack(session.tabId);
1715
- return true;
1716
- },
1717
- );
1718
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1719
- } catch (error) {
1720
- return sendBrowseSessionError(reply, error);
1721
- }
1722
- });
1723
-
1724
- // POST /v1/browse/forward — navigate forward
1725
- app.post("/v1/browse/forward", async (req, reply) => {
1726
- try {
1727
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1728
- const { session } = await withSerializedStrictBrowseSession(
1729
- browseSessions,
1730
- browseClient,
1731
- requestedSessionId(req),
1732
- async (session) => {
1733
- await brokerForSession(session).goForward(session.tabId);
1734
- return true;
1735
- },
1736
- );
1737
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1738
- } catch (error) {
1739
- return sendBrowseSessionError(reply, error);
1740
- }
1741
- });
1742
-
1743
- // POST /v1/browse/sync — checkpoint capture, keep the tab open, queue index+publish
1744
- app.post("/v1/browse/sync", async (req, reply) => {
1745
- try {
1746
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1747
- const { session, result: syncResult } = await withSerializedStrictBrowseSession(
1748
- browseSessions,
1749
- browseClient,
1750
- requestedSessionId(req),
1751
- async (session) => {
1752
- const syncResult = await flushBrowseCapture(session, { queueIndex: true, queuePublish: true });
1753
- await restartBrowseCapture(session);
1754
- return syncResult;
1755
- },
1756
- );
1757
-
1758
- return reply.send({
1759
- ok: true,
1760
- checkpointed: true,
1761
- session_id: session.sessionId,
1762
- tab_id: session.tabId,
1763
- indexed: syncResult.indexed,
1764
- mode: syncResult.mode,
1765
- domain: syncResult.domain,
1766
- skill_id: syncResult.skill_id,
1767
- endpoint_count: syncResult.endpoint_count,
1768
- endpoints: syncResult.endpoints,
1769
- request_count: syncResult.request_count,
1770
- pipeline: syncResult.pipeline,
1771
- publish_policy: syncResult.publish_policy,
1772
- background_publish_queued: syncResult.background_publish_queued,
1773
- next_step: buildCheckpointNextStep("sync", syncResult, session.sessionId),
1774
- });
1775
- } catch (error) {
1776
- return sendBrowseSessionError(reply, error);
1777
- }
1778
- });
1779
-
1780
- // POST /v1/browse/close — checkpoint capture, queue index+publish, save auth, close tab
1781
- app.post("/v1/browse/close", async (req, reply) => {
1782
- try {
1783
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1784
- const { session, result: syncResult } = await withSerializedStrictBrowseSession(
1785
- browseSessions,
1786
- browseClient,
1787
- requestedSessionId(req),
1788
- async (session) => {
1789
- const broker = brokerForSession(session);
1790
- if (session.domain) {
1791
- await saveAuthProfileBestEffort(session.tabId, session.domain, "browse_close");
1792
- }
1793
- const syncResult = await flushBrowseCapture(session, { queueIndex: true, queuePublish: true });
1794
- await broker.closeTab(session.tabId).catch(() => {});
1795
- removeBrowseSession(browseSessions, session.sessionId);
1796
- return syncResult;
1797
- },
1798
- );
1799
- return reply.send({
1800
- ok: true,
1801
- checkpointed: true,
1802
- session_id: session.sessionId,
1803
- indexed: syncResult.indexed,
1804
- mode: syncResult.mode,
1805
- endpoint_count: syncResult.endpoint_count,
1806
- request_count: syncResult.request_count,
1807
- pipeline: syncResult.pipeline,
1808
- publish_policy: syncResult.publish_policy,
1809
- background_publish_queued: syncResult.background_publish_queued,
1810
- auth_saved: session.domain || null,
1811
- next_step: buildCheckpointNextStep("close", syncResult, session.sessionId),
1812
- });
1813
- } catch (error) {
1814
- return sendBrowseSessionError(reply, error);
1815
- }
1816
- });
1817
- }
1818
-
1819
- function saveTrace(trace: unknown) {
1820
- if (!existsSync(TRACES_DIR)) mkdirSync(TRACES_DIR, { recursive: true });
1821
- const t = trace as { trace_id: string };
1822
- writeFileSync(join(TRACES_DIR, `${t.trace_id}.json`), JSON.stringify(trace, null, 2));
1823
- }