unbrowse 3.1.0 → 3.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/dist/cli.js +454 -96
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +695 -46
  4. package/dist/server.js +25994 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/manifest.json +1 -1
  7. package/runtime-src/agent-outcome.ts +0 -166
  8. package/runtime-src/analytics-session.ts +0 -55
  9. package/runtime-src/api/browse-index.ts +0 -317
  10. package/runtime-src/api/browse-session.ts +0 -572
  11. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  12. package/runtime-src/api/browse-submit.ts +0 -1184
  13. package/runtime-src/api/routes.ts +0 -1823
  14. package/runtime-src/auth/browser-cookies.ts +0 -423
  15. package/runtime-src/auth/index.ts +0 -535
  16. package/runtime-src/auth/runtime.ts +0 -116
  17. package/runtime-src/browser/index.ts +0 -659
  18. package/runtime-src/browser/types.ts +0 -41
  19. package/runtime-src/build-info.generated.ts +0 -6
  20. package/runtime-src/capture/index.ts +0 -1794
  21. package/runtime-src/capture/prefetch.ts +0 -95
  22. package/runtime-src/capture/rsc.ts +0 -45
  23. package/runtime-src/cli/shortcuts.ts +0 -273
  24. package/runtime-src/cli.ts +0 -1572
  25. package/runtime-src/client/graph-client.ts +0 -100
  26. package/runtime-src/client/index.ts +0 -1425
  27. package/runtime-src/debug-trace.ts +0 -18
  28. package/runtime-src/domain.ts +0 -38
  29. package/runtime-src/execution/index.ts +0 -3397
  30. package/runtime-src/execution/retry.ts +0 -46
  31. package/runtime-src/execution/robots.ts +0 -167
  32. package/runtime-src/execution/search-forms.ts +0 -188
  33. package/runtime-src/extraction/index.ts +0 -1507
  34. package/runtime-src/foundry/publish-bundle.ts +0 -392
  35. package/runtime-src/graph/agent-augment.ts +0 -315
  36. package/runtime-src/graph/index.ts +0 -1524
  37. package/runtime-src/graph/local-fixtures.ts +0 -393
  38. package/runtime-src/graph/local-harness.ts +0 -646
  39. package/runtime-src/graph/planner.ts +0 -411
  40. package/runtime-src/graph/session.ts +0 -294
  41. package/runtime-src/graph/trace-store.ts +0 -136
  42. package/runtime-src/index.ts +0 -24
  43. package/runtime-src/indexer/index.ts +0 -465
  44. package/runtime-src/intent-match.ts +0 -1515
  45. package/runtime-src/kuri/client.ts +0 -1839
  46. package/runtime-src/logger.ts +0 -30
  47. package/runtime-src/marketplace/index.ts +0 -103
  48. package/runtime-src/mcp.ts +0 -1747
  49. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  50. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  51. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  52. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  53. package/runtime-src/orchestrator/index.ts +0 -4480
  54. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  55. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  56. package/runtime-src/payments/cascade.ts +0 -137
  57. package/runtime-src/payments/index.ts +0 -270
  58. package/runtime-src/payments/lobster-pay.ts +0 -182
  59. package/runtime-src/payments/wallet.ts +0 -98
  60. package/runtime-src/publish/review-context.ts +0 -93
  61. package/runtime-src/publish/sanitize.ts +0 -197
  62. package/runtime-src/publish/schema-review.ts +0 -192
  63. package/runtime-src/publish-admission.ts +0 -388
  64. package/runtime-src/ratelimit/index.ts +0 -23
  65. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  66. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  67. package/runtime-src/reverse-engineer/index.ts +0 -1551
  68. package/runtime-src/router.ts +0 -17
  69. package/runtime-src/routing-telemetry.ts +0 -395
  70. package/runtime-src/runtime/browser-access.ts +0 -11
  71. package/runtime-src/runtime/browser-auth.ts +0 -12
  72. package/runtime-src/runtime/browser-host.ts +0 -48
  73. package/runtime-src/runtime/lifecycle.ts +0 -17
  74. package/runtime-src/runtime/local-server.ts +0 -311
  75. package/runtime-src/runtime/paths.ts +0 -99
  76. package/runtime-src/runtime/setup.ts +0 -251
  77. package/runtime-src/runtime/supervisor.ts +0 -69
  78. package/runtime-src/runtime/update-hints.ts +0 -351
  79. package/runtime-src/server.ts +0 -100
  80. package/runtime-src/session-logs.ts +0 -142
  81. package/runtime-src/settings.ts +0 -221
  82. package/runtime-src/single-binary.ts +0 -143
  83. package/runtime-src/site-policy.ts +0 -54
  84. package/runtime-src/stale-cleanup-runner.ts +0 -144
  85. package/runtime-src/stale-cleanup.ts +0 -133
  86. package/runtime-src/telemetry-attribution.ts +0 -120
  87. package/runtime-src/telemetry.ts +0 -253
  88. package/runtime-src/template-params.ts +0 -141
  89. package/runtime-src/transform/drift.ts +0 -60
  90. package/runtime-src/transform/index.ts +0 -277
  91. package/runtime-src/types/index.ts +0 -1
  92. package/runtime-src/types/skill.ts +0 -912
  93. package/runtime-src/vault/index.ts +0 -196
  94. package/runtime-src/verification/auth-gate.ts +0 -8
  95. package/runtime-src/verification/candidates.ts +0 -27
  96. package/runtime-src/verification/index.ts +0 -120
  97. package/runtime-src/verification/matrix.ts +0 -30
  98. package/runtime-src/version.ts +0 -148
  99. package/runtime-src/workflow/artifact.ts +0 -161
  100. package/runtime-src/workflow/compile.ts +0 -808
  101. package/runtime-src/workflow/publish.ts +0 -225
  102. package/runtime-src/workflow/runtime.ts +0 -213
@@ -1,1823 +0,0 @@
1
- import type { FastifyInstance } from "fastify";
2
- import * as kuri from "../kuri/client.js";
3
- import type { KuriHarEntry } from "../kuri/client.js";
4
- import { extractEndpoints, extractAuthHeaders } from "../reverse-engineer/index.js";
5
- import { INTERCEPTOR_SCRIPT, enrichPassiveCaptureRequests, injectInterceptor } from "../capture/index.js";
6
- import { indexSkillLocally, mergeAgentReview, publishIndexedSkill, queueBackgroundIndex } from "../indexer/index.js";
7
- import { nanoid } from "nanoid";
8
- import type { ExecutionTrace, OrchestrationTiming, ProjectionOptions, SkillManifest } from "../types/index.js";
9
- import { mergeEndpoints } from "../marketplace/index.js";
10
- import { buildSkillOperationGraph, getEndpointDescriptionMetadata, getSkillChunk, toAgentSkillChunkView } from "../graph/index.js";
11
- import { augmentEndpointsWithAgent } from "../graph/agent-augment.js";
12
- import { findExistingSkillForDomain, cachePublishedSkill } from "../client/index.js";
13
- import { storeCredential } from "../vault/index.js";
14
- import { generateLocalDescription, writeSkillSnapshot, buildResolveCacheKey, getDomainReuseKey, domainSkillCache, persistDomainCache, scopedCacheKey, snapshotPathForCacheKey, invalidateRouteCacheForDomain, summarizeSchema, extractSampleValues } from "../orchestrator/index.js";
15
- import { TRACE_VERSION, CODE_HASH, DEFAULT_BACKEND_URL, GIT_SHA, PACKAGE_VERSION } from "../version.js";
16
- import { promoteExplicitExecution, resolveAndExecute, type OrchestratorResult } from "../orchestrator/index.js";
17
- import { getSkill } from "../marketplace/index.js";
18
- import { executeSkill, rankEndpoints } from "../execution/index.js";
19
- import {
20
- extractBrowserAuth,
21
- importBrowserCookiesIntoTab,
22
- loginWithBrowserFallback,
23
- loadAuthProfileBestEffort,
24
- saveAuthProfileBestEffort,
25
- } from "../auth/index.js";
26
- import { recordFeedback, recordDiagnostics, recordExecution, getApiKey, getRecentLocalSkill, recordAnalyticsSession, type AnalyticsSessionPayload } from "../client/index.js";
27
- import { ROUTE_LIMITS } from "../ratelimit/index.js";
28
- import { listRecentSessionsForDomain } from "../session-logs.js";
29
- import { attachAgentOutcomeHints } from "../agent-outcome.js";
30
- import { writeFileSync, existsSync, mkdirSync } from "fs";
31
- import { join } from "path";
32
- import {
33
- BrowseSessionError,
34
- createRegisteredBrowseSession,
35
- extractBrowseFailureMessage,
36
- getOrCreateNavigateBrowseSession,
37
- isBrowseSessionLive,
38
- isRecoverableBrowseFailure,
39
- type BrowseSession,
40
- withSerializedStrictBrowseSession,
41
- removeBrowseSession,
42
- } from "./browse-session.js";
43
- import { cacheBrowseRequests, harEntriesToRawRequests } from "./browse-index.js";
44
- import { isUrlWaitHint, resolveSubmitWaitHint, submitBrowseForm } from "./browse-submit.js";
45
- import { cleanupStaleSkills } from "../stale-cleanup-runner.js";
46
- import {
47
- decideCheckpointPublish,
48
- decideExplicitPublish,
49
- getCapturePipelineSettings,
50
- updateCapturePipelineSettings,
51
- } from "../settings.js";
52
- import { publishFoundryBundle } from "../foundry/publish-bundle.js";
53
- import { buildEndpointReviewContext } from "../publish/review-context.js";
54
- import { applyWorkflowSchemaReviews } from "../publish/schema-review.js";
55
- import { readWorkflowArtifact, writeWorkflowArtifact } from "../workflow/artifact.js";
56
-
57
- const BETA_API_URL = process.env.UNBROWSE_BACKEND_URL || DEFAULT_BACKEND_URL;
58
-
59
- const TRACES_DIR = process.env.TRACES_DIR ?? join(process.cwd(), "traces");
60
- const BROWSE_BROKER_MAX = Math.max(1, Number(process.env.KURI_MULTI_BROKER_MAX ?? "2"));
61
- const BROWSE_BROKER_BASE_PORT = Number(process.env.KURI_PORT ?? "7700");
62
-
63
- type AnalyticsSessionResult = {
64
- trace: Pick<ExecutionTrace, "trace_id" | "started_at" | "completed_at" | "endpoint_id" | "trace_version" | "success" | "tokens_saved" | "tokens_saved_pct" | "api_call_count">;
65
- timing?: Pick<OrchestrationTiming, "source" | "time_saved_ms" | "time_saved_pct" | "cost_saved_uc" | "tokens_saved" | "tokens_saved_pct">;
66
- source?: OrchestratorResult["source"];
67
- };
68
-
69
- export function buildAnalyticsSessionPayload(
70
- result: AnalyticsSessionResult,
71
- opts: {
72
- browser_mode?: AnalyticsSessionPayload["browser_mode"];
73
- discovery_queries: number;
74
- cached_skill_calls?: number;
75
- fresh_index_calls?: number;
76
- },
77
- ): AnalyticsSessionPayload {
78
- const source = result.timing?.source ?? result.source;
79
- const apiCalls = result.trace.api_call_count ?? (result.trace.endpoint_id ? 1 : 0);
80
- const browserMode = opts.browser_mode ?? (
81
- source === "live-capture" || source === "first-pass" || source === "browser-action"
82
- ? "default"
83
- : "replaced"
84
- );
85
- const cachedSkillCalls = opts.cached_skill_calls ?? (
86
- apiCalls > 0 && source !== "live-capture" && source !== "first-pass" ? 1 : 0
87
- );
88
- const freshIndexCalls = opts.fresh_index_calls ?? (
89
- apiCalls > 0 && (source === "live-capture" || source === "first-pass") ? 1 : 0
90
- );
91
-
92
- return {
93
- session_id: result.trace.trace_id,
94
- started_at: result.trace.started_at,
95
- completed_at: result.trace.completed_at,
96
- trace_version: result.trace.trace_version ?? TRACE_VERSION,
97
- api_calls: apiCalls,
98
- discovery_queries: opts.discovery_queries,
99
- cached_skill_calls: cachedSkillCalls,
100
- fresh_index_calls: freshIndexCalls,
101
- browser_mode: browserMode,
102
- success: result.trace.success ?? true,
103
- source,
104
- time_saved_ms: result.timing?.time_saved_ms,
105
- time_saved_pct: result.timing?.time_saved_pct,
106
- tokens_saved: result.trace.tokens_saved ?? result.timing?.tokens_saved,
107
- tokens_saved_pct: result.trace.tokens_saved_pct ?? result.timing?.tokens_saved_pct,
108
- cost_saved_uc: result.timing?.cost_saved_uc,
109
- };
110
- }
111
-
112
-
113
- /** Process HAR entries into routes and queue local index, with remote share opt-in only. */
114
- function passiveIndexFromRequests(
115
- requests: RawRequest[],
116
- pageUrl: string,
117
- options: { publishAfterIndex?: boolean } = {},
118
- ): void {
119
- if (requests.length === 0) return;
120
-
121
- let domain: string;
122
- try { domain = new URL(pageUrl).hostname; } catch { return; }
123
- const intent = `browse ${domain}`;
124
-
125
- // Fire-and-forget — full pipeline runs async
126
- void (async () => {
127
- try {
128
- // 1. Extract endpoints from captured traffic
129
- const rawEndpoints = extractEndpoints(requests, undefined, { pageUrl, finalUrl: pageUrl });
130
- if (rawEndpoints.length === 0) {
131
- console.error(`[passive-index] ${domain}: 0 endpoints from ${requests.length} requests`);
132
- return;
133
- }
134
-
135
- // 2. Extract and store auth credentials (cookies + sensitive headers)
136
- const capturedAuthHeaders = extractAuthHeaders(requests);
137
- if (Object.keys(capturedAuthHeaders).length > 0) {
138
- const authKey = `${domain}-session`;
139
- await storeCredential(authKey, JSON.stringify({ headers: capturedAuthHeaders }));
140
- }
141
-
142
- // 3. Merge with existing skill for this domain (never reduce endpoint count)
143
- const existingSkill = findExistingSkillForDomain(domain, intent);
144
- const mergedEndpoints = existingSkill
145
- ? mergeEndpoints(existingSkill.endpoints, rawEndpoints)
146
- : rawEndpoints;
147
- // Guard: if passive capture found fewer endpoints than what exists, keep the richer set
148
- if (existingSkill && mergedEndpoints.length < existingSkill.endpoints.length) {
149
- console.error(`[passive-index] ${domain}: skipping — would reduce ${existingSkill.endpoints.length} → ${mergedEndpoints.length} endpoints`);
150
- return;
151
- }
152
-
153
- // 4. Generate descriptions for endpoints without them (enables BM25 ranking)
154
- for (const ep of mergedEndpoints) {
155
- if (!ep.description) {
156
- ep.description = generateLocalDescription(ep);
157
- }
158
- }
159
-
160
- // 5. Skip LLM-based augmentation — the calling agent IS the LLM.
161
- // Endpoint descriptions come from generateLocalDescription (heuristic).
162
- // The agent reviews endpoints in the deferral response and picks the right one.
163
- const enrichedEndpoints = mergedEndpoints;
164
-
165
- // 6. Build operation dependency graph
166
- const operationGraph = buildSkillOperationGraph(enrichedEndpoints);
167
-
168
- // 7. Assemble full skill manifest
169
- const skill: SkillManifest = {
170
- skill_id: existingSkill?.skill_id ?? nanoid(),
171
- version: "1.0.0",
172
- schema_version: "1",
173
- lifecycle: "active" as const,
174
- execution_type: "http" as const,
175
- created_at: existingSkill?.created_at ?? new Date().toISOString(),
176
- updated_at: new Date().toISOString(),
177
- name: domain,
178
- intent_signature: intent,
179
- domain,
180
- description: `API skill for ${domain}`,
181
- owner_type: "agent" as const,
182
- endpoints: enrichedEndpoints,
183
- operation_graph: operationGraph,
184
- intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
185
- };
186
-
187
- // 8. Cache locally for immediate reuse — write to BOTH the published skill cache
188
- // AND the domain skill snapshot so resolveAndExecute finds it on next call
189
- try { cachePublishedSkill(skill); } catch { /* best-effort */ }
190
-
191
- // Write domain skill snapshot (keyed by resolve cache key)
192
- const bgCacheKey = buildResolveCacheKey(domain, intent, pageUrl);
193
- const bgScopedKey = scopedCacheKey("global", bgCacheKey);
194
- writeSkillSnapshot(bgScopedKey, skill);
195
-
196
- // Update domain-level reuse cache
197
- const bgDomainKey = getDomainReuseKey(pageUrl ?? domain);
198
- if (bgDomainKey) {
199
- domainSkillCache.set(bgDomainKey, {
200
- skillId: skill.skill_id,
201
- localSkillPath: snapshotPathForCacheKey(bgScopedKey),
202
- ts: Date.now(),
203
- });
204
- persistDomainCache();
205
- }
206
-
207
- // 9. Queue local index, and only remote-share when the caller explicitly asked for it.
208
- const cacheKey = `passive:${domain}:${Date.now()}`;
209
- queueBackgroundIndex({
210
- skill,
211
- domain,
212
- intent,
213
- contextUrl: pageUrl,
214
- cacheKey,
215
- publishAfterIndex: options.publishAfterIndex === true,
216
- });
217
-
218
- console.error(`[passive-index] ${domain}: ${enrichedEndpoints.length} endpoints indexed from ${requests.length} requests`);
219
- } catch (err) {
220
- console.error(`[passive-index] ${domain} failed: ${err instanceof Error ? err.message : err}`);
221
- }
222
- })();
223
- }
224
-
225
- /** Convenience wrapper: convert HAR entries and run passive indexing */
226
- function passiveIndexHar(
227
- entries: KuriHarEntry[],
228
- pageUrl: string,
229
- options: { publishAfterIndex?: boolean } = {},
230
- ): void {
231
- passiveIndexFromRequests(harEntriesToRawRequests(entries), pageUrl, options);
232
- }
233
- // ── Browse session state (module-level so orchestrator can register sessions) ──
234
- const browseSessions = new Map<string, BrowseSession>();
235
-
236
- function browseBrokerPorts(): number[] {
237
- return Array.from({ length: BROWSE_BROKER_MAX }, (_, index) => BROWSE_BROKER_BASE_PORT + index);
238
- }
239
-
240
- function brokerForSession(session: BrowseSession | undefined): kuri.KuriClient {
241
- if (session?.client) return session.client as kuri.KuriClient;
242
- if (session?.brokerPort !== undefined) return kuri.getKuriClient(session.brokerPort);
243
- return kuri.getKuriClient();
244
- }
245
-
246
- function selectBrowseBrokerClient(requestedSessionId?: string): kuri.KuriClient {
247
- if (requestedSessionId) {
248
- const existing = browseSessions.get(requestedSessionId);
249
- if (existing?.client) return existing.client as kuri.KuriClient;
250
- if (existing) return brokerForSession(existing);
251
- }
252
-
253
- const loads = new Map<number, number>(browseBrokerPorts().map((port) => [port, 0]));
254
- for (const session of browseSessions.values()) {
255
- const port = session.brokerPort ?? BROWSE_BROKER_BASE_PORT;
256
- loads.set(port, (loads.get(port) ?? 0) + 1);
257
- }
258
- const [selectedPort] = [...loads.entries()].sort((a, b) => a[1] - b[1] || a[0] - b[0])[0] ?? [BROWSE_BROKER_BASE_PORT, 0];
259
- return kuri.getKuriClient(selectedPort);
260
- }
261
-
262
- async function loadSkillForMutation(skillId: string, clientScope?: string): Promise<SkillManifest | null> {
263
- let skill = getRecentLocalSkill(skillId, clientScope);
264
- if (!skill) {
265
- for (const [, entry] of domainSkillCache) {
266
- if (entry.skillId === skillId && entry.localSkillPath) {
267
- try { skill = JSON.parse(require("fs").readFileSync(entry.localSkillPath, "utf-8")); } catch {}
268
- break;
269
- }
270
- }
271
- }
272
- if (!skill) skill = await getSkill(skillId, clientScope);
273
- return skill;
274
- }
275
-
276
- function buildSkillIndexJob(skill: SkillManifest, clientScope?: string): {
277
- skill: SkillManifest;
278
- domain: string;
279
- intent: string;
280
- clientScope?: string;
281
- cacheKey: string;
282
- } {
283
- const intent = skill.intent_signature || `browse ${skill.domain}`;
284
- return {
285
- skill,
286
- domain: skill.domain,
287
- intent,
288
- clientScope,
289
- cacheKey: buildResolveCacheKey(skill.domain, intent, undefined),
290
- };
291
- }
292
-
293
- /** Register a browse session from the orchestrator (Phase 4 handoff) */
294
- export function registerBrowseSession(tabId: string, url: string, domain: string): BrowseSession {
295
- const client = kuri.getKuriClient();
296
- return createRegisteredBrowseSession(browseSessions, {
297
- tabId,
298
- url,
299
- harActive: true,
300
- domain,
301
- brokerPort: client.getPort(),
302
- client,
303
- });
304
- }
305
-
306
- // ── /v1/stats cache ──────────────────────────────────────────────────
307
- let statsCache: { data: unknown; ts: number } | null = null;
308
- const STATS_CACHE_TTL = 5 * 60 * 1000; // 5 minutes
309
-
310
- async function fetchStats() {
311
- if (statsCache && Date.now() - statsCache.ts < STATS_CACHE_TTL) {
312
- return statsCache.data;
313
- }
314
-
315
- const npmPoint = (pkg: string, range: string) =>
316
- fetch(`https://api.npmjs.org/downloads/point/${range}/${pkg}`)
317
- .then(r => r.json() as Promise<{ downloads?: number }>);
318
-
319
- const npmRange = (pkg: string) =>
320
- fetch(`https://api.npmjs.org/downloads/range/last-month/${pkg}`)
321
- .then(r => r.json() as Promise<{ downloads?: Array<{ day: string; downloads: number }> }>);
322
-
323
- const externalCalls: Promise<unknown>[] = [
324
- npmPoint("unbrowse", "last-month"),
325
- npmPoint("unbrowse-openclaw", "last-month"),
326
- npmPoint("unbrowse", "1970-01-01:2099-12-31"),
327
- npmPoint("unbrowse-openclaw", "1970-01-01:2099-12-31"),
328
- npmRange("unbrowse"),
329
- npmRange("unbrowse-openclaw"),
330
- fetch("https://api.github.com/repos/anthropic-ai/unbrowse", {
331
- headers: { "User-Agent": "unbrowse-stats" },
332
- }).then(r => r.json() as Promise<Record<string, unknown>>),
333
- ];
334
-
335
- // Only call Unkey analytics if the key is available as an env var
336
- const unkeyAnalyticsKey = process.env.UNKEY_ANALYTICS_KEY;
337
- if (unkeyAnalyticsKey) {
338
- externalCalls.push(
339
- fetch("https://api.unkey.com/v2/analytics.getVerifications", {
340
- method: "POST",
341
- headers: {
342
- Authorization: `Bearer ${unkeyAnalyticsKey}`,
343
- "Content-Type": "application/json",
344
- },
345
- body: JSON.stringify({ apiId: "api_2bUScBc8U6JNsXLrhfHwfqzXHJDi" }),
346
- }).then(r => r.json() as Promise<unknown>),
347
- );
348
- }
349
-
350
- const [
351
- unbrowse30d, plugin30d,
352
- unbrowseAll, pluginAll,
353
- unbrowseDaily, pluginDaily,
354
- github,
355
- ...rest
356
- ] = await Promise.allSettled(externalCalls);
357
- const unkey = rest[0]; // may be undefined if no key
358
-
359
- const val = <T>(r: PromiseSettledResult<T> | undefined): T | null =>
360
- r?.status === "fulfilled" ? r.value : null;
361
-
362
- // npm numbers
363
- const u30 = val(unbrowse30d)?.downloads ?? null;
364
- const p30 = val(plugin30d)?.downloads ?? null;
365
- const uAll = val(unbrowseAll)?.downloads ?? null;
366
- const pAll = val(pluginAll)?.downloads ?? null;
367
-
368
- // daily breakdown — merge the two packages by day
369
- const uDays = val(unbrowseDaily)?.downloads ?? [];
370
- const pDays = val(pluginDaily)?.downloads ?? [];
371
- const dayMap = new Map<string, { unbrowse: number; plugin: number }>();
372
- for (const d of uDays) dayMap.set(d.day, { unbrowse: d.downloads, plugin: 0 });
373
- for (const d of pDays) {
374
- const entry = dayMap.get(d.day);
375
- if (entry) entry.plugin = d.downloads;
376
- else dayMap.set(d.day, { unbrowse: 0, plugin: d.downloads });
377
- }
378
- const daily = [...dayMap.entries()]
379
- .sort(([a], [b]) => a.localeCompare(b))
380
- .map(([day, v]) => ({ day, unbrowse: v.unbrowse, plugin: v.plugin, total: v.unbrowse + v.plugin }));
381
-
382
- // github
383
- const gh = val(github);
384
- const githubData = gh && typeof gh === "object"
385
- ? {
386
- stars: (gh as Record<string, number>).stargazers_count ?? null,
387
- forks: (gh as Record<string, number>).forks_count ?? null,
388
- open_issues: (gh as Record<string, number>).open_issues_count ?? null,
389
- watchers: (gh as Record<string, number>).watchers_count ?? null,
390
- }
391
- : { stars: null, forks: null, open_issues: null, watchers: null };
392
-
393
- // unkey
394
- let agentsData: { total_api_calls_30d: number | null; note?: string } = {
395
- total_api_calls_30d: null,
396
- note: "unkey analytics unavailable",
397
- };
398
- const uk = val(unkey);
399
- if (uk && Array.isArray(uk)) {
400
- const total = (uk as Array<{ total?: number }>).reduce((s, v) => s + (v.total ?? 0), 0);
401
- agentsData = { total_api_calls_30d: total };
402
- } else if (uk && typeof uk === "object" && (uk as Record<string, unknown>).total != null) {
403
- agentsData = { total_api_calls_30d: (uk as Record<string, number>).total };
404
- }
405
-
406
- const data = {
407
- npm: {
408
- unbrowse: { last_30d: u30, all_time: uAll },
409
- openclaw_plugin: { last_30d: p30, all_time: pAll },
410
- combined: {
411
- last_30d: u30 != null && p30 != null ? u30 + p30 : (u30 ?? p30),
412
- all_time: uAll != null && pAll != null ? uAll + pAll : (uAll ?? pAll),
413
- },
414
- daily,
415
- },
416
- github: githubData,
417
- agents: agentsData,
418
- fetched_at: new Date().toISOString(),
419
- };
420
-
421
- statsCache = { data, ts: Date.now() };
422
- return data;
423
- }
424
-
425
- export async function registerRoutes(app: FastifyInstance) {
426
- const clientScopeFor = (req: { headers: Record<string, unknown>; id: string }) =>
427
- (typeof req.headers["x-unbrowse-client-id"] === "string" && req.headers["x-unbrowse-client-id"].trim())
428
- ? req.headers["x-unbrowse-client-id"].trim()
429
- : req.id;
430
-
431
- function checkpointPublishCommand(skillId: string | null, confirmPublish = false): string {
432
- return skillId
433
- ? `unbrowse publish --skill ${skillId}${confirmPublish ? " --confirm-publish" : ""}`
434
- : `unbrowse publish --skill <skill_id>${confirmPublish ? " --confirm-publish" : ""}`;
435
- }
436
-
437
- function buildPublishFailureNextStep(skillId: string, validationErrors?: string[]): string {
438
- const reviewErrors = (validationErrors ?? []).filter((error) => error.startsWith("review_required:"));
439
- if (reviewErrors.length > 0) {
440
- return `Remote share blocked: ${reviewErrors.length} endpoint(s) still need review. Re-run ${checkpointPublishCommand(skillId)} to inspect review_context, then publish again with reviewed endpoints.`;
441
- }
442
- return `Remote share did not complete. Inspect validation_errors, adjust the contract locally, then retry ${checkpointPublishCommand(skillId, true)}.`;
443
- }
444
-
445
- function buildCheckpointNextStep(
446
- action: "sync" | "close",
447
- result: {
448
- skill_id: string | null;
449
- pipeline: {
450
- index_queued: boolean;
451
- publish_queued: boolean;
452
- };
453
- publish_policy: {
454
- mode: "auto" | "disabled" | "blacklisted" | "prompt";
455
- reason: string;
456
- matched_domain?: string;
457
- };
458
- },
459
- sessionId?: string,
460
- ): string {
461
- const sessionHint = sessionId ? ` --session ${sessionId}` : "";
462
- const publishCommand = checkpointPublishCommand(
463
- result.skill_id,
464
- result.publish_policy.mode === "blacklisted" || result.publish_policy.mode === "prompt",
465
- );
466
-
467
- if (!result.pipeline.index_queued) {
468
- return action === "sync"
469
- ? `Checkpoint recorded, but no new capture was available to index. Continue browsing, then run \`unbrowse close${sessionHint}\` for the final checkpoint.`
470
- : "Final checkpoint recorded, but no new capture was available to index or publish.";
471
- }
472
-
473
- if (result.publish_policy.mode === "auto") {
474
- return action === "sync"
475
- ? `Checkpoint saved. Background index + publish queued. Continue browsing, then run \`unbrowse close${sessionHint}\` for the final checkpoint.`
476
- : "Final checkpoint saved. Background index + publish queued. Inspect the indexed contract or wait for publish to complete.";
477
- }
478
-
479
- const base = result.publish_policy.mode === "disabled"
480
- ? "Checkpoint saved. Local index queued, but auto-publish is disabled in settings."
481
- : `Checkpoint saved. Local index queued, but auto-publish did not run: ${result.publish_policy.reason}`;
482
-
483
- const suffix = result.skill_id
484
- ? ` Review the indexed contract, then run \`${publishCommand}\` only if you own this index.`
485
- : "";
486
-
487
- if (action === "sync") {
488
- return `${base} Continue browsing, then run \`unbrowse close${sessionHint}\` when done.${suffix}`;
489
- }
490
- return `${base}${suffix}`;
491
- }
492
-
493
- // Auth gate: block all routes except /health when no API key is configured
494
- app.addHook("onRequest", async (req, reply) => {
495
- if (req.url === "/health" || req.url === "/v1/stats" || req.url.startsWith("/v1/settings")) return;
496
-
497
- const key = getApiKey();
498
- if (!key) {
499
- return reply.code(401).send({
500
- error: "api_key_required",
501
- message: "No API key configured. Restart the server to auto-register, or run: bash scripts/setup.sh",
502
- docs_url: "https://unbrowse.ai",
503
- });
504
- }
505
- });
506
-
507
- app.get("/v1/settings", async (_req, reply) => {
508
- return reply.send({
509
- capture_pipeline: getCapturePipelineSettings(),
510
- });
511
- });
512
-
513
- app.post("/v1/settings", async (req, reply) => {
514
- const body = (req.body ?? {}) as {
515
- auto_publish_checkpoints?: boolean;
516
- publish_domain_blacklist?: string[];
517
- publish_domain_promptlist?: string[];
518
- clear_publish_domain_blacklist?: boolean;
519
- clear_publish_domain_promptlist?: boolean;
520
- };
521
-
522
- const settings = updateCapturePipelineSettings({
523
- auto_publish_checkpoints: typeof body.auto_publish_checkpoints === "boolean"
524
- ? body.auto_publish_checkpoints
525
- : undefined,
526
- publish_domain_blacklist: Array.isArray(body.publish_domain_blacklist)
527
- ? body.publish_domain_blacklist
528
- : undefined,
529
- publish_domain_promptlist: Array.isArray(body.publish_domain_promptlist)
530
- ? body.publish_domain_promptlist
531
- : undefined,
532
- clear_publish_domain_blacklist: body.clear_publish_domain_blacklist === true,
533
- clear_publish_domain_promptlist: body.clear_publish_domain_promptlist === true,
534
- });
535
-
536
- return reply.send({
537
- ok: true,
538
- capture_pipeline: settings,
539
- next_step: settings.auto_publish_checkpoints
540
- ? "Auto-publish after sync/close is enabled unless a domain rule blocks it."
541
- : "Auto-publish after sync/close is disabled. Use index for local recompute and publish only when you explicitly want remote share.",
542
- });
543
- });
544
-
545
- // POST /v1/intent/resolve
546
- app.post("/v1/intent/resolve", { config: { rateLimit: ROUTE_LIMITS["/v1/intent/resolve"] } }, async (req, reply) => {
547
- const clientScope = clientScopeFor(req);
548
- const { intent, params, context, projection, confirm_unsafe, confirm_third_party_terms, dry_run, force_capture } = req.body as {
549
- intent: string;
550
- params?: Record<string, unknown>;
551
- context?: { url?: string; domain?: string };
552
- projection?: ProjectionOptions;
553
- confirm_unsafe?: boolean;
554
- confirm_third_party_terms?: boolean;
555
- dry_run?: boolean;
556
- force_capture?: boolean;
557
- };
558
- if (!intent) return reply.code(400).send({ error: "intent required" });
559
- try {
560
- const result = await resolveAndExecute(intent, params ?? {}, context, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, force_capture, client_scope: clientScope });
561
-
562
- // Surface timing breakdown
563
- const res = attachAgentOutcomeHints({ ...result } as Record<string, unknown>, {
564
- skill: result.skill,
565
- endpointId: result.trace.endpoint_id,
566
- timing: result.timing,
567
- });
568
- if (result.timing) {
569
- res.timing = result.timing;
570
- }
571
-
572
- // If the orchestrator already included available_endpoints in result (deferral),
573
- // also append them at the top level for backward compatibility.
574
- const innerResult = result.result as Record<string, unknown> | null;
575
- if (innerResult?.available_endpoints && !res.available_endpoints) {
576
- res.available_endpoints = innerResult.available_endpoints;
577
- }
578
-
579
- await recordAnalyticsSession(buildAnalyticsSessionPayload(result, {
580
- discovery_queries: 1,
581
- })).catch(() => {});
582
-
583
- return reply.send(res);
584
- } catch (err) {
585
- return reply.code(500).send({ error: (err as Error).message });
586
- }
587
- });
588
-
589
- // GET /v1/skills/:skill_id — local route so skill lookups hit disk cache before proxying to backend
590
- app.get("/v1/skills/:skill_id", async (req, reply) => {
591
- const clientScope = clientScopeFor(req);
592
- const { skill_id } = req.params as { skill_id: string };
593
- // Check local caches: recent skills → domain snapshots → marketplace
594
- let skill = getRecentLocalSkill(skill_id, clientScope);
595
- if (!skill) {
596
- for (const [, entry] of domainSkillCache) {
597
- if (entry.skillId === skill_id && entry.localSkillPath) {
598
- try { skill = JSON.parse(require("fs").readFileSync(entry.localSkillPath, "utf-8")); } catch {}
599
- break;
600
- }
601
- }
602
- }
603
- if (!skill) skill = await getSkill(skill_id, clientScope);
604
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
605
- return reply.send(skill);
606
- });
607
-
608
- // POST /v1/skills/:skill_id/review — agent submits reviewed descriptions + synthetic examples
609
- app.post("/v1/skills/:skill_id/review", async (req, reply) => {
610
- const clientScope = clientScopeFor(req);
611
- const { skill_id } = req.params as { skill_id: string };
612
- const { endpoints: reviews } = req.body as {
613
- endpoints: import("../types/index.js").EndpointReviewPayload[];
614
- };
615
- if (!reviews?.length) return reply.code(400).send({ error: "endpoints[] required" });
616
-
617
- let skill = await loadSkillForMutation(skill_id, clientScope);
618
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
619
-
620
- const updated = mergeAgentReview(skill.endpoints, reviews);
621
- skill.endpoints = updated;
622
- skill.updated_at = new Date().toISOString();
623
- const updatedWorkflowArtifact = applyWorkflowSchemaReviews(readWorkflowArtifact(skill.skill_id), reviews);
624
- if (updatedWorkflowArtifact) writeWorkflowArtifact(updatedWorkflowArtifact);
625
-
626
- const indexed = await indexSkillLocally(buildSkillIndexJob(skill, clientScope));
627
- return reply.send({
628
- ok: true,
629
- skill_id: indexed.skill.skill_id,
630
- endpoints_updated: reviews.length,
631
- indexed: true,
632
- publish_status: "indexed",
633
- endpoint_count: indexed.skill.endpoints.length,
634
- });
635
- });
636
-
637
- // POST /v1/skills/:skill_id/index — local-only graph/export recompute from cached state
638
- app.post("/v1/skills/:skill_id/index", async (req, reply) => {
639
- const clientScope = clientScopeFor(req);
640
- const { skill_id } = req.params as { skill_id: string };
641
- const skill = await loadSkillForMutation(skill_id, clientScope);
642
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
643
-
644
- const indexed = await indexSkillLocally(buildSkillIndexJob(skill, clientScope));
645
- return reply.send({
646
- ok: true,
647
- skill_id: indexed.skill.skill_id,
648
- indexed: true,
649
- publish_status: "indexed",
650
- endpoint_count: indexed.skill.endpoints.length,
651
- domain: indexed.domain,
652
- next_step: `Local contracts re-indexed. Review them, then run ${checkpointPublishCommand(indexed.skill.skill_id)} when you explicitly want remote share.`,
653
- });
654
- });
655
-
656
- // POST /v1/skills/:skill_id/publish — two-phase agent-driven publish
657
- // Phase 1 (no endpoints body): re-index locally, then return endpoints needing descriptions
658
- // Phase 2 (with endpoints): merge descriptions, re-index locally, then publish remotely
659
- app.post("/v1/skills/:skill_id/publish", async (req, reply) => {
660
- const clientScope = clientScopeFor(req);
661
- const { skill_id } = req.params as { skill_id: string };
662
- const { endpoints: reviews, confirm_publish } = (req.body as {
663
- endpoints?: import("../types/index.js").EndpointReviewPayload[];
664
- confirm_publish?: boolean;
665
- }) ?? {};
666
-
667
- let skill = await loadSkillForMutation(skill_id, clientScope);
668
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
669
-
670
- if (reviews?.length) {
671
- const publishDecision = decideExplicitPublish(skill.domain, confirm_publish === true);
672
- if (!publishDecision.allowed) {
673
- return reply.code(409).send({
674
- error: "publish_confirmation_required",
675
- domain: skill.domain,
676
- publish_policy: {
677
- mode: publishDecision.mode,
678
- reason: publishDecision.reason,
679
- matched_domain: publishDecision.matchedDomain,
680
- },
681
- next_step: `Re-run ${checkpointPublishCommand(skill.skill_id, true)} only if you own this index.`,
682
- });
683
- }
684
- const updated = mergeAgentReview(skill.endpoints, reviews);
685
- skill.endpoints = updated;
686
- skill.updated_at = new Date().toISOString();
687
- const updatedWorkflowArtifact = applyWorkflowSchemaReviews(readWorkflowArtifact(skill.skill_id), reviews);
688
- if (updatedWorkflowArtifact) writeWorkflowArtifact(updatedWorkflowArtifact);
689
- const indexed = await indexSkillLocally(buildSkillIndexJob(skill, clientScope));
690
- const publishResult = await publishIndexedSkill(indexed);
691
- return reply.send({
692
- ok: true,
693
- skill_id: skill.skill_id,
694
- endpoints_updated: reviews.length,
695
- indexed: true,
696
- published: publishResult.published,
697
- publish_status: publishResult.publishStatus,
698
- ...(publishResult.publishedAt ? { published_at: publishResult.publishedAt } : {}),
699
- ...(publishResult.validationErrors ? { validation_errors: publishResult.validationErrors } : {}),
700
- next_step: publishResult.publishStatus === "published"
701
- ? "Remote share completed. Re-run resolve/skill inspection to use the published contract."
702
- : buildPublishFailureNextStep(skill.skill_id, publishResult.validationErrors),
703
- });
704
- }
705
-
706
- const indexed = await indexSkillLocally(buildSkillIndexJob(skill, clientScope));
707
- const ranked = rankEndpoints(indexed.skill.endpoints, indexed.skill.intent_signature, indexed.skill.domain);
708
- const endpoints_to_describe = ranked.map((r) => {
709
- const descriptionMeta = getEndpointDescriptionMetadata(r.endpoint);
710
- return {
711
- endpoint_id: r.endpoint.endpoint_id,
712
- method: r.endpoint.method,
713
- url: r.endpoint.url_template.length > 120
714
- ? r.endpoint.url_template.slice(0, 120) + "..."
715
- : r.endpoint.url_template,
716
- current_description: descriptionMeta.display,
717
- description_source: descriptionMeta.source,
718
- description_needs_review: descriptionMeta.needs_review,
719
- ...(descriptionMeta.warning ? { description_warning: descriptionMeta.warning } : {}),
720
- schema_summary: r.endpoint.response_schema
721
- ? summarizeSchema(r.endpoint.response_schema)
722
- : null,
723
- sample_values: extractSampleValues(r.endpoint.semantic?.example_response_compact),
724
- input_params: r.endpoint.semantic?.requires?.map((b) => ({
725
- key: b.key,
726
- type: b.type ?? b.semantic_type,
727
- required: b.required ?? false,
728
- example: b.example_value,
729
- })) ?? [],
730
- dom_extraction: !!r.endpoint.dom_extraction,
731
- review_context: buildEndpointReviewContext(indexed.skill, r.endpoint.endpoint_id),
732
- _fill_description:
733
- "DESCRIBE THIS ENDPOINT — what it returns, key params, action type, and any audience/eligibility/pricing/validity constraints",
734
- };
735
- });
736
-
737
- return reply.send({
738
- skill_id: indexed.skill.skill_id,
739
- domain: indexed.skill.domain,
740
- indexed: true,
741
- publish_status: "indexed",
742
- endpoint_count: indexed.skill.endpoints.length,
743
- endpoints_to_describe,
744
- next_step:
745
- `Fill each endpoint's description using review_context (deps, request_schema, response_schema, provenance, trigger page) plus any audience/eligibility/pricing/validity caveats, then call: unbrowse publish --skill ${indexed.skill.skill_id} --endpoints '[{endpoint_id, description, action_kind, resource_kind}]'`,
746
- _next_step:
747
- `Fill each endpoint's description using review_context (deps, request_schema, response_schema, provenance, trigger page) plus any audience/eligibility/pricing/validity caveats, then call: unbrowse publish --skill ${indexed.skill.skill_id} --endpoints '[{endpoint_id, description, action_kind, resource_kind}]'`,
748
- });
749
- });
750
-
751
- // POST /v1/foundry/publish-bundle — derive bundle/share/host artifacts from one preset
752
- app.post("/v1/foundry/publish-bundle", async (req, reply) => {
753
- const { preset_path, site_url, hosts } = (req.body as {
754
- preset_path?: string;
755
- site_url?: string;
756
- hosts?: string[];
757
- }) ?? {};
758
-
759
- if (!preset_path?.trim()) {
760
- return reply.code(400).send({ error: "preset_path is required" });
761
- }
762
-
763
- try {
764
- const result = publishFoundryBundle({
765
- presetPath: preset_path,
766
- ...(site_url?.trim() ? { siteUrl: site_url } : {}),
767
- ...(Array.isArray(hosts) && hosts.length > 0 ? {
768
- hosts: hosts.filter((host): host is "codex" | "claude" | "openclaw" =>
769
- host === "codex" || host === "claude" || host === "openclaw",
770
- ),
771
- } : {}),
772
- });
773
- return reply.send(result);
774
- } catch (error) {
775
- return reply.code(400).send({
776
- error: error instanceof Error ? error.message : String(error),
777
- });
778
- }
779
- });
780
- // POST /v1/skills/:skill_id/chunk — dynamic subgraph load for the current intent/bindings
781
- app.post("/v1/skills/:skill_id/chunk", async (req, reply) => {
782
- const clientScope = clientScopeFor(req);
783
- const { skill_id } = req.params as { skill_id: string };
784
- const { intent, operation_id, known_bindings, max_operations } = req.body as {
785
- intent?: string;
786
- operation_id?: string;
787
- known_bindings?: Record<string, unknown>;
788
- max_operations?: number;
789
- };
790
- const skill = getRecentLocalSkill(skill_id, clientScope) ?? await getSkill(skill_id, clientScope);
791
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
792
- return reply.send(toAgentSkillChunkView(getSkillChunk(skill, {
793
- intent,
794
- seed_operation_id: operation_id,
795
- known_bindings,
796
- max_operations,
797
- })));
798
- });
799
-
800
- // POST /v1/skills/:skill_id/execute
801
- app.post("/v1/skills/:skill_id/execute", { config: { rateLimit: ROUTE_LIMITS["/v1/skills/:skill_id/execute"] } }, async (req, reply) => {
802
- const clientScope = clientScopeFor(req);
803
- const { skill_id } = req.params as { skill_id: string };
804
- const { params, projection, confirm_unsafe, confirm_third_party_terms, dry_run, intent, context_url } = req.body as {
805
- params?: Record<string, unknown>;
806
- projection?: ProjectionOptions;
807
- confirm_unsafe?: boolean;
808
- confirm_third_party_terms?: boolean;
809
- dry_run?: boolean;
810
- intent?: string;
811
- context_url?: string;
812
- };
813
- // Check local caches first: recent skills → domain snapshots → marketplace
814
- let skill = getRecentLocalSkill(skill_id, clientScope);
815
- if (!skill) {
816
- // Check domain snapshot cache — passively indexed skills live here
817
- const { findExistingSkillForDomain: findLocal } = await import("../client/index.js");
818
- for (const [, entry] of domainSkillCache) {
819
- if (entry.skillId === skill_id && entry.localSkillPath) {
820
- try {
821
- skill = JSON.parse(require("fs").readFileSync(entry.localSkillPath, "utf-8"));
822
- } catch { /* snapshot read failed */ }
823
- break;
824
- }
825
- }
826
- }
827
- if (!skill) skill = await getSkill(skill_id, clientScope);
828
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
829
- const execParams = {
830
- ...(params ?? {}),
831
- ...(context_url && typeof params?.url !== "string" ? { url: context_url } : {}),
832
- };
833
- try {
834
- const execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, intent, contextUrl: context_url, client_scope: clientScope });
835
- saveTrace(execResult.trace);
836
- if (execResult.trace.endpoint_id) {
837
- recordExecution(skill.skill_id, execResult.trace.endpoint_id, execResult.trace, skill).catch(() => {});
838
- }
839
- if (execResult.trace.success) {
840
- promoteExplicitExecution(
841
- clientScope,
842
- intent || skill.intent_signature,
843
- context_url || (typeof execParams.url === "string" ? execParams.url : undefined),
844
- skill,
845
- execResult.trace.endpoint_id,
846
- execResult.result,
847
- );
848
- }
849
-
850
- // Auto-recovery: if endpoint returned 404 (stale), re-capture via orchestrator
851
- if (
852
- execResult.trace.status_code === 404 &&
853
- skill.domain &&
854
- skill.intent_signature &&
855
- skill.execution_type !== "browser-capture"
856
- ) {
857
- try {
858
- const recoveryUrl =
859
- context_url ||
860
- (typeof execParams.url === "string" && execParams.url) ||
861
- skill.endpoints.find((endpoint) => typeof endpoint.trigger_url === "string" && endpoint.trigger_url)?.trigger_url ||
862
- `https://${skill.domain}`;
863
- const freshResult = await resolveAndExecute(
864
- intent || skill.intent_signature,
865
- { ...execParams, url: recoveryUrl },
866
- { url: recoveryUrl },
867
- projection,
868
- { confirm_unsafe, confirm_third_party_terms, dry_run, intent: intent || skill.intent_signature, client_scope: clientScope }
869
- );
870
- saveTrace(freshResult.trace);
871
- if (freshResult.trace?.skill_id && freshResult.trace?.endpoint_id) {
872
- recordExecution(freshResult.trace.skill_id, freshResult.trace.endpoint_id, freshResult.trace, skill).catch(() => {});
873
- }
874
- await recordAnalyticsSession(buildAnalyticsSessionPayload(freshResult, {
875
- discovery_queries: 1,
876
- })).catch(() => {});
877
- const recovered = attachAgentOutcomeHints({
878
- ...freshResult,
879
- _recovery: {
880
- reason: "stale_endpoint_404",
881
- original_skill_id: skill_id,
882
- message: "Original endpoint returned 404. Auto-recovered with fresh capture.",
883
- },
884
- } as Record<string, unknown>, {
885
- skill: freshResult.skill ?? skill,
886
- endpointId: freshResult.trace.endpoint_id,
887
- timing: freshResult.timing,
888
- });
889
- return reply.send({
890
- ...recovered,
891
- });
892
- } catch {
893
- // Recovery failed — return original 404 with guidance
894
- }
895
- }
896
-
897
- await recordAnalyticsSession(buildAnalyticsSessionPayload(execResult, {
898
- discovery_queries: 0,
899
- })).catch(() => {});
900
-
901
- const response = attachAgentOutcomeHints({ ...execResult } as Record<string, unknown>, {
902
- skill,
903
- endpointId: execResult.trace.endpoint_id,
904
- });
905
- return reply.send(response);
906
- } catch (err) {
907
- return reply.code(500).send({ error: (err as Error).message });
908
- }
909
- });
910
-
911
- // POST /v1/skills/:skill_id/auth -- store credentials (cookies/headers) for a skill
912
- app.post("/v1/skills/:skill_id/auth", async (req, reply) => {
913
- const { skill_id } = req.params as { skill_id: string };
914
- const skill = await getSkill(skill_id);
915
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
916
-
917
- const body = req.body as {
918
- cookies?: Array<{ name: string; value: string; domain: string; path?: string }>;
919
- headers?: Record<string, string>;
920
- };
921
- if (!body.cookies && !body.headers) {
922
- return reply.code(400).send({ error: "Provide cookies or headers" });
923
- }
924
-
925
- const ref = `${skill.domain}-session`;
926
- await storeCredential(ref, JSON.stringify({ cookies: body.cookies ?? [], headers: body.headers ?? {} }));
927
-
928
- // Patch the skill manifest to reference the stored credentials
929
- if (!skill.auth_profile_ref) {
930
- await publishSkill({ ...skill, auth_profile_ref: ref });
931
- }
932
-
933
- return reply.send({ ok: true, auth_profile_ref: ref });
934
- });
935
-
936
- // POST /v1/auth/login — interactive OAuth flow or direct browser cookie extraction
937
- app.post("/v1/auth/login", { config: { rateLimit: ROUTE_LIMITS["/v1/auth/login"] } }, async (req, reply) => {
938
- const {
939
- url,
940
- browser,
941
- chrome_profile,
942
- firefox_profile,
943
- chromium_profile,
944
- chromium_user_data_dir,
945
- chromium_cookie_db_path,
946
- safe_storage_service,
947
- browser_name,
948
- } = req.body as {
949
- url: string;
950
- browser?: "auto" | "firefox" | "chrome" | "chromium";
951
- chrome_profile?: string;
952
- firefox_profile?: string;
953
- chromium_profile?: string;
954
- chromium_user_data_dir?: string;
955
- chromium_cookie_db_path?: string;
956
- safe_storage_service?: string;
957
- browser_name?: string;
958
- };
959
- if (!url) return reply.code(400).send({ error: "url required" });
960
- try {
961
- const result = await loginWithBrowserFallback(url, {
962
- browser,
963
- chromeProfile: chrome_profile,
964
- firefoxProfile: firefox_profile,
965
- chromium: {
966
- profile: chromium_profile,
967
- userDataDir: chromium_user_data_dir,
968
- cookieDbPath: chromium_cookie_db_path,
969
- safeStorageService: safe_storage_service,
970
- browserName: browser_name,
971
- },
972
- });
973
- return reply.send(result);
974
- } catch (err) {
975
- return reply.code(500).send({ error: (err as Error).message });
976
- }
977
- });
978
-
979
- // POST /v1/auth/steal — extract cookies from Firefox/Chrome/custom Chromium-family SQLite DBs.
980
- // No browser launch, Chrome can stay open. Higher rate limit since it's instant.
981
- app.post("/v1/auth/steal", { config: { rateLimit: { max: 30, timeWindow: "1 minute" } } }, async (req, reply) => {
982
- const {
983
- url,
984
- browser,
985
- chrome_profile,
986
- firefox_profile,
987
- chromium_profile,
988
- chromium_user_data_dir,
989
- chromium_cookie_db_path,
990
- safe_storage_service,
991
- browser_name,
992
- } = req.body as {
993
- url: string;
994
- browser?: "auto" | "firefox" | "chrome" | "chromium";
995
- chrome_profile?: string;
996
- firefox_profile?: string;
997
- chromium_profile?: string;
998
- chromium_user_data_dir?: string;
999
- chromium_cookie_db_path?: string;
1000
- safe_storage_service?: string;
1001
- browser_name?: string;
1002
- };
1003
- if (!url) return reply.code(400).send({ error: "url required" });
1004
- try {
1005
- const domain = new URL(url).hostname;
1006
- const result = await extractBrowserAuth(domain, {
1007
- browser,
1008
- chromeProfile: chrome_profile,
1009
- firefoxProfile: firefox_profile,
1010
- chromium: {
1011
- profile: chromium_profile,
1012
- userDataDir: chromium_user_data_dir,
1013
- cookieDbPath: chromium_cookie_db_path,
1014
- safeStorageService: safe_storage_service,
1015
- browserName: browser_name,
1016
- },
1017
- });
1018
- return reply.send(result);
1019
- } catch (err) {
1020
- return reply.code(500).send({ error: (err as Error).message });
1021
- }
1022
- });
1023
-
1024
- // POST /v1/skills/:skill_id/verify — trigger verification
1025
- app.post("/v1/skills/:skill_id/verify", async (req, reply) => {
1026
- const { skill_id } = req.params as { skill_id: string };
1027
- const skill = await getSkill(skill_id);
1028
- if (!skill) return reply.code(404).send({ error: "Skill not found" });
1029
- try {
1030
- const { verifySkill } = await import("../verification/index.js");
1031
- const results = await verifySkill(skill);
1032
- return reply.send({ skill_id, verification: results });
1033
- } catch (err) {
1034
- return reply.code(500).send({ error: (err as Error).message });
1035
- }
1036
- });
1037
-
1038
- // POST /v1/stale/cleanup — verify active skills, mark dead endpoints stale, drop local cache entries
1039
- app.post("/v1/stale/cleanup", async (req, reply) => {
1040
- const body = (req.body ?? {}) as {
1041
- skill_id?: string;
1042
- domain?: string;
1043
- limit?: number;
1044
- };
1045
- try {
1046
- const result = await cleanupStaleSkills({
1047
- skill_id: typeof body.skill_id === "string" ? body.skill_id : undefined,
1048
- domain: typeof body.domain === "string" ? body.domain : undefined,
1049
- limit: typeof body.limit === "number" ? body.limit : undefined,
1050
- });
1051
- return reply.send(result);
1052
- } catch (err) {
1053
- return reply.code(500).send({ error: (err as Error).message });
1054
- }
1055
- });
1056
-
1057
- // POST /v1/feedback — submit execution feedback with optional diagnostics
1058
- app.post("/v1/feedback", async (req, reply) => {
1059
- const { skill_id, target_id, endpoint_id, rating, outcome, diagnostics } = req.body as {
1060
- skill_id?: string;
1061
- target_id?: string;
1062
- endpoint_id?: string;
1063
- rating?: number;
1064
- outcome?: string;
1065
- diagnostics?: {
1066
- total_ms?: number;
1067
- bottleneck?: string;
1068
- wrong_endpoint?: boolean;
1069
- expected_data?: string;
1070
- got_data?: string;
1071
- trace_version?: string;
1072
- };
1073
- };
1074
- const resolvedSkillId = skill_id ?? target_id;
1075
- if (!resolvedSkillId || !endpoint_id || rating == null) {
1076
- return reply.code(400).send({ error: "skill_id, endpoint_id, and rating required" });
1077
- }
1078
- try {
1079
- const avg_rating = await recordFeedback(resolvedSkillId, endpoint_id, rating);
1080
- // Forward diagnostics to backend for version-grouped analysis
1081
- if (diagnostics) {
1082
- recordDiagnostics(resolvedSkillId, endpoint_id, diagnostics).catch(() => {});
1083
- }
1084
- return reply.send({ ok: true, avg_rating });
1085
- } catch (err) {
1086
- return reply.code(500).send({ error: (err as Error).message });
1087
- }
1088
- });
1089
-
1090
- // GET /v1/stats — public, no auth required
1091
- app.get("/v1/stats", async (_req, reply) => {
1092
- try {
1093
- const data = await fetchStats();
1094
- return reply.send(data);
1095
- } catch (err) {
1096
- return reply.code(500).send({ error: (err as Error).message });
1097
- }
1098
- });
1099
-
1100
- // GET /health
1101
- app.get("/health", async (_req, reply) => reply.send({
1102
- status: "ok",
1103
- package_version: PACKAGE_VERSION,
1104
- trace_version: TRACE_VERSION,
1105
- code_hash: CODE_HASH,
1106
- git_sha: GIT_SHA,
1107
- }));
1108
-
1109
- // GET /v1/sessions/:domain — read local trace/debug files instead of proxying to backend
1110
- app.get("/v1/sessions/:domain", async (req, reply) => {
1111
- const { domain } = req.params as { domain: string };
1112
- const query = req.query as { limit?: string | number };
1113
- const limitRaw = typeof query.limit === "number" ? query.limit : Number(query.limit ?? 10);
1114
- const limit = Number.isFinite(limitRaw) ? Math.min(Math.max(limitRaw, 1), 50) : 10;
1115
- return reply.send({
1116
- domain,
1117
- sessions: listRecentSessionsForDomain(TRACES_DIR, domain, limit),
1118
- });
1119
- });
1120
-
1121
- // Catch-all proxy: forward unmatched /v1/* routes to the configured backend origin
1122
- app.all("/v1/*", async (req, reply) => {
1123
- const key = getApiKey();
1124
- const upstream = `${BETA_API_URL}${req.url}`;
1125
- const headers: Record<string, string> = { "Content-Type": "application/json" };
1126
- if (key) headers["Authorization"] = `Bearer ${key}`;
1127
-
1128
- try {
1129
- const res = await fetch(upstream, {
1130
- method: req.method,
1131
- headers,
1132
- body: req.method !== "GET" && req.method !== "HEAD" ? JSON.stringify(req.body) : undefined,
1133
- });
1134
- const text = await res.text();
1135
- try {
1136
- return reply.code(res.status).send(JSON.parse(text));
1137
- } catch {
1138
- return reply.code(res.status).send({ error: text || `Upstream returned ${res.status}` });
1139
- }
1140
- } catch (err) {
1141
- return reply.code(502).send({ error: `Proxy to beta-api failed: ${(err as Error).message}` });
1142
- }
1143
- });
1144
-
1145
- // ── Browse session management ─────────────────────────────────────────
1146
- // Kuri browser actions with passive HAR indexing. The server manages a
1147
- // per-session tab + HAR state so every action the agent takes through
1148
- // the CLI is passively captured and indexed.
1149
-
1150
- // browseSessions is module-level (shared with orchestrator via registerBrowseSession)
1151
-
1152
- function requestedSessionId(req: { body?: unknown; query?: unknown }): string | undefined {
1153
- const body = req.body && typeof req.body === "object" ? req.body as Record<string, unknown> : null;
1154
- if (typeof body?.session_id === "string" && body.session_id.trim()) return body.session_id;
1155
- const query = req.query && typeof req.query === "object" ? req.query as Record<string, unknown> : null;
1156
- if (typeof query?.session_id === "string" && query.session_id.trim()) return query.session_id;
1157
- return undefined;
1158
- }
1159
-
1160
- function sendBrowseSessionError(reply: { code: (statusCode: number) => { send: (body: unknown) => unknown } }, error: unknown): unknown {
1161
- if (error instanceof BrowseSessionError) {
1162
- return reply.code(error.statusCode).send({ error: error.code });
1163
- }
1164
- if (isRecoverableBrowseFailure(error)) {
1165
- return reply.code(502).send({
1166
- error: "recoverable_browse_failure",
1167
- message: extractBrowseFailureMessage(error) ?? "recoverable_browse_failure",
1168
- recoverable: true,
1169
- });
1170
- }
1171
- throw error;
1172
- }
1173
-
1174
- /** Extract registrable domain for auth profile naming */
1175
- function profileName(url: string): string {
1176
- try { return new URL(url).hostname.replace(/^www\./, ""); } catch { return "unknown"; }
1177
- }
1178
-
1179
- async function restartBrowseCapture(session: BrowseSession): Promise<void> {
1180
- const broker = brokerForSession(session);
1181
- const load = await broker.waitForLoad(session.tabId, 2_000).catch(() => null);
1182
- if (load && load.status === "timeout") {
1183
- session.harActive = false;
1184
- return;
1185
- }
1186
- await broker.networkEnable(session.tabId).catch(() => {});
1187
- await broker.harStart(session.tabId).catch(() => {});
1188
- await broker.scriptInject(session.tabId, INTERCEPTOR_SCRIPT).catch(() => {});
1189
- session.harActive = true;
1190
- await injectInterceptor(session.tabId).catch(() => {});
1191
- }
1192
- async function flushBrowseCapture(
1193
- session: BrowseSession,
1194
- options: { queueIndex?: boolean; queuePublish?: boolean } = {},
1195
- ): Promise<{
1196
- indexed: boolean;
1197
- mode: "http" | "dom" | "none";
1198
- domain: string;
1199
- skill_id: string | null;
1200
- endpoint_count: number;
1201
- endpoints: Array<{
1202
- endpoint_id: string;
1203
- method: string;
1204
- url_template: string;
1205
- description?: string;
1206
- trigger_url?: string;
1207
- action_kind?: string;
1208
- resource_kind?: string;
1209
- }>;
1210
- request_count: number;
1211
- pipeline: {
1212
- index_queued: boolean;
1213
- publish_queued: boolean;
1214
- };
1215
- publish_policy: {
1216
- mode: "auto" | "disabled" | "blacklisted" | "prompt";
1217
- reason: string;
1218
- matched_domain?: string;
1219
- };
1220
- background_publish_queued: boolean;
1221
- }> {
1222
- let harEntries: KuriHarEntry[] = [];
1223
- if (session.harActive) {
1224
- try {
1225
- const { entries } = await brokerForSession(session).harStop(session.tabId);
1226
- harEntries = entries;
1227
- } catch { /* non-fatal */ }
1228
- }
1229
- session.harActive = false;
1230
-
1231
- const allRequests = await enrichPassiveCaptureRequests({
1232
- tabId: session.tabId,
1233
- captureUrl: session.url,
1234
- harEntries,
1235
- intent: `browse ${session.domain || profileName(session.url)}`,
1236
- });
1237
- const syncResult = await cacheBrowseRequests({
1238
- sessionUrl: session.url,
1239
- sessionDomain: session.domain,
1240
- requests: allRequests,
1241
- getPageHtml: () => brokerForSession(session).getPageHtml(session.tabId),
1242
- intent: `browse ${session.domain || profileName(session.url)}`,
1243
- });
1244
-
1245
- let indexQueued = false;
1246
- let publishQueued = false;
1247
- const publishDecision = options.queuePublish
1248
- ? decideCheckpointPublish(syncResult.domain)
1249
- : {
1250
- publishQueued: false,
1251
- mode: "disabled" as const,
1252
- reason: "Remote publish not requested for this checkpoint.",
1253
- };
1254
- if (options.queueIndex) {
1255
- if (syncResult.skill) {
1256
- queueBackgroundIndex({
1257
- skill: { ...syncResult.skill },
1258
- domain: syncResult.domain,
1259
- intent: syncResult.skill.intent_signature || `browse ${syncResult.domain}`,
1260
- contextUrl: session.url,
1261
- cacheKey: `browse-submit:${syncResult.domain}:${Date.now()}`,
1262
- publishAfterIndex: publishDecision.publishQueued,
1263
- });
1264
- indexQueued = true;
1265
- publishQueued = publishDecision.publishQueued;
1266
- } else if (allRequests.length > 0) {
1267
- passiveIndexFromRequests(allRequests, session.url, {
1268
- publishAfterIndex: publishDecision.publishQueued,
1269
- });
1270
- indexQueued = true;
1271
- publishQueued = publishDecision.publishQueued;
1272
- }
1273
- }
1274
- return {
1275
- indexed: syncResult.indexed,
1276
- mode: syncResult.mode,
1277
- domain: syncResult.domain,
1278
- skill_id: syncResult.skill?.skill_id ?? null,
1279
- endpoint_count: syncResult.skill?.endpoints.length ?? 0,
1280
- endpoints: (syncResult.skill?.endpoints ?? []).map((endpoint) => ({
1281
- endpoint_id: endpoint.endpoint_id,
1282
- method: endpoint.method,
1283
- url_template: endpoint.url_template,
1284
- description: endpoint.description,
1285
- trigger_url: endpoint.trigger_url,
1286
- action_kind: endpoint.semantic?.action_kind,
1287
- resource_kind: endpoint.semantic?.resource_kind,
1288
- })),
1289
- request_count: allRequests.length,
1290
- pipeline: {
1291
- index_queued: indexQueued,
1292
- publish_queued: publishQueued,
1293
- },
1294
- publish_policy: {
1295
- mode: publishDecision.mode,
1296
- reason: publishDecision.reason,
1297
- ...(publishDecision.matchedDomain ? { matched_domain: publishDecision.matchedDomain } : {}),
1298
- },
1299
- background_publish_queued: publishQueued,
1300
- };
1301
- }
1302
-
1303
- // POST /v1/browse/go — navigate to URL
1304
- app.post("/v1/browse/go", async (req, reply) => {
1305
- const { url } = req.body as { url: string };
1306
- if (!url) return reply.code(400).send({ error: "url required" });
1307
- try {
1308
- const sessionId = requestedSessionId(req);
1309
- const browseClient = selectBrowseBrokerClient(sessionId);
1310
- const navigateSession = async (session: BrowseSession) => {
1311
- const broker = brokerForSession(session);
1312
- const newDomain = profileName(url);
1313
-
1314
- if (session.harActive && session.url !== "about:blank") {
1315
- try {
1316
- const { entries } = await broker.harStop(session.tabId);
1317
- passiveIndexHar(entries, session.url, { publishAfterIndex: false });
1318
- } catch { /* non-fatal */ }
1319
- session.harActive = false;
1320
- }
1321
-
1322
- if (session.domain && session.domain !== newDomain) {
1323
- await saveAuthProfileBestEffort(session.tabId, session.domain, "browse_go");
1324
- }
1325
-
1326
- let cookiesInjected = 0;
1327
- if (newDomain && newDomain !== session.domain) {
1328
- // Check if the browser already has fresh session cookies for this domain.
1329
- // If so, skip vault/profile cookie injection — browser cookies are fresher
1330
- // and injecting stale vault cookies (e.g. JSESSIONID) causes HTTP 400 on
1331
- // sites like LinkedIn that validate CSRF alignment.
1332
- const browserHasFreshSession = await (async () => {
1333
- try {
1334
- const { extractBrowserCookies } = await import("../auth/browser-cookies.js");
1335
- const { cookies } = extractBrowserCookies(newDomain);
1336
- // Consider the session fresh if we have session-like cookies that aren't expired
1337
- const now = Date.now() / 1000;
1338
- return cookies.some((c) =>
1339
- (c.httpOnly || c.secure) && (!c.expires || c.expires > now),
1340
- );
1341
- } catch { return false; }
1342
- })();
1343
-
1344
- if (browserHasFreshSession) {
1345
- // Import browser cookies via CDP (they're fresh from Chrome's jar)
1346
- cookiesInjected = await importBrowserCookiesIntoTab(session.tabId, newDomain);
1347
- } else {
1348
- // No fresh browser cookies — load from vault/auth profile
1349
- cookiesInjected = await importBrowserCookiesIntoTab(session.tabId, newDomain);
1350
- await loadAuthProfileBestEffort(session.tabId, newDomain, "browse_go");
1351
- }
1352
- }
1353
-
1354
- await restartBrowseCapture(session);
1355
-
1356
- await broker.navigate(session.tabId, url);
1357
- await broker.navigate(session.tabId, url);
1358
- const finalUrl = await broker.getCurrentUrl(session.tabId).catch(() => url);
1359
- session.url = typeof finalUrl === "string" && finalUrl.startsWith("http") ? finalUrl : url;
1360
- session.domain = profileName(session.url);
1361
- await injectInterceptor(session.tabId);
1362
- const stillLive = await isBrowseSessionLive(session, browseClient).catch(() => false);
1363
- if (!stillLive) throw { error: "CDP command failed" };
1364
-
1365
- return { cookiesInjected };
1366
- };
1367
-
1368
- let session: BrowseSession;
1369
- let result: { cookiesInjected: number };
1370
- if (sessionId) {
1371
- const navigated = await withSerializedStrictBrowseSession(
1372
- browseSessions,
1373
- browseClient,
1374
- sessionId,
1375
- navigateSession,
1376
- );
1377
- session = navigated.session;
1378
- result = navigated.result;
1379
- } else {
1380
- session = await getOrCreateNavigateBrowseSession(
1381
- browseSessions,
1382
- browseClient,
1383
- injectInterceptor,
1384
- );
1385
- result = await navigateSession(session);
1386
- }
1387
-
1388
- return reply.send({
1389
- ok: true,
1390
- session_id: session.sessionId,
1391
- url: session.url,
1392
- tab_id: session.tabId,
1393
- auth_profile: session.domain,
1394
- ...(result.cookiesInjected > 0 ? { cookies_injected: result.cookiesInjected } : {}),
1395
- });
1396
- } catch (error) {
1397
- return sendBrowseSessionError(reply, error);
1398
- }
1399
- });
1400
-
1401
- // POST /v1/browse/submit — submit active form; same-origin fetch+rehydrate is explicit opt-in
1402
- app.post("/v1/browse/submit", async (req, reply) => {
1403
- const {
1404
- form_selector: formSelector,
1405
- submit_selector: submitSelector,
1406
- wait_for: waitFor,
1407
- same_origin_fetch_fallback: sameOriginFetchFallback,
1408
- timeout_ms: timeoutMs,
1409
- assist_site_state: assistSiteState,
1410
- } = (req.body as {
1411
- form_selector?: string;
1412
- submit_selector?: string;
1413
- wait_for?: string;
1414
- same_origin_fetch_fallback?: boolean;
1415
- timeout_ms?: number;
1416
- assist_site_state?: boolean;
1417
- }) ?? {};
1418
-
1419
- try {
1420
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1421
- const { session, result } = await withSerializedStrictBrowseSession(
1422
- browseSessions,
1423
- browseClient,
1424
- requestedSessionId(req),
1425
- async (session) => submitBrowseForm(
1426
- {
1427
- client: brokerForSession(session),
1428
- session,
1429
- flushCapture: async (session) => await flushBrowseCapture(session, { queueBackgroundPublish: false }),
1430
- flushCapture: async (session) => await flushBrowseCapture(session, { queueBackgroundPublish: false }),
1431
- restartCapture: restartBrowseCapture,
1432
- rehydratePlugins: (tabId) => brokerForSession(session).bestEffortRehydratePlugins(tabId),
1433
- },
1434
- {
1435
- formSelector,
1436
- submitSelector,
1437
- waitFor,
1438
- sameOriginFetchFallback,
1439
- timeoutMs,
1440
- assistSiteState,
1441
- },
1442
- ),
1443
- (result) => !result.ok && result.recoverable === true,
1444
- );
1445
-
1446
- let activeSession = session;
1447
- const hintedDestination = result.wait_for && isUrlWaitHint(result.wait_for)
1448
- ? resolveSubmitWaitHint(activeSession.url || "about:blank", result.wait_for)
1449
- : null;
1450
- const rawResultUrl = typeof result.url === "string" ? result.url : "";
1451
- activeSession.url = rawResultUrl || await brokerForSession(activeSession).getCurrentUrl(activeSession.tabId).catch(() => activeSession.url);
1452
- if (result.ok && hintedDestination && (!rawResultUrl || !rawResultUrl.includes(result.wait_for ?? ""))) {
1453
- activeSession.url = hintedDestination;
1454
- }
1455
- activeSession.domain = profileName(activeSession.url);
1456
- const stillLive = await isBrowseSessionLive(activeSession, browseClient).catch(() => false);
1457
- if (!stillLive) {
1458
- removeBrowseSession(browseSessions, activeSession.sessionId);
1459
- throw new BrowseSessionError("session_expired");
1460
- }
1461
-
1462
- const statusCode = result.ok ? 200 : (result.recoverable ? 502 : 400);
1463
- const sessionHint = `--session ${activeSession.sessionId}`;
1464
- const nextStep = result.ok
1465
- ? `If more UI steps remain, continue the flow. Run \`unbrowse sync ${sessionHint}\` after meaningful transitions, then \`unbrowse close ${sessionHint}\` when you're done to checkpoint the final capture, save auth, and queue the background pipeline.`
1466
- : `Inspect the page state with \`unbrowse snap ${sessionHint} --filter interactive\`, then retry submit with selectors or a wait hint if needed.`;
1467
- return reply.code(statusCode).send({
1468
- ...result,
1469
- session_id: activeSession.sessionId,
1470
- next_step: nextStep,
1471
- recovered: false,
1472
- tab_id: activeSession.tabId,
1473
- url: activeSession.url,
1474
- });
1475
- } catch (error) {
1476
- return sendBrowseSessionError(reply, error);
1477
- }
1478
- });
1479
-
1480
- // POST /v1/browse/snap — a11y snapshot
1481
- app.post("/v1/browse/snap", async (req, reply) => {
1482
- const { filter } = (req.body as { filter?: string; session_id?: string }) ?? {};
1483
- try {
1484
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1485
- const { session, result: snapshot } = await withSerializedStrictBrowseSession(
1486
- browseSessions,
1487
- browseClient,
1488
- requestedSessionId(req),
1489
- async (session) => brokerForSession(session).snapshot(session.tabId, filter),
1490
- );
1491
- return reply.send({ snapshot, session_id: session.sessionId, tab_id: session.tabId });
1492
- } catch (error) {
1493
- return sendBrowseSessionError(reply, error);
1494
- }
1495
- });
1496
-
1497
- // POST /v1/browse/click — click by ref
1498
- app.post("/v1/browse/click", async (req, reply) => {
1499
- const { ref } = req.body as { ref: string; session_id?: string };
1500
- if (!ref) return reply.code(400).send({ error: "ref required" });
1501
- try {
1502
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1503
- const { session } = await withSerializedStrictBrowseSession(
1504
- browseSessions,
1505
- browseClient,
1506
- requestedSessionId(req),
1507
- async (session) => {
1508
- await brokerForSession(session).click(session.tabId, ref);
1509
- return true;
1510
- },
1511
- );
1512
- await isBrowseSessionLive(session, browseClient).catch(() => false);
1513
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId, url: session.url });
1514
- } catch (error) {
1515
- return sendBrowseSessionError(reply, error);
1516
- }
1517
- });
1518
-
1519
- // POST /v1/browse/fill — fill input by ref
1520
- app.post("/v1/browse/fill", async (req, reply) => {
1521
- const { ref, value } = req.body as { ref: string; value: string; session_id?: string };
1522
- if (!ref || value === undefined) return reply.code(400).send({ error: "ref and value required" });
1523
- try {
1524
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1525
- const { session } = await withSerializedStrictBrowseSession(
1526
- browseSessions,
1527
- browseClient,
1528
- requestedSessionId(req),
1529
- async (session) => {
1530
- await brokerForSession(session).fill(session.tabId, ref, value);
1531
- return true;
1532
- },
1533
- );
1534
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1535
- } catch (error) {
1536
- return sendBrowseSessionError(reply, error);
1537
- }
1538
- });
1539
-
1540
- // POST /v1/browse/type — keyboard type
1541
- app.post("/v1/browse/type", async (req, reply) => {
1542
- const { text } = req.body as { text: string; session_id?: string };
1543
- if (!text) return reply.code(400).send({ error: "text required" });
1544
- try {
1545
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1546
- const { session } = await withSerializedStrictBrowseSession(
1547
- browseSessions,
1548
- browseClient,
1549
- requestedSessionId(req),
1550
- async (session) => {
1551
- await brokerForSession(session).keyboardType(session.tabId, text);
1552
- return true;
1553
- },
1554
- );
1555
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1556
- } catch (error) {
1557
- return sendBrowseSessionError(reply, error);
1558
- }
1559
- });
1560
-
1561
- // POST /v1/browse/press — press key
1562
- app.post("/v1/browse/press", async (req, reply) => {
1563
- const { key } = req.body as { key: string; session_id?: string };
1564
- if (!key) return reply.code(400).send({ error: "key required" });
1565
- try {
1566
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1567
- const { session } = await withSerializedStrictBrowseSession(
1568
- browseSessions,
1569
- browseClient,
1570
- requestedSessionId(req),
1571
- async (session) => {
1572
- await brokerForSession(session).press(session.tabId, key);
1573
- return true;
1574
- },
1575
- );
1576
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1577
- } catch (error) {
1578
- return sendBrowseSessionError(reply, error);
1579
- }
1580
- });
1581
-
1582
- // POST /v1/browse/select — select option by ref
1583
- app.post("/v1/browse/select", async (req, reply) => {
1584
- const { ref, value } = req.body as { ref: string; value: string; session_id?: string };
1585
- if (!ref || value === undefined) return reply.code(400).send({ error: "ref and value required" });
1586
- try {
1587
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1588
- const { session } = await withSerializedStrictBrowseSession(
1589
- browseSessions,
1590
- browseClient,
1591
- requestedSessionId(req),
1592
- async (session) => {
1593
- await brokerForSession(session).select(session.tabId, ref, value);
1594
- return true;
1595
- },
1596
- );
1597
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1598
- } catch (error) {
1599
- return sendBrowseSessionError(reply, error);
1600
- }
1601
- });
1602
-
1603
- // POST /v1/browse/scroll — scroll
1604
- app.post("/v1/browse/scroll", async (req, reply) => {
1605
- const { direction, amount } = (req.body as { direction?: string; amount?: number; session_id?: string }) ?? {};
1606
- try {
1607
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1608
- const { session } = await withSerializedStrictBrowseSession(
1609
- browseSessions,
1610
- browseClient,
1611
- requestedSessionId(req),
1612
- async (session) => {
1613
- await brokerForSession(session).scroll(session.tabId, (direction as any) ?? "down", amount);
1614
- return true;
1615
- },
1616
- );
1617
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1618
- } catch (error) {
1619
- return sendBrowseSessionError(reply, error);
1620
- }
1621
- });
1622
-
1623
- // GET /v1/browse/screenshot — capture screenshot
1624
- app.get("/v1/browse/screenshot", async (req, reply) => {
1625
- try {
1626
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1627
- const { session, result: data } = await withSerializedStrictBrowseSession(
1628
- browseSessions,
1629
- browseClient,
1630
- requestedSessionId(req),
1631
- async (session) => brokerForSession(session).screenshot(session.tabId),
1632
- );
1633
- return reply.send({ screenshot: data, session_id: session.sessionId, tab_id: session.tabId });
1634
- } catch (error) {
1635
- return sendBrowseSessionError(reply, error);
1636
- }
1637
- });
1638
-
1639
- // GET /v1/browse/text — page text
1640
- app.get("/v1/browse/text", async (req, reply) => {
1641
- try {
1642
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1643
- const { session, result: text } = await withSerializedStrictBrowseSession(
1644
- browseSessions,
1645
- browseClient,
1646
- requestedSessionId(req),
1647
- async (session) => brokerForSession(session).getText(session.tabId),
1648
- );
1649
- return reply.send({ text, session_id: session.sessionId, tab_id: session.tabId });
1650
- } catch (error) {
1651
- return sendBrowseSessionError(reply, error);
1652
- }
1653
- });
1654
-
1655
- // GET /v1/browse/markdown — page as markdown
1656
- app.get("/v1/browse/markdown", async (req, reply) => {
1657
- try {
1658
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1659
- const { session, result: markdown } = await withSerializedStrictBrowseSession(
1660
- browseSessions,
1661
- browseClient,
1662
- requestedSessionId(req),
1663
- async (session) => brokerForSession(session).getMarkdown(session.tabId),
1664
- );
1665
- return reply.send({ markdown, session_id: session.sessionId, tab_id: session.tabId });
1666
- } catch (error) {
1667
- return sendBrowseSessionError(reply, error);
1668
- }
1669
- });
1670
-
1671
- // GET /v1/browse/cookies — page cookies
1672
- app.get("/v1/browse/cookies", async (req, reply) => {
1673
- try {
1674
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1675
- const { session, result: cookies } = await withSerializedStrictBrowseSession(
1676
- browseSessions,
1677
- browseClient,
1678
- requestedSessionId(req),
1679
- async (session) => brokerForSession(session).getCookies(session.tabId),
1680
- );
1681
- return reply.send({ cookies, session_id: session.sessionId, tab_id: session.tabId });
1682
- } catch (error) {
1683
- return sendBrowseSessionError(reply, error);
1684
- }
1685
- });
1686
-
1687
- // POST /v1/browse/eval — evaluate JS
1688
- app.post("/v1/browse/eval", async (req, reply) => {
1689
- const { expression } = req.body as { expression: string; session_id?: string };
1690
- if (!expression) return reply.code(400).send({ error: "expression required" });
1691
- try {
1692
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1693
- const { session, result } = await withSerializedStrictBrowseSession(
1694
- browseSessions,
1695
- browseClient,
1696
- requestedSessionId(req),
1697
- async (session) => brokerForSession(session).evaluate(session.tabId, expression),
1698
- );
1699
- return reply.send({ result, session_id: session.sessionId, tab_id: session.tabId });
1700
- } catch (error) {
1701
- return sendBrowseSessionError(reply, error);
1702
- }
1703
- });
1704
-
1705
- // POST /v1/browse/back — navigate back
1706
- app.post("/v1/browse/back", async (req, reply) => {
1707
- try {
1708
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1709
- const { session } = await withSerializedStrictBrowseSession(
1710
- browseSessions,
1711
- browseClient,
1712
- requestedSessionId(req),
1713
- async (session) => {
1714
- await brokerForSession(session).goBack(session.tabId);
1715
- return true;
1716
- },
1717
- );
1718
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1719
- } catch (error) {
1720
- return sendBrowseSessionError(reply, error);
1721
- }
1722
- });
1723
-
1724
- // POST /v1/browse/forward — navigate forward
1725
- app.post("/v1/browse/forward", async (req, reply) => {
1726
- try {
1727
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1728
- const { session } = await withSerializedStrictBrowseSession(
1729
- browseSessions,
1730
- browseClient,
1731
- requestedSessionId(req),
1732
- async (session) => {
1733
- await brokerForSession(session).goForward(session.tabId);
1734
- return true;
1735
- },
1736
- );
1737
- return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
1738
- } catch (error) {
1739
- return sendBrowseSessionError(reply, error);
1740
- }
1741
- });
1742
-
1743
- // POST /v1/browse/sync — checkpoint capture, keep the tab open, queue index+publish
1744
- app.post("/v1/browse/sync", async (req, reply) => {
1745
- try {
1746
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1747
- const { session, result: syncResult } = await withSerializedStrictBrowseSession(
1748
- browseSessions,
1749
- browseClient,
1750
- requestedSessionId(req),
1751
- async (session) => {
1752
- const syncResult = await flushBrowseCapture(session, { queueIndex: true, queuePublish: true });
1753
- await restartBrowseCapture(session);
1754
- return syncResult;
1755
- },
1756
- );
1757
-
1758
- return reply.send({
1759
- ok: true,
1760
- checkpointed: true,
1761
- session_id: session.sessionId,
1762
- tab_id: session.tabId,
1763
- indexed: syncResult.indexed,
1764
- mode: syncResult.mode,
1765
- domain: syncResult.domain,
1766
- skill_id: syncResult.skill_id,
1767
- endpoint_count: syncResult.endpoint_count,
1768
- endpoints: syncResult.endpoints,
1769
- request_count: syncResult.request_count,
1770
- pipeline: syncResult.pipeline,
1771
- publish_policy: syncResult.publish_policy,
1772
- background_publish_queued: syncResult.background_publish_queued,
1773
- next_step: buildCheckpointNextStep("sync", syncResult, session.sessionId),
1774
- });
1775
- } catch (error) {
1776
- return sendBrowseSessionError(reply, error);
1777
- }
1778
- });
1779
-
1780
- // POST /v1/browse/close — checkpoint capture, queue index+publish, save auth, close tab
1781
- app.post("/v1/browse/close", async (req, reply) => {
1782
- try {
1783
- const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
1784
- const { session, result: syncResult } = await withSerializedStrictBrowseSession(
1785
- browseSessions,
1786
- browseClient,
1787
- requestedSessionId(req),
1788
- async (session) => {
1789
- const broker = brokerForSession(session);
1790
- if (session.domain) {
1791
- await saveAuthProfileBestEffort(session.tabId, session.domain, "browse_close");
1792
- }
1793
- const syncResult = await flushBrowseCapture(session, { queueIndex: true, queuePublish: true });
1794
- await broker.closeTab(session.tabId).catch(() => {});
1795
- removeBrowseSession(browseSessions, session.sessionId);
1796
- return syncResult;
1797
- },
1798
- );
1799
- return reply.send({
1800
- ok: true,
1801
- checkpointed: true,
1802
- session_id: session.sessionId,
1803
- indexed: syncResult.indexed,
1804
- mode: syncResult.mode,
1805
- endpoint_count: syncResult.endpoint_count,
1806
- request_count: syncResult.request_count,
1807
- pipeline: syncResult.pipeline,
1808
- publish_policy: syncResult.publish_policy,
1809
- background_publish_queued: syncResult.background_publish_queued,
1810
- auth_saved: session.domain || null,
1811
- next_step: buildCheckpointNextStep("close", syncResult, session.sessionId),
1812
- });
1813
- } catch (error) {
1814
- return sendBrowseSessionError(reply, error);
1815
- }
1816
- });
1817
- }
1818
-
1819
- function saveTrace(trace: unknown) {
1820
- if (!existsSync(TRACES_DIR)) mkdirSync(TRACES_DIR, { recursive: true });
1821
- const t = trace as { trace_id: string };
1822
- writeFileSync(join(TRACES_DIR, `${t.trace_id}.json`), JSON.stringify(trace, null, 2));
1823
- }