unbrowse 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +455 -96
- package/dist/index.js +2 -6
- package/dist/mcp.js +695 -46
- package/dist/server.js +25811 -0
- package/package.json +1 -2
- package/vendor/kuri/darwin-arm64/kuri +0 -0
- package/vendor/kuri/darwin-x64/kuri +0 -0
- package/vendor/kuri/linux-arm64/kuri +0 -0
- package/vendor/kuri/linux-x64/kuri +0 -0
- package/vendor/kuri/manifest.json +7 -10
- package/runtime-src/agent-outcome.ts +0 -166
- package/runtime-src/analytics-session.ts +0 -55
- package/runtime-src/api/browse-index.ts +0 -317
- package/runtime-src/api/browse-session.ts +0 -572
- package/runtime-src/api/browse-submit-prereqs.ts +0 -48
- package/runtime-src/api/browse-submit.ts +0 -1184
- package/runtime-src/api/routes.ts +0 -1823
- package/runtime-src/auth/browser-cookies.ts +0 -423
- package/runtime-src/auth/index.ts +0 -535
- package/runtime-src/auth/runtime.ts +0 -116
- package/runtime-src/browser/index.ts +0 -659
- package/runtime-src/browser/types.ts +0 -41
- package/runtime-src/build-info.generated.ts +0 -6
- package/runtime-src/capture/index.ts +0 -1794
- package/runtime-src/capture/prefetch.ts +0 -95
- package/runtime-src/capture/rsc.ts +0 -45
- package/runtime-src/cli/shortcuts.ts +0 -273
- package/runtime-src/cli.ts +0 -1572
- package/runtime-src/client/graph-client.ts +0 -100
- package/runtime-src/client/index.ts +0 -1425
- package/runtime-src/debug-trace.ts +0 -18
- package/runtime-src/domain.ts +0 -38
- package/runtime-src/execution/index.ts +0 -3397
- package/runtime-src/execution/retry.ts +0 -46
- package/runtime-src/execution/robots.ts +0 -167
- package/runtime-src/execution/search-forms.ts +0 -188
- package/runtime-src/extraction/index.ts +0 -1507
- package/runtime-src/foundry/publish-bundle.ts +0 -392
- package/runtime-src/graph/agent-augment.ts +0 -315
- package/runtime-src/graph/index.ts +0 -1524
- package/runtime-src/graph/local-fixtures.ts +0 -393
- package/runtime-src/graph/local-harness.ts +0 -646
- package/runtime-src/graph/planner.ts +0 -411
- package/runtime-src/graph/session.ts +0 -294
- package/runtime-src/graph/trace-store.ts +0 -136
- package/runtime-src/index.ts +0 -24
- package/runtime-src/indexer/index.ts +0 -465
- package/runtime-src/intent-match.ts +0 -1515
- package/runtime-src/kuri/client.ts +0 -1839
- package/runtime-src/logger.ts +0 -30
- package/runtime-src/marketplace/index.ts +0 -103
- package/runtime-src/mcp.ts +0 -1747
- package/runtime-src/orchestrator/browser-agent.ts +0 -374
- package/runtime-src/orchestrator/dag-advisor.ts +0 -59
- package/runtime-src/orchestrator/dag-feedback.ts +0 -257
- package/runtime-src/orchestrator/first-pass-action.ts +0 -403
- package/runtime-src/orchestrator/index.ts +0 -4480
- package/runtime-src/orchestrator/passive-publish.ts +0 -187
- package/runtime-src/orchestrator/timing-economics.ts +0 -80
- package/runtime-src/payments/cascade.ts +0 -137
- package/runtime-src/payments/index.ts +0 -270
- package/runtime-src/payments/lobster-pay.ts +0 -182
- package/runtime-src/payments/wallet.ts +0 -98
- package/runtime-src/publish/review-context.ts +0 -93
- package/runtime-src/publish/sanitize.ts +0 -197
- package/runtime-src/publish/schema-review.ts +0 -192
- package/runtime-src/publish-admission.ts +0 -388
- package/runtime-src/ratelimit/index.ts +0 -23
- package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
- package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
- package/runtime-src/reverse-engineer/index.ts +0 -1551
- package/runtime-src/router.ts +0 -17
- package/runtime-src/routing-telemetry.ts +0 -395
- package/runtime-src/runtime/browser-access.ts +0 -11
- package/runtime-src/runtime/browser-auth.ts +0 -12
- package/runtime-src/runtime/browser-host.ts +0 -48
- package/runtime-src/runtime/lifecycle.ts +0 -17
- package/runtime-src/runtime/local-server.ts +0 -311
- package/runtime-src/runtime/paths.ts +0 -99
- package/runtime-src/runtime/setup.ts +0 -251
- package/runtime-src/runtime/supervisor.ts +0 -69
- package/runtime-src/runtime/update-hints.ts +0 -351
- package/runtime-src/server.ts +0 -100
- package/runtime-src/session-logs.ts +0 -142
- package/runtime-src/settings.ts +0 -221
- package/runtime-src/single-binary.ts +0 -143
- package/runtime-src/site-policy.ts +0 -54
- package/runtime-src/stale-cleanup-runner.ts +0 -144
- package/runtime-src/stale-cleanup.ts +0 -133
- package/runtime-src/telemetry-attribution.ts +0 -120
- package/runtime-src/telemetry.ts +0 -253
- package/runtime-src/template-params.ts +0 -141
- package/runtime-src/transform/drift.ts +0 -60
- package/runtime-src/transform/index.ts +0 -277
- package/runtime-src/types/index.ts +0 -1
- package/runtime-src/types/skill.ts +0 -912
- package/runtime-src/vault/index.ts +0 -196
- package/runtime-src/verification/auth-gate.ts +0 -8
- package/runtime-src/verification/candidates.ts +0 -27
- package/runtime-src/verification/index.ts +0 -120
- package/runtime-src/verification/matrix.ts +0 -30
- package/runtime-src/version.ts +0 -148
- package/runtime-src/workflow/artifact.ts +0 -161
- package/runtime-src/workflow/compile.ts +0 -808
- package/runtime-src/workflow/publish.ts +0 -225
- package/runtime-src/workflow/runtime.ts +0 -213
- package/vendor/kuri/win-x64/kuri.exe +0 -0
|
@@ -1,1823 +0,0 @@
|
|
|
1
|
-
import type { FastifyInstance } from "fastify";
|
|
2
|
-
import * as kuri from "../kuri/client.js";
|
|
3
|
-
import type { KuriHarEntry } from "../kuri/client.js";
|
|
4
|
-
import { extractEndpoints, extractAuthHeaders } from "../reverse-engineer/index.js";
|
|
5
|
-
import { INTERCEPTOR_SCRIPT, enrichPassiveCaptureRequests, injectInterceptor } from "../capture/index.js";
|
|
6
|
-
import { indexSkillLocally, mergeAgentReview, publishIndexedSkill, queueBackgroundIndex } from "../indexer/index.js";
|
|
7
|
-
import { nanoid } from "nanoid";
|
|
8
|
-
import type { ExecutionTrace, OrchestrationTiming, ProjectionOptions, SkillManifest } from "../types/index.js";
|
|
9
|
-
import { mergeEndpoints } from "../marketplace/index.js";
|
|
10
|
-
import { buildSkillOperationGraph, getEndpointDescriptionMetadata, getSkillChunk, toAgentSkillChunkView } from "../graph/index.js";
|
|
11
|
-
import { augmentEndpointsWithAgent } from "../graph/agent-augment.js";
|
|
12
|
-
import { findExistingSkillForDomain, cachePublishedSkill } from "../client/index.js";
|
|
13
|
-
import { storeCredential } from "../vault/index.js";
|
|
14
|
-
import { generateLocalDescription, writeSkillSnapshot, buildResolveCacheKey, getDomainReuseKey, domainSkillCache, persistDomainCache, scopedCacheKey, snapshotPathForCacheKey, invalidateRouteCacheForDomain, summarizeSchema, extractSampleValues } from "../orchestrator/index.js";
|
|
15
|
-
import { TRACE_VERSION, CODE_HASH, DEFAULT_BACKEND_URL, GIT_SHA, PACKAGE_VERSION } from "../version.js";
|
|
16
|
-
import { promoteExplicitExecution, resolveAndExecute, type OrchestratorResult } from "../orchestrator/index.js";
|
|
17
|
-
import { getSkill } from "../marketplace/index.js";
|
|
18
|
-
import { executeSkill, rankEndpoints } from "../execution/index.js";
|
|
19
|
-
import {
|
|
20
|
-
extractBrowserAuth,
|
|
21
|
-
importBrowserCookiesIntoTab,
|
|
22
|
-
loginWithBrowserFallback,
|
|
23
|
-
loadAuthProfileBestEffort,
|
|
24
|
-
saveAuthProfileBestEffort,
|
|
25
|
-
} from "../auth/index.js";
|
|
26
|
-
import { recordFeedback, recordDiagnostics, recordExecution, getApiKey, getRecentLocalSkill, recordAnalyticsSession, type AnalyticsSessionPayload } from "../client/index.js";
|
|
27
|
-
import { ROUTE_LIMITS } from "../ratelimit/index.js";
|
|
28
|
-
import { listRecentSessionsForDomain } from "../session-logs.js";
|
|
29
|
-
import { attachAgentOutcomeHints } from "../agent-outcome.js";
|
|
30
|
-
import { writeFileSync, existsSync, mkdirSync } from "fs";
|
|
31
|
-
import { join } from "path";
|
|
32
|
-
import {
|
|
33
|
-
BrowseSessionError,
|
|
34
|
-
createRegisteredBrowseSession,
|
|
35
|
-
extractBrowseFailureMessage,
|
|
36
|
-
getOrCreateNavigateBrowseSession,
|
|
37
|
-
isBrowseSessionLive,
|
|
38
|
-
isRecoverableBrowseFailure,
|
|
39
|
-
type BrowseSession,
|
|
40
|
-
withSerializedStrictBrowseSession,
|
|
41
|
-
removeBrowseSession,
|
|
42
|
-
} from "./browse-session.js";
|
|
43
|
-
import { cacheBrowseRequests, harEntriesToRawRequests } from "./browse-index.js";
|
|
44
|
-
import { isUrlWaitHint, resolveSubmitWaitHint, submitBrowseForm } from "./browse-submit.js";
|
|
45
|
-
import { cleanupStaleSkills } from "../stale-cleanup-runner.js";
|
|
46
|
-
import {
|
|
47
|
-
decideCheckpointPublish,
|
|
48
|
-
decideExplicitPublish,
|
|
49
|
-
getCapturePipelineSettings,
|
|
50
|
-
updateCapturePipelineSettings,
|
|
51
|
-
} from "../settings.js";
|
|
52
|
-
import { publishFoundryBundle } from "../foundry/publish-bundle.js";
|
|
53
|
-
import { buildEndpointReviewContext } from "../publish/review-context.js";
|
|
54
|
-
import { applyWorkflowSchemaReviews } from "../publish/schema-review.js";
|
|
55
|
-
import { readWorkflowArtifact, writeWorkflowArtifact } from "../workflow/artifact.js";
|
|
56
|
-
|
|
57
|
-
const BETA_API_URL = process.env.UNBROWSE_BACKEND_URL || DEFAULT_BACKEND_URL;
|
|
58
|
-
|
|
59
|
-
const TRACES_DIR = process.env.TRACES_DIR ?? join(process.cwd(), "traces");
|
|
60
|
-
const BROWSE_BROKER_MAX = Math.max(1, Number(process.env.KURI_MULTI_BROKER_MAX ?? "2"));
|
|
61
|
-
const BROWSE_BROKER_BASE_PORT = Number(process.env.KURI_PORT ?? "7700");
|
|
62
|
-
|
|
63
|
-
type AnalyticsSessionResult = {
|
|
64
|
-
trace: Pick<ExecutionTrace, "trace_id" | "started_at" | "completed_at" | "endpoint_id" | "trace_version" | "success" | "tokens_saved" | "tokens_saved_pct" | "api_call_count">;
|
|
65
|
-
timing?: Pick<OrchestrationTiming, "source" | "time_saved_ms" | "time_saved_pct" | "cost_saved_uc" | "tokens_saved" | "tokens_saved_pct">;
|
|
66
|
-
source?: OrchestratorResult["source"];
|
|
67
|
-
};
|
|
68
|
-
|
|
69
|
-
export function buildAnalyticsSessionPayload(
|
|
70
|
-
result: AnalyticsSessionResult,
|
|
71
|
-
opts: {
|
|
72
|
-
browser_mode?: AnalyticsSessionPayload["browser_mode"];
|
|
73
|
-
discovery_queries: number;
|
|
74
|
-
cached_skill_calls?: number;
|
|
75
|
-
fresh_index_calls?: number;
|
|
76
|
-
},
|
|
77
|
-
): AnalyticsSessionPayload {
|
|
78
|
-
const source = result.timing?.source ?? result.source;
|
|
79
|
-
const apiCalls = result.trace.api_call_count ?? (result.trace.endpoint_id ? 1 : 0);
|
|
80
|
-
const browserMode = opts.browser_mode ?? (
|
|
81
|
-
source === "live-capture" || source === "first-pass" || source === "browser-action"
|
|
82
|
-
? "default"
|
|
83
|
-
: "replaced"
|
|
84
|
-
);
|
|
85
|
-
const cachedSkillCalls = opts.cached_skill_calls ?? (
|
|
86
|
-
apiCalls > 0 && source !== "live-capture" && source !== "first-pass" ? 1 : 0
|
|
87
|
-
);
|
|
88
|
-
const freshIndexCalls = opts.fresh_index_calls ?? (
|
|
89
|
-
apiCalls > 0 && (source === "live-capture" || source === "first-pass") ? 1 : 0
|
|
90
|
-
);
|
|
91
|
-
|
|
92
|
-
return {
|
|
93
|
-
session_id: result.trace.trace_id,
|
|
94
|
-
started_at: result.trace.started_at,
|
|
95
|
-
completed_at: result.trace.completed_at,
|
|
96
|
-
trace_version: result.trace.trace_version ?? TRACE_VERSION,
|
|
97
|
-
api_calls: apiCalls,
|
|
98
|
-
discovery_queries: opts.discovery_queries,
|
|
99
|
-
cached_skill_calls: cachedSkillCalls,
|
|
100
|
-
fresh_index_calls: freshIndexCalls,
|
|
101
|
-
browser_mode: browserMode,
|
|
102
|
-
success: result.trace.success ?? true,
|
|
103
|
-
source,
|
|
104
|
-
time_saved_ms: result.timing?.time_saved_ms,
|
|
105
|
-
time_saved_pct: result.timing?.time_saved_pct,
|
|
106
|
-
tokens_saved: result.trace.tokens_saved ?? result.timing?.tokens_saved,
|
|
107
|
-
tokens_saved_pct: result.trace.tokens_saved_pct ?? result.timing?.tokens_saved_pct,
|
|
108
|
-
cost_saved_uc: result.timing?.cost_saved_uc,
|
|
109
|
-
};
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
/** Process HAR entries into routes and queue local index, with remote share opt-in only. */
|
|
114
|
-
function passiveIndexFromRequests(
|
|
115
|
-
requests: RawRequest[],
|
|
116
|
-
pageUrl: string,
|
|
117
|
-
options: { publishAfterIndex?: boolean } = {},
|
|
118
|
-
): void {
|
|
119
|
-
if (requests.length === 0) return;
|
|
120
|
-
|
|
121
|
-
let domain: string;
|
|
122
|
-
try { domain = new URL(pageUrl).hostname; } catch { return; }
|
|
123
|
-
const intent = `browse ${domain}`;
|
|
124
|
-
|
|
125
|
-
// Fire-and-forget — full pipeline runs async
|
|
126
|
-
void (async () => {
|
|
127
|
-
try {
|
|
128
|
-
// 1. Extract endpoints from captured traffic
|
|
129
|
-
const rawEndpoints = extractEndpoints(requests, undefined, { pageUrl, finalUrl: pageUrl });
|
|
130
|
-
if (rawEndpoints.length === 0) {
|
|
131
|
-
console.error(`[passive-index] ${domain}: 0 endpoints from ${requests.length} requests`);
|
|
132
|
-
return;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
// 2. Extract and store auth credentials (cookies + sensitive headers)
|
|
136
|
-
const capturedAuthHeaders = extractAuthHeaders(requests);
|
|
137
|
-
if (Object.keys(capturedAuthHeaders).length > 0) {
|
|
138
|
-
const authKey = `${domain}-session`;
|
|
139
|
-
await storeCredential(authKey, JSON.stringify({ headers: capturedAuthHeaders }));
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
// 3. Merge with existing skill for this domain (never reduce endpoint count)
|
|
143
|
-
const existingSkill = findExistingSkillForDomain(domain, intent);
|
|
144
|
-
const mergedEndpoints = existingSkill
|
|
145
|
-
? mergeEndpoints(existingSkill.endpoints, rawEndpoints)
|
|
146
|
-
: rawEndpoints;
|
|
147
|
-
// Guard: if passive capture found fewer endpoints than what exists, keep the richer set
|
|
148
|
-
if (existingSkill && mergedEndpoints.length < existingSkill.endpoints.length) {
|
|
149
|
-
console.error(`[passive-index] ${domain}: skipping — would reduce ${existingSkill.endpoints.length} → ${mergedEndpoints.length} endpoints`);
|
|
150
|
-
return;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
// 4. Generate descriptions for endpoints without them (enables BM25 ranking)
|
|
154
|
-
for (const ep of mergedEndpoints) {
|
|
155
|
-
if (!ep.description) {
|
|
156
|
-
ep.description = generateLocalDescription(ep);
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
// 5. Skip LLM-based augmentation — the calling agent IS the LLM.
|
|
161
|
-
// Endpoint descriptions come from generateLocalDescription (heuristic).
|
|
162
|
-
// The agent reviews endpoints in the deferral response and picks the right one.
|
|
163
|
-
const enrichedEndpoints = mergedEndpoints;
|
|
164
|
-
|
|
165
|
-
// 6. Build operation dependency graph
|
|
166
|
-
const operationGraph = buildSkillOperationGraph(enrichedEndpoints);
|
|
167
|
-
|
|
168
|
-
// 7. Assemble full skill manifest
|
|
169
|
-
const skill: SkillManifest = {
|
|
170
|
-
skill_id: existingSkill?.skill_id ?? nanoid(),
|
|
171
|
-
version: "1.0.0",
|
|
172
|
-
schema_version: "1",
|
|
173
|
-
lifecycle: "active" as const,
|
|
174
|
-
execution_type: "http" as const,
|
|
175
|
-
created_at: existingSkill?.created_at ?? new Date().toISOString(),
|
|
176
|
-
updated_at: new Date().toISOString(),
|
|
177
|
-
name: domain,
|
|
178
|
-
intent_signature: intent,
|
|
179
|
-
domain,
|
|
180
|
-
description: `API skill for ${domain}`,
|
|
181
|
-
owner_type: "agent" as const,
|
|
182
|
-
endpoints: enrichedEndpoints,
|
|
183
|
-
operation_graph: operationGraph,
|
|
184
|
-
intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
|
|
185
|
-
};
|
|
186
|
-
|
|
187
|
-
// 8. Cache locally for immediate reuse — write to BOTH the published skill cache
|
|
188
|
-
// AND the domain skill snapshot so resolveAndExecute finds it on next call
|
|
189
|
-
try { cachePublishedSkill(skill); } catch { /* best-effort */ }
|
|
190
|
-
|
|
191
|
-
// Write domain skill snapshot (keyed by resolve cache key)
|
|
192
|
-
const bgCacheKey = buildResolveCacheKey(domain, intent, pageUrl);
|
|
193
|
-
const bgScopedKey = scopedCacheKey("global", bgCacheKey);
|
|
194
|
-
writeSkillSnapshot(bgScopedKey, skill);
|
|
195
|
-
|
|
196
|
-
// Update domain-level reuse cache
|
|
197
|
-
const bgDomainKey = getDomainReuseKey(pageUrl ?? domain);
|
|
198
|
-
if (bgDomainKey) {
|
|
199
|
-
domainSkillCache.set(bgDomainKey, {
|
|
200
|
-
skillId: skill.skill_id,
|
|
201
|
-
localSkillPath: snapshotPathForCacheKey(bgScopedKey),
|
|
202
|
-
ts: Date.now(),
|
|
203
|
-
});
|
|
204
|
-
persistDomainCache();
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
// 9. Queue local index, and only remote-share when the caller explicitly asked for it.
|
|
208
|
-
const cacheKey = `passive:${domain}:${Date.now()}`;
|
|
209
|
-
queueBackgroundIndex({
|
|
210
|
-
skill,
|
|
211
|
-
domain,
|
|
212
|
-
intent,
|
|
213
|
-
contextUrl: pageUrl,
|
|
214
|
-
cacheKey,
|
|
215
|
-
publishAfterIndex: options.publishAfterIndex === true,
|
|
216
|
-
});
|
|
217
|
-
|
|
218
|
-
console.error(`[passive-index] ${domain}: ${enrichedEndpoints.length} endpoints indexed from ${requests.length} requests`);
|
|
219
|
-
} catch (err) {
|
|
220
|
-
console.error(`[passive-index] ${domain} failed: ${err instanceof Error ? err.message : err}`);
|
|
221
|
-
}
|
|
222
|
-
})();
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
/** Convenience wrapper: convert HAR entries and run passive indexing */
|
|
226
|
-
function passiveIndexHar(
|
|
227
|
-
entries: KuriHarEntry[],
|
|
228
|
-
pageUrl: string,
|
|
229
|
-
options: { publishAfterIndex?: boolean } = {},
|
|
230
|
-
): void {
|
|
231
|
-
passiveIndexFromRequests(harEntriesToRawRequests(entries), pageUrl, options);
|
|
232
|
-
}
|
|
233
|
-
// ── Browse session state (module-level so orchestrator can register sessions) ──
|
|
234
|
-
const browseSessions = new Map<string, BrowseSession>();
|
|
235
|
-
|
|
236
|
-
function browseBrokerPorts(): number[] {
|
|
237
|
-
return Array.from({ length: BROWSE_BROKER_MAX }, (_, index) => BROWSE_BROKER_BASE_PORT + index);
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
function brokerForSession(session: BrowseSession | undefined): kuri.KuriClient {
|
|
241
|
-
if (session?.client) return session.client as kuri.KuriClient;
|
|
242
|
-
if (session?.brokerPort !== undefined) return kuri.getKuriClient(session.brokerPort);
|
|
243
|
-
return kuri.getKuriClient();
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
function selectBrowseBrokerClient(requestedSessionId?: string): kuri.KuriClient {
|
|
247
|
-
if (requestedSessionId) {
|
|
248
|
-
const existing = browseSessions.get(requestedSessionId);
|
|
249
|
-
if (existing?.client) return existing.client as kuri.KuriClient;
|
|
250
|
-
if (existing) return brokerForSession(existing);
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
const loads = new Map<number, number>(browseBrokerPorts().map((port) => [port, 0]));
|
|
254
|
-
for (const session of browseSessions.values()) {
|
|
255
|
-
const port = session.brokerPort ?? BROWSE_BROKER_BASE_PORT;
|
|
256
|
-
loads.set(port, (loads.get(port) ?? 0) + 1);
|
|
257
|
-
}
|
|
258
|
-
const [selectedPort] = [...loads.entries()].sort((a, b) => a[1] - b[1] || a[0] - b[0])[0] ?? [BROWSE_BROKER_BASE_PORT, 0];
|
|
259
|
-
return kuri.getKuriClient(selectedPort);
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
async function loadSkillForMutation(skillId: string, clientScope?: string): Promise<SkillManifest | null> {
|
|
263
|
-
let skill = getRecentLocalSkill(skillId, clientScope);
|
|
264
|
-
if (!skill) {
|
|
265
|
-
for (const [, entry] of domainSkillCache) {
|
|
266
|
-
if (entry.skillId === skillId && entry.localSkillPath) {
|
|
267
|
-
try { skill = JSON.parse(require("fs").readFileSync(entry.localSkillPath, "utf-8")); } catch {}
|
|
268
|
-
break;
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
if (!skill) skill = await getSkill(skillId, clientScope);
|
|
273
|
-
return skill;
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
function buildSkillIndexJob(skill: SkillManifest, clientScope?: string): {
|
|
277
|
-
skill: SkillManifest;
|
|
278
|
-
domain: string;
|
|
279
|
-
intent: string;
|
|
280
|
-
clientScope?: string;
|
|
281
|
-
cacheKey: string;
|
|
282
|
-
} {
|
|
283
|
-
const intent = skill.intent_signature || `browse ${skill.domain}`;
|
|
284
|
-
return {
|
|
285
|
-
skill,
|
|
286
|
-
domain: skill.domain,
|
|
287
|
-
intent,
|
|
288
|
-
clientScope,
|
|
289
|
-
cacheKey: buildResolveCacheKey(skill.domain, intent, undefined),
|
|
290
|
-
};
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
/** Register a browse session from the orchestrator (Phase 4 handoff) */
|
|
294
|
-
export function registerBrowseSession(tabId: string, url: string, domain: string): BrowseSession {
|
|
295
|
-
const client = kuri.getKuriClient();
|
|
296
|
-
return createRegisteredBrowseSession(browseSessions, {
|
|
297
|
-
tabId,
|
|
298
|
-
url,
|
|
299
|
-
harActive: true,
|
|
300
|
-
domain,
|
|
301
|
-
brokerPort: client.getPort(),
|
|
302
|
-
client,
|
|
303
|
-
});
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
// ── /v1/stats cache ──────────────────────────────────────────────────
|
|
307
|
-
let statsCache: { data: unknown; ts: number } | null = null;
|
|
308
|
-
const STATS_CACHE_TTL = 5 * 60 * 1000; // 5 minutes
|
|
309
|
-
|
|
310
|
-
async function fetchStats() {
|
|
311
|
-
if (statsCache && Date.now() - statsCache.ts < STATS_CACHE_TTL) {
|
|
312
|
-
return statsCache.data;
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
const npmPoint = (pkg: string, range: string) =>
|
|
316
|
-
fetch(`https://api.npmjs.org/downloads/point/${range}/${pkg}`)
|
|
317
|
-
.then(r => r.json() as Promise<{ downloads?: number }>);
|
|
318
|
-
|
|
319
|
-
const npmRange = (pkg: string) =>
|
|
320
|
-
fetch(`https://api.npmjs.org/downloads/range/last-month/${pkg}`)
|
|
321
|
-
.then(r => r.json() as Promise<{ downloads?: Array<{ day: string; downloads: number }> }>);
|
|
322
|
-
|
|
323
|
-
const externalCalls: Promise<unknown>[] = [
|
|
324
|
-
npmPoint("unbrowse", "last-month"),
|
|
325
|
-
npmPoint("unbrowse-openclaw", "last-month"),
|
|
326
|
-
npmPoint("unbrowse", "1970-01-01:2099-12-31"),
|
|
327
|
-
npmPoint("unbrowse-openclaw", "1970-01-01:2099-12-31"),
|
|
328
|
-
npmRange("unbrowse"),
|
|
329
|
-
npmRange("unbrowse-openclaw"),
|
|
330
|
-
fetch("https://api.github.com/repos/anthropic-ai/unbrowse", {
|
|
331
|
-
headers: { "User-Agent": "unbrowse-stats" },
|
|
332
|
-
}).then(r => r.json() as Promise<Record<string, unknown>>),
|
|
333
|
-
];
|
|
334
|
-
|
|
335
|
-
// Only call Unkey analytics if the key is available as an env var
|
|
336
|
-
const unkeyAnalyticsKey = process.env.UNKEY_ANALYTICS_KEY;
|
|
337
|
-
if (unkeyAnalyticsKey) {
|
|
338
|
-
externalCalls.push(
|
|
339
|
-
fetch("https://api.unkey.com/v2/analytics.getVerifications", {
|
|
340
|
-
method: "POST",
|
|
341
|
-
headers: {
|
|
342
|
-
Authorization: `Bearer ${unkeyAnalyticsKey}`,
|
|
343
|
-
"Content-Type": "application/json",
|
|
344
|
-
},
|
|
345
|
-
body: JSON.stringify({ apiId: "api_2bUScBc8U6JNsXLrhfHwfqzXHJDi" }),
|
|
346
|
-
}).then(r => r.json() as Promise<unknown>),
|
|
347
|
-
);
|
|
348
|
-
}
|
|
349
|
-
|
|
350
|
-
const [
|
|
351
|
-
unbrowse30d, plugin30d,
|
|
352
|
-
unbrowseAll, pluginAll,
|
|
353
|
-
unbrowseDaily, pluginDaily,
|
|
354
|
-
github,
|
|
355
|
-
...rest
|
|
356
|
-
] = await Promise.allSettled(externalCalls);
|
|
357
|
-
const unkey = rest[0]; // may be undefined if no key
|
|
358
|
-
|
|
359
|
-
const val = <T>(r: PromiseSettledResult<T> | undefined): T | null =>
|
|
360
|
-
r?.status === "fulfilled" ? r.value : null;
|
|
361
|
-
|
|
362
|
-
// npm numbers
|
|
363
|
-
const u30 = val(unbrowse30d)?.downloads ?? null;
|
|
364
|
-
const p30 = val(plugin30d)?.downloads ?? null;
|
|
365
|
-
const uAll = val(unbrowseAll)?.downloads ?? null;
|
|
366
|
-
const pAll = val(pluginAll)?.downloads ?? null;
|
|
367
|
-
|
|
368
|
-
// daily breakdown — merge the two packages by day
|
|
369
|
-
const uDays = val(unbrowseDaily)?.downloads ?? [];
|
|
370
|
-
const pDays = val(pluginDaily)?.downloads ?? [];
|
|
371
|
-
const dayMap = new Map<string, { unbrowse: number; plugin: number }>();
|
|
372
|
-
for (const d of uDays) dayMap.set(d.day, { unbrowse: d.downloads, plugin: 0 });
|
|
373
|
-
for (const d of pDays) {
|
|
374
|
-
const entry = dayMap.get(d.day);
|
|
375
|
-
if (entry) entry.plugin = d.downloads;
|
|
376
|
-
else dayMap.set(d.day, { unbrowse: 0, plugin: d.downloads });
|
|
377
|
-
}
|
|
378
|
-
const daily = [...dayMap.entries()]
|
|
379
|
-
.sort(([a], [b]) => a.localeCompare(b))
|
|
380
|
-
.map(([day, v]) => ({ day, unbrowse: v.unbrowse, plugin: v.plugin, total: v.unbrowse + v.plugin }));
|
|
381
|
-
|
|
382
|
-
// github
|
|
383
|
-
const gh = val(github);
|
|
384
|
-
const githubData = gh && typeof gh === "object"
|
|
385
|
-
? {
|
|
386
|
-
stars: (gh as Record<string, number>).stargazers_count ?? null,
|
|
387
|
-
forks: (gh as Record<string, number>).forks_count ?? null,
|
|
388
|
-
open_issues: (gh as Record<string, number>).open_issues_count ?? null,
|
|
389
|
-
watchers: (gh as Record<string, number>).watchers_count ?? null,
|
|
390
|
-
}
|
|
391
|
-
: { stars: null, forks: null, open_issues: null, watchers: null };
|
|
392
|
-
|
|
393
|
-
// unkey
|
|
394
|
-
let agentsData: { total_api_calls_30d: number | null; note?: string } = {
|
|
395
|
-
total_api_calls_30d: null,
|
|
396
|
-
note: "unkey analytics unavailable",
|
|
397
|
-
};
|
|
398
|
-
const uk = val(unkey);
|
|
399
|
-
if (uk && Array.isArray(uk)) {
|
|
400
|
-
const total = (uk as Array<{ total?: number }>).reduce((s, v) => s + (v.total ?? 0), 0);
|
|
401
|
-
agentsData = { total_api_calls_30d: total };
|
|
402
|
-
} else if (uk && typeof uk === "object" && (uk as Record<string, unknown>).total != null) {
|
|
403
|
-
agentsData = { total_api_calls_30d: (uk as Record<string, number>).total };
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
const data = {
|
|
407
|
-
npm: {
|
|
408
|
-
unbrowse: { last_30d: u30, all_time: uAll },
|
|
409
|
-
openclaw_plugin: { last_30d: p30, all_time: pAll },
|
|
410
|
-
combined: {
|
|
411
|
-
last_30d: u30 != null && p30 != null ? u30 + p30 : (u30 ?? p30),
|
|
412
|
-
all_time: uAll != null && pAll != null ? uAll + pAll : (uAll ?? pAll),
|
|
413
|
-
},
|
|
414
|
-
daily,
|
|
415
|
-
},
|
|
416
|
-
github: githubData,
|
|
417
|
-
agents: agentsData,
|
|
418
|
-
fetched_at: new Date().toISOString(),
|
|
419
|
-
};
|
|
420
|
-
|
|
421
|
-
statsCache = { data, ts: Date.now() };
|
|
422
|
-
return data;
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
export async function registerRoutes(app: FastifyInstance) {
|
|
426
|
-
const clientScopeFor = (req: { headers: Record<string, unknown>; id: string }) =>
|
|
427
|
-
(typeof req.headers["x-unbrowse-client-id"] === "string" && req.headers["x-unbrowse-client-id"].trim())
|
|
428
|
-
? req.headers["x-unbrowse-client-id"].trim()
|
|
429
|
-
: req.id;
|
|
430
|
-
|
|
431
|
-
function checkpointPublishCommand(skillId: string | null, confirmPublish = false): string {
|
|
432
|
-
return skillId
|
|
433
|
-
? `unbrowse publish --skill ${skillId}${confirmPublish ? " --confirm-publish" : ""}`
|
|
434
|
-
: `unbrowse publish --skill <skill_id>${confirmPublish ? " --confirm-publish" : ""}`;
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
function buildPublishFailureNextStep(skillId: string, validationErrors?: string[]): string {
|
|
438
|
-
const reviewErrors = (validationErrors ?? []).filter((error) => error.startsWith("review_required:"));
|
|
439
|
-
if (reviewErrors.length > 0) {
|
|
440
|
-
return `Remote share blocked: ${reviewErrors.length} endpoint(s) still need review. Re-run ${checkpointPublishCommand(skillId)} to inspect review_context, then publish again with reviewed endpoints.`;
|
|
441
|
-
}
|
|
442
|
-
return `Remote share did not complete. Inspect validation_errors, adjust the contract locally, then retry ${checkpointPublishCommand(skillId, true)}.`;
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
function buildCheckpointNextStep(
|
|
446
|
-
action: "sync" | "close",
|
|
447
|
-
result: {
|
|
448
|
-
skill_id: string | null;
|
|
449
|
-
pipeline: {
|
|
450
|
-
index_queued: boolean;
|
|
451
|
-
publish_queued: boolean;
|
|
452
|
-
};
|
|
453
|
-
publish_policy: {
|
|
454
|
-
mode: "auto" | "disabled" | "blacklisted" | "prompt";
|
|
455
|
-
reason: string;
|
|
456
|
-
matched_domain?: string;
|
|
457
|
-
};
|
|
458
|
-
},
|
|
459
|
-
sessionId?: string,
|
|
460
|
-
): string {
|
|
461
|
-
const sessionHint = sessionId ? ` --session ${sessionId}` : "";
|
|
462
|
-
const publishCommand = checkpointPublishCommand(
|
|
463
|
-
result.skill_id,
|
|
464
|
-
result.publish_policy.mode === "blacklisted" || result.publish_policy.mode === "prompt",
|
|
465
|
-
);
|
|
466
|
-
|
|
467
|
-
if (!result.pipeline.index_queued) {
|
|
468
|
-
return action === "sync"
|
|
469
|
-
? `Checkpoint recorded, but no new capture was available to index. Continue browsing, then run \`unbrowse close${sessionHint}\` for the final checkpoint.`
|
|
470
|
-
: "Final checkpoint recorded, but no new capture was available to index or publish.";
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
if (result.publish_policy.mode === "auto") {
|
|
474
|
-
return action === "sync"
|
|
475
|
-
? `Checkpoint saved. Background index + publish queued. Continue browsing, then run \`unbrowse close${sessionHint}\` for the final checkpoint.`
|
|
476
|
-
: "Final checkpoint saved. Background index + publish queued. Inspect the indexed contract or wait for publish to complete.";
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
const base = result.publish_policy.mode === "disabled"
|
|
480
|
-
? "Checkpoint saved. Local index queued, but auto-publish is disabled in settings."
|
|
481
|
-
: `Checkpoint saved. Local index queued, but auto-publish did not run: ${result.publish_policy.reason}`;
|
|
482
|
-
|
|
483
|
-
const suffix = result.skill_id
|
|
484
|
-
? ` Review the indexed contract, then run \`${publishCommand}\` only if you own this index.`
|
|
485
|
-
: "";
|
|
486
|
-
|
|
487
|
-
if (action === "sync") {
|
|
488
|
-
return `${base} Continue browsing, then run \`unbrowse close${sessionHint}\` when done.${suffix}`;
|
|
489
|
-
}
|
|
490
|
-
return `${base}${suffix}`;
|
|
491
|
-
}
|
|
492
|
-
|
|
493
|
-
// Auth gate: block all routes except /health when no API key is configured
|
|
494
|
-
app.addHook("onRequest", async (req, reply) => {
|
|
495
|
-
if (req.url === "/health" || req.url === "/v1/stats" || req.url.startsWith("/v1/settings")) return;
|
|
496
|
-
|
|
497
|
-
const key = getApiKey();
|
|
498
|
-
if (!key) {
|
|
499
|
-
return reply.code(401).send({
|
|
500
|
-
error: "api_key_required",
|
|
501
|
-
message: "No API key configured. Restart the server to auto-register, or run: bash scripts/setup.sh",
|
|
502
|
-
docs_url: "https://unbrowse.ai",
|
|
503
|
-
});
|
|
504
|
-
}
|
|
505
|
-
});
|
|
506
|
-
|
|
507
|
-
app.get("/v1/settings", async (_req, reply) => {
|
|
508
|
-
return reply.send({
|
|
509
|
-
capture_pipeline: getCapturePipelineSettings(),
|
|
510
|
-
});
|
|
511
|
-
});
|
|
512
|
-
|
|
513
|
-
app.post("/v1/settings", async (req, reply) => {
|
|
514
|
-
const body = (req.body ?? {}) as {
|
|
515
|
-
auto_publish_checkpoints?: boolean;
|
|
516
|
-
publish_domain_blacklist?: string[];
|
|
517
|
-
publish_domain_promptlist?: string[];
|
|
518
|
-
clear_publish_domain_blacklist?: boolean;
|
|
519
|
-
clear_publish_domain_promptlist?: boolean;
|
|
520
|
-
};
|
|
521
|
-
|
|
522
|
-
const settings = updateCapturePipelineSettings({
|
|
523
|
-
auto_publish_checkpoints: typeof body.auto_publish_checkpoints === "boolean"
|
|
524
|
-
? body.auto_publish_checkpoints
|
|
525
|
-
: undefined,
|
|
526
|
-
publish_domain_blacklist: Array.isArray(body.publish_domain_blacklist)
|
|
527
|
-
? body.publish_domain_blacklist
|
|
528
|
-
: undefined,
|
|
529
|
-
publish_domain_promptlist: Array.isArray(body.publish_domain_promptlist)
|
|
530
|
-
? body.publish_domain_promptlist
|
|
531
|
-
: undefined,
|
|
532
|
-
clear_publish_domain_blacklist: body.clear_publish_domain_blacklist === true,
|
|
533
|
-
clear_publish_domain_promptlist: body.clear_publish_domain_promptlist === true,
|
|
534
|
-
});
|
|
535
|
-
|
|
536
|
-
return reply.send({
|
|
537
|
-
ok: true,
|
|
538
|
-
capture_pipeline: settings,
|
|
539
|
-
next_step: settings.auto_publish_checkpoints
|
|
540
|
-
? "Auto-publish after sync/close is enabled unless a domain rule blocks it."
|
|
541
|
-
: "Auto-publish after sync/close is disabled. Use index for local recompute and publish only when you explicitly want remote share.",
|
|
542
|
-
});
|
|
543
|
-
});
|
|
544
|
-
|
|
545
|
-
// POST /v1/intent/resolve
|
|
546
|
-
app.post("/v1/intent/resolve", { config: { rateLimit: ROUTE_LIMITS["/v1/intent/resolve"] } }, async (req, reply) => {
|
|
547
|
-
const clientScope = clientScopeFor(req);
|
|
548
|
-
const { intent, params, context, projection, confirm_unsafe, confirm_third_party_terms, dry_run, force_capture } = req.body as {
|
|
549
|
-
intent: string;
|
|
550
|
-
params?: Record<string, unknown>;
|
|
551
|
-
context?: { url?: string; domain?: string };
|
|
552
|
-
projection?: ProjectionOptions;
|
|
553
|
-
confirm_unsafe?: boolean;
|
|
554
|
-
confirm_third_party_terms?: boolean;
|
|
555
|
-
dry_run?: boolean;
|
|
556
|
-
force_capture?: boolean;
|
|
557
|
-
};
|
|
558
|
-
if (!intent) return reply.code(400).send({ error: "intent required" });
|
|
559
|
-
try {
|
|
560
|
-
const result = await resolveAndExecute(intent, params ?? {}, context, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, force_capture, client_scope: clientScope });
|
|
561
|
-
|
|
562
|
-
// Surface timing breakdown
|
|
563
|
-
const res = attachAgentOutcomeHints({ ...result } as Record<string, unknown>, {
|
|
564
|
-
skill: result.skill,
|
|
565
|
-
endpointId: result.trace.endpoint_id,
|
|
566
|
-
timing: result.timing,
|
|
567
|
-
});
|
|
568
|
-
if (result.timing) {
|
|
569
|
-
res.timing = result.timing;
|
|
570
|
-
}
|
|
571
|
-
|
|
572
|
-
// If the orchestrator already included available_endpoints in result (deferral),
|
|
573
|
-
// also append them at the top level for backward compatibility.
|
|
574
|
-
const innerResult = result.result as Record<string, unknown> | null;
|
|
575
|
-
if (innerResult?.available_endpoints && !res.available_endpoints) {
|
|
576
|
-
res.available_endpoints = innerResult.available_endpoints;
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
await recordAnalyticsSession(buildAnalyticsSessionPayload(result, {
|
|
580
|
-
discovery_queries: 1,
|
|
581
|
-
})).catch(() => {});
|
|
582
|
-
|
|
583
|
-
return reply.send(res);
|
|
584
|
-
} catch (err) {
|
|
585
|
-
return reply.code(500).send({ error: (err as Error).message });
|
|
586
|
-
}
|
|
587
|
-
});
|
|
588
|
-
|
|
589
|
-
// GET /v1/skills/:skill_id — local route so skill lookups hit disk cache before proxying to backend
|
|
590
|
-
app.get("/v1/skills/:skill_id", async (req, reply) => {
|
|
591
|
-
const clientScope = clientScopeFor(req);
|
|
592
|
-
const { skill_id } = req.params as { skill_id: string };
|
|
593
|
-
// Check local caches: recent skills → domain snapshots → marketplace
|
|
594
|
-
let skill = getRecentLocalSkill(skill_id, clientScope);
|
|
595
|
-
if (!skill) {
|
|
596
|
-
for (const [, entry] of domainSkillCache) {
|
|
597
|
-
if (entry.skillId === skill_id && entry.localSkillPath) {
|
|
598
|
-
try { skill = JSON.parse(require("fs").readFileSync(entry.localSkillPath, "utf-8")); } catch {}
|
|
599
|
-
break;
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
}
|
|
603
|
-
if (!skill) skill = await getSkill(skill_id, clientScope);
|
|
604
|
-
if (!skill) return reply.code(404).send({ error: "Skill not found" });
|
|
605
|
-
return reply.send(skill);
|
|
606
|
-
});
|
|
607
|
-
|
|
608
|
-
// POST /v1/skills/:skill_id/review — agent submits reviewed descriptions + synthetic examples
|
|
609
|
-
app.post("/v1/skills/:skill_id/review", async (req, reply) => {
|
|
610
|
-
const clientScope = clientScopeFor(req);
|
|
611
|
-
const { skill_id } = req.params as { skill_id: string };
|
|
612
|
-
const { endpoints: reviews } = req.body as {
|
|
613
|
-
endpoints: import("../types/index.js").EndpointReviewPayload[];
|
|
614
|
-
};
|
|
615
|
-
if (!reviews?.length) return reply.code(400).send({ error: "endpoints[] required" });
|
|
616
|
-
|
|
617
|
-
let skill = await loadSkillForMutation(skill_id, clientScope);
|
|
618
|
-
if (!skill) return reply.code(404).send({ error: "Skill not found" });
|
|
619
|
-
|
|
620
|
-
const updated = mergeAgentReview(skill.endpoints, reviews);
|
|
621
|
-
skill.endpoints = updated;
|
|
622
|
-
skill.updated_at = new Date().toISOString();
|
|
623
|
-
const updatedWorkflowArtifact = applyWorkflowSchemaReviews(readWorkflowArtifact(skill.skill_id), reviews);
|
|
624
|
-
if (updatedWorkflowArtifact) writeWorkflowArtifact(updatedWorkflowArtifact);
|
|
625
|
-
|
|
626
|
-
const indexed = await indexSkillLocally(buildSkillIndexJob(skill, clientScope));
|
|
627
|
-
return reply.send({
|
|
628
|
-
ok: true,
|
|
629
|
-
skill_id: indexed.skill.skill_id,
|
|
630
|
-
endpoints_updated: reviews.length,
|
|
631
|
-
indexed: true,
|
|
632
|
-
publish_status: "indexed",
|
|
633
|
-
endpoint_count: indexed.skill.endpoints.length,
|
|
634
|
-
});
|
|
635
|
-
});
|
|
636
|
-
|
|
637
|
-
// POST /v1/skills/:skill_id/index — local-only graph/export recompute from cached state
|
|
638
|
-
app.post("/v1/skills/:skill_id/index", async (req, reply) => {
|
|
639
|
-
const clientScope = clientScopeFor(req);
|
|
640
|
-
const { skill_id } = req.params as { skill_id: string };
|
|
641
|
-
const skill = await loadSkillForMutation(skill_id, clientScope);
|
|
642
|
-
if (!skill) return reply.code(404).send({ error: "Skill not found" });
|
|
643
|
-
|
|
644
|
-
const indexed = await indexSkillLocally(buildSkillIndexJob(skill, clientScope));
|
|
645
|
-
return reply.send({
|
|
646
|
-
ok: true,
|
|
647
|
-
skill_id: indexed.skill.skill_id,
|
|
648
|
-
indexed: true,
|
|
649
|
-
publish_status: "indexed",
|
|
650
|
-
endpoint_count: indexed.skill.endpoints.length,
|
|
651
|
-
domain: indexed.domain,
|
|
652
|
-
next_step: `Local contracts re-indexed. Review them, then run ${checkpointPublishCommand(indexed.skill.skill_id)} when you explicitly want remote share.`,
|
|
653
|
-
});
|
|
654
|
-
});
|
|
655
|
-
|
|
656
|
-
// POST /v1/skills/:skill_id/publish — two-phase agent-driven publish
|
|
657
|
-
// Phase 1 (no endpoints body): re-index locally, then return endpoints needing descriptions
|
|
658
|
-
// Phase 2 (with endpoints): merge descriptions, re-index locally, then publish remotely
|
|
659
|
-
app.post("/v1/skills/:skill_id/publish", async (req, reply) => {
|
|
660
|
-
const clientScope = clientScopeFor(req);
|
|
661
|
-
const { skill_id } = req.params as { skill_id: string };
|
|
662
|
-
const { endpoints: reviews, confirm_publish } = (req.body as {
|
|
663
|
-
endpoints?: import("../types/index.js").EndpointReviewPayload[];
|
|
664
|
-
confirm_publish?: boolean;
|
|
665
|
-
}) ?? {};
|
|
666
|
-
|
|
667
|
-
let skill = await loadSkillForMutation(skill_id, clientScope);
|
|
668
|
-
if (!skill) return reply.code(404).send({ error: "Skill not found" });
|
|
669
|
-
|
|
670
|
-
if (reviews?.length) {
|
|
671
|
-
const publishDecision = decideExplicitPublish(skill.domain, confirm_publish === true);
|
|
672
|
-
if (!publishDecision.allowed) {
|
|
673
|
-
return reply.code(409).send({
|
|
674
|
-
error: "publish_confirmation_required",
|
|
675
|
-
domain: skill.domain,
|
|
676
|
-
publish_policy: {
|
|
677
|
-
mode: publishDecision.mode,
|
|
678
|
-
reason: publishDecision.reason,
|
|
679
|
-
matched_domain: publishDecision.matchedDomain,
|
|
680
|
-
},
|
|
681
|
-
next_step: `Re-run ${checkpointPublishCommand(skill.skill_id, true)} only if you own this index.`,
|
|
682
|
-
});
|
|
683
|
-
}
|
|
684
|
-
const updated = mergeAgentReview(skill.endpoints, reviews);
|
|
685
|
-
skill.endpoints = updated;
|
|
686
|
-
skill.updated_at = new Date().toISOString();
|
|
687
|
-
const updatedWorkflowArtifact = applyWorkflowSchemaReviews(readWorkflowArtifact(skill.skill_id), reviews);
|
|
688
|
-
if (updatedWorkflowArtifact) writeWorkflowArtifact(updatedWorkflowArtifact);
|
|
689
|
-
const indexed = await indexSkillLocally(buildSkillIndexJob(skill, clientScope));
|
|
690
|
-
const publishResult = await publishIndexedSkill(indexed);
|
|
691
|
-
return reply.send({
|
|
692
|
-
ok: true,
|
|
693
|
-
skill_id: skill.skill_id,
|
|
694
|
-
endpoints_updated: reviews.length,
|
|
695
|
-
indexed: true,
|
|
696
|
-
published: publishResult.published,
|
|
697
|
-
publish_status: publishResult.publishStatus,
|
|
698
|
-
...(publishResult.publishedAt ? { published_at: publishResult.publishedAt } : {}),
|
|
699
|
-
...(publishResult.validationErrors ? { validation_errors: publishResult.validationErrors } : {}),
|
|
700
|
-
next_step: publishResult.publishStatus === "published"
|
|
701
|
-
? "Remote share completed. Re-run resolve/skill inspection to use the published contract."
|
|
702
|
-
: buildPublishFailureNextStep(skill.skill_id, publishResult.validationErrors),
|
|
703
|
-
});
|
|
704
|
-
}
|
|
705
|
-
|
|
706
|
-
const indexed = await indexSkillLocally(buildSkillIndexJob(skill, clientScope));
|
|
707
|
-
const ranked = rankEndpoints(indexed.skill.endpoints, indexed.skill.intent_signature, indexed.skill.domain);
|
|
708
|
-
const endpoints_to_describe = ranked.map((r) => {
|
|
709
|
-
const descriptionMeta = getEndpointDescriptionMetadata(r.endpoint);
|
|
710
|
-
return {
|
|
711
|
-
endpoint_id: r.endpoint.endpoint_id,
|
|
712
|
-
method: r.endpoint.method,
|
|
713
|
-
url: r.endpoint.url_template.length > 120
|
|
714
|
-
? r.endpoint.url_template.slice(0, 120) + "..."
|
|
715
|
-
: r.endpoint.url_template,
|
|
716
|
-
current_description: descriptionMeta.display,
|
|
717
|
-
description_source: descriptionMeta.source,
|
|
718
|
-
description_needs_review: descriptionMeta.needs_review,
|
|
719
|
-
...(descriptionMeta.warning ? { description_warning: descriptionMeta.warning } : {}),
|
|
720
|
-
schema_summary: r.endpoint.response_schema
|
|
721
|
-
? summarizeSchema(r.endpoint.response_schema)
|
|
722
|
-
: null,
|
|
723
|
-
sample_values: extractSampleValues(r.endpoint.semantic?.example_response_compact),
|
|
724
|
-
input_params: r.endpoint.semantic?.requires?.map((b) => ({
|
|
725
|
-
key: b.key,
|
|
726
|
-
type: b.type ?? b.semantic_type,
|
|
727
|
-
required: b.required ?? false,
|
|
728
|
-
example: b.example_value,
|
|
729
|
-
})) ?? [],
|
|
730
|
-
dom_extraction: !!r.endpoint.dom_extraction,
|
|
731
|
-
review_context: buildEndpointReviewContext(indexed.skill, r.endpoint.endpoint_id),
|
|
732
|
-
_fill_description:
|
|
733
|
-
"DESCRIBE THIS ENDPOINT — what it returns, key params, action type, and any audience/eligibility/pricing/validity constraints",
|
|
734
|
-
};
|
|
735
|
-
});
|
|
736
|
-
|
|
737
|
-
return reply.send({
|
|
738
|
-
skill_id: indexed.skill.skill_id,
|
|
739
|
-
domain: indexed.skill.domain,
|
|
740
|
-
indexed: true,
|
|
741
|
-
publish_status: "indexed",
|
|
742
|
-
endpoint_count: indexed.skill.endpoints.length,
|
|
743
|
-
endpoints_to_describe,
|
|
744
|
-
next_step:
|
|
745
|
-
`Fill each endpoint's description using review_context (deps, request_schema, response_schema, provenance, trigger page) plus any audience/eligibility/pricing/validity caveats, then call: unbrowse publish --skill ${indexed.skill.skill_id} --endpoints '[{endpoint_id, description, action_kind, resource_kind}]'`,
|
|
746
|
-
_next_step:
|
|
747
|
-
`Fill each endpoint's description using review_context (deps, request_schema, response_schema, provenance, trigger page) plus any audience/eligibility/pricing/validity caveats, then call: unbrowse publish --skill ${indexed.skill.skill_id} --endpoints '[{endpoint_id, description, action_kind, resource_kind}]'`,
|
|
748
|
-
});
|
|
749
|
-
});
|
|
750
|
-
|
|
751
|
-
// POST /v1/foundry/publish-bundle — derive bundle/share/host artifacts from one preset
|
|
752
|
-
app.post("/v1/foundry/publish-bundle", async (req, reply) => {
|
|
753
|
-
const { preset_path, site_url, hosts } = (req.body as {
|
|
754
|
-
preset_path?: string;
|
|
755
|
-
site_url?: string;
|
|
756
|
-
hosts?: string[];
|
|
757
|
-
}) ?? {};
|
|
758
|
-
|
|
759
|
-
if (!preset_path?.trim()) {
|
|
760
|
-
return reply.code(400).send({ error: "preset_path is required" });
|
|
761
|
-
}
|
|
762
|
-
|
|
763
|
-
try {
|
|
764
|
-
const result = publishFoundryBundle({
|
|
765
|
-
presetPath: preset_path,
|
|
766
|
-
...(site_url?.trim() ? { siteUrl: site_url } : {}),
|
|
767
|
-
...(Array.isArray(hosts) && hosts.length > 0 ? {
|
|
768
|
-
hosts: hosts.filter((host): host is "codex" | "claude" | "openclaw" =>
|
|
769
|
-
host === "codex" || host === "claude" || host === "openclaw",
|
|
770
|
-
),
|
|
771
|
-
} : {}),
|
|
772
|
-
});
|
|
773
|
-
return reply.send(result);
|
|
774
|
-
} catch (error) {
|
|
775
|
-
return reply.code(400).send({
|
|
776
|
-
error: error instanceof Error ? error.message : String(error),
|
|
777
|
-
});
|
|
778
|
-
}
|
|
779
|
-
});
|
|
780
|
-
// POST /v1/skills/:skill_id/chunk — dynamic subgraph load for the current intent/bindings
|
|
781
|
-
app.post("/v1/skills/:skill_id/chunk", async (req, reply) => {
|
|
782
|
-
const clientScope = clientScopeFor(req);
|
|
783
|
-
const { skill_id } = req.params as { skill_id: string };
|
|
784
|
-
const { intent, operation_id, known_bindings, max_operations } = req.body as {
|
|
785
|
-
intent?: string;
|
|
786
|
-
operation_id?: string;
|
|
787
|
-
known_bindings?: Record<string, unknown>;
|
|
788
|
-
max_operations?: number;
|
|
789
|
-
};
|
|
790
|
-
const skill = getRecentLocalSkill(skill_id, clientScope) ?? await getSkill(skill_id, clientScope);
|
|
791
|
-
if (!skill) return reply.code(404).send({ error: "Skill not found" });
|
|
792
|
-
return reply.send(toAgentSkillChunkView(getSkillChunk(skill, {
|
|
793
|
-
intent,
|
|
794
|
-
seed_operation_id: operation_id,
|
|
795
|
-
known_bindings,
|
|
796
|
-
max_operations,
|
|
797
|
-
})));
|
|
798
|
-
});
|
|
799
|
-
|
|
800
|
-
// POST /v1/skills/:skill_id/execute
|
|
801
|
-
app.post("/v1/skills/:skill_id/execute", { config: { rateLimit: ROUTE_LIMITS["/v1/skills/:skill_id/execute"] } }, async (req, reply) => {
|
|
802
|
-
const clientScope = clientScopeFor(req);
|
|
803
|
-
const { skill_id } = req.params as { skill_id: string };
|
|
804
|
-
const { params, projection, confirm_unsafe, confirm_third_party_terms, dry_run, intent, context_url } = req.body as {
|
|
805
|
-
params?: Record<string, unknown>;
|
|
806
|
-
projection?: ProjectionOptions;
|
|
807
|
-
confirm_unsafe?: boolean;
|
|
808
|
-
confirm_third_party_terms?: boolean;
|
|
809
|
-
dry_run?: boolean;
|
|
810
|
-
intent?: string;
|
|
811
|
-
context_url?: string;
|
|
812
|
-
};
|
|
813
|
-
// Check local caches first: recent skills → domain snapshots → marketplace
|
|
814
|
-
let skill = getRecentLocalSkill(skill_id, clientScope);
|
|
815
|
-
if (!skill) {
|
|
816
|
-
// Check domain snapshot cache — passively indexed skills live here
|
|
817
|
-
const { findExistingSkillForDomain: findLocal } = await import("../client/index.js");
|
|
818
|
-
for (const [, entry] of domainSkillCache) {
|
|
819
|
-
if (entry.skillId === skill_id && entry.localSkillPath) {
|
|
820
|
-
try {
|
|
821
|
-
skill = JSON.parse(require("fs").readFileSync(entry.localSkillPath, "utf-8"));
|
|
822
|
-
} catch { /* snapshot read failed */ }
|
|
823
|
-
break;
|
|
824
|
-
}
|
|
825
|
-
}
|
|
826
|
-
}
|
|
827
|
-
if (!skill) skill = await getSkill(skill_id, clientScope);
|
|
828
|
-
if (!skill) return reply.code(404).send({ error: "Skill not found" });
|
|
829
|
-
const execParams = {
|
|
830
|
-
...(params ?? {}),
|
|
831
|
-
...(context_url && typeof params?.url !== "string" ? { url: context_url } : {}),
|
|
832
|
-
};
|
|
833
|
-
try {
|
|
834
|
-
const execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, intent, contextUrl: context_url, client_scope: clientScope });
|
|
835
|
-
saveTrace(execResult.trace);
|
|
836
|
-
if (execResult.trace.endpoint_id) {
|
|
837
|
-
recordExecution(skill.skill_id, execResult.trace.endpoint_id, execResult.trace, skill).catch(() => {});
|
|
838
|
-
}
|
|
839
|
-
if (execResult.trace.success) {
|
|
840
|
-
promoteExplicitExecution(
|
|
841
|
-
clientScope,
|
|
842
|
-
intent || skill.intent_signature,
|
|
843
|
-
context_url || (typeof execParams.url === "string" ? execParams.url : undefined),
|
|
844
|
-
skill,
|
|
845
|
-
execResult.trace.endpoint_id,
|
|
846
|
-
execResult.result,
|
|
847
|
-
);
|
|
848
|
-
}
|
|
849
|
-
|
|
850
|
-
// Auto-recovery: if endpoint returned 404 (stale), re-capture via orchestrator
|
|
851
|
-
if (
|
|
852
|
-
execResult.trace.status_code === 404 &&
|
|
853
|
-
skill.domain &&
|
|
854
|
-
skill.intent_signature &&
|
|
855
|
-
skill.execution_type !== "browser-capture"
|
|
856
|
-
) {
|
|
857
|
-
try {
|
|
858
|
-
const recoveryUrl =
|
|
859
|
-
context_url ||
|
|
860
|
-
(typeof execParams.url === "string" && execParams.url) ||
|
|
861
|
-
skill.endpoints.find((endpoint) => typeof endpoint.trigger_url === "string" && endpoint.trigger_url)?.trigger_url ||
|
|
862
|
-
`https://${skill.domain}`;
|
|
863
|
-
const freshResult = await resolveAndExecute(
|
|
864
|
-
intent || skill.intent_signature,
|
|
865
|
-
{ ...execParams, url: recoveryUrl },
|
|
866
|
-
{ url: recoveryUrl },
|
|
867
|
-
projection,
|
|
868
|
-
{ confirm_unsafe, confirm_third_party_terms, dry_run, intent: intent || skill.intent_signature, client_scope: clientScope }
|
|
869
|
-
);
|
|
870
|
-
saveTrace(freshResult.trace);
|
|
871
|
-
if (freshResult.trace?.skill_id && freshResult.trace?.endpoint_id) {
|
|
872
|
-
recordExecution(freshResult.trace.skill_id, freshResult.trace.endpoint_id, freshResult.trace, skill).catch(() => {});
|
|
873
|
-
}
|
|
874
|
-
await recordAnalyticsSession(buildAnalyticsSessionPayload(freshResult, {
|
|
875
|
-
discovery_queries: 1,
|
|
876
|
-
})).catch(() => {});
|
|
877
|
-
const recovered = attachAgentOutcomeHints({
|
|
878
|
-
...freshResult,
|
|
879
|
-
_recovery: {
|
|
880
|
-
reason: "stale_endpoint_404",
|
|
881
|
-
original_skill_id: skill_id,
|
|
882
|
-
message: "Original endpoint returned 404. Auto-recovered with fresh capture.",
|
|
883
|
-
},
|
|
884
|
-
} as Record<string, unknown>, {
|
|
885
|
-
skill: freshResult.skill ?? skill,
|
|
886
|
-
endpointId: freshResult.trace.endpoint_id,
|
|
887
|
-
timing: freshResult.timing,
|
|
888
|
-
});
|
|
889
|
-
return reply.send({
|
|
890
|
-
...recovered,
|
|
891
|
-
});
|
|
892
|
-
} catch {
|
|
893
|
-
// Recovery failed — return original 404 with guidance
|
|
894
|
-
}
|
|
895
|
-
}
|
|
896
|
-
|
|
897
|
-
await recordAnalyticsSession(buildAnalyticsSessionPayload(execResult, {
|
|
898
|
-
discovery_queries: 0,
|
|
899
|
-
})).catch(() => {});
|
|
900
|
-
|
|
901
|
-
const response = attachAgentOutcomeHints({ ...execResult } as Record<string, unknown>, {
|
|
902
|
-
skill,
|
|
903
|
-
endpointId: execResult.trace.endpoint_id,
|
|
904
|
-
});
|
|
905
|
-
return reply.send(response);
|
|
906
|
-
} catch (err) {
|
|
907
|
-
return reply.code(500).send({ error: (err as Error).message });
|
|
908
|
-
}
|
|
909
|
-
});
|
|
910
|
-
|
|
911
|
-
// POST /v1/skills/:skill_id/auth -- store credentials (cookies/headers) for a skill
|
|
912
|
-
app.post("/v1/skills/:skill_id/auth", async (req, reply) => {
|
|
913
|
-
const { skill_id } = req.params as { skill_id: string };
|
|
914
|
-
const skill = await getSkill(skill_id);
|
|
915
|
-
if (!skill) return reply.code(404).send({ error: "Skill not found" });
|
|
916
|
-
|
|
917
|
-
const body = req.body as {
|
|
918
|
-
cookies?: Array<{ name: string; value: string; domain: string; path?: string }>;
|
|
919
|
-
headers?: Record<string, string>;
|
|
920
|
-
};
|
|
921
|
-
if (!body.cookies && !body.headers) {
|
|
922
|
-
return reply.code(400).send({ error: "Provide cookies or headers" });
|
|
923
|
-
}
|
|
924
|
-
|
|
925
|
-
const ref = `${skill.domain}-session`;
|
|
926
|
-
await storeCredential(ref, JSON.stringify({ cookies: body.cookies ?? [], headers: body.headers ?? {} }));
|
|
927
|
-
|
|
928
|
-
// Patch the skill manifest to reference the stored credentials
|
|
929
|
-
if (!skill.auth_profile_ref) {
|
|
930
|
-
await publishSkill({ ...skill, auth_profile_ref: ref });
|
|
931
|
-
}
|
|
932
|
-
|
|
933
|
-
return reply.send({ ok: true, auth_profile_ref: ref });
|
|
934
|
-
});
|
|
935
|
-
|
|
936
|
-
// POST /v1/auth/login — interactive OAuth flow or direct browser cookie extraction
|
|
937
|
-
app.post("/v1/auth/login", { config: { rateLimit: ROUTE_LIMITS["/v1/auth/login"] } }, async (req, reply) => {
|
|
938
|
-
const {
|
|
939
|
-
url,
|
|
940
|
-
browser,
|
|
941
|
-
chrome_profile,
|
|
942
|
-
firefox_profile,
|
|
943
|
-
chromium_profile,
|
|
944
|
-
chromium_user_data_dir,
|
|
945
|
-
chromium_cookie_db_path,
|
|
946
|
-
safe_storage_service,
|
|
947
|
-
browser_name,
|
|
948
|
-
} = req.body as {
|
|
949
|
-
url: string;
|
|
950
|
-
browser?: "auto" | "firefox" | "chrome" | "chromium";
|
|
951
|
-
chrome_profile?: string;
|
|
952
|
-
firefox_profile?: string;
|
|
953
|
-
chromium_profile?: string;
|
|
954
|
-
chromium_user_data_dir?: string;
|
|
955
|
-
chromium_cookie_db_path?: string;
|
|
956
|
-
safe_storage_service?: string;
|
|
957
|
-
browser_name?: string;
|
|
958
|
-
};
|
|
959
|
-
if (!url) return reply.code(400).send({ error: "url required" });
|
|
960
|
-
try {
|
|
961
|
-
const result = await loginWithBrowserFallback(url, {
|
|
962
|
-
browser,
|
|
963
|
-
chromeProfile: chrome_profile,
|
|
964
|
-
firefoxProfile: firefox_profile,
|
|
965
|
-
chromium: {
|
|
966
|
-
profile: chromium_profile,
|
|
967
|
-
userDataDir: chromium_user_data_dir,
|
|
968
|
-
cookieDbPath: chromium_cookie_db_path,
|
|
969
|
-
safeStorageService: safe_storage_service,
|
|
970
|
-
browserName: browser_name,
|
|
971
|
-
},
|
|
972
|
-
});
|
|
973
|
-
return reply.send(result);
|
|
974
|
-
} catch (err) {
|
|
975
|
-
return reply.code(500).send({ error: (err as Error).message });
|
|
976
|
-
}
|
|
977
|
-
});
|
|
978
|
-
|
|
979
|
-
// POST /v1/auth/steal — extract cookies from Firefox/Chrome/custom Chromium-family SQLite DBs.
|
|
980
|
-
// No browser launch, Chrome can stay open. Higher rate limit since it's instant.
|
|
981
|
-
app.post("/v1/auth/steal", { config: { rateLimit: { max: 30, timeWindow: "1 minute" } } }, async (req, reply) => {
|
|
982
|
-
const {
|
|
983
|
-
url,
|
|
984
|
-
browser,
|
|
985
|
-
chrome_profile,
|
|
986
|
-
firefox_profile,
|
|
987
|
-
chromium_profile,
|
|
988
|
-
chromium_user_data_dir,
|
|
989
|
-
chromium_cookie_db_path,
|
|
990
|
-
safe_storage_service,
|
|
991
|
-
browser_name,
|
|
992
|
-
} = req.body as {
|
|
993
|
-
url: string;
|
|
994
|
-
browser?: "auto" | "firefox" | "chrome" | "chromium";
|
|
995
|
-
chrome_profile?: string;
|
|
996
|
-
firefox_profile?: string;
|
|
997
|
-
chromium_profile?: string;
|
|
998
|
-
chromium_user_data_dir?: string;
|
|
999
|
-
chromium_cookie_db_path?: string;
|
|
1000
|
-
safe_storage_service?: string;
|
|
1001
|
-
browser_name?: string;
|
|
1002
|
-
};
|
|
1003
|
-
if (!url) return reply.code(400).send({ error: "url required" });
|
|
1004
|
-
try {
|
|
1005
|
-
const domain = new URL(url).hostname;
|
|
1006
|
-
const result = await extractBrowserAuth(domain, {
|
|
1007
|
-
browser,
|
|
1008
|
-
chromeProfile: chrome_profile,
|
|
1009
|
-
firefoxProfile: firefox_profile,
|
|
1010
|
-
chromium: {
|
|
1011
|
-
profile: chromium_profile,
|
|
1012
|
-
userDataDir: chromium_user_data_dir,
|
|
1013
|
-
cookieDbPath: chromium_cookie_db_path,
|
|
1014
|
-
safeStorageService: safe_storage_service,
|
|
1015
|
-
browserName: browser_name,
|
|
1016
|
-
},
|
|
1017
|
-
});
|
|
1018
|
-
return reply.send(result);
|
|
1019
|
-
} catch (err) {
|
|
1020
|
-
return reply.code(500).send({ error: (err as Error).message });
|
|
1021
|
-
}
|
|
1022
|
-
});
|
|
1023
|
-
|
|
1024
|
-
// POST /v1/skills/:skill_id/verify — trigger verification
|
|
1025
|
-
app.post("/v1/skills/:skill_id/verify", async (req, reply) => {
|
|
1026
|
-
const { skill_id } = req.params as { skill_id: string };
|
|
1027
|
-
const skill = await getSkill(skill_id);
|
|
1028
|
-
if (!skill) return reply.code(404).send({ error: "Skill not found" });
|
|
1029
|
-
try {
|
|
1030
|
-
const { verifySkill } = await import("../verification/index.js");
|
|
1031
|
-
const results = await verifySkill(skill);
|
|
1032
|
-
return reply.send({ skill_id, verification: results });
|
|
1033
|
-
} catch (err) {
|
|
1034
|
-
return reply.code(500).send({ error: (err as Error).message });
|
|
1035
|
-
}
|
|
1036
|
-
});
|
|
1037
|
-
|
|
1038
|
-
// POST /v1/stale/cleanup — verify active skills, mark dead endpoints stale, drop local cache entries
|
|
1039
|
-
app.post("/v1/stale/cleanup", async (req, reply) => {
|
|
1040
|
-
const body = (req.body ?? {}) as {
|
|
1041
|
-
skill_id?: string;
|
|
1042
|
-
domain?: string;
|
|
1043
|
-
limit?: number;
|
|
1044
|
-
};
|
|
1045
|
-
try {
|
|
1046
|
-
const result = await cleanupStaleSkills({
|
|
1047
|
-
skill_id: typeof body.skill_id === "string" ? body.skill_id : undefined,
|
|
1048
|
-
domain: typeof body.domain === "string" ? body.domain : undefined,
|
|
1049
|
-
limit: typeof body.limit === "number" ? body.limit : undefined,
|
|
1050
|
-
});
|
|
1051
|
-
return reply.send(result);
|
|
1052
|
-
} catch (err) {
|
|
1053
|
-
return reply.code(500).send({ error: (err as Error).message });
|
|
1054
|
-
}
|
|
1055
|
-
});
|
|
1056
|
-
|
|
1057
|
-
// POST /v1/feedback — submit execution feedback with optional diagnostics
|
|
1058
|
-
app.post("/v1/feedback", async (req, reply) => {
|
|
1059
|
-
const { skill_id, target_id, endpoint_id, rating, outcome, diagnostics } = req.body as {
|
|
1060
|
-
skill_id?: string;
|
|
1061
|
-
target_id?: string;
|
|
1062
|
-
endpoint_id?: string;
|
|
1063
|
-
rating?: number;
|
|
1064
|
-
outcome?: string;
|
|
1065
|
-
diagnostics?: {
|
|
1066
|
-
total_ms?: number;
|
|
1067
|
-
bottleneck?: string;
|
|
1068
|
-
wrong_endpoint?: boolean;
|
|
1069
|
-
expected_data?: string;
|
|
1070
|
-
got_data?: string;
|
|
1071
|
-
trace_version?: string;
|
|
1072
|
-
};
|
|
1073
|
-
};
|
|
1074
|
-
const resolvedSkillId = skill_id ?? target_id;
|
|
1075
|
-
if (!resolvedSkillId || !endpoint_id || rating == null) {
|
|
1076
|
-
return reply.code(400).send({ error: "skill_id, endpoint_id, and rating required" });
|
|
1077
|
-
}
|
|
1078
|
-
try {
|
|
1079
|
-
const avg_rating = await recordFeedback(resolvedSkillId, endpoint_id, rating);
|
|
1080
|
-
// Forward diagnostics to backend for version-grouped analysis
|
|
1081
|
-
if (diagnostics) {
|
|
1082
|
-
recordDiagnostics(resolvedSkillId, endpoint_id, diagnostics).catch(() => {});
|
|
1083
|
-
}
|
|
1084
|
-
return reply.send({ ok: true, avg_rating });
|
|
1085
|
-
} catch (err) {
|
|
1086
|
-
return reply.code(500).send({ error: (err as Error).message });
|
|
1087
|
-
}
|
|
1088
|
-
});
|
|
1089
|
-
|
|
1090
|
-
// GET /v1/stats — public, no auth required
|
|
1091
|
-
app.get("/v1/stats", async (_req, reply) => {
|
|
1092
|
-
try {
|
|
1093
|
-
const data = await fetchStats();
|
|
1094
|
-
return reply.send(data);
|
|
1095
|
-
} catch (err) {
|
|
1096
|
-
return reply.code(500).send({ error: (err as Error).message });
|
|
1097
|
-
}
|
|
1098
|
-
});
|
|
1099
|
-
|
|
1100
|
-
// GET /health
|
|
1101
|
-
app.get("/health", async (_req, reply) => reply.send({
|
|
1102
|
-
status: "ok",
|
|
1103
|
-
package_version: PACKAGE_VERSION,
|
|
1104
|
-
trace_version: TRACE_VERSION,
|
|
1105
|
-
code_hash: CODE_HASH,
|
|
1106
|
-
git_sha: GIT_SHA,
|
|
1107
|
-
}));
|
|
1108
|
-
|
|
1109
|
-
// GET /v1/sessions/:domain — read local trace/debug files instead of proxying to backend
|
|
1110
|
-
app.get("/v1/sessions/:domain", async (req, reply) => {
|
|
1111
|
-
const { domain } = req.params as { domain: string };
|
|
1112
|
-
const query = req.query as { limit?: string | number };
|
|
1113
|
-
const limitRaw = typeof query.limit === "number" ? query.limit : Number(query.limit ?? 10);
|
|
1114
|
-
const limit = Number.isFinite(limitRaw) ? Math.min(Math.max(limitRaw, 1), 50) : 10;
|
|
1115
|
-
return reply.send({
|
|
1116
|
-
domain,
|
|
1117
|
-
sessions: listRecentSessionsForDomain(TRACES_DIR, domain, limit),
|
|
1118
|
-
});
|
|
1119
|
-
});
|
|
1120
|
-
|
|
1121
|
-
// Catch-all proxy: forward unmatched /v1/* routes to the configured backend origin
|
|
1122
|
-
app.all("/v1/*", async (req, reply) => {
|
|
1123
|
-
const key = getApiKey();
|
|
1124
|
-
const upstream = `${BETA_API_URL}${req.url}`;
|
|
1125
|
-
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
|
1126
|
-
if (key) headers["Authorization"] = `Bearer ${key}`;
|
|
1127
|
-
|
|
1128
|
-
try {
|
|
1129
|
-
const res = await fetch(upstream, {
|
|
1130
|
-
method: req.method,
|
|
1131
|
-
headers,
|
|
1132
|
-
body: req.method !== "GET" && req.method !== "HEAD" ? JSON.stringify(req.body) : undefined,
|
|
1133
|
-
});
|
|
1134
|
-
const text = await res.text();
|
|
1135
|
-
try {
|
|
1136
|
-
return reply.code(res.status).send(JSON.parse(text));
|
|
1137
|
-
} catch {
|
|
1138
|
-
return reply.code(res.status).send({ error: text || `Upstream returned ${res.status}` });
|
|
1139
|
-
}
|
|
1140
|
-
} catch (err) {
|
|
1141
|
-
return reply.code(502).send({ error: `Proxy to beta-api failed: ${(err as Error).message}` });
|
|
1142
|
-
}
|
|
1143
|
-
});
|
|
1144
|
-
|
|
1145
|
-
// ── Browse session management ─────────────────────────────────────────
|
|
1146
|
-
// Kuri browser actions with passive HAR indexing. The server manages a
|
|
1147
|
-
// per-session tab + HAR state so every action the agent takes through
|
|
1148
|
-
// the CLI is passively captured and indexed.
|
|
1149
|
-
|
|
1150
|
-
// browseSessions is module-level (shared with orchestrator via registerBrowseSession)
|
|
1151
|
-
|
|
1152
|
-
function requestedSessionId(req: { body?: unknown; query?: unknown }): string | undefined {
|
|
1153
|
-
const body = req.body && typeof req.body === "object" ? req.body as Record<string, unknown> : null;
|
|
1154
|
-
if (typeof body?.session_id === "string" && body.session_id.trim()) return body.session_id;
|
|
1155
|
-
const query = req.query && typeof req.query === "object" ? req.query as Record<string, unknown> : null;
|
|
1156
|
-
if (typeof query?.session_id === "string" && query.session_id.trim()) return query.session_id;
|
|
1157
|
-
return undefined;
|
|
1158
|
-
}
|
|
1159
|
-
|
|
1160
|
-
function sendBrowseSessionError(reply: { code: (statusCode: number) => { send: (body: unknown) => unknown } }, error: unknown): unknown {
|
|
1161
|
-
if (error instanceof BrowseSessionError) {
|
|
1162
|
-
return reply.code(error.statusCode).send({ error: error.code });
|
|
1163
|
-
}
|
|
1164
|
-
if (isRecoverableBrowseFailure(error)) {
|
|
1165
|
-
return reply.code(502).send({
|
|
1166
|
-
error: "recoverable_browse_failure",
|
|
1167
|
-
message: extractBrowseFailureMessage(error) ?? "recoverable_browse_failure",
|
|
1168
|
-
recoverable: true,
|
|
1169
|
-
});
|
|
1170
|
-
}
|
|
1171
|
-
throw error;
|
|
1172
|
-
}
|
|
1173
|
-
|
|
1174
|
-
/** Extract registrable domain for auth profile naming */
|
|
1175
|
-
function profileName(url: string): string {
|
|
1176
|
-
try { return new URL(url).hostname.replace(/^www\./, ""); } catch { return "unknown"; }
|
|
1177
|
-
}
|
|
1178
|
-
|
|
1179
|
-
async function restartBrowseCapture(session: BrowseSession): Promise<void> {
|
|
1180
|
-
const broker = brokerForSession(session);
|
|
1181
|
-
const load = await broker.waitForLoad(session.tabId, 2_000).catch(() => null);
|
|
1182
|
-
if (load && load.status === "timeout") {
|
|
1183
|
-
session.harActive = false;
|
|
1184
|
-
return;
|
|
1185
|
-
}
|
|
1186
|
-
await broker.networkEnable(session.tabId).catch(() => {});
|
|
1187
|
-
await broker.harStart(session.tabId).catch(() => {});
|
|
1188
|
-
await broker.scriptInject(session.tabId, INTERCEPTOR_SCRIPT).catch(() => {});
|
|
1189
|
-
session.harActive = true;
|
|
1190
|
-
await injectInterceptor(session.tabId).catch(() => {});
|
|
1191
|
-
}
|
|
1192
|
-
async function flushBrowseCapture(
|
|
1193
|
-
session: BrowseSession,
|
|
1194
|
-
options: { queueIndex?: boolean; queuePublish?: boolean } = {},
|
|
1195
|
-
): Promise<{
|
|
1196
|
-
indexed: boolean;
|
|
1197
|
-
mode: "http" | "dom" | "none";
|
|
1198
|
-
domain: string;
|
|
1199
|
-
skill_id: string | null;
|
|
1200
|
-
endpoint_count: number;
|
|
1201
|
-
endpoints: Array<{
|
|
1202
|
-
endpoint_id: string;
|
|
1203
|
-
method: string;
|
|
1204
|
-
url_template: string;
|
|
1205
|
-
description?: string;
|
|
1206
|
-
trigger_url?: string;
|
|
1207
|
-
action_kind?: string;
|
|
1208
|
-
resource_kind?: string;
|
|
1209
|
-
}>;
|
|
1210
|
-
request_count: number;
|
|
1211
|
-
pipeline: {
|
|
1212
|
-
index_queued: boolean;
|
|
1213
|
-
publish_queued: boolean;
|
|
1214
|
-
};
|
|
1215
|
-
publish_policy: {
|
|
1216
|
-
mode: "auto" | "disabled" | "blacklisted" | "prompt";
|
|
1217
|
-
reason: string;
|
|
1218
|
-
matched_domain?: string;
|
|
1219
|
-
};
|
|
1220
|
-
background_publish_queued: boolean;
|
|
1221
|
-
}> {
|
|
1222
|
-
let harEntries: KuriHarEntry[] = [];
|
|
1223
|
-
if (session.harActive) {
|
|
1224
|
-
try {
|
|
1225
|
-
const { entries } = await brokerForSession(session).harStop(session.tabId);
|
|
1226
|
-
harEntries = entries;
|
|
1227
|
-
} catch { /* non-fatal */ }
|
|
1228
|
-
}
|
|
1229
|
-
session.harActive = false;
|
|
1230
|
-
|
|
1231
|
-
const allRequests = await enrichPassiveCaptureRequests({
|
|
1232
|
-
tabId: session.tabId,
|
|
1233
|
-
captureUrl: session.url,
|
|
1234
|
-
harEntries,
|
|
1235
|
-
intent: `browse ${session.domain || profileName(session.url)}`,
|
|
1236
|
-
});
|
|
1237
|
-
const syncResult = await cacheBrowseRequests({
|
|
1238
|
-
sessionUrl: session.url,
|
|
1239
|
-
sessionDomain: session.domain,
|
|
1240
|
-
requests: allRequests,
|
|
1241
|
-
getPageHtml: () => brokerForSession(session).getPageHtml(session.tabId),
|
|
1242
|
-
intent: `browse ${session.domain || profileName(session.url)}`,
|
|
1243
|
-
});
|
|
1244
|
-
|
|
1245
|
-
let indexQueued = false;
|
|
1246
|
-
let publishQueued = false;
|
|
1247
|
-
const publishDecision = options.queuePublish
|
|
1248
|
-
? decideCheckpointPublish(syncResult.domain)
|
|
1249
|
-
: {
|
|
1250
|
-
publishQueued: false,
|
|
1251
|
-
mode: "disabled" as const,
|
|
1252
|
-
reason: "Remote publish not requested for this checkpoint.",
|
|
1253
|
-
};
|
|
1254
|
-
if (options.queueIndex) {
|
|
1255
|
-
if (syncResult.skill) {
|
|
1256
|
-
queueBackgroundIndex({
|
|
1257
|
-
skill: { ...syncResult.skill },
|
|
1258
|
-
domain: syncResult.domain,
|
|
1259
|
-
intent: syncResult.skill.intent_signature || `browse ${syncResult.domain}`,
|
|
1260
|
-
contextUrl: session.url,
|
|
1261
|
-
cacheKey: `browse-submit:${syncResult.domain}:${Date.now()}`,
|
|
1262
|
-
publishAfterIndex: publishDecision.publishQueued,
|
|
1263
|
-
});
|
|
1264
|
-
indexQueued = true;
|
|
1265
|
-
publishQueued = publishDecision.publishQueued;
|
|
1266
|
-
} else if (allRequests.length > 0) {
|
|
1267
|
-
passiveIndexFromRequests(allRequests, session.url, {
|
|
1268
|
-
publishAfterIndex: publishDecision.publishQueued,
|
|
1269
|
-
});
|
|
1270
|
-
indexQueued = true;
|
|
1271
|
-
publishQueued = publishDecision.publishQueued;
|
|
1272
|
-
}
|
|
1273
|
-
}
|
|
1274
|
-
return {
|
|
1275
|
-
indexed: syncResult.indexed,
|
|
1276
|
-
mode: syncResult.mode,
|
|
1277
|
-
domain: syncResult.domain,
|
|
1278
|
-
skill_id: syncResult.skill?.skill_id ?? null,
|
|
1279
|
-
endpoint_count: syncResult.skill?.endpoints.length ?? 0,
|
|
1280
|
-
endpoints: (syncResult.skill?.endpoints ?? []).map((endpoint) => ({
|
|
1281
|
-
endpoint_id: endpoint.endpoint_id,
|
|
1282
|
-
method: endpoint.method,
|
|
1283
|
-
url_template: endpoint.url_template,
|
|
1284
|
-
description: endpoint.description,
|
|
1285
|
-
trigger_url: endpoint.trigger_url,
|
|
1286
|
-
action_kind: endpoint.semantic?.action_kind,
|
|
1287
|
-
resource_kind: endpoint.semantic?.resource_kind,
|
|
1288
|
-
})),
|
|
1289
|
-
request_count: allRequests.length,
|
|
1290
|
-
pipeline: {
|
|
1291
|
-
index_queued: indexQueued,
|
|
1292
|
-
publish_queued: publishQueued,
|
|
1293
|
-
},
|
|
1294
|
-
publish_policy: {
|
|
1295
|
-
mode: publishDecision.mode,
|
|
1296
|
-
reason: publishDecision.reason,
|
|
1297
|
-
...(publishDecision.matchedDomain ? { matched_domain: publishDecision.matchedDomain } : {}),
|
|
1298
|
-
},
|
|
1299
|
-
background_publish_queued: publishQueued,
|
|
1300
|
-
};
|
|
1301
|
-
}
|
|
1302
|
-
|
|
1303
|
-
// POST /v1/browse/go — navigate to URL
|
|
1304
|
-
app.post("/v1/browse/go", async (req, reply) => {
|
|
1305
|
-
const { url } = req.body as { url: string };
|
|
1306
|
-
if (!url) return reply.code(400).send({ error: "url required" });
|
|
1307
|
-
try {
|
|
1308
|
-
const sessionId = requestedSessionId(req);
|
|
1309
|
-
const browseClient = selectBrowseBrokerClient(sessionId);
|
|
1310
|
-
const navigateSession = async (session: BrowseSession) => {
|
|
1311
|
-
const broker = brokerForSession(session);
|
|
1312
|
-
const newDomain = profileName(url);
|
|
1313
|
-
|
|
1314
|
-
if (session.harActive && session.url !== "about:blank") {
|
|
1315
|
-
try {
|
|
1316
|
-
const { entries } = await broker.harStop(session.tabId);
|
|
1317
|
-
passiveIndexHar(entries, session.url, { publishAfterIndex: false });
|
|
1318
|
-
} catch { /* non-fatal */ }
|
|
1319
|
-
session.harActive = false;
|
|
1320
|
-
}
|
|
1321
|
-
|
|
1322
|
-
if (session.domain && session.domain !== newDomain) {
|
|
1323
|
-
await saveAuthProfileBestEffort(session.tabId, session.domain, "browse_go");
|
|
1324
|
-
}
|
|
1325
|
-
|
|
1326
|
-
let cookiesInjected = 0;
|
|
1327
|
-
if (newDomain && newDomain !== session.domain) {
|
|
1328
|
-
// Check if the browser already has fresh session cookies for this domain.
|
|
1329
|
-
// If so, skip vault/profile cookie injection — browser cookies are fresher
|
|
1330
|
-
// and injecting stale vault cookies (e.g. JSESSIONID) causes HTTP 400 on
|
|
1331
|
-
// sites like LinkedIn that validate CSRF alignment.
|
|
1332
|
-
const browserHasFreshSession = await (async () => {
|
|
1333
|
-
try {
|
|
1334
|
-
const { extractBrowserCookies } = await import("../auth/browser-cookies.js");
|
|
1335
|
-
const { cookies } = extractBrowserCookies(newDomain);
|
|
1336
|
-
// Consider the session fresh if we have session-like cookies that aren't expired
|
|
1337
|
-
const now = Date.now() / 1000;
|
|
1338
|
-
return cookies.some((c) =>
|
|
1339
|
-
(c.httpOnly || c.secure) && (!c.expires || c.expires > now),
|
|
1340
|
-
);
|
|
1341
|
-
} catch { return false; }
|
|
1342
|
-
})();
|
|
1343
|
-
|
|
1344
|
-
if (browserHasFreshSession) {
|
|
1345
|
-
// Import browser cookies via CDP (they're fresh from Chrome's jar)
|
|
1346
|
-
cookiesInjected = await importBrowserCookiesIntoTab(session.tabId, newDomain);
|
|
1347
|
-
} else {
|
|
1348
|
-
// No fresh browser cookies — load from vault/auth profile
|
|
1349
|
-
cookiesInjected = await importBrowserCookiesIntoTab(session.tabId, newDomain);
|
|
1350
|
-
await loadAuthProfileBestEffort(session.tabId, newDomain, "browse_go");
|
|
1351
|
-
}
|
|
1352
|
-
}
|
|
1353
|
-
|
|
1354
|
-
await restartBrowseCapture(session);
|
|
1355
|
-
|
|
1356
|
-
await broker.navigate(session.tabId, url);
|
|
1357
|
-
await broker.navigate(session.tabId, url);
|
|
1358
|
-
const finalUrl = await broker.getCurrentUrl(session.tabId).catch(() => url);
|
|
1359
|
-
session.url = typeof finalUrl === "string" && finalUrl.startsWith("http") ? finalUrl : url;
|
|
1360
|
-
session.domain = profileName(session.url);
|
|
1361
|
-
await injectInterceptor(session.tabId);
|
|
1362
|
-
const stillLive = await isBrowseSessionLive(session, browseClient).catch(() => false);
|
|
1363
|
-
if (!stillLive) throw { error: "CDP command failed" };
|
|
1364
|
-
|
|
1365
|
-
return { cookiesInjected };
|
|
1366
|
-
};
|
|
1367
|
-
|
|
1368
|
-
let session: BrowseSession;
|
|
1369
|
-
let result: { cookiesInjected: number };
|
|
1370
|
-
if (sessionId) {
|
|
1371
|
-
const navigated = await withSerializedStrictBrowseSession(
|
|
1372
|
-
browseSessions,
|
|
1373
|
-
browseClient,
|
|
1374
|
-
sessionId,
|
|
1375
|
-
navigateSession,
|
|
1376
|
-
);
|
|
1377
|
-
session = navigated.session;
|
|
1378
|
-
result = navigated.result;
|
|
1379
|
-
} else {
|
|
1380
|
-
session = await getOrCreateNavigateBrowseSession(
|
|
1381
|
-
browseSessions,
|
|
1382
|
-
browseClient,
|
|
1383
|
-
injectInterceptor,
|
|
1384
|
-
);
|
|
1385
|
-
result = await navigateSession(session);
|
|
1386
|
-
}
|
|
1387
|
-
|
|
1388
|
-
return reply.send({
|
|
1389
|
-
ok: true,
|
|
1390
|
-
session_id: session.sessionId,
|
|
1391
|
-
url: session.url,
|
|
1392
|
-
tab_id: session.tabId,
|
|
1393
|
-
auth_profile: session.domain,
|
|
1394
|
-
...(result.cookiesInjected > 0 ? { cookies_injected: result.cookiesInjected } : {}),
|
|
1395
|
-
});
|
|
1396
|
-
} catch (error) {
|
|
1397
|
-
return sendBrowseSessionError(reply, error);
|
|
1398
|
-
}
|
|
1399
|
-
});
|
|
1400
|
-
|
|
1401
|
-
// POST /v1/browse/submit — submit active form; same-origin fetch+rehydrate is explicit opt-in
|
|
1402
|
-
app.post("/v1/browse/submit", async (req, reply) => {
|
|
1403
|
-
const {
|
|
1404
|
-
form_selector: formSelector,
|
|
1405
|
-
submit_selector: submitSelector,
|
|
1406
|
-
wait_for: waitFor,
|
|
1407
|
-
same_origin_fetch_fallback: sameOriginFetchFallback,
|
|
1408
|
-
timeout_ms: timeoutMs,
|
|
1409
|
-
assist_site_state: assistSiteState,
|
|
1410
|
-
} = (req.body as {
|
|
1411
|
-
form_selector?: string;
|
|
1412
|
-
submit_selector?: string;
|
|
1413
|
-
wait_for?: string;
|
|
1414
|
-
same_origin_fetch_fallback?: boolean;
|
|
1415
|
-
timeout_ms?: number;
|
|
1416
|
-
assist_site_state?: boolean;
|
|
1417
|
-
}) ?? {};
|
|
1418
|
-
|
|
1419
|
-
try {
|
|
1420
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1421
|
-
const { session, result } = await withSerializedStrictBrowseSession(
|
|
1422
|
-
browseSessions,
|
|
1423
|
-
browseClient,
|
|
1424
|
-
requestedSessionId(req),
|
|
1425
|
-
async (session) => submitBrowseForm(
|
|
1426
|
-
{
|
|
1427
|
-
client: brokerForSession(session),
|
|
1428
|
-
session,
|
|
1429
|
-
flushCapture: async (session) => await flushBrowseCapture(session, { queueBackgroundPublish: false }),
|
|
1430
|
-
flushCapture: async (session) => await flushBrowseCapture(session, { queueBackgroundPublish: false }),
|
|
1431
|
-
restartCapture: restartBrowseCapture,
|
|
1432
|
-
rehydratePlugins: (tabId) => brokerForSession(session).bestEffortRehydratePlugins(tabId),
|
|
1433
|
-
},
|
|
1434
|
-
{
|
|
1435
|
-
formSelector,
|
|
1436
|
-
submitSelector,
|
|
1437
|
-
waitFor,
|
|
1438
|
-
sameOriginFetchFallback,
|
|
1439
|
-
timeoutMs,
|
|
1440
|
-
assistSiteState,
|
|
1441
|
-
},
|
|
1442
|
-
),
|
|
1443
|
-
(result) => !result.ok && result.recoverable === true,
|
|
1444
|
-
);
|
|
1445
|
-
|
|
1446
|
-
let activeSession = session;
|
|
1447
|
-
const hintedDestination = result.wait_for && isUrlWaitHint(result.wait_for)
|
|
1448
|
-
? resolveSubmitWaitHint(activeSession.url || "about:blank", result.wait_for)
|
|
1449
|
-
: null;
|
|
1450
|
-
const rawResultUrl = typeof result.url === "string" ? result.url : "";
|
|
1451
|
-
activeSession.url = rawResultUrl || await brokerForSession(activeSession).getCurrentUrl(activeSession.tabId).catch(() => activeSession.url);
|
|
1452
|
-
if (result.ok && hintedDestination && (!rawResultUrl || !rawResultUrl.includes(result.wait_for ?? ""))) {
|
|
1453
|
-
activeSession.url = hintedDestination;
|
|
1454
|
-
}
|
|
1455
|
-
activeSession.domain = profileName(activeSession.url);
|
|
1456
|
-
const stillLive = await isBrowseSessionLive(activeSession, browseClient).catch(() => false);
|
|
1457
|
-
if (!stillLive) {
|
|
1458
|
-
removeBrowseSession(browseSessions, activeSession.sessionId);
|
|
1459
|
-
throw new BrowseSessionError("session_expired");
|
|
1460
|
-
}
|
|
1461
|
-
|
|
1462
|
-
const statusCode = result.ok ? 200 : (result.recoverable ? 502 : 400);
|
|
1463
|
-
const sessionHint = `--session ${activeSession.sessionId}`;
|
|
1464
|
-
const nextStep = result.ok
|
|
1465
|
-
? `If more UI steps remain, continue the flow. Run \`unbrowse sync ${sessionHint}\` after meaningful transitions, then \`unbrowse close ${sessionHint}\` when you're done to checkpoint the final capture, save auth, and queue the background pipeline.`
|
|
1466
|
-
: `Inspect the page state with \`unbrowse snap ${sessionHint} --filter interactive\`, then retry submit with selectors or a wait hint if needed.`;
|
|
1467
|
-
return reply.code(statusCode).send({
|
|
1468
|
-
...result,
|
|
1469
|
-
session_id: activeSession.sessionId,
|
|
1470
|
-
next_step: nextStep,
|
|
1471
|
-
recovered: false,
|
|
1472
|
-
tab_id: activeSession.tabId,
|
|
1473
|
-
url: activeSession.url,
|
|
1474
|
-
});
|
|
1475
|
-
} catch (error) {
|
|
1476
|
-
return sendBrowseSessionError(reply, error);
|
|
1477
|
-
}
|
|
1478
|
-
});
|
|
1479
|
-
|
|
1480
|
-
// POST /v1/browse/snap — a11y snapshot
|
|
1481
|
-
app.post("/v1/browse/snap", async (req, reply) => {
|
|
1482
|
-
const { filter } = (req.body as { filter?: string; session_id?: string }) ?? {};
|
|
1483
|
-
try {
|
|
1484
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1485
|
-
const { session, result: snapshot } = await withSerializedStrictBrowseSession(
|
|
1486
|
-
browseSessions,
|
|
1487
|
-
browseClient,
|
|
1488
|
-
requestedSessionId(req),
|
|
1489
|
-
async (session) => brokerForSession(session).snapshot(session.tabId, filter),
|
|
1490
|
-
);
|
|
1491
|
-
return reply.send({ snapshot, session_id: session.sessionId, tab_id: session.tabId });
|
|
1492
|
-
} catch (error) {
|
|
1493
|
-
return sendBrowseSessionError(reply, error);
|
|
1494
|
-
}
|
|
1495
|
-
});
|
|
1496
|
-
|
|
1497
|
-
// POST /v1/browse/click — click by ref
|
|
1498
|
-
app.post("/v1/browse/click", async (req, reply) => {
|
|
1499
|
-
const { ref } = req.body as { ref: string; session_id?: string };
|
|
1500
|
-
if (!ref) return reply.code(400).send({ error: "ref required" });
|
|
1501
|
-
try {
|
|
1502
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1503
|
-
const { session } = await withSerializedStrictBrowseSession(
|
|
1504
|
-
browseSessions,
|
|
1505
|
-
browseClient,
|
|
1506
|
-
requestedSessionId(req),
|
|
1507
|
-
async (session) => {
|
|
1508
|
-
await brokerForSession(session).click(session.tabId, ref);
|
|
1509
|
-
return true;
|
|
1510
|
-
},
|
|
1511
|
-
);
|
|
1512
|
-
await isBrowseSessionLive(session, browseClient).catch(() => false);
|
|
1513
|
-
return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId, url: session.url });
|
|
1514
|
-
} catch (error) {
|
|
1515
|
-
return sendBrowseSessionError(reply, error);
|
|
1516
|
-
}
|
|
1517
|
-
});
|
|
1518
|
-
|
|
1519
|
-
// POST /v1/browse/fill — fill input by ref
|
|
1520
|
-
app.post("/v1/browse/fill", async (req, reply) => {
|
|
1521
|
-
const { ref, value } = req.body as { ref: string; value: string; session_id?: string };
|
|
1522
|
-
if (!ref || value === undefined) return reply.code(400).send({ error: "ref and value required" });
|
|
1523
|
-
try {
|
|
1524
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1525
|
-
const { session } = await withSerializedStrictBrowseSession(
|
|
1526
|
-
browseSessions,
|
|
1527
|
-
browseClient,
|
|
1528
|
-
requestedSessionId(req),
|
|
1529
|
-
async (session) => {
|
|
1530
|
-
await brokerForSession(session).fill(session.tabId, ref, value);
|
|
1531
|
-
return true;
|
|
1532
|
-
},
|
|
1533
|
-
);
|
|
1534
|
-
return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
|
|
1535
|
-
} catch (error) {
|
|
1536
|
-
return sendBrowseSessionError(reply, error);
|
|
1537
|
-
}
|
|
1538
|
-
});
|
|
1539
|
-
|
|
1540
|
-
// POST /v1/browse/type — keyboard type
|
|
1541
|
-
app.post("/v1/browse/type", async (req, reply) => {
|
|
1542
|
-
const { text } = req.body as { text: string; session_id?: string };
|
|
1543
|
-
if (!text) return reply.code(400).send({ error: "text required" });
|
|
1544
|
-
try {
|
|
1545
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1546
|
-
const { session } = await withSerializedStrictBrowseSession(
|
|
1547
|
-
browseSessions,
|
|
1548
|
-
browseClient,
|
|
1549
|
-
requestedSessionId(req),
|
|
1550
|
-
async (session) => {
|
|
1551
|
-
await brokerForSession(session).keyboardType(session.tabId, text);
|
|
1552
|
-
return true;
|
|
1553
|
-
},
|
|
1554
|
-
);
|
|
1555
|
-
return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
|
|
1556
|
-
} catch (error) {
|
|
1557
|
-
return sendBrowseSessionError(reply, error);
|
|
1558
|
-
}
|
|
1559
|
-
});
|
|
1560
|
-
|
|
1561
|
-
// POST /v1/browse/press — press key
|
|
1562
|
-
app.post("/v1/browse/press", async (req, reply) => {
|
|
1563
|
-
const { key } = req.body as { key: string; session_id?: string };
|
|
1564
|
-
if (!key) return reply.code(400).send({ error: "key required" });
|
|
1565
|
-
try {
|
|
1566
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1567
|
-
const { session } = await withSerializedStrictBrowseSession(
|
|
1568
|
-
browseSessions,
|
|
1569
|
-
browseClient,
|
|
1570
|
-
requestedSessionId(req),
|
|
1571
|
-
async (session) => {
|
|
1572
|
-
await brokerForSession(session).press(session.tabId, key);
|
|
1573
|
-
return true;
|
|
1574
|
-
},
|
|
1575
|
-
);
|
|
1576
|
-
return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
|
|
1577
|
-
} catch (error) {
|
|
1578
|
-
return sendBrowseSessionError(reply, error);
|
|
1579
|
-
}
|
|
1580
|
-
});
|
|
1581
|
-
|
|
1582
|
-
// POST /v1/browse/select — select option by ref
|
|
1583
|
-
app.post("/v1/browse/select", async (req, reply) => {
|
|
1584
|
-
const { ref, value } = req.body as { ref: string; value: string; session_id?: string };
|
|
1585
|
-
if (!ref || value === undefined) return reply.code(400).send({ error: "ref and value required" });
|
|
1586
|
-
try {
|
|
1587
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1588
|
-
const { session } = await withSerializedStrictBrowseSession(
|
|
1589
|
-
browseSessions,
|
|
1590
|
-
browseClient,
|
|
1591
|
-
requestedSessionId(req),
|
|
1592
|
-
async (session) => {
|
|
1593
|
-
await brokerForSession(session).select(session.tabId, ref, value);
|
|
1594
|
-
return true;
|
|
1595
|
-
},
|
|
1596
|
-
);
|
|
1597
|
-
return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
|
|
1598
|
-
} catch (error) {
|
|
1599
|
-
return sendBrowseSessionError(reply, error);
|
|
1600
|
-
}
|
|
1601
|
-
});
|
|
1602
|
-
|
|
1603
|
-
// POST /v1/browse/scroll — scroll
|
|
1604
|
-
app.post("/v1/browse/scroll", async (req, reply) => {
|
|
1605
|
-
const { direction, amount } = (req.body as { direction?: string; amount?: number; session_id?: string }) ?? {};
|
|
1606
|
-
try {
|
|
1607
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1608
|
-
const { session } = await withSerializedStrictBrowseSession(
|
|
1609
|
-
browseSessions,
|
|
1610
|
-
browseClient,
|
|
1611
|
-
requestedSessionId(req),
|
|
1612
|
-
async (session) => {
|
|
1613
|
-
await brokerForSession(session).scroll(session.tabId, (direction as any) ?? "down", amount);
|
|
1614
|
-
return true;
|
|
1615
|
-
},
|
|
1616
|
-
);
|
|
1617
|
-
return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
|
|
1618
|
-
} catch (error) {
|
|
1619
|
-
return sendBrowseSessionError(reply, error);
|
|
1620
|
-
}
|
|
1621
|
-
});
|
|
1622
|
-
|
|
1623
|
-
// GET /v1/browse/screenshot — capture screenshot
|
|
1624
|
-
app.get("/v1/browse/screenshot", async (req, reply) => {
|
|
1625
|
-
try {
|
|
1626
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1627
|
-
const { session, result: data } = await withSerializedStrictBrowseSession(
|
|
1628
|
-
browseSessions,
|
|
1629
|
-
browseClient,
|
|
1630
|
-
requestedSessionId(req),
|
|
1631
|
-
async (session) => brokerForSession(session).screenshot(session.tabId),
|
|
1632
|
-
);
|
|
1633
|
-
return reply.send({ screenshot: data, session_id: session.sessionId, tab_id: session.tabId });
|
|
1634
|
-
} catch (error) {
|
|
1635
|
-
return sendBrowseSessionError(reply, error);
|
|
1636
|
-
}
|
|
1637
|
-
});
|
|
1638
|
-
|
|
1639
|
-
// GET /v1/browse/text — page text
|
|
1640
|
-
app.get("/v1/browse/text", async (req, reply) => {
|
|
1641
|
-
try {
|
|
1642
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1643
|
-
const { session, result: text } = await withSerializedStrictBrowseSession(
|
|
1644
|
-
browseSessions,
|
|
1645
|
-
browseClient,
|
|
1646
|
-
requestedSessionId(req),
|
|
1647
|
-
async (session) => brokerForSession(session).getText(session.tabId),
|
|
1648
|
-
);
|
|
1649
|
-
return reply.send({ text, session_id: session.sessionId, tab_id: session.tabId });
|
|
1650
|
-
} catch (error) {
|
|
1651
|
-
return sendBrowseSessionError(reply, error);
|
|
1652
|
-
}
|
|
1653
|
-
});
|
|
1654
|
-
|
|
1655
|
-
// GET /v1/browse/markdown — page as markdown
|
|
1656
|
-
app.get("/v1/browse/markdown", async (req, reply) => {
|
|
1657
|
-
try {
|
|
1658
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1659
|
-
const { session, result: markdown } = await withSerializedStrictBrowseSession(
|
|
1660
|
-
browseSessions,
|
|
1661
|
-
browseClient,
|
|
1662
|
-
requestedSessionId(req),
|
|
1663
|
-
async (session) => brokerForSession(session).getMarkdown(session.tabId),
|
|
1664
|
-
);
|
|
1665
|
-
return reply.send({ markdown, session_id: session.sessionId, tab_id: session.tabId });
|
|
1666
|
-
} catch (error) {
|
|
1667
|
-
return sendBrowseSessionError(reply, error);
|
|
1668
|
-
}
|
|
1669
|
-
});
|
|
1670
|
-
|
|
1671
|
-
// GET /v1/browse/cookies — page cookies
|
|
1672
|
-
app.get("/v1/browse/cookies", async (req, reply) => {
|
|
1673
|
-
try {
|
|
1674
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1675
|
-
const { session, result: cookies } = await withSerializedStrictBrowseSession(
|
|
1676
|
-
browseSessions,
|
|
1677
|
-
browseClient,
|
|
1678
|
-
requestedSessionId(req),
|
|
1679
|
-
async (session) => brokerForSession(session).getCookies(session.tabId),
|
|
1680
|
-
);
|
|
1681
|
-
return reply.send({ cookies, session_id: session.sessionId, tab_id: session.tabId });
|
|
1682
|
-
} catch (error) {
|
|
1683
|
-
return sendBrowseSessionError(reply, error);
|
|
1684
|
-
}
|
|
1685
|
-
});
|
|
1686
|
-
|
|
1687
|
-
// POST /v1/browse/eval — evaluate JS
|
|
1688
|
-
app.post("/v1/browse/eval", async (req, reply) => {
|
|
1689
|
-
const { expression } = req.body as { expression: string; session_id?: string };
|
|
1690
|
-
if (!expression) return reply.code(400).send({ error: "expression required" });
|
|
1691
|
-
try {
|
|
1692
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1693
|
-
const { session, result } = await withSerializedStrictBrowseSession(
|
|
1694
|
-
browseSessions,
|
|
1695
|
-
browseClient,
|
|
1696
|
-
requestedSessionId(req),
|
|
1697
|
-
async (session) => brokerForSession(session).evaluate(session.tabId, expression),
|
|
1698
|
-
);
|
|
1699
|
-
return reply.send({ result, session_id: session.sessionId, tab_id: session.tabId });
|
|
1700
|
-
} catch (error) {
|
|
1701
|
-
return sendBrowseSessionError(reply, error);
|
|
1702
|
-
}
|
|
1703
|
-
});
|
|
1704
|
-
|
|
1705
|
-
// POST /v1/browse/back — navigate back
|
|
1706
|
-
app.post("/v1/browse/back", async (req, reply) => {
|
|
1707
|
-
try {
|
|
1708
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1709
|
-
const { session } = await withSerializedStrictBrowseSession(
|
|
1710
|
-
browseSessions,
|
|
1711
|
-
browseClient,
|
|
1712
|
-
requestedSessionId(req),
|
|
1713
|
-
async (session) => {
|
|
1714
|
-
await brokerForSession(session).goBack(session.tabId);
|
|
1715
|
-
return true;
|
|
1716
|
-
},
|
|
1717
|
-
);
|
|
1718
|
-
return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
|
|
1719
|
-
} catch (error) {
|
|
1720
|
-
return sendBrowseSessionError(reply, error);
|
|
1721
|
-
}
|
|
1722
|
-
});
|
|
1723
|
-
|
|
1724
|
-
// POST /v1/browse/forward — navigate forward
|
|
1725
|
-
app.post("/v1/browse/forward", async (req, reply) => {
|
|
1726
|
-
try {
|
|
1727
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1728
|
-
const { session } = await withSerializedStrictBrowseSession(
|
|
1729
|
-
browseSessions,
|
|
1730
|
-
browseClient,
|
|
1731
|
-
requestedSessionId(req),
|
|
1732
|
-
async (session) => {
|
|
1733
|
-
await brokerForSession(session).goForward(session.tabId);
|
|
1734
|
-
return true;
|
|
1735
|
-
},
|
|
1736
|
-
);
|
|
1737
|
-
return reply.send({ ok: true, session_id: session.sessionId, tab_id: session.tabId });
|
|
1738
|
-
} catch (error) {
|
|
1739
|
-
return sendBrowseSessionError(reply, error);
|
|
1740
|
-
}
|
|
1741
|
-
});
|
|
1742
|
-
|
|
1743
|
-
// POST /v1/browse/sync — checkpoint capture, keep the tab open, queue index+publish
|
|
1744
|
-
app.post("/v1/browse/sync", async (req, reply) => {
|
|
1745
|
-
try {
|
|
1746
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1747
|
-
const { session, result: syncResult } = await withSerializedStrictBrowseSession(
|
|
1748
|
-
browseSessions,
|
|
1749
|
-
browseClient,
|
|
1750
|
-
requestedSessionId(req),
|
|
1751
|
-
async (session) => {
|
|
1752
|
-
const syncResult = await flushBrowseCapture(session, { queueIndex: true, queuePublish: true });
|
|
1753
|
-
await restartBrowseCapture(session);
|
|
1754
|
-
return syncResult;
|
|
1755
|
-
},
|
|
1756
|
-
);
|
|
1757
|
-
|
|
1758
|
-
return reply.send({
|
|
1759
|
-
ok: true,
|
|
1760
|
-
checkpointed: true,
|
|
1761
|
-
session_id: session.sessionId,
|
|
1762
|
-
tab_id: session.tabId,
|
|
1763
|
-
indexed: syncResult.indexed,
|
|
1764
|
-
mode: syncResult.mode,
|
|
1765
|
-
domain: syncResult.domain,
|
|
1766
|
-
skill_id: syncResult.skill_id,
|
|
1767
|
-
endpoint_count: syncResult.endpoint_count,
|
|
1768
|
-
endpoints: syncResult.endpoints,
|
|
1769
|
-
request_count: syncResult.request_count,
|
|
1770
|
-
pipeline: syncResult.pipeline,
|
|
1771
|
-
publish_policy: syncResult.publish_policy,
|
|
1772
|
-
background_publish_queued: syncResult.background_publish_queued,
|
|
1773
|
-
next_step: buildCheckpointNextStep("sync", syncResult, session.sessionId),
|
|
1774
|
-
});
|
|
1775
|
-
} catch (error) {
|
|
1776
|
-
return sendBrowseSessionError(reply, error);
|
|
1777
|
-
}
|
|
1778
|
-
});
|
|
1779
|
-
|
|
1780
|
-
// POST /v1/browse/close — checkpoint capture, queue index+publish, save auth, close tab
|
|
1781
|
-
app.post("/v1/browse/close", async (req, reply) => {
|
|
1782
|
-
try {
|
|
1783
|
-
const browseClient = selectBrowseBrokerClient(requestedSessionId(req));
|
|
1784
|
-
const { session, result: syncResult } = await withSerializedStrictBrowseSession(
|
|
1785
|
-
browseSessions,
|
|
1786
|
-
browseClient,
|
|
1787
|
-
requestedSessionId(req),
|
|
1788
|
-
async (session) => {
|
|
1789
|
-
const broker = brokerForSession(session);
|
|
1790
|
-
if (session.domain) {
|
|
1791
|
-
await saveAuthProfileBestEffort(session.tabId, session.domain, "browse_close");
|
|
1792
|
-
}
|
|
1793
|
-
const syncResult = await flushBrowseCapture(session, { queueIndex: true, queuePublish: true });
|
|
1794
|
-
await broker.closeTab(session.tabId).catch(() => {});
|
|
1795
|
-
removeBrowseSession(browseSessions, session.sessionId);
|
|
1796
|
-
return syncResult;
|
|
1797
|
-
},
|
|
1798
|
-
);
|
|
1799
|
-
return reply.send({
|
|
1800
|
-
ok: true,
|
|
1801
|
-
checkpointed: true,
|
|
1802
|
-
session_id: session.sessionId,
|
|
1803
|
-
indexed: syncResult.indexed,
|
|
1804
|
-
mode: syncResult.mode,
|
|
1805
|
-
endpoint_count: syncResult.endpoint_count,
|
|
1806
|
-
request_count: syncResult.request_count,
|
|
1807
|
-
pipeline: syncResult.pipeline,
|
|
1808
|
-
publish_policy: syncResult.publish_policy,
|
|
1809
|
-
background_publish_queued: syncResult.background_publish_queued,
|
|
1810
|
-
auth_saved: session.domain || null,
|
|
1811
|
-
next_step: buildCheckpointNextStep("close", syncResult, session.sessionId),
|
|
1812
|
-
});
|
|
1813
|
-
} catch (error) {
|
|
1814
|
-
return sendBrowseSessionError(reply, error);
|
|
1815
|
-
}
|
|
1816
|
-
});
|
|
1817
|
-
}
|
|
1818
|
-
|
|
1819
|
-
function saveTrace(trace: unknown) {
|
|
1820
|
-
if (!existsSync(TRACES_DIR)) mkdirSync(TRACES_DIR, { recursive: true });
|
|
1821
|
-
const t = trace as { trace_id: string };
|
|
1822
|
-
writeFileSync(join(TRACES_DIR, `${t.trace_id}.json`), JSON.stringify(trace, null, 2));
|
|
1823
|
-
}
|