unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/cli.js +79 -33
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +73 -22
  4. package/dist/server.js +25994 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/manifest.json +1 -1
  7. package/runtime-src/agent-outcome.ts +0 -166
  8. package/runtime-src/analytics-session.ts +0 -55
  9. package/runtime-src/api/browse-index.ts +0 -343
  10. package/runtime-src/api/browse-session.ts +0 -572
  11. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  12. package/runtime-src/api/browse-submit.ts +0 -1184
  13. package/runtime-src/api/routes.ts +0 -1833
  14. package/runtime-src/auth/browser-cookies.ts +0 -423
  15. package/runtime-src/auth/index.ts +0 -535
  16. package/runtime-src/auth/runtime.ts +0 -116
  17. package/runtime-src/browser/index.ts +0 -659
  18. package/runtime-src/browser/types.ts +0 -41
  19. package/runtime-src/build-info.generated.ts +0 -6
  20. package/runtime-src/capture/index.ts +0 -1929
  21. package/runtime-src/capture/prefetch.ts +0 -95
  22. package/runtime-src/capture/rsc.ts +0 -45
  23. package/runtime-src/cli/shortcuts.ts +0 -273
  24. package/runtime-src/cli.ts +0 -1734
  25. package/runtime-src/client/graph-client.ts +0 -100
  26. package/runtime-src/client/index.ts +0 -1425
  27. package/runtime-src/debug-trace.ts +0 -18
  28. package/runtime-src/domain.ts +0 -38
  29. package/runtime-src/execution/index.ts +0 -3407
  30. package/runtime-src/execution/retry.ts +0 -46
  31. package/runtime-src/execution/robots.ts +0 -167
  32. package/runtime-src/execution/search-forms.ts +0 -188
  33. package/runtime-src/execution/token-resolver.ts +0 -135
  34. package/runtime-src/extraction/index.ts +0 -1507
  35. package/runtime-src/foundry/publish-bundle.ts +0 -392
  36. package/runtime-src/graph/agent-augment.ts +0 -315
  37. package/runtime-src/graph/index.ts +0 -1532
  38. package/runtime-src/graph/local-fixtures.ts +0 -393
  39. package/runtime-src/graph/local-harness.ts +0 -646
  40. package/runtime-src/graph/planner.ts +0 -411
  41. package/runtime-src/graph/session.ts +0 -294
  42. package/runtime-src/graph/trace-store.ts +0 -136
  43. package/runtime-src/impact-log.ts +0 -227
  44. package/runtime-src/index.ts +0 -24
  45. package/runtime-src/indexer/index.ts +0 -465
  46. package/runtime-src/intent-match.ts +0 -1515
  47. package/runtime-src/kuri/client.ts +0 -1839
  48. package/runtime-src/logger.ts +0 -30
  49. package/runtime-src/marketplace/index.ts +0 -111
  50. package/runtime-src/mcp.ts +0 -1911
  51. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  52. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  53. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  54. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  55. package/runtime-src/orchestrator/index.ts +0 -4480
  56. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  57. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  58. package/runtime-src/payments/cascade.ts +0 -137
  59. package/runtime-src/payments/index.ts +0 -270
  60. package/runtime-src/payments/lobster-pay.ts +0 -182
  61. package/runtime-src/payments/wallet.ts +0 -98
  62. package/runtime-src/publish/review-context.ts +0 -93
  63. package/runtime-src/publish/sanitize.ts +0 -197
  64. package/runtime-src/publish/schema-review.ts +0 -192
  65. package/runtime-src/publish-admission.ts +0 -388
  66. package/runtime-src/ratelimit/index.ts +0 -23
  67. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  68. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  69. package/runtime-src/reverse-engineer/index.ts +0 -1551
  70. package/runtime-src/reverse-engineer/token-sources.ts +0 -379
  71. package/runtime-src/router.ts +0 -17
  72. package/runtime-src/routing-telemetry.ts +0 -395
  73. package/runtime-src/runtime/browser-access.ts +0 -11
  74. package/runtime-src/runtime/browser-auth.ts +0 -12
  75. package/runtime-src/runtime/browser-host.ts +0 -48
  76. package/runtime-src/runtime/lifecycle.ts +0 -17
  77. package/runtime-src/runtime/local-server.ts +0 -311
  78. package/runtime-src/runtime/paths.ts +0 -99
  79. package/runtime-src/runtime/setup.ts +0 -251
  80. package/runtime-src/runtime/supervisor.ts +0 -69
  81. package/runtime-src/runtime/update-hints.ts +0 -351
  82. package/runtime-src/server.ts +0 -100
  83. package/runtime-src/session-logs.ts +0 -142
  84. package/runtime-src/settings.ts +0 -221
  85. package/runtime-src/single-binary.ts +0 -143
  86. package/runtime-src/site-policy.ts +0 -54
  87. package/runtime-src/stale-cleanup-runner.ts +0 -144
  88. package/runtime-src/stale-cleanup.ts +0 -133
  89. package/runtime-src/telemetry-attribution.ts +0 -120
  90. package/runtime-src/telemetry.ts +0 -253
  91. package/runtime-src/template-params.ts +0 -141
  92. package/runtime-src/transform/drift.ts +0 -60
  93. package/runtime-src/transform/index.ts +0 -277
  94. package/runtime-src/types/index.ts +0 -1
  95. package/runtime-src/types/skill.ts +0 -931
  96. package/runtime-src/vault/index.ts +0 -196
  97. package/runtime-src/verification/auth-gate.ts +0 -8
  98. package/runtime-src/verification/candidates.ts +0 -27
  99. package/runtime-src/verification/index.ts +0 -120
  100. package/runtime-src/verification/matrix.ts +0 -30
  101. package/runtime-src/version.ts +0 -148
  102. package/runtime-src/workflow/artifact.ts +0 -161
  103. package/runtime-src/workflow/compile.ts +0 -808
  104. package/runtime-src/workflow/publish.ts +0 -225
  105. package/runtime-src/workflow/runtime.ts +0 -213
@@ -1,3407 +0,0 @@
1
- import { executeInBrowser, triggerAndIntercept } from "../capture/index.js";
2
- import { captureSession } from "../capture/index.js";
3
- import type { CaptureResult, RawRequest } from "../capture/index.js";
4
- import { extractEndpoints, extractAuthHeaders, type ExtractionContext } from "../reverse-engineer/index.js";
5
- import { scanBundlesForRoutes } from "../reverse-engineer/bundle-scanner.js";
6
- import { publishSkill, mergeEndpoints } from "../marketplace/index.js";
7
- import { updateEndpointScore } from "../marketplace/index.js";
8
- import { getCredential, storeCredential, deleteCredential } from "../vault/index.js";
9
- import { getStoredAuth, getAuthCookies, refreshAuthFromBrowser } from "../auth/index.js";
10
- import { resolvePreExecutionAuth } from "../auth/dependency-runtime.js";
11
- import { authRuntime } from "../auth/runtime.js";
12
- import { applyProjection, inferSchema } from "../transform/index.js";
13
- import { detectSchemaDrift } from "../transform/drift.js";
14
- import { recordExecution, recordTransaction, cachePublishedSkill, findExistingSkillForDomain, getLocalWalletContext, updateEndpointSchema } from "../client/index.js";
15
- import { validateManifest } from "../client/index.js";
16
- import { withRetry, isRetryableStatus } from "./retry.js";
17
- import type { EndpointDescriptor, ExecutionOptions, ExecutionTrace, ProjectionOptions, SkillManifest } from "../types/index.js";
18
- import { nanoid } from "nanoid";
19
- import { getRegistrableDomain } from "../domain.js";
20
- import { extractFromDOM, extractFromDOMWithHint } from "../extraction/index.js";
21
- import { buildSkillOperationGraph, getEndpointDescriptionMetadata, inferEndpointSemantic, resolveEndpointSemantic } from "../graph/index.js";
22
- import { log } from "../logger.js";
23
- import { TRACE_VERSION } from "../version.js";
24
- import { buildQueryBindingMap, extractTemplateQueryBindings, mergeContextTemplateParams } from "../template-params.js";
25
- import { assessIntentResult, projectIntentData } from "../intent-match.js";
26
- import { isStructuredSearchForm, detectSearchForms, type SearchFormSpec } from "./search-forms.js";
27
- import { attributeLifecycle, type LifecycleEvent, type LifecyclePhase } from "../runtime/lifecycle.js";
28
- import { queueBackgroundIndex } from "../indexer/index.js";
29
- import {
30
- writeSkillSnapshot,
31
- domainSkillCache,
32
- persistDomainCache,
33
- getDomainReuseKey,
34
- scopedCacheKey,
35
- buildResolveCacheKey,
36
- snapshotPathForCacheKey,
37
- generateLocalDescription,
38
- } from "../orchestrator/index.js";
39
- import { checkPaymentRequirement } from "../payments/index.js";
40
- import { isAllowedByRobots } from "./robots.js";
41
- import { annotateEndpointPolicy, endpointRequiresThirdPartyTermsConfirmation, getEndpointPolicy } from "../site-policy.js";
42
- import {
43
- mergeWorkflowArtifacts,
44
- readWorkflowArtifact,
45
- recordWorkflowRecipeOutcome,
46
- writeWorkflowArtifact,
47
- } from "../workflow/artifact.js";
48
- import { buildWorkflowArtifactFromCapture } from "../workflow/compile.js";
49
- import {
50
- needsWorkflowTokenRefresh,
51
- pickWorkflowRecipe,
52
- resolveWorkflowBindings,
53
- translateWorkflowStrategy,
54
- validateWorkflowReplayParams,
55
- } from "../workflow/runtime.js";
56
- import { buildWorkflowPublishArtifact, writeWorkflowPublishArtifact } from "../workflow/publish.js";
57
- /** Stamp every trace with the code version hash for telemetry tracking */
58
- function stampTrace(trace: ExecutionTrace): ExecutionTrace {
59
- trace.trace_version = TRACE_VERSION;
60
- return trace;
61
- }
62
-
63
- const DEFAULT_BROWSER_UA =
64
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36";
65
-
66
- function cloneReplayBody(body: unknown): unknown {
67
- if (!body || typeof body !== "object") return body;
68
- if (Array.isArray(body)) return body.map((entry) => cloneReplayBody(entry));
69
- return { ...(body as Record<string, unknown>) };
70
- }
71
-
72
- function serializeReplayBody(body: unknown, headers: Record<string, string>): BodyInit | undefined {
73
- if (body == null) return undefined;
74
- if (typeof body === "string") return body;
75
- if (body instanceof URLSearchParams) return body.toString();
76
- if (body instanceof FormData) return body;
77
- const contentType = headers["content-type"] ?? headers["Content-Type"] ?? "";
78
- if (/application\/x-www-form-urlencoded/i.test(contentType) && body && typeof body === "object" && !Array.isArray(body)) {
79
- const params = new URLSearchParams();
80
- for (const [key, value] of Object.entries(body as Record<string, unknown>)) {
81
- if (value == null) continue;
82
- params.set(key, String(value));
83
- }
84
- return params.toString();
85
- }
86
- return JSON.stringify(body);
87
- }
88
-
89
- async function reloadExecutionAuthState(
90
- skill: SkillManifest,
91
- epDomain: string,
92
- authHeaders: Record<string, string>,
93
- cookies: Array<{ name: string; value: string; domain: string }>,
94
- ): Promise<void> {
95
- for (const key of Object.keys(authHeaders)) delete authHeaders[key];
96
- cookies.splice(0, cookies.length);
97
-
98
- if (skill.auth_profile_ref) {
99
- const stored = await getCredential(skill.auth_profile_ref);
100
- if (stored) {
101
- try {
102
- const parsed = JSON.parse(stored) as {
103
- headers?: Record<string, string>;
104
- cookies?: Array<{ name: string; value: string; domain: string }>;
105
- };
106
- Object.assign(authHeaders, parsed.headers ?? {});
107
- cookies.push(...(parsed.cookies ?? []));
108
- } catch {
109
- /* ignore malformed auth state */
110
- }
111
- }
112
- }
113
-
114
- if (cookies.length === 0) {
115
- try {
116
- const resolved = await getAuthCookies(epDomain, {
117
- autoExtract: !!skill.auth_profile_ref,
118
- });
119
- if (resolved?.length) cookies.push(...resolved);
120
- } catch {
121
- /* ignore */
122
- }
123
- }
124
-
125
- if (Object.keys(authHeaders).length === 0) {
126
- try {
127
- const sessionKey = `${getRegistrableDomain(epDomain)}-session`;
128
- const sessionData = await getCredential(sessionKey);
129
- if (sessionData) {
130
- const parsed = JSON.parse(sessionData) as { headers?: Record<string, string>; cookies?: typeof cookies };
131
- if (parsed.headers) Object.assign(authHeaders, parsed.headers);
132
- if (parsed.cookies && cookies.length === 0) cookies.push(...parsed.cookies);
133
- }
134
- } catch {
135
- /* ignore */
136
- }
137
- }
138
- }
139
-
140
- function persistWorkflowArtifactForCapture(
141
- artifactSkill: SkillManifest,
142
- captured: Pick<CaptureResult, "requests" | "har_lineage_id" | "final_url" | "html" | "js_bundles" | "cookies">,
143
- capturedAuthHeaders?: Record<string, string>,
144
- ): void {
145
- try {
146
- if (process.env.UNBROWSE_DEBUG_WORKFLOW === "1") {
147
- log(
148
- "workflow",
149
- `capture artifact attempt skill=${artifactSkill.skill_id} requests=${captured.requests.length} final_url=${captured.final_url}`,
150
- );
151
- }
152
- const nextArtifact = mergeWorkflowArtifacts(
153
- buildWorkflowArtifactFromCapture(artifactSkill, captured, { authHeaders: capturedAuthHeaders }),
154
- readWorkflowArtifact(artifactSkill.skill_id),
155
- );
156
- const writtenPath = writeWorkflowArtifact(nextArtifact);
157
- const exportPath = writeWorkflowPublishArtifact(buildWorkflowPublishArtifact(artifactSkill, nextArtifact, {
158
- publishStatus: "captured",
159
- }));
160
- if (process.env.UNBROWSE_DEBUG_WORKFLOW === "1") {
161
- log(
162
- "workflow",
163
- `capture persisted skill=${artifactSkill.skill_id} requests=${captured.requests.length} path=${writtenPath ?? "write-failed"} export=${exportPath ?? "write-failed"}`,
164
- );
165
- }
166
- } catch (error) {
167
- const message = error instanceof Error ? error.message : String(error);
168
- log("workflow", `capture persistence failed for ${artifactSkill.skill_id}: ${message}`);
169
- }
170
- }
171
-
172
- // ---------------------------------------------------------------------------
173
- // Quality gate — validate extracted data before marketplace publishing
174
- // ---------------------------------------------------------------------------
175
-
176
- interface QualityResult {
177
- valid: boolean;
178
- quality_note?: string;
179
- }
180
-
181
- const VALID_VERIFICATION_STATUSES = new Set(["verified", "unverified", "failed", "pending"]);
182
-
183
- function normalizeEndpointForManifest(endpoint: EndpointDescriptor): EndpointDescriptor {
184
- const verification_status = VALID_VERIFICATION_STATUSES.has(endpoint.verification_status)
185
- ? endpoint.verification_status
186
- : (endpoint.response_schema || endpoint.dom_extraction ? "verified" : "pending");
187
- return { ...endpoint, verification_status };
188
- }
189
-
190
- async function prepareLearnedEndpoints(
191
- endpoints: EndpointDescriptor[],
192
- _intent: string,
193
- _domain: string,
194
- ): Promise<EndpointDescriptor[]> {
195
- return endpoints.map(normalizeEndpointForManifest);
196
- }
197
-
198
- function intentWantsStructuredRecords(intent?: string): boolean {
199
- return /\b(search|list|find|get|fetch|timeline|feed|trending)\b/i.test(intent ?? "");
200
- }
201
-
202
- export function isBundleInferredEndpoint(endpoint: Pick<EndpointDescriptor, "description">): boolean {
203
- return /inferred from js bundle/i.test(endpoint.description ?? "");
204
- }
205
-
206
- function isSupportEvidenceEndpoint(endpoint: EndpointDescriptor): boolean {
207
- if (isCanonicalReplayEndpoint(endpoint)) return true;
208
- if (endpoint.dom_extraction && endpoint.response_schema) return true;
209
- if (isBundleInferredEndpoint(endpoint)) return false;
210
- return !!endpoint.response_schema;
211
- }
212
-
213
- function looksLikeUiChromeText(value: string): boolean {
214
- const lower = value.toLowerCase();
215
- let hits = 0;
216
- for (const token of [
217
- "advanced search",
218
- "pull requests",
219
- "discussions",
220
- "languages",
221
- "more languages",
222
- "owner",
223
- "number of stars",
224
- "number of forks",
225
- "date created",
226
- "date pushed",
227
- "public private",
228
- "results",
229
- ]) {
230
- if (lower.includes(token)) hits++;
231
- }
232
- return hits >= 2;
233
- }
234
-
235
- /** Detect concatenated values like "AAPLApple" or "Inc978,583" */
236
- function isConcatenatedValue(s: string): boolean {
237
- // Uppercase ticker jammed onto capitalized word: AAPLApple, NVDANvidia
238
- if (/[A-Z]{2,}[A-Z][a-z]/.test(s)) return true;
239
- // Word ending in letter immediately followed by digits: Inc978, Corp123
240
- if (/[a-zA-Z]\d{3,}/.test(s)) return true;
241
- return false;
242
- }
243
-
244
- /**
245
- * Validate extraction quality. Always returns data to the caller —
246
- * this only gates whether we publish to the marketplace.
247
- */
248
- export function validateExtractionQuality(data: unknown, confidence: number, intent?: string): QualityResult {
249
- // 1. Min confidence
250
- if (confidence < 0.5) {
251
- return { valid: false, quality_note: `confidence too low (${confidence.toFixed(2)} < 0.5)` };
252
- }
253
-
254
- // Only validate arrays (repeated data structures)
255
- if (!Array.isArray(data)) return { valid: true };
256
- if (data.length === 0) return { valid: true };
257
- if (intentWantsStructuredRecords(intent) && data.every((item) => !item || typeof item !== "object" || Array.isArray(item))) {
258
- return { valid: false, quality_note: "primitive rows only — expected structured records" };
259
- }
260
-
261
- const stringRows = data.filter((item): item is string => typeof item === "string");
262
- if (stringRows.length === data.length) {
263
- const uiChromeRows = stringRows.filter((item) => looksLikeUiChromeText(item));
264
- if (uiChromeRows.length / stringRows.length >= 0.5) {
265
- return { valid: false, quality_note: "ui chrome text detected instead of structured records" };
266
- }
267
- }
268
-
269
- // 2. Deduplication check
270
- const serialized = data.map((item) => JSON.stringify(item));
271
- const unique = new Set(serialized);
272
- const dupeRatio = 1 - unique.size / serialized.length;
273
- if (dupeRatio > 0.5) {
274
- return { valid: false, quality_note: `${Math.round(dupeRatio * 100)}% duplicate rows` };
275
- }
276
-
277
- // 3. Concatenation detection
278
- let totalStrings = 0;
279
- let concatStrings = 0;
280
- for (const item of data) {
281
- if (item && typeof item === "object") {
282
- for (const val of Object.values(item as Record<string, unknown>)) {
283
- if (typeof val === "string" && val.length > 3) {
284
- totalStrings++;
285
- if (isConcatenatedValue(val)) concatStrings++;
286
- }
287
- }
288
- }
289
- }
290
- if (totalStrings > 0 && concatStrings / totalStrings > 0.3) {
291
- return { valid: false, quality_note: `${Math.round((concatStrings / totalStrings) * 100)}% concatenated values detected` };
292
- }
293
-
294
- // 4. Diversity check — reject if all items share the same link/title (nav chrome)
295
- if (data.length >= 3) {
296
- for (const field of ["link", "href", "url", "title"]) {
297
- const vals = data
298
- .map((item) => (item && typeof item === "object" ? (item as Record<string, unknown>)[field] : undefined))
299
- .filter((v) => v != null);
300
- if (vals.length >= 3) {
301
- const uniqueVals = new Set(vals.map(String));
302
- if (uniqueVals.size === 1) {
303
- return { valid: false, quality_note: `all items share the same "${field}" — likely navigation chrome` };
304
- }
305
- }
306
- }
307
- }
308
-
309
- return { valid: true };
310
- }
311
-
312
- export interface ExecutionResult {
313
- trace: ExecutionTrace;
314
- result: unknown;
315
- learned_skill?: SkillManifest;
316
- }
317
-
318
- export function projectResultForIntent(data: unknown, intent?: string): unknown {
319
- return projectIntentData(data, intent);
320
- }
321
-
322
- function inferActionKindFromIntent(intent: string): string {
323
- const lower = intent.toLowerCase();
324
- if (/\b(search|find|lookup)\b/.test(lower)) return "search";
325
- if (/\b(list|feed|timeline|trending)\b/.test(lower)) return "list";
326
- return "detail";
327
- }
328
-
329
- function sanitizeNavigationQueryParams(url: URL): URL {
330
- const out = new URL(url.toString());
331
- for (const key of [...out.searchParams.keys()]) {
332
- const lower = key.toLowerCase();
333
- if (lower === "url" || lower === "context_url" || lower === "intent" || lower === "redirect" || lower === "redirect_url") {
334
- out.searchParams.delete(key);
335
- }
336
- }
337
- return out;
338
- }
339
-
340
- function restoreTemplatePlaceholderEncoding(url: string): string {
341
- // Only restore template placeholders like {variable_name}, not arbitrary JSON braces.
342
- // Template placeholders: %7Bword_chars%7D (no spaces, no quotes, no colons inside)
343
- return url.replace(/%7B(\w+)%7D/gi, "{$1}");
344
- }
345
-
346
- function compactSchemaSample(value: unknown, depth = 0): unknown {
347
- if (depth >= 4) return Array.isArray(value) ? [] : value && typeof value === "object" ? "[truncated]" : value;
348
- if (Array.isArray(value)) return value.slice(0, 3).map((item) => compactSchemaSample(item, depth + 1));
349
- if (value && typeof value === "object") {
350
- return Object.fromEntries(
351
- Object.entries(value as Record<string, unknown>)
352
- .slice(0, 20)
353
- .map(([key, next]) => [key, compactSchemaSample(next, depth + 1)]),
354
- );
355
- }
356
- return value;
357
- }
358
-
359
- function isDocumentLikeUrl(url: string): boolean {
360
- try {
361
- const parsed = new URL(url);
362
- return !/\/api\/|graphql|\/rest\/|\/rpc\/|\/ajax\/|\/1\.1\/|\/2\/|voyager/i.test(parsed.pathname);
363
- } catch {
364
- return false;
365
- }
366
- }
367
-
368
- export function resolveExecutionUrlTemplate(
369
- endpoint: EndpointDescriptor,
370
- contextUrl?: string,
371
- ): string {
372
- if (!contextUrl) return endpoint.url_template;
373
- if (endpoint.method !== "GET") return endpoint.url_template;
374
- if (!isDocumentLikeUrl(endpoint.url_template)) return endpoint.url_template;
375
- if (endpoint.trigger_url && !isDocumentLikeUrl(endpoint.trigger_url)) return endpoint.url_template;
376
- return contextUrl;
377
- }
378
-
379
- export function shouldIgnoreLearnedBrowserStrategy(
380
- endpoint: EndpointDescriptor,
381
- resolvedUrl: string,
382
- ): boolean {
383
- return endpoint.method === "GET" && !endpoint.dom_extraction && !isDocumentLikeUrl(resolvedUrl);
384
- }
385
-
386
- function deriveStructuredDataReplay(url: string, mode: "concrete" | "template"): string {
387
- try {
388
- const parsed = new URL(url);
389
- const host = parsed.hostname.replace(/^www\./, "");
390
- const pathname = parsed.pathname.replace(/\/+$/, "") || "/";
391
- const templateParam = (primary: string): string => `{${primary}}`;
392
-
393
- if (host === "mastodon.social") {
394
- if (pathname === "/search" || pathname === "/search/") {
395
- if (mode === "concrete" && !parsed.searchParams.get("q")) return url;
396
- const replay = new URL("https://mastodon.social/api/v2/search");
397
- replay.searchParams.set("q", mode === "template" ? templateParam("q") : (parsed.searchParams.get("q") ?? ""));
398
- replay.searchParams.set("resolve", "false");
399
- replay.searchParams.set("type", "statuses");
400
- replay.searchParams.set("limit", "20");
401
- return restoreTemplatePlaceholderEncoding(replay.toString());
402
- }
403
- if (pathname === "/public") {
404
- const replay = new URL("https://mastodon.social/api/v1/timelines/public");
405
- replay.searchParams.set("limit", "20");
406
- return replay.toString();
407
- }
408
- }
409
-
410
- if (host === "gitlab.com") {
411
- if (pathname === "/explore/projects") {
412
- if (mode === "concrete" && !parsed.searchParams.get("name") && !parsed.searchParams.get("search")) return url;
413
- const replay = new URL("https://gitlab.com/api/v4/projects");
414
- const search = parsed.searchParams.get("search");
415
- const name = parsed.searchParams.get("name");
416
- replay.searchParams.set(
417
- "search",
418
- mode === "template"
419
- ? (search && search.length > 0 ? "{search}" : name && name.length > 0 ? "{name}" : "{q}")
420
- : (name ?? search ?? ""),
421
- );
422
- replay.searchParams.set("simple", "true");
423
- replay.searchParams.set("per_page", "20");
424
- return restoreTemplatePlaceholderEncoding(replay.toString());
425
- }
426
-
427
- const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
428
- const reserved = new Set([
429
- "-",
430
- "api",
431
- "admin",
432
- "dashboard",
433
- "explore",
434
- "groups",
435
- "help",
436
- "oauth",
437
- "profile",
438
- "projects",
439
- "search",
440
- "session",
441
- "users",
442
- ]);
443
- if (segments.length === 2 && !reserved.has(segments[0])) {
444
- return `https://gitlab.com/api/v4/projects/${encodeURIComponent(`${segments[0]}/${segments[1]}`)}`;
445
- }
446
- }
447
-
448
- if (host === "github.com") {
449
- if (pathname === "/search") {
450
- if (mode === "concrete" && !parsed.searchParams.get("q")) return url;
451
- const replay = new URL("https://api.github.com/search/repositories");
452
- replay.searchParams.set(
453
- "q",
454
- mode === "template" ? "{q}" : (parsed.searchParams.get("q") ?? ""),
455
- );
456
- replay.searchParams.set("per_page", "20");
457
- return restoreTemplatePlaceholderEncoding(replay.toString());
458
- }
459
- }
460
-
461
- if (host === "hn.algolia.com") {
462
- if (pathname === "/" || pathname === "") {
463
- if (mode === "concrete" && !parsed.searchParams.get("q") && !parsed.searchParams.get("query")) return url;
464
- const replay = new URL("https://hn.algolia.com/api/v1/search");
465
- replay.searchParams.set(
466
- "query",
467
- mode === "template" ? "{q}" : (parsed.searchParams.get("q") ?? parsed.searchParams.get("query") ?? ""),
468
- );
469
- replay.searchParams.set("tags", "story");
470
- return restoreTemplatePlaceholderEncoding(replay.toString());
471
- }
472
- }
473
-
474
- if (host === "huggingface.co") {
475
- if (pathname === "/models" || pathname === "/models/") {
476
- if (mode === "concrete" && !parsed.searchParams.get("search") && !parsed.searchParams.get("q")) return url;
477
- const replay = new URL("https://huggingface.co/api/models");
478
- replay.searchParams.set(
479
- "search",
480
- mode === "template"
481
- ? (parsed.searchParams.get("search") ? "{search}" : parsed.searchParams.get("q") ? "{q}" : "{search}")
482
- : (parsed.searchParams.get("search") ?? parsed.searchParams.get("q") ?? ""),
483
- );
484
- replay.searchParams.set("limit", "20");
485
- return restoreTemplatePlaceholderEncoding(replay.toString());
486
- }
487
- }
488
-
489
- if (host === "developer.mozilla.org") {
490
- const locale = pathname.split("/").filter(Boolean)[0] || "en-US";
491
- if (pathname.endsWith("/search") || pathname === "/search") {
492
- if (mode === "concrete" && !parsed.searchParams.get("q")) return url;
493
- const replay = new URL("https://developer.mozilla.org/api/v1/search");
494
- replay.searchParams.set(
495
- "q",
496
- mode === "template" ? "{q}" : (parsed.searchParams.get("q") ?? ""),
497
- );
498
- replay.searchParams.set(
499
- "page",
500
- mode === "template"
501
- ? (parsed.searchParams.get("page") ? "{page}" : "1")
502
- : (parsed.searchParams.get("page") ?? "1"),
503
- );
504
- replay.searchParams.set("page_size", parsed.searchParams.get("page_size") ?? "20");
505
- replay.searchParams.set(
506
- "locale",
507
- mode === "template"
508
- ? (parsed.searchParams.get("locale") ? "{locale}" : locale)
509
- : (parsed.searchParams.get("locale") ?? locale),
510
- );
511
- return restoreTemplatePlaceholderEncoding(replay.toString());
512
- }
513
- }
514
-
515
- if (host === "dev.to") {
516
- const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
517
- if (segments[0] === "t" && segments[1]) {
518
- const replay = new URL("https://dev.to/api/articles");
519
- replay.searchParams.set("tag", mode === "template" ? "{tag}" : segments[1]);
520
- replay.searchParams.set("per_page", "20");
521
- return restoreTemplatePlaceholderEncoding(replay.toString());
522
- }
523
- }
524
-
525
- if (host === "npmjs.com") {
526
- if (pathname === "/search" || pathname === "/search/") {
527
- if (mode === "concrete" && !parsed.searchParams.get("q")) return url;
528
- const replay = new URL("https://registry.npmjs.org/-/v1/search");
529
- replay.searchParams.set("text", mode === "template" ? templateParam("q") : (parsed.searchParams.get("q") ?? ""));
530
- replay.searchParams.set("size", "20");
531
- return restoreTemplatePlaceholderEncoding(replay.toString());
532
- }
533
-
534
- const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
535
- if (segments[0] === "package" && segments.length >= 2) {
536
- const versionIndex = segments.indexOf("v");
537
- const packageName = segments.slice(1, versionIndex === -1 ? undefined : versionIndex).join("/");
538
- if (packageName) return `https://registry.npmjs.org/${encodeURIComponent(packageName)}`;
539
- }
540
- }
541
-
542
- if (host === "pypi.org") {
543
- const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
544
- if (segments[0] === "project" && segments[1]) {
545
- return `https://pypi.org/pypi/${encodeURIComponent(segments[1])}/json`;
546
- }
547
- }
548
-
549
- if (host === "pub.dev") {
550
- const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
551
- if (segments[0] === "packages" && segments[1]) {
552
- return `https://pub.dev/api/packages/${encodeURIComponent(segments[1])}`;
553
- }
554
- }
555
-
556
- if (host === "hub.docker.com") {
557
- if (pathname === "/search" || pathname === "/search/") {
558
- if (mode === "concrete" && !parsed.searchParams.get("q") && !parsed.searchParams.get("query")) return url;
559
- const replay = new URL("https://hub.docker.com/v2/search/repositories/");
560
- replay.searchParams.set(
561
- "query",
562
- mode === "template"
563
- ? (parsed.searchParams.get("query") ? "{query}" : "{q}")
564
- : (parsed.searchParams.get("q") ?? parsed.searchParams.get("query") ?? ""),
565
- );
566
- replay.searchParams.set("page_size", "20");
567
- return restoreTemplatePlaceholderEncoding(replay.toString());
568
- }
569
-
570
- const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
571
- if (segments[0] === "r" && segments.length >= 4 && segments[3] === "tags") {
572
- return `https://hub.docker.com/v2/repositories/${encodeURIComponent(segments[1])}/${encodeURIComponent(segments[2])}/tags/?page_size=25`;
573
- }
574
- }
575
-
576
- if (host === "rubygems.org") {
577
- const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
578
- if (segments[0] === "gems" && segments[1]) {
579
- return `https://rubygems.org/api/v1/gems/${encodeURIComponent(segments[1])}.json`;
580
- }
581
- }
582
-
583
- if (host === "stackoverflow.com" || host === "serverfault.com" || host === "superuser.com") {
584
- const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
585
- if (segments[0] === "questions" && segments[1] === "tagged" && segments[2]) {
586
- const replay = new URL("https://api.stackexchange.com/2.3/questions");
587
- replay.searchParams.set("site", host === "stackoverflow.com" ? "stackoverflow" : host.replace(/\.com$/, ""));
588
- replay.searchParams.set("tagged", mode === "template" ? "{tag}" : segments[2]);
589
- replay.searchParams.set("order", "desc");
590
- replay.searchParams.set("sort", "activity");
591
- replay.searchParams.set("pagesize", "20");
592
- replay.searchParams.set("filter", "default");
593
- return restoreTemplatePlaceholderEncoding(replay.toString());
594
- }
595
- }
596
-
597
- if (host === "jmail.world") {
598
- if (pathname === "/search" || pathname === "/search/") {
599
- if (mode === "concrete" && !parsed.searchParams.get("q")) return url;
600
- const replay = new URL("https://jmail.world/api/emails/search");
601
- replay.searchParams.set("q", mode === "template" ? "{q}" : (parsed.searchParams.get("q") ?? ""));
602
- replay.searchParams.set("limit", mode === "template" ? "{limit}" : (parsed.searchParams.get("limit") ?? "50"));
603
- replay.searchParams.set("page", mode === "template" ? "{page}" : (parsed.searchParams.get("page") ?? "1"));
604
- replay.searchParams.set("source", mode === "template" ? "{source}" : (parsed.searchParams.get("source") ?? "all"));
605
- return restoreTemplatePlaceholderEncoding(replay.toString());
606
- }
607
- }
608
-
609
- if (host !== "reddit.com" && host !== "old.reddit.com" && host !== "np.reddit.com") return url;
610
- if (/\.json$/i.test(parsed.pathname) || /\/api\/|\/svc\/|graphql/i.test(parsed.pathname)) return url;
611
- if (parsed.pathname === "/search" || parsed.pathname === "/search/") {
612
- if (mode === "concrete" && !parsed.searchParams.get("q")) return url;
613
- parsed.pathname = "/search.json";
614
- if (mode === "template" && !parsed.searchParams.get("q")) parsed.searchParams.set("q", "{q}");
615
- return parsed.toString();
616
- }
617
- if (parsed.pathname.startsWith("/r/") || parsed.pathname.startsWith("/comments/")) {
618
- parsed.pathname = `${parsed.pathname.replace(/\/$/, "")}/.json`;
619
- return parsed.toString();
620
- }
621
- return url;
622
- } catch {
623
- return url;
624
- }
625
- }
626
-
627
- export function deriveStructuredDataReplayUrl(url: string): string {
628
- return deriveStructuredDataReplay(url, "concrete");
629
- }
630
-
631
- export function deriveStructuredDataReplayTemplate(url: string): string {
632
- return deriveStructuredDataReplay(url, "template");
633
- }
634
-
635
- export function deriveStructuredDataReplayCandidates(url: string): string[] {
636
- const primary = deriveStructuredDataReplayUrl(url);
637
- const out = new Set<string>([primary]);
638
- try {
639
- const parsed = new URL(primary);
640
- const host = parsed.hostname.replace(/^www\./, "");
641
- if (host === "reddit.com" || host === "np.reddit.com") {
642
- parsed.hostname = "old.reddit.com";
643
- out.add(parsed.toString());
644
- }
645
- } catch {
646
- // ignore
647
- }
648
- return [...out];
649
- }
650
-
651
- export function deriveStructuredDataReplayCandidatesFromInputs(
652
- url: string,
653
- params: Record<string, unknown> = {},
654
- ): string[] {
655
- const seeded = new Set<string>();
656
- const replayTemplate = deriveStructuredDataReplayTemplate(url);
657
- if (replayTemplate !== url) {
658
- const seededReplay = interpolate(
659
- replayTemplate,
660
- mergeContextTemplateParams(params, replayTemplate, url),
661
- );
662
- if (!/\{[^}]+\}/.test(seededReplay)) seeded.add(seededReplay);
663
- }
664
- for (const replayUrl of deriveStructuredDataReplayCandidates(url)) seeded.add(replayUrl);
665
- return [...seeded];
666
- }
667
-
668
- export function buildStructuredReplayHeaders(
669
- originalUrl: string,
670
- replayUrl: string,
671
- baseHeaders: Record<string, string>,
672
- ): Record<string, string> {
673
- const headers = { ...baseHeaders };
674
- try {
675
- const replayTarget = new URL(replayUrl);
676
- const originalTarget = new URL(originalUrl);
677
- const host = replayTarget.hostname.replace(/^www\./, "");
678
- const needsApiReplayHeaders =
679
- replayTarget.hostname !== originalTarget.hostname ||
680
- /\/api\/|graphql|\/rest\/|\/rpc\/|\/v\d+\//i.test(replayTarget.pathname);
681
- if (needsApiReplayHeaders) {
682
- headers["user-agent"] ??= DEFAULT_BROWSER_UA;
683
- headers["accept-language"] ??= "en-US,en;q=0.9";
684
- headers["referer"] ??= originalTarget.toString();
685
- headers["accept"] ??= "application/json,text/plain,*/*";
686
- }
687
- if (host === "reddit.com" || host === "old.reddit.com" || host === "np.reddit.com") {
688
- headers["user-agent"] ??= DEFAULT_BROWSER_UA;
689
- headers["accept-language"] ??= "en-US,en;q=0.9";
690
- headers["referer"] ??= originalTarget.toString();
691
- headers["accept"] = "application/json,text/plain,*/*";
692
- }
693
- } catch {
694
- return headers;
695
- }
696
- return headers;
697
- }
698
-
699
- function normalizeReplayHeaders(
700
- ...bags: Array<Record<string, string> | undefined>
701
- ): Record<string, string> {
702
- const normalized: Record<string, string> = {};
703
- for (const bag of bags) {
704
- for (const [key, value] of Object.entries(bag ?? {})) {
705
- if (typeof value !== "string") continue;
706
- const trimmed = value.trim();
707
- if (!trimmed) continue;
708
- normalized[key.toLowerCase()] = trimmed;
709
- }
710
- }
711
- return normalized;
712
- }
713
-
714
- function shouldFallbackToBrowserReplay(
715
- data: unknown,
716
- endpoint: EndpointDescriptor,
717
- intent?: string,
718
- contextUrl?: string,
719
- ): boolean {
720
- const replayUrl = resolveExecutionUrlTemplate(endpoint, contextUrl);
721
- if (!isDocumentLikeUrl(replayUrl)) return false;
722
- if (typeof data === "string") return isHtml(data) || isSpaShell(data);
723
- const assessment = assessIntentResult(data, intent);
724
- return assessment.verdict === "fail";
725
- }
726
-
727
- function buildSampleRequestFromUrl(url: string): Record<string, unknown> {
728
- try {
729
- return Object.fromEntries(sanitizeNavigationQueryParams(new URL(url)).searchParams.entries());
730
- } catch {
731
- return {};
732
- }
733
- }
734
-
735
- export function buildPageArtifactCapture(
736
- url: string,
737
- intent: string,
738
- html: string,
739
- authRequired = false,
740
- ): {
741
- endpoint?: EndpointDescriptor;
742
- result?: { data: unknown; _extraction: Record<string, unknown> };
743
- quality_note?: string;
744
- search_form?: SearchFormSpec;
745
- } {
746
- const extracted = extractFromDOM(html, intent);
747
- if (!extracted.data || extracted.confidence <= 0.2) return {};
748
- const quality = validateExtractionQuality(extracted.data, extracted.confidence, intent);
749
- if (!quality.valid) {
750
- return { quality_note: quality.quality_note ?? "low_quality_dom_extraction" };
751
- }
752
- const semanticAssessment = assessIntentResult(extracted.data, intent);
753
- if (semanticAssessment.verdict === "fail") {
754
- return { quality_note: semanticAssessment.reason };
755
- }
756
-
757
- // Detect structured search forms from the captured HTML
758
- const searchForms = detectSearchForms(html);
759
- const validSearchForm = searchForms.find((spec: SearchFormSpec) => isStructuredSearchForm(spec));
760
-
761
- const response_schema = inferSchema([extracted.data]);
762
- const endpoint: EndpointDescriptor = {
763
- endpoint_id: nanoid(),
764
- method: "GET",
765
- url_template: templatizeQueryParams(url),
766
- idempotency: "safe" as const,
767
- verification_status: "verified" as const,
768
- reliability_score: extracted.confidence,
769
- description: validSearchForm
770
- ? `Captured search form artifact for ${intent}`
771
- : `Captured page artifact for ${intent}`,
772
- response_schema,
773
- dom_extraction: {
774
- extraction_method: extracted.extraction_method,
775
- confidence: extracted.confidence,
776
- ...(extracted.selector ? { selector: extracted.selector } : {}),
777
- ...(validSearchForm ? { search_form: validSearchForm } : {}),
778
- },
779
- trigger_url: url,
780
- };
781
- endpoint.semantic = {
782
- ...inferEndpointSemantic(endpoint, {
783
- sampleResponse: extracted.data,
784
- sampleRequest: buildSampleRequestFromUrl(url),
785
- observedAt: new Date().toISOString(),
786
- sampleRequestUrl: url,
787
- }),
788
- ...(authRequired ? { auth_required: true } : {}),
789
- };
790
- if (validSearchForm && endpoint.semantic) {
791
- endpoint.semantic.action_kind = "search";
792
- }
793
-
794
- if (validSearchForm) {
795
- log("execution", `detected structured search form: ${validSearchForm.form_selector} with ${validSearchForm.fields.length} fields`);
796
- }
797
-
798
- return {
799
- endpoint,
800
- result: {
801
- data: extracted.data,
802
- _extraction: {
803
- method: extracted.extraction_method,
804
- confidence: extracted.confidence,
805
- source: "dom-fallback",
806
- ...(validSearchForm ? { search_form_detected: true } : {}),
807
- },
808
- },
809
- ...(validSearchForm ? { search_form: validSearchForm } : {}),
810
- };
811
- }
812
-
813
- export function buildCanonicalDocumentEndpoint(
814
- url: string,
815
- intent: string,
816
- authRequired = false,
817
- ): EndpointDescriptor | undefined {
818
- const replayUrl = deriveStructuredDataReplayUrl(url);
819
- const replayTemplate = deriveStructuredDataReplayTemplate(url);
820
- if (replayUrl === url && replayTemplate === url) return undefined;
821
- const endpoint: EndpointDescriptor = {
822
- endpoint_id: nanoid(),
823
- method: "GET",
824
- url_template: replayTemplate !== url ? replayTemplate : replayUrl,
825
- idempotency: "safe",
826
- verification_status: "verified",
827
- reliability_score: 0.9,
828
- description: `Structured replay for ${intent}`,
829
- trigger_url: url,
830
- };
831
- endpoint.semantic = {
832
- ...inferEndpointSemantic(endpoint, {
833
- sampleRequest: buildSampleRequestFromUrl(url),
834
- observedAt: new Date().toISOString(),
835
- sampleRequestUrl: url,
836
- }),
837
- ...(authRequired ? { auth_required: true } : {}),
838
- };
839
- return endpoint;
840
- }
841
-
842
- export function isCanonicalReplayEndpoint(endpoint: Pick<EndpointDescriptor, "method" | "url_template" | "trigger_url">): boolean {
843
- if (endpoint.method !== "GET" || !endpoint.trigger_url) return false;
844
- try {
845
- const concrete = deriveStructuredDataReplayUrl(endpoint.trigger_url);
846
- const template = deriveStructuredDataReplayTemplate(endpoint.trigger_url);
847
- return endpoint.url_template === concrete || endpoint.url_template === template;
848
- } catch {
849
- return false;
850
- }
851
- }
852
-
853
- async function trySeedStructuredDocumentSkill(
854
- skill: SkillManifest,
855
- url: string,
856
- intent: string,
857
- params: Record<string, unknown>,
858
- targetDomain: string,
859
- authHeaders: Record<string, string> | undefined,
860
- cookies: Array<{ name: string; value: string; domain: string }> | undefined,
861
- usedStoredAuth: boolean,
862
- ): Promise<ExecutionResult | undefined> {
863
- const canonicalDocumentEndpoint = buildCanonicalDocumentEndpoint(url, intent, usedStoredAuth);
864
- if (!canonicalDocumentEndpoint) return undefined;
865
-
866
- const replayUrls = deriveStructuredDataReplayCandidatesFromInputs(url, params);
867
- let headers: Record<string, string> = {
868
- accept: "application/json,text/plain,*/*",
869
- ...(canonicalDocumentEndpoint.headers_template ?? {}),
870
- ...(authHeaders ?? {}),
871
- };
872
- if (cookies && cookies.length > 0) {
873
- headers.cookie = cookies.map((c) => {
874
- const v = c.value.startsWith('"') && c.value.endsWith('"') ? c.value.slice(1, -1) : c.value;
875
- return `${c.name}=${v}`;
876
- }).join("; ");
877
- }
878
-
879
- let data: unknown;
880
- let passed = false;
881
- let successfulReplayUrl = replayUrls[0] ?? url;
882
- for (const replayUrl of replayUrls) {
883
- const res = await fetch(replayUrl, {
884
- method: "GET",
885
- headers: buildStructuredReplayHeaders(url, replayUrl, headers),
886
- redirect: "follow",
887
- });
888
- const text = await res.text();
889
- try { data = JSON.parse(text); } catch { data = text; }
890
-
891
- const assessment = assessIntentResult(data, intent);
892
- if (assessment.verdict === "pass") {
893
- passed = true;
894
- successfulReplayUrl = replayUrl;
895
- break;
896
- }
897
- }
898
- if (!passed) return undefined;
899
-
900
- const semanticSample = compactSchemaSample(data);
901
- canonicalDocumentEndpoint.response_schema = inferSchema([semanticSample]);
902
- canonicalDocumentEndpoint.semantic = {
903
- ...inferEndpointSemantic(canonicalDocumentEndpoint, {
904
- sampleResponse: semanticSample,
905
- sampleRequest: {
906
- ...buildSampleRequestFromUrl(url),
907
- ...params,
908
- },
909
- observedAt: new Date().toISOString(),
910
- sampleRequestUrl: url,
911
- }),
912
- ...(usedStoredAuth ? { auth_required: true } : {}),
913
- };
914
-
915
- const domain = getRegistrableDomain(targetDomain);
916
- const existingSkill = findExistingSkillForDomain(domain, intent);
917
- const localEndpoints = await prepareLearnedEndpoints(
918
- existingSkill
919
- ? mergeEndpoints(existingSkill.endpoints, [canonicalDocumentEndpoint])
920
- : [canonicalDocumentEndpoint],
921
- intent,
922
- domain,
923
- );
924
-
925
- const localDraft: SkillManifest = {
926
- skill_id: existingSkill?.skill_id ?? nanoid(),
927
- version: "1.0.0",
928
- schema_version: "1",
929
- lifecycle: "active" as const,
930
- execution_type: "http" as const,
931
- created_at: existingSkill?.created_at ?? new Date().toISOString(),
932
- updated_at: new Date().toISOString(),
933
- name: domain,
934
- intent_signature: intent,
935
- domain,
936
- description: `API skill for ${domain}`,
937
- owner_type: "agent" as const,
938
- endpoints: localEndpoints,
939
- operation_graph: buildSkillOperationGraph(localEndpoints),
940
- intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
941
- };
942
-
943
- let learned: SkillManifest = localDraft;
944
- const validation = await validateManifest({ ...localDraft, skill_id: "__validate__" });
945
- if (validation.valid) {
946
- try {
947
- const { operation_graph: _graph, ...publishDraft } = localDraft;
948
- const published = await publishSkill(publishDraft);
949
- learned = {
950
- ...published,
951
- endpoints: localEndpoints,
952
- operation_graph: localDraft.operation_graph,
953
- };
954
- } catch {
955
- learned = localDraft;
956
- }
957
- }
958
- try { cachePublishedSkill(learned); } catch { /* best-effort */ }
959
- const seededRequest: RawRequest = {
960
- url: successfulReplayUrl,
961
- method: "GET",
962
- request_headers: headers,
963
- response_status: 200,
964
- response_headers: {
965
- "content-type": typeof data === "string" ? "text/plain" : "application/json",
966
- },
967
- response_body: typeof data === "string" ? data : JSON.stringify(data),
968
- timestamp: new Date().toISOString(),
969
- };
970
- persistWorkflowArtifactForCapture(
971
- learned,
972
- {
973
- requests: [seededRequest],
974
- har_lineage_id: `seeded:${learned.skill_id}:canonical-document`,
975
- final_url: successfulReplayUrl,
976
- cookies: cookies ?? [],
977
- js_bundles: new Map(),
978
- },
979
- authHeaders,
980
- );
981
-
982
- const trace: ExecutionTrace = stampTrace({
983
- trace_id: nanoid(),
984
- skill_id: learned.skill_id,
985
- endpoint_id: "browser-capture",
986
- started_at: new Date().toISOString(),
987
- completed_at: new Date().toISOString(),
988
- success: true,
989
- result: {
990
- learned_skill_id: learned.skill_id,
991
- endpoints_discovered: 1,
992
- seeded_from: "canonical_document",
993
- },
994
- });
995
- return {
996
- trace,
997
- result: trace.result,
998
- learned_skill: learned,
999
- };
1000
- }
1001
-
1002
- async function trySeedPublicDocumentFetchSkill(
1003
- skill: SkillManifest,
1004
- url: string,
1005
- intent: string,
1006
- targetDomain: string,
1007
- authHeaders: Record<string, string> | undefined,
1008
- cookies: Array<{ name: string; value: string; domain: string }> | undefined,
1009
- usedStoredAuth: boolean,
1010
- ): Promise<ExecutionResult | undefined> {
1011
- const headers: Record<string, string> = {
1012
- accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
1013
- "user-agent": DEFAULT_BROWSER_UA,
1014
- "accept-language": "en-US,en;q=0.9",
1015
- ...(authHeaders ?? {}),
1016
- };
1017
- if (cookies && cookies.length > 0) {
1018
- headers.cookie = cookies.map((c) => {
1019
- const v = c.value.startsWith('"') && c.value.endsWith('"') ? c.value.slice(1, -1) : c.value;
1020
- return `${c.name}=${v}`;
1021
- }).join("; ");
1022
- }
1023
-
1024
- const response = await fetch(url, {
1025
- method: "GET",
1026
- headers: buildStructuredReplayHeaders(url, url, headers),
1027
- redirect: "follow",
1028
- });
1029
- const html = await response.text();
1030
- if (!isHtml(html) || isSpaShell(html)) return undefined;
1031
-
1032
- const built = buildPageArtifactCapture(response.url || url, intent, html, usedStoredAuth);
1033
- if (!built.endpoint) return undefined;
1034
-
1035
- const domain = getRegistrableDomain(targetDomain);
1036
- const existingSkill = findExistingSkillForDomain(domain, intent);
1037
- const localEndpoints = await prepareLearnedEndpoints(
1038
- existingSkill
1039
- ? mergeEndpoints(existingSkill.endpoints, [built.endpoint])
1040
- : [built.endpoint],
1041
- intent,
1042
- domain,
1043
- );
1044
-
1045
- const localDraft: SkillManifest = {
1046
- skill_id: existingSkill?.skill_id ?? nanoid(),
1047
- version: "1.0.0",
1048
- schema_version: "1",
1049
- lifecycle: "active" as const,
1050
- execution_type: "http" as const,
1051
- created_at: existingSkill?.created_at ?? new Date().toISOString(),
1052
- updated_at: new Date().toISOString(),
1053
- name: domain,
1054
- intent_signature: intent,
1055
- domain,
1056
- description: `API skill for ${domain}`,
1057
- owner_type: "agent" as const,
1058
- endpoints: localEndpoints,
1059
- operation_graph: buildSkillOperationGraph(localEndpoints),
1060
- intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
1061
- ...(usedStoredAuth ? { auth_profile_ref: `${domain}-session` } : {}),
1062
- };
1063
-
1064
- let learned: SkillManifest = localDraft;
1065
- const validation = await validateManifest({ ...localDraft, skill_id: "__validate__" });
1066
- if (validation.valid) {
1067
- try {
1068
- const { operation_graph: _graph, ...publishDraft } = localDraft;
1069
- const published = await publishSkill(publishDraft);
1070
- learned = {
1071
- ...published,
1072
- endpoints: localEndpoints,
1073
- operation_graph: localDraft.operation_graph,
1074
- };
1075
- } catch {
1076
- learned = localDraft;
1077
- }
1078
- }
1079
- try { cachePublishedSkill(learned); } catch { /* best-effort */ }
1080
- const seededRequest: RawRequest = {
1081
- url,
1082
- method: "GET",
1083
- request_headers: headers,
1084
- response_status: response.status,
1085
- response_headers: Object.fromEntries(response.headers.entries()),
1086
- response_body: html,
1087
- timestamp: new Date().toISOString(),
1088
- };
1089
- persistWorkflowArtifactForCapture(
1090
- learned,
1091
- {
1092
- requests: [seededRequest],
1093
- har_lineage_id: `seeded:${learned.skill_id}:document-fetch`,
1094
- final_url: response.url || url,
1095
- html,
1096
- cookies: cookies ?? [],
1097
- js_bundles: new Map(),
1098
- },
1099
- authHeaders,
1100
- );
1101
-
1102
- const trace: ExecutionTrace = stampTrace({
1103
- trace_id: nanoid(),
1104
- skill_id: learned.skill_id,
1105
- endpoint_id: "browser-capture",
1106
- started_at: new Date().toISOString(),
1107
- completed_at: new Date().toISOString(),
1108
- success: true,
1109
- result: {
1110
- learned_skill_id: learned.skill_id,
1111
- endpoints_discovered: 1,
1112
- seeded_from: "document_fetch",
1113
- },
1114
- });
1115
- return {
1116
- trace,
1117
- result: trace.result,
1118
- learned_skill: learned,
1119
- };
1120
- }
1121
-
1122
- export async function executeSkill(
1123
- skill: SkillManifest,
1124
- params: Record<string, unknown> = {},
1125
- projection?: ProjectionOptions,
1126
- options?: ExecutionOptions
1127
- ): Promise<ExecutionResult> {
1128
- if (skill.execution_type === "browser-capture") {
1129
- return executeBrowserCapture(skill, params, options);
1130
- }
1131
-
1132
- // Allow targeting a specific endpoint by ID — never silently fall back
1133
- if (params.endpoint_id) {
1134
- const target = skill.endpoints.find((e) => e.endpoint_id === params.endpoint_id);
1135
- if (target) {
1136
- const { endpoint_id: _, ...cleanParams } = params;
1137
- return executeEndpoint(skill, target, cleanParams, projection, options);
1138
- }
1139
- // Agent explicitly chose this endpoint — don't silently swap to a different one
1140
- log("exec", `endpoint ${params.endpoint_id} not found in skill ${skill.skill_id} (${skill.endpoints.length} endpoints: ${skill.endpoints.map(e => e.endpoint_id).join(", ")})`);
1141
- const trace: ExecutionTrace = {
1142
- trace_id: nanoid(),
1143
- skill_id: skill.skill_id,
1144
- endpoint_id: String(params.endpoint_id),
1145
- started_at: new Date().toISOString(),
1146
- completed_at: new Date().toISOString(),
1147
- success: false,
1148
- error: `endpoint_not_found: ${params.endpoint_id} not in skill ${skill.skill_id}`,
1149
- };
1150
- return {
1151
- trace,
1152
- result: {
1153
- error: "endpoint_not_found",
1154
- message: `Endpoint ${params.endpoint_id} not found in skill ${skill.skill_id}. Available: ${skill.endpoints.map(e => `${e.endpoint_id} (${e.description?.slice(0, 50)})`).join(", ")}`,
1155
- available_endpoints: skill.endpoints.map(e => ({ endpoint_id: e.endpoint_id, description: e.description })),
1156
- },
1157
- };
1158
- }
1159
-
1160
- // Use the caller's intent for ranking when available, fall back to skill's original intent
1161
- const endpoint = selectBestEndpoint(skill.endpoints, options?.intent ?? skill.intent_signature, skill.domain, options?.contextUrl);
1162
- return executeEndpoint(skill, endpoint, params, projection, options);
1163
- }
1164
-
1165
- async function executeBrowserCapture(
1166
- skill: SkillManifest,
1167
- params: Record<string, unknown>,
1168
- options?: ExecutionOptions,
1169
- ): Promise<ExecutionResult> {
1170
- const fallbackUrl =
1171
- (typeof params.context_url === "string" && params.context_url) ||
1172
- skill.endpoints.find((endpoint) => typeof endpoint.trigger_url === "string" && endpoint.trigger_url)?.trigger_url ||
1173
- skill.endpoints.find((endpoint) => !/\{[^}]+\}/.test(endpoint.url_template))?.url_template ||
1174
- "";
1175
- const url = typeof params.url === "string" ? params.url : String(params.url ?? fallbackUrl);
1176
- const intent = String(params.intent ?? skill.intent_signature);
1177
- if (!url) throw new Error("browser-capture skill requires params.url");
1178
-
1179
- const startedAt = new Date().toISOString();
1180
- const traceId = nanoid();
1181
- const targetDomain = new URL(url).hostname;
1182
-
1183
- // BUG-002/003 fix: auto-load vault cookies for the target domain
1184
- let authHeaders = params.auth_headers as Record<string, string> | undefined;
1185
- let cookies = params.cookies as Array<{ name: string; value: string; domain: string }> | undefined;
1186
- let usedStoredAuth = !!(cookies && cookies.length > 0) || !!(authHeaders && Object.keys(authHeaders).length > 0);
1187
-
1188
- // Auto-resolve cookies from vault, falling back to browser extraction
1189
- if (!cookies || cookies.length === 0) {
1190
- const resolved = await getAuthCookies(targetDomain, { autoExtract: true });
1191
- if (resolved && resolved.length > 0) {
1192
- cookies = resolved;
1193
- usedStoredAuth = true;
1194
- }
1195
- }
1196
- const seeded = await trySeedStructuredDocumentSkill(
1197
- skill,
1198
- url,
1199
- intent,
1200
- params,
1201
- targetDomain,
1202
- authHeaders,
1203
- cookies,
1204
- usedStoredAuth,
1205
- );
1206
- if (seeded) return seeded;
1207
- const documentSeed = await trySeedPublicDocumentFetchSkill(
1208
- skill,
1209
- url,
1210
- intent,
1211
- targetDomain,
1212
- authHeaders,
1213
- cookies,
1214
- usedStoredAuth,
1215
- );
1216
- if (documentSeed) return documentSeed;
1217
- let captured;
1218
- try {
1219
- captured = await captureSession(url, authHeaders, cookies, intent);
1220
- } catch (captureErr: unknown) {
1221
- const err = captureErr as Error & { code?: string; login_url?: string };
1222
- if (err.code === "auth_required") {
1223
- const trace: ExecutionTrace = stampTrace({
1224
- trace_id: traceId,
1225
- skill_id: skill.skill_id,
1226
- endpoint_id: "browser-capture",
1227
- started_at: startedAt,
1228
- completed_at: new Date().toISOString(),
1229
- success: false,
1230
- error: "auth_required",
1231
- });
1232
- return {
1233
- trace,
1234
- result: {
1235
- error: "auth_required",
1236
- provider: "cloudflare",
1237
- login_url: err.login_url ?? url,
1238
- message: `Site is blocked by Cloudflare WAF. Run: unbrowse login --url "${url}" to authenticate interactively.`,
1239
- },
1240
- };
1241
- }
1242
- const message = captureErr instanceof Error ? captureErr.message : String(captureErr);
1243
- const normalizedError = /unable to connect/i.test(message)
1244
- ? "connection_failed"
1245
- : /timed out/i.test(message)
1246
- ? "capture_timeout"
1247
- : "capture_failed";
1248
- const trace: ExecutionTrace = stampTrace({
1249
- trace_id: traceId,
1250
- skill_id: skill.skill_id,
1251
- endpoint_id: "browser-capture",
1252
- started_at: startedAt,
1253
- completed_at: new Date().toISOString(),
1254
- success: false,
1255
- error: normalizedError,
1256
- });
1257
- return {
1258
- trace,
1259
- result: {
1260
- error: normalizedError,
1261
- message,
1262
- },
1263
- };
1264
- }
1265
-
1266
- const finalDomain = (() => {
1267
- try { return new URL(captured.final_url).hostname; } catch { return targetDomain; }
1268
- })();
1269
- const AUTH_PROVIDERS = /accounts\.google\.com|login\.microsoftonline\.com|auth0\.com|cognito-idp\.|appleid\.apple\.com|github\.com|facebook\.com/i;
1270
- const LOGIN_PATHS = /\/(login|signin|sign-in|sso|auth|uas\/login|checkpoint|oauth)/i;
1271
-
1272
- const redirectedToAuth = finalDomain !== targetDomain && AUTH_PROVIDERS.test(finalDomain);
1273
- const redirectedToLogin = captured.final_url !== url && (() => { try { return LOGIN_PATHS.test(new URL(String(captured.final_url)).pathname); } catch { return false; } })();
1274
-
1275
- if (redirectedToAuth || redirectedToLogin) {
1276
- const trace: ExecutionTrace = stampTrace({
1277
- trace_id: traceId,
1278
- skill_id: skill.skill_id,
1279
- endpoint_id: "browser-capture",
1280
- started_at: startedAt,
1281
- completed_at: new Date().toISOString(),
1282
- success: false,
1283
- error: "auth_required",
1284
- });
1285
- return {
1286
- trace,
1287
- result: {
1288
- error: "auth_required",
1289
- provider: getRegistrableDomain(finalDomain),
1290
- login_url: captured.final_url,
1291
- message: `Site requires authentication. Call POST /v1/auth/login with {"url": "${captured.final_url}"} to log in interactively, or pass cookies via params.cookies / headers via params.auth_headers.`,
1292
- },
1293
- };
1294
- }
1295
-
1296
- const endpoints = extractEndpoints(captured.requests, captured.ws_messages, { pageUrl: url, finalUrl: captured.final_url, intent });
1297
-
1298
- // Detect structured search forms from captured HTML and attach to search-like endpoints
1299
- if (captured.html) {
1300
- const detectedForms = detectSearchForms(captured.html);
1301
- if (detectedForms.length > 0) {
1302
- for (const ep of endpoints) {
1303
- if (!ep.search_form && ep.method === "GET") {
1304
- const matchingForm = detectedForms.find((f) => isStructuredSearchForm(f));
1305
- if (matchingForm) {
1306
- ep.search_form = matchingForm;
1307
- break; // attach the best form to the first search-like GET endpoint
1308
- }
1309
- }
1310
- }
1311
- }
1312
- }
1313
-
1314
- // JS bundle scanning: discover API routes not seen in network traffic
1315
- if (captured.js_bundles && captured.js_bundles.size > 0) {
1316
- const pageOrigin = new URL(url).origin;
1317
- const bundleRoutes = scanBundlesForRoutes(captured.js_bundles, pageOrigin);
1318
-
1319
- // Build set of already-discovered URL paths for deduplication
1320
- const networkPaths = new Set<string>();
1321
- for (const ep of endpoints) {
1322
- try {
1323
- const normalized = new URL(ep.url_template).pathname
1324
- .replace(/\{[^}]+\}/g, "*")
1325
- .replace(/\/+$/, "");
1326
- networkPaths.add(normalized);
1327
- } catch { /* skip */ }
1328
- }
1329
-
1330
- let added = 0;
1331
- for (const route of bundleRoutes) {
1332
- const normalized = route.path.replace(/\/+$/, "");
1333
- if (networkPaths.has(normalized)) continue;
1334
-
1335
- // Check if a network endpoint's wildcard pattern matches this route
1336
- let isDup = false;
1337
- for (const np of networkPaths) {
1338
- if (np.includes("*")) {
1339
- const re = new RegExp("^" + np.replace(/\*/g, "[^/]+") + "$");
1340
- if (re.test(normalized)) { isDup = true; break; }
1341
- }
1342
- }
1343
- if (isDup) continue;
1344
-
1345
- // Build query template from bundle-inferred param names
1346
- let epUrl = route.url;
1347
- let epQuery: Record<string, unknown> | undefined;
1348
- let queryParamNames = route.query_params ? [...route.query_params] : [];
1349
- if (queryParamNames.length === 0) {
1350
- try {
1351
- const triggerUrl = new URL(url);
1352
- const triggerParams = [...triggerUrl.searchParams.keys()].filter((k) =>
1353
- /^(q|query|search|term|type|tag|sort|page)$/i.test(k)
1354
- );
1355
- if (triggerParams.length > 0 && /\/(search|lookup|find)\b/i.test(route.path)) {
1356
- queryParamNames = triggerParams;
1357
- }
1358
- } catch { /* skip */ }
1359
- }
1360
- if (queryParamNames.length > 0) {
1361
- epQuery = {};
1362
- for (const p of queryParamNames) epQuery[p] = "";
1363
- const qStr = queryParamNames.map((k) => `${encodeURIComponent(k)}={${k}}`).join("&");
1364
- epUrl = `${route.url}?${qStr}`;
1365
- }
1366
-
1367
- endpoints.push({
1368
- endpoint_id: nanoid(),
1369
- method: "GET",
1370
- url_template: epUrl,
1371
- query: epQuery,
1372
- idempotency: "safe",
1373
- verification_status: "pending",
1374
- reliability_score: 0.2,
1375
- description: `Inferred from JS bundle (${route.match_type}). Not observed in network traffic.`,
1376
- trigger_url: url,
1377
- });
1378
- added++;
1379
- networkPaths.add(normalized);
1380
- }
1381
-
1382
- if (added > 0) {
1383
- log("execution", `added ${added} inferred endpoints from JS bundle scanning`);
1384
- }
1385
- }
1386
-
1387
- const cleanEndpoints = endpoints.filter((ep) => {
1388
- try {
1389
- const host = new URL(ep.url_template).hostname;
1390
- return !AUTH_PROVIDERS.test(host) && !LOGIN_PATHS.test(new URL(ep.url_template).pathname);
1391
- } catch { return true; }
1392
- });
1393
-
1394
- const domain = captured.domain;
1395
-
1396
- // Persist session cookies + auth headers so server-fetch works without browser.
1397
- // extractAuthHeaders collects everything sanitizeHeaders strips from skill manifests
1398
- // (authorization, x-csrf-token, api keys, etc.) — stored encrypted in vault.
1399
- let auth_profile_ref: string | undefined;
1400
- const capturedAuthHeaders = extractAuthHeaders(captured.requests);
1401
-
1402
- if ((captured.cookies && captured.cookies.length > 0) || Object.keys(capturedAuthHeaders).length > 0) {
1403
- auth_profile_ref = `${domain}-session`;
1404
- await storeCredential(auth_profile_ref, JSON.stringify({
1405
- cookies: captured.cookies ?? [],
1406
- headers: Object.keys(capturedAuthHeaders).length > 0 ? capturedAuthHeaders : undefined,
1407
- }));
1408
- }
1409
-
1410
- // BUG-004 fix: set auth_profile_ref when vault has stored auth for this domain
1411
- if (!auth_profile_ref) {
1412
- const vaultKey = `auth:${targetDomain}`;
1413
- const hasStoredAuth = (await getCredential(vaultKey)) != null;
1414
- if (hasStoredAuth) auth_profile_ref = vaultKey;
1415
- }
1416
- const authBackedCapture = usedStoredAuth || !!auth_profile_ref;
1417
- if (authBackedCapture) {
1418
- for (const endpoint of cleanEndpoints) {
1419
- endpoint.semantic = {
1420
- ...(endpoint.semantic ?? {}),
1421
- action_kind: endpoint.semantic?.action_kind ?? "fetch",
1422
- resource_kind: endpoint.semantic?.resource_kind ?? "resource",
1423
- auth_required: true,
1424
- };
1425
- }
1426
- }
1427
-
1428
- const canonicalDocumentEndpoint = buildCanonicalDocumentEndpoint(url, intent, authBackedCapture);
1429
- if (
1430
- canonicalDocumentEndpoint &&
1431
- !cleanEndpoints.some((endpoint) => endpoint.method === canonicalDocumentEndpoint.method && endpoint.url_template === canonicalDocumentEndpoint.url_template)
1432
- ) {
1433
- cleanEndpoints.push(canonicalDocumentEndpoint);
1434
- }
1435
-
1436
- const pageArtifact = captured.html
1437
- ? buildPageArtifactCapture(url, intent, captured.html, authBackedCapture)
1438
- : {};
1439
- const domArtifactEndpoint = pageArtifact.endpoint;
1440
- const domArtifactResult = pageArtifact.result;
1441
- const inferredOnlyCapture = cleanEndpoints.length > 0 && cleanEndpoints.every((endpoint) => isBundleInferredEndpoint(endpoint));
1442
- const hasSupportEvidence = cleanEndpoints.some((endpoint) => isSupportEvidenceEndpoint(endpoint)) || !!domArtifactEndpoint;
1443
-
1444
- if (inferredOnlyCapture && !hasSupportEvidence) {
1445
- const trace: ExecutionTrace = stampTrace({
1446
- trace_id: traceId,
1447
- skill_id: skill.skill_id,
1448
- endpoint_id: "browser-capture",
1449
- started_at: startedAt,
1450
- completed_at: new Date().toISOString(),
1451
- success: false,
1452
- error: "bundle_routes_only",
1453
- });
1454
- return {
1455
- trace,
1456
- result: {
1457
- error: "no_endpoints",
1458
- message: `Only bundle-inferred routes were found at ${url}; no observed API responses or structured DOM data were validated.`,
1459
- },
1460
- };
1461
- }
1462
-
1463
- if (cleanEndpoints.length === 0) {
1464
- // DOM fallback: extract structured data from rendered page, learn a DOM skill
1465
- if (domArtifactEndpoint && domArtifactResult) {
1466
- const existingDomSkill = findExistingSkillForDomain(domain, intent);
1467
- const domEndpoints = await prepareLearnedEndpoints(
1468
- existingDomSkill
1469
- ? mergeEndpoints(existingDomSkill.endpoints, [domArtifactEndpoint])
1470
- : [domArtifactEndpoint],
1471
- intent,
1472
- domain,
1473
- );
1474
- const domDraft: SkillManifest = {
1475
- skill_id: existingDomSkill?.skill_id ?? nanoid(),
1476
- version: "1.0.0",
1477
- schema_version: "1",
1478
- lifecycle: "active" as const,
1479
- execution_type: "http" as const,
1480
- created_at: new Date().toISOString(),
1481
- updated_at: new Date().toISOString(),
1482
- name: domain,
1483
- intent_signature: intent,
1484
- domain,
1485
- description: `API skill for ${domain}`,
1486
- owner_type: "agent" as const,
1487
- endpoints: domEndpoints,
1488
- operation_graph: buildSkillOperationGraph(domEndpoints),
1489
- intents: Array.from(new Set([...(existingDomSkill?.intents ?? []), intent])),
1490
- ...(auth_profile_ref ? { auth_profile_ref } : {}),
1491
- };
1492
-
1493
- // Only publish to marketplace if quality passes
1494
- let learned: SkillManifest | undefined = domDraft;
1495
- try {
1496
- const validation = await validateManifest({ ...domDraft, skill_id: "__validate__" });
1497
- if (validation.valid) {
1498
- learned = await publishSkill(domDraft);
1499
- }
1500
- } catch { /* publish failure is non-fatal */ }
1501
- if (learned) {
1502
- try { cachePublishedSkill(learned, options?.client_scope); } catch { /* local cache best-effort */ }
1503
- persistWorkflowArtifactForCapture(learned, captured, capturedAuthHeaders);
1504
- }
1505
-
1506
- const trace: ExecutionTrace = stampTrace({
1507
- trace_id: traceId,
1508
- skill_id: learned?.skill_id ?? skill.skill_id,
1509
- endpoint_id: domArtifactEndpoint.endpoint_id,
1510
- started_at: startedAt,
1511
- completed_at: new Date().toISOString(),
1512
- success: true,
1513
- result: domArtifactResult.data,
1514
- });
1515
- // Always return data to the caller — quality gate only blocks publishing
1516
- return {
1517
- trace,
1518
- result: domArtifactResult,
1519
- learned_skill: learned,
1520
- };
1521
- }
1522
-
1523
- if (pageArtifact.quality_note) {
1524
- const trace: ExecutionTrace = stampTrace({
1525
- trace_id: traceId,
1526
- skill_id: skill.skill_id,
1527
- endpoint_id: "browser-capture",
1528
- started_at: startedAt,
1529
- completed_at: new Date().toISOString(),
1530
- success: false,
1531
- error: pageArtifact.quality_note,
1532
- });
1533
- return {
1534
- trace,
1535
- result: {
1536
- error: "low_quality_dom_extraction",
1537
- message: `Structured DOM extraction was rejected for ${url}: ${pageArtifact.quality_note}`,
1538
- },
1539
- };
1540
- }
1541
-
1542
- const trace: ExecutionTrace = stampTrace({
1543
- trace_id: traceId,
1544
- skill_id: skill.skill_id,
1545
- endpoint_id: "browser-capture",
1546
- started_at: startedAt,
1547
- completed_at: new Date().toISOString(),
1548
- success: false,
1549
- error: "no_endpoints",
1550
- });
1551
- return {
1552
- trace,
1553
- result: {
1554
- error: "no_endpoints",
1555
- message: `No API endpoints or structured DOM data found at ${url}. The site may require authentication or may not expose machine-readable data from this page.`,
1556
- },
1557
- };
1558
- }
1559
-
1560
- // Reuse existing skill for this domain to preserve skill_id and learned exec_strategy.
1561
- // This prevents duplicate skills accumulating in the marketplace on re-captures.
1562
- const existingSkill = findExistingSkillForDomain(domain, intent);
1563
- if (existingSkill) {
1564
- // Carry forward learned exec_strategy from old endpoints to matching new ones
1565
- for (const ep of cleanEndpoints) {
1566
- if (ep.exec_strategy) continue;
1567
- // Match by URL template (endpoint_id changes on re-capture)
1568
- const oldMatch = existingSkill.endpoints.find(
1569
- (old) => old.url_template === ep.url_template && old.method === ep.method
1570
- );
1571
- if (oldMatch?.exec_strategy) {
1572
- ep.exec_strategy = oldMatch.exec_strategy;
1573
- }
1574
- }
1575
- }
1576
-
1577
- // Keep all captured endpoints locally so the resolver can use WS-backed skills,
1578
- // but only publish HTTP endpoints until backend validation supports WS manifests.
1579
- const learnedEndpoints = domArtifactEndpoint
1580
- ? [...cleanEndpoints, domArtifactEndpoint]
1581
- : cleanEndpoints;
1582
- const localEndpoints = await prepareLearnedEndpoints(
1583
- existingSkill
1584
- ? mergeEndpoints(existingSkill.endpoints, learnedEndpoints)
1585
- : learnedEndpoints,
1586
- intent,
1587
- domain,
1588
- );
1589
- const publishableEndpoints = localEndpoints.filter((ep) => ep.method !== "WS");
1590
-
1591
- const localDraft: SkillManifest = {
1592
- skill_id: existingSkill?.skill_id ?? nanoid(),
1593
- version: "1.0.0",
1594
- schema_version: "1",
1595
- lifecycle: "active" as const,
1596
- execution_type: "http" as const,
1597
- created_at: existingSkill?.created_at ?? new Date().toISOString(),
1598
- updated_at: new Date().toISOString(),
1599
- name: domain,
1600
- intent_signature: intent,
1601
- domain,
1602
- description: `API skill for ${domain}`,
1603
- owner_type: "agent" as const,
1604
- endpoints: localEndpoints,
1605
- intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
1606
- ...(auth_profile_ref ? { auth_profile_ref } : {}),
1607
- };
1608
- // Generate local descriptions immediately so BM25 ranking works on first cache hit
1609
- for (const ep of localDraft.endpoints) {
1610
- if (!ep.description) {
1611
- ep.description = generateLocalDescription(ep);
1612
- }
1613
- }
1614
-
1615
- // PHASE 2: Write local cache IMMEDIATELY (~1ms) — populates cache before auto-exec
1616
- const bgCacheKey = buildResolveCacheKey(domain, intent, url);
1617
- const bgScopedKey = scopedCacheKey(options?.client_scope ?? "global", bgCacheKey);
1618
- writeSkillSnapshot(bgScopedKey, localDraft);
1619
- const bgDomainKey = getDomainReuseKey(url ?? domain);
1620
- if (bgDomainKey) {
1621
- domainSkillCache.set(bgDomainKey, {
1622
- skillId: localDraft.skill_id,
1623
- localSkillPath: snapshotPathForCacheKey(bgScopedKey),
1624
- ts: Date.now(),
1625
- });
1626
- persistDomainCache();
1627
- }
1628
-
1629
- // PHASE 2: Queue heavy work for background (graph + validate + publish)
1630
- queueBackgroundIndex({
1631
- skill: { ...localDraft },
1632
- domain,
1633
- intent,
1634
- contextUrl: url,
1635
- clientScope: options?.client_scope,
1636
- cacheKey: bgCacheKey,
1637
- });
1638
-
1639
- // Return the local draft as learned_skill — no blocking on marketplace publish
1640
- let learned: SkillManifest = localDraft;
1641
- try { cachePublishedSkill(localDraft, options?.client_scope); } catch { /* best-effort */ }
1642
- persistWorkflowArtifactForCapture(localDraft, captured, capturedAuthHeaders);
1643
-
1644
- // Attribute lifecycle phases for this capture-to-publish flow
1645
- const completedAt = new Date().toISOString();
1646
- const captureDurationMs = new Date(completedAt).getTime() - new Date(startedAt).getTime();
1647
- const lifecycleEvents: LifecycleEvent[] = [
1648
- { phase: "capture", skill_id: learned.skill_id, timestamp: startedAt, duration_ms: captureDurationMs, source: "live-capture" },
1649
- { phase: "publish", skill_id: learned.skill_id, timestamp: completedAt, duration_ms: 0, source: publishableEndpoints.length > 0 ? "marketplace" : "cache" },
1650
- ];
1651
- const lifecycleAttribution = attributeLifecycle(lifecycleEvents);
1652
- log("execution", `lifecycle attribution: capture=${lifecycleAttribution.get("capture") ?? 0}ms, publish=${lifecycleAttribution.get("publish") ?? 0}ms`);
1653
-
1654
- const trace: ExecutionTrace = stampTrace({
1655
- trace_id: traceId,
1656
- skill_id: learned.skill_id,
1657
- endpoint_id: "browser-capture",
1658
- started_at: startedAt,
1659
- completed_at: completedAt,
1660
- success: true,
1661
- result: { learned_skill_id: learned.skill_id, endpoints_discovered: cleanEndpoints.length },
1662
- });
1663
-
1664
- // Detect tracking-only capture: all endpoints lack a response_schema, meaning no real
1665
- // JSON data was returned — the site likely gated its API behind authentication.
1666
- // Only flag this when no auth was used (so a retry with auth has a chance of succeeding).
1667
- const hasMeaningfulEndpoint = cleanEndpoints.some((ep) => isSupportEvidenceEndpoint(ep));
1668
- const authRecommended = !usedStoredAuth && !hasMeaningfulEndpoint && !inferredOnlyCapture;
1669
-
1670
- return {
1671
- trace,
1672
- result: {
1673
- ...(trace.result as Record<string, unknown>),
1674
- ...(authRecommended ? {
1675
- auth_recommended: true,
1676
- auth_hint: `No data endpoints found — ${domain} likely requires authentication. ` +
1677
- `Store browser cookies for this domain via the auth endpoints, then retry this capture.`,
1678
- } : {}),
1679
- },
1680
- learned_skill: learned,
1681
- };
1682
- }
1683
-
1684
- async function tryHttpFetch(
1685
- url: string,
1686
- authHeaders: Record<string, string>,
1687
- cookies: Array<{ name: string; value: string; domain: string }>,
1688
- ): Promise<{ html: string; final_url: string } | null> {
1689
- try {
1690
- const headers: Record<string, string> = {
1691
- "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
1692
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
1693
- "Accept-Language": "en-US,en;q=0.9",
1694
- "Cache-Control": "no-cache",
1695
- ...authHeaders,
1696
- };
1697
- if (cookies && cookies.length > 0) {
1698
- headers["Cookie"] = cookies.map(c => `${c.name}=${c.value}`).join("; ");
1699
- }
1700
- const controller = new AbortController();
1701
- const timeout = setTimeout(() => controller.abort(), 10_000);
1702
- const res = await fetch(url, {
1703
- headers,
1704
- signal: controller.signal,
1705
- redirect: "follow",
1706
- });
1707
- clearTimeout(timeout);
1708
- if (res.status !== 200) return null;
1709
- const ct = res.headers.get("content-type") ?? "";
1710
- if (!ct.includes("text/html") && !ct.includes("application/xhtml")) return null;
1711
- const html = await res.text();
1712
- if (!html || html.length < 1024) return null;
1713
- return { html, final_url: res.url || url };
1714
- } catch {
1715
- return null;
1716
- }
1717
- }
1718
-
1719
- /** When extraction returns "multiple" candidates, pick the best one's data to avoid duplicates */
1720
- function flattenExtracted(data: unknown): unknown {
1721
- if (!Array.isArray(data)) return data;
1722
- const first = data[0];
1723
- if (first && typeof first === "object" && "type" in first && "data" in first && "relevance_score" in first) {
1724
- return data.reduce((best: any, cur: any) => (cur.relevance_score ?? 0) > (best.relevance_score ?? 0) ? cur : best).data;
1725
- }
1726
- return data;
1727
- }
1728
-
1729
- async function executeDomExtractionEndpoint(
1730
- endpoint: EndpointDescriptor,
1731
- url: string,
1732
- intent: string,
1733
- authHeaders: Record<string, string>,
1734
- cookies: Array<{ name: string; value: string; domain: string }>,
1735
- ): Promise<{ data: unknown; status: number; trace_id: string }> {
1736
- // SSR fast-path: try plain HTTP fetch before browser
1737
- const ssrResult = await tryHttpFetch(url, authHeaders, cookies);
1738
- if (ssrResult) {
1739
- const ssrExtracted = extractFromDOMWithHint(ssrResult.html, intent, endpoint.dom_extraction);
1740
- if (ssrExtracted.data) {
1741
- const ssrQuality = validateExtractionQuality(ssrExtracted.data, ssrExtracted.confidence, intent);
1742
- if (ssrQuality.valid) {
1743
- const ssrSemantic = assessIntentResult(ssrExtracted.data, intent);
1744
- if (ssrSemantic.verdict !== "fail") {
1745
- console.log(`[ssr-fast] hit — extracted via HTTP fetch`);
1746
- return {
1747
- data: flattenExtracted(ssrExtracted.data),
1748
- status: 200,
1749
- trace_id: nanoid(),
1750
- };
1751
- }
1752
- }
1753
- }
1754
- console.log(`[ssr-fast] miss, falling back to browser`);
1755
- } else {
1756
- console.log(`[ssr-fast] miss, falling back to browser`);
1757
- }
1758
-
1759
- // Browser fallback — captures both intercepted API requests AND page HTML
1760
- const captured = await captureSession(url, authHeaders, cookies, intent);
1761
-
1762
- // Check intercepted requests first — if the site's JS made API calls,
1763
- // those have the actual filtered data (not the initial HTML page load)
1764
- if (captured.requests.length > 0) {
1765
- const { extractEndpoints: extractEps } = await import("../reverse-engineer/index.js");
1766
- const apiEndpoints = extractEps(captured.requests, undefined, { pageUrl: url, finalUrl: captured.final_url });
1767
- const jsonEndpoints = apiEndpoints.filter(ep => ep.response_schema && !ep.dom_extraction);
1768
- if (jsonEndpoints.length > 0) {
1769
- // Found real API responses — return the best one's data
1770
- const best = jsonEndpoints[0];
1771
- const matchingReq = captured.requests.find(r =>
1772
- r.url.includes(best.url_template.split("?")[0].split("{")[0]) &&
1773
- r.response_body && r.response_status >= 200 && r.response_status < 400
1774
- );
1775
- if (matchingReq?.response_body) {
1776
- try {
1777
- const data = JSON.parse(matchingReq.response_body);
1778
- console.log(`[dom-exec] found API response from browser capture: ${matchingReq.url.substring(0, 80)}`);
1779
- return { data, status: matchingReq.response_status, trace_id: nanoid() };
1780
- } catch { /* not JSON, fall through to DOM extraction */ }
1781
- }
1782
- }
1783
- }
1784
-
1785
- // Fall back to DOM extraction from rendered HTML
1786
- const html = captured.html ?? "";
1787
- const extracted = extractFromDOMWithHint(html, intent, endpoint.dom_extraction);
1788
- if (extracted.data) {
1789
- const quality = validateExtractionQuality(extracted.data, extracted.confidence, intent);
1790
- if (!quality.valid) {
1791
- return {
1792
- data: {
1793
- error: "low_quality_dom_extraction",
1794
- message: `Structured DOM extraction was rejected: ${quality.quality_note ?? "low quality extraction"}`,
1795
- },
1796
- status: 422,
1797
- trace_id: nanoid(),
1798
- };
1799
- }
1800
- const semanticAssessment = assessIntentResult(extracted.data, intent);
1801
- if (semanticAssessment.verdict === "fail") {
1802
- return {
1803
- data: {
1804
- error: "low_quality_dom_extraction",
1805
- message: `Structured DOM extraction was rejected: ${semanticAssessment.reason}`,
1806
- },
1807
- status: 422,
1808
- trace_id: nanoid(),
1809
- };
1810
- }
1811
- return {
1812
- data: flattenExtracted(extracted.data),
1813
- status: 200,
1814
- trace_id: nanoid(),
1815
- };
1816
- }
1817
- return {
1818
- data: html,
1819
- status: 200,
1820
- trace_id: nanoid(),
1821
- };
1822
- }
1823
-
1824
-
1825
- export async function executeEndpoint(
1826
- skill: SkillManifest,
1827
- endpoint: EndpointDescriptor,
1828
- params: Record<string, unknown> = {},
1829
- projection?: ProjectionOptions,
1830
- options?: ExecutionOptions
1831
- ): Promise<ExecutionResult> {
1832
- endpoint = annotateEndpointPolicy(endpoint);
1833
- const workflowArtifact = readWorkflowArtifact(skill.skill_id);
1834
- const workflowRecipe = pickWorkflowRecipe(workflowArtifact, endpoint.endpoint_id);
1835
- const reservedMetaParams = new Set(["endpoint_id", "url", "context_url", "intent"]);
1836
- // WebSocket endpoint: connect, collect messages, return
1837
- if (endpoint.method === "WS") {
1838
- const startedAt = new Date().toISOString();
1839
- const traceId = nanoid();
1840
- try {
1841
- const { WebSocket } = await import("ws");
1842
- const messages: string[] = [];
1843
- await new Promise<void>((resolve, reject) => {
1844
- const ws = new WebSocket(endpoint.url_template);
1845
- const timeout = setTimeout(() => { ws.close(); resolve(); }, 7000);
1846
- ws.on("message", (data: Buffer | string) => {
1847
- messages.push(data.toString());
1848
- });
1849
- ws.on("error", (err: Error) => { clearTimeout(timeout); reject(err); });
1850
- ws.on("close", () => { clearTimeout(timeout); resolve(); });
1851
- });
1852
- const parsed = messages.map((m) => { try { return JSON.parse(m); } catch { return m; } });
1853
- const trace: ExecutionTrace = stampTrace({
1854
- trace_id: traceId, skill_id: skill.skill_id, endpoint_id: endpoint.endpoint_id,
1855
- started_at: startedAt, completed_at: new Date().toISOString(), success: true, result: parsed,
1856
- });
1857
- let resultData: unknown = parsed;
1858
- if (projection?.raw) {
1859
- // Explicit raw — skip projection
1860
- } else if (projection) {
1861
- resultData = applyProjection(parsed, projection);
1862
- }
1863
- return {
1864
- trace, result: resultData,
1865
- };
1866
- } catch (err) {
1867
- const trace: ExecutionTrace = stampTrace({
1868
- trace_id: traceId, skill_id: skill.skill_id, endpoint_id: endpoint.endpoint_id,
1869
- started_at: startedAt, completed_at: new Date().toISOString(), success: false,
1870
- error: String(err),
1871
- });
1872
- return { trace, result: { error: String(err) } };
1873
- }
1874
- }
1875
-
1876
- // Payment gate — check if marketplace skill requires payment before executing
1877
- if (!skill.skill_id.startsWith("local:") && skill.execution_type === "http" && skill.owner_type !== "agent") {
1878
- const wallet = getLocalWalletContext();
1879
- const gate = await checkPaymentRequirement(skill.skill_id, endpoint.endpoint_id, {
1880
- wallet_configured: !!wallet.wallet_address,
1881
- });
1882
- if (gate.status === "payment_required" || gate.status === "wallet_not_configured" || gate.status === "insufficient_balance") {
1883
- // If lobster wallet is available, let execution proceed —
1884
- // the client-level apiRequest will handle 402 pay-and-retry automatically.
1885
- let lobsterAvailable = false;
1886
- try {
1887
- const { isLobsterAvailable } = await import("../payments/lobster-pay.js");
1888
- lobsterAvailable = isLobsterAvailable();
1889
- } catch {}
1890
-
1891
- if (lobsterAvailable && gate.status === "payment_required") {
1892
- console.log(`[payment] ${skill.skill_id}: lobster available — proceeding with auto-pay`);
1893
- } else {
1894
- const trace: ExecutionTrace = stampTrace({
1895
- trace_id: nanoid(),
1896
- skill_id: skill.skill_id,
1897
- endpoint_id: endpoint.endpoint_id,
1898
- started_at: new Date().toISOString(),
1899
- completed_at: new Date().toISOString(),
1900
- success: false,
1901
- status_code: 402,
1902
- error: "payment_required",
1903
- });
1904
- return {
1905
- trace,
1906
- result: {
1907
- error: "payment_required",
1908
- price_usd: gate.requirement?.amount,
1909
- payment_status: gate.status,
1910
- message: gate.message,
1911
- wallet_provider: wallet.wallet_provider ?? "lobster.cash",
1912
- wallet_address: wallet.wallet_address,
1913
- indexing_fallback_available: true,
1914
- },
1915
- };
1916
- }
1917
- }
1918
- }
1919
-
1920
- // Mutation safety / policy gate
1921
- if (endpoint.method !== "GET") {
1922
- if (workflowRecipe?.mutation_guard.block_reason) {
1923
- return {
1924
- trace: stampTrace({
1925
- trace_id: nanoid(),
1926
- skill_id: skill.skill_id,
1927
- endpoint_id: endpoint.endpoint_id,
1928
- started_at: new Date().toISOString(),
1929
- completed_at: new Date().toISOString(),
1930
- success: false,
1931
- error: "workflow_blocked",
1932
- }),
1933
- result: {
1934
- error: "workflow_blocked",
1935
- message: workflowRecipe.mutation_guard.block_reason,
1936
- },
1937
- };
1938
- }
1939
- if (options?.dry_run) {
1940
- // Merge path_params defaults for dry_run preview too
1941
- const dryParams = { ...params };
1942
- if (endpoint.path_params) {
1943
- for (const [k, v] of Object.entries(endpoint.path_params)) {
1944
- if (dryParams[k] == null) dryParams[k] = v;
1945
- }
1946
- }
1947
- if (endpoint.body_params) {
1948
- for (const [k, v] of Object.entries(endpoint.body_params)) {
1949
- if (dryParams[k] == null) dryParams[k] = v;
1950
- }
1951
- }
1952
- const url = interpolate(endpoint.url_template, dryParams);
1953
- const body = endpoint.body ? interpolateObj(endpoint.body, dryParams) : undefined;
1954
- return {
1955
- trace: stampTrace({
1956
- trace_id: nanoid(),
1957
- skill_id: skill.skill_id,
1958
- endpoint_id: endpoint.endpoint_id,
1959
- started_at: new Date().toISOString(),
1960
- completed_at: new Date().toISOString(),
1961
- success: false,
1962
- error: "dry_run",
1963
- }),
1964
- result: {
1965
- dry_run: true,
1966
- ...(endpoint.policy ? { site_policy: endpoint.policy } : {}),
1967
- would_execute: { method: endpoint.method, url, body },
1968
- },
1969
- };
1970
- }
1971
- if (endpointRequiresThirdPartyTermsConfirmation(endpoint) && !options?.confirm_third_party_terms) {
1972
- const policy = getEndpointPolicy(endpoint)!;
1973
- return {
1974
- trace: stampTrace({
1975
- trace_id: nanoid(),
1976
- skill_id: skill.skill_id,
1977
- endpoint_id: endpoint.endpoint_id,
1978
- started_at: new Date().toISOString(),
1979
- completed_at: new Date().toISOString(),
1980
- success: false,
1981
- error: "third_party_terms_confirmation_required",
1982
- }),
1983
- result: {
1984
- error: "third_party_terms_confirmation_required",
1985
- message: `This endpoint may violate third-party site terms for ${policy.policy_domain}. Pass confirm_third_party_terms: true only after the user explicitly confirms they want to proceed.`,
1986
- policy_domain: policy.policy_domain,
1987
- policy_reason: policy.reason,
1988
- },
1989
- };
1990
- }
1991
- if (endpoint.idempotency === "unsafe" && !options?.confirm_unsafe) {
1992
- return {
1993
- trace: stampTrace({
1994
- trace_id: nanoid(),
1995
- skill_id: skill.skill_id,
1996
- endpoint_id: endpoint.endpoint_id,
1997
- started_at: new Date().toISOString(),
1998
- completed_at: new Date().toISOString(),
1999
- success: false,
2000
- error: "confirmation_required",
2001
- }),
2002
- result: {
2003
- error: "confirmation_required",
2004
- message: `This endpoint (${endpoint.method} ${endpoint.url_template}) is marked as unsafe. Pass confirm_unsafe: true to proceed.`,
2005
- },
2006
- };
2007
- }
2008
- }
2009
-
2010
- const startedAt = new Date().toISOString();
2011
- const authHeaders: Record<string, string> = {};
2012
- const cookies: Array<{ name: string; value: string; domain: string }> = [];
2013
-
2014
- // Endpoint domain — used for cookie resolution, strategy caching, auth refresh
2015
- const epDomain = (() => { try { return new URL(endpoint.url_template).hostname; } catch { return skill.domain; } })();
2016
- await reloadExecutionAuthState(skill, epDomain, authHeaders, cookies);
2017
-
2018
- // If endpoint has auth_tokens bindings, always resolve fresh tokens.
2019
- // Vault headers may be stale — the DAG knows how to get fresh ones.
2020
- log("exec", `auth_tokens check: ${endpoint.auth_tokens?.length ?? 0} bindings on ${endpoint.endpoint_id}`);
2021
- if (endpoint.auth_tokens?.length) {
2022
- try {
2023
- const { resolveAuthTokens } = await import("./token-resolver.js");
2024
- const resolved = await resolveAuthTokens(endpoint, cookies, authHeaders);
2025
- log("exec", `token resolver returned ${Object.keys(resolved).length} headers: ${Object.keys(resolved).join(",") || "none"}`);
2026
- Object.assign(authHeaders, resolved);
2027
- } catch (e) { log("exec", `token resolver failed: ${e}`); }
2028
- }
2029
-
2030
- log("exec", `endpoint ${endpoint.endpoint_id}: cookies=${cookies.length} authHeaders=${Object.keys(authHeaders).length} hasAuth=${cookies.length > 0 || Object.keys(authHeaders).length > 0}`);
2031
-
2032
- // BUG-006: Merge path_params defaults — user params override captured defaults
2033
- let mergedParams = mergeContextTemplateParams(params, endpoint.url_template, options?.contextUrl);
2034
- if (endpoint.path_params && typeof endpoint.path_params === "object") {
2035
- for (const [k, v] of Object.entries(endpoint.path_params)) {
2036
- if (mergedParams[k] == null) {
2037
- mergedParams[k] = v;
2038
- }
2039
- }
2040
- }
2041
- if (endpoint.body_params && typeof endpoint.body_params === "object") {
2042
- for (const [k, v] of Object.entries(endpoint.body_params)) {
2043
- if (mergedParams[k] == null) {
2044
- mergedParams[k] = v;
2045
- }
2046
- }
2047
- }
2048
-
2049
- // Merge captured query params into URL — user params override endpoint defaults
2050
- let urlTemplate = resolveExecutionUrlTemplate(endpoint, options?.contextUrl);
2051
- if (endpoint.query && typeof endpoint.query === "object" && Object.keys(endpoint.query).length > 0) {
2052
- try {
2053
- const u = new URL(urlTemplate);
2054
- const queryBindings = extractTemplateQueryBindings(endpoint.url_template);
2055
- for (const [k, v] of Object.entries(endpoint.query)) {
2056
- const bindingKey = queryBindings[k];
2057
- // User params override captured query defaults
2058
- if (bindingKey && mergedParams[bindingKey] != null) {
2059
- u.searchParams.set(k, String(mergedParams[bindingKey]));
2060
- } else if (mergedParams[k] != null) {
2061
- u.searchParams.set(k, String(mergedParams[k]));
2062
- } else if (v != null) {
2063
- u.searchParams.set(k, String(v));
2064
- }
2065
- }
2066
- urlTemplate = restoreTemplatePlaceholderEncoding(u.toString());
2067
- } catch {
2068
- // URL parse failure — skip query merge
2069
- }
2070
- }
2071
- let url = interpolate(urlTemplate, mergedParams);
2072
- let body = endpoint.body ? interpolateObj(endpoint.body, mergedParams) : undefined;
2073
-
2074
- if (workflowRecipe) {
2075
- const validationErrors = validateWorkflowReplayParams(workflowRecipe, mergedParams);
2076
- if (validationErrors.length > 0) {
2077
- return {
2078
- trace: stampTrace({
2079
- trace_id: nanoid(),
2080
- skill_id: skill.skill_id,
2081
- endpoint_id: endpoint.endpoint_id,
2082
- started_at: startedAt,
2083
- completed_at: new Date().toISOString(),
2084
- success: false,
2085
- error: "invalid_replay_params",
2086
- }),
2087
- result: {
2088
- error: "invalid_replay_params",
2089
- message: "Replay parameters did not satisfy the published workflow contract.",
2090
- validation_errors: validationErrors,
2091
- replay_contract: workflowRecipe.replay_contract,
2092
- },
2093
- };
2094
- }
2095
- }
2096
-
2097
- const isSafe = endpoint.method === "GET";
2098
- let workflowBindings = workflowRecipe && workflowArtifact
2099
- ? resolveWorkflowBindings(workflowRecipe, {
2100
- cookies,
2101
- authHeaders,
2102
- body,
2103
- artifact: workflowArtifact,
2104
- })
2105
- : null;
2106
- if (workflowBindings?.bodyOverride !== undefined) {
2107
- body = workflowBindings.bodyOverride;
2108
- }
2109
-
2110
- // Append leftover params as query string on GET requests.
2111
- // Params already consumed by path_params, endpoint.query, or {template} vars are skipped.
2112
- if (isSafe && Object.keys(params).length > 0) {
2113
- const consumedKeys = new Set<string>([
2114
- ...reservedMetaParams,
2115
- ...Object.keys(endpoint.path_params ?? {}),
2116
- ...Object.keys(endpoint.query ?? {}),
2117
- ]);
2118
- for (const [rawKey, bindingKey] of Object.entries(extractTemplateQueryBindings(endpoint.url_template))) {
2119
- consumedKeys.add(rawKey);
2120
- consumedKeys.add(bindingKey);
2121
- }
2122
- if (isCanonicalReplayEndpoint(endpoint)) {
2123
- try {
2124
- for (const key of new URL(endpoint.trigger_url!).searchParams.keys()) consumedKeys.add(key);
2125
- } catch {
2126
- /* ignore */
2127
- }
2128
- }
2129
- // Also mark keys that appeared as {var} in the original URL template
2130
- const templateVarRe = /\{(\w+)\}/g;
2131
- let m: RegExpExecArray | null;
2132
- while ((m = templateVarRe.exec(endpoint.url_template)) !== null) {
2133
- consumedKeys.add(m[1]);
2134
- }
2135
- const leftover = Object.entries(params).filter(([k]) => !consumedKeys.has(k) && params[k] != null);
2136
- if (leftover.length > 0) {
2137
- try {
2138
- const u = new URL(url);
2139
- for (const [k, v] of leftover) {
2140
- u.searchParams.set(k, String(v));
2141
- }
2142
- url = u.toString();
2143
- } catch { /* URL parse failure — skip */ }
2144
- }
2145
- }
2146
-
2147
- const hasAuthContext =
2148
- cookies.length > 0 ||
2149
- Object.keys(authHeaders).length > 0 ||
2150
- !!skill.auth_profile_ref ||
2151
- endpoint.semantic?.auth_required === true;
2152
-
2153
- // robots.txt compliance gate — block disallowed paths before any network call.
2154
- if (!options?.skip_robots_check && !hasAuthContext) {
2155
- const allowed = await isAllowedByRobots(url);
2156
- if (!allowed) {
2157
- const traceId = nanoid();
2158
- log("exec", `robots.txt blocked ${url}`);
2159
- return {
2160
- trace: stampTrace({
2161
- trace_id: traceId,
2162
- skill_id: skill.skill_id,
2163
- endpoint_id: endpoint.endpoint_id,
2164
- started_at: startedAt,
2165
- completed_at: new Date().toISOString(),
2166
- success: false,
2167
- error: "robots_txt_disallowed",
2168
- }),
2169
- result: {
2170
- error: "robots_txt_disallowed",
2171
- message: `robots.txt disallows access to ${url} for the Unbrowse user-agent.`,
2172
- },
2173
- };
2174
- }
2175
- }
2176
- const structuredReplayUrl = isSafe ? deriveStructuredDataReplayUrl(url) : url;
2177
- const hasStructuredReplay = structuredReplayUrl !== url;
2178
-
2179
- const serverFetch = async (
2180
- extraHeaders: Record<string, string> = {},
2181
- bodyOverride: unknown = body,
2182
- ): Promise<{ data: unknown; status: number; trace_id: string }> => {
2183
- const endpointHeaders = normalizeReplayHeaders(endpoint.headers_template);
2184
- const sessionHeaders = normalizeReplayHeaders(authHeaders);
2185
-
2186
- // Default accept to JSON, but never overwrite the endpoint's own accept header
2187
- // (e.g. LinkedIn uses "application/vnd.linkedin.normalized+json+2.1")
2188
- const defaultAccept: Record<string, string> = (!endpoint.dom_extraction && !endpointHeaders["accept"] && !sessionHeaders["accept"])
2189
- ? { "accept": "application/json" } : {};
2190
- const headers: Record<string, string> = {
2191
- ...defaultAccept,
2192
- ...endpointHeaders,
2193
- ...sessionHeaders,
2194
- ...normalizeReplayHeaders(extraHeaders),
2195
- };
2196
- // Strip browser-only headers that cause issues server-side
2197
- delete headers["sec-ch-ua"];
2198
- delete headers["sec-ch-ua-mobile"];
2199
- delete headers["sec-ch-ua-platform"];
2200
- delete headers["upgrade-insecure-requests"];
2201
-
2202
- // Inject cookies as Cookie header — same as a browser would send.
2203
- // Strip enclosing quotes from values — Chrome's SQLite stores them quoted
2204
- // but the Cookie header must send them unquoted (RFC 6265 §4.1.1).
2205
- if (cookies.length > 0) {
2206
- const cookieStr = cookies.map((c) => {
2207
- const v = c.value.startsWith('"') && c.value.endsWith('"') ? c.value.slice(1, -1) : c.value;
2208
- return `${c.name}=${v}`;
2209
- }).join("; ");
2210
- headers["cookie"] = cookieStr;
2211
-
2212
- // CSRF token auto-detection (bird pattern): many sites require CSRF tokens
2213
- // as both a cookie AND a header. The cookie value is always fresher than
2214
- // any stored vault header, so it ALWAYS overrides.
2215
- const csrfCookie = cookies.find((c) =>
2216
- /^(ct0|csrf_token|_csrf|csrftoken|XSRF-TOKEN|_xsrf)$/i.test(c.name)
2217
- );
2218
- if (csrfCookie) {
2219
- const v = csrfCookie.value.startsWith('"') && csrfCookie.value.endsWith('"') ? csrfCookie.value.slice(1, -1) : csrfCookie.value;
2220
- headers["x-csrf-token"] = v;
2221
- headers["x-xsrf-token"] = v;
2222
- }
2223
- }
2224
-
2225
- if (endpoint.csrf_plan && cookies.length > 0) {
2226
- const csrfCookie = cookies.find((c) =>
2227
- endpoint.csrf_plan!.extractor_sequence.some((name) => name.toLowerCase() === c.name.toLowerCase()),
2228
- );
2229
- if (csrfCookie) {
2230
- const v = csrfCookie.value.startsWith('"') && csrfCookie.value.endsWith('"') ? csrfCookie.value.slice(1, -1) : csrfCookie.value;
2231
- if (endpoint.csrf_plan.source === "cookie" || endpoint.csrf_plan.source === "header") {
2232
- headers[endpoint.csrf_plan.param_name.toLowerCase()] = v;
2233
- } else if (endpoint.csrf_plan.source === "form" && body && typeof body === "object" && !Array.isArray(body)) {
2234
- (body as Record<string, unknown>)[endpoint.csrf_plan.param_name] ??= v;
2235
- }
2236
- }
2237
- }
2238
-
2239
- const replayUrls = hasStructuredReplay ? deriveStructuredDataReplayCandidates(structuredReplayUrl) : [structuredReplayUrl];
2240
- let last: { data: unknown; status: number } = { data: null, status: 0 };
2241
-
2242
- for (const replayUrl of replayUrls) {
2243
- const replayHeaders = buildStructuredReplayHeaders(url, replayUrl, headers);
2244
- log("exec", `server-fetch: ${endpoint.method} ${replayUrl.substring(0, 80)} csrf=${replayHeaders["x-csrf-token"]?.substring(0, 10)}... cookies=${(replayHeaders["cookie"]?.length ?? 0)}chars`);
2245
- const res = await fetch(replayUrl, {
2246
- method: endpoint.method,
2247
- headers: replayHeaders,
2248
- body: serializeReplayBody(bodyOverride, replayHeaders),
2249
- redirect: "follow",
2250
- });
2251
- let data: unknown;
2252
- const text = await res.text();
2253
- try { data = JSON.parse(text); } catch { data = text; }
2254
- last = { data, status: res.status };
2255
- if (res.ok && !(typeof data === "string" && isHtml(data))) {
2256
- return { data, status: res.status, trace_id: nanoid() };
2257
- }
2258
- }
2259
-
2260
- return { data: last.data, status: last.status, trace_id: nanoid() };
2261
- };
2262
-
2263
- const browserCall = () => executeInBrowser(
2264
- url,
2265
- endpoint.method,
2266
- endpoint.headers_template ?? {},
2267
- body,
2268
- authHeaders,
2269
- cookies
2270
- );
2271
-
2272
- let result: { data: unknown; status: number; trace_id: string };
2273
- const hasAuth = cookies.length > 0 || Object.keys(authHeaders).length > 0;
2274
- const preferredWorkflowStrategy = workflowRecipe?.steps[0]?.strategy
2275
- ? translateWorkflowStrategy(workflowRecipe.steps[0].strategy)
2276
- : undefined;
2277
- let workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy;
2278
-
2279
- if (endpoint.dom_extraction && isSafe) {
2280
- if (hasStructuredReplay) {
2281
- result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
2282
- if (shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
2283
- result = await executeDomExtractionEndpoint(
2284
- endpoint,
2285
- url,
2286
- options?.intent ?? skill.intent_signature,
2287
- authHeaders,
2288
- cookies,
2289
- );
2290
- }
2291
- } else {
2292
- result = await executeDomExtractionEndpoint(
2293
- endpoint,
2294
- url,
2295
- options?.intent ?? skill.intent_signature,
2296
- authHeaders,
2297
- cookies,
2298
- );
2299
- }
2300
- } else if (hasAuth) {
2301
- // Authed execution: learned strategy → skip doomed tiers
2302
- // 1. Server fetch (fast — works for Twitter, simple APIs)
2303
- // 2. Trigger-and-intercept (navigate to page, let site's JS make the call)
2304
- // 3. Browser in-page fetch (last resort)
2305
- let strategy: "server" | "trigger-intercept" | "browser" | undefined;
2306
-
2307
- // Endpoint-level learned strategy (strong signal — proven for this specific endpoint).
2308
- // Domain-level prediction is only used as a tiebreaker, never to skip server-fetch entirely,
2309
- // because different endpoints on the same domain may have different requirements.
2310
- const endpointStrategy = preferredWorkflowStrategy ?? endpoint.exec_strategy;
2311
-
2312
- if (hasStructuredReplay) {
2313
- result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
2314
- if (result.status >= 200 && result.status < 400 && !shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
2315
- strategy = "server";
2316
- workflowChosenStrategy = "server";
2317
- } else if (endpoint.trigger_url && isSafe) {
2318
- // Build trigger URL with agent's params applied — don't replay original captured search
2319
- let triggerUrl = endpoint.trigger_url;
2320
- if (Object.keys(mergedParams).length > 0) {
2321
- try {
2322
- const tu = new URL(endpoint.trigger_url);
2323
- for (const [k, v] of Object.entries(mergedParams)) {
2324
- if (v != null && !reservedMetaParams.has(k)) {
2325
- tu.searchParams.set(k, String(v));
2326
- }
2327
- }
2328
- triggerUrl = tu.toString();
2329
- } catch { /* keep original trigger_url */ }
2330
- }
2331
- result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
2332
- strategy = "trigger-intercept";
2333
- workflowChosenStrategy = "trigger-intercept";
2334
- } else {
2335
- result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
2336
- strategy = "browser";
2337
- workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
2338
- }
2339
- } else if (endpointStrategy === "server") {
2340
- // Proven: server-fetch works for this endpoint
2341
- result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
2342
- if (shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
2343
- result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
2344
- strategy = "browser";
2345
- workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
2346
- } else {
2347
- strategy = "server";
2348
- workflowChosenStrategy = "server";
2349
- }
2350
- } else if (endpointStrategy === "trigger-intercept" && endpoint.trigger_url && isSafe) {
2351
- // Proven: this endpoint needs trigger-intercept
2352
- let triggerUrl = endpoint.trigger_url;
2353
- if (Object.keys(mergedParams).length > 0) {
2354
- try {
2355
- const tu = new URL(endpoint.trigger_url);
2356
- for (const [k, v] of Object.entries(mergedParams)) {
2357
- if (v != null && !reservedMetaParams.has(k)) tu.searchParams.set(k, String(v));
2358
- }
2359
- triggerUrl = tu.toString();
2360
- } catch { /* keep original */ }
2361
- }
2362
- log("exec", `using learned strategy trigger-intercept via ${triggerUrl}`);
2363
- result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
2364
- strategy = "trigger-intercept";
2365
- workflowChosenStrategy = "trigger-intercept";
2366
- } else if (endpointStrategy === "browser") {
2367
- if (shouldIgnoreLearnedBrowserStrategy(endpoint, url)) {
2368
- result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
2369
- if (result.status >= 200 && result.status < 400 && !shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
2370
- strategy = "server";
2371
- workflowChosenStrategy = "server";
2372
- } else {
2373
- log("exec", `server replay rejected stale learned browser strategy for ${endpoint.endpoint_id}; falling back to browser`);
2374
- result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
2375
- strategy = "browser";
2376
- workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
2377
- }
2378
- } else {
2379
- log("exec", `using learned strategy browser`);
2380
- result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
2381
- strategy = "browser";
2382
- workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
2383
- }
2384
- } else {
2385
- // No endpoint-level strategy — always try server-fetch first (fastest path).
2386
- // Fall back to trigger-intercept or browser if server returns 4xx.
2387
- try {
2388
- result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
2389
- if (result.status >= 200 && result.status < 400) {
2390
- // For API endpoints, trust the server response — no fallback to browser
2391
- const isApiEndpoint = /\/(api|graphql)\b/i.test(endpoint.url_template) || /\.(json)(\?|$)/.test(endpoint.url_template);
2392
- if (!isApiEndpoint && shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
2393
- result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
2394
- strategy = "browser";
2395
- workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
2396
- } else {
2397
- strategy = "server";
2398
- workflowChosenStrategy = "server";
2399
- }
2400
- } else {
2401
- log("exec", `server fetch returned ${result.status}, falling back`);
2402
- if (endpoint.trigger_url && isSafe) {
2403
- let triggerUrl = endpoint.trigger_url;
2404
- if (Object.keys(mergedParams).length > 0) {
2405
- try {
2406
- const tu = new URL(endpoint.trigger_url);
2407
- for (const [k, v] of Object.entries(mergedParams)) {
2408
- if (v != null && !reservedMetaParams.has(k)) tu.searchParams.set(k, String(v));
2409
- }
2410
- triggerUrl = tu.toString();
2411
- } catch { /* keep original */ }
2412
- }
2413
- result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
2414
- strategy = "trigger-intercept";
2415
- workflowChosenStrategy = "trigger-intercept";
2416
- } else {
2417
- result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
2418
- strategy = "browser";
2419
- workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
2420
- }
2421
- }
2422
- } catch {
2423
- result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
2424
- strategy = "browser";
2425
- workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
2426
- }
2427
- }
2428
-
2429
- // Persist learned strategy at endpoint level only.
2430
- // Domain-level cache removed: it over-generalizes (e.g., one 400 on LinkedIn
2431
- // locked all endpoints into trigger-intercept even though server-fetch works for most).
2432
- if (strategy && result.status >= 200 && result.status < 400 && strategy !== endpoint.exec_strategy) {
2433
- log("exec", `learned exec_strategy=${strategy} for endpoint ${endpoint.endpoint_id}`);
2434
- endpoint.exec_strategy = strategy;
2435
- try { cachePublishedSkill(skill, options?.client_scope); } catch (e) { log("exec", `failed to cache strategy: ${e}`); }
2436
- }
2437
- } else if (isSafe) {
2438
- // No auth: fetch-first for safe GETs — fall back to browser if SPA shell or error
2439
- try {
2440
- result = await withRetry(() => serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride), (r) => isRetryableStatus(r.status));
2441
- if (typeof result.data === "string" && isHtml(result.data)) {
2442
- if (isSpaShell(result.data)) {
2443
- result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
2444
- }
2445
- }
2446
- } catch {
2447
- result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
2448
- }
2449
- } else {
2450
- // No auth, non-GET: server fetch
2451
- result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
2452
- }
2453
-
2454
- if (workflowRecipe && workflowArtifact && needsWorkflowTokenRefresh(result.status)) {
2455
- const refreshed = await refreshAuthFromBrowser(epDomain);
2456
- if (refreshed) {
2457
- await reloadExecutionAuthState(skill, epDomain, authHeaders, cookies);
2458
- workflowBindings = resolveWorkflowBindings(workflowRecipe, {
2459
- cookies,
2460
- authHeaders,
2461
- body,
2462
- artifact: workflowArtifact,
2463
- });
2464
- result = await serverFetch(workflowBindings.extraHeaders, workflowBindings.bodyOverride);
2465
- if (result.status >= 200 && result.status < 400) {
2466
- workflowChosenStrategy = workflowChosenStrategy ?? "server";
2467
- }
2468
- }
2469
- }
2470
- const { status, trace_id } = result;
2471
- let data = result.data;
2472
-
2473
- const trace: ExecutionTrace = stampTrace({
2474
- trace_id,
2475
- skill_id: skill.skill_id,
2476
- endpoint_id: endpoint.endpoint_id,
2477
- started_at: startedAt,
2478
- completed_at: new Date().toISOString(),
2479
- success: status >= 200 && status < 300,
2480
- status_code: status,
2481
- });
2482
-
2483
- if (!trace.success) {
2484
- trace.error = status === 404
2485
- ? `HTTP 404 — endpoint may be stale. Re-run via POST /v1/intent/resolve to get fresh endpoints.`
2486
- : `HTTP ${status}`;
2487
- } else {
2488
- trace.result = data;
2489
- }
2490
-
2491
- // Stale credential detection: on 401/403, attempt auth recovery before giving up.
2492
- // Chain: authRuntime.refreshSession (lightweight) → refreshAuthFromBrowser (re-extract)
2493
- // → authRuntime.loginIfNeeded (full interactive login)
2494
- if (status === 401 || status === 403) {
2495
- let authRecovered = false;
2496
- try {
2497
- // 1. Lightweight session refresh via authRuntime
2498
- const sessionRefreshed = await authRuntime.refreshSession(epDomain);
2499
- if (sessionRefreshed) {
2500
- log("auth", `session refreshed via authRuntime for ${epDomain} — retry should succeed`);
2501
- authRecovered = true;
2502
- }
2503
-
2504
- // 2. Re-extract cookies from browser SQLite (bird pattern)
2505
- if (!authRecovered) {
2506
- const browserRefreshed = await refreshAuthFromBrowser(epDomain);
2507
- if (browserRefreshed) {
2508
- log("auth", `credentials refreshed from browser for ${epDomain}`);
2509
- authRecovered = true;
2510
- }
2511
- }
2512
-
2513
- // 3. Full login flow via authRuntime as last resort
2514
- if (!authRecovered) {
2515
- const loginResult = await authRuntime.loginIfNeeded(epDomain);
2516
- if (loginResult) {
2517
- log("auth", `loginIfNeeded succeeded for ${epDomain}`);
2518
- authRecovered = true;
2519
- }
2520
- }
2521
-
2522
- if (authRecovered) {
2523
- trace.error = `${trace.error} (credentials refreshed — retry should succeed)`;
2524
- } else {
2525
- // No recovery path worked — delete stale credentials
2526
- if (skill.auth_profile_ref) {
2527
- await deleteCredential(skill.auth_profile_ref);
2528
- }
2529
- trace.error = `${trace.error} (stale credentials — re-authenticate via /v1/auth/login)`;
2530
- }
2531
- } catch {
2532
- if (skill.auth_profile_ref) {
2533
- await deleteCredential(skill.auth_profile_ref);
2534
- }
2535
- trace.error = `${trace.error} (stale credential deleted)`;
2536
- }
2537
- }
2538
-
2539
- // Schema drift detection on re-execution
2540
- if (trace.success && endpoint.response_schema && data != null) {
2541
- const drift = detectSchemaDrift(endpoint.response_schema, data);
2542
- if (drift.drifted) {
2543
- trace.drift = drift;
2544
- }
2545
- }
2546
-
2547
- // HTML→JSON post-processing: if the endpoint returned HTML instead of JSON,
2548
- // pipe it through DOM extraction to produce structured data.
2549
- // Always extract — returning raw HTML to an agent is never useful.
2550
- if (trace.success && typeof data === "string" && isHtml(data)) {
2551
- const intent = options?.intent || skill.intent_signature;
2552
- if (!endpoint.dom_extraction) {
2553
- trace.success = false;
2554
- trace.error = "unexpected_html_response";
2555
- data = {
2556
- error: "unexpected_html_response",
2557
- message: `Endpoint returned HTML instead of API data for intent "${intent}"`,
2558
- };
2559
- trace.result = data;
2560
- } else {
2561
- const extracted = extractFromDOM(data, intent);
2562
- if (extracted.data) {
2563
- const quality = validateExtractionQuality(extracted.data, extracted.confidence, intent);
2564
- const semanticAssessment = quality.valid ? assessIntentResult(extracted.data, intent) : { verdict: "fail" as const, reason: quality.quality_note ?? "low_quality_dom_extraction" };
2565
- if (quality.valid && semanticAssessment.verdict !== "fail") {
2566
- data = {
2567
- data: extracted.data,
2568
- _extraction: {
2569
- method: extracted.extraction_method,
2570
- confidence: extracted.confidence,
2571
- source: "html-postprocess",
2572
- },
2573
- };
2574
- trace.result = data;
2575
- } else {
2576
- trace.success = false;
2577
- trace.error = semanticAssessment.reason ?? quality.quality_note ?? "low_quality_dom_extraction";
2578
- data = {
2579
- error: "low_quality_dom_extraction",
2580
- message: `Structured DOM extraction was rejected: ${semanticAssessment.reason ?? quality.quality_note ?? "low quality extraction"}`,
2581
- };
2582
- trace.result = data;
2583
- }
2584
- }
2585
- }
2586
- }
2587
-
2588
- const effectiveIntent = options?.intent ?? skill.intent_signature;
2589
- if (trace.success && effectiveIntent && data != null) {
2590
- const semanticAssessment = assessIntentResult(data, effectiveIntent);
2591
- if (semanticAssessment.verdict === "fail") {
2592
- trace.success = false;
2593
- trace.error = semanticAssessment.reason;
2594
- data = {
2595
- error: "intent_mismatch",
2596
- message: `Execution result did not satisfy intent "${effectiveIntent}": ${semanticAssessment.reason}`,
2597
- projected: semanticAssessment.projected,
2598
- };
2599
- trace.result = data;
2600
- }
2601
- }
2602
-
2603
- // Backfill response_schema on first successful execution — push to marketplace so all agents benefit
2604
- if (trace.success && !endpoint.response_schema && data != null && typeof data !== "string") {
2605
- try {
2606
- const inferred = inferSchema([data]);
2607
- if (inferred.type !== "object" || inferred.properties) {
2608
- log("exec", `learned response_schema for endpoint ${endpoint.endpoint_id} (${Object.keys(inferred.properties ?? {}).length} top-level props)`);
2609
- endpoint.response_schema = inferred;
2610
- trace.schema_backfilled = true;
2611
- cachePublishedSkill(skill, options?.client_scope);
2612
- updateEndpointSchema(skill.skill_id, endpoint.endpoint_id, inferred).catch(() => {});
2613
- }
2614
- } catch {}
2615
- }
2616
-
2617
- // Record execution for reliability scoring (fire-and-forget — don't block response)
2618
- recordExecution(skill.skill_id, endpoint.endpoint_id, trace, skill).catch(() => {});
2619
-
2620
- // Record transaction if this was a paid execution (fire-and-forget)
2621
- if (trace.success && options?.payment_verified === true && skill.base_price_usd && skill.base_price_usd > 0) {
2622
- const consumerConfig = (() => {
2623
- try { return JSON.parse(require("fs").readFileSync(require("os").homedir() + "/.unbrowse/config.json", "utf-8")); }
2624
- catch { return {}; }
2625
- })();
2626
- if (consumerConfig.agent_id) {
2627
- const wallet = getLocalWalletContext();
2628
- recordTransaction({
2629
- transaction_id: trace.trace_id,
2630
- consumer_id: consumerConfig.agent_id,
2631
- creator_id: skill.indexer_id,
2632
- skill_id: skill.skill_id,
2633
- endpoint_id: endpoint.endpoint_id,
2634
- price_usd: skill.base_price_usd,
2635
- payment_proof: wallet.wallet_address ? `wallet:${wallet.wallet_address}` : undefined,
2636
- }).catch(() => {});
2637
- }
2638
- }
2639
- // Apply field projection
2640
- let resultData = data;
2641
- if (projection?.raw) {
2642
- // Explicit raw request — skip projection
2643
- } else if (projection && trace.success) {
2644
- resultData = applyProjection(data, projection);
2645
- } else if (trace.success) {
2646
- resultData = projectResultForIntent(data, effectiveIntent);
2647
- }
2648
-
2649
- if (workflowArtifact && workflowRecipe && workflowChosenStrategy) {
2650
- const updatedWorkflow = recordWorkflowRecipeOutcome(
2651
- workflowArtifact,
2652
- endpoint.endpoint_id,
2653
- workflowChosenStrategy,
2654
- {
2655
- success: trace.success,
2656
- status,
2657
- error: trace.error,
2658
- selectedBindings: workflowBindings?.selectedBindings,
2659
- },
2660
- );
2661
- const writtenPath = writeWorkflowArtifact(updatedWorkflow);
2662
- if (process.env.UNBROWSE_DEBUG_WORKFLOW === "1") {
2663
- log(
2664
- "workflow",
2665
- `execution persisted skill=${skill.skill_id} endpoint=${endpoint.endpoint_id} strategy=${workflowChosenStrategy} path=${writtenPath ?? "write-failed"}`,
2666
- );
2667
- }
2668
- trace.result = trace.result;
2669
- (trace as ExecutionTrace & { workflow_selected_bindings?: unknown; workflow_strategy?: string }).workflow_selected_bindings = workflowBindings?.selectedBindings;
2670
- (trace as ExecutionTrace & { workflow_selected_bindings?: unknown; workflow_strategy?: string }).workflow_strategy = workflowChosenStrategy;
2671
- }
2672
-
2673
- return {
2674
- trace, result: resultData,
2675
- };
2676
- }
2677
-
2678
- /**
2679
- * Convert query params in a URL to template variables.
2680
- * e.g. /search?q=books&page=1 → /search?q={q}&page={page}
2681
- * Path stays untouched — only query string is templatized.
2682
- */
2683
- export function templatizeQueryParams(url: string): string {
2684
- try {
2685
- const u = sanitizeNavigationQueryParams(new URL(url));
2686
- if (u.search.length <= 1) return url; // no query params
2687
- const params = new URLSearchParams(u.search);
2688
- const templated = new URLSearchParams();
2689
- const bindings = buildQueryBindingMap(params.keys());
2690
- const seen = new Set<string>();
2691
- for (const [key] of params) {
2692
- if (seen.has(key)) continue;
2693
- seen.add(key);
2694
- templated.set(key, `{${bindings[key] ?? key}}`);
2695
- }
2696
- return `${u.origin}${u.pathname}?${templated.toString().replace(/%7B/g, "{").replace(/%7D/g, "}")}`;
2697
- } catch {
2698
- return url;
2699
- }
2700
- }
2701
-
2702
- function interpolate(template: string, params: Record<string, unknown>): string {
2703
- // Split URL into base and query string to properly encode query params
2704
- const qIdx = template.indexOf("?");
2705
- if (qIdx === -1) {
2706
- return template.replace(/\{(\w+)\}/g, (_, k) =>
2707
- params[k] != null ? String(params[k]) : `{${k}}`
2708
- );
2709
- }
2710
-
2711
- const base = template.substring(0, qIdx);
2712
- const query = template.substring(qIdx + 1);
2713
-
2714
- // Interpolate base path without encoding
2715
- const interpolatedBase = base.replace(/\{(\w+)\}/g, (_, k) =>
2716
- params[k] != null ? String(params[k]) : `{${k}}`
2717
- );
2718
-
2719
- // Interpolate query params with URL encoding
2720
- const interpolatedQuery = query.replace(/\{(\w+)\}/g, (_, k) =>
2721
- params[k] != null ? encodeURIComponent(String(params[k])) : `{${k}}`
2722
- );
2723
-
2724
- return `${interpolatedBase}?${interpolatedQuery}`;
2725
- }
2726
-
2727
- function interpolateObj(
2728
- obj: Record<string, unknown>,
2729
- params: Record<string, unknown>
2730
- ): Record<string, unknown> {
2731
- return JSON.parse(
2732
- JSON.stringify(obj).replace(/"(\{(\w+)\})"/g, (_, _full, k) =>
2733
- params[k] != null ? JSON.stringify(params[k]) : `"{${k}}"`
2734
- )
2735
- ) as Record<string, unknown>;
2736
- }
2737
-
2738
- /**
2739
- * BUG-004 fix: select best endpoint by schema richness, not just "first safe GET".
2740
- * Prefers: safe endpoints with object/array response_schema > safe without > unsafe.
2741
- */
2742
- // --- BM25 scoring for intent→endpoint relevance ---
2743
-
2744
- /** Minimal stemmer: strip trailing s/es/ed/ing for matching */
2745
- function stem(word: string): string {
2746
- if (word.endsWith("ies") && word.length > 4) return word.slice(0, -3) + "y";
2747
- // "messages" → "message" (not "messag"), "classes" → "class", "pages" → "page"
2748
- if (word.endsWith("ses") || word.endsWith("ges") || word.endsWith("ces") || word.endsWith("zes")) return word.slice(0, -1);
2749
- if (word.endsWith("es") && word.length > 4) return word.slice(0, -2);
2750
- if (word.endsWith("s") && !word.endsWith("ss") && word.length > 3) return word.slice(0, -1);
2751
- // "bookmarked" → "bookmark", "saved" → "save", "liked" → "like"
2752
- if (word.endsWith("ed") && word.length > 4) return word.slice(0, -2);
2753
- // "loading" → "load", "trending" → "trend" (but not "thing", "ring")
2754
- if (word.endsWith("ing") && word.length > 5) return word.slice(0, -3);
2755
- return word;
2756
- }
2757
-
2758
- const BM25_K1 = 1.2;
2759
- const BM25_B = 0.75;
2760
- const STOPWORDS = new Set([
2761
- "the", "a", "an", "and", "or", "of", "to", "in", "for", "on", "with", "from",
2762
- "get", "all", "this", "that", "is", "are", "was", "be", "it", "at", "by", "not",
2763
- "com", "www", "https", "http", "html", "htm",
2764
- ]);
2765
-
2766
- /** Expand tokens with synonyms/related terms for better recall */
2767
- const SYNONYMS: Record<string, string[]> = {
2768
- price: ["price", "prices", "pricing", "cost", "usd", "quote", "rate", "value", "market"],
2769
- token: ["token", "tokens", "coin", "coins", "crypto", "currency", "asset"],
2770
- search: ["search", "query", "find", "lookup", "filter", "dex"],
2771
- chart: ["chart", "charts", "graph", "history", "ohlcv", "candle", "candles", "kline"],
2772
- trade: ["trade", "trades", "swap", "swaps", "order", "orders", "transaction", "transactions"],
2773
- volume: ["volume", "vol", "liquidity", "tvl"],
2774
- pair: ["pair", "pairs", "pool", "pools"],
2775
- trending: ["trending", "top", "hot", "gainers", "losers", "movers"],
2776
- user: ["user", "users", "account", "accounts", "profile", "profiles", "member"],
2777
- list: ["list", "lists", "all", "index", "browse", "catalog"],
2778
- feed: ["feed", "feeds", "timeline", "stream", "home", "cards", "feedCards"],
2779
- post: ["post", "posts", "article", "articles", "update", "updates", "content", "entry"],
2780
- comment: ["comment", "comments", "reply", "replies", "discussion", "thread"],
2781
- message: ["message", "messages", "messaging", "inbox", "conversation", "conversations", "chat"],
2782
- notification: ["notification", "notifications", "alert", "alerts", "bell"],
2783
- connection: ["connection", "connections", "follower", "followers", "following", "network", "contact", "contacts", "invitation", "invitations"],
2784
- profile: ["profile", "profiles", "identity", "about", "bio", "member"],
2785
- recommend: ["recommend", "recommendation", "recommendations", "suggested", "suggestion", "suggestions", "forYou"],
2786
- bookmark: ["bookmark", "bookmarks", "bookmarked", "saved", "save", "favorite", "favourites"],
2787
- news: ["news", "headline", "headlines", "story", "stories", "storylines"],
2788
- dashboard: ["dashboard", "overview", "summary", "home", "main"],
2789
- };
2790
-
2791
- function normalizeTokenText(text: string): string {
2792
- return text
2793
- .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
2794
- .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
2795
- .replace(/([a-zA-Z])(\d)/g, "$1 $2")
2796
- .replace(/(\d)([a-zA-Z])/g, "$1 $2");
2797
- }
2798
-
2799
- function tokenize(text: string): string[] {
2800
- return normalizeTokenText(text).toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((w) => w.length > 1 && !STOPWORDS.has(w));
2801
- }
2802
-
2803
- /** Expand intent tokens with synonyms + stemmed variants for better matching */
2804
- function expandQuery(tokens: string[]): string[] {
2805
- const expanded = new Set(tokens);
2806
- for (const t of tokens) {
2807
- const stemmed = stem(t);
2808
- expanded.add(stemmed);
2809
- // Look up synonyms by: raw token, stemmed token, or any SYNONYMS key that stems to the same value
2810
- // (e.g. "messages" → stem "messag" matches SYNONYMS["message"] → stem "messag")
2811
- let syns = SYNONYMS[t] ?? SYNONYMS[stemmed];
2812
- if (!syns) {
2813
- for (const key of Object.keys(SYNONYMS)) {
2814
- if (stem(key) === stemmed) { syns = SYNONYMS[key]; break; }
2815
- }
2816
- }
2817
- if (syns) for (const s of syns) { expanded.add(s); expanded.add(stem(s)); }
2818
- }
2819
- return [...expanded];
2820
- }
2821
-
2822
- /** Build a "document" from an endpoint: URL path segments + query params + schema property names */
2823
- function endpointToTokens(ep: EndpointDescriptor): string[] {
2824
- const tokens: string[] = [];
2825
- try {
2826
- const u = new URL(ep.url_template);
2827
- // Path segments — split on delimiters AND camelCase to extract meaningful words
2828
- // e.g. "BookmarkFoldersSlice" → ["Bookmark", "Folders", "Slice"]
2829
- const rawSegments = u.pathname.split(/[/\-_.{}]/).filter((s) => s.length > 1 && !/^v\d+$/.test(s));
2830
- for (const seg of rawSegments) {
2831
- tokens.push(seg);
2832
- // Also split camelCase: "BookmarkFoldersSlice" → ["Bookmark", "Folders", "Slice"]
2833
- const camelParts = seg.split(/(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])/).filter((s) => s.length > 1);
2834
- if (camelParts.length > 1) tokens.push(...camelParts);
2835
- }
2836
- // Hostname subdomains (e.g. "api" from api.dexscreener.com — strong signal)
2837
- const hostParts = u.hostname.split(".");
2838
- tokens.push(...hostParts.filter((s) => s.length > 2 && s !== "www" && s !== "com" && s !== "org" && s !== "net" && s !== "io"));
2839
- // Query param names and values
2840
- for (const [key, val] of u.searchParams.entries()) {
2841
- tokens.push(key);
2842
- if (val.length > 1 && val.length < 50) {
2843
- tokens.push(...val.split(/[/\-_.]/).filter((s) => s.length > 1));
2844
- } else if (val.length >= 50) {
2845
- // Long values (e.g. graphql queryId): split on camelCase and delimiters to extract meaningful words
2846
- const parts = val.split(/[/\-_.()]/).flatMap((s) => s.split(/(?<=[a-z])(?=[A-Z])/)).filter((s) => s.length > 1);
2847
- tokens.push(...parts.slice(0, 10)); // cap to avoid noise from hashes
2848
- }
2849
- }
2850
- } catch { /* skip */ }
2851
- // Schema property names (strong signal — these describe the response data)
2852
- if (ep.response_schema?.properties) {
2853
- tokens.push(...Object.keys(ep.response_schema.properties));
2854
- // Also add nested property names (1 level deep)
2855
- for (const val of Object.values(ep.response_schema.properties) as Array<{ properties?: Record<string, unknown> }>) {
2856
- if (val?.properties) tokens.push(...Object.keys(val.properties));
2857
- }
2858
- }
2859
- // Trigger URL path segments — reveals which page triggered this API call
2860
- // e.g., trigger_url="/i/bookmarks" adds "bookmarks" token for BM25 matching
2861
- if (ep.trigger_url) {
2862
- try {
2863
- const tu = new URL(ep.trigger_url);
2864
- tokens.push(...tu.pathname.split(/[/\-_.{}]/).filter((s) => s.length > 1 && !/^(i|app|en|v\d+)$/.test(s)));
2865
- } catch { /* skip */ }
2866
- }
2867
- // LLM-generated description — strongest semantic signal for intent matching.
2868
- // Tokenized words are added 3x to boost their BM25 weight over noisy URL tokens.
2869
- if (ep.description) {
2870
- const descTokens = ep.description.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((w) => w.length > 1 && !STOPWORDS.has(w));
2871
- for (let i = 0; i < 3; i++) tokens.push(...descTokens);
2872
- }
2873
- return tokens.map((t) => stem(t.toLowerCase()));
2874
- }
2875
-
2876
- function bm25Score(query: string[], doc: string[], avgDl: number, docCount: number, docFreqs: Map<string, number>): number {
2877
- const dl = doc.length;
2878
- const tf = new Map<string, number>();
2879
- for (const t of doc) tf.set(t, (tf.get(t) ?? 0) + 1);
2880
-
2881
- let score = 0;
2882
- for (const term of query) {
2883
- const freq = tf.get(term) ?? 0;
2884
- if (freq === 0) continue;
2885
- // Real IDF: log((N - df + 0.5) / (df + 0.5) + 1) — terms appearing in fewer docs score higher
2886
- const df = docFreqs.get(term) ?? 0;
2887
- const idf = Math.log((docCount - df + 0.5) / (df + 0.5) + 1);
2888
- const num = freq * (BM25_K1 + 1);
2889
- const denom = freq + BM25_K1 * (1 - BM25_B + BM25_B * (dl / avgDl));
2890
- score += idf * (num / denom);
2891
- }
2892
- return score;
2893
- }
2894
-
2895
- export interface RankedEndpoint {
2896
- endpoint: EndpointDescriptor;
2897
- score: number;
2898
- }
2899
-
2900
- function intentResourceKinds(intent?: string): string[] {
2901
- const lower = (intent ?? "").toLowerCase();
2902
- if (/\b(person|people|profile|profiles|user|users|member|members)\b/.test(lower)) return ["person", "people", "profile", "user", "member"];
2903
- if (/\b(company|organization|org)\b/.test(lower)) return ["company", "organization", "org", "business"];
2904
- if (/\b(post|posts|tweet|tweets|status|statuses|feed|timeline|stream|home)\b/.test(lower)) return ["post", "tweet", "status", "message", "feed", "timeline", "update"];
2905
- if (/\b(topic|topics|trend|trending|hashtag|hashtags)\b/.test(lower)) return ["topic", "trend", "hashtag"];
2906
- if (/\b(repo|repository|repositories)\b/.test(lower)) return ["repo", "repository", "project"];
2907
- return [];
2908
- }
2909
-
2910
- function intentActionKinds(intent?: string): string[] {
2911
- const lower = (intent ?? "").toLowerCase();
2912
- if (/\b(feed|timeline|stream|home)\b/.test(lower)) return ["list", "feed", "timeline"];
2913
- if (/\b(search|find|lookup)\b/.test(lower)) return ["search", "list"];
2914
- if (/\b(get|fetch|view)\b/.test(lower)) return ["detail", "get", "fetch"];
2915
- if (/\b(list|browse|discover|trending|top|latest)\b/.test(lower)) return ["list", "search"];
2916
- return [];
2917
- }
2918
-
2919
- function isEntityDetailIntent(intent?: string): boolean {
2920
- const lower = (intent ?? "").toLowerCase();
2921
- return /\b(get|fetch|view)\b/.test(lower) && /\b(company|organization|org|business|person|people|profile|profiles|user|users|member|members)\b/.test(lower);
2922
- }
2923
-
2924
- function semanticIntentAdjustment(endpoint: EndpointDescriptor, intent?: string): number {
2925
- const semantic = resolveEndpointSemantic(endpoint);
2926
- if (!semantic || !intent) return 0;
2927
- const resourceKinds = intentResourceKinds(intent);
2928
- const actionKinds = intentActionKinds(intent);
2929
- let delta = 0;
2930
-
2931
- const resource = (semantic.resource_kind ?? "").toLowerCase();
2932
- const action = (semantic.action_kind ?? "").toLowerCase();
2933
- const negatives = new Set((semantic.negative_tags ?? []).map((tag) => tag.toLowerCase()));
2934
- const haystack = [
2935
- endpoint.url_template,
2936
- endpoint.description ?? "",
2937
- semantic.description_out ?? "",
2938
- semantic.response_summary ?? "",
2939
- ].join(" ").toLowerCase();
2940
- const uiScaffold = /(sharebox|closedsharebox|mailbox|messaging|conversation|notification|notifications|alerts?|presence|badging|launchpad|previewbanner|main_feed|feedtype)/i.test(haystack);
2941
-
2942
- if (resourceKinds.length > 0) {
2943
- if (resourceKinds.some((kind) => resource.includes(kind) || kind.includes(resource))) delta += 80;
2944
- else if (resource) delta -= 90;
2945
- }
2946
-
2947
- if (actionKinds.length > 0) {
2948
- if (actionKinds.some((kind) => action.includes(kind) || kind.includes(action))) delta += 25;
2949
- else if (action) delta -= 25;
2950
- }
2951
-
2952
- if (negatives.has("config") || negatives.has("telemetry") || negatives.has("experiment") || negatives.has("auth")) {
2953
- delta -= 60;
2954
- }
2955
- if (negatives.has("adjacent") || negatives.has("ads")) {
2956
- delta -= 90;
2957
- }
2958
- if (uiScaffold && (resourceKinds.length > 0 || actionKinds.length > 0)) {
2959
- delta -= 220;
2960
- }
2961
-
2962
- return delta;
2963
- }
2964
-
2965
- /**
2966
- * Rank endpoints by relevance to intent using BM25 + structural bonuses.
2967
- * Exported so routes.ts can surface the ranked list to the agent.
2968
- */
2969
- export function rankEndpoints(endpoints: EndpointDescriptor[], intent?: string, skillDomain?: string, contextUrl?: string): RankedEndpoint[] {
2970
- // --- Hard-filter: hosts that NEVER contain useful data ---
2971
- const NOISE_HOSTS = /(id5-sync\.com|btloader\.com|presage\.io|onetrust\.com|adsrvr\.org|googlesyndication\.com|adtrafficquality\.google|amazon-adsystem\.com|crazyegg\.com|challenges\.cloudflare\.com|google-analytics\.com|doubleclick\.net|gstatic\.com|accounts\.google\.com|login\.microsoftonline\.com|auth0\.com|cognito-idp\.|protechts\.net|demdex\.net|datadoghq\.com|fullstory\.com|launchdarkly\.com|intercom\.io|sentry\.io|segment\.io|amplitude\.com|mixpanel\.com|hotjar\.com|clarity\.ms|googletagmanager\.com|walletconnect\.com|cloudflareinsights\.com|fonts\.googleapis\.com|recaptcha|waa-pa\.|signaler-pa\.|ogads-pa\.|reddit\.com\/pixels?|pixel-config\.|dns-finder\.com|cookieconsentpub|firebase\.googleapis\.com|firebaseinstallations\.googleapis\.com|identitytoolkit\.googleapis\.com|securetoken\.googleapis\.com|apis\.google\.com|connect\.facebook\.net|bat\.bing\.com|static\.cloudflareinsights\.com|cdn\.mxpnl\.com|js\.hs-analytics\.net|snap\.licdn\.com|clc\.stackoverflow\.com|px\.ads|t\.co\/i|analytics\.|telemetry\.|stats\.)/i;
2972
-
2973
- // Noise URL path patterns — tracking, telemetry, logging
2974
- const NOISE_PATHS = /\/(track|pixel|telemetry|beacon|csp-report|litms|demdex|analytics|protechts|collect|tr\/|gen_204|generate_204|log$|logging|heartbeat|metrics|consent|sodar|tag$|event$|events$|impression|pageview|click|__)/i;
2975
-
2976
- // Auth/session/config — on-domain but not data
2977
- const AUTH_CONFIG_PATHS = /\/(csrf_meta|logged_in_user|analytics_user_data|onboarding|geolocation|auth|login|logout|register|signup|session|webConfig|config\.json|manifest\.json|robots\.txt|sitemap|favicon|opensearch|service-worker|sw\.js)\b/i;
2978
-
2979
- // Session plumbing — infrastructure endpoints no user would ever want as data.
2980
- // Only true noise: account config, badge counts, feature flags, telemetry, DM settings.
2981
- // NOT filtered: HomeTimeline, Bookmarks, Notifications, UserByScreenName, etc. — real data.
2982
- const SESSION_PLUMBING = /(account\/settings|account\/multi|badge_count|DataSaverMode|permissionsState|email_phone_info|live_pipeline|user_flow|strato\/column|ces\/p2|IntercomStarter|getAltText|fleetline|FeatureHelper|VerifiedAvatar|ScheduledPromotion|DirectCall|DmSettings|PinnedTimeline)/i;
2983
-
2984
- // Static assets
2985
- const STATIC_ASSET_PATTERNS = /\.(woff2?|ttf|eot|css|js|mjs|png|jpg|jpeg|gif|svg|ico|webp|avif|mp4|mp3|wav|riv|lottie|wasm)(\?|%3F|$)/i;
2986
-
2987
- // Animation/UI asset paths
2988
- const UI_ASSET_PATHS = /\/(rive|lottie|animations?|sprites?|assets\/static)\//i;
2989
- const filtered = endpoints.filter((ep) => {
2990
- if (ep.method === "HEAD" || ep.method === "OPTIONS") return false;
2991
- if (ep.verification_status === "disabled") return false;
2992
- if (STATIC_ASSET_PATTERNS.test(ep.url_template)) return false;
2993
- if (UI_ASSET_PATHS.test(ep.url_template)) return false;
2994
- try {
2995
- const host = new URL(ep.url_template).hostname;
2996
- if (NOISE_HOSTS.test(host)) return false;
2997
- } catch { /* skip */ }
2998
- if (NOISE_PATHS.test(ep.url_template)) return false;
2999
- if (AUTH_CONFIG_PATHS.test(ep.url_template)) return false;
3000
- if (SESSION_PLUMBING.test(ep.url_template)) return false;
3001
- return true;
3002
- });
3003
-
3004
- const nonDisabled = endpoints.filter((ep) => ep.verification_status !== "disabled");
3005
- const candidates = filtered.length > 0 ? filtered : nonDisabled;
3006
- if (candidates.length === 0) return [];
3007
- const intentLower = (intent ?? "").toLowerCase();
3008
-
3009
- function endpointHaystack(ep: EndpointDescriptor): string {
3010
- return `${ep.url_template} ${ep.description ?? ""} ${JSON.stringify(ep.response_schema ?? {})} ${JSON.stringify(resolveEndpointSemantic(ep) ?? {})}`.toLowerCase();
3011
- }
3012
-
3013
- function isPlausibleForIntent(ep: EndpointDescriptor): boolean {
3014
- if (!intentLower) return true;
3015
- const haystack = endpointHaystack(ep);
3016
-
3017
- if (/\b(stock|stocks|ticker|tickers|quote|quotes)\b/.test(intentLower)) {
3018
- const hasPositive = /(symbol|ticker|regularmarketprice|currentprice|marketcap|currency|change_percent|changepercent|quote)/i.test(haystack);
3019
- const hasNegative = /(news|article|articles|video|story|stories|thumbnail|image|author)/i.test(haystack);
3020
- return hasPositive && !hasNegative;
3021
- }
3022
-
3023
- if (/\b(product|products|item|items)\b/.test(intentLower)) {
3024
- const hasPositive = /(product|products|itemstacks|items\[\]|price|rating|review|reviewcount|numberofreviews|brand|sku|usitemid|catalogproducttype)/i.test(haystack);
3025
- const hasNegative = /(captcha|robot|human|bootstrapdata|traceparent|nonce|psych|isomorphicsessionid|persistedqueriesconfig|errorloggingconfig|renderviewid|headerobj|initialtempodata|wcpbeacon)/i.test(haystack);
3026
- return hasPositive && !hasNegative;
3027
- }
3028
-
3029
- if (/\b(channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(intentLower)) {
3030
- const hasEntitySignal = /(\/guilds\b|\/channels\b|\bguilds?\b|\bchannels?\b|\bservers?\b|\bworkspaces?\b)/i.test(haystack);
3031
- const hasFieldSignal = /\b(ids?|names?|icon|member_count|topic|description)\b/i.test(haystack);
3032
- const hasPositive = hasEntitySignal && hasFieldSignal;
3033
- const hasNegative = /(affinit|preview|quests|survey|referrals?|promotions?|science|detectable|applications\/public|\/games\b|entitlements?|billing|subscriptions?|collectibles?|gifts?|experiments?|connections?|status|incidents?|scheduled-maintenances?)/i.test(haystack);
3034
- return hasPositive && !hasNegative;
3035
- }
3036
-
3037
- return true;
3038
- }
3039
-
3040
- const plausibilityScopedIntent = /\b(stock|stocks|ticker|tickers|quote|quotes|product|products|item|items|channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(intentLower);
3041
- const plausibleCandidates = candidates.filter((ep) => isPlausibleForIntent(ep));
3042
- if (plausibilityScopedIntent && plausibleCandidates.length === 0) return [];
3043
- const rankedCandidates = plausibleCandidates.length > 0 ? plausibleCandidates : candidates;
3044
- const canonicalReplayTriggers = new Set(
3045
- rankedCandidates
3046
- .filter((ep) => isCanonicalReplayEndpoint(ep))
3047
- .map((ep) => ep.trigger_url)
3048
- .filter((value): value is string => !!value),
3049
- );
3050
- const structuredApiTriggers = new Set(
3051
- rankedCandidates
3052
- .filter((ep) => {
3053
- const url = ep.url_template.toLowerCase();
3054
- const looksLikeApiEndpoint = /\/api\/|graphql|\/rest\/|\/rpc\/|voyager/i.test(url);
3055
- return !!ep.trigger_url && !ep.dom_extraction && (looksLikeApiEndpoint || !!ep.response_schema || ep.method === "WS");
3056
- })
3057
- .map((ep) => ep.trigger_url)
3058
- .filter((value): value is string => !!value),
3059
- );
3060
-
3061
- // Tokenize intent with synonym expansion for better recall
3062
- const rawTokens = intent ? tokenize(intent) : [];
3063
- const queryTokens = rawTokens.length > 0 ? expandQuery(rawTokens) : [];
3064
- const docs = rankedCandidates.map((ep) => endpointToTokens(ep));
3065
- const avgDl = docs.reduce((sum, d) => sum + d.length, 0) / docs.length || 1;
3066
-
3067
- // Build corpus-level document frequencies for real IDF
3068
- const docFreqs = new Map<string, number>();
3069
- for (const doc of docs) {
3070
- const seen = new Set(doc);
3071
- for (const t of seen) docFreqs.set(t, (docFreqs.get(t) ?? 0) + 1);
3072
- }
3073
- const docCount = docs.length;
3074
-
3075
- // Meta/support/promo/config path patterns — not primary data
3076
- const META_PATHS = /\/(annotation|insight|sentiment|vote|portfolio|summary_button|summary_card|tagmetric|quick_add|notifications?|preferences|settings|onboarding|public\/active|remoteConfig|banner\/metadata|embedded-wallets|glow\/get-rendered)/i;
3077
-
3078
- // Data format indicators
3079
- const DATA_INDICATORS = /\.(json|xml|csv)(\?|$)|\/api\//i;
3080
-
3081
- // Currency/time patterns — strong price/financial signal
3082
- const CURRENCY_TIME_PATTERNS = /\/(usd|eur|gbp|btc|eth|sol|cny|jpy|krw|24_hours|7_days|30_days|1_year|max|hourly|daily|weekly|price|prices|market|markets|ticker|tickers|quote|quotes|ohlcv?|candles?|klines?)\b/i;
3083
-
3084
- // API subdomain pattern — "api.example.com" or "io.example.com" strongly suggests data endpoint
3085
- const API_SUBDOMAIN = /^(api|io|data|feed|stream|ws)\./i;
3086
- const LIST_INTENT = /\b(search|list|find|trending|top|latest|discover|browse)\b/i;
3087
- const STATUS_INTENT = /\b(status|incident|outage|maintenance|uptime|degraded)\b/i;
3088
- const COMMS_INTENT = /\b(guilds?|channels?|messages?|dms?|servers?|threads?|chat)\b/i;
3089
- const COMMS_PATH = /\/(guilds?|channels?|messages?|threads?|conversations?|affinities)\b/i;
3090
- const DISCORD_META_PATHS = /\/(referrals?|promotions?|science|entitlements?|billing|subscriptions?|collectibles?|gifts?|experiments?)\b/i;
3091
- const SESSION_BOUND_QUERY = /[?&](?:[^=]*?(crumb|csrf|xsrf|token|session|auth|signature|nonce))=\{/i;
3092
- const COMPANY_INTENT = /\b(company|companies|organization|organisations|business|org)\b/i;
3093
- const PROFILE_INTENT = /\b(person|people|profile|profiles|user|users|member|members)\b/i;
3094
- const PRODUCT_DETAIL_INTENT = /\b(product|products|item|items|listing|listings)\b/i.test(intent ?? "");
3095
- const ENTITY_DETAIL_INTENT = isEntityDetailIntent(intent);
3096
-
3097
- const scored = rankedCandidates.map((ep, i) => {
3098
- let score = 0;
3099
- let pathname = "";
3100
- let hostname = "";
3101
- let contextPath = "";
3102
- let contextLeaf = "";
3103
- let contextQueryKeys = new Set<string>();
3104
- const semantic = resolveEndpointSemantic(ep);
3105
- const descriptionMeta = getEndpointDescriptionMetadata({
3106
- description: ep.description,
3107
- semantic,
3108
- });
3109
- try {
3110
- const u = new URL(ep.url_template);
3111
- pathname = u.pathname;
3112
- hostname = u.hostname;
3113
- } catch { /* skip */ }
3114
- try {
3115
- if (contextUrl) {
3116
- const cu = new URL(contextUrl);
3117
- contextPath = cu.pathname;
3118
- const contextSegs = cu.pathname.split("/").filter(Boolean);
3119
- contextLeaf = contextSegs.length > 0 ? decodeURIComponent(contextSegs[contextSegs.length - 1] ?? "").toLowerCase() : "";
3120
- contextQueryKeys = new Set([...cu.searchParams.keys()]);
3121
- }
3122
- } catch { /* skip */ }
3123
-
3124
- // === BM25 relevance to intent (primary signal, weighted heavily) ===
3125
- if (queryTokens.length > 0) {
3126
- score += bm25Score(queryTokens, docs[i], avgDl, docCount, docFreqs) * 20;
3127
- }
3128
-
3129
- // === Description match bonus — separate from BM25 to avoid IDF dilution ===
3130
- // When an endpoint has a description, compute direct token overlap with RAW intent
3131
- // (not synonym-expanded, to avoid dilution). Each matching core intent token gives a
3132
- // massive bonus that overrides structural noise from schema richness.
3133
- if (descriptionMeta.source === "agent" && descriptionMeta.display && rawTokens.length > 0) {
3134
- const descTokens = new Set(
3135
- descriptionMeta.display.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/)
3136
- .filter((w) => w.length > 1 && !STOPWORDS.has(w))
3137
- .map((w) => stem(w))
3138
- );
3139
- // Use raw intent tokens (not expanded) — "feed" and "post" are the core signal
3140
- const rawStems = new Set(rawTokens.map((t) => stem(t)));
3141
- let matches = 0;
3142
- for (const t of rawStems) {
3143
- if (descTokens.has(t)) matches++;
3144
- }
3145
- // Each matching core token = +100 points. "feed" matching gives +100,
3146
- // "feed" + "post" matching gives +200, etc.
3147
- score += matches * 100;
3148
- }
3149
-
3150
- // === Structural bonuses ===
3151
- if (ep.dom_extraction) score += 25;
3152
- if (descriptionMeta.needs_review && ep.dom_extraction) score -= 140;
3153
- if (isCanonicalReplayEndpoint(ep)) score += 160;
3154
- if (ep.idempotency === "safe" || ep.method === "GET") score += 5;
3155
- if (isBundleInferredEndpoint(ep) && !ep.response_schema) score -= 180;
3156
- score += semanticIntentAdjustment(ep, intent);
3157
-
3158
- // Rich schema = likely structured data endpoint
3159
- if (ep.response_schema) {
3160
- score += 5;
3161
- if (ep.response_schema.type === "array") score += 10;
3162
- else if (ep.response_schema.type === "object" && ep.response_schema.properties) {
3163
- const propCount = Object.keys(ep.response_schema.properties).length;
3164
- score += Math.min(propCount * 2, 20);
3165
- }
3166
- }
3167
- score += ep.reliability_score * 5;
3168
- if (ep.method === "WS" && ep.response_schema) score += 3;
3169
-
3170
- // === Domain affinity ===
3171
- if (skillDomain) {
3172
- try {
3173
- if (hostname === skillDomain || hostname.endsWith(`.${skillDomain}`)) {
3174
- score += 15;
3175
- // Extra bonus for API subdomains on the skill domain
3176
- if (API_SUBDOMAIN.test(hostname)) score += 15;
3177
- } else {
3178
- // Off-domain = almost never right
3179
- score -= 30;
3180
- }
3181
- } catch { /* skip */ }
3182
- }
3183
-
3184
- // API subdomain bonus even without skill domain context
3185
- if (API_SUBDOMAIN.test(hostname)) score += 10;
3186
-
3187
- // Strongly penalize dedicated status/statuspage endpoints unless the user explicitly
3188
- // asked for status/incidents/maintenance. These often hijack root-domain skills.
3189
- if (!STATUS_INTENT.test(intent ?? "")) {
3190
- if (/^(status|statuspage)\./i.test(hostname) || /\/(scheduled-maintenances|incidents|components|status|uptime|summary)\b/i.test(pathname)) {
3191
- score -= 80;
3192
- }
3193
- }
3194
-
3195
- // === Data-relevance signals ===
3196
- if (DATA_INDICATORS.test(ep.url_template)) score += 5;
3197
- if (CURRENCY_TIME_PATTERNS.test(pathname)) score += 15;
3198
- if (intent && COMMS_INTENT.test(intent) && COMMS_PATH.test(pathname)) score += 45;
3199
- if (intent && COMMS_INTENT.test(intent) && DISCORD_META_PATHS.test(pathname)) score -= 220;
3200
- if (/\b(stock|stocks|ticker|tickers|quote|quotes)\b/i.test(intent ?? "")) {
3201
- const quoteHaystack = `${ep.url_template} ${ep.description ?? ""} ${JSON.stringify(ep.response_schema ?? {})} ${JSON.stringify(semantic)}`.toLowerCase();
3202
- if (/\/chart\b/i.test(pathname) && /(regularmarketprice|currentprice|previousclose|chartpreviousclose|price)/i.test(quoteHaystack)) {
3203
- score += 120;
3204
- }
3205
- if (SESSION_BOUND_QUERY.test(ep.url_template)) {
3206
- score -= 170;
3207
- }
3208
- }
3209
-
3210
- // Deep paths with meaningful segments = likely data endpoints
3211
- const pathDepth = pathname.split("/").filter((s) => s.length > 0).length;
3212
- if (pathDepth >= 3) score += 5;
3213
-
3214
- // === Context URL match — endpoint was captured from the page the user is asking about ===
3215
- if (contextUrl && ep.trigger_url) {
3216
- try {
3217
- const contextPath = new URL(contextUrl).pathname;
3218
- const triggerPath = new URL(ep.trigger_url).pathname;
3219
- if (triggerPath === contextPath) score += 20;
3220
- } catch { /* skip */ }
3221
- }
3222
-
3223
- // Direct endpoint/context path match. Stronger than trigger_url because marketplace
3224
- // skills may have stale or missing trigger_url, but url_template still reflects intent.
3225
- if (contextPath) {
3226
- if (pathname === contextPath) score += 45;
3227
- else if (pathname.startsWith(contextPath) || contextPath.startsWith(pathname)) score += 20;
3228
-
3229
- const contextSegs = contextPath.split("/").filter(Boolean);
3230
- const endpointSegs = pathname.split("/").filter(Boolean);
3231
- if (contextSegs.length > 0 && endpointSegs.length > 0 && contextSegs[0] === endpointSegs[0]) {
3232
- score += 12;
3233
- }
3234
-
3235
- if (contextQueryKeys.size > 0) {
3236
- let matchedKeys = 0;
3237
- for (const key of contextQueryKeys) {
3238
- if (ep.url_template.includes(`${key}=`) || ep.url_template.includes(`{${key}}`)) matchedKeys++;
3239
- }
3240
- score += matchedKeys * 12;
3241
- if (matchedKeys === 0 && /\/search\b/.test(contextPath)) score -= 20;
3242
- }
3243
- }
3244
-
3245
- const looksLikeApiEndpoint = /\/api\/|graphql|\/rest\/|\/rpc\/|voyager/i.test(ep.url_template);
3246
- const looksLikeDocumentRoute = !!contextPath && pathname === contextPath && !looksLikeApiEndpoint;
3247
- const isCapturedPageArtifact = /captured (?:search form |page )?artifact/i.test(ep.description ?? "");
3248
- const hasCanonicalReplaySibling = !!ep.trigger_url && canonicalReplayTriggers.has(ep.trigger_url);
3249
- const hasStructuredApiSibling = !!ep.trigger_url && structuredApiTriggers.has(ep.trigger_url);
3250
- const triggerPath = (() => {
3251
- try {
3252
- return ep.trigger_url ? new URL(ep.trigger_url).pathname : "";
3253
- } catch {
3254
- return "";
3255
- }
3256
- })();
3257
- const exactContextDocument =
3258
- PRODUCT_DETAIL_INTENT &&
3259
- !!contextPath &&
3260
- (pathname === contextPath || triggerPath === contextPath);
3261
- const mismatchedContextDocument =
3262
- !!contextPath &&
3263
- (isCapturedPageArtifact || looksLikeDocumentRoute) &&
3264
- pathname !== contextPath &&
3265
- triggerPath !== contextPath;
3266
-
3267
- if (ENTITY_DETAIL_INTENT && looksLikeDocumentRoute && !exactContextDocument) {
3268
- score -= 55;
3269
- }
3270
- if (ENTITY_DETAIL_INTENT && isCapturedPageArtifact && !exactContextDocument) {
3271
- score -= 200;
3272
- }
3273
- if (ENTITY_DETAIL_INTENT && mismatchedContextDocument) {
3274
- score -= 420;
3275
- }
3276
- if (intent && COMMS_INTENT.test(intent) && looksLikeDocumentRoute) {
3277
- score -= 180;
3278
- }
3279
- if (intent && COMMS_INTENT.test(intent) && isCapturedPageArtifact) {
3280
- score -= 1000;
3281
- }
3282
-
3283
- if (intent && COMPANY_INTENT.test(intent)) {
3284
- const companyHaystack = `${ep.url_template} ${ep.description ?? ""} ${JSON.stringify(ep.response_schema ?? {})}`.toLowerCase();
3285
- if (/(organization|company|companies|org)/i.test(companyHaystack) && looksLikeApiEndpoint) score += 110;
3286
- if (/(mailbox|messaging|messagecenter|notifications?|inbox|launchpad|identity|sharebox)/i.test(companyHaystack)) score -= 140;
3287
- if (/(organizationdashcompanies|universalname|companyprofile|organizationprofile|aboutthisprofile|organizationresult|companyresult)/i.test(companyHaystack)) score += 95;
3288
- if (looksLikeDocumentRoute) score -= 35;
3289
- }
3290
-
3291
- if (intent && PROFILE_INTENT.test(intent)) {
3292
- const profileHaystack = `${ep.url_template} ${ep.description ?? ""} ${JSON.stringify(ep.response_schema ?? {})}`.toLowerCase();
3293
- if (/(sidebar|recommend|recommendations|suggested|spotlight|timeline|tweets|following|followers)/i.test(profileHaystack)) score -= 90;
3294
- if (/(sharebox|closedsharebox|mailbox|messaging|conversation|alerts?|notification|presence|badging|feedtype|main_feed)/i.test(profileHaystack)) score -= 180;
3295
- if (/(userbyscreenname|profile|profiles|memberprofile|identityprofile|person)/i.test(profileHaystack) && looksLikeApiEndpoint) score += 80;
3296
- if (/(search\/results\/people|searchcluster|searchresult|public_identifier|headline|mini_profile|memberresult)/i.test(profileHaystack)) score += 95;
3297
- }
3298
-
3299
- if (intent && /\b(feed|timeline|stream|home)\b/i.test(intent) && /\b(post|posts|status|statuses|update|updates)\b/i.test(intent)) {
3300
- const feedHaystack = `${ep.url_template} ${ep.description ?? ""} ${JSON.stringify(semantic)}`.toLowerCase();
3301
- if (/(voyagerfeeddashmainfeed|voyagerfeeddashfeedupdates|mainfeed|feedupdates|main_feed)/i.test(feedHaystack)) score += 170;
3302
- if (/(identitydashprofiles|voyageridentity|storylines|newsdashstorylines|globalnav|launchpad|mailbox|notification|presence)/i.test(feedHaystack)) score -= 150;
3303
- }
3304
- if (intent && /\b(search|list|find|feed|timeline|stream|home|latest|trending|discover|browse)\b/i.test(intent) && /\b(post|posts|tweet|tweets|status|statuses|update|updates)\b/i.test(intent)) {
3305
- const contentHaystack = `${ep.url_template} ${ep.description ?? ""} ${JSON.stringify(semantic)}`.toLowerCase();
3306
- if (looksLikeApiEndpoint && /(search|timeline|feed|stream|result|results|entries|posts|tweets|statuses|updates)/i.test(contentHaystack)) score += 180;
3307
- if (/(sidebar|recommend|recommendations|usersbyrestids|user details|profile|profiles|followers|following|people|spotlight)/i.test(contentHaystack)) score -= 140;
3308
- if (isCapturedPageArtifact && hasStructuredApiSibling) score -= 320;
3309
- else if (looksLikeDocumentRoute && hasStructuredApiSibling) score -= 200;
3310
- }
3311
-
3312
- const requestHint = JSON.stringify(semantic.example_request ?? {}).toLowerCase();
3313
- const endpointHint = `${ep.url_template} ${ep.description ?? ""}`.toLowerCase();
3314
- const hasConcreteEntityRoute =
3315
- ENTITY_DETAIL_INTENT &&
3316
- !!contextLeaf &&
3317
- !/^(search|explore|trending|tabs|home|for-you|foryou|latest|live|people|posts|videos)$/.test(contextLeaf);
3318
- if (hasConcreteEntityRoute) {
3319
- if (requestHint.includes(contextLeaf)) score += 120;
3320
- else if (endpointHint.includes(contextLeaf)) score += 40;
3321
- if (/(screen_name|screenname|username|userby|slug|vanity|universalname|public_identifier|identifier)/i.test(endpointHint + " " + requestHint)) score += 55;
3322
- if (/(restids|usersbyrestids|recommendations|timeline|tweets|following|followers)/i.test(endpointHint + " " + requestHint)) score -= 70;
3323
- }
3324
-
3325
- // Penalize fixed entity/detail pages when the user asked for a list/search flow.
3326
- const isStaticEntityPath = /^\/[^/{?]+\/[^/{?]+$/.test(pathname);
3327
- if (intent && LIST_INTENT.test(intent) && isStaticEntityPath) {
3328
- score -= 35;
3329
- }
3330
-
3331
- // Reward endpoints whose path explicitly names the list/search surface the user is on.
3332
- if (intent && LIST_INTENT.test(intent) && /\/(search|trending|discover|explore)\b/i.test(pathname)) {
3333
- score += 30;
3334
- }
3335
-
3336
- // === Penalties ===
3337
- if (META_PATHS.test(pathname)) score -= 15;
3338
- if (DISCORD_META_PATHS.test(pathname)) score -= 35;
3339
- if (SESSION_PLUMBING.test(pathname) || SESSION_PLUMBING.test(ep.url_template)) score -= 30;
3340
- if (isBundleInferredEndpoint(ep) && !ep.response_schema) score -= 40;
3341
-
3342
- // Penalize root/short paths (homepage, config, init)
3343
- if (pathname.length <= 2) score -= 10;
3344
-
3345
- // Penalize POST endpoints that aren't explicitly API calls (likely tracking/events)
3346
- if (ep.method === "POST" && !DATA_INDICATORS.test(ep.url_template) && !ep.response_schema) {
3347
- score -= 15;
3348
- }
3349
-
3350
- if (intent && COMMS_INTENT.test(intent) && isCapturedPageArtifact) {
3351
- score = Math.min(score, -400);
3352
- }
3353
- if (hasCanonicalReplaySibling && ep.dom_extraction && !isCanonicalReplayEndpoint(ep)) {
3354
- score -= 260;
3355
- }
3356
- if (descriptionMeta.needs_review && isCapturedPageArtifact) {
3357
- score -= 120;
3358
- }
3359
-
3360
- return { endpoint: ep, score };
3361
- });
3362
-
3363
- scored.sort((a, b) => b.score - a.score);
3364
- if (plausibilityScopedIntent && scored[0] && scored[0].score < 0) return [];
3365
- return scored;
3366
- }
3367
-
3368
- function selectBestEndpoint(endpoints: EndpointDescriptor[], intent?: string, skillDomain?: string, contextUrl?: string): EndpointDescriptor {
3369
- if (endpoints.length === 0) throw new Error("No endpoints available");
3370
- if (endpoints.length === 1) return endpoints[0];
3371
-
3372
- const ranked = rankEndpoints(endpoints, intent, skillDomain, contextUrl);
3373
- if (ranked.length === 0) throw new Error("All endpoints are disabled");
3374
- return ranked[0].endpoint;
3375
- }
3376
-
3377
- /** Detect if a string response is HTML rather than JSON/plaintext */
3378
- function isHtml(text: string): boolean {
3379
- const trimmed = text.trimStart().slice(0, 200).toLowerCase();
3380
- return trimmed.startsWith("<!doctype html") ||
3381
- trimmed.startsWith("<html") ||
3382
- (trimmed.includes("<head") && trimmed.includes("<body"));
3383
- }
3384
-
3385
- /**
3386
- * Detect if HTML is an empty SPA shell that needs JS to render.
3387
- * SPA shells have a near-empty body (just a <div id="root"> or similar)
3388
- * with all content loaded by JavaScript bundles.
3389
- * SSR pages have substantial text content in the body already.
3390
- */
3391
- function isSpaShell(html: string): boolean {
3392
- // Quick heuristic: extract body content and check if it has meaningful text
3393
- const bodyMatch = html.match(/<body[^>]*>([\s\S]*)<\/body>/i);
3394
- if (!bodyMatch) return true; // no body at all — treat as SPA shell
3395
- const body = bodyMatch[1];
3396
-
3397
- // Strip script/style tags and HTML tags to get raw text
3398
- const text = body
3399
- .replace(/<script[\s\S]*?<\/script>/gi, "")
3400
- .replace(/<style[\s\S]*?<\/style>/gi, "")
3401
- .replace(/<[^>]+>/g, " ")
3402
- .replace(/\s+/g, " ")
3403
- .trim();
3404
-
3405
- // SPA shells have very little text — just "Loading..." or empty divs
3406
- return text.length < 200;
3407
- }