unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +79 -33
- package/dist/index.js +2 -6
- package/dist/mcp.js +73 -22
- package/dist/server.js +25994 -0
- package/package.json +1 -2
- package/vendor/kuri/manifest.json +1 -1
- package/runtime-src/agent-outcome.ts +0 -166
- package/runtime-src/analytics-session.ts +0 -55
- package/runtime-src/api/browse-index.ts +0 -343
- package/runtime-src/api/browse-session.ts +0 -572
- package/runtime-src/api/browse-submit-prereqs.ts +0 -48
- package/runtime-src/api/browse-submit.ts +0 -1184
- package/runtime-src/api/routes.ts +0 -1833
- package/runtime-src/auth/browser-cookies.ts +0 -423
- package/runtime-src/auth/index.ts +0 -535
- package/runtime-src/auth/runtime.ts +0 -116
- package/runtime-src/browser/index.ts +0 -659
- package/runtime-src/browser/types.ts +0 -41
- package/runtime-src/build-info.generated.ts +0 -6
- package/runtime-src/capture/index.ts +0 -1929
- package/runtime-src/capture/prefetch.ts +0 -95
- package/runtime-src/capture/rsc.ts +0 -45
- package/runtime-src/cli/shortcuts.ts +0 -273
- package/runtime-src/cli.ts +0 -1734
- package/runtime-src/client/graph-client.ts +0 -100
- package/runtime-src/client/index.ts +0 -1425
- package/runtime-src/debug-trace.ts +0 -18
- package/runtime-src/domain.ts +0 -38
- package/runtime-src/execution/index.ts +0 -3407
- package/runtime-src/execution/retry.ts +0 -46
- package/runtime-src/execution/robots.ts +0 -167
- package/runtime-src/execution/search-forms.ts +0 -188
- package/runtime-src/execution/token-resolver.ts +0 -135
- package/runtime-src/extraction/index.ts +0 -1507
- package/runtime-src/foundry/publish-bundle.ts +0 -392
- package/runtime-src/graph/agent-augment.ts +0 -315
- package/runtime-src/graph/index.ts +0 -1532
- package/runtime-src/graph/local-fixtures.ts +0 -393
- package/runtime-src/graph/local-harness.ts +0 -646
- package/runtime-src/graph/planner.ts +0 -411
- package/runtime-src/graph/session.ts +0 -294
- package/runtime-src/graph/trace-store.ts +0 -136
- package/runtime-src/impact-log.ts +0 -227
- package/runtime-src/index.ts +0 -24
- package/runtime-src/indexer/index.ts +0 -465
- package/runtime-src/intent-match.ts +0 -1515
- package/runtime-src/kuri/client.ts +0 -1839
- package/runtime-src/logger.ts +0 -30
- package/runtime-src/marketplace/index.ts +0 -111
- package/runtime-src/mcp.ts +0 -1911
- package/runtime-src/orchestrator/browser-agent.ts +0 -374
- package/runtime-src/orchestrator/dag-advisor.ts +0 -59
- package/runtime-src/orchestrator/dag-feedback.ts +0 -257
- package/runtime-src/orchestrator/first-pass-action.ts +0 -403
- package/runtime-src/orchestrator/index.ts +0 -4480
- package/runtime-src/orchestrator/passive-publish.ts +0 -187
- package/runtime-src/orchestrator/timing-economics.ts +0 -80
- package/runtime-src/payments/cascade.ts +0 -137
- package/runtime-src/payments/index.ts +0 -270
- package/runtime-src/payments/lobster-pay.ts +0 -182
- package/runtime-src/payments/wallet.ts +0 -98
- package/runtime-src/publish/review-context.ts +0 -93
- package/runtime-src/publish/sanitize.ts +0 -197
- package/runtime-src/publish/schema-review.ts +0 -192
- package/runtime-src/publish-admission.ts +0 -388
- package/runtime-src/ratelimit/index.ts +0 -23
- package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
- package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
- package/runtime-src/reverse-engineer/index.ts +0 -1551
- package/runtime-src/reverse-engineer/token-sources.ts +0 -379
- package/runtime-src/router.ts +0 -17
- package/runtime-src/routing-telemetry.ts +0 -395
- package/runtime-src/runtime/browser-access.ts +0 -11
- package/runtime-src/runtime/browser-auth.ts +0 -12
- package/runtime-src/runtime/browser-host.ts +0 -48
- package/runtime-src/runtime/lifecycle.ts +0 -17
- package/runtime-src/runtime/local-server.ts +0 -311
- package/runtime-src/runtime/paths.ts +0 -99
- package/runtime-src/runtime/setup.ts +0 -251
- package/runtime-src/runtime/supervisor.ts +0 -69
- package/runtime-src/runtime/update-hints.ts +0 -351
- package/runtime-src/server.ts +0 -100
- package/runtime-src/session-logs.ts +0 -142
- package/runtime-src/settings.ts +0 -221
- package/runtime-src/single-binary.ts +0 -143
- package/runtime-src/site-policy.ts +0 -54
- package/runtime-src/stale-cleanup-runner.ts +0 -144
- package/runtime-src/stale-cleanup.ts +0 -133
- package/runtime-src/telemetry-attribution.ts +0 -120
- package/runtime-src/telemetry.ts +0 -253
- package/runtime-src/template-params.ts +0 -141
- package/runtime-src/transform/drift.ts +0 -60
- package/runtime-src/transform/index.ts +0 -277
- package/runtime-src/types/index.ts +0 -1
- package/runtime-src/types/skill.ts +0 -931
- package/runtime-src/vault/index.ts +0 -196
- package/runtime-src/verification/auth-gate.ts +0 -8
- package/runtime-src/verification/candidates.ts +0 -27
- package/runtime-src/verification/index.ts +0 -120
- package/runtime-src/verification/matrix.ts +0 -30
- package/runtime-src/version.ts +0 -148
- package/runtime-src/workflow/artifact.ts +0 -161
- package/runtime-src/workflow/compile.ts +0 -808
- package/runtime-src/workflow/publish.ts +0 -225
- package/runtime-src/workflow/runtime.ts +0 -213
|
@@ -1,3407 +0,0 @@
|
|
|
1
|
-
import { executeInBrowser, triggerAndIntercept } from "../capture/index.js";
|
|
2
|
-
import { captureSession } from "../capture/index.js";
|
|
3
|
-
import type { CaptureResult, RawRequest } from "../capture/index.js";
|
|
4
|
-
import { extractEndpoints, extractAuthHeaders, type ExtractionContext } from "../reverse-engineer/index.js";
|
|
5
|
-
import { scanBundlesForRoutes } from "../reverse-engineer/bundle-scanner.js";
|
|
6
|
-
import { publishSkill, mergeEndpoints } from "../marketplace/index.js";
|
|
7
|
-
import { updateEndpointScore } from "../marketplace/index.js";
|
|
8
|
-
import { getCredential, storeCredential, deleteCredential } from "../vault/index.js";
|
|
9
|
-
import { getStoredAuth, getAuthCookies, refreshAuthFromBrowser } from "../auth/index.js";
|
|
10
|
-
import { resolvePreExecutionAuth } from "../auth/dependency-runtime.js";
|
|
11
|
-
import { authRuntime } from "../auth/runtime.js";
|
|
12
|
-
import { applyProjection, inferSchema } from "../transform/index.js";
|
|
13
|
-
import { detectSchemaDrift } from "../transform/drift.js";
|
|
14
|
-
import { recordExecution, recordTransaction, cachePublishedSkill, findExistingSkillForDomain, getLocalWalletContext, updateEndpointSchema } from "../client/index.js";
|
|
15
|
-
import { validateManifest } from "../client/index.js";
|
|
16
|
-
import { withRetry, isRetryableStatus } from "./retry.js";
|
|
17
|
-
import type { EndpointDescriptor, ExecutionOptions, ExecutionTrace, ProjectionOptions, SkillManifest } from "../types/index.js";
|
|
18
|
-
import { nanoid } from "nanoid";
|
|
19
|
-
import { getRegistrableDomain } from "../domain.js";
|
|
20
|
-
import { extractFromDOM, extractFromDOMWithHint } from "../extraction/index.js";
|
|
21
|
-
import { buildSkillOperationGraph, getEndpointDescriptionMetadata, inferEndpointSemantic, resolveEndpointSemantic } from "../graph/index.js";
|
|
22
|
-
import { log } from "../logger.js";
|
|
23
|
-
import { TRACE_VERSION } from "../version.js";
|
|
24
|
-
import { buildQueryBindingMap, extractTemplateQueryBindings, mergeContextTemplateParams } from "../template-params.js";
|
|
25
|
-
import { assessIntentResult, projectIntentData } from "../intent-match.js";
|
|
26
|
-
import { isStructuredSearchForm, detectSearchForms, type SearchFormSpec } from "./search-forms.js";
|
|
27
|
-
import { attributeLifecycle, type LifecycleEvent, type LifecyclePhase } from "../runtime/lifecycle.js";
|
|
28
|
-
import { queueBackgroundIndex } from "../indexer/index.js";
|
|
29
|
-
import {
|
|
30
|
-
writeSkillSnapshot,
|
|
31
|
-
domainSkillCache,
|
|
32
|
-
persistDomainCache,
|
|
33
|
-
getDomainReuseKey,
|
|
34
|
-
scopedCacheKey,
|
|
35
|
-
buildResolveCacheKey,
|
|
36
|
-
snapshotPathForCacheKey,
|
|
37
|
-
generateLocalDescription,
|
|
38
|
-
} from "../orchestrator/index.js";
|
|
39
|
-
import { checkPaymentRequirement } from "../payments/index.js";
|
|
40
|
-
import { isAllowedByRobots } from "./robots.js";
|
|
41
|
-
import { annotateEndpointPolicy, endpointRequiresThirdPartyTermsConfirmation, getEndpointPolicy } from "../site-policy.js";
|
|
42
|
-
import {
|
|
43
|
-
mergeWorkflowArtifacts,
|
|
44
|
-
readWorkflowArtifact,
|
|
45
|
-
recordWorkflowRecipeOutcome,
|
|
46
|
-
writeWorkflowArtifact,
|
|
47
|
-
} from "../workflow/artifact.js";
|
|
48
|
-
import { buildWorkflowArtifactFromCapture } from "../workflow/compile.js";
|
|
49
|
-
import {
|
|
50
|
-
needsWorkflowTokenRefresh,
|
|
51
|
-
pickWorkflowRecipe,
|
|
52
|
-
resolveWorkflowBindings,
|
|
53
|
-
translateWorkflowStrategy,
|
|
54
|
-
validateWorkflowReplayParams,
|
|
55
|
-
} from "../workflow/runtime.js";
|
|
56
|
-
import { buildWorkflowPublishArtifact, writeWorkflowPublishArtifact } from "../workflow/publish.js";
|
|
57
|
-
/** Stamp every trace with the code version hash for telemetry tracking */
|
|
58
|
-
function stampTrace(trace: ExecutionTrace): ExecutionTrace {
|
|
59
|
-
trace.trace_version = TRACE_VERSION;
|
|
60
|
-
return trace;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
const DEFAULT_BROWSER_UA =
|
|
64
|
-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36";
|
|
65
|
-
|
|
66
|
-
function cloneReplayBody(body: unknown): unknown {
|
|
67
|
-
if (!body || typeof body !== "object") return body;
|
|
68
|
-
if (Array.isArray(body)) return body.map((entry) => cloneReplayBody(entry));
|
|
69
|
-
return { ...(body as Record<string, unknown>) };
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
function serializeReplayBody(body: unknown, headers: Record<string, string>): BodyInit | undefined {
|
|
73
|
-
if (body == null) return undefined;
|
|
74
|
-
if (typeof body === "string") return body;
|
|
75
|
-
if (body instanceof URLSearchParams) return body.toString();
|
|
76
|
-
if (body instanceof FormData) return body;
|
|
77
|
-
const contentType = headers["content-type"] ?? headers["Content-Type"] ?? "";
|
|
78
|
-
if (/application\/x-www-form-urlencoded/i.test(contentType) && body && typeof body === "object" && !Array.isArray(body)) {
|
|
79
|
-
const params = new URLSearchParams();
|
|
80
|
-
for (const [key, value] of Object.entries(body as Record<string, unknown>)) {
|
|
81
|
-
if (value == null) continue;
|
|
82
|
-
params.set(key, String(value));
|
|
83
|
-
}
|
|
84
|
-
return params.toString();
|
|
85
|
-
}
|
|
86
|
-
return JSON.stringify(body);
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
async function reloadExecutionAuthState(
|
|
90
|
-
skill: SkillManifest,
|
|
91
|
-
epDomain: string,
|
|
92
|
-
authHeaders: Record<string, string>,
|
|
93
|
-
cookies: Array<{ name: string; value: string; domain: string }>,
|
|
94
|
-
): Promise<void> {
|
|
95
|
-
for (const key of Object.keys(authHeaders)) delete authHeaders[key];
|
|
96
|
-
cookies.splice(0, cookies.length);
|
|
97
|
-
|
|
98
|
-
if (skill.auth_profile_ref) {
|
|
99
|
-
const stored = await getCredential(skill.auth_profile_ref);
|
|
100
|
-
if (stored) {
|
|
101
|
-
try {
|
|
102
|
-
const parsed = JSON.parse(stored) as {
|
|
103
|
-
headers?: Record<string, string>;
|
|
104
|
-
cookies?: Array<{ name: string; value: string; domain: string }>;
|
|
105
|
-
};
|
|
106
|
-
Object.assign(authHeaders, parsed.headers ?? {});
|
|
107
|
-
cookies.push(...(parsed.cookies ?? []));
|
|
108
|
-
} catch {
|
|
109
|
-
/* ignore malformed auth state */
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
if (cookies.length === 0) {
|
|
115
|
-
try {
|
|
116
|
-
const resolved = await getAuthCookies(epDomain, {
|
|
117
|
-
autoExtract: !!skill.auth_profile_ref,
|
|
118
|
-
});
|
|
119
|
-
if (resolved?.length) cookies.push(...resolved);
|
|
120
|
-
} catch {
|
|
121
|
-
/* ignore */
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
if (Object.keys(authHeaders).length === 0) {
|
|
126
|
-
try {
|
|
127
|
-
const sessionKey = `${getRegistrableDomain(epDomain)}-session`;
|
|
128
|
-
const sessionData = await getCredential(sessionKey);
|
|
129
|
-
if (sessionData) {
|
|
130
|
-
const parsed = JSON.parse(sessionData) as { headers?: Record<string, string>; cookies?: typeof cookies };
|
|
131
|
-
if (parsed.headers) Object.assign(authHeaders, parsed.headers);
|
|
132
|
-
if (parsed.cookies && cookies.length === 0) cookies.push(...parsed.cookies);
|
|
133
|
-
}
|
|
134
|
-
} catch {
|
|
135
|
-
/* ignore */
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
function persistWorkflowArtifactForCapture(
|
|
141
|
-
artifactSkill: SkillManifest,
|
|
142
|
-
captured: Pick<CaptureResult, "requests" | "har_lineage_id" | "final_url" | "html" | "js_bundles" | "cookies">,
|
|
143
|
-
capturedAuthHeaders?: Record<string, string>,
|
|
144
|
-
): void {
|
|
145
|
-
try {
|
|
146
|
-
if (process.env.UNBROWSE_DEBUG_WORKFLOW === "1") {
|
|
147
|
-
log(
|
|
148
|
-
"workflow",
|
|
149
|
-
`capture artifact attempt skill=${artifactSkill.skill_id} requests=${captured.requests.length} final_url=${captured.final_url}`,
|
|
150
|
-
);
|
|
151
|
-
}
|
|
152
|
-
const nextArtifact = mergeWorkflowArtifacts(
|
|
153
|
-
buildWorkflowArtifactFromCapture(artifactSkill, captured, { authHeaders: capturedAuthHeaders }),
|
|
154
|
-
readWorkflowArtifact(artifactSkill.skill_id),
|
|
155
|
-
);
|
|
156
|
-
const writtenPath = writeWorkflowArtifact(nextArtifact);
|
|
157
|
-
const exportPath = writeWorkflowPublishArtifact(buildWorkflowPublishArtifact(artifactSkill, nextArtifact, {
|
|
158
|
-
publishStatus: "captured",
|
|
159
|
-
}));
|
|
160
|
-
if (process.env.UNBROWSE_DEBUG_WORKFLOW === "1") {
|
|
161
|
-
log(
|
|
162
|
-
"workflow",
|
|
163
|
-
`capture persisted skill=${artifactSkill.skill_id} requests=${captured.requests.length} path=${writtenPath ?? "write-failed"} export=${exportPath ?? "write-failed"}`,
|
|
164
|
-
);
|
|
165
|
-
}
|
|
166
|
-
} catch (error) {
|
|
167
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
168
|
-
log("workflow", `capture persistence failed for ${artifactSkill.skill_id}: ${message}`);
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
// ---------------------------------------------------------------------------
|
|
173
|
-
// Quality gate — validate extracted data before marketplace publishing
|
|
174
|
-
// ---------------------------------------------------------------------------
|
|
175
|
-
|
|
176
|
-
interface QualityResult {
|
|
177
|
-
valid: boolean;
|
|
178
|
-
quality_note?: string;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
const VALID_VERIFICATION_STATUSES = new Set(["verified", "unverified", "failed", "pending"]);
|
|
182
|
-
|
|
183
|
-
function normalizeEndpointForManifest(endpoint: EndpointDescriptor): EndpointDescriptor {
|
|
184
|
-
const verification_status = VALID_VERIFICATION_STATUSES.has(endpoint.verification_status)
|
|
185
|
-
? endpoint.verification_status
|
|
186
|
-
: (endpoint.response_schema || endpoint.dom_extraction ? "verified" : "pending");
|
|
187
|
-
return { ...endpoint, verification_status };
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
async function prepareLearnedEndpoints(
|
|
191
|
-
endpoints: EndpointDescriptor[],
|
|
192
|
-
_intent: string,
|
|
193
|
-
_domain: string,
|
|
194
|
-
): Promise<EndpointDescriptor[]> {
|
|
195
|
-
return endpoints.map(normalizeEndpointForManifest);
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
function intentWantsStructuredRecords(intent?: string): boolean {
|
|
199
|
-
return /\b(search|list|find|get|fetch|timeline|feed|trending)\b/i.test(intent ?? "");
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
export function isBundleInferredEndpoint(endpoint: Pick<EndpointDescriptor, "description">): boolean {
|
|
203
|
-
return /inferred from js bundle/i.test(endpoint.description ?? "");
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
function isSupportEvidenceEndpoint(endpoint: EndpointDescriptor): boolean {
|
|
207
|
-
if (isCanonicalReplayEndpoint(endpoint)) return true;
|
|
208
|
-
if (endpoint.dom_extraction && endpoint.response_schema) return true;
|
|
209
|
-
if (isBundleInferredEndpoint(endpoint)) return false;
|
|
210
|
-
return !!endpoint.response_schema;
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
function looksLikeUiChromeText(value: string): boolean {
|
|
214
|
-
const lower = value.toLowerCase();
|
|
215
|
-
let hits = 0;
|
|
216
|
-
for (const token of [
|
|
217
|
-
"advanced search",
|
|
218
|
-
"pull requests",
|
|
219
|
-
"discussions",
|
|
220
|
-
"languages",
|
|
221
|
-
"more languages",
|
|
222
|
-
"owner",
|
|
223
|
-
"number of stars",
|
|
224
|
-
"number of forks",
|
|
225
|
-
"date created",
|
|
226
|
-
"date pushed",
|
|
227
|
-
"public private",
|
|
228
|
-
"results",
|
|
229
|
-
]) {
|
|
230
|
-
if (lower.includes(token)) hits++;
|
|
231
|
-
}
|
|
232
|
-
return hits >= 2;
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
/** Detect concatenated values like "AAPLApple" or "Inc978,583" */
|
|
236
|
-
function isConcatenatedValue(s: string): boolean {
|
|
237
|
-
// Uppercase ticker jammed onto capitalized word: AAPLApple, NVDANvidia
|
|
238
|
-
if (/[A-Z]{2,}[A-Z][a-z]/.test(s)) return true;
|
|
239
|
-
// Word ending in letter immediately followed by digits: Inc978, Corp123
|
|
240
|
-
if (/[a-zA-Z]\d{3,}/.test(s)) return true;
|
|
241
|
-
return false;
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
/**
|
|
245
|
-
* Validate extraction quality. Always returns data to the caller —
|
|
246
|
-
* this only gates whether we publish to the marketplace.
|
|
247
|
-
*/
|
|
248
|
-
export function validateExtractionQuality(data: unknown, confidence: number, intent?: string): QualityResult {
|
|
249
|
-
// 1. Min confidence
|
|
250
|
-
if (confidence < 0.5) {
|
|
251
|
-
return { valid: false, quality_note: `confidence too low (${confidence.toFixed(2)} < 0.5)` };
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
// Only validate arrays (repeated data structures)
|
|
255
|
-
if (!Array.isArray(data)) return { valid: true };
|
|
256
|
-
if (data.length === 0) return { valid: true };
|
|
257
|
-
if (intentWantsStructuredRecords(intent) && data.every((item) => !item || typeof item !== "object" || Array.isArray(item))) {
|
|
258
|
-
return { valid: false, quality_note: "primitive rows only — expected structured records" };
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
const stringRows = data.filter((item): item is string => typeof item === "string");
|
|
262
|
-
if (stringRows.length === data.length) {
|
|
263
|
-
const uiChromeRows = stringRows.filter((item) => looksLikeUiChromeText(item));
|
|
264
|
-
if (uiChromeRows.length / stringRows.length >= 0.5) {
|
|
265
|
-
return { valid: false, quality_note: "ui chrome text detected instead of structured records" };
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
// 2. Deduplication check
|
|
270
|
-
const serialized = data.map((item) => JSON.stringify(item));
|
|
271
|
-
const unique = new Set(serialized);
|
|
272
|
-
const dupeRatio = 1 - unique.size / serialized.length;
|
|
273
|
-
if (dupeRatio > 0.5) {
|
|
274
|
-
return { valid: false, quality_note: `${Math.round(dupeRatio * 100)}% duplicate rows` };
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
// 3. Concatenation detection
|
|
278
|
-
let totalStrings = 0;
|
|
279
|
-
let concatStrings = 0;
|
|
280
|
-
for (const item of data) {
|
|
281
|
-
if (item && typeof item === "object") {
|
|
282
|
-
for (const val of Object.values(item as Record<string, unknown>)) {
|
|
283
|
-
if (typeof val === "string" && val.length > 3) {
|
|
284
|
-
totalStrings++;
|
|
285
|
-
if (isConcatenatedValue(val)) concatStrings++;
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
if (totalStrings > 0 && concatStrings / totalStrings > 0.3) {
|
|
291
|
-
return { valid: false, quality_note: `${Math.round((concatStrings / totalStrings) * 100)}% concatenated values detected` };
|
|
292
|
-
}
|
|
293
|
-
|
|
294
|
-
// 4. Diversity check — reject if all items share the same link/title (nav chrome)
|
|
295
|
-
if (data.length >= 3) {
|
|
296
|
-
for (const field of ["link", "href", "url", "title"]) {
|
|
297
|
-
const vals = data
|
|
298
|
-
.map((item) => (item && typeof item === "object" ? (item as Record<string, unknown>)[field] : undefined))
|
|
299
|
-
.filter((v) => v != null);
|
|
300
|
-
if (vals.length >= 3) {
|
|
301
|
-
const uniqueVals = new Set(vals.map(String));
|
|
302
|
-
if (uniqueVals.size === 1) {
|
|
303
|
-
return { valid: false, quality_note: `all items share the same "${field}" — likely navigation chrome` };
|
|
304
|
-
}
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
return { valid: true };
|
|
310
|
-
}
|
|
311
|
-
|
|
312
|
-
export interface ExecutionResult {
|
|
313
|
-
trace: ExecutionTrace;
|
|
314
|
-
result: unknown;
|
|
315
|
-
learned_skill?: SkillManifest;
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
export function projectResultForIntent(data: unknown, intent?: string): unknown {
|
|
319
|
-
return projectIntentData(data, intent);
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
function inferActionKindFromIntent(intent: string): string {
|
|
323
|
-
const lower = intent.toLowerCase();
|
|
324
|
-
if (/\b(search|find|lookup)\b/.test(lower)) return "search";
|
|
325
|
-
if (/\b(list|feed|timeline|trending)\b/.test(lower)) return "list";
|
|
326
|
-
return "detail";
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
function sanitizeNavigationQueryParams(url: URL): URL {
|
|
330
|
-
const out = new URL(url.toString());
|
|
331
|
-
for (const key of [...out.searchParams.keys()]) {
|
|
332
|
-
const lower = key.toLowerCase();
|
|
333
|
-
if (lower === "url" || lower === "context_url" || lower === "intent" || lower === "redirect" || lower === "redirect_url") {
|
|
334
|
-
out.searchParams.delete(key);
|
|
335
|
-
}
|
|
336
|
-
}
|
|
337
|
-
return out;
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
function restoreTemplatePlaceholderEncoding(url: string): string {
|
|
341
|
-
// Only restore template placeholders like {variable_name}, not arbitrary JSON braces.
|
|
342
|
-
// Template placeholders: %7Bword_chars%7D (no spaces, no quotes, no colons inside)
|
|
343
|
-
return url.replace(/%7B(\w+)%7D/gi, "{$1}");
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
function compactSchemaSample(value: unknown, depth = 0): unknown {
|
|
347
|
-
if (depth >= 4) return Array.isArray(value) ? [] : value && typeof value === "object" ? "[truncated]" : value;
|
|
348
|
-
if (Array.isArray(value)) return value.slice(0, 3).map((item) => compactSchemaSample(item, depth + 1));
|
|
349
|
-
if (value && typeof value === "object") {
|
|
350
|
-
return Object.fromEntries(
|
|
351
|
-
Object.entries(value as Record<string, unknown>)
|
|
352
|
-
.slice(0, 20)
|
|
353
|
-
.map(([key, next]) => [key, compactSchemaSample(next, depth + 1)]),
|
|
354
|
-
);
|
|
355
|
-
}
|
|
356
|
-
return value;
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
function isDocumentLikeUrl(url: string): boolean {
|
|
360
|
-
try {
|
|
361
|
-
const parsed = new URL(url);
|
|
362
|
-
return !/\/api\/|graphql|\/rest\/|\/rpc\/|\/ajax\/|\/1\.1\/|\/2\/|voyager/i.test(parsed.pathname);
|
|
363
|
-
} catch {
|
|
364
|
-
return false;
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
export function resolveExecutionUrlTemplate(
|
|
369
|
-
endpoint: EndpointDescriptor,
|
|
370
|
-
contextUrl?: string,
|
|
371
|
-
): string {
|
|
372
|
-
if (!contextUrl) return endpoint.url_template;
|
|
373
|
-
if (endpoint.method !== "GET") return endpoint.url_template;
|
|
374
|
-
if (!isDocumentLikeUrl(endpoint.url_template)) return endpoint.url_template;
|
|
375
|
-
if (endpoint.trigger_url && !isDocumentLikeUrl(endpoint.trigger_url)) return endpoint.url_template;
|
|
376
|
-
return contextUrl;
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
export function shouldIgnoreLearnedBrowserStrategy(
|
|
380
|
-
endpoint: EndpointDescriptor,
|
|
381
|
-
resolvedUrl: string,
|
|
382
|
-
): boolean {
|
|
383
|
-
return endpoint.method === "GET" && !endpoint.dom_extraction && !isDocumentLikeUrl(resolvedUrl);
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
function deriveStructuredDataReplay(url: string, mode: "concrete" | "template"): string {
|
|
387
|
-
try {
|
|
388
|
-
const parsed = new URL(url);
|
|
389
|
-
const host = parsed.hostname.replace(/^www\./, "");
|
|
390
|
-
const pathname = parsed.pathname.replace(/\/+$/, "") || "/";
|
|
391
|
-
const templateParam = (primary: string): string => `{${primary}}`;
|
|
392
|
-
|
|
393
|
-
if (host === "mastodon.social") {
|
|
394
|
-
if (pathname === "/search" || pathname === "/search/") {
|
|
395
|
-
if (mode === "concrete" && !parsed.searchParams.get("q")) return url;
|
|
396
|
-
const replay = new URL("https://mastodon.social/api/v2/search");
|
|
397
|
-
replay.searchParams.set("q", mode === "template" ? templateParam("q") : (parsed.searchParams.get("q") ?? ""));
|
|
398
|
-
replay.searchParams.set("resolve", "false");
|
|
399
|
-
replay.searchParams.set("type", "statuses");
|
|
400
|
-
replay.searchParams.set("limit", "20");
|
|
401
|
-
return restoreTemplatePlaceholderEncoding(replay.toString());
|
|
402
|
-
}
|
|
403
|
-
if (pathname === "/public") {
|
|
404
|
-
const replay = new URL("https://mastodon.social/api/v1/timelines/public");
|
|
405
|
-
replay.searchParams.set("limit", "20");
|
|
406
|
-
return replay.toString();
|
|
407
|
-
}
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
if (host === "gitlab.com") {
|
|
411
|
-
if (pathname === "/explore/projects") {
|
|
412
|
-
if (mode === "concrete" && !parsed.searchParams.get("name") && !parsed.searchParams.get("search")) return url;
|
|
413
|
-
const replay = new URL("https://gitlab.com/api/v4/projects");
|
|
414
|
-
const search = parsed.searchParams.get("search");
|
|
415
|
-
const name = parsed.searchParams.get("name");
|
|
416
|
-
replay.searchParams.set(
|
|
417
|
-
"search",
|
|
418
|
-
mode === "template"
|
|
419
|
-
? (search && search.length > 0 ? "{search}" : name && name.length > 0 ? "{name}" : "{q}")
|
|
420
|
-
: (name ?? search ?? ""),
|
|
421
|
-
);
|
|
422
|
-
replay.searchParams.set("simple", "true");
|
|
423
|
-
replay.searchParams.set("per_page", "20");
|
|
424
|
-
return restoreTemplatePlaceholderEncoding(replay.toString());
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
|
|
428
|
-
const reserved = new Set([
|
|
429
|
-
"-",
|
|
430
|
-
"api",
|
|
431
|
-
"admin",
|
|
432
|
-
"dashboard",
|
|
433
|
-
"explore",
|
|
434
|
-
"groups",
|
|
435
|
-
"help",
|
|
436
|
-
"oauth",
|
|
437
|
-
"profile",
|
|
438
|
-
"projects",
|
|
439
|
-
"search",
|
|
440
|
-
"session",
|
|
441
|
-
"users",
|
|
442
|
-
]);
|
|
443
|
-
if (segments.length === 2 && !reserved.has(segments[0])) {
|
|
444
|
-
return `https://gitlab.com/api/v4/projects/${encodeURIComponent(`${segments[0]}/${segments[1]}`)}`;
|
|
445
|
-
}
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
if (host === "github.com") {
|
|
449
|
-
if (pathname === "/search") {
|
|
450
|
-
if (mode === "concrete" && !parsed.searchParams.get("q")) return url;
|
|
451
|
-
const replay = new URL("https://api.github.com/search/repositories");
|
|
452
|
-
replay.searchParams.set(
|
|
453
|
-
"q",
|
|
454
|
-
mode === "template" ? "{q}" : (parsed.searchParams.get("q") ?? ""),
|
|
455
|
-
);
|
|
456
|
-
replay.searchParams.set("per_page", "20");
|
|
457
|
-
return restoreTemplatePlaceholderEncoding(replay.toString());
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
if (host === "hn.algolia.com") {
|
|
462
|
-
if (pathname === "/" || pathname === "") {
|
|
463
|
-
if (mode === "concrete" && !parsed.searchParams.get("q") && !parsed.searchParams.get("query")) return url;
|
|
464
|
-
const replay = new URL("https://hn.algolia.com/api/v1/search");
|
|
465
|
-
replay.searchParams.set(
|
|
466
|
-
"query",
|
|
467
|
-
mode === "template" ? "{q}" : (parsed.searchParams.get("q") ?? parsed.searchParams.get("query") ?? ""),
|
|
468
|
-
);
|
|
469
|
-
replay.searchParams.set("tags", "story");
|
|
470
|
-
return restoreTemplatePlaceholderEncoding(replay.toString());
|
|
471
|
-
}
|
|
472
|
-
}
|
|
473
|
-
|
|
474
|
-
if (host === "huggingface.co") {
|
|
475
|
-
if (pathname === "/models" || pathname === "/models/") {
|
|
476
|
-
if (mode === "concrete" && !parsed.searchParams.get("search") && !parsed.searchParams.get("q")) return url;
|
|
477
|
-
const replay = new URL("https://huggingface.co/api/models");
|
|
478
|
-
replay.searchParams.set(
|
|
479
|
-
"search",
|
|
480
|
-
mode === "template"
|
|
481
|
-
? (parsed.searchParams.get("search") ? "{search}" : parsed.searchParams.get("q") ? "{q}" : "{search}")
|
|
482
|
-
: (parsed.searchParams.get("search") ?? parsed.searchParams.get("q") ?? ""),
|
|
483
|
-
);
|
|
484
|
-
replay.searchParams.set("limit", "20");
|
|
485
|
-
return restoreTemplatePlaceholderEncoding(replay.toString());
|
|
486
|
-
}
|
|
487
|
-
}
|
|
488
|
-
|
|
489
|
-
if (host === "developer.mozilla.org") {
|
|
490
|
-
const locale = pathname.split("/").filter(Boolean)[0] || "en-US";
|
|
491
|
-
if (pathname.endsWith("/search") || pathname === "/search") {
|
|
492
|
-
if (mode === "concrete" && !parsed.searchParams.get("q")) return url;
|
|
493
|
-
const replay = new URL("https://developer.mozilla.org/api/v1/search");
|
|
494
|
-
replay.searchParams.set(
|
|
495
|
-
"q",
|
|
496
|
-
mode === "template" ? "{q}" : (parsed.searchParams.get("q") ?? ""),
|
|
497
|
-
);
|
|
498
|
-
replay.searchParams.set(
|
|
499
|
-
"page",
|
|
500
|
-
mode === "template"
|
|
501
|
-
? (parsed.searchParams.get("page") ? "{page}" : "1")
|
|
502
|
-
: (parsed.searchParams.get("page") ?? "1"),
|
|
503
|
-
);
|
|
504
|
-
replay.searchParams.set("page_size", parsed.searchParams.get("page_size") ?? "20");
|
|
505
|
-
replay.searchParams.set(
|
|
506
|
-
"locale",
|
|
507
|
-
mode === "template"
|
|
508
|
-
? (parsed.searchParams.get("locale") ? "{locale}" : locale)
|
|
509
|
-
: (parsed.searchParams.get("locale") ?? locale),
|
|
510
|
-
);
|
|
511
|
-
return restoreTemplatePlaceholderEncoding(replay.toString());
|
|
512
|
-
}
|
|
513
|
-
}
|
|
514
|
-
|
|
515
|
-
if (host === "dev.to") {
|
|
516
|
-
const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
|
|
517
|
-
if (segments[0] === "t" && segments[1]) {
|
|
518
|
-
const replay = new URL("https://dev.to/api/articles");
|
|
519
|
-
replay.searchParams.set("tag", mode === "template" ? "{tag}" : segments[1]);
|
|
520
|
-
replay.searchParams.set("per_page", "20");
|
|
521
|
-
return restoreTemplatePlaceholderEncoding(replay.toString());
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
if (host === "npmjs.com") {
|
|
526
|
-
if (pathname === "/search" || pathname === "/search/") {
|
|
527
|
-
if (mode === "concrete" && !parsed.searchParams.get("q")) return url;
|
|
528
|
-
const replay = new URL("https://registry.npmjs.org/-/v1/search");
|
|
529
|
-
replay.searchParams.set("text", mode === "template" ? templateParam("q") : (parsed.searchParams.get("q") ?? ""));
|
|
530
|
-
replay.searchParams.set("size", "20");
|
|
531
|
-
return restoreTemplatePlaceholderEncoding(replay.toString());
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
|
|
535
|
-
if (segments[0] === "package" && segments.length >= 2) {
|
|
536
|
-
const versionIndex = segments.indexOf("v");
|
|
537
|
-
const packageName = segments.slice(1, versionIndex === -1 ? undefined : versionIndex).join("/");
|
|
538
|
-
if (packageName) return `https://registry.npmjs.org/${encodeURIComponent(packageName)}`;
|
|
539
|
-
}
|
|
540
|
-
}
|
|
541
|
-
|
|
542
|
-
if (host === "pypi.org") {
|
|
543
|
-
const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
|
|
544
|
-
if (segments[0] === "project" && segments[1]) {
|
|
545
|
-
return `https://pypi.org/pypi/${encodeURIComponent(segments[1])}/json`;
|
|
546
|
-
}
|
|
547
|
-
}
|
|
548
|
-
|
|
549
|
-
if (host === "pub.dev") {
|
|
550
|
-
const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
|
|
551
|
-
if (segments[0] === "packages" && segments[1]) {
|
|
552
|
-
return `https://pub.dev/api/packages/${encodeURIComponent(segments[1])}`;
|
|
553
|
-
}
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
if (host === "hub.docker.com") {
|
|
557
|
-
if (pathname === "/search" || pathname === "/search/") {
|
|
558
|
-
if (mode === "concrete" && !parsed.searchParams.get("q") && !parsed.searchParams.get("query")) return url;
|
|
559
|
-
const replay = new URL("https://hub.docker.com/v2/search/repositories/");
|
|
560
|
-
replay.searchParams.set(
|
|
561
|
-
"query",
|
|
562
|
-
mode === "template"
|
|
563
|
-
? (parsed.searchParams.get("query") ? "{query}" : "{q}")
|
|
564
|
-
: (parsed.searchParams.get("q") ?? parsed.searchParams.get("query") ?? ""),
|
|
565
|
-
);
|
|
566
|
-
replay.searchParams.set("page_size", "20");
|
|
567
|
-
return restoreTemplatePlaceholderEncoding(replay.toString());
|
|
568
|
-
}
|
|
569
|
-
|
|
570
|
-
const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
|
|
571
|
-
if (segments[0] === "r" && segments.length >= 4 && segments[3] === "tags") {
|
|
572
|
-
return `https://hub.docker.com/v2/repositories/${encodeURIComponent(segments[1])}/${encodeURIComponent(segments[2])}/tags/?page_size=25`;
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
|
|
576
|
-
if (host === "rubygems.org") {
|
|
577
|
-
const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
|
|
578
|
-
if (segments[0] === "gems" && segments[1]) {
|
|
579
|
-
return `https://rubygems.org/api/v1/gems/${encodeURIComponent(segments[1])}.json`;
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
if (host === "stackoverflow.com" || host === "serverfault.com" || host === "superuser.com") {
|
|
584
|
-
const segments = pathname.split("/").filter(Boolean).map((segment) => decodeURIComponent(segment));
|
|
585
|
-
if (segments[0] === "questions" && segments[1] === "tagged" && segments[2]) {
|
|
586
|
-
const replay = new URL("https://api.stackexchange.com/2.3/questions");
|
|
587
|
-
replay.searchParams.set("site", host === "stackoverflow.com" ? "stackoverflow" : host.replace(/\.com$/, ""));
|
|
588
|
-
replay.searchParams.set("tagged", mode === "template" ? "{tag}" : segments[2]);
|
|
589
|
-
replay.searchParams.set("order", "desc");
|
|
590
|
-
replay.searchParams.set("sort", "activity");
|
|
591
|
-
replay.searchParams.set("pagesize", "20");
|
|
592
|
-
replay.searchParams.set("filter", "default");
|
|
593
|
-
return restoreTemplatePlaceholderEncoding(replay.toString());
|
|
594
|
-
}
|
|
595
|
-
}
|
|
596
|
-
|
|
597
|
-
if (host === "jmail.world") {
|
|
598
|
-
if (pathname === "/search" || pathname === "/search/") {
|
|
599
|
-
if (mode === "concrete" && !parsed.searchParams.get("q")) return url;
|
|
600
|
-
const replay = new URL("https://jmail.world/api/emails/search");
|
|
601
|
-
replay.searchParams.set("q", mode === "template" ? "{q}" : (parsed.searchParams.get("q") ?? ""));
|
|
602
|
-
replay.searchParams.set("limit", mode === "template" ? "{limit}" : (parsed.searchParams.get("limit") ?? "50"));
|
|
603
|
-
replay.searchParams.set("page", mode === "template" ? "{page}" : (parsed.searchParams.get("page") ?? "1"));
|
|
604
|
-
replay.searchParams.set("source", mode === "template" ? "{source}" : (parsed.searchParams.get("source") ?? "all"));
|
|
605
|
-
return restoreTemplatePlaceholderEncoding(replay.toString());
|
|
606
|
-
}
|
|
607
|
-
}
|
|
608
|
-
|
|
609
|
-
if (host !== "reddit.com" && host !== "old.reddit.com" && host !== "np.reddit.com") return url;
|
|
610
|
-
if (/\.json$/i.test(parsed.pathname) || /\/api\/|\/svc\/|graphql/i.test(parsed.pathname)) return url;
|
|
611
|
-
if (parsed.pathname === "/search" || parsed.pathname === "/search/") {
|
|
612
|
-
if (mode === "concrete" && !parsed.searchParams.get("q")) return url;
|
|
613
|
-
parsed.pathname = "/search.json";
|
|
614
|
-
if (mode === "template" && !parsed.searchParams.get("q")) parsed.searchParams.set("q", "{q}");
|
|
615
|
-
return parsed.toString();
|
|
616
|
-
}
|
|
617
|
-
if (parsed.pathname.startsWith("/r/") || parsed.pathname.startsWith("/comments/")) {
|
|
618
|
-
parsed.pathname = `${parsed.pathname.replace(/\/$/, "")}/.json`;
|
|
619
|
-
return parsed.toString();
|
|
620
|
-
}
|
|
621
|
-
return url;
|
|
622
|
-
} catch {
|
|
623
|
-
return url;
|
|
624
|
-
}
|
|
625
|
-
}
|
|
626
|
-
|
|
627
|
-
export function deriveStructuredDataReplayUrl(url: string): string {
|
|
628
|
-
return deriveStructuredDataReplay(url, "concrete");
|
|
629
|
-
}
|
|
630
|
-
|
|
631
|
-
export function deriveStructuredDataReplayTemplate(url: string): string {
|
|
632
|
-
return deriveStructuredDataReplay(url, "template");
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
export function deriveStructuredDataReplayCandidates(url: string): string[] {
|
|
636
|
-
const primary = deriveStructuredDataReplayUrl(url);
|
|
637
|
-
const out = new Set<string>([primary]);
|
|
638
|
-
try {
|
|
639
|
-
const parsed = new URL(primary);
|
|
640
|
-
const host = parsed.hostname.replace(/^www\./, "");
|
|
641
|
-
if (host === "reddit.com" || host === "np.reddit.com") {
|
|
642
|
-
parsed.hostname = "old.reddit.com";
|
|
643
|
-
out.add(parsed.toString());
|
|
644
|
-
}
|
|
645
|
-
} catch {
|
|
646
|
-
// ignore
|
|
647
|
-
}
|
|
648
|
-
return [...out];
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
export function deriveStructuredDataReplayCandidatesFromInputs(
|
|
652
|
-
url: string,
|
|
653
|
-
params: Record<string, unknown> = {},
|
|
654
|
-
): string[] {
|
|
655
|
-
const seeded = new Set<string>();
|
|
656
|
-
const replayTemplate = deriveStructuredDataReplayTemplate(url);
|
|
657
|
-
if (replayTemplate !== url) {
|
|
658
|
-
const seededReplay = interpolate(
|
|
659
|
-
replayTemplate,
|
|
660
|
-
mergeContextTemplateParams(params, replayTemplate, url),
|
|
661
|
-
);
|
|
662
|
-
if (!/\{[^}]+\}/.test(seededReplay)) seeded.add(seededReplay);
|
|
663
|
-
}
|
|
664
|
-
for (const replayUrl of deriveStructuredDataReplayCandidates(url)) seeded.add(replayUrl);
|
|
665
|
-
return [...seeded];
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
export function buildStructuredReplayHeaders(
|
|
669
|
-
originalUrl: string,
|
|
670
|
-
replayUrl: string,
|
|
671
|
-
baseHeaders: Record<string, string>,
|
|
672
|
-
): Record<string, string> {
|
|
673
|
-
const headers = { ...baseHeaders };
|
|
674
|
-
try {
|
|
675
|
-
const replayTarget = new URL(replayUrl);
|
|
676
|
-
const originalTarget = new URL(originalUrl);
|
|
677
|
-
const host = replayTarget.hostname.replace(/^www\./, "");
|
|
678
|
-
const needsApiReplayHeaders =
|
|
679
|
-
replayTarget.hostname !== originalTarget.hostname ||
|
|
680
|
-
/\/api\/|graphql|\/rest\/|\/rpc\/|\/v\d+\//i.test(replayTarget.pathname);
|
|
681
|
-
if (needsApiReplayHeaders) {
|
|
682
|
-
headers["user-agent"] ??= DEFAULT_BROWSER_UA;
|
|
683
|
-
headers["accept-language"] ??= "en-US,en;q=0.9";
|
|
684
|
-
headers["referer"] ??= originalTarget.toString();
|
|
685
|
-
headers["accept"] ??= "application/json,text/plain,*/*";
|
|
686
|
-
}
|
|
687
|
-
if (host === "reddit.com" || host === "old.reddit.com" || host === "np.reddit.com") {
|
|
688
|
-
headers["user-agent"] ??= DEFAULT_BROWSER_UA;
|
|
689
|
-
headers["accept-language"] ??= "en-US,en;q=0.9";
|
|
690
|
-
headers["referer"] ??= originalTarget.toString();
|
|
691
|
-
headers["accept"] = "application/json,text/plain,*/*";
|
|
692
|
-
}
|
|
693
|
-
} catch {
|
|
694
|
-
return headers;
|
|
695
|
-
}
|
|
696
|
-
return headers;
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
function normalizeReplayHeaders(
|
|
700
|
-
...bags: Array<Record<string, string> | undefined>
|
|
701
|
-
): Record<string, string> {
|
|
702
|
-
const normalized: Record<string, string> = {};
|
|
703
|
-
for (const bag of bags) {
|
|
704
|
-
for (const [key, value] of Object.entries(bag ?? {})) {
|
|
705
|
-
if (typeof value !== "string") continue;
|
|
706
|
-
const trimmed = value.trim();
|
|
707
|
-
if (!trimmed) continue;
|
|
708
|
-
normalized[key.toLowerCase()] = trimmed;
|
|
709
|
-
}
|
|
710
|
-
}
|
|
711
|
-
return normalized;
|
|
712
|
-
}
|
|
713
|
-
|
|
714
|
-
function shouldFallbackToBrowserReplay(
|
|
715
|
-
data: unknown,
|
|
716
|
-
endpoint: EndpointDescriptor,
|
|
717
|
-
intent?: string,
|
|
718
|
-
contextUrl?: string,
|
|
719
|
-
): boolean {
|
|
720
|
-
const replayUrl = resolveExecutionUrlTemplate(endpoint, contextUrl);
|
|
721
|
-
if (!isDocumentLikeUrl(replayUrl)) return false;
|
|
722
|
-
if (typeof data === "string") return isHtml(data) || isSpaShell(data);
|
|
723
|
-
const assessment = assessIntentResult(data, intent);
|
|
724
|
-
return assessment.verdict === "fail";
|
|
725
|
-
}
|
|
726
|
-
|
|
727
|
-
function buildSampleRequestFromUrl(url: string): Record<string, unknown> {
|
|
728
|
-
try {
|
|
729
|
-
return Object.fromEntries(sanitizeNavigationQueryParams(new URL(url)).searchParams.entries());
|
|
730
|
-
} catch {
|
|
731
|
-
return {};
|
|
732
|
-
}
|
|
733
|
-
}
|
|
734
|
-
|
|
735
|
-
export function buildPageArtifactCapture(
|
|
736
|
-
url: string,
|
|
737
|
-
intent: string,
|
|
738
|
-
html: string,
|
|
739
|
-
authRequired = false,
|
|
740
|
-
): {
|
|
741
|
-
endpoint?: EndpointDescriptor;
|
|
742
|
-
result?: { data: unknown; _extraction: Record<string, unknown> };
|
|
743
|
-
quality_note?: string;
|
|
744
|
-
search_form?: SearchFormSpec;
|
|
745
|
-
} {
|
|
746
|
-
const extracted = extractFromDOM(html, intent);
|
|
747
|
-
if (!extracted.data || extracted.confidence <= 0.2) return {};
|
|
748
|
-
const quality = validateExtractionQuality(extracted.data, extracted.confidence, intent);
|
|
749
|
-
if (!quality.valid) {
|
|
750
|
-
return { quality_note: quality.quality_note ?? "low_quality_dom_extraction" };
|
|
751
|
-
}
|
|
752
|
-
const semanticAssessment = assessIntentResult(extracted.data, intent);
|
|
753
|
-
if (semanticAssessment.verdict === "fail") {
|
|
754
|
-
return { quality_note: semanticAssessment.reason };
|
|
755
|
-
}
|
|
756
|
-
|
|
757
|
-
// Detect structured search forms from the captured HTML
|
|
758
|
-
const searchForms = detectSearchForms(html);
|
|
759
|
-
const validSearchForm = searchForms.find((spec: SearchFormSpec) => isStructuredSearchForm(spec));
|
|
760
|
-
|
|
761
|
-
const response_schema = inferSchema([extracted.data]);
|
|
762
|
-
const endpoint: EndpointDescriptor = {
|
|
763
|
-
endpoint_id: nanoid(),
|
|
764
|
-
method: "GET",
|
|
765
|
-
url_template: templatizeQueryParams(url),
|
|
766
|
-
idempotency: "safe" as const,
|
|
767
|
-
verification_status: "verified" as const,
|
|
768
|
-
reliability_score: extracted.confidence,
|
|
769
|
-
description: validSearchForm
|
|
770
|
-
? `Captured search form artifact for ${intent}`
|
|
771
|
-
: `Captured page artifact for ${intent}`,
|
|
772
|
-
response_schema,
|
|
773
|
-
dom_extraction: {
|
|
774
|
-
extraction_method: extracted.extraction_method,
|
|
775
|
-
confidence: extracted.confidence,
|
|
776
|
-
...(extracted.selector ? { selector: extracted.selector } : {}),
|
|
777
|
-
...(validSearchForm ? { search_form: validSearchForm } : {}),
|
|
778
|
-
},
|
|
779
|
-
trigger_url: url,
|
|
780
|
-
};
|
|
781
|
-
endpoint.semantic = {
|
|
782
|
-
...inferEndpointSemantic(endpoint, {
|
|
783
|
-
sampleResponse: extracted.data,
|
|
784
|
-
sampleRequest: buildSampleRequestFromUrl(url),
|
|
785
|
-
observedAt: new Date().toISOString(),
|
|
786
|
-
sampleRequestUrl: url,
|
|
787
|
-
}),
|
|
788
|
-
...(authRequired ? { auth_required: true } : {}),
|
|
789
|
-
};
|
|
790
|
-
if (validSearchForm && endpoint.semantic) {
|
|
791
|
-
endpoint.semantic.action_kind = "search";
|
|
792
|
-
}
|
|
793
|
-
|
|
794
|
-
if (validSearchForm) {
|
|
795
|
-
log("execution", `detected structured search form: ${validSearchForm.form_selector} with ${validSearchForm.fields.length} fields`);
|
|
796
|
-
}
|
|
797
|
-
|
|
798
|
-
return {
|
|
799
|
-
endpoint,
|
|
800
|
-
result: {
|
|
801
|
-
data: extracted.data,
|
|
802
|
-
_extraction: {
|
|
803
|
-
method: extracted.extraction_method,
|
|
804
|
-
confidence: extracted.confidence,
|
|
805
|
-
source: "dom-fallback",
|
|
806
|
-
...(validSearchForm ? { search_form_detected: true } : {}),
|
|
807
|
-
},
|
|
808
|
-
},
|
|
809
|
-
...(validSearchForm ? { search_form: validSearchForm } : {}),
|
|
810
|
-
};
|
|
811
|
-
}
|
|
812
|
-
|
|
813
|
-
export function buildCanonicalDocumentEndpoint(
|
|
814
|
-
url: string,
|
|
815
|
-
intent: string,
|
|
816
|
-
authRequired = false,
|
|
817
|
-
): EndpointDescriptor | undefined {
|
|
818
|
-
const replayUrl = deriveStructuredDataReplayUrl(url);
|
|
819
|
-
const replayTemplate = deriveStructuredDataReplayTemplate(url);
|
|
820
|
-
if (replayUrl === url && replayTemplate === url) return undefined;
|
|
821
|
-
const endpoint: EndpointDescriptor = {
|
|
822
|
-
endpoint_id: nanoid(),
|
|
823
|
-
method: "GET",
|
|
824
|
-
url_template: replayTemplate !== url ? replayTemplate : replayUrl,
|
|
825
|
-
idempotency: "safe",
|
|
826
|
-
verification_status: "verified",
|
|
827
|
-
reliability_score: 0.9,
|
|
828
|
-
description: `Structured replay for ${intent}`,
|
|
829
|
-
trigger_url: url,
|
|
830
|
-
};
|
|
831
|
-
endpoint.semantic = {
|
|
832
|
-
...inferEndpointSemantic(endpoint, {
|
|
833
|
-
sampleRequest: buildSampleRequestFromUrl(url),
|
|
834
|
-
observedAt: new Date().toISOString(),
|
|
835
|
-
sampleRequestUrl: url,
|
|
836
|
-
}),
|
|
837
|
-
...(authRequired ? { auth_required: true } : {}),
|
|
838
|
-
};
|
|
839
|
-
return endpoint;
|
|
840
|
-
}
|
|
841
|
-
|
|
842
|
-
export function isCanonicalReplayEndpoint(endpoint: Pick<EndpointDescriptor, "method" | "url_template" | "trigger_url">): boolean {
|
|
843
|
-
if (endpoint.method !== "GET" || !endpoint.trigger_url) return false;
|
|
844
|
-
try {
|
|
845
|
-
const concrete = deriveStructuredDataReplayUrl(endpoint.trigger_url);
|
|
846
|
-
const template = deriveStructuredDataReplayTemplate(endpoint.trigger_url);
|
|
847
|
-
return endpoint.url_template === concrete || endpoint.url_template === template;
|
|
848
|
-
} catch {
|
|
849
|
-
return false;
|
|
850
|
-
}
|
|
851
|
-
}
|
|
852
|
-
|
|
853
|
-
async function trySeedStructuredDocumentSkill(
|
|
854
|
-
skill: SkillManifest,
|
|
855
|
-
url: string,
|
|
856
|
-
intent: string,
|
|
857
|
-
params: Record<string, unknown>,
|
|
858
|
-
targetDomain: string,
|
|
859
|
-
authHeaders: Record<string, string> | undefined,
|
|
860
|
-
cookies: Array<{ name: string; value: string; domain: string }> | undefined,
|
|
861
|
-
usedStoredAuth: boolean,
|
|
862
|
-
): Promise<ExecutionResult | undefined> {
|
|
863
|
-
const canonicalDocumentEndpoint = buildCanonicalDocumentEndpoint(url, intent, usedStoredAuth);
|
|
864
|
-
if (!canonicalDocumentEndpoint) return undefined;
|
|
865
|
-
|
|
866
|
-
const replayUrls = deriveStructuredDataReplayCandidatesFromInputs(url, params);
|
|
867
|
-
let headers: Record<string, string> = {
|
|
868
|
-
accept: "application/json,text/plain,*/*",
|
|
869
|
-
...(canonicalDocumentEndpoint.headers_template ?? {}),
|
|
870
|
-
...(authHeaders ?? {}),
|
|
871
|
-
};
|
|
872
|
-
if (cookies && cookies.length > 0) {
|
|
873
|
-
headers.cookie = cookies.map((c) => {
|
|
874
|
-
const v = c.value.startsWith('"') && c.value.endsWith('"') ? c.value.slice(1, -1) : c.value;
|
|
875
|
-
return `${c.name}=${v}`;
|
|
876
|
-
}).join("; ");
|
|
877
|
-
}
|
|
878
|
-
|
|
879
|
-
let data: unknown;
|
|
880
|
-
let passed = false;
|
|
881
|
-
let successfulReplayUrl = replayUrls[0] ?? url;
|
|
882
|
-
for (const replayUrl of replayUrls) {
|
|
883
|
-
const res = await fetch(replayUrl, {
|
|
884
|
-
method: "GET",
|
|
885
|
-
headers: buildStructuredReplayHeaders(url, replayUrl, headers),
|
|
886
|
-
redirect: "follow",
|
|
887
|
-
});
|
|
888
|
-
const text = await res.text();
|
|
889
|
-
try { data = JSON.parse(text); } catch { data = text; }
|
|
890
|
-
|
|
891
|
-
const assessment = assessIntentResult(data, intent);
|
|
892
|
-
if (assessment.verdict === "pass") {
|
|
893
|
-
passed = true;
|
|
894
|
-
successfulReplayUrl = replayUrl;
|
|
895
|
-
break;
|
|
896
|
-
}
|
|
897
|
-
}
|
|
898
|
-
if (!passed) return undefined;
|
|
899
|
-
|
|
900
|
-
const semanticSample = compactSchemaSample(data);
|
|
901
|
-
canonicalDocumentEndpoint.response_schema = inferSchema([semanticSample]);
|
|
902
|
-
canonicalDocumentEndpoint.semantic = {
|
|
903
|
-
...inferEndpointSemantic(canonicalDocumentEndpoint, {
|
|
904
|
-
sampleResponse: semanticSample,
|
|
905
|
-
sampleRequest: {
|
|
906
|
-
...buildSampleRequestFromUrl(url),
|
|
907
|
-
...params,
|
|
908
|
-
},
|
|
909
|
-
observedAt: new Date().toISOString(),
|
|
910
|
-
sampleRequestUrl: url,
|
|
911
|
-
}),
|
|
912
|
-
...(usedStoredAuth ? { auth_required: true } : {}),
|
|
913
|
-
};
|
|
914
|
-
|
|
915
|
-
const domain = getRegistrableDomain(targetDomain);
|
|
916
|
-
const existingSkill = findExistingSkillForDomain(domain, intent);
|
|
917
|
-
const localEndpoints = await prepareLearnedEndpoints(
|
|
918
|
-
existingSkill
|
|
919
|
-
? mergeEndpoints(existingSkill.endpoints, [canonicalDocumentEndpoint])
|
|
920
|
-
: [canonicalDocumentEndpoint],
|
|
921
|
-
intent,
|
|
922
|
-
domain,
|
|
923
|
-
);
|
|
924
|
-
|
|
925
|
-
const localDraft: SkillManifest = {
|
|
926
|
-
skill_id: existingSkill?.skill_id ?? nanoid(),
|
|
927
|
-
version: "1.0.0",
|
|
928
|
-
schema_version: "1",
|
|
929
|
-
lifecycle: "active" as const,
|
|
930
|
-
execution_type: "http" as const,
|
|
931
|
-
created_at: existingSkill?.created_at ?? new Date().toISOString(),
|
|
932
|
-
updated_at: new Date().toISOString(),
|
|
933
|
-
name: domain,
|
|
934
|
-
intent_signature: intent,
|
|
935
|
-
domain,
|
|
936
|
-
description: `API skill for ${domain}`,
|
|
937
|
-
owner_type: "agent" as const,
|
|
938
|
-
endpoints: localEndpoints,
|
|
939
|
-
operation_graph: buildSkillOperationGraph(localEndpoints),
|
|
940
|
-
intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
|
|
941
|
-
};
|
|
942
|
-
|
|
943
|
-
let learned: SkillManifest = localDraft;
|
|
944
|
-
const validation = await validateManifest({ ...localDraft, skill_id: "__validate__" });
|
|
945
|
-
if (validation.valid) {
|
|
946
|
-
try {
|
|
947
|
-
const { operation_graph: _graph, ...publishDraft } = localDraft;
|
|
948
|
-
const published = await publishSkill(publishDraft);
|
|
949
|
-
learned = {
|
|
950
|
-
...published,
|
|
951
|
-
endpoints: localEndpoints,
|
|
952
|
-
operation_graph: localDraft.operation_graph,
|
|
953
|
-
};
|
|
954
|
-
} catch {
|
|
955
|
-
learned = localDraft;
|
|
956
|
-
}
|
|
957
|
-
}
|
|
958
|
-
try { cachePublishedSkill(learned); } catch { /* best-effort */ }
|
|
959
|
-
const seededRequest: RawRequest = {
|
|
960
|
-
url: successfulReplayUrl,
|
|
961
|
-
method: "GET",
|
|
962
|
-
request_headers: headers,
|
|
963
|
-
response_status: 200,
|
|
964
|
-
response_headers: {
|
|
965
|
-
"content-type": typeof data === "string" ? "text/plain" : "application/json",
|
|
966
|
-
},
|
|
967
|
-
response_body: typeof data === "string" ? data : JSON.stringify(data),
|
|
968
|
-
timestamp: new Date().toISOString(),
|
|
969
|
-
};
|
|
970
|
-
persistWorkflowArtifactForCapture(
|
|
971
|
-
learned,
|
|
972
|
-
{
|
|
973
|
-
requests: [seededRequest],
|
|
974
|
-
har_lineage_id: `seeded:${learned.skill_id}:canonical-document`,
|
|
975
|
-
final_url: successfulReplayUrl,
|
|
976
|
-
cookies: cookies ?? [],
|
|
977
|
-
js_bundles: new Map(),
|
|
978
|
-
},
|
|
979
|
-
authHeaders,
|
|
980
|
-
);
|
|
981
|
-
|
|
982
|
-
const trace: ExecutionTrace = stampTrace({
|
|
983
|
-
trace_id: nanoid(),
|
|
984
|
-
skill_id: learned.skill_id,
|
|
985
|
-
endpoint_id: "browser-capture",
|
|
986
|
-
started_at: new Date().toISOString(),
|
|
987
|
-
completed_at: new Date().toISOString(),
|
|
988
|
-
success: true,
|
|
989
|
-
result: {
|
|
990
|
-
learned_skill_id: learned.skill_id,
|
|
991
|
-
endpoints_discovered: 1,
|
|
992
|
-
seeded_from: "canonical_document",
|
|
993
|
-
},
|
|
994
|
-
});
|
|
995
|
-
return {
|
|
996
|
-
trace,
|
|
997
|
-
result: trace.result,
|
|
998
|
-
learned_skill: learned,
|
|
999
|
-
};
|
|
1000
|
-
}
|
|
1001
|
-
|
|
1002
|
-
async function trySeedPublicDocumentFetchSkill(
|
|
1003
|
-
skill: SkillManifest,
|
|
1004
|
-
url: string,
|
|
1005
|
-
intent: string,
|
|
1006
|
-
targetDomain: string,
|
|
1007
|
-
authHeaders: Record<string, string> | undefined,
|
|
1008
|
-
cookies: Array<{ name: string; value: string; domain: string }> | undefined,
|
|
1009
|
-
usedStoredAuth: boolean,
|
|
1010
|
-
): Promise<ExecutionResult | undefined> {
|
|
1011
|
-
const headers: Record<string, string> = {
|
|
1012
|
-
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
1013
|
-
"user-agent": DEFAULT_BROWSER_UA,
|
|
1014
|
-
"accept-language": "en-US,en;q=0.9",
|
|
1015
|
-
...(authHeaders ?? {}),
|
|
1016
|
-
};
|
|
1017
|
-
if (cookies && cookies.length > 0) {
|
|
1018
|
-
headers.cookie = cookies.map((c) => {
|
|
1019
|
-
const v = c.value.startsWith('"') && c.value.endsWith('"') ? c.value.slice(1, -1) : c.value;
|
|
1020
|
-
return `${c.name}=${v}`;
|
|
1021
|
-
}).join("; ");
|
|
1022
|
-
}
|
|
1023
|
-
|
|
1024
|
-
const response = await fetch(url, {
|
|
1025
|
-
method: "GET",
|
|
1026
|
-
headers: buildStructuredReplayHeaders(url, url, headers),
|
|
1027
|
-
redirect: "follow",
|
|
1028
|
-
});
|
|
1029
|
-
const html = await response.text();
|
|
1030
|
-
if (!isHtml(html) || isSpaShell(html)) return undefined;
|
|
1031
|
-
|
|
1032
|
-
const built = buildPageArtifactCapture(response.url || url, intent, html, usedStoredAuth);
|
|
1033
|
-
if (!built.endpoint) return undefined;
|
|
1034
|
-
|
|
1035
|
-
const domain = getRegistrableDomain(targetDomain);
|
|
1036
|
-
const existingSkill = findExistingSkillForDomain(domain, intent);
|
|
1037
|
-
const localEndpoints = await prepareLearnedEndpoints(
|
|
1038
|
-
existingSkill
|
|
1039
|
-
? mergeEndpoints(existingSkill.endpoints, [built.endpoint])
|
|
1040
|
-
: [built.endpoint],
|
|
1041
|
-
intent,
|
|
1042
|
-
domain,
|
|
1043
|
-
);
|
|
1044
|
-
|
|
1045
|
-
const localDraft: SkillManifest = {
|
|
1046
|
-
skill_id: existingSkill?.skill_id ?? nanoid(),
|
|
1047
|
-
version: "1.0.0",
|
|
1048
|
-
schema_version: "1",
|
|
1049
|
-
lifecycle: "active" as const,
|
|
1050
|
-
execution_type: "http" as const,
|
|
1051
|
-
created_at: existingSkill?.created_at ?? new Date().toISOString(),
|
|
1052
|
-
updated_at: new Date().toISOString(),
|
|
1053
|
-
name: domain,
|
|
1054
|
-
intent_signature: intent,
|
|
1055
|
-
domain,
|
|
1056
|
-
description: `API skill for ${domain}`,
|
|
1057
|
-
owner_type: "agent" as const,
|
|
1058
|
-
endpoints: localEndpoints,
|
|
1059
|
-
operation_graph: buildSkillOperationGraph(localEndpoints),
|
|
1060
|
-
intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
|
|
1061
|
-
...(usedStoredAuth ? { auth_profile_ref: `${domain}-session` } : {}),
|
|
1062
|
-
};
|
|
1063
|
-
|
|
1064
|
-
let learned: SkillManifest = localDraft;
|
|
1065
|
-
const validation = await validateManifest({ ...localDraft, skill_id: "__validate__" });
|
|
1066
|
-
if (validation.valid) {
|
|
1067
|
-
try {
|
|
1068
|
-
const { operation_graph: _graph, ...publishDraft } = localDraft;
|
|
1069
|
-
const published = await publishSkill(publishDraft);
|
|
1070
|
-
learned = {
|
|
1071
|
-
...published,
|
|
1072
|
-
endpoints: localEndpoints,
|
|
1073
|
-
operation_graph: localDraft.operation_graph,
|
|
1074
|
-
};
|
|
1075
|
-
} catch {
|
|
1076
|
-
learned = localDraft;
|
|
1077
|
-
}
|
|
1078
|
-
}
|
|
1079
|
-
try { cachePublishedSkill(learned); } catch { /* best-effort */ }
|
|
1080
|
-
const seededRequest: RawRequest = {
|
|
1081
|
-
url,
|
|
1082
|
-
method: "GET",
|
|
1083
|
-
request_headers: headers,
|
|
1084
|
-
response_status: response.status,
|
|
1085
|
-
response_headers: Object.fromEntries(response.headers.entries()),
|
|
1086
|
-
response_body: html,
|
|
1087
|
-
timestamp: new Date().toISOString(),
|
|
1088
|
-
};
|
|
1089
|
-
persistWorkflowArtifactForCapture(
|
|
1090
|
-
learned,
|
|
1091
|
-
{
|
|
1092
|
-
requests: [seededRequest],
|
|
1093
|
-
har_lineage_id: `seeded:${learned.skill_id}:document-fetch`,
|
|
1094
|
-
final_url: response.url || url,
|
|
1095
|
-
html,
|
|
1096
|
-
cookies: cookies ?? [],
|
|
1097
|
-
js_bundles: new Map(),
|
|
1098
|
-
},
|
|
1099
|
-
authHeaders,
|
|
1100
|
-
);
|
|
1101
|
-
|
|
1102
|
-
const trace: ExecutionTrace = stampTrace({
|
|
1103
|
-
trace_id: nanoid(),
|
|
1104
|
-
skill_id: learned.skill_id,
|
|
1105
|
-
endpoint_id: "browser-capture",
|
|
1106
|
-
started_at: new Date().toISOString(),
|
|
1107
|
-
completed_at: new Date().toISOString(),
|
|
1108
|
-
success: true,
|
|
1109
|
-
result: {
|
|
1110
|
-
learned_skill_id: learned.skill_id,
|
|
1111
|
-
endpoints_discovered: 1,
|
|
1112
|
-
seeded_from: "document_fetch",
|
|
1113
|
-
},
|
|
1114
|
-
});
|
|
1115
|
-
return {
|
|
1116
|
-
trace,
|
|
1117
|
-
result: trace.result,
|
|
1118
|
-
learned_skill: learned,
|
|
1119
|
-
};
|
|
1120
|
-
}
|
|
1121
|
-
|
|
1122
|
-
export async function executeSkill(
|
|
1123
|
-
skill: SkillManifest,
|
|
1124
|
-
params: Record<string, unknown> = {},
|
|
1125
|
-
projection?: ProjectionOptions,
|
|
1126
|
-
options?: ExecutionOptions
|
|
1127
|
-
): Promise<ExecutionResult> {
|
|
1128
|
-
if (skill.execution_type === "browser-capture") {
|
|
1129
|
-
return executeBrowserCapture(skill, params, options);
|
|
1130
|
-
}
|
|
1131
|
-
|
|
1132
|
-
// Allow targeting a specific endpoint by ID — never silently fall back
|
|
1133
|
-
if (params.endpoint_id) {
|
|
1134
|
-
const target = skill.endpoints.find((e) => e.endpoint_id === params.endpoint_id);
|
|
1135
|
-
if (target) {
|
|
1136
|
-
const { endpoint_id: _, ...cleanParams } = params;
|
|
1137
|
-
return executeEndpoint(skill, target, cleanParams, projection, options);
|
|
1138
|
-
}
|
|
1139
|
-
// Agent explicitly chose this endpoint — don't silently swap to a different one
|
|
1140
|
-
log("exec", `endpoint ${params.endpoint_id} not found in skill ${skill.skill_id} (${skill.endpoints.length} endpoints: ${skill.endpoints.map(e => e.endpoint_id).join(", ")})`);
|
|
1141
|
-
const trace: ExecutionTrace = {
|
|
1142
|
-
trace_id: nanoid(),
|
|
1143
|
-
skill_id: skill.skill_id,
|
|
1144
|
-
endpoint_id: String(params.endpoint_id),
|
|
1145
|
-
started_at: new Date().toISOString(),
|
|
1146
|
-
completed_at: new Date().toISOString(),
|
|
1147
|
-
success: false,
|
|
1148
|
-
error: `endpoint_not_found: ${params.endpoint_id} not in skill ${skill.skill_id}`,
|
|
1149
|
-
};
|
|
1150
|
-
return {
|
|
1151
|
-
trace,
|
|
1152
|
-
result: {
|
|
1153
|
-
error: "endpoint_not_found",
|
|
1154
|
-
message: `Endpoint ${params.endpoint_id} not found in skill ${skill.skill_id}. Available: ${skill.endpoints.map(e => `${e.endpoint_id} (${e.description?.slice(0, 50)})`).join(", ")}`,
|
|
1155
|
-
available_endpoints: skill.endpoints.map(e => ({ endpoint_id: e.endpoint_id, description: e.description })),
|
|
1156
|
-
},
|
|
1157
|
-
};
|
|
1158
|
-
}
|
|
1159
|
-
|
|
1160
|
-
// Use the caller's intent for ranking when available, fall back to skill's original intent
|
|
1161
|
-
const endpoint = selectBestEndpoint(skill.endpoints, options?.intent ?? skill.intent_signature, skill.domain, options?.contextUrl);
|
|
1162
|
-
return executeEndpoint(skill, endpoint, params, projection, options);
|
|
1163
|
-
}
|
|
1164
|
-
|
|
1165
|
-
async function executeBrowserCapture(
|
|
1166
|
-
skill: SkillManifest,
|
|
1167
|
-
params: Record<string, unknown>,
|
|
1168
|
-
options?: ExecutionOptions,
|
|
1169
|
-
): Promise<ExecutionResult> {
|
|
1170
|
-
const fallbackUrl =
|
|
1171
|
-
(typeof params.context_url === "string" && params.context_url) ||
|
|
1172
|
-
skill.endpoints.find((endpoint) => typeof endpoint.trigger_url === "string" && endpoint.trigger_url)?.trigger_url ||
|
|
1173
|
-
skill.endpoints.find((endpoint) => !/\{[^}]+\}/.test(endpoint.url_template))?.url_template ||
|
|
1174
|
-
"";
|
|
1175
|
-
const url = typeof params.url === "string" ? params.url : String(params.url ?? fallbackUrl);
|
|
1176
|
-
const intent = String(params.intent ?? skill.intent_signature);
|
|
1177
|
-
if (!url) throw new Error("browser-capture skill requires params.url");
|
|
1178
|
-
|
|
1179
|
-
const startedAt = new Date().toISOString();
|
|
1180
|
-
const traceId = nanoid();
|
|
1181
|
-
const targetDomain = new URL(url).hostname;
|
|
1182
|
-
|
|
1183
|
-
// BUG-002/003 fix: auto-load vault cookies for the target domain
|
|
1184
|
-
let authHeaders = params.auth_headers as Record<string, string> | undefined;
|
|
1185
|
-
let cookies = params.cookies as Array<{ name: string; value: string; domain: string }> | undefined;
|
|
1186
|
-
let usedStoredAuth = !!(cookies && cookies.length > 0) || !!(authHeaders && Object.keys(authHeaders).length > 0);
|
|
1187
|
-
|
|
1188
|
-
// Auto-resolve cookies from vault, falling back to browser extraction
|
|
1189
|
-
if (!cookies || cookies.length === 0) {
|
|
1190
|
-
const resolved = await getAuthCookies(targetDomain, { autoExtract: true });
|
|
1191
|
-
if (resolved && resolved.length > 0) {
|
|
1192
|
-
cookies = resolved;
|
|
1193
|
-
usedStoredAuth = true;
|
|
1194
|
-
}
|
|
1195
|
-
}
|
|
1196
|
-
const seeded = await trySeedStructuredDocumentSkill(
|
|
1197
|
-
skill,
|
|
1198
|
-
url,
|
|
1199
|
-
intent,
|
|
1200
|
-
params,
|
|
1201
|
-
targetDomain,
|
|
1202
|
-
authHeaders,
|
|
1203
|
-
cookies,
|
|
1204
|
-
usedStoredAuth,
|
|
1205
|
-
);
|
|
1206
|
-
if (seeded) return seeded;
|
|
1207
|
-
const documentSeed = await trySeedPublicDocumentFetchSkill(
|
|
1208
|
-
skill,
|
|
1209
|
-
url,
|
|
1210
|
-
intent,
|
|
1211
|
-
targetDomain,
|
|
1212
|
-
authHeaders,
|
|
1213
|
-
cookies,
|
|
1214
|
-
usedStoredAuth,
|
|
1215
|
-
);
|
|
1216
|
-
if (documentSeed) return documentSeed;
|
|
1217
|
-
let captured;
|
|
1218
|
-
try {
|
|
1219
|
-
captured = await captureSession(url, authHeaders, cookies, intent);
|
|
1220
|
-
} catch (captureErr: unknown) {
|
|
1221
|
-
const err = captureErr as Error & { code?: string; login_url?: string };
|
|
1222
|
-
if (err.code === "auth_required") {
|
|
1223
|
-
const trace: ExecutionTrace = stampTrace({
|
|
1224
|
-
trace_id: traceId,
|
|
1225
|
-
skill_id: skill.skill_id,
|
|
1226
|
-
endpoint_id: "browser-capture",
|
|
1227
|
-
started_at: startedAt,
|
|
1228
|
-
completed_at: new Date().toISOString(),
|
|
1229
|
-
success: false,
|
|
1230
|
-
error: "auth_required",
|
|
1231
|
-
});
|
|
1232
|
-
return {
|
|
1233
|
-
trace,
|
|
1234
|
-
result: {
|
|
1235
|
-
error: "auth_required",
|
|
1236
|
-
provider: "cloudflare",
|
|
1237
|
-
login_url: err.login_url ?? url,
|
|
1238
|
-
message: `Site is blocked by Cloudflare WAF. Run: unbrowse login --url "${url}" to authenticate interactively.`,
|
|
1239
|
-
},
|
|
1240
|
-
};
|
|
1241
|
-
}
|
|
1242
|
-
const message = captureErr instanceof Error ? captureErr.message : String(captureErr);
|
|
1243
|
-
const normalizedError = /unable to connect/i.test(message)
|
|
1244
|
-
? "connection_failed"
|
|
1245
|
-
: /timed out/i.test(message)
|
|
1246
|
-
? "capture_timeout"
|
|
1247
|
-
: "capture_failed";
|
|
1248
|
-
const trace: ExecutionTrace = stampTrace({
|
|
1249
|
-
trace_id: traceId,
|
|
1250
|
-
skill_id: skill.skill_id,
|
|
1251
|
-
endpoint_id: "browser-capture",
|
|
1252
|
-
started_at: startedAt,
|
|
1253
|
-
completed_at: new Date().toISOString(),
|
|
1254
|
-
success: false,
|
|
1255
|
-
error: normalizedError,
|
|
1256
|
-
});
|
|
1257
|
-
return {
|
|
1258
|
-
trace,
|
|
1259
|
-
result: {
|
|
1260
|
-
error: normalizedError,
|
|
1261
|
-
message,
|
|
1262
|
-
},
|
|
1263
|
-
};
|
|
1264
|
-
}
|
|
1265
|
-
|
|
1266
|
-
const finalDomain = (() => {
|
|
1267
|
-
try { return new URL(captured.final_url).hostname; } catch { return targetDomain; }
|
|
1268
|
-
})();
|
|
1269
|
-
const AUTH_PROVIDERS = /accounts\.google\.com|login\.microsoftonline\.com|auth0\.com|cognito-idp\.|appleid\.apple\.com|github\.com|facebook\.com/i;
|
|
1270
|
-
const LOGIN_PATHS = /\/(login|signin|sign-in|sso|auth|uas\/login|checkpoint|oauth)/i;
|
|
1271
|
-
|
|
1272
|
-
const redirectedToAuth = finalDomain !== targetDomain && AUTH_PROVIDERS.test(finalDomain);
|
|
1273
|
-
const redirectedToLogin = captured.final_url !== url && (() => { try { return LOGIN_PATHS.test(new URL(String(captured.final_url)).pathname); } catch { return false; } })();
|
|
1274
|
-
|
|
1275
|
-
if (redirectedToAuth || redirectedToLogin) {
|
|
1276
|
-
const trace: ExecutionTrace = stampTrace({
|
|
1277
|
-
trace_id: traceId,
|
|
1278
|
-
skill_id: skill.skill_id,
|
|
1279
|
-
endpoint_id: "browser-capture",
|
|
1280
|
-
started_at: startedAt,
|
|
1281
|
-
completed_at: new Date().toISOString(),
|
|
1282
|
-
success: false,
|
|
1283
|
-
error: "auth_required",
|
|
1284
|
-
});
|
|
1285
|
-
return {
|
|
1286
|
-
trace,
|
|
1287
|
-
result: {
|
|
1288
|
-
error: "auth_required",
|
|
1289
|
-
provider: getRegistrableDomain(finalDomain),
|
|
1290
|
-
login_url: captured.final_url,
|
|
1291
|
-
message: `Site requires authentication. Call POST /v1/auth/login with {"url": "${captured.final_url}"} to log in interactively, or pass cookies via params.cookies / headers via params.auth_headers.`,
|
|
1292
|
-
},
|
|
1293
|
-
};
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
|
-
const endpoints = extractEndpoints(captured.requests, captured.ws_messages, { pageUrl: url, finalUrl: captured.final_url, intent });
|
|
1297
|
-
|
|
1298
|
-
// Detect structured search forms from captured HTML and attach to search-like endpoints
|
|
1299
|
-
if (captured.html) {
|
|
1300
|
-
const detectedForms = detectSearchForms(captured.html);
|
|
1301
|
-
if (detectedForms.length > 0) {
|
|
1302
|
-
for (const ep of endpoints) {
|
|
1303
|
-
if (!ep.search_form && ep.method === "GET") {
|
|
1304
|
-
const matchingForm = detectedForms.find((f) => isStructuredSearchForm(f));
|
|
1305
|
-
if (matchingForm) {
|
|
1306
|
-
ep.search_form = matchingForm;
|
|
1307
|
-
break; // attach the best form to the first search-like GET endpoint
|
|
1308
|
-
}
|
|
1309
|
-
}
|
|
1310
|
-
}
|
|
1311
|
-
}
|
|
1312
|
-
}
|
|
1313
|
-
|
|
1314
|
-
// JS bundle scanning: discover API routes not seen in network traffic
|
|
1315
|
-
if (captured.js_bundles && captured.js_bundles.size > 0) {
|
|
1316
|
-
const pageOrigin = new URL(url).origin;
|
|
1317
|
-
const bundleRoutes = scanBundlesForRoutes(captured.js_bundles, pageOrigin);
|
|
1318
|
-
|
|
1319
|
-
// Build set of already-discovered URL paths for deduplication
|
|
1320
|
-
const networkPaths = new Set<string>();
|
|
1321
|
-
for (const ep of endpoints) {
|
|
1322
|
-
try {
|
|
1323
|
-
const normalized = new URL(ep.url_template).pathname
|
|
1324
|
-
.replace(/\{[^}]+\}/g, "*")
|
|
1325
|
-
.replace(/\/+$/, "");
|
|
1326
|
-
networkPaths.add(normalized);
|
|
1327
|
-
} catch { /* skip */ }
|
|
1328
|
-
}
|
|
1329
|
-
|
|
1330
|
-
let added = 0;
|
|
1331
|
-
for (const route of bundleRoutes) {
|
|
1332
|
-
const normalized = route.path.replace(/\/+$/, "");
|
|
1333
|
-
if (networkPaths.has(normalized)) continue;
|
|
1334
|
-
|
|
1335
|
-
// Check if a network endpoint's wildcard pattern matches this route
|
|
1336
|
-
let isDup = false;
|
|
1337
|
-
for (const np of networkPaths) {
|
|
1338
|
-
if (np.includes("*")) {
|
|
1339
|
-
const re = new RegExp("^" + np.replace(/\*/g, "[^/]+") + "$");
|
|
1340
|
-
if (re.test(normalized)) { isDup = true; break; }
|
|
1341
|
-
}
|
|
1342
|
-
}
|
|
1343
|
-
if (isDup) continue;
|
|
1344
|
-
|
|
1345
|
-
// Build query template from bundle-inferred param names
|
|
1346
|
-
let epUrl = route.url;
|
|
1347
|
-
let epQuery: Record<string, unknown> | undefined;
|
|
1348
|
-
let queryParamNames = route.query_params ? [...route.query_params] : [];
|
|
1349
|
-
if (queryParamNames.length === 0) {
|
|
1350
|
-
try {
|
|
1351
|
-
const triggerUrl = new URL(url);
|
|
1352
|
-
const triggerParams = [...triggerUrl.searchParams.keys()].filter((k) =>
|
|
1353
|
-
/^(q|query|search|term|type|tag|sort|page)$/i.test(k)
|
|
1354
|
-
);
|
|
1355
|
-
if (triggerParams.length > 0 && /\/(search|lookup|find)\b/i.test(route.path)) {
|
|
1356
|
-
queryParamNames = triggerParams;
|
|
1357
|
-
}
|
|
1358
|
-
} catch { /* skip */ }
|
|
1359
|
-
}
|
|
1360
|
-
if (queryParamNames.length > 0) {
|
|
1361
|
-
epQuery = {};
|
|
1362
|
-
for (const p of queryParamNames) epQuery[p] = "";
|
|
1363
|
-
const qStr = queryParamNames.map((k) => `${encodeURIComponent(k)}={${k}}`).join("&");
|
|
1364
|
-
epUrl = `${route.url}?${qStr}`;
|
|
1365
|
-
}
|
|
1366
|
-
|
|
1367
|
-
endpoints.push({
|
|
1368
|
-
endpoint_id: nanoid(),
|
|
1369
|
-
method: "GET",
|
|
1370
|
-
url_template: epUrl,
|
|
1371
|
-
query: epQuery,
|
|
1372
|
-
idempotency: "safe",
|
|
1373
|
-
verification_status: "pending",
|
|
1374
|
-
reliability_score: 0.2,
|
|
1375
|
-
description: `Inferred from JS bundle (${route.match_type}). Not observed in network traffic.`,
|
|
1376
|
-
trigger_url: url,
|
|
1377
|
-
});
|
|
1378
|
-
added++;
|
|
1379
|
-
networkPaths.add(normalized);
|
|
1380
|
-
}
|
|
1381
|
-
|
|
1382
|
-
if (added > 0) {
|
|
1383
|
-
log("execution", `added ${added} inferred endpoints from JS bundle scanning`);
|
|
1384
|
-
}
|
|
1385
|
-
}
|
|
1386
|
-
|
|
1387
|
-
const cleanEndpoints = endpoints.filter((ep) => {
|
|
1388
|
-
try {
|
|
1389
|
-
const host = new URL(ep.url_template).hostname;
|
|
1390
|
-
return !AUTH_PROVIDERS.test(host) && !LOGIN_PATHS.test(new URL(ep.url_template).pathname);
|
|
1391
|
-
} catch { return true; }
|
|
1392
|
-
});
|
|
1393
|
-
|
|
1394
|
-
const domain = captured.domain;
|
|
1395
|
-
|
|
1396
|
-
// Persist session cookies + auth headers so server-fetch works without browser.
|
|
1397
|
-
// extractAuthHeaders collects everything sanitizeHeaders strips from skill manifests
|
|
1398
|
-
// (authorization, x-csrf-token, api keys, etc.) — stored encrypted in vault.
|
|
1399
|
-
let auth_profile_ref: string | undefined;
|
|
1400
|
-
const capturedAuthHeaders = extractAuthHeaders(captured.requests);
|
|
1401
|
-
|
|
1402
|
-
if ((captured.cookies && captured.cookies.length > 0) || Object.keys(capturedAuthHeaders).length > 0) {
|
|
1403
|
-
auth_profile_ref = `${domain}-session`;
|
|
1404
|
-
await storeCredential(auth_profile_ref, JSON.stringify({
|
|
1405
|
-
cookies: captured.cookies ?? [],
|
|
1406
|
-
headers: Object.keys(capturedAuthHeaders).length > 0 ? capturedAuthHeaders : undefined,
|
|
1407
|
-
}));
|
|
1408
|
-
}
|
|
1409
|
-
|
|
1410
|
-
// BUG-004 fix: set auth_profile_ref when vault has stored auth for this domain
|
|
1411
|
-
if (!auth_profile_ref) {
|
|
1412
|
-
const vaultKey = `auth:${targetDomain}`;
|
|
1413
|
-
const hasStoredAuth = (await getCredential(vaultKey)) != null;
|
|
1414
|
-
if (hasStoredAuth) auth_profile_ref = vaultKey;
|
|
1415
|
-
}
|
|
1416
|
-
const authBackedCapture = usedStoredAuth || !!auth_profile_ref;
|
|
1417
|
-
if (authBackedCapture) {
|
|
1418
|
-
for (const endpoint of cleanEndpoints) {
|
|
1419
|
-
endpoint.semantic = {
|
|
1420
|
-
...(endpoint.semantic ?? {}),
|
|
1421
|
-
action_kind: endpoint.semantic?.action_kind ?? "fetch",
|
|
1422
|
-
resource_kind: endpoint.semantic?.resource_kind ?? "resource",
|
|
1423
|
-
auth_required: true,
|
|
1424
|
-
};
|
|
1425
|
-
}
|
|
1426
|
-
}
|
|
1427
|
-
|
|
1428
|
-
const canonicalDocumentEndpoint = buildCanonicalDocumentEndpoint(url, intent, authBackedCapture);
|
|
1429
|
-
if (
|
|
1430
|
-
canonicalDocumentEndpoint &&
|
|
1431
|
-
!cleanEndpoints.some((endpoint) => endpoint.method === canonicalDocumentEndpoint.method && endpoint.url_template === canonicalDocumentEndpoint.url_template)
|
|
1432
|
-
) {
|
|
1433
|
-
cleanEndpoints.push(canonicalDocumentEndpoint);
|
|
1434
|
-
}
|
|
1435
|
-
|
|
1436
|
-
const pageArtifact = captured.html
|
|
1437
|
-
? buildPageArtifactCapture(url, intent, captured.html, authBackedCapture)
|
|
1438
|
-
: {};
|
|
1439
|
-
const domArtifactEndpoint = pageArtifact.endpoint;
|
|
1440
|
-
const domArtifactResult = pageArtifact.result;
|
|
1441
|
-
const inferredOnlyCapture = cleanEndpoints.length > 0 && cleanEndpoints.every((endpoint) => isBundleInferredEndpoint(endpoint));
|
|
1442
|
-
const hasSupportEvidence = cleanEndpoints.some((endpoint) => isSupportEvidenceEndpoint(endpoint)) || !!domArtifactEndpoint;
|
|
1443
|
-
|
|
1444
|
-
if (inferredOnlyCapture && !hasSupportEvidence) {
|
|
1445
|
-
const trace: ExecutionTrace = stampTrace({
|
|
1446
|
-
trace_id: traceId,
|
|
1447
|
-
skill_id: skill.skill_id,
|
|
1448
|
-
endpoint_id: "browser-capture",
|
|
1449
|
-
started_at: startedAt,
|
|
1450
|
-
completed_at: new Date().toISOString(),
|
|
1451
|
-
success: false,
|
|
1452
|
-
error: "bundle_routes_only",
|
|
1453
|
-
});
|
|
1454
|
-
return {
|
|
1455
|
-
trace,
|
|
1456
|
-
result: {
|
|
1457
|
-
error: "no_endpoints",
|
|
1458
|
-
message: `Only bundle-inferred routes were found at ${url}; no observed API responses or structured DOM data were validated.`,
|
|
1459
|
-
},
|
|
1460
|
-
};
|
|
1461
|
-
}
|
|
1462
|
-
|
|
1463
|
-
if (cleanEndpoints.length === 0) {
|
|
1464
|
-
// DOM fallback: extract structured data from rendered page, learn a DOM skill
|
|
1465
|
-
if (domArtifactEndpoint && domArtifactResult) {
|
|
1466
|
-
const existingDomSkill = findExistingSkillForDomain(domain, intent);
|
|
1467
|
-
const domEndpoints = await prepareLearnedEndpoints(
|
|
1468
|
-
existingDomSkill
|
|
1469
|
-
? mergeEndpoints(existingDomSkill.endpoints, [domArtifactEndpoint])
|
|
1470
|
-
: [domArtifactEndpoint],
|
|
1471
|
-
intent,
|
|
1472
|
-
domain,
|
|
1473
|
-
);
|
|
1474
|
-
const domDraft: SkillManifest = {
|
|
1475
|
-
skill_id: existingDomSkill?.skill_id ?? nanoid(),
|
|
1476
|
-
version: "1.0.0",
|
|
1477
|
-
schema_version: "1",
|
|
1478
|
-
lifecycle: "active" as const,
|
|
1479
|
-
execution_type: "http" as const,
|
|
1480
|
-
created_at: new Date().toISOString(),
|
|
1481
|
-
updated_at: new Date().toISOString(),
|
|
1482
|
-
name: domain,
|
|
1483
|
-
intent_signature: intent,
|
|
1484
|
-
domain,
|
|
1485
|
-
description: `API skill for ${domain}`,
|
|
1486
|
-
owner_type: "agent" as const,
|
|
1487
|
-
endpoints: domEndpoints,
|
|
1488
|
-
operation_graph: buildSkillOperationGraph(domEndpoints),
|
|
1489
|
-
intents: Array.from(new Set([...(existingDomSkill?.intents ?? []), intent])),
|
|
1490
|
-
...(auth_profile_ref ? { auth_profile_ref } : {}),
|
|
1491
|
-
};
|
|
1492
|
-
|
|
1493
|
-
// Only publish to marketplace if quality passes
|
|
1494
|
-
let learned: SkillManifest | undefined = domDraft;
|
|
1495
|
-
try {
|
|
1496
|
-
const validation = await validateManifest({ ...domDraft, skill_id: "__validate__" });
|
|
1497
|
-
if (validation.valid) {
|
|
1498
|
-
learned = await publishSkill(domDraft);
|
|
1499
|
-
}
|
|
1500
|
-
} catch { /* publish failure is non-fatal */ }
|
|
1501
|
-
if (learned) {
|
|
1502
|
-
try { cachePublishedSkill(learned, options?.client_scope); } catch { /* local cache best-effort */ }
|
|
1503
|
-
persistWorkflowArtifactForCapture(learned, captured, capturedAuthHeaders);
|
|
1504
|
-
}
|
|
1505
|
-
|
|
1506
|
-
const trace: ExecutionTrace = stampTrace({
|
|
1507
|
-
trace_id: traceId,
|
|
1508
|
-
skill_id: learned?.skill_id ?? skill.skill_id,
|
|
1509
|
-
endpoint_id: domArtifactEndpoint.endpoint_id,
|
|
1510
|
-
started_at: startedAt,
|
|
1511
|
-
completed_at: new Date().toISOString(),
|
|
1512
|
-
success: true,
|
|
1513
|
-
result: domArtifactResult.data,
|
|
1514
|
-
});
|
|
1515
|
-
// Always return data to the caller — quality gate only blocks publishing
|
|
1516
|
-
return {
|
|
1517
|
-
trace,
|
|
1518
|
-
result: domArtifactResult,
|
|
1519
|
-
learned_skill: learned,
|
|
1520
|
-
};
|
|
1521
|
-
}
|
|
1522
|
-
|
|
1523
|
-
if (pageArtifact.quality_note) {
|
|
1524
|
-
const trace: ExecutionTrace = stampTrace({
|
|
1525
|
-
trace_id: traceId,
|
|
1526
|
-
skill_id: skill.skill_id,
|
|
1527
|
-
endpoint_id: "browser-capture",
|
|
1528
|
-
started_at: startedAt,
|
|
1529
|
-
completed_at: new Date().toISOString(),
|
|
1530
|
-
success: false,
|
|
1531
|
-
error: pageArtifact.quality_note,
|
|
1532
|
-
});
|
|
1533
|
-
return {
|
|
1534
|
-
trace,
|
|
1535
|
-
result: {
|
|
1536
|
-
error: "low_quality_dom_extraction",
|
|
1537
|
-
message: `Structured DOM extraction was rejected for ${url}: ${pageArtifact.quality_note}`,
|
|
1538
|
-
},
|
|
1539
|
-
};
|
|
1540
|
-
}
|
|
1541
|
-
|
|
1542
|
-
const trace: ExecutionTrace = stampTrace({
|
|
1543
|
-
trace_id: traceId,
|
|
1544
|
-
skill_id: skill.skill_id,
|
|
1545
|
-
endpoint_id: "browser-capture",
|
|
1546
|
-
started_at: startedAt,
|
|
1547
|
-
completed_at: new Date().toISOString(),
|
|
1548
|
-
success: false,
|
|
1549
|
-
error: "no_endpoints",
|
|
1550
|
-
});
|
|
1551
|
-
return {
|
|
1552
|
-
trace,
|
|
1553
|
-
result: {
|
|
1554
|
-
error: "no_endpoints",
|
|
1555
|
-
message: `No API endpoints or structured DOM data found at ${url}. The site may require authentication or may not expose machine-readable data from this page.`,
|
|
1556
|
-
},
|
|
1557
|
-
};
|
|
1558
|
-
}
|
|
1559
|
-
|
|
1560
|
-
// Reuse existing skill for this domain to preserve skill_id and learned exec_strategy.
|
|
1561
|
-
// This prevents duplicate skills accumulating in the marketplace on re-captures.
|
|
1562
|
-
const existingSkill = findExistingSkillForDomain(domain, intent);
|
|
1563
|
-
if (existingSkill) {
|
|
1564
|
-
// Carry forward learned exec_strategy from old endpoints to matching new ones
|
|
1565
|
-
for (const ep of cleanEndpoints) {
|
|
1566
|
-
if (ep.exec_strategy) continue;
|
|
1567
|
-
// Match by URL template (endpoint_id changes on re-capture)
|
|
1568
|
-
const oldMatch = existingSkill.endpoints.find(
|
|
1569
|
-
(old) => old.url_template === ep.url_template && old.method === ep.method
|
|
1570
|
-
);
|
|
1571
|
-
if (oldMatch?.exec_strategy) {
|
|
1572
|
-
ep.exec_strategy = oldMatch.exec_strategy;
|
|
1573
|
-
}
|
|
1574
|
-
}
|
|
1575
|
-
}
|
|
1576
|
-
|
|
1577
|
-
// Keep all captured endpoints locally so the resolver can use WS-backed skills,
|
|
1578
|
-
// but only publish HTTP endpoints until backend validation supports WS manifests.
|
|
1579
|
-
const learnedEndpoints = domArtifactEndpoint
|
|
1580
|
-
? [...cleanEndpoints, domArtifactEndpoint]
|
|
1581
|
-
: cleanEndpoints;
|
|
1582
|
-
const localEndpoints = await prepareLearnedEndpoints(
|
|
1583
|
-
existingSkill
|
|
1584
|
-
? mergeEndpoints(existingSkill.endpoints, learnedEndpoints)
|
|
1585
|
-
: learnedEndpoints,
|
|
1586
|
-
intent,
|
|
1587
|
-
domain,
|
|
1588
|
-
);
|
|
1589
|
-
const publishableEndpoints = localEndpoints.filter((ep) => ep.method !== "WS");
|
|
1590
|
-
|
|
1591
|
-
const localDraft: SkillManifest = {
|
|
1592
|
-
skill_id: existingSkill?.skill_id ?? nanoid(),
|
|
1593
|
-
version: "1.0.0",
|
|
1594
|
-
schema_version: "1",
|
|
1595
|
-
lifecycle: "active" as const,
|
|
1596
|
-
execution_type: "http" as const,
|
|
1597
|
-
created_at: existingSkill?.created_at ?? new Date().toISOString(),
|
|
1598
|
-
updated_at: new Date().toISOString(),
|
|
1599
|
-
name: domain,
|
|
1600
|
-
intent_signature: intent,
|
|
1601
|
-
domain,
|
|
1602
|
-
description: `API skill for ${domain}`,
|
|
1603
|
-
owner_type: "agent" as const,
|
|
1604
|
-
endpoints: localEndpoints,
|
|
1605
|
-
intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
|
|
1606
|
-
...(auth_profile_ref ? { auth_profile_ref } : {}),
|
|
1607
|
-
};
|
|
1608
|
-
// Generate local descriptions immediately so BM25 ranking works on first cache hit
|
|
1609
|
-
for (const ep of localDraft.endpoints) {
|
|
1610
|
-
if (!ep.description) {
|
|
1611
|
-
ep.description = generateLocalDescription(ep);
|
|
1612
|
-
}
|
|
1613
|
-
}
|
|
1614
|
-
|
|
1615
|
-
// PHASE 2: Write local cache IMMEDIATELY (~1ms) — populates cache before auto-exec
|
|
1616
|
-
const bgCacheKey = buildResolveCacheKey(domain, intent, url);
|
|
1617
|
-
const bgScopedKey = scopedCacheKey(options?.client_scope ?? "global", bgCacheKey);
|
|
1618
|
-
writeSkillSnapshot(bgScopedKey, localDraft);
|
|
1619
|
-
const bgDomainKey = getDomainReuseKey(url ?? domain);
|
|
1620
|
-
if (bgDomainKey) {
|
|
1621
|
-
domainSkillCache.set(bgDomainKey, {
|
|
1622
|
-
skillId: localDraft.skill_id,
|
|
1623
|
-
localSkillPath: snapshotPathForCacheKey(bgScopedKey),
|
|
1624
|
-
ts: Date.now(),
|
|
1625
|
-
});
|
|
1626
|
-
persistDomainCache();
|
|
1627
|
-
}
|
|
1628
|
-
|
|
1629
|
-
// PHASE 2: Queue heavy work for background (graph + validate + publish)
|
|
1630
|
-
queueBackgroundIndex({
|
|
1631
|
-
skill: { ...localDraft },
|
|
1632
|
-
domain,
|
|
1633
|
-
intent,
|
|
1634
|
-
contextUrl: url,
|
|
1635
|
-
clientScope: options?.client_scope,
|
|
1636
|
-
cacheKey: bgCacheKey,
|
|
1637
|
-
});
|
|
1638
|
-
|
|
1639
|
-
// Return the local draft as learned_skill — no blocking on marketplace publish
|
|
1640
|
-
let learned: SkillManifest = localDraft;
|
|
1641
|
-
try { cachePublishedSkill(localDraft, options?.client_scope); } catch { /* best-effort */ }
|
|
1642
|
-
persistWorkflowArtifactForCapture(localDraft, captured, capturedAuthHeaders);
|
|
1643
|
-
|
|
1644
|
-
// Attribute lifecycle phases for this capture-to-publish flow
|
|
1645
|
-
const completedAt = new Date().toISOString();
|
|
1646
|
-
const captureDurationMs = new Date(completedAt).getTime() - new Date(startedAt).getTime();
|
|
1647
|
-
const lifecycleEvents: LifecycleEvent[] = [
|
|
1648
|
-
{ phase: "capture", skill_id: learned.skill_id, timestamp: startedAt, duration_ms: captureDurationMs, source: "live-capture" },
|
|
1649
|
-
{ phase: "publish", skill_id: learned.skill_id, timestamp: completedAt, duration_ms: 0, source: publishableEndpoints.length > 0 ? "marketplace" : "cache" },
|
|
1650
|
-
];
|
|
1651
|
-
const lifecycleAttribution = attributeLifecycle(lifecycleEvents);
|
|
1652
|
-
log("execution", `lifecycle attribution: capture=${lifecycleAttribution.get("capture") ?? 0}ms, publish=${lifecycleAttribution.get("publish") ?? 0}ms`);
|
|
1653
|
-
|
|
1654
|
-
const trace: ExecutionTrace = stampTrace({
|
|
1655
|
-
trace_id: traceId,
|
|
1656
|
-
skill_id: learned.skill_id,
|
|
1657
|
-
endpoint_id: "browser-capture",
|
|
1658
|
-
started_at: startedAt,
|
|
1659
|
-
completed_at: completedAt,
|
|
1660
|
-
success: true,
|
|
1661
|
-
result: { learned_skill_id: learned.skill_id, endpoints_discovered: cleanEndpoints.length },
|
|
1662
|
-
});
|
|
1663
|
-
|
|
1664
|
-
// Detect tracking-only capture: all endpoints lack a response_schema, meaning no real
|
|
1665
|
-
// JSON data was returned — the site likely gated its API behind authentication.
|
|
1666
|
-
// Only flag this when no auth was used (so a retry with auth has a chance of succeeding).
|
|
1667
|
-
const hasMeaningfulEndpoint = cleanEndpoints.some((ep) => isSupportEvidenceEndpoint(ep));
|
|
1668
|
-
const authRecommended = !usedStoredAuth && !hasMeaningfulEndpoint && !inferredOnlyCapture;
|
|
1669
|
-
|
|
1670
|
-
return {
|
|
1671
|
-
trace,
|
|
1672
|
-
result: {
|
|
1673
|
-
...(trace.result as Record<string, unknown>),
|
|
1674
|
-
...(authRecommended ? {
|
|
1675
|
-
auth_recommended: true,
|
|
1676
|
-
auth_hint: `No data endpoints found — ${domain} likely requires authentication. ` +
|
|
1677
|
-
`Store browser cookies for this domain via the auth endpoints, then retry this capture.`,
|
|
1678
|
-
} : {}),
|
|
1679
|
-
},
|
|
1680
|
-
learned_skill: learned,
|
|
1681
|
-
};
|
|
1682
|
-
}
|
|
1683
|
-
|
|
1684
|
-
async function tryHttpFetch(
|
|
1685
|
-
url: string,
|
|
1686
|
-
authHeaders: Record<string, string>,
|
|
1687
|
-
cookies: Array<{ name: string; value: string; domain: string }>,
|
|
1688
|
-
): Promise<{ html: string; final_url: string } | null> {
|
|
1689
|
-
try {
|
|
1690
|
-
const headers: Record<string, string> = {
|
|
1691
|
-
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
1692
|
-
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
1693
|
-
"Accept-Language": "en-US,en;q=0.9",
|
|
1694
|
-
"Cache-Control": "no-cache",
|
|
1695
|
-
...authHeaders,
|
|
1696
|
-
};
|
|
1697
|
-
if (cookies && cookies.length > 0) {
|
|
1698
|
-
headers["Cookie"] = cookies.map(c => `${c.name}=${c.value}`).join("; ");
|
|
1699
|
-
}
|
|
1700
|
-
const controller = new AbortController();
|
|
1701
|
-
const timeout = setTimeout(() => controller.abort(), 10_000);
|
|
1702
|
-
const res = await fetch(url, {
|
|
1703
|
-
headers,
|
|
1704
|
-
signal: controller.signal,
|
|
1705
|
-
redirect: "follow",
|
|
1706
|
-
});
|
|
1707
|
-
clearTimeout(timeout);
|
|
1708
|
-
if (res.status !== 200) return null;
|
|
1709
|
-
const ct = res.headers.get("content-type") ?? "";
|
|
1710
|
-
if (!ct.includes("text/html") && !ct.includes("application/xhtml")) return null;
|
|
1711
|
-
const html = await res.text();
|
|
1712
|
-
if (!html || html.length < 1024) return null;
|
|
1713
|
-
return { html, final_url: res.url || url };
|
|
1714
|
-
} catch {
|
|
1715
|
-
return null;
|
|
1716
|
-
}
|
|
1717
|
-
}
|
|
1718
|
-
|
|
1719
|
-
/** When extraction returns "multiple" candidates, pick the best one's data to avoid duplicates */
|
|
1720
|
-
function flattenExtracted(data: unknown): unknown {
|
|
1721
|
-
if (!Array.isArray(data)) return data;
|
|
1722
|
-
const first = data[0];
|
|
1723
|
-
if (first && typeof first === "object" && "type" in first && "data" in first && "relevance_score" in first) {
|
|
1724
|
-
return data.reduce((best: any, cur: any) => (cur.relevance_score ?? 0) > (best.relevance_score ?? 0) ? cur : best).data;
|
|
1725
|
-
}
|
|
1726
|
-
return data;
|
|
1727
|
-
}
|
|
1728
|
-
|
|
1729
|
-
async function executeDomExtractionEndpoint(
|
|
1730
|
-
endpoint: EndpointDescriptor,
|
|
1731
|
-
url: string,
|
|
1732
|
-
intent: string,
|
|
1733
|
-
authHeaders: Record<string, string>,
|
|
1734
|
-
cookies: Array<{ name: string; value: string; domain: string }>,
|
|
1735
|
-
): Promise<{ data: unknown; status: number; trace_id: string }> {
|
|
1736
|
-
// SSR fast-path: try plain HTTP fetch before browser
|
|
1737
|
-
const ssrResult = await tryHttpFetch(url, authHeaders, cookies);
|
|
1738
|
-
if (ssrResult) {
|
|
1739
|
-
const ssrExtracted = extractFromDOMWithHint(ssrResult.html, intent, endpoint.dom_extraction);
|
|
1740
|
-
if (ssrExtracted.data) {
|
|
1741
|
-
const ssrQuality = validateExtractionQuality(ssrExtracted.data, ssrExtracted.confidence, intent);
|
|
1742
|
-
if (ssrQuality.valid) {
|
|
1743
|
-
const ssrSemantic = assessIntentResult(ssrExtracted.data, intent);
|
|
1744
|
-
if (ssrSemantic.verdict !== "fail") {
|
|
1745
|
-
console.log(`[ssr-fast] hit — extracted via HTTP fetch`);
|
|
1746
|
-
return {
|
|
1747
|
-
data: flattenExtracted(ssrExtracted.data),
|
|
1748
|
-
status: 200,
|
|
1749
|
-
trace_id: nanoid(),
|
|
1750
|
-
};
|
|
1751
|
-
}
|
|
1752
|
-
}
|
|
1753
|
-
}
|
|
1754
|
-
console.log(`[ssr-fast] miss, falling back to browser`);
|
|
1755
|
-
} else {
|
|
1756
|
-
console.log(`[ssr-fast] miss, falling back to browser`);
|
|
1757
|
-
}
|
|
1758
|
-
|
|
1759
|
-
// Browser fallback — captures both intercepted API requests AND page HTML
|
|
1760
|
-
const captured = await captureSession(url, authHeaders, cookies, intent);
|
|
1761
|
-
|
|
1762
|
-
// Check intercepted requests first — if the site's JS made API calls,
|
|
1763
|
-
// those have the actual filtered data (not the initial HTML page load)
|
|
1764
|
-
if (captured.requests.length > 0) {
|
|
1765
|
-
const { extractEndpoints: extractEps } = await import("../reverse-engineer/index.js");
|
|
1766
|
-
const apiEndpoints = extractEps(captured.requests, undefined, { pageUrl: url, finalUrl: captured.final_url });
|
|
1767
|
-
const jsonEndpoints = apiEndpoints.filter(ep => ep.response_schema && !ep.dom_extraction);
|
|
1768
|
-
if (jsonEndpoints.length > 0) {
|
|
1769
|
-
// Found real API responses — return the best one's data
|
|
1770
|
-
const best = jsonEndpoints[0];
|
|
1771
|
-
const matchingReq = captured.requests.find(r =>
|
|
1772
|
-
r.url.includes(best.url_template.split("?")[0].split("{")[0]) &&
|
|
1773
|
-
r.response_body && r.response_status >= 200 && r.response_status < 400
|
|
1774
|
-
);
|
|
1775
|
-
if (matchingReq?.response_body) {
|
|
1776
|
-
try {
|
|
1777
|
-
const data = JSON.parse(matchingReq.response_body);
|
|
1778
|
-
console.log(`[dom-exec] found API response from browser capture: ${matchingReq.url.substring(0, 80)}`);
|
|
1779
|
-
return { data, status: matchingReq.response_status, trace_id: nanoid() };
|
|
1780
|
-
} catch { /* not JSON, fall through to DOM extraction */ }
|
|
1781
|
-
}
|
|
1782
|
-
}
|
|
1783
|
-
}
|
|
1784
|
-
|
|
1785
|
-
// Fall back to DOM extraction from rendered HTML
|
|
1786
|
-
const html = captured.html ?? "";
|
|
1787
|
-
const extracted = extractFromDOMWithHint(html, intent, endpoint.dom_extraction);
|
|
1788
|
-
if (extracted.data) {
|
|
1789
|
-
const quality = validateExtractionQuality(extracted.data, extracted.confidence, intent);
|
|
1790
|
-
if (!quality.valid) {
|
|
1791
|
-
return {
|
|
1792
|
-
data: {
|
|
1793
|
-
error: "low_quality_dom_extraction",
|
|
1794
|
-
message: `Structured DOM extraction was rejected: ${quality.quality_note ?? "low quality extraction"}`,
|
|
1795
|
-
},
|
|
1796
|
-
status: 422,
|
|
1797
|
-
trace_id: nanoid(),
|
|
1798
|
-
};
|
|
1799
|
-
}
|
|
1800
|
-
const semanticAssessment = assessIntentResult(extracted.data, intent);
|
|
1801
|
-
if (semanticAssessment.verdict === "fail") {
|
|
1802
|
-
return {
|
|
1803
|
-
data: {
|
|
1804
|
-
error: "low_quality_dom_extraction",
|
|
1805
|
-
message: `Structured DOM extraction was rejected: ${semanticAssessment.reason}`,
|
|
1806
|
-
},
|
|
1807
|
-
status: 422,
|
|
1808
|
-
trace_id: nanoid(),
|
|
1809
|
-
};
|
|
1810
|
-
}
|
|
1811
|
-
return {
|
|
1812
|
-
data: flattenExtracted(extracted.data),
|
|
1813
|
-
status: 200,
|
|
1814
|
-
trace_id: nanoid(),
|
|
1815
|
-
};
|
|
1816
|
-
}
|
|
1817
|
-
return {
|
|
1818
|
-
data: html,
|
|
1819
|
-
status: 200,
|
|
1820
|
-
trace_id: nanoid(),
|
|
1821
|
-
};
|
|
1822
|
-
}
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
export async function executeEndpoint(
|
|
1826
|
-
skill: SkillManifest,
|
|
1827
|
-
endpoint: EndpointDescriptor,
|
|
1828
|
-
params: Record<string, unknown> = {},
|
|
1829
|
-
projection?: ProjectionOptions,
|
|
1830
|
-
options?: ExecutionOptions
|
|
1831
|
-
): Promise<ExecutionResult> {
|
|
1832
|
-
endpoint = annotateEndpointPolicy(endpoint);
|
|
1833
|
-
const workflowArtifact = readWorkflowArtifact(skill.skill_id);
|
|
1834
|
-
const workflowRecipe = pickWorkflowRecipe(workflowArtifact, endpoint.endpoint_id);
|
|
1835
|
-
const reservedMetaParams = new Set(["endpoint_id", "url", "context_url", "intent"]);
|
|
1836
|
-
// WebSocket endpoint: connect, collect messages, return
|
|
1837
|
-
if (endpoint.method === "WS") {
|
|
1838
|
-
const startedAt = new Date().toISOString();
|
|
1839
|
-
const traceId = nanoid();
|
|
1840
|
-
try {
|
|
1841
|
-
const { WebSocket } = await import("ws");
|
|
1842
|
-
const messages: string[] = [];
|
|
1843
|
-
await new Promise<void>((resolve, reject) => {
|
|
1844
|
-
const ws = new WebSocket(endpoint.url_template);
|
|
1845
|
-
const timeout = setTimeout(() => { ws.close(); resolve(); }, 7000);
|
|
1846
|
-
ws.on("message", (data: Buffer | string) => {
|
|
1847
|
-
messages.push(data.toString());
|
|
1848
|
-
});
|
|
1849
|
-
ws.on("error", (err: Error) => { clearTimeout(timeout); reject(err); });
|
|
1850
|
-
ws.on("close", () => { clearTimeout(timeout); resolve(); });
|
|
1851
|
-
});
|
|
1852
|
-
const parsed = messages.map((m) => { try { return JSON.parse(m); } catch { return m; } });
|
|
1853
|
-
const trace: ExecutionTrace = stampTrace({
|
|
1854
|
-
trace_id: traceId, skill_id: skill.skill_id, endpoint_id: endpoint.endpoint_id,
|
|
1855
|
-
started_at: startedAt, completed_at: new Date().toISOString(), success: true, result: parsed,
|
|
1856
|
-
});
|
|
1857
|
-
let resultData: unknown = parsed;
|
|
1858
|
-
if (projection?.raw) {
|
|
1859
|
-
// Explicit raw — skip projection
|
|
1860
|
-
} else if (projection) {
|
|
1861
|
-
resultData = applyProjection(parsed, projection);
|
|
1862
|
-
}
|
|
1863
|
-
return {
|
|
1864
|
-
trace, result: resultData,
|
|
1865
|
-
};
|
|
1866
|
-
} catch (err) {
|
|
1867
|
-
const trace: ExecutionTrace = stampTrace({
|
|
1868
|
-
trace_id: traceId, skill_id: skill.skill_id, endpoint_id: endpoint.endpoint_id,
|
|
1869
|
-
started_at: startedAt, completed_at: new Date().toISOString(), success: false,
|
|
1870
|
-
error: String(err),
|
|
1871
|
-
});
|
|
1872
|
-
return { trace, result: { error: String(err) } };
|
|
1873
|
-
}
|
|
1874
|
-
}
|
|
1875
|
-
|
|
1876
|
-
// Payment gate — check if marketplace skill requires payment before executing
|
|
1877
|
-
if (!skill.skill_id.startsWith("local:") && skill.execution_type === "http" && skill.owner_type !== "agent") {
|
|
1878
|
-
const wallet = getLocalWalletContext();
|
|
1879
|
-
const gate = await checkPaymentRequirement(skill.skill_id, endpoint.endpoint_id, {
|
|
1880
|
-
wallet_configured: !!wallet.wallet_address,
|
|
1881
|
-
});
|
|
1882
|
-
if (gate.status === "payment_required" || gate.status === "wallet_not_configured" || gate.status === "insufficient_balance") {
|
|
1883
|
-
// If lobster wallet is available, let execution proceed —
|
|
1884
|
-
// the client-level apiRequest will handle 402 pay-and-retry automatically.
|
|
1885
|
-
let lobsterAvailable = false;
|
|
1886
|
-
try {
|
|
1887
|
-
const { isLobsterAvailable } = await import("../payments/lobster-pay.js");
|
|
1888
|
-
lobsterAvailable = isLobsterAvailable();
|
|
1889
|
-
} catch {}
|
|
1890
|
-
|
|
1891
|
-
if (lobsterAvailable && gate.status === "payment_required") {
|
|
1892
|
-
console.log(`[payment] ${skill.skill_id}: lobster available — proceeding with auto-pay`);
|
|
1893
|
-
} else {
|
|
1894
|
-
const trace: ExecutionTrace = stampTrace({
|
|
1895
|
-
trace_id: nanoid(),
|
|
1896
|
-
skill_id: skill.skill_id,
|
|
1897
|
-
endpoint_id: endpoint.endpoint_id,
|
|
1898
|
-
started_at: new Date().toISOString(),
|
|
1899
|
-
completed_at: new Date().toISOString(),
|
|
1900
|
-
success: false,
|
|
1901
|
-
status_code: 402,
|
|
1902
|
-
error: "payment_required",
|
|
1903
|
-
});
|
|
1904
|
-
return {
|
|
1905
|
-
trace,
|
|
1906
|
-
result: {
|
|
1907
|
-
error: "payment_required",
|
|
1908
|
-
price_usd: gate.requirement?.amount,
|
|
1909
|
-
payment_status: gate.status,
|
|
1910
|
-
message: gate.message,
|
|
1911
|
-
wallet_provider: wallet.wallet_provider ?? "lobster.cash",
|
|
1912
|
-
wallet_address: wallet.wallet_address,
|
|
1913
|
-
indexing_fallback_available: true,
|
|
1914
|
-
},
|
|
1915
|
-
};
|
|
1916
|
-
}
|
|
1917
|
-
}
|
|
1918
|
-
}
|
|
1919
|
-
|
|
1920
|
-
// Mutation safety / policy gate
|
|
1921
|
-
if (endpoint.method !== "GET") {
|
|
1922
|
-
if (workflowRecipe?.mutation_guard.block_reason) {
|
|
1923
|
-
return {
|
|
1924
|
-
trace: stampTrace({
|
|
1925
|
-
trace_id: nanoid(),
|
|
1926
|
-
skill_id: skill.skill_id,
|
|
1927
|
-
endpoint_id: endpoint.endpoint_id,
|
|
1928
|
-
started_at: new Date().toISOString(),
|
|
1929
|
-
completed_at: new Date().toISOString(),
|
|
1930
|
-
success: false,
|
|
1931
|
-
error: "workflow_blocked",
|
|
1932
|
-
}),
|
|
1933
|
-
result: {
|
|
1934
|
-
error: "workflow_blocked",
|
|
1935
|
-
message: workflowRecipe.mutation_guard.block_reason,
|
|
1936
|
-
},
|
|
1937
|
-
};
|
|
1938
|
-
}
|
|
1939
|
-
if (options?.dry_run) {
|
|
1940
|
-
// Merge path_params defaults for dry_run preview too
|
|
1941
|
-
const dryParams = { ...params };
|
|
1942
|
-
if (endpoint.path_params) {
|
|
1943
|
-
for (const [k, v] of Object.entries(endpoint.path_params)) {
|
|
1944
|
-
if (dryParams[k] == null) dryParams[k] = v;
|
|
1945
|
-
}
|
|
1946
|
-
}
|
|
1947
|
-
if (endpoint.body_params) {
|
|
1948
|
-
for (const [k, v] of Object.entries(endpoint.body_params)) {
|
|
1949
|
-
if (dryParams[k] == null) dryParams[k] = v;
|
|
1950
|
-
}
|
|
1951
|
-
}
|
|
1952
|
-
const url = interpolate(endpoint.url_template, dryParams);
|
|
1953
|
-
const body = endpoint.body ? interpolateObj(endpoint.body, dryParams) : undefined;
|
|
1954
|
-
return {
|
|
1955
|
-
trace: stampTrace({
|
|
1956
|
-
trace_id: nanoid(),
|
|
1957
|
-
skill_id: skill.skill_id,
|
|
1958
|
-
endpoint_id: endpoint.endpoint_id,
|
|
1959
|
-
started_at: new Date().toISOString(),
|
|
1960
|
-
completed_at: new Date().toISOString(),
|
|
1961
|
-
success: false,
|
|
1962
|
-
error: "dry_run",
|
|
1963
|
-
}),
|
|
1964
|
-
result: {
|
|
1965
|
-
dry_run: true,
|
|
1966
|
-
...(endpoint.policy ? { site_policy: endpoint.policy } : {}),
|
|
1967
|
-
would_execute: { method: endpoint.method, url, body },
|
|
1968
|
-
},
|
|
1969
|
-
};
|
|
1970
|
-
}
|
|
1971
|
-
if (endpointRequiresThirdPartyTermsConfirmation(endpoint) && !options?.confirm_third_party_terms) {
|
|
1972
|
-
const policy = getEndpointPolicy(endpoint)!;
|
|
1973
|
-
return {
|
|
1974
|
-
trace: stampTrace({
|
|
1975
|
-
trace_id: nanoid(),
|
|
1976
|
-
skill_id: skill.skill_id,
|
|
1977
|
-
endpoint_id: endpoint.endpoint_id,
|
|
1978
|
-
started_at: new Date().toISOString(),
|
|
1979
|
-
completed_at: new Date().toISOString(),
|
|
1980
|
-
success: false,
|
|
1981
|
-
error: "third_party_terms_confirmation_required",
|
|
1982
|
-
}),
|
|
1983
|
-
result: {
|
|
1984
|
-
error: "third_party_terms_confirmation_required",
|
|
1985
|
-
message: `This endpoint may violate third-party site terms for ${policy.policy_domain}. Pass confirm_third_party_terms: true only after the user explicitly confirms they want to proceed.`,
|
|
1986
|
-
policy_domain: policy.policy_domain,
|
|
1987
|
-
policy_reason: policy.reason,
|
|
1988
|
-
},
|
|
1989
|
-
};
|
|
1990
|
-
}
|
|
1991
|
-
if (endpoint.idempotency === "unsafe" && !options?.confirm_unsafe) {
|
|
1992
|
-
return {
|
|
1993
|
-
trace: stampTrace({
|
|
1994
|
-
trace_id: nanoid(),
|
|
1995
|
-
skill_id: skill.skill_id,
|
|
1996
|
-
endpoint_id: endpoint.endpoint_id,
|
|
1997
|
-
started_at: new Date().toISOString(),
|
|
1998
|
-
completed_at: new Date().toISOString(),
|
|
1999
|
-
success: false,
|
|
2000
|
-
error: "confirmation_required",
|
|
2001
|
-
}),
|
|
2002
|
-
result: {
|
|
2003
|
-
error: "confirmation_required",
|
|
2004
|
-
message: `This endpoint (${endpoint.method} ${endpoint.url_template}) is marked as unsafe. Pass confirm_unsafe: true to proceed.`,
|
|
2005
|
-
},
|
|
2006
|
-
};
|
|
2007
|
-
}
|
|
2008
|
-
}
|
|
2009
|
-
|
|
2010
|
-
const startedAt = new Date().toISOString();
|
|
2011
|
-
const authHeaders: Record<string, string> = {};
|
|
2012
|
-
const cookies: Array<{ name: string; value: string; domain: string }> = [];
|
|
2013
|
-
|
|
2014
|
-
// Endpoint domain — used for cookie resolution, strategy caching, auth refresh
|
|
2015
|
-
const epDomain = (() => { try { return new URL(endpoint.url_template).hostname; } catch { return skill.domain; } })();
|
|
2016
|
-
await reloadExecutionAuthState(skill, epDomain, authHeaders, cookies);
|
|
2017
|
-
|
|
2018
|
-
// If endpoint has auth_tokens bindings, always resolve fresh tokens.
|
|
2019
|
-
// Vault headers may be stale — the DAG knows how to get fresh ones.
|
|
2020
|
-
log("exec", `auth_tokens check: ${endpoint.auth_tokens?.length ?? 0} bindings on ${endpoint.endpoint_id}`);
|
|
2021
|
-
if (endpoint.auth_tokens?.length) {
|
|
2022
|
-
try {
|
|
2023
|
-
const { resolveAuthTokens } = await import("./token-resolver.js");
|
|
2024
|
-
const resolved = await resolveAuthTokens(endpoint, cookies, authHeaders);
|
|
2025
|
-
log("exec", `token resolver returned ${Object.keys(resolved).length} headers: ${Object.keys(resolved).join(",") || "none"}`);
|
|
2026
|
-
Object.assign(authHeaders, resolved);
|
|
2027
|
-
} catch (e) { log("exec", `token resolver failed: ${e}`); }
|
|
2028
|
-
}
|
|
2029
|
-
|
|
2030
|
-
log("exec", `endpoint ${endpoint.endpoint_id}: cookies=${cookies.length} authHeaders=${Object.keys(authHeaders).length} hasAuth=${cookies.length > 0 || Object.keys(authHeaders).length > 0}`);
|
|
2031
|
-
|
|
2032
|
-
// BUG-006: Merge path_params defaults — user params override captured defaults
|
|
2033
|
-
let mergedParams = mergeContextTemplateParams(params, endpoint.url_template, options?.contextUrl);
|
|
2034
|
-
if (endpoint.path_params && typeof endpoint.path_params === "object") {
|
|
2035
|
-
for (const [k, v] of Object.entries(endpoint.path_params)) {
|
|
2036
|
-
if (mergedParams[k] == null) {
|
|
2037
|
-
mergedParams[k] = v;
|
|
2038
|
-
}
|
|
2039
|
-
}
|
|
2040
|
-
}
|
|
2041
|
-
if (endpoint.body_params && typeof endpoint.body_params === "object") {
|
|
2042
|
-
for (const [k, v] of Object.entries(endpoint.body_params)) {
|
|
2043
|
-
if (mergedParams[k] == null) {
|
|
2044
|
-
mergedParams[k] = v;
|
|
2045
|
-
}
|
|
2046
|
-
}
|
|
2047
|
-
}
|
|
2048
|
-
|
|
2049
|
-
// Merge captured query params into URL — user params override endpoint defaults
|
|
2050
|
-
let urlTemplate = resolveExecutionUrlTemplate(endpoint, options?.contextUrl);
|
|
2051
|
-
if (endpoint.query && typeof endpoint.query === "object" && Object.keys(endpoint.query).length > 0) {
|
|
2052
|
-
try {
|
|
2053
|
-
const u = new URL(urlTemplate);
|
|
2054
|
-
const queryBindings = extractTemplateQueryBindings(endpoint.url_template);
|
|
2055
|
-
for (const [k, v] of Object.entries(endpoint.query)) {
|
|
2056
|
-
const bindingKey = queryBindings[k];
|
|
2057
|
-
// User params override captured query defaults
|
|
2058
|
-
if (bindingKey && mergedParams[bindingKey] != null) {
|
|
2059
|
-
u.searchParams.set(k, String(mergedParams[bindingKey]));
|
|
2060
|
-
} else if (mergedParams[k] != null) {
|
|
2061
|
-
u.searchParams.set(k, String(mergedParams[k]));
|
|
2062
|
-
} else if (v != null) {
|
|
2063
|
-
u.searchParams.set(k, String(v));
|
|
2064
|
-
}
|
|
2065
|
-
}
|
|
2066
|
-
urlTemplate = restoreTemplatePlaceholderEncoding(u.toString());
|
|
2067
|
-
} catch {
|
|
2068
|
-
// URL parse failure — skip query merge
|
|
2069
|
-
}
|
|
2070
|
-
}
|
|
2071
|
-
let url = interpolate(urlTemplate, mergedParams);
|
|
2072
|
-
let body = endpoint.body ? interpolateObj(endpoint.body, mergedParams) : undefined;
|
|
2073
|
-
|
|
2074
|
-
if (workflowRecipe) {
|
|
2075
|
-
const validationErrors = validateWorkflowReplayParams(workflowRecipe, mergedParams);
|
|
2076
|
-
if (validationErrors.length > 0) {
|
|
2077
|
-
return {
|
|
2078
|
-
trace: stampTrace({
|
|
2079
|
-
trace_id: nanoid(),
|
|
2080
|
-
skill_id: skill.skill_id,
|
|
2081
|
-
endpoint_id: endpoint.endpoint_id,
|
|
2082
|
-
started_at: startedAt,
|
|
2083
|
-
completed_at: new Date().toISOString(),
|
|
2084
|
-
success: false,
|
|
2085
|
-
error: "invalid_replay_params",
|
|
2086
|
-
}),
|
|
2087
|
-
result: {
|
|
2088
|
-
error: "invalid_replay_params",
|
|
2089
|
-
message: "Replay parameters did not satisfy the published workflow contract.",
|
|
2090
|
-
validation_errors: validationErrors,
|
|
2091
|
-
replay_contract: workflowRecipe.replay_contract,
|
|
2092
|
-
},
|
|
2093
|
-
};
|
|
2094
|
-
}
|
|
2095
|
-
}
|
|
2096
|
-
|
|
2097
|
-
const isSafe = endpoint.method === "GET";
|
|
2098
|
-
let workflowBindings = workflowRecipe && workflowArtifact
|
|
2099
|
-
? resolveWorkflowBindings(workflowRecipe, {
|
|
2100
|
-
cookies,
|
|
2101
|
-
authHeaders,
|
|
2102
|
-
body,
|
|
2103
|
-
artifact: workflowArtifact,
|
|
2104
|
-
})
|
|
2105
|
-
: null;
|
|
2106
|
-
if (workflowBindings?.bodyOverride !== undefined) {
|
|
2107
|
-
body = workflowBindings.bodyOverride;
|
|
2108
|
-
}
|
|
2109
|
-
|
|
2110
|
-
// Append leftover params as query string on GET requests.
|
|
2111
|
-
// Params already consumed by path_params, endpoint.query, or {template} vars are skipped.
|
|
2112
|
-
if (isSafe && Object.keys(params).length > 0) {
|
|
2113
|
-
const consumedKeys = new Set<string>([
|
|
2114
|
-
...reservedMetaParams,
|
|
2115
|
-
...Object.keys(endpoint.path_params ?? {}),
|
|
2116
|
-
...Object.keys(endpoint.query ?? {}),
|
|
2117
|
-
]);
|
|
2118
|
-
for (const [rawKey, bindingKey] of Object.entries(extractTemplateQueryBindings(endpoint.url_template))) {
|
|
2119
|
-
consumedKeys.add(rawKey);
|
|
2120
|
-
consumedKeys.add(bindingKey);
|
|
2121
|
-
}
|
|
2122
|
-
if (isCanonicalReplayEndpoint(endpoint)) {
|
|
2123
|
-
try {
|
|
2124
|
-
for (const key of new URL(endpoint.trigger_url!).searchParams.keys()) consumedKeys.add(key);
|
|
2125
|
-
} catch {
|
|
2126
|
-
/* ignore */
|
|
2127
|
-
}
|
|
2128
|
-
}
|
|
2129
|
-
// Also mark keys that appeared as {var} in the original URL template
|
|
2130
|
-
const templateVarRe = /\{(\w+)\}/g;
|
|
2131
|
-
let m: RegExpExecArray | null;
|
|
2132
|
-
while ((m = templateVarRe.exec(endpoint.url_template)) !== null) {
|
|
2133
|
-
consumedKeys.add(m[1]);
|
|
2134
|
-
}
|
|
2135
|
-
const leftover = Object.entries(params).filter(([k]) => !consumedKeys.has(k) && params[k] != null);
|
|
2136
|
-
if (leftover.length > 0) {
|
|
2137
|
-
try {
|
|
2138
|
-
const u = new URL(url);
|
|
2139
|
-
for (const [k, v] of leftover) {
|
|
2140
|
-
u.searchParams.set(k, String(v));
|
|
2141
|
-
}
|
|
2142
|
-
url = u.toString();
|
|
2143
|
-
} catch { /* URL parse failure — skip */ }
|
|
2144
|
-
}
|
|
2145
|
-
}
|
|
2146
|
-
|
|
2147
|
-
const hasAuthContext =
|
|
2148
|
-
cookies.length > 0 ||
|
|
2149
|
-
Object.keys(authHeaders).length > 0 ||
|
|
2150
|
-
!!skill.auth_profile_ref ||
|
|
2151
|
-
endpoint.semantic?.auth_required === true;
|
|
2152
|
-
|
|
2153
|
-
// robots.txt compliance gate — block disallowed paths before any network call.
|
|
2154
|
-
if (!options?.skip_robots_check && !hasAuthContext) {
|
|
2155
|
-
const allowed = await isAllowedByRobots(url);
|
|
2156
|
-
if (!allowed) {
|
|
2157
|
-
const traceId = nanoid();
|
|
2158
|
-
log("exec", `robots.txt blocked ${url}`);
|
|
2159
|
-
return {
|
|
2160
|
-
trace: stampTrace({
|
|
2161
|
-
trace_id: traceId,
|
|
2162
|
-
skill_id: skill.skill_id,
|
|
2163
|
-
endpoint_id: endpoint.endpoint_id,
|
|
2164
|
-
started_at: startedAt,
|
|
2165
|
-
completed_at: new Date().toISOString(),
|
|
2166
|
-
success: false,
|
|
2167
|
-
error: "robots_txt_disallowed",
|
|
2168
|
-
}),
|
|
2169
|
-
result: {
|
|
2170
|
-
error: "robots_txt_disallowed",
|
|
2171
|
-
message: `robots.txt disallows access to ${url} for the Unbrowse user-agent.`,
|
|
2172
|
-
},
|
|
2173
|
-
};
|
|
2174
|
-
}
|
|
2175
|
-
}
|
|
2176
|
-
const structuredReplayUrl = isSafe ? deriveStructuredDataReplayUrl(url) : url;
|
|
2177
|
-
const hasStructuredReplay = structuredReplayUrl !== url;
|
|
2178
|
-
|
|
2179
|
-
const serverFetch = async (
|
|
2180
|
-
extraHeaders: Record<string, string> = {},
|
|
2181
|
-
bodyOverride: unknown = body,
|
|
2182
|
-
): Promise<{ data: unknown; status: number; trace_id: string }> => {
|
|
2183
|
-
const endpointHeaders = normalizeReplayHeaders(endpoint.headers_template);
|
|
2184
|
-
const sessionHeaders = normalizeReplayHeaders(authHeaders);
|
|
2185
|
-
|
|
2186
|
-
// Default accept to JSON, but never overwrite the endpoint's own accept header
|
|
2187
|
-
// (e.g. LinkedIn uses "application/vnd.linkedin.normalized+json+2.1")
|
|
2188
|
-
const defaultAccept: Record<string, string> = (!endpoint.dom_extraction && !endpointHeaders["accept"] && !sessionHeaders["accept"])
|
|
2189
|
-
? { "accept": "application/json" } : {};
|
|
2190
|
-
const headers: Record<string, string> = {
|
|
2191
|
-
...defaultAccept,
|
|
2192
|
-
...endpointHeaders,
|
|
2193
|
-
...sessionHeaders,
|
|
2194
|
-
...normalizeReplayHeaders(extraHeaders),
|
|
2195
|
-
};
|
|
2196
|
-
// Strip browser-only headers that cause issues server-side
|
|
2197
|
-
delete headers["sec-ch-ua"];
|
|
2198
|
-
delete headers["sec-ch-ua-mobile"];
|
|
2199
|
-
delete headers["sec-ch-ua-platform"];
|
|
2200
|
-
delete headers["upgrade-insecure-requests"];
|
|
2201
|
-
|
|
2202
|
-
// Inject cookies as Cookie header — same as a browser would send.
|
|
2203
|
-
// Strip enclosing quotes from values — Chrome's SQLite stores them quoted
|
|
2204
|
-
// but the Cookie header must send them unquoted (RFC 6265 §4.1.1).
|
|
2205
|
-
if (cookies.length > 0) {
|
|
2206
|
-
const cookieStr = cookies.map((c) => {
|
|
2207
|
-
const v = c.value.startsWith('"') && c.value.endsWith('"') ? c.value.slice(1, -1) : c.value;
|
|
2208
|
-
return `${c.name}=${v}`;
|
|
2209
|
-
}).join("; ");
|
|
2210
|
-
headers["cookie"] = cookieStr;
|
|
2211
|
-
|
|
2212
|
-
// CSRF token auto-detection (bird pattern): many sites require CSRF tokens
|
|
2213
|
-
// as both a cookie AND a header. The cookie value is always fresher than
|
|
2214
|
-
// any stored vault header, so it ALWAYS overrides.
|
|
2215
|
-
const csrfCookie = cookies.find((c) =>
|
|
2216
|
-
/^(ct0|csrf_token|_csrf|csrftoken|XSRF-TOKEN|_xsrf)$/i.test(c.name)
|
|
2217
|
-
);
|
|
2218
|
-
if (csrfCookie) {
|
|
2219
|
-
const v = csrfCookie.value.startsWith('"') && csrfCookie.value.endsWith('"') ? csrfCookie.value.slice(1, -1) : csrfCookie.value;
|
|
2220
|
-
headers["x-csrf-token"] = v;
|
|
2221
|
-
headers["x-xsrf-token"] = v;
|
|
2222
|
-
}
|
|
2223
|
-
}
|
|
2224
|
-
|
|
2225
|
-
if (endpoint.csrf_plan && cookies.length > 0) {
|
|
2226
|
-
const csrfCookie = cookies.find((c) =>
|
|
2227
|
-
endpoint.csrf_plan!.extractor_sequence.some((name) => name.toLowerCase() === c.name.toLowerCase()),
|
|
2228
|
-
);
|
|
2229
|
-
if (csrfCookie) {
|
|
2230
|
-
const v = csrfCookie.value.startsWith('"') && csrfCookie.value.endsWith('"') ? csrfCookie.value.slice(1, -1) : csrfCookie.value;
|
|
2231
|
-
if (endpoint.csrf_plan.source === "cookie" || endpoint.csrf_plan.source === "header") {
|
|
2232
|
-
headers[endpoint.csrf_plan.param_name.toLowerCase()] = v;
|
|
2233
|
-
} else if (endpoint.csrf_plan.source === "form" && body && typeof body === "object" && !Array.isArray(body)) {
|
|
2234
|
-
(body as Record<string, unknown>)[endpoint.csrf_plan.param_name] ??= v;
|
|
2235
|
-
}
|
|
2236
|
-
}
|
|
2237
|
-
}
|
|
2238
|
-
|
|
2239
|
-
const replayUrls = hasStructuredReplay ? deriveStructuredDataReplayCandidates(structuredReplayUrl) : [structuredReplayUrl];
|
|
2240
|
-
let last: { data: unknown; status: number } = { data: null, status: 0 };
|
|
2241
|
-
|
|
2242
|
-
for (const replayUrl of replayUrls) {
|
|
2243
|
-
const replayHeaders = buildStructuredReplayHeaders(url, replayUrl, headers);
|
|
2244
|
-
log("exec", `server-fetch: ${endpoint.method} ${replayUrl.substring(0, 80)} csrf=${replayHeaders["x-csrf-token"]?.substring(0, 10)}... cookies=${(replayHeaders["cookie"]?.length ?? 0)}chars`);
|
|
2245
|
-
const res = await fetch(replayUrl, {
|
|
2246
|
-
method: endpoint.method,
|
|
2247
|
-
headers: replayHeaders,
|
|
2248
|
-
body: serializeReplayBody(bodyOverride, replayHeaders),
|
|
2249
|
-
redirect: "follow",
|
|
2250
|
-
});
|
|
2251
|
-
let data: unknown;
|
|
2252
|
-
const text = await res.text();
|
|
2253
|
-
try { data = JSON.parse(text); } catch { data = text; }
|
|
2254
|
-
last = { data, status: res.status };
|
|
2255
|
-
if (res.ok && !(typeof data === "string" && isHtml(data))) {
|
|
2256
|
-
return { data, status: res.status, trace_id: nanoid() };
|
|
2257
|
-
}
|
|
2258
|
-
}
|
|
2259
|
-
|
|
2260
|
-
return { data: last.data, status: last.status, trace_id: nanoid() };
|
|
2261
|
-
};
|
|
2262
|
-
|
|
2263
|
-
const browserCall = () => executeInBrowser(
|
|
2264
|
-
url,
|
|
2265
|
-
endpoint.method,
|
|
2266
|
-
endpoint.headers_template ?? {},
|
|
2267
|
-
body,
|
|
2268
|
-
authHeaders,
|
|
2269
|
-
cookies
|
|
2270
|
-
);
|
|
2271
|
-
|
|
2272
|
-
let result: { data: unknown; status: number; trace_id: string };
|
|
2273
|
-
const hasAuth = cookies.length > 0 || Object.keys(authHeaders).length > 0;
|
|
2274
|
-
const preferredWorkflowStrategy = workflowRecipe?.steps[0]?.strategy
|
|
2275
|
-
? translateWorkflowStrategy(workflowRecipe.steps[0].strategy)
|
|
2276
|
-
: undefined;
|
|
2277
|
-
let workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy;
|
|
2278
|
-
|
|
2279
|
-
if (endpoint.dom_extraction && isSafe) {
|
|
2280
|
-
if (hasStructuredReplay) {
|
|
2281
|
-
result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
|
|
2282
|
-
if (shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
|
|
2283
|
-
result = await executeDomExtractionEndpoint(
|
|
2284
|
-
endpoint,
|
|
2285
|
-
url,
|
|
2286
|
-
options?.intent ?? skill.intent_signature,
|
|
2287
|
-
authHeaders,
|
|
2288
|
-
cookies,
|
|
2289
|
-
);
|
|
2290
|
-
}
|
|
2291
|
-
} else {
|
|
2292
|
-
result = await executeDomExtractionEndpoint(
|
|
2293
|
-
endpoint,
|
|
2294
|
-
url,
|
|
2295
|
-
options?.intent ?? skill.intent_signature,
|
|
2296
|
-
authHeaders,
|
|
2297
|
-
cookies,
|
|
2298
|
-
);
|
|
2299
|
-
}
|
|
2300
|
-
} else if (hasAuth) {
|
|
2301
|
-
// Authed execution: learned strategy → skip doomed tiers
|
|
2302
|
-
// 1. Server fetch (fast — works for Twitter, simple APIs)
|
|
2303
|
-
// 2. Trigger-and-intercept (navigate to page, let site's JS make the call)
|
|
2304
|
-
// 3. Browser in-page fetch (last resort)
|
|
2305
|
-
let strategy: "server" | "trigger-intercept" | "browser" | undefined;
|
|
2306
|
-
|
|
2307
|
-
// Endpoint-level learned strategy (strong signal — proven for this specific endpoint).
|
|
2308
|
-
// Domain-level prediction is only used as a tiebreaker, never to skip server-fetch entirely,
|
|
2309
|
-
// because different endpoints on the same domain may have different requirements.
|
|
2310
|
-
const endpointStrategy = preferredWorkflowStrategy ?? endpoint.exec_strategy;
|
|
2311
|
-
|
|
2312
|
-
if (hasStructuredReplay) {
|
|
2313
|
-
result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
|
|
2314
|
-
if (result.status >= 200 && result.status < 400 && !shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
|
|
2315
|
-
strategy = "server";
|
|
2316
|
-
workflowChosenStrategy = "server";
|
|
2317
|
-
} else if (endpoint.trigger_url && isSafe) {
|
|
2318
|
-
// Build trigger URL with agent's params applied — don't replay original captured search
|
|
2319
|
-
let triggerUrl = endpoint.trigger_url;
|
|
2320
|
-
if (Object.keys(mergedParams).length > 0) {
|
|
2321
|
-
try {
|
|
2322
|
-
const tu = new URL(endpoint.trigger_url);
|
|
2323
|
-
for (const [k, v] of Object.entries(mergedParams)) {
|
|
2324
|
-
if (v != null && !reservedMetaParams.has(k)) {
|
|
2325
|
-
tu.searchParams.set(k, String(v));
|
|
2326
|
-
}
|
|
2327
|
-
}
|
|
2328
|
-
triggerUrl = tu.toString();
|
|
2329
|
-
} catch { /* keep original trigger_url */ }
|
|
2330
|
-
}
|
|
2331
|
-
result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
|
|
2332
|
-
strategy = "trigger-intercept";
|
|
2333
|
-
workflowChosenStrategy = "trigger-intercept";
|
|
2334
|
-
} else {
|
|
2335
|
-
result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
|
|
2336
|
-
strategy = "browser";
|
|
2337
|
-
workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
|
|
2338
|
-
}
|
|
2339
|
-
} else if (endpointStrategy === "server") {
|
|
2340
|
-
// Proven: server-fetch works for this endpoint
|
|
2341
|
-
result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
|
|
2342
|
-
if (shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
|
|
2343
|
-
result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
|
|
2344
|
-
strategy = "browser";
|
|
2345
|
-
workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
|
|
2346
|
-
} else {
|
|
2347
|
-
strategy = "server";
|
|
2348
|
-
workflowChosenStrategy = "server";
|
|
2349
|
-
}
|
|
2350
|
-
} else if (endpointStrategy === "trigger-intercept" && endpoint.trigger_url && isSafe) {
|
|
2351
|
-
// Proven: this endpoint needs trigger-intercept
|
|
2352
|
-
let triggerUrl = endpoint.trigger_url;
|
|
2353
|
-
if (Object.keys(mergedParams).length > 0) {
|
|
2354
|
-
try {
|
|
2355
|
-
const tu = new URL(endpoint.trigger_url);
|
|
2356
|
-
for (const [k, v] of Object.entries(mergedParams)) {
|
|
2357
|
-
if (v != null && !reservedMetaParams.has(k)) tu.searchParams.set(k, String(v));
|
|
2358
|
-
}
|
|
2359
|
-
triggerUrl = tu.toString();
|
|
2360
|
-
} catch { /* keep original */ }
|
|
2361
|
-
}
|
|
2362
|
-
log("exec", `using learned strategy trigger-intercept via ${triggerUrl}`);
|
|
2363
|
-
result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
|
|
2364
|
-
strategy = "trigger-intercept";
|
|
2365
|
-
workflowChosenStrategy = "trigger-intercept";
|
|
2366
|
-
} else if (endpointStrategy === "browser") {
|
|
2367
|
-
if (shouldIgnoreLearnedBrowserStrategy(endpoint, url)) {
|
|
2368
|
-
result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
|
|
2369
|
-
if (result.status >= 200 && result.status < 400 && !shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
|
|
2370
|
-
strategy = "server";
|
|
2371
|
-
workflowChosenStrategy = "server";
|
|
2372
|
-
} else {
|
|
2373
|
-
log("exec", `server replay rejected stale learned browser strategy for ${endpoint.endpoint_id}; falling back to browser`);
|
|
2374
|
-
result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
|
|
2375
|
-
strategy = "browser";
|
|
2376
|
-
workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
|
|
2377
|
-
}
|
|
2378
|
-
} else {
|
|
2379
|
-
log("exec", `using learned strategy browser`);
|
|
2380
|
-
result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
|
|
2381
|
-
strategy = "browser";
|
|
2382
|
-
workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
|
|
2383
|
-
}
|
|
2384
|
-
} else {
|
|
2385
|
-
// No endpoint-level strategy — always try server-fetch first (fastest path).
|
|
2386
|
-
// Fall back to trigger-intercept or browser if server returns 4xx.
|
|
2387
|
-
try {
|
|
2388
|
-
result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
|
|
2389
|
-
if (result.status >= 200 && result.status < 400) {
|
|
2390
|
-
// For API endpoints, trust the server response — no fallback to browser
|
|
2391
|
-
const isApiEndpoint = /\/(api|graphql)\b/i.test(endpoint.url_template) || /\.(json)(\?|$)/.test(endpoint.url_template);
|
|
2392
|
-
if (!isApiEndpoint && shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
|
|
2393
|
-
result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
|
|
2394
|
-
strategy = "browser";
|
|
2395
|
-
workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
|
|
2396
|
-
} else {
|
|
2397
|
-
strategy = "server";
|
|
2398
|
-
workflowChosenStrategy = "server";
|
|
2399
|
-
}
|
|
2400
|
-
} else {
|
|
2401
|
-
log("exec", `server fetch returned ${result.status}, falling back`);
|
|
2402
|
-
if (endpoint.trigger_url && isSafe) {
|
|
2403
|
-
let triggerUrl = endpoint.trigger_url;
|
|
2404
|
-
if (Object.keys(mergedParams).length > 0) {
|
|
2405
|
-
try {
|
|
2406
|
-
const tu = new URL(endpoint.trigger_url);
|
|
2407
|
-
for (const [k, v] of Object.entries(mergedParams)) {
|
|
2408
|
-
if (v != null && !reservedMetaParams.has(k)) tu.searchParams.set(k, String(v));
|
|
2409
|
-
}
|
|
2410
|
-
triggerUrl = tu.toString();
|
|
2411
|
-
} catch { /* keep original */ }
|
|
2412
|
-
}
|
|
2413
|
-
result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
|
|
2414
|
-
strategy = "trigger-intercept";
|
|
2415
|
-
workflowChosenStrategy = "trigger-intercept";
|
|
2416
|
-
} else {
|
|
2417
|
-
result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
|
|
2418
|
-
strategy = "browser";
|
|
2419
|
-
workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
|
|
2420
|
-
}
|
|
2421
|
-
}
|
|
2422
|
-
} catch {
|
|
2423
|
-
result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
|
|
2424
|
-
strategy = "browser";
|
|
2425
|
-
workflowChosenStrategy = workflowRecipe?.steps[0]?.strategy === "browser-action" ? "browser-action" : "browser-fetch";
|
|
2426
|
-
}
|
|
2427
|
-
}
|
|
2428
|
-
|
|
2429
|
-
// Persist learned strategy at endpoint level only.
|
|
2430
|
-
// Domain-level cache removed: it over-generalizes (e.g., one 400 on LinkedIn
|
|
2431
|
-
// locked all endpoints into trigger-intercept even though server-fetch works for most).
|
|
2432
|
-
if (strategy && result.status >= 200 && result.status < 400 && strategy !== endpoint.exec_strategy) {
|
|
2433
|
-
log("exec", `learned exec_strategy=${strategy} for endpoint ${endpoint.endpoint_id}`);
|
|
2434
|
-
endpoint.exec_strategy = strategy;
|
|
2435
|
-
try { cachePublishedSkill(skill, options?.client_scope); } catch (e) { log("exec", `failed to cache strategy: ${e}`); }
|
|
2436
|
-
}
|
|
2437
|
-
} else if (isSafe) {
|
|
2438
|
-
// No auth: fetch-first for safe GETs — fall back to browser if SPA shell or error
|
|
2439
|
-
try {
|
|
2440
|
-
result = await withRetry(() => serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride), (r) => isRetryableStatus(r.status));
|
|
2441
|
-
if (typeof result.data === "string" && isHtml(result.data)) {
|
|
2442
|
-
if (isSpaShell(result.data)) {
|
|
2443
|
-
result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
|
|
2444
|
-
}
|
|
2445
|
-
}
|
|
2446
|
-
} catch {
|
|
2447
|
-
result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
|
|
2448
|
-
}
|
|
2449
|
-
} else {
|
|
2450
|
-
// No auth, non-GET: server fetch
|
|
2451
|
-
result = await serverFetch(workflowBindings?.extraHeaders, workflowBindings?.bodyOverride);
|
|
2452
|
-
}
|
|
2453
|
-
|
|
2454
|
-
if (workflowRecipe && workflowArtifact && needsWorkflowTokenRefresh(result.status)) {
|
|
2455
|
-
const refreshed = await refreshAuthFromBrowser(epDomain);
|
|
2456
|
-
if (refreshed) {
|
|
2457
|
-
await reloadExecutionAuthState(skill, epDomain, authHeaders, cookies);
|
|
2458
|
-
workflowBindings = resolveWorkflowBindings(workflowRecipe, {
|
|
2459
|
-
cookies,
|
|
2460
|
-
authHeaders,
|
|
2461
|
-
body,
|
|
2462
|
-
artifact: workflowArtifact,
|
|
2463
|
-
});
|
|
2464
|
-
result = await serverFetch(workflowBindings.extraHeaders, workflowBindings.bodyOverride);
|
|
2465
|
-
if (result.status >= 200 && result.status < 400) {
|
|
2466
|
-
workflowChosenStrategy = workflowChosenStrategy ?? "server";
|
|
2467
|
-
}
|
|
2468
|
-
}
|
|
2469
|
-
}
|
|
2470
|
-
const { status, trace_id } = result;
|
|
2471
|
-
let data = result.data;
|
|
2472
|
-
|
|
2473
|
-
const trace: ExecutionTrace = stampTrace({
|
|
2474
|
-
trace_id,
|
|
2475
|
-
skill_id: skill.skill_id,
|
|
2476
|
-
endpoint_id: endpoint.endpoint_id,
|
|
2477
|
-
started_at: startedAt,
|
|
2478
|
-
completed_at: new Date().toISOString(),
|
|
2479
|
-
success: status >= 200 && status < 300,
|
|
2480
|
-
status_code: status,
|
|
2481
|
-
});
|
|
2482
|
-
|
|
2483
|
-
if (!trace.success) {
|
|
2484
|
-
trace.error = status === 404
|
|
2485
|
-
? `HTTP 404 — endpoint may be stale. Re-run via POST /v1/intent/resolve to get fresh endpoints.`
|
|
2486
|
-
: `HTTP ${status}`;
|
|
2487
|
-
} else {
|
|
2488
|
-
trace.result = data;
|
|
2489
|
-
}
|
|
2490
|
-
|
|
2491
|
-
// Stale credential detection: on 401/403, attempt auth recovery before giving up.
|
|
2492
|
-
// Chain: authRuntime.refreshSession (lightweight) → refreshAuthFromBrowser (re-extract)
|
|
2493
|
-
// → authRuntime.loginIfNeeded (full interactive login)
|
|
2494
|
-
if (status === 401 || status === 403) {
|
|
2495
|
-
let authRecovered = false;
|
|
2496
|
-
try {
|
|
2497
|
-
// 1. Lightweight session refresh via authRuntime
|
|
2498
|
-
const sessionRefreshed = await authRuntime.refreshSession(epDomain);
|
|
2499
|
-
if (sessionRefreshed) {
|
|
2500
|
-
log("auth", `session refreshed via authRuntime for ${epDomain} — retry should succeed`);
|
|
2501
|
-
authRecovered = true;
|
|
2502
|
-
}
|
|
2503
|
-
|
|
2504
|
-
// 2. Re-extract cookies from browser SQLite (bird pattern)
|
|
2505
|
-
if (!authRecovered) {
|
|
2506
|
-
const browserRefreshed = await refreshAuthFromBrowser(epDomain);
|
|
2507
|
-
if (browserRefreshed) {
|
|
2508
|
-
log("auth", `credentials refreshed from browser for ${epDomain}`);
|
|
2509
|
-
authRecovered = true;
|
|
2510
|
-
}
|
|
2511
|
-
}
|
|
2512
|
-
|
|
2513
|
-
// 3. Full login flow via authRuntime as last resort
|
|
2514
|
-
if (!authRecovered) {
|
|
2515
|
-
const loginResult = await authRuntime.loginIfNeeded(epDomain);
|
|
2516
|
-
if (loginResult) {
|
|
2517
|
-
log("auth", `loginIfNeeded succeeded for ${epDomain}`);
|
|
2518
|
-
authRecovered = true;
|
|
2519
|
-
}
|
|
2520
|
-
}
|
|
2521
|
-
|
|
2522
|
-
if (authRecovered) {
|
|
2523
|
-
trace.error = `${trace.error} (credentials refreshed — retry should succeed)`;
|
|
2524
|
-
} else {
|
|
2525
|
-
// No recovery path worked — delete stale credentials
|
|
2526
|
-
if (skill.auth_profile_ref) {
|
|
2527
|
-
await deleteCredential(skill.auth_profile_ref);
|
|
2528
|
-
}
|
|
2529
|
-
trace.error = `${trace.error} (stale credentials — re-authenticate via /v1/auth/login)`;
|
|
2530
|
-
}
|
|
2531
|
-
} catch {
|
|
2532
|
-
if (skill.auth_profile_ref) {
|
|
2533
|
-
await deleteCredential(skill.auth_profile_ref);
|
|
2534
|
-
}
|
|
2535
|
-
trace.error = `${trace.error} (stale credential deleted)`;
|
|
2536
|
-
}
|
|
2537
|
-
}
|
|
2538
|
-
|
|
2539
|
-
// Schema drift detection on re-execution
|
|
2540
|
-
if (trace.success && endpoint.response_schema && data != null) {
|
|
2541
|
-
const drift = detectSchemaDrift(endpoint.response_schema, data);
|
|
2542
|
-
if (drift.drifted) {
|
|
2543
|
-
trace.drift = drift;
|
|
2544
|
-
}
|
|
2545
|
-
}
|
|
2546
|
-
|
|
2547
|
-
// HTML→JSON post-processing: if the endpoint returned HTML instead of JSON,
|
|
2548
|
-
// pipe it through DOM extraction to produce structured data.
|
|
2549
|
-
// Always extract — returning raw HTML to an agent is never useful.
|
|
2550
|
-
if (trace.success && typeof data === "string" && isHtml(data)) {
|
|
2551
|
-
const intent = options?.intent || skill.intent_signature;
|
|
2552
|
-
if (!endpoint.dom_extraction) {
|
|
2553
|
-
trace.success = false;
|
|
2554
|
-
trace.error = "unexpected_html_response";
|
|
2555
|
-
data = {
|
|
2556
|
-
error: "unexpected_html_response",
|
|
2557
|
-
message: `Endpoint returned HTML instead of API data for intent "${intent}"`,
|
|
2558
|
-
};
|
|
2559
|
-
trace.result = data;
|
|
2560
|
-
} else {
|
|
2561
|
-
const extracted = extractFromDOM(data, intent);
|
|
2562
|
-
if (extracted.data) {
|
|
2563
|
-
const quality = validateExtractionQuality(extracted.data, extracted.confidence, intent);
|
|
2564
|
-
const semanticAssessment = quality.valid ? assessIntentResult(extracted.data, intent) : { verdict: "fail" as const, reason: quality.quality_note ?? "low_quality_dom_extraction" };
|
|
2565
|
-
if (quality.valid && semanticAssessment.verdict !== "fail") {
|
|
2566
|
-
data = {
|
|
2567
|
-
data: extracted.data,
|
|
2568
|
-
_extraction: {
|
|
2569
|
-
method: extracted.extraction_method,
|
|
2570
|
-
confidence: extracted.confidence,
|
|
2571
|
-
source: "html-postprocess",
|
|
2572
|
-
},
|
|
2573
|
-
};
|
|
2574
|
-
trace.result = data;
|
|
2575
|
-
} else {
|
|
2576
|
-
trace.success = false;
|
|
2577
|
-
trace.error = semanticAssessment.reason ?? quality.quality_note ?? "low_quality_dom_extraction";
|
|
2578
|
-
data = {
|
|
2579
|
-
error: "low_quality_dom_extraction",
|
|
2580
|
-
message: `Structured DOM extraction was rejected: ${semanticAssessment.reason ?? quality.quality_note ?? "low quality extraction"}`,
|
|
2581
|
-
};
|
|
2582
|
-
trace.result = data;
|
|
2583
|
-
}
|
|
2584
|
-
}
|
|
2585
|
-
}
|
|
2586
|
-
}
|
|
2587
|
-
|
|
2588
|
-
const effectiveIntent = options?.intent ?? skill.intent_signature;
|
|
2589
|
-
if (trace.success && effectiveIntent && data != null) {
|
|
2590
|
-
const semanticAssessment = assessIntentResult(data, effectiveIntent);
|
|
2591
|
-
if (semanticAssessment.verdict === "fail") {
|
|
2592
|
-
trace.success = false;
|
|
2593
|
-
trace.error = semanticAssessment.reason;
|
|
2594
|
-
data = {
|
|
2595
|
-
error: "intent_mismatch",
|
|
2596
|
-
message: `Execution result did not satisfy intent "${effectiveIntent}": ${semanticAssessment.reason}`,
|
|
2597
|
-
projected: semanticAssessment.projected,
|
|
2598
|
-
};
|
|
2599
|
-
trace.result = data;
|
|
2600
|
-
}
|
|
2601
|
-
}
|
|
2602
|
-
|
|
2603
|
-
// Backfill response_schema on first successful execution — push to marketplace so all agents benefit
|
|
2604
|
-
if (trace.success && !endpoint.response_schema && data != null && typeof data !== "string") {
|
|
2605
|
-
try {
|
|
2606
|
-
const inferred = inferSchema([data]);
|
|
2607
|
-
if (inferred.type !== "object" || inferred.properties) {
|
|
2608
|
-
log("exec", `learned response_schema for endpoint ${endpoint.endpoint_id} (${Object.keys(inferred.properties ?? {}).length} top-level props)`);
|
|
2609
|
-
endpoint.response_schema = inferred;
|
|
2610
|
-
trace.schema_backfilled = true;
|
|
2611
|
-
cachePublishedSkill(skill, options?.client_scope);
|
|
2612
|
-
updateEndpointSchema(skill.skill_id, endpoint.endpoint_id, inferred).catch(() => {});
|
|
2613
|
-
}
|
|
2614
|
-
} catch {}
|
|
2615
|
-
}
|
|
2616
|
-
|
|
2617
|
-
// Record execution for reliability scoring (fire-and-forget — don't block response)
|
|
2618
|
-
recordExecution(skill.skill_id, endpoint.endpoint_id, trace, skill).catch(() => {});
|
|
2619
|
-
|
|
2620
|
-
// Record transaction if this was a paid execution (fire-and-forget)
|
|
2621
|
-
if (trace.success && options?.payment_verified === true && skill.base_price_usd && skill.base_price_usd > 0) {
|
|
2622
|
-
const consumerConfig = (() => {
|
|
2623
|
-
try { return JSON.parse(require("fs").readFileSync(require("os").homedir() + "/.unbrowse/config.json", "utf-8")); }
|
|
2624
|
-
catch { return {}; }
|
|
2625
|
-
})();
|
|
2626
|
-
if (consumerConfig.agent_id) {
|
|
2627
|
-
const wallet = getLocalWalletContext();
|
|
2628
|
-
recordTransaction({
|
|
2629
|
-
transaction_id: trace.trace_id,
|
|
2630
|
-
consumer_id: consumerConfig.agent_id,
|
|
2631
|
-
creator_id: skill.indexer_id,
|
|
2632
|
-
skill_id: skill.skill_id,
|
|
2633
|
-
endpoint_id: endpoint.endpoint_id,
|
|
2634
|
-
price_usd: skill.base_price_usd,
|
|
2635
|
-
payment_proof: wallet.wallet_address ? `wallet:${wallet.wallet_address}` : undefined,
|
|
2636
|
-
}).catch(() => {});
|
|
2637
|
-
}
|
|
2638
|
-
}
|
|
2639
|
-
// Apply field projection
|
|
2640
|
-
let resultData = data;
|
|
2641
|
-
if (projection?.raw) {
|
|
2642
|
-
// Explicit raw request — skip projection
|
|
2643
|
-
} else if (projection && trace.success) {
|
|
2644
|
-
resultData = applyProjection(data, projection);
|
|
2645
|
-
} else if (trace.success) {
|
|
2646
|
-
resultData = projectResultForIntent(data, effectiveIntent);
|
|
2647
|
-
}
|
|
2648
|
-
|
|
2649
|
-
if (workflowArtifact && workflowRecipe && workflowChosenStrategy) {
|
|
2650
|
-
const updatedWorkflow = recordWorkflowRecipeOutcome(
|
|
2651
|
-
workflowArtifact,
|
|
2652
|
-
endpoint.endpoint_id,
|
|
2653
|
-
workflowChosenStrategy,
|
|
2654
|
-
{
|
|
2655
|
-
success: trace.success,
|
|
2656
|
-
status,
|
|
2657
|
-
error: trace.error,
|
|
2658
|
-
selectedBindings: workflowBindings?.selectedBindings,
|
|
2659
|
-
},
|
|
2660
|
-
);
|
|
2661
|
-
const writtenPath = writeWorkflowArtifact(updatedWorkflow);
|
|
2662
|
-
if (process.env.UNBROWSE_DEBUG_WORKFLOW === "1") {
|
|
2663
|
-
log(
|
|
2664
|
-
"workflow",
|
|
2665
|
-
`execution persisted skill=${skill.skill_id} endpoint=${endpoint.endpoint_id} strategy=${workflowChosenStrategy} path=${writtenPath ?? "write-failed"}`,
|
|
2666
|
-
);
|
|
2667
|
-
}
|
|
2668
|
-
trace.result = trace.result;
|
|
2669
|
-
(trace as ExecutionTrace & { workflow_selected_bindings?: unknown; workflow_strategy?: string }).workflow_selected_bindings = workflowBindings?.selectedBindings;
|
|
2670
|
-
(trace as ExecutionTrace & { workflow_selected_bindings?: unknown; workflow_strategy?: string }).workflow_strategy = workflowChosenStrategy;
|
|
2671
|
-
}
|
|
2672
|
-
|
|
2673
|
-
return {
|
|
2674
|
-
trace, result: resultData,
|
|
2675
|
-
};
|
|
2676
|
-
}
|
|
2677
|
-
|
|
2678
|
-
/**
|
|
2679
|
-
* Convert query params in a URL to template variables.
|
|
2680
|
-
* e.g. /search?q=books&page=1 → /search?q={q}&page={page}
|
|
2681
|
-
* Path stays untouched — only query string is templatized.
|
|
2682
|
-
*/
|
|
2683
|
-
export function templatizeQueryParams(url: string): string {
|
|
2684
|
-
try {
|
|
2685
|
-
const u = sanitizeNavigationQueryParams(new URL(url));
|
|
2686
|
-
if (u.search.length <= 1) return url; // no query params
|
|
2687
|
-
const params = new URLSearchParams(u.search);
|
|
2688
|
-
const templated = new URLSearchParams();
|
|
2689
|
-
const bindings = buildQueryBindingMap(params.keys());
|
|
2690
|
-
const seen = new Set<string>();
|
|
2691
|
-
for (const [key] of params) {
|
|
2692
|
-
if (seen.has(key)) continue;
|
|
2693
|
-
seen.add(key);
|
|
2694
|
-
templated.set(key, `{${bindings[key] ?? key}}`);
|
|
2695
|
-
}
|
|
2696
|
-
return `${u.origin}${u.pathname}?${templated.toString().replace(/%7B/g, "{").replace(/%7D/g, "}")}`;
|
|
2697
|
-
} catch {
|
|
2698
|
-
return url;
|
|
2699
|
-
}
|
|
2700
|
-
}
|
|
2701
|
-
|
|
2702
|
-
function interpolate(template: string, params: Record<string, unknown>): string {
|
|
2703
|
-
// Split URL into base and query string to properly encode query params
|
|
2704
|
-
const qIdx = template.indexOf("?");
|
|
2705
|
-
if (qIdx === -1) {
|
|
2706
|
-
return template.replace(/\{(\w+)\}/g, (_, k) =>
|
|
2707
|
-
params[k] != null ? String(params[k]) : `{${k}}`
|
|
2708
|
-
);
|
|
2709
|
-
}
|
|
2710
|
-
|
|
2711
|
-
const base = template.substring(0, qIdx);
|
|
2712
|
-
const query = template.substring(qIdx + 1);
|
|
2713
|
-
|
|
2714
|
-
// Interpolate base path without encoding
|
|
2715
|
-
const interpolatedBase = base.replace(/\{(\w+)\}/g, (_, k) =>
|
|
2716
|
-
params[k] != null ? String(params[k]) : `{${k}}`
|
|
2717
|
-
);
|
|
2718
|
-
|
|
2719
|
-
// Interpolate query params with URL encoding
|
|
2720
|
-
const interpolatedQuery = query.replace(/\{(\w+)\}/g, (_, k) =>
|
|
2721
|
-
params[k] != null ? encodeURIComponent(String(params[k])) : `{${k}}`
|
|
2722
|
-
);
|
|
2723
|
-
|
|
2724
|
-
return `${interpolatedBase}?${interpolatedQuery}`;
|
|
2725
|
-
}
|
|
2726
|
-
|
|
2727
|
-
function interpolateObj(
|
|
2728
|
-
obj: Record<string, unknown>,
|
|
2729
|
-
params: Record<string, unknown>
|
|
2730
|
-
): Record<string, unknown> {
|
|
2731
|
-
return JSON.parse(
|
|
2732
|
-
JSON.stringify(obj).replace(/"(\{(\w+)\})"/g, (_, _full, k) =>
|
|
2733
|
-
params[k] != null ? JSON.stringify(params[k]) : `"{${k}}"`
|
|
2734
|
-
)
|
|
2735
|
-
) as Record<string, unknown>;
|
|
2736
|
-
}
|
|
2737
|
-
|
|
2738
|
-
/**
|
|
2739
|
-
* BUG-004 fix: select best endpoint by schema richness, not just "first safe GET".
|
|
2740
|
-
* Prefers: safe endpoints with object/array response_schema > safe without > unsafe.
|
|
2741
|
-
*/
|
|
2742
|
-
// --- BM25 scoring for intent→endpoint relevance ---
|
|
2743
|
-
|
|
2744
|
-
/** Minimal stemmer: strip trailing s/es/ed/ing for matching */
|
|
2745
|
-
function stem(word: string): string {
|
|
2746
|
-
if (word.endsWith("ies") && word.length > 4) return word.slice(0, -3) + "y";
|
|
2747
|
-
// "messages" → "message" (not "messag"), "classes" → "class", "pages" → "page"
|
|
2748
|
-
if (word.endsWith("ses") || word.endsWith("ges") || word.endsWith("ces") || word.endsWith("zes")) return word.slice(0, -1);
|
|
2749
|
-
if (word.endsWith("es") && word.length > 4) return word.slice(0, -2);
|
|
2750
|
-
if (word.endsWith("s") && !word.endsWith("ss") && word.length > 3) return word.slice(0, -1);
|
|
2751
|
-
// "bookmarked" → "bookmark", "saved" → "save", "liked" → "like"
|
|
2752
|
-
if (word.endsWith("ed") && word.length > 4) return word.slice(0, -2);
|
|
2753
|
-
// "loading" → "load", "trending" → "trend" (but not "thing", "ring")
|
|
2754
|
-
if (word.endsWith("ing") && word.length > 5) return word.slice(0, -3);
|
|
2755
|
-
return word;
|
|
2756
|
-
}
|
|
2757
|
-
|
|
2758
|
-
const BM25_K1 = 1.2;
|
|
2759
|
-
const BM25_B = 0.75;
|
|
2760
|
-
const STOPWORDS = new Set([
|
|
2761
|
-
"the", "a", "an", "and", "or", "of", "to", "in", "for", "on", "with", "from",
|
|
2762
|
-
"get", "all", "this", "that", "is", "are", "was", "be", "it", "at", "by", "not",
|
|
2763
|
-
"com", "www", "https", "http", "html", "htm",
|
|
2764
|
-
]);
|
|
2765
|
-
|
|
2766
|
-
/** Expand tokens with synonyms/related terms for better recall */
|
|
2767
|
-
const SYNONYMS: Record<string, string[]> = {
|
|
2768
|
-
price: ["price", "prices", "pricing", "cost", "usd", "quote", "rate", "value", "market"],
|
|
2769
|
-
token: ["token", "tokens", "coin", "coins", "crypto", "currency", "asset"],
|
|
2770
|
-
search: ["search", "query", "find", "lookup", "filter", "dex"],
|
|
2771
|
-
chart: ["chart", "charts", "graph", "history", "ohlcv", "candle", "candles", "kline"],
|
|
2772
|
-
trade: ["trade", "trades", "swap", "swaps", "order", "orders", "transaction", "transactions"],
|
|
2773
|
-
volume: ["volume", "vol", "liquidity", "tvl"],
|
|
2774
|
-
pair: ["pair", "pairs", "pool", "pools"],
|
|
2775
|
-
trending: ["trending", "top", "hot", "gainers", "losers", "movers"],
|
|
2776
|
-
user: ["user", "users", "account", "accounts", "profile", "profiles", "member"],
|
|
2777
|
-
list: ["list", "lists", "all", "index", "browse", "catalog"],
|
|
2778
|
-
feed: ["feed", "feeds", "timeline", "stream", "home", "cards", "feedCards"],
|
|
2779
|
-
post: ["post", "posts", "article", "articles", "update", "updates", "content", "entry"],
|
|
2780
|
-
comment: ["comment", "comments", "reply", "replies", "discussion", "thread"],
|
|
2781
|
-
message: ["message", "messages", "messaging", "inbox", "conversation", "conversations", "chat"],
|
|
2782
|
-
notification: ["notification", "notifications", "alert", "alerts", "bell"],
|
|
2783
|
-
connection: ["connection", "connections", "follower", "followers", "following", "network", "contact", "contacts", "invitation", "invitations"],
|
|
2784
|
-
profile: ["profile", "profiles", "identity", "about", "bio", "member"],
|
|
2785
|
-
recommend: ["recommend", "recommendation", "recommendations", "suggested", "suggestion", "suggestions", "forYou"],
|
|
2786
|
-
bookmark: ["bookmark", "bookmarks", "bookmarked", "saved", "save", "favorite", "favourites"],
|
|
2787
|
-
news: ["news", "headline", "headlines", "story", "stories", "storylines"],
|
|
2788
|
-
dashboard: ["dashboard", "overview", "summary", "home", "main"],
|
|
2789
|
-
};
|
|
2790
|
-
|
|
2791
|
-
function normalizeTokenText(text: string): string {
|
|
2792
|
-
return text
|
|
2793
|
-
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
2794
|
-
.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
|
|
2795
|
-
.replace(/([a-zA-Z])(\d)/g, "$1 $2")
|
|
2796
|
-
.replace(/(\d)([a-zA-Z])/g, "$1 $2");
|
|
2797
|
-
}
|
|
2798
|
-
|
|
2799
|
-
function tokenize(text: string): string[] {
|
|
2800
|
-
return normalizeTokenText(text).toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((w) => w.length > 1 && !STOPWORDS.has(w));
|
|
2801
|
-
}
|
|
2802
|
-
|
|
2803
|
-
/** Expand intent tokens with synonyms + stemmed variants for better matching */
|
|
2804
|
-
function expandQuery(tokens: string[]): string[] {
|
|
2805
|
-
const expanded = new Set(tokens);
|
|
2806
|
-
for (const t of tokens) {
|
|
2807
|
-
const stemmed = stem(t);
|
|
2808
|
-
expanded.add(stemmed);
|
|
2809
|
-
// Look up synonyms by: raw token, stemmed token, or any SYNONYMS key that stems to the same value
|
|
2810
|
-
// (e.g. "messages" → stem "messag" matches SYNONYMS["message"] → stem "messag")
|
|
2811
|
-
let syns = SYNONYMS[t] ?? SYNONYMS[stemmed];
|
|
2812
|
-
if (!syns) {
|
|
2813
|
-
for (const key of Object.keys(SYNONYMS)) {
|
|
2814
|
-
if (stem(key) === stemmed) { syns = SYNONYMS[key]; break; }
|
|
2815
|
-
}
|
|
2816
|
-
}
|
|
2817
|
-
if (syns) for (const s of syns) { expanded.add(s); expanded.add(stem(s)); }
|
|
2818
|
-
}
|
|
2819
|
-
return [...expanded];
|
|
2820
|
-
}
|
|
2821
|
-
|
|
2822
|
-
/** Build a "document" from an endpoint: URL path segments + query params + schema property names */
|
|
2823
|
-
function endpointToTokens(ep: EndpointDescriptor): string[] {
|
|
2824
|
-
const tokens: string[] = [];
|
|
2825
|
-
try {
|
|
2826
|
-
const u = new URL(ep.url_template);
|
|
2827
|
-
// Path segments — split on delimiters AND camelCase to extract meaningful words
|
|
2828
|
-
// e.g. "BookmarkFoldersSlice" → ["Bookmark", "Folders", "Slice"]
|
|
2829
|
-
const rawSegments = u.pathname.split(/[/\-_.{}]/).filter((s) => s.length > 1 && !/^v\d+$/.test(s));
|
|
2830
|
-
for (const seg of rawSegments) {
|
|
2831
|
-
tokens.push(seg);
|
|
2832
|
-
// Also split camelCase: "BookmarkFoldersSlice" → ["Bookmark", "Folders", "Slice"]
|
|
2833
|
-
const camelParts = seg.split(/(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])/).filter((s) => s.length > 1);
|
|
2834
|
-
if (camelParts.length > 1) tokens.push(...camelParts);
|
|
2835
|
-
}
|
|
2836
|
-
// Hostname subdomains (e.g. "api" from api.dexscreener.com — strong signal)
|
|
2837
|
-
const hostParts = u.hostname.split(".");
|
|
2838
|
-
tokens.push(...hostParts.filter((s) => s.length > 2 && s !== "www" && s !== "com" && s !== "org" && s !== "net" && s !== "io"));
|
|
2839
|
-
// Query param names and values
|
|
2840
|
-
for (const [key, val] of u.searchParams.entries()) {
|
|
2841
|
-
tokens.push(key);
|
|
2842
|
-
if (val.length > 1 && val.length < 50) {
|
|
2843
|
-
tokens.push(...val.split(/[/\-_.]/).filter((s) => s.length > 1));
|
|
2844
|
-
} else if (val.length >= 50) {
|
|
2845
|
-
// Long values (e.g. graphql queryId): split on camelCase and delimiters to extract meaningful words
|
|
2846
|
-
const parts = val.split(/[/\-_.()]/).flatMap((s) => s.split(/(?<=[a-z])(?=[A-Z])/)).filter((s) => s.length > 1);
|
|
2847
|
-
tokens.push(...parts.slice(0, 10)); // cap to avoid noise from hashes
|
|
2848
|
-
}
|
|
2849
|
-
}
|
|
2850
|
-
} catch { /* skip */ }
|
|
2851
|
-
// Schema property names (strong signal — these describe the response data)
|
|
2852
|
-
if (ep.response_schema?.properties) {
|
|
2853
|
-
tokens.push(...Object.keys(ep.response_schema.properties));
|
|
2854
|
-
// Also add nested property names (1 level deep)
|
|
2855
|
-
for (const val of Object.values(ep.response_schema.properties) as Array<{ properties?: Record<string, unknown> }>) {
|
|
2856
|
-
if (val?.properties) tokens.push(...Object.keys(val.properties));
|
|
2857
|
-
}
|
|
2858
|
-
}
|
|
2859
|
-
// Trigger URL path segments — reveals which page triggered this API call
|
|
2860
|
-
// e.g., trigger_url="/i/bookmarks" adds "bookmarks" token for BM25 matching
|
|
2861
|
-
if (ep.trigger_url) {
|
|
2862
|
-
try {
|
|
2863
|
-
const tu = new URL(ep.trigger_url);
|
|
2864
|
-
tokens.push(...tu.pathname.split(/[/\-_.{}]/).filter((s) => s.length > 1 && !/^(i|app|en|v\d+)$/.test(s)));
|
|
2865
|
-
} catch { /* skip */ }
|
|
2866
|
-
}
|
|
2867
|
-
// LLM-generated description — strongest semantic signal for intent matching.
|
|
2868
|
-
// Tokenized words are added 3x to boost their BM25 weight over noisy URL tokens.
|
|
2869
|
-
if (ep.description) {
|
|
2870
|
-
const descTokens = ep.description.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((w) => w.length > 1 && !STOPWORDS.has(w));
|
|
2871
|
-
for (let i = 0; i < 3; i++) tokens.push(...descTokens);
|
|
2872
|
-
}
|
|
2873
|
-
return tokens.map((t) => stem(t.toLowerCase()));
|
|
2874
|
-
}
|
|
2875
|
-
|
|
2876
|
-
function bm25Score(query: string[], doc: string[], avgDl: number, docCount: number, docFreqs: Map<string, number>): number {
|
|
2877
|
-
const dl = doc.length;
|
|
2878
|
-
const tf = new Map<string, number>();
|
|
2879
|
-
for (const t of doc) tf.set(t, (tf.get(t) ?? 0) + 1);
|
|
2880
|
-
|
|
2881
|
-
let score = 0;
|
|
2882
|
-
for (const term of query) {
|
|
2883
|
-
const freq = tf.get(term) ?? 0;
|
|
2884
|
-
if (freq === 0) continue;
|
|
2885
|
-
// Real IDF: log((N - df + 0.5) / (df + 0.5) + 1) — terms appearing in fewer docs score higher
|
|
2886
|
-
const df = docFreqs.get(term) ?? 0;
|
|
2887
|
-
const idf = Math.log((docCount - df + 0.5) / (df + 0.5) + 1);
|
|
2888
|
-
const num = freq * (BM25_K1 + 1);
|
|
2889
|
-
const denom = freq + BM25_K1 * (1 - BM25_B + BM25_B * (dl / avgDl));
|
|
2890
|
-
score += idf * (num / denom);
|
|
2891
|
-
}
|
|
2892
|
-
return score;
|
|
2893
|
-
}
|
|
2894
|
-
|
|
2895
|
-
export interface RankedEndpoint {
|
|
2896
|
-
endpoint: EndpointDescriptor;
|
|
2897
|
-
score: number;
|
|
2898
|
-
}
|
|
2899
|
-
|
|
2900
|
-
function intentResourceKinds(intent?: string): string[] {
|
|
2901
|
-
const lower = (intent ?? "").toLowerCase();
|
|
2902
|
-
if (/\b(person|people|profile|profiles|user|users|member|members)\b/.test(lower)) return ["person", "people", "profile", "user", "member"];
|
|
2903
|
-
if (/\b(company|organization|org)\b/.test(lower)) return ["company", "organization", "org", "business"];
|
|
2904
|
-
if (/\b(post|posts|tweet|tweets|status|statuses|feed|timeline|stream|home)\b/.test(lower)) return ["post", "tweet", "status", "message", "feed", "timeline", "update"];
|
|
2905
|
-
if (/\b(topic|topics|trend|trending|hashtag|hashtags)\b/.test(lower)) return ["topic", "trend", "hashtag"];
|
|
2906
|
-
if (/\b(repo|repository|repositories)\b/.test(lower)) return ["repo", "repository", "project"];
|
|
2907
|
-
return [];
|
|
2908
|
-
}
|
|
2909
|
-
|
|
2910
|
-
function intentActionKinds(intent?: string): string[] {
|
|
2911
|
-
const lower = (intent ?? "").toLowerCase();
|
|
2912
|
-
if (/\b(feed|timeline|stream|home)\b/.test(lower)) return ["list", "feed", "timeline"];
|
|
2913
|
-
if (/\b(search|find|lookup)\b/.test(lower)) return ["search", "list"];
|
|
2914
|
-
if (/\b(get|fetch|view)\b/.test(lower)) return ["detail", "get", "fetch"];
|
|
2915
|
-
if (/\b(list|browse|discover|trending|top|latest)\b/.test(lower)) return ["list", "search"];
|
|
2916
|
-
return [];
|
|
2917
|
-
}
|
|
2918
|
-
|
|
2919
|
-
function isEntityDetailIntent(intent?: string): boolean {
|
|
2920
|
-
const lower = (intent ?? "").toLowerCase();
|
|
2921
|
-
return /\b(get|fetch|view)\b/.test(lower) && /\b(company|organization|org|business|person|people|profile|profiles|user|users|member|members)\b/.test(lower);
|
|
2922
|
-
}
|
|
2923
|
-
|
|
2924
|
-
function semanticIntentAdjustment(endpoint: EndpointDescriptor, intent?: string): number {
|
|
2925
|
-
const semantic = resolveEndpointSemantic(endpoint);
|
|
2926
|
-
if (!semantic || !intent) return 0;
|
|
2927
|
-
const resourceKinds = intentResourceKinds(intent);
|
|
2928
|
-
const actionKinds = intentActionKinds(intent);
|
|
2929
|
-
let delta = 0;
|
|
2930
|
-
|
|
2931
|
-
const resource = (semantic.resource_kind ?? "").toLowerCase();
|
|
2932
|
-
const action = (semantic.action_kind ?? "").toLowerCase();
|
|
2933
|
-
const negatives = new Set((semantic.negative_tags ?? []).map((tag) => tag.toLowerCase()));
|
|
2934
|
-
const haystack = [
|
|
2935
|
-
endpoint.url_template,
|
|
2936
|
-
endpoint.description ?? "",
|
|
2937
|
-
semantic.description_out ?? "",
|
|
2938
|
-
semantic.response_summary ?? "",
|
|
2939
|
-
].join(" ").toLowerCase();
|
|
2940
|
-
const uiScaffold = /(sharebox|closedsharebox|mailbox|messaging|conversation|notification|notifications|alerts?|presence|badging|launchpad|previewbanner|main_feed|feedtype)/i.test(haystack);
|
|
2941
|
-
|
|
2942
|
-
if (resourceKinds.length > 0) {
|
|
2943
|
-
if (resourceKinds.some((kind) => resource.includes(kind) || kind.includes(resource))) delta += 80;
|
|
2944
|
-
else if (resource) delta -= 90;
|
|
2945
|
-
}
|
|
2946
|
-
|
|
2947
|
-
if (actionKinds.length > 0) {
|
|
2948
|
-
if (actionKinds.some((kind) => action.includes(kind) || kind.includes(action))) delta += 25;
|
|
2949
|
-
else if (action) delta -= 25;
|
|
2950
|
-
}
|
|
2951
|
-
|
|
2952
|
-
if (negatives.has("config") || negatives.has("telemetry") || negatives.has("experiment") || negatives.has("auth")) {
|
|
2953
|
-
delta -= 60;
|
|
2954
|
-
}
|
|
2955
|
-
if (negatives.has("adjacent") || negatives.has("ads")) {
|
|
2956
|
-
delta -= 90;
|
|
2957
|
-
}
|
|
2958
|
-
if (uiScaffold && (resourceKinds.length > 0 || actionKinds.length > 0)) {
|
|
2959
|
-
delta -= 220;
|
|
2960
|
-
}
|
|
2961
|
-
|
|
2962
|
-
return delta;
|
|
2963
|
-
}
|
|
2964
|
-
|
|
2965
|
-
/**
|
|
2966
|
-
* Rank endpoints by relevance to intent using BM25 + structural bonuses.
|
|
2967
|
-
* Exported so routes.ts can surface the ranked list to the agent.
|
|
2968
|
-
*/
|
|
2969
|
-
export function rankEndpoints(endpoints: EndpointDescriptor[], intent?: string, skillDomain?: string, contextUrl?: string): RankedEndpoint[] {
|
|
2970
|
-
// --- Hard-filter: hosts that NEVER contain useful data ---
|
|
2971
|
-
const NOISE_HOSTS = /(id5-sync\.com|btloader\.com|presage\.io|onetrust\.com|adsrvr\.org|googlesyndication\.com|adtrafficquality\.google|amazon-adsystem\.com|crazyegg\.com|challenges\.cloudflare\.com|google-analytics\.com|doubleclick\.net|gstatic\.com|accounts\.google\.com|login\.microsoftonline\.com|auth0\.com|cognito-idp\.|protechts\.net|demdex\.net|datadoghq\.com|fullstory\.com|launchdarkly\.com|intercom\.io|sentry\.io|segment\.io|amplitude\.com|mixpanel\.com|hotjar\.com|clarity\.ms|googletagmanager\.com|walletconnect\.com|cloudflareinsights\.com|fonts\.googleapis\.com|recaptcha|waa-pa\.|signaler-pa\.|ogads-pa\.|reddit\.com\/pixels?|pixel-config\.|dns-finder\.com|cookieconsentpub|firebase\.googleapis\.com|firebaseinstallations\.googleapis\.com|identitytoolkit\.googleapis\.com|securetoken\.googleapis\.com|apis\.google\.com|connect\.facebook\.net|bat\.bing\.com|static\.cloudflareinsights\.com|cdn\.mxpnl\.com|js\.hs-analytics\.net|snap\.licdn\.com|clc\.stackoverflow\.com|px\.ads|t\.co\/i|analytics\.|telemetry\.|stats\.)/i;
|
|
2972
|
-
|
|
2973
|
-
// Noise URL path patterns — tracking, telemetry, logging
|
|
2974
|
-
const NOISE_PATHS = /\/(track|pixel|telemetry|beacon|csp-report|litms|demdex|analytics|protechts|collect|tr\/|gen_204|generate_204|log$|logging|heartbeat|metrics|consent|sodar|tag$|event$|events$|impression|pageview|click|__)/i;
|
|
2975
|
-
|
|
2976
|
-
// Auth/session/config — on-domain but not data
|
|
2977
|
-
const AUTH_CONFIG_PATHS = /\/(csrf_meta|logged_in_user|analytics_user_data|onboarding|geolocation|auth|login|logout|register|signup|session|webConfig|config\.json|manifest\.json|robots\.txt|sitemap|favicon|opensearch|service-worker|sw\.js)\b/i;
|
|
2978
|
-
|
|
2979
|
-
// Session plumbing — infrastructure endpoints no user would ever want as data.
|
|
2980
|
-
// Only true noise: account config, badge counts, feature flags, telemetry, DM settings.
|
|
2981
|
-
// NOT filtered: HomeTimeline, Bookmarks, Notifications, UserByScreenName, etc. — real data.
|
|
2982
|
-
const SESSION_PLUMBING = /(account\/settings|account\/multi|badge_count|DataSaverMode|permissionsState|email_phone_info|live_pipeline|user_flow|strato\/column|ces\/p2|IntercomStarter|getAltText|fleetline|FeatureHelper|VerifiedAvatar|ScheduledPromotion|DirectCall|DmSettings|PinnedTimeline)/i;
|
|
2983
|
-
|
|
2984
|
-
// Static assets
|
|
2985
|
-
const STATIC_ASSET_PATTERNS = /\.(woff2?|ttf|eot|css|js|mjs|png|jpg|jpeg|gif|svg|ico|webp|avif|mp4|mp3|wav|riv|lottie|wasm)(\?|%3F|$)/i;
|
|
2986
|
-
|
|
2987
|
-
// Animation/UI asset paths
|
|
2988
|
-
const UI_ASSET_PATHS = /\/(rive|lottie|animations?|sprites?|assets\/static)\//i;
|
|
2989
|
-
const filtered = endpoints.filter((ep) => {
|
|
2990
|
-
if (ep.method === "HEAD" || ep.method === "OPTIONS") return false;
|
|
2991
|
-
if (ep.verification_status === "disabled") return false;
|
|
2992
|
-
if (STATIC_ASSET_PATTERNS.test(ep.url_template)) return false;
|
|
2993
|
-
if (UI_ASSET_PATHS.test(ep.url_template)) return false;
|
|
2994
|
-
try {
|
|
2995
|
-
const host = new URL(ep.url_template).hostname;
|
|
2996
|
-
if (NOISE_HOSTS.test(host)) return false;
|
|
2997
|
-
} catch { /* skip */ }
|
|
2998
|
-
if (NOISE_PATHS.test(ep.url_template)) return false;
|
|
2999
|
-
if (AUTH_CONFIG_PATHS.test(ep.url_template)) return false;
|
|
3000
|
-
if (SESSION_PLUMBING.test(ep.url_template)) return false;
|
|
3001
|
-
return true;
|
|
3002
|
-
});
|
|
3003
|
-
|
|
3004
|
-
const nonDisabled = endpoints.filter((ep) => ep.verification_status !== "disabled");
|
|
3005
|
-
const candidates = filtered.length > 0 ? filtered : nonDisabled;
|
|
3006
|
-
if (candidates.length === 0) return [];
|
|
3007
|
-
const intentLower = (intent ?? "").toLowerCase();
|
|
3008
|
-
|
|
3009
|
-
function endpointHaystack(ep: EndpointDescriptor): string {
|
|
3010
|
-
return `${ep.url_template} ${ep.description ?? ""} ${JSON.stringify(ep.response_schema ?? {})} ${JSON.stringify(resolveEndpointSemantic(ep) ?? {})}`.toLowerCase();
|
|
3011
|
-
}
|
|
3012
|
-
|
|
3013
|
-
function isPlausibleForIntent(ep: EndpointDescriptor): boolean {
|
|
3014
|
-
if (!intentLower) return true;
|
|
3015
|
-
const haystack = endpointHaystack(ep);
|
|
3016
|
-
|
|
3017
|
-
if (/\b(stock|stocks|ticker|tickers|quote|quotes)\b/.test(intentLower)) {
|
|
3018
|
-
const hasPositive = /(symbol|ticker|regularmarketprice|currentprice|marketcap|currency|change_percent|changepercent|quote)/i.test(haystack);
|
|
3019
|
-
const hasNegative = /(news|article|articles|video|story|stories|thumbnail|image|author)/i.test(haystack);
|
|
3020
|
-
return hasPositive && !hasNegative;
|
|
3021
|
-
}
|
|
3022
|
-
|
|
3023
|
-
if (/\b(product|products|item|items)\b/.test(intentLower)) {
|
|
3024
|
-
const hasPositive = /(product|products|itemstacks|items\[\]|price|rating|review|reviewcount|numberofreviews|brand|sku|usitemid|catalogproducttype)/i.test(haystack);
|
|
3025
|
-
const hasNegative = /(captcha|robot|human|bootstrapdata|traceparent|nonce|psych|isomorphicsessionid|persistedqueriesconfig|errorloggingconfig|renderviewid|headerobj|initialtempodata|wcpbeacon)/i.test(haystack);
|
|
3026
|
-
return hasPositive && !hasNegative;
|
|
3027
|
-
}
|
|
3028
|
-
|
|
3029
|
-
if (/\b(channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(intentLower)) {
|
|
3030
|
-
const hasEntitySignal = /(\/guilds\b|\/channels\b|\bguilds?\b|\bchannels?\b|\bservers?\b|\bworkspaces?\b)/i.test(haystack);
|
|
3031
|
-
const hasFieldSignal = /\b(ids?|names?|icon|member_count|topic|description)\b/i.test(haystack);
|
|
3032
|
-
const hasPositive = hasEntitySignal && hasFieldSignal;
|
|
3033
|
-
const hasNegative = /(affinit|preview|quests|survey|referrals?|promotions?|science|detectable|applications\/public|\/games\b|entitlements?|billing|subscriptions?|collectibles?|gifts?|experiments?|connections?|status|incidents?|scheduled-maintenances?)/i.test(haystack);
|
|
3034
|
-
return hasPositive && !hasNegative;
|
|
3035
|
-
}
|
|
3036
|
-
|
|
3037
|
-
return true;
|
|
3038
|
-
}
|
|
3039
|
-
|
|
3040
|
-
const plausibilityScopedIntent = /\b(stock|stocks|ticker|tickers|quote|quotes|product|products|item|items|channel|channels|server|servers|guild|guilds|workspace|workspaces)\b/.test(intentLower);
|
|
3041
|
-
const plausibleCandidates = candidates.filter((ep) => isPlausibleForIntent(ep));
|
|
3042
|
-
if (plausibilityScopedIntent && plausibleCandidates.length === 0) return [];
|
|
3043
|
-
const rankedCandidates = plausibleCandidates.length > 0 ? plausibleCandidates : candidates;
|
|
3044
|
-
const canonicalReplayTriggers = new Set(
|
|
3045
|
-
rankedCandidates
|
|
3046
|
-
.filter((ep) => isCanonicalReplayEndpoint(ep))
|
|
3047
|
-
.map((ep) => ep.trigger_url)
|
|
3048
|
-
.filter((value): value is string => !!value),
|
|
3049
|
-
);
|
|
3050
|
-
const structuredApiTriggers = new Set(
|
|
3051
|
-
rankedCandidates
|
|
3052
|
-
.filter((ep) => {
|
|
3053
|
-
const url = ep.url_template.toLowerCase();
|
|
3054
|
-
const looksLikeApiEndpoint = /\/api\/|graphql|\/rest\/|\/rpc\/|voyager/i.test(url);
|
|
3055
|
-
return !!ep.trigger_url && !ep.dom_extraction && (looksLikeApiEndpoint || !!ep.response_schema || ep.method === "WS");
|
|
3056
|
-
})
|
|
3057
|
-
.map((ep) => ep.trigger_url)
|
|
3058
|
-
.filter((value): value is string => !!value),
|
|
3059
|
-
);
|
|
3060
|
-
|
|
3061
|
-
// Tokenize intent with synonym expansion for better recall
|
|
3062
|
-
const rawTokens = intent ? tokenize(intent) : [];
|
|
3063
|
-
const queryTokens = rawTokens.length > 0 ? expandQuery(rawTokens) : [];
|
|
3064
|
-
const docs = rankedCandidates.map((ep) => endpointToTokens(ep));
|
|
3065
|
-
const avgDl = docs.reduce((sum, d) => sum + d.length, 0) / docs.length || 1;
|
|
3066
|
-
|
|
3067
|
-
// Build corpus-level document frequencies for real IDF
|
|
3068
|
-
const docFreqs = new Map<string, number>();
|
|
3069
|
-
for (const doc of docs) {
|
|
3070
|
-
const seen = new Set(doc);
|
|
3071
|
-
for (const t of seen) docFreqs.set(t, (docFreqs.get(t) ?? 0) + 1);
|
|
3072
|
-
}
|
|
3073
|
-
const docCount = docs.length;
|
|
3074
|
-
|
|
3075
|
-
// Meta/support/promo/config path patterns — not primary data
|
|
3076
|
-
const META_PATHS = /\/(annotation|insight|sentiment|vote|portfolio|summary_button|summary_card|tagmetric|quick_add|notifications?|preferences|settings|onboarding|public\/active|remoteConfig|banner\/metadata|embedded-wallets|glow\/get-rendered)/i;
|
|
3077
|
-
|
|
3078
|
-
// Data format indicators
|
|
3079
|
-
const DATA_INDICATORS = /\.(json|xml|csv)(\?|$)|\/api\//i;
|
|
3080
|
-
|
|
3081
|
-
// Currency/time patterns — strong price/financial signal
|
|
3082
|
-
const CURRENCY_TIME_PATTERNS = /\/(usd|eur|gbp|btc|eth|sol|cny|jpy|krw|24_hours|7_days|30_days|1_year|max|hourly|daily|weekly|price|prices|market|markets|ticker|tickers|quote|quotes|ohlcv?|candles?|klines?)\b/i;
|
|
3083
|
-
|
|
3084
|
-
// API subdomain pattern — "api.example.com" or "io.example.com" strongly suggests data endpoint
|
|
3085
|
-
const API_SUBDOMAIN = /^(api|io|data|feed|stream|ws)\./i;
|
|
3086
|
-
const LIST_INTENT = /\b(search|list|find|trending|top|latest|discover|browse)\b/i;
|
|
3087
|
-
const STATUS_INTENT = /\b(status|incident|outage|maintenance|uptime|degraded)\b/i;
|
|
3088
|
-
const COMMS_INTENT = /\b(guilds?|channels?|messages?|dms?|servers?|threads?|chat)\b/i;
|
|
3089
|
-
const COMMS_PATH = /\/(guilds?|channels?|messages?|threads?|conversations?|affinities)\b/i;
|
|
3090
|
-
const DISCORD_META_PATHS = /\/(referrals?|promotions?|science|entitlements?|billing|subscriptions?|collectibles?|gifts?|experiments?)\b/i;
|
|
3091
|
-
const SESSION_BOUND_QUERY = /[?&](?:[^=]*?(crumb|csrf|xsrf|token|session|auth|signature|nonce))=\{/i;
|
|
3092
|
-
const COMPANY_INTENT = /\b(company|companies|organization|organisations|business|org)\b/i;
|
|
3093
|
-
const PROFILE_INTENT = /\b(person|people|profile|profiles|user|users|member|members)\b/i;
|
|
3094
|
-
const PRODUCT_DETAIL_INTENT = /\b(product|products|item|items|listing|listings)\b/i.test(intent ?? "");
|
|
3095
|
-
const ENTITY_DETAIL_INTENT = isEntityDetailIntent(intent);
|
|
3096
|
-
|
|
3097
|
-
const scored = rankedCandidates.map((ep, i) => {
|
|
3098
|
-
let score = 0;
|
|
3099
|
-
let pathname = "";
|
|
3100
|
-
let hostname = "";
|
|
3101
|
-
let contextPath = "";
|
|
3102
|
-
let contextLeaf = "";
|
|
3103
|
-
let contextQueryKeys = new Set<string>();
|
|
3104
|
-
const semantic = resolveEndpointSemantic(ep);
|
|
3105
|
-
const descriptionMeta = getEndpointDescriptionMetadata({
|
|
3106
|
-
description: ep.description,
|
|
3107
|
-
semantic,
|
|
3108
|
-
});
|
|
3109
|
-
try {
|
|
3110
|
-
const u = new URL(ep.url_template);
|
|
3111
|
-
pathname = u.pathname;
|
|
3112
|
-
hostname = u.hostname;
|
|
3113
|
-
} catch { /* skip */ }
|
|
3114
|
-
try {
|
|
3115
|
-
if (contextUrl) {
|
|
3116
|
-
const cu = new URL(contextUrl);
|
|
3117
|
-
contextPath = cu.pathname;
|
|
3118
|
-
const contextSegs = cu.pathname.split("/").filter(Boolean);
|
|
3119
|
-
contextLeaf = contextSegs.length > 0 ? decodeURIComponent(contextSegs[contextSegs.length - 1] ?? "").toLowerCase() : "";
|
|
3120
|
-
contextQueryKeys = new Set([...cu.searchParams.keys()]);
|
|
3121
|
-
}
|
|
3122
|
-
} catch { /* skip */ }
|
|
3123
|
-
|
|
3124
|
-
// === BM25 relevance to intent (primary signal, weighted heavily) ===
|
|
3125
|
-
if (queryTokens.length > 0) {
|
|
3126
|
-
score += bm25Score(queryTokens, docs[i], avgDl, docCount, docFreqs) * 20;
|
|
3127
|
-
}
|
|
3128
|
-
|
|
3129
|
-
// === Description match bonus — separate from BM25 to avoid IDF dilution ===
|
|
3130
|
-
// When an endpoint has a description, compute direct token overlap with RAW intent
|
|
3131
|
-
// (not synonym-expanded, to avoid dilution). Each matching core intent token gives a
|
|
3132
|
-
// massive bonus that overrides structural noise from schema richness.
|
|
3133
|
-
if (descriptionMeta.source === "agent" && descriptionMeta.display && rawTokens.length > 0) {
|
|
3134
|
-
const descTokens = new Set(
|
|
3135
|
-
descriptionMeta.display.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/)
|
|
3136
|
-
.filter((w) => w.length > 1 && !STOPWORDS.has(w))
|
|
3137
|
-
.map((w) => stem(w))
|
|
3138
|
-
);
|
|
3139
|
-
// Use raw intent tokens (not expanded) — "feed" and "post" are the core signal
|
|
3140
|
-
const rawStems = new Set(rawTokens.map((t) => stem(t)));
|
|
3141
|
-
let matches = 0;
|
|
3142
|
-
for (const t of rawStems) {
|
|
3143
|
-
if (descTokens.has(t)) matches++;
|
|
3144
|
-
}
|
|
3145
|
-
// Each matching core token = +100 points. "feed" matching gives +100,
|
|
3146
|
-
// "feed" + "post" matching gives +200, etc.
|
|
3147
|
-
score += matches * 100;
|
|
3148
|
-
}
|
|
3149
|
-
|
|
3150
|
-
// === Structural bonuses ===
|
|
3151
|
-
if (ep.dom_extraction) score += 25;
|
|
3152
|
-
if (descriptionMeta.needs_review && ep.dom_extraction) score -= 140;
|
|
3153
|
-
if (isCanonicalReplayEndpoint(ep)) score += 160;
|
|
3154
|
-
if (ep.idempotency === "safe" || ep.method === "GET") score += 5;
|
|
3155
|
-
if (isBundleInferredEndpoint(ep) && !ep.response_schema) score -= 180;
|
|
3156
|
-
score += semanticIntentAdjustment(ep, intent);
|
|
3157
|
-
|
|
3158
|
-
// Rich schema = likely structured data endpoint
|
|
3159
|
-
if (ep.response_schema) {
|
|
3160
|
-
score += 5;
|
|
3161
|
-
if (ep.response_schema.type === "array") score += 10;
|
|
3162
|
-
else if (ep.response_schema.type === "object" && ep.response_schema.properties) {
|
|
3163
|
-
const propCount = Object.keys(ep.response_schema.properties).length;
|
|
3164
|
-
score += Math.min(propCount * 2, 20);
|
|
3165
|
-
}
|
|
3166
|
-
}
|
|
3167
|
-
score += ep.reliability_score * 5;
|
|
3168
|
-
if (ep.method === "WS" && ep.response_schema) score += 3;
|
|
3169
|
-
|
|
3170
|
-
// === Domain affinity ===
|
|
3171
|
-
if (skillDomain) {
|
|
3172
|
-
try {
|
|
3173
|
-
if (hostname === skillDomain || hostname.endsWith(`.${skillDomain}`)) {
|
|
3174
|
-
score += 15;
|
|
3175
|
-
// Extra bonus for API subdomains on the skill domain
|
|
3176
|
-
if (API_SUBDOMAIN.test(hostname)) score += 15;
|
|
3177
|
-
} else {
|
|
3178
|
-
// Off-domain = almost never right
|
|
3179
|
-
score -= 30;
|
|
3180
|
-
}
|
|
3181
|
-
} catch { /* skip */ }
|
|
3182
|
-
}
|
|
3183
|
-
|
|
3184
|
-
// API subdomain bonus even without skill domain context
|
|
3185
|
-
if (API_SUBDOMAIN.test(hostname)) score += 10;
|
|
3186
|
-
|
|
3187
|
-
// Strongly penalize dedicated status/statuspage endpoints unless the user explicitly
|
|
3188
|
-
// asked for status/incidents/maintenance. These often hijack root-domain skills.
|
|
3189
|
-
if (!STATUS_INTENT.test(intent ?? "")) {
|
|
3190
|
-
if (/^(status|statuspage)\./i.test(hostname) || /\/(scheduled-maintenances|incidents|components|status|uptime|summary)\b/i.test(pathname)) {
|
|
3191
|
-
score -= 80;
|
|
3192
|
-
}
|
|
3193
|
-
}
|
|
3194
|
-
|
|
3195
|
-
// === Data-relevance signals ===
|
|
3196
|
-
if (DATA_INDICATORS.test(ep.url_template)) score += 5;
|
|
3197
|
-
if (CURRENCY_TIME_PATTERNS.test(pathname)) score += 15;
|
|
3198
|
-
if (intent && COMMS_INTENT.test(intent) && COMMS_PATH.test(pathname)) score += 45;
|
|
3199
|
-
if (intent && COMMS_INTENT.test(intent) && DISCORD_META_PATHS.test(pathname)) score -= 220;
|
|
3200
|
-
if (/\b(stock|stocks|ticker|tickers|quote|quotes)\b/i.test(intent ?? "")) {
|
|
3201
|
-
const quoteHaystack = `${ep.url_template} ${ep.description ?? ""} ${JSON.stringify(ep.response_schema ?? {})} ${JSON.stringify(semantic)}`.toLowerCase();
|
|
3202
|
-
if (/\/chart\b/i.test(pathname) && /(regularmarketprice|currentprice|previousclose|chartpreviousclose|price)/i.test(quoteHaystack)) {
|
|
3203
|
-
score += 120;
|
|
3204
|
-
}
|
|
3205
|
-
if (SESSION_BOUND_QUERY.test(ep.url_template)) {
|
|
3206
|
-
score -= 170;
|
|
3207
|
-
}
|
|
3208
|
-
}
|
|
3209
|
-
|
|
3210
|
-
// Deep paths with meaningful segments = likely data endpoints
|
|
3211
|
-
const pathDepth = pathname.split("/").filter((s) => s.length > 0).length;
|
|
3212
|
-
if (pathDepth >= 3) score += 5;
|
|
3213
|
-
|
|
3214
|
-
// === Context URL match — endpoint was captured from the page the user is asking about ===
|
|
3215
|
-
if (contextUrl && ep.trigger_url) {
|
|
3216
|
-
try {
|
|
3217
|
-
const contextPath = new URL(contextUrl).pathname;
|
|
3218
|
-
const triggerPath = new URL(ep.trigger_url).pathname;
|
|
3219
|
-
if (triggerPath === contextPath) score += 20;
|
|
3220
|
-
} catch { /* skip */ }
|
|
3221
|
-
}
|
|
3222
|
-
|
|
3223
|
-
// Direct endpoint/context path match. Stronger than trigger_url because marketplace
|
|
3224
|
-
// skills may have stale or missing trigger_url, but url_template still reflects intent.
|
|
3225
|
-
if (contextPath) {
|
|
3226
|
-
if (pathname === contextPath) score += 45;
|
|
3227
|
-
else if (pathname.startsWith(contextPath) || contextPath.startsWith(pathname)) score += 20;
|
|
3228
|
-
|
|
3229
|
-
const contextSegs = contextPath.split("/").filter(Boolean);
|
|
3230
|
-
const endpointSegs = pathname.split("/").filter(Boolean);
|
|
3231
|
-
if (contextSegs.length > 0 && endpointSegs.length > 0 && contextSegs[0] === endpointSegs[0]) {
|
|
3232
|
-
score += 12;
|
|
3233
|
-
}
|
|
3234
|
-
|
|
3235
|
-
if (contextQueryKeys.size > 0) {
|
|
3236
|
-
let matchedKeys = 0;
|
|
3237
|
-
for (const key of contextQueryKeys) {
|
|
3238
|
-
if (ep.url_template.includes(`${key}=`) || ep.url_template.includes(`{${key}}`)) matchedKeys++;
|
|
3239
|
-
}
|
|
3240
|
-
score += matchedKeys * 12;
|
|
3241
|
-
if (matchedKeys === 0 && /\/search\b/.test(contextPath)) score -= 20;
|
|
3242
|
-
}
|
|
3243
|
-
}
|
|
3244
|
-
|
|
3245
|
-
const looksLikeApiEndpoint = /\/api\/|graphql|\/rest\/|\/rpc\/|voyager/i.test(ep.url_template);
|
|
3246
|
-
const looksLikeDocumentRoute = !!contextPath && pathname === contextPath && !looksLikeApiEndpoint;
|
|
3247
|
-
const isCapturedPageArtifact = /captured (?:search form |page )?artifact/i.test(ep.description ?? "");
|
|
3248
|
-
const hasCanonicalReplaySibling = !!ep.trigger_url && canonicalReplayTriggers.has(ep.trigger_url);
|
|
3249
|
-
const hasStructuredApiSibling = !!ep.trigger_url && structuredApiTriggers.has(ep.trigger_url);
|
|
3250
|
-
const triggerPath = (() => {
|
|
3251
|
-
try {
|
|
3252
|
-
return ep.trigger_url ? new URL(ep.trigger_url).pathname : "";
|
|
3253
|
-
} catch {
|
|
3254
|
-
return "";
|
|
3255
|
-
}
|
|
3256
|
-
})();
|
|
3257
|
-
const exactContextDocument =
|
|
3258
|
-
PRODUCT_DETAIL_INTENT &&
|
|
3259
|
-
!!contextPath &&
|
|
3260
|
-
(pathname === contextPath || triggerPath === contextPath);
|
|
3261
|
-
const mismatchedContextDocument =
|
|
3262
|
-
!!contextPath &&
|
|
3263
|
-
(isCapturedPageArtifact || looksLikeDocumentRoute) &&
|
|
3264
|
-
pathname !== contextPath &&
|
|
3265
|
-
triggerPath !== contextPath;
|
|
3266
|
-
|
|
3267
|
-
if (ENTITY_DETAIL_INTENT && looksLikeDocumentRoute && !exactContextDocument) {
|
|
3268
|
-
score -= 55;
|
|
3269
|
-
}
|
|
3270
|
-
if (ENTITY_DETAIL_INTENT && isCapturedPageArtifact && !exactContextDocument) {
|
|
3271
|
-
score -= 200;
|
|
3272
|
-
}
|
|
3273
|
-
if (ENTITY_DETAIL_INTENT && mismatchedContextDocument) {
|
|
3274
|
-
score -= 420;
|
|
3275
|
-
}
|
|
3276
|
-
if (intent && COMMS_INTENT.test(intent) && looksLikeDocumentRoute) {
|
|
3277
|
-
score -= 180;
|
|
3278
|
-
}
|
|
3279
|
-
if (intent && COMMS_INTENT.test(intent) && isCapturedPageArtifact) {
|
|
3280
|
-
score -= 1000;
|
|
3281
|
-
}
|
|
3282
|
-
|
|
3283
|
-
if (intent && COMPANY_INTENT.test(intent)) {
|
|
3284
|
-
const companyHaystack = `${ep.url_template} ${ep.description ?? ""} ${JSON.stringify(ep.response_schema ?? {})}`.toLowerCase();
|
|
3285
|
-
if (/(organization|company|companies|org)/i.test(companyHaystack) && looksLikeApiEndpoint) score += 110;
|
|
3286
|
-
if (/(mailbox|messaging|messagecenter|notifications?|inbox|launchpad|identity|sharebox)/i.test(companyHaystack)) score -= 140;
|
|
3287
|
-
if (/(organizationdashcompanies|universalname|companyprofile|organizationprofile|aboutthisprofile|organizationresult|companyresult)/i.test(companyHaystack)) score += 95;
|
|
3288
|
-
if (looksLikeDocumentRoute) score -= 35;
|
|
3289
|
-
}
|
|
3290
|
-
|
|
3291
|
-
if (intent && PROFILE_INTENT.test(intent)) {
|
|
3292
|
-
const profileHaystack = `${ep.url_template} ${ep.description ?? ""} ${JSON.stringify(ep.response_schema ?? {})}`.toLowerCase();
|
|
3293
|
-
if (/(sidebar|recommend|recommendations|suggested|spotlight|timeline|tweets|following|followers)/i.test(profileHaystack)) score -= 90;
|
|
3294
|
-
if (/(sharebox|closedsharebox|mailbox|messaging|conversation|alerts?|notification|presence|badging|feedtype|main_feed)/i.test(profileHaystack)) score -= 180;
|
|
3295
|
-
if (/(userbyscreenname|profile|profiles|memberprofile|identityprofile|person)/i.test(profileHaystack) && looksLikeApiEndpoint) score += 80;
|
|
3296
|
-
if (/(search\/results\/people|searchcluster|searchresult|public_identifier|headline|mini_profile|memberresult)/i.test(profileHaystack)) score += 95;
|
|
3297
|
-
}
|
|
3298
|
-
|
|
3299
|
-
if (intent && /\b(feed|timeline|stream|home)\b/i.test(intent) && /\b(post|posts|status|statuses|update|updates)\b/i.test(intent)) {
|
|
3300
|
-
const feedHaystack = `${ep.url_template} ${ep.description ?? ""} ${JSON.stringify(semantic)}`.toLowerCase();
|
|
3301
|
-
if (/(voyagerfeeddashmainfeed|voyagerfeeddashfeedupdates|mainfeed|feedupdates|main_feed)/i.test(feedHaystack)) score += 170;
|
|
3302
|
-
if (/(identitydashprofiles|voyageridentity|storylines|newsdashstorylines|globalnav|launchpad|mailbox|notification|presence)/i.test(feedHaystack)) score -= 150;
|
|
3303
|
-
}
|
|
3304
|
-
if (intent && /\b(search|list|find|feed|timeline|stream|home|latest|trending|discover|browse)\b/i.test(intent) && /\b(post|posts|tweet|tweets|status|statuses|update|updates)\b/i.test(intent)) {
|
|
3305
|
-
const contentHaystack = `${ep.url_template} ${ep.description ?? ""} ${JSON.stringify(semantic)}`.toLowerCase();
|
|
3306
|
-
if (looksLikeApiEndpoint && /(search|timeline|feed|stream|result|results|entries|posts|tweets|statuses|updates)/i.test(contentHaystack)) score += 180;
|
|
3307
|
-
if (/(sidebar|recommend|recommendations|usersbyrestids|user details|profile|profiles|followers|following|people|spotlight)/i.test(contentHaystack)) score -= 140;
|
|
3308
|
-
if (isCapturedPageArtifact && hasStructuredApiSibling) score -= 320;
|
|
3309
|
-
else if (looksLikeDocumentRoute && hasStructuredApiSibling) score -= 200;
|
|
3310
|
-
}
|
|
3311
|
-
|
|
3312
|
-
const requestHint = JSON.stringify(semantic.example_request ?? {}).toLowerCase();
|
|
3313
|
-
const endpointHint = `${ep.url_template} ${ep.description ?? ""}`.toLowerCase();
|
|
3314
|
-
const hasConcreteEntityRoute =
|
|
3315
|
-
ENTITY_DETAIL_INTENT &&
|
|
3316
|
-
!!contextLeaf &&
|
|
3317
|
-
!/^(search|explore|trending|tabs|home|for-you|foryou|latest|live|people|posts|videos)$/.test(contextLeaf);
|
|
3318
|
-
if (hasConcreteEntityRoute) {
|
|
3319
|
-
if (requestHint.includes(contextLeaf)) score += 120;
|
|
3320
|
-
else if (endpointHint.includes(contextLeaf)) score += 40;
|
|
3321
|
-
if (/(screen_name|screenname|username|userby|slug|vanity|universalname|public_identifier|identifier)/i.test(endpointHint + " " + requestHint)) score += 55;
|
|
3322
|
-
if (/(restids|usersbyrestids|recommendations|timeline|tweets|following|followers)/i.test(endpointHint + " " + requestHint)) score -= 70;
|
|
3323
|
-
}
|
|
3324
|
-
|
|
3325
|
-
// Penalize fixed entity/detail pages when the user asked for a list/search flow.
|
|
3326
|
-
const isStaticEntityPath = /^\/[^/{?]+\/[^/{?]+$/.test(pathname);
|
|
3327
|
-
if (intent && LIST_INTENT.test(intent) && isStaticEntityPath) {
|
|
3328
|
-
score -= 35;
|
|
3329
|
-
}
|
|
3330
|
-
|
|
3331
|
-
// Reward endpoints whose path explicitly names the list/search surface the user is on.
|
|
3332
|
-
if (intent && LIST_INTENT.test(intent) && /\/(search|trending|discover|explore)\b/i.test(pathname)) {
|
|
3333
|
-
score += 30;
|
|
3334
|
-
}
|
|
3335
|
-
|
|
3336
|
-
// === Penalties ===
|
|
3337
|
-
if (META_PATHS.test(pathname)) score -= 15;
|
|
3338
|
-
if (DISCORD_META_PATHS.test(pathname)) score -= 35;
|
|
3339
|
-
if (SESSION_PLUMBING.test(pathname) || SESSION_PLUMBING.test(ep.url_template)) score -= 30;
|
|
3340
|
-
if (isBundleInferredEndpoint(ep) && !ep.response_schema) score -= 40;
|
|
3341
|
-
|
|
3342
|
-
// Penalize root/short paths (homepage, config, init)
|
|
3343
|
-
if (pathname.length <= 2) score -= 10;
|
|
3344
|
-
|
|
3345
|
-
// Penalize POST endpoints that aren't explicitly API calls (likely tracking/events)
|
|
3346
|
-
if (ep.method === "POST" && !DATA_INDICATORS.test(ep.url_template) && !ep.response_schema) {
|
|
3347
|
-
score -= 15;
|
|
3348
|
-
}
|
|
3349
|
-
|
|
3350
|
-
if (intent && COMMS_INTENT.test(intent) && isCapturedPageArtifact) {
|
|
3351
|
-
score = Math.min(score, -400);
|
|
3352
|
-
}
|
|
3353
|
-
if (hasCanonicalReplaySibling && ep.dom_extraction && !isCanonicalReplayEndpoint(ep)) {
|
|
3354
|
-
score -= 260;
|
|
3355
|
-
}
|
|
3356
|
-
if (descriptionMeta.needs_review && isCapturedPageArtifact) {
|
|
3357
|
-
score -= 120;
|
|
3358
|
-
}
|
|
3359
|
-
|
|
3360
|
-
return { endpoint: ep, score };
|
|
3361
|
-
});
|
|
3362
|
-
|
|
3363
|
-
scored.sort((a, b) => b.score - a.score);
|
|
3364
|
-
if (plausibilityScopedIntent && scored[0] && scored[0].score < 0) return [];
|
|
3365
|
-
return scored;
|
|
3366
|
-
}
|
|
3367
|
-
|
|
3368
|
-
function selectBestEndpoint(endpoints: EndpointDescriptor[], intent?: string, skillDomain?: string, contextUrl?: string): EndpointDescriptor {
|
|
3369
|
-
if (endpoints.length === 0) throw new Error("No endpoints available");
|
|
3370
|
-
if (endpoints.length === 1) return endpoints[0];
|
|
3371
|
-
|
|
3372
|
-
const ranked = rankEndpoints(endpoints, intent, skillDomain, contextUrl);
|
|
3373
|
-
if (ranked.length === 0) throw new Error("All endpoints are disabled");
|
|
3374
|
-
return ranked[0].endpoint;
|
|
3375
|
-
}
|
|
3376
|
-
|
|
3377
|
-
/** Detect if a string response is HTML rather than JSON/plaintext */
|
|
3378
|
-
function isHtml(text: string): boolean {
|
|
3379
|
-
const trimmed = text.trimStart().slice(0, 200).toLowerCase();
|
|
3380
|
-
return trimmed.startsWith("<!doctype html") ||
|
|
3381
|
-
trimmed.startsWith("<html") ||
|
|
3382
|
-
(trimmed.includes("<head") && trimmed.includes("<body"));
|
|
3383
|
-
}
|
|
3384
|
-
|
|
3385
|
-
/**
|
|
3386
|
-
* Detect if HTML is an empty SPA shell that needs JS to render.
|
|
3387
|
-
* SPA shells have a near-empty body (just a <div id="root"> or similar)
|
|
3388
|
-
* with all content loaded by JavaScript bundles.
|
|
3389
|
-
* SSR pages have substantial text content in the body already.
|
|
3390
|
-
*/
|
|
3391
|
-
function isSpaShell(html: string): boolean {
|
|
3392
|
-
// Quick heuristic: extract body content and check if it has meaningful text
|
|
3393
|
-
const bodyMatch = html.match(/<body[^>]*>([\s\S]*)<\/body>/i);
|
|
3394
|
-
if (!bodyMatch) return true; // no body at all — treat as SPA shell
|
|
3395
|
-
const body = bodyMatch[1];
|
|
3396
|
-
|
|
3397
|
-
// Strip script/style tags and HTML tags to get raw text
|
|
3398
|
-
const text = body
|
|
3399
|
-
.replace(/<script[\s\S]*?<\/script>/gi, "")
|
|
3400
|
-
.replace(/<style[\s\S]*?<\/style>/gi, "")
|
|
3401
|
-
.replace(/<[^>]+>/g, " ")
|
|
3402
|
-
.replace(/\s+/g, " ")
|
|
3403
|
-
.trim();
|
|
3404
|
-
|
|
3405
|
-
// SPA shells have very little text — just "Loading..." or empty divs
|
|
3406
|
-
return text.length < 200;
|
|
3407
|
-
}
|