unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/cli.js +79 -33
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +73 -22
  4. package/dist/server.js +25994 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/manifest.json +1 -1
  7. package/runtime-src/agent-outcome.ts +0 -166
  8. package/runtime-src/analytics-session.ts +0 -55
  9. package/runtime-src/api/browse-index.ts +0 -343
  10. package/runtime-src/api/browse-session.ts +0 -572
  11. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  12. package/runtime-src/api/browse-submit.ts +0 -1184
  13. package/runtime-src/api/routes.ts +0 -1833
  14. package/runtime-src/auth/browser-cookies.ts +0 -423
  15. package/runtime-src/auth/index.ts +0 -535
  16. package/runtime-src/auth/runtime.ts +0 -116
  17. package/runtime-src/browser/index.ts +0 -659
  18. package/runtime-src/browser/types.ts +0 -41
  19. package/runtime-src/build-info.generated.ts +0 -6
  20. package/runtime-src/capture/index.ts +0 -1929
  21. package/runtime-src/capture/prefetch.ts +0 -95
  22. package/runtime-src/capture/rsc.ts +0 -45
  23. package/runtime-src/cli/shortcuts.ts +0 -273
  24. package/runtime-src/cli.ts +0 -1734
  25. package/runtime-src/client/graph-client.ts +0 -100
  26. package/runtime-src/client/index.ts +0 -1425
  27. package/runtime-src/debug-trace.ts +0 -18
  28. package/runtime-src/domain.ts +0 -38
  29. package/runtime-src/execution/index.ts +0 -3407
  30. package/runtime-src/execution/retry.ts +0 -46
  31. package/runtime-src/execution/robots.ts +0 -167
  32. package/runtime-src/execution/search-forms.ts +0 -188
  33. package/runtime-src/execution/token-resolver.ts +0 -135
  34. package/runtime-src/extraction/index.ts +0 -1507
  35. package/runtime-src/foundry/publish-bundle.ts +0 -392
  36. package/runtime-src/graph/agent-augment.ts +0 -315
  37. package/runtime-src/graph/index.ts +0 -1532
  38. package/runtime-src/graph/local-fixtures.ts +0 -393
  39. package/runtime-src/graph/local-harness.ts +0 -646
  40. package/runtime-src/graph/planner.ts +0 -411
  41. package/runtime-src/graph/session.ts +0 -294
  42. package/runtime-src/graph/trace-store.ts +0 -136
  43. package/runtime-src/impact-log.ts +0 -227
  44. package/runtime-src/index.ts +0 -24
  45. package/runtime-src/indexer/index.ts +0 -465
  46. package/runtime-src/intent-match.ts +0 -1515
  47. package/runtime-src/kuri/client.ts +0 -1839
  48. package/runtime-src/logger.ts +0 -30
  49. package/runtime-src/marketplace/index.ts +0 -111
  50. package/runtime-src/mcp.ts +0 -1911
  51. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  52. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  53. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  54. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  55. package/runtime-src/orchestrator/index.ts +0 -4480
  56. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  57. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  58. package/runtime-src/payments/cascade.ts +0 -137
  59. package/runtime-src/payments/index.ts +0 -270
  60. package/runtime-src/payments/lobster-pay.ts +0 -182
  61. package/runtime-src/payments/wallet.ts +0 -98
  62. package/runtime-src/publish/review-context.ts +0 -93
  63. package/runtime-src/publish/sanitize.ts +0 -197
  64. package/runtime-src/publish/schema-review.ts +0 -192
  65. package/runtime-src/publish-admission.ts +0 -388
  66. package/runtime-src/ratelimit/index.ts +0 -23
  67. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  68. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  69. package/runtime-src/reverse-engineer/index.ts +0 -1551
  70. package/runtime-src/reverse-engineer/token-sources.ts +0 -379
  71. package/runtime-src/router.ts +0 -17
  72. package/runtime-src/routing-telemetry.ts +0 -395
  73. package/runtime-src/runtime/browser-access.ts +0 -11
  74. package/runtime-src/runtime/browser-auth.ts +0 -12
  75. package/runtime-src/runtime/browser-host.ts +0 -48
  76. package/runtime-src/runtime/lifecycle.ts +0 -17
  77. package/runtime-src/runtime/local-server.ts +0 -311
  78. package/runtime-src/runtime/paths.ts +0 -99
  79. package/runtime-src/runtime/setup.ts +0 -251
  80. package/runtime-src/runtime/supervisor.ts +0 -69
  81. package/runtime-src/runtime/update-hints.ts +0 -351
  82. package/runtime-src/server.ts +0 -100
  83. package/runtime-src/session-logs.ts +0 -142
  84. package/runtime-src/settings.ts +0 -221
  85. package/runtime-src/single-binary.ts +0 -143
  86. package/runtime-src/site-policy.ts +0 -54
  87. package/runtime-src/stale-cleanup-runner.ts +0 -144
  88. package/runtime-src/stale-cleanup.ts +0 -133
  89. package/runtime-src/telemetry-attribution.ts +0 -120
  90. package/runtime-src/telemetry.ts +0 -253
  91. package/runtime-src/template-params.ts +0 -141
  92. package/runtime-src/transform/drift.ts +0 -60
  93. package/runtime-src/transform/index.ts +0 -277
  94. package/runtime-src/types/index.ts +0 -1
  95. package/runtime-src/types/skill.ts +0 -931
  96. package/runtime-src/vault/index.ts +0 -196
  97. package/runtime-src/verification/auth-gate.ts +0 -8
  98. package/runtime-src/verification/candidates.ts +0 -27
  99. package/runtime-src/verification/index.ts +0 -120
  100. package/runtime-src/verification/matrix.ts +0 -30
  101. package/runtime-src/version.ts +0 -148
  102. package/runtime-src/workflow/artifact.ts +0 -161
  103. package/runtime-src/workflow/compile.ts +0 -808
  104. package/runtime-src/workflow/publish.ts +0 -225
  105. package/runtime-src/workflow/runtime.ts +0 -213
@@ -1,808 +0,0 @@
1
- import { nanoid } from "nanoid";
2
- import { load } from "cheerio";
3
- import type { CaptureResult, RawRequest } from "../capture/index.js";
4
- import type {
5
- EndpointDescriptor,
6
- SkillManifest,
7
- TokenBinding,
8
- WorkflowActionStep,
9
- WorkflowArtifact,
10
- WorkflowBootstrapHint,
11
- WorkflowDomFieldHint,
12
- WorkflowDomOptionHint,
13
- WorkflowMetaHint,
14
- WorkflowNextStateSpec,
15
- WorkflowPaymentRequirementSpec,
16
- WorkflowParameterConstraint,
17
- WorkflowParameterSourceHint,
18
- WorkflowParameterSpec,
19
- WorkflowPrerequisiteSpec,
20
- WorkflowReplayContract,
21
- WorkflowRecipe,
22
- WorkflowStep,
23
- WorkflowStepStrategy,
24
- WorkflowTokenCandidate,
25
- } from "../types/index.js";
26
-
27
- const TOKEN_NAME_RE = /(csrf|xsrf|authenticity|nonce|crumb|signature|token|session)/i;
28
-
29
- function normalizeName(value: string): string {
30
- return value.toLowerCase().replace(/[^a-z0-9]+/g, "");
31
- }
32
-
33
- function parseCookieHeader(header: string | undefined): Record<string, string> {
34
- if (!header) return {};
35
- return Object.fromEntries(
36
- header.split(";")
37
- .map((part) => part.trim())
38
- .filter(Boolean)
39
- .map((part) => {
40
- const idx = part.indexOf("=");
41
- return idx === -1 ? [part, ""] : [part.slice(0, idx).trim(), part.slice(idx + 1).trim()];
42
- }),
43
- );
44
- }
45
-
46
- function parseRequestBody(raw: string | undefined): Record<string, unknown> | undefined {
47
- if (!raw) return undefined;
48
- try {
49
- const parsed = JSON.parse(raw);
50
- return parsed && typeof parsed === "object" && !Array.isArray(parsed)
51
- ? parsed as Record<string, unknown>
52
- : undefined;
53
- } catch {
54
- try {
55
- const params = new URLSearchParams(raw);
56
- const out: Record<string, unknown> = {};
57
- for (const [key, value] of params) out[key] = value;
58
- return Object.keys(out).length > 0 ? out : undefined;
59
- } catch {
60
- return undefined;
61
- }
62
- }
63
- }
64
-
65
- function collectPrimitivePaths(
66
- value: unknown,
67
- prefix = "",
68
- out: WorkflowBootstrapHint[] = [],
69
- ): WorkflowBootstrapHint[] {
70
- if (value == null) return out;
71
- if (Array.isArray(value)) {
72
- value.slice(0, 5).forEach((entry, index) => collectPrimitivePaths(entry, `${prefix}[${index}]`, out));
73
- return out;
74
- }
75
- if (typeof value === "object") {
76
- Object.entries(value as Record<string, unknown>)
77
- .slice(0, 12)
78
- .forEach(([key, next]) => collectPrimitivePaths(next, prefix ? `${prefix}.${key}` : key, out));
79
- return out;
80
- }
81
- if (typeof value === "string" && TOKEN_NAME_RE.test(prefix)) {
82
- out.push({ path: prefix, value });
83
- }
84
- return out;
85
- }
86
-
87
- export interface HtmlWorkflowHints {
88
- dom_form_hints: WorkflowDomFieldHint[];
89
- dom_option_hints: WorkflowDomOptionHint[];
90
- meta_hints: WorkflowMetaHint[];
91
- bootstrap_hints: WorkflowBootstrapHint[];
92
- }
93
-
94
- function domSelectorForField(
95
- tagName: string,
96
- fieldName?: string,
97
- fieldId?: string,
98
- ): string | undefined {
99
- if (fieldId) return `#${fieldId}`;
100
- if (fieldName) return `${tagName}[name="${fieldName}"]`;
101
- return undefined;
102
- }
103
-
104
- export function extractHtmlWorkflowHints(html?: string): HtmlWorkflowHints {
105
- if (!html) return { dom_form_hints: [], dom_option_hints: [], meta_hints: [], bootstrap_hints: [] };
106
- const $ = load(html);
107
- const dom_form_hints: WorkflowDomFieldHint[] = [];
108
- const dom_option_hints: WorkflowDomOptionHint[] = [];
109
- const meta_hints: WorkflowMetaHint[] = [];
110
- const bootstrap_hints: WorkflowBootstrapHint[] = [];
111
-
112
- $("input[type='hidden'], input[name]").each((_, el) => {
113
- const field_name = $(el).attr("name")?.trim();
114
- if (!field_name) return;
115
- if (!TOKEN_NAME_RE.test(field_name)) return;
116
- dom_form_hints.push({
117
- form_selector: $(el).closest("form").attr("id")
118
- ? `form#${$(el).closest("form").attr("id")}`
119
- : undefined,
120
- field_name,
121
- value: $(el).attr("value") ?? undefined,
122
- field_type: $(el).attr("type") ?? undefined,
123
- selector: domSelectorForField(
124
- el.tagName?.toLowerCase?.() ?? "input",
125
- field_name,
126
- $(el).attr("id") ?? undefined,
127
- ),
128
- required: $(el).attr("required") != null,
129
- pattern: $(el).attr("pattern") ?? undefined,
130
- min: $(el).attr("min") ?? undefined,
131
- max: $(el).attr("max") ?? undefined,
132
- });
133
- });
134
-
135
- const optionMap = new Map<string, WorkflowDomOptionHint>();
136
- $("select[name], input[type='radio'][name], input[type='checkbox'][name]").each((_, el) => {
137
- const field_name = ($(el).attr("name") || "").trim();
138
- if (!field_name) return;
139
- const form_selector = $(el).closest("form").attr("id")
140
- ? `form#${$(el).closest("form").attr("id")}`
141
- : undefined;
142
- const key = `${form_selector ?? ""}::${field_name}`;
143
- const existing = optionMap.get(key) ?? {
144
- form_selector,
145
- field_name,
146
- values: [],
147
- labels: [],
148
- field_type: $(el).attr("type") ?? (el.tagName?.toLowerCase?.() === "select" ? "select" : undefined),
149
- };
150
- if (el.tagName?.toLowerCase?.() === "select") {
151
- $(el).find("option").each((_, optionEl) => {
152
- const value = ($(optionEl).attr("value") || $(optionEl).text() || "").trim();
153
- if (!value) return;
154
- if (!existing.values.includes(value)) existing.values.push(value);
155
- const label = $(optionEl).text().trim();
156
- if (label && !existing.labels?.includes(label)) existing.labels?.push(label);
157
- });
158
- } else {
159
- const value = ($(el).attr("value") || "").trim();
160
- if (value && !existing.values.includes(value)) existing.values.push(value);
161
- const label = $(el).closest("label").text().trim();
162
- if (label && !existing.labels?.includes(label)) existing.labels?.push(label);
163
- }
164
- optionMap.set(key, existing);
165
- });
166
- dom_option_hints.push(...optionMap.values().filter((hint) => hint.values.length > 0));
167
-
168
- $("meta[name], meta[property]").each((_, el) => {
169
- const key = ($(el).attr("name") || $(el).attr("property") || "").trim();
170
- const value = ($(el).attr("content") || "").trim();
171
- if (!key || !value || !TOKEN_NAME_RE.test(key)) return;
172
- meta_hints.push({ key, value });
173
- });
174
-
175
- $("script[type='application/json'], script[type='application/ld+json']").each((_, el) => {
176
- const text = $(el).contents().text().trim();
177
- if (!text) return;
178
- try {
179
- const parsed = JSON.parse(text);
180
- collectPrimitivePaths(parsed, "", bootstrap_hints);
181
- } catch {
182
- /* ignore */
183
- }
184
- });
185
-
186
- const inlineMatches = html.matchAll(/["']([A-Za-z0-9_.-]*(?:csrf|xsrf|token|nonce|crumb|signature)[A-Za-z0-9_.-]*)["']\s*:\s*["']([^"']+)["']/gi);
187
- for (const match of inlineMatches) {
188
- bootstrap_hints.push({ path: match[1], value: match[2] });
189
- }
190
-
191
- return { dom_form_hints, dom_option_hints, meta_hints, bootstrap_hints };
192
- }
193
-
194
- function stripTemplate(urlTemplate: string): string {
195
- return urlTemplate.replace(/\{[^}]+\}/g, "");
196
- }
197
-
198
- function requestMatchesEndpoint(req: RawRequest, endpoint: EndpointDescriptor): boolean {
199
- if (req.method.toUpperCase() !== endpoint.method.toUpperCase()) return false;
200
- try {
201
- const reqUrl = new URL(req.url);
202
- const endpointUrl = new URL(stripTemplate(endpoint.url_template));
203
- if (reqUrl.origin !== endpointUrl.origin) return false;
204
- return reqUrl.pathname.startsWith(endpointUrl.pathname) || endpointUrl.pathname.startsWith(reqUrl.pathname);
205
- } catch {
206
- return false;
207
- }
208
- }
209
-
210
- function candidateConfidence(targetName: string, sourceName: string, exactValueMatch: boolean): number {
211
- let score = exactValueMatch ? 0.95 : 0.45;
212
- if (normalizeName(targetName) === normalizeName(sourceName)) score += 0.15;
213
- if (normalizeName(targetName).includes(normalizeName(sourceName)) || normalizeName(sourceName).includes(normalizeName(targetName))) score += 0.1;
214
- return Math.min(score, 0.99);
215
- }
216
-
217
- function inferTokenBindingsForRequest(
218
- req: RawRequest,
219
- htmlHints: HtmlWorkflowHints,
220
- ): TokenBinding[] {
221
- const headers = Object.fromEntries(Object.entries(req.request_headers).map(([key, value]) => [key.toLowerCase(), value]));
222
- const body = parseRequestBody(req.request_body);
223
- const responseHeaders = Object.fromEntries(Object.entries(req.response_headers).map(([key, value]) => [key.toLowerCase(), value]));
224
- const cookies = parseCookieHeader(headers.cookie);
225
-
226
- const sources: Array<{ kind: WorkflowTokenCandidate["source_kind"]; name: string; value?: string; path?: string }> = [
227
- ...Object.entries(cookies).map(([name, value]) => ({ kind: "cookie" as const, name, value })),
228
- ...Object.entries(responseHeaders).map(([name, value]) => ({ kind: "response_header" as const, name, value })),
229
- ...htmlHints.dom_form_hints.map((hint) => ({ kind: "hidden_input" as const, name: hint.field_name, value: hint.value, path: hint.form_selector })),
230
- ...htmlHints.meta_hints.map((hint) => ({ kind: "meta" as const, name: hint.key, value: hint.value })),
231
- ...htmlHints.bootstrap_hints.map((hint) => ({ kind: "bootstrap_json" as const, name: hint.path.split(".").pop() ?? hint.path, value: hint.value, path: hint.path })),
232
- ];
233
-
234
- const targets: Array<{ location: "header" | "body"; name: string; value?: string }> = [
235
- ...Object.entries(headers)
236
- .filter(([name]) => name !== "cookie")
237
- .filter(([name, value]) => TOKEN_NAME_RE.test(name) || sources.some((source) => source.value && source.value === value))
238
- .map(([name, value]) => ({ location: "header" as const, name, value })),
239
- ...Object.entries(body ?? {})
240
- .filter(([name, value]) => TOKEN_NAME_RE.test(name) || sources.some((source) => source.value && source.value === String(value)))
241
- .map(([name, value]) => ({ location: "body" as const, name, value: value == null ? undefined : String(value) })),
242
- ];
243
-
244
- return targets.map((target) => {
245
- const candidates = sources
246
- .filter((source) => {
247
- if (!source.value) return normalizeName(target.name).includes(normalizeName(source.name));
248
- return source.value === target.value || normalizeName(target.name).includes(normalizeName(source.name)) || normalizeName(source.name).includes(normalizeName(target.name));
249
- })
250
- .map((source) => ({
251
- source_kind: source.kind,
252
- source_name: source.name,
253
- source_path: source.path,
254
- observed_value: source.value,
255
- confidence: candidateConfidence(target.name, source.name, source.value === target.value),
256
- }))
257
- .sort((lhs, rhs) => rhs.confidence - lhs.confidence);
258
-
259
- return {
260
- binding_id: nanoid(),
261
- target_location: target.location,
262
- target_name: target.name,
263
- refresh_on_statuses: [401, 403, 419, 422],
264
- candidates,
265
- };
266
- }).filter((binding) => binding.candidates.length > 0);
267
- }
268
-
269
- function isSensitiveWorkflowField(name: string): boolean {
270
- return TOKEN_NAME_RE.test(name);
271
- }
272
-
273
- function inferValueType(value: unknown): WorkflowParameterSpec["type"] {
274
- if (typeof value === "boolean") return "boolean";
275
- if (typeof value === "number") return Number.isInteger(value) ? "integer" : "number";
276
- if (Array.isArray(value)) return "array";
277
- if (value && typeof value === "object") return "object";
278
- if (typeof value === "string") {
279
- if (/^(true|false)$/i.test(value)) return "boolean";
280
- if (/^-?\d+$/.test(value)) return "integer";
281
- if (/^-?\d+\.\d+$/.test(value)) return "number";
282
- }
283
- return "string";
284
- }
285
-
286
- function inferValueFormat(name: string, value: unknown): string | undefined {
287
- const sample = typeof value === "string" ? value : "";
288
- if (/^\d{4}-\d{2}-\d{2}$/.test(sample)) return "date";
289
- if (sample && /^https?:\/\//i.test(sample)) return "url";
290
- if (sample && /^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(sample)) return "email";
291
- if (/date/i.test(name) && /^\d{4}-\d{2}-\d{2}$/.test(sample)) return "date";
292
- return undefined;
293
- }
294
-
295
- function summarizeResponseSchema(endpoint: EndpointDescriptor): string | undefined {
296
- const objectKeys = Object.keys(endpoint.response_schema?.properties ?? {});
297
- if (objectKeys.length > 0) return objectKeys.slice(0, 8).join(", ");
298
- const arrayKeys = Object.keys(endpoint.response_schema?.items?.properties ?? {});
299
- if (arrayKeys.length > 0) return arrayKeys.slice(0, 8).join(", ");
300
- return endpoint.response_schema ? endpoint.response_schema.type : undefined;
301
- }
302
-
303
- function templatePathParamNames(urlTemplate: string): string[] {
304
- return [...urlTemplate.matchAll(/\{([^}]+)\}/g)].map((match) => match[1]);
305
- }
306
-
307
- function inferQuerySamples(matchedRequests: RawRequest[]): Map<string, unknown> {
308
- const samples = new Map<string, unknown>();
309
- for (const req of matchedRequests) {
310
- try {
311
- const url = new URL(req.url);
312
- for (const [key, value] of url.searchParams.entries()) {
313
- if (!samples.has(key)) samples.set(key, value);
314
- }
315
- } catch {
316
- /* ignore */
317
- }
318
- }
319
- return samples;
320
- }
321
-
322
- function parameterDescription(
323
- endpoint: EndpointDescriptor,
324
- name: string,
325
- location: WorkflowParameterSpec["location"],
326
- userSupplied: boolean,
327
- derivedFrom: string[],
328
- ): string {
329
- const semantic = endpoint.semantic?.requires?.find((binding) => binding.key === name)?.description;
330
- if (semantic) return semantic;
331
- if (!userSupplied && derivedFrom.length > 0) {
332
- return `Derived ${location} parameter populated from ${derivedFrom.join(", ")}.`;
333
- }
334
- return `Observed ${location} parameter for ${endpoint.method} ${endpoint.url_template}.`;
335
- }
336
-
337
- function uniquePrimitiveValues(values: unknown[]): Array<string | number | boolean> {
338
- const out: Array<string | number | boolean> = [];
339
- for (const value of values) {
340
- if (typeof value !== "string" && typeof value !== "number" && typeof value !== "boolean") continue;
341
- if (!out.includes(value)) out.push(value);
342
- }
343
- return out;
344
- }
345
-
346
- function buildParameterSpecs(
347
- endpoint: EndpointDescriptor,
348
- matchedRequests: RawRequest[],
349
- htmlHints: HtmlWorkflowHints,
350
- tokenBindings: TokenBinding[],
351
- ): WorkflowParameterSpec[] {
352
- type ParamAccumulator = {
353
- name: string;
354
- location: WorkflowParameterSpec["location"];
355
- values: unknown[];
356
- defaultValue?: unknown;
357
- required: boolean;
358
- userSupplied: boolean;
359
- derivedFrom: string[];
360
- constraints: WorkflowParameterConstraint[];
361
- sourceHints: WorkflowParameterSourceHint[];
362
- };
363
- const acc = new Map<string, ParamAccumulator>();
364
- const getKey = (location: WorkflowParameterSpec["location"], name: string) => `${location}:${name}`;
365
- const ensure = (location: WorkflowParameterSpec["location"], name: string): ParamAccumulator => {
366
- const key = getKey(location, name);
367
- const existing = acc.get(key);
368
- if (existing) return existing;
369
- const created: ParamAccumulator = {
370
- name,
371
- location,
372
- values: [],
373
- required: false,
374
- userSupplied: !isSensitiveWorkflowField(name),
375
- derivedFrom: [],
376
- constraints: [],
377
- sourceHints: [],
378
- };
379
- acc.set(key, created);
380
- return created;
381
- };
382
- const pushConstraint = (target: ParamAccumulator, constraint: WorkflowParameterConstraint): void => {
383
- if (target.constraints.some((entry) => entry.kind === constraint.kind && entry.value === constraint.value && entry.description === constraint.description)) return;
384
- target.constraints.push(constraint);
385
- };
386
- const pushSourceHint = (target: ParamAccumulator, hint: WorkflowParameterSourceHint): void => {
387
- if (target.sourceHints.some((entry) => entry.source_kind === hint.source_kind && entry.source_name === hint.source_name && entry.source_path === hint.source_path)) return;
388
- target.sourceHints.push(hint);
389
- };
390
- const querySamples = inferQuerySamples(matchedRequests);
391
-
392
- for (const name of templatePathParamNames(endpoint.url_template)) {
393
- const target = ensure("path", name);
394
- target.required = true;
395
- target.defaultValue = endpoint.path_params?.[name];
396
- if (endpoint.path_params?.[name] != null) target.values.push(endpoint.path_params[name]);
397
- pushSourceHint(target, { source_kind: "path_template", source_name: name, confidence: 0.99 });
398
- }
399
-
400
- for (const [name, value] of Object.entries(endpoint.query ?? {})) {
401
- const target = ensure("query", name);
402
- target.defaultValue = value;
403
- target.values.push(value);
404
- pushSourceHint(target, { source_kind: "query_default", source_name: name, confidence: 0.9 });
405
- }
406
- for (const [name, value] of querySamples.entries()) {
407
- const target = ensure("query", name);
408
- target.values.push(value);
409
- pushSourceHint(target, { source_kind: "observed_query", source_name: name, confidence: 0.99 });
410
- }
411
-
412
- for (const [name, value] of Object.entries(endpoint.body_params ?? {})) {
413
- const target = ensure("body", name);
414
- target.defaultValue = value;
415
- target.values.push(value);
416
- target.required = true;
417
- pushSourceHint(target, { source_kind: "body_default", source_name: name, confidence: 0.95 });
418
- }
419
- for (const [name, value] of Object.entries(endpoint.body ?? {})) {
420
- const target = ensure("body", name);
421
- target.values.push(value);
422
- target.required = true;
423
- pushSourceHint(target, { source_kind: "body_default", source_name: name, confidence: 0.8 });
424
- }
425
- for (const req of matchedRequests) {
426
- for (const [name, value] of Object.entries(parseRequestBody(req.request_body) ?? {})) {
427
- const target = ensure("body", name);
428
- target.values.push(value);
429
- target.required = true;
430
- pushSourceHint(target, { source_kind: "observed_body", source_name: name, confidence: 0.99 });
431
- }
432
- }
433
-
434
- for (const binding of tokenBindings) {
435
- const location = binding.target_location === "header" ? "header" : "body";
436
- const target = ensure(location, binding.target_name);
437
- target.required = true;
438
- target.userSupplied = false;
439
- for (const candidate of binding.candidates) {
440
- pushSourceHint(target, {
441
- source_kind: candidate.source_kind,
442
- source_name: candidate.source_name,
443
- source_path: candidate.source_path,
444
- confidence: candidate.confidence,
445
- });
446
- const derivedLabel = candidate.source_path ? `${candidate.source_kind}:${candidate.source_path}` : `${candidate.source_kind}:${candidate.source_name}`;
447
- if (!target.derivedFrom.includes(derivedLabel)) target.derivedFrom.push(derivedLabel);
448
- }
449
- }
450
-
451
- for (const hint of htmlHints.dom_form_hints) {
452
- const bodyTarget = acc.get(getKey("body", hint.field_name));
453
- if (!bodyTarget) continue;
454
- if (hint.pattern) pushConstraint(bodyTarget, { kind: "pattern", value: hint.pattern });
455
- if (hint.min) pushConstraint(bodyTarget, { kind: "min", value: hint.min });
456
- if (hint.max) pushConstraint(bodyTarget, { kind: "max", value: hint.max });
457
- if (hint.required) bodyTarget.required = true;
458
- if (hint.selector) {
459
- if (!bodyTarget.derivedFrom.includes(`dom:${hint.selector}`)) bodyTarget.derivedFrom.push(`dom:${hint.selector}`);
460
- }
461
- }
462
-
463
- for (const optionHint of htmlHints.dom_option_hints) {
464
- const target = acc.get(getKey("body", optionHint.field_name)) ?? acc.get(getKey("query", optionHint.field_name));
465
- if (!target) continue;
466
- const values = uniquePrimitiveValues(optionHint.values);
467
- if (values.length > 0) pushConstraint(target, { kind: "enum", description: `Allowed values observed for ${optionHint.field_name}.` });
468
- target.values.push(...values);
469
- }
470
-
471
- for (const binding of endpoint.semantic?.requires ?? []) {
472
- const target = acc.get(getKey("path", binding.key))
473
- ?? acc.get(getKey("query", binding.key))
474
- ?? acc.get(getKey("body", binding.key))
475
- ?? acc.get(getKey("header", binding.key));
476
- if (!target) continue;
477
- if (binding.required) target.required = true;
478
- if (binding.example_value != null) target.values.push(binding.example_value);
479
- pushSourceHint(target, { source_kind: "semantic_requires", source_name: binding.key, confidence: 0.7 });
480
- }
481
-
482
- return Array.from(acc.values()).map((entry) => {
483
- const sampleValue = entry.values.find((value) => value != null);
484
- const format = inferValueFormat(entry.name, sampleValue);
485
- const enumValues = uniquePrimitiveValues(entry.values).slice(0, 12);
486
- const type = inferValueType(sampleValue);
487
- const sensitive = isSensitiveWorkflowField(entry.name);
488
- const constraints = [...entry.constraints];
489
- if (format) constraints.push({ kind: "format", value: format });
490
- if (enumValues.length > 1) constraints.push({ kind: "enum", description: `Allowed values observed for ${entry.name}.` });
491
- return {
492
- name: entry.name,
493
- location: entry.location,
494
- description: parameterDescription(endpoint, entry.name, entry.location, entry.userSupplied, entry.derivedFrom),
495
- type,
496
- required: entry.required,
497
- user_supplied: entry.userSupplied,
498
- ...(sensitive ? {} : entry.defaultValue !== undefined ? { default_value: entry.defaultValue } : {}),
499
- ...(sensitive ? {} : sampleValue !== undefined ? { example_value: sampleValue } : {}),
500
- ...(format ? { format } : {}),
501
- ...(enumValues.length > 1 ? { enum_values: enumValues } : {}),
502
- ...(entry.derivedFrom.length > 0 ? { derived_from: entry.derivedFrom } : {}),
503
- ...(constraints.length > 0 ? { constraints } : {}),
504
- source_hints: entry.sourceHints,
505
- };
506
- }).sort((lhs, rhs) => {
507
- const order = ["path", "query", "body", "header"];
508
- return order.indexOf(lhs.location) - order.indexOf(rhs.location) || lhs.name.localeCompare(rhs.name);
509
- });
510
- }
511
-
512
- function buildPrerequisiteSpecs(
513
- skill: SkillManifest,
514
- endpoint: EndpointDescriptor,
515
- htmlHints: HtmlWorkflowHints,
516
- tokenBindings: TokenBinding[],
517
- actionSequence?: WorkflowActionStep[],
518
- ): WorkflowPrerequisiteSpec[] {
519
- const prerequisites: WorkflowPrerequisiteSpec[] = [];
520
- if (skill.auth_profile_ref || endpoint.semantic?.auth_required) {
521
- prerequisites.push({
522
- prerequisite_id: nanoid(),
523
- kind: "authenticated-session",
524
- name: "authenticated_session",
525
- description: "Requires authenticated browser or stored auth state before replay.",
526
- required: true,
527
- derived_from: skill.auth_profile_ref,
528
- });
529
- }
530
- if (endpoint.trigger_url) {
531
- prerequisites.push({
532
- prerequisite_id: nanoid(),
533
- kind: "page-context",
534
- name: "trigger_url",
535
- description: "Observed from a specific page context during traversal.",
536
- required: true,
537
- derived_from: endpoint.trigger_url,
538
- });
539
- }
540
- for (const binding of tokenBindings) {
541
- const top = binding.candidates[0];
542
- prerequisites.push({
543
- prerequisite_id: nanoid(),
544
- kind: "token-binding",
545
- name: binding.target_name,
546
- description: `Replay derives ${binding.target_name} from observed ${top?.source_kind ?? "token"} state.`,
547
- required: true,
548
- source_kind: top?.source_kind,
549
- source_name: top?.source_name,
550
- derived_from: top?.source_path,
551
- });
552
- }
553
- for (const hint of htmlHints.dom_form_hints.filter((entry) => entry.required || !!entry.pattern || !!entry.min || !!entry.max)) {
554
- prerequisites.push({
555
- prerequisite_id: nanoid(),
556
- kind: "hidden-field",
557
- name: hint.field_name,
558
- description: "Observed DOM field constraint recorded during traversal.",
559
- required: !!hint.required,
560
- selector: hint.selector,
561
- derived_from: hint.form_selector,
562
- });
563
- }
564
- if (actionSequence && actionSequence.length > 0) {
565
- prerequisites.push({
566
- prerequisite_id: nanoid(),
567
- kind: "browser-action",
568
- name: "action_sequence",
569
- description: `Observed ${actionSequence.length} browser action${actionSequence.length === 1 ? "" : "s"} before request.`,
570
- required: false,
571
- });
572
- }
573
- return prerequisites;
574
- }
575
-
576
- function buildNextStateSpecs(
577
- endpoint: EndpointDescriptor,
578
- finalUrl: string,
579
- ): WorkflowNextStateSpec[] {
580
- const nextState: WorkflowNextStateSpec[] = [];
581
- if (finalUrl) {
582
- nextState.push({
583
- kind: "page_url",
584
- value: finalUrl,
585
- description: "Observed page destination after successful traversal.",
586
- });
587
- }
588
- const schemaSummary = summarizeResponseSchema(endpoint);
589
- if (schemaSummary) {
590
- nextState.push({
591
- kind: "response_schema",
592
- value: schemaSummary,
593
- description: "Observed response shape summary for replay validation.",
594
- });
595
- }
596
- return nextState;
597
- }
598
-
599
- function buildPaymentRequirement(
600
- skill: SkillManifest,
601
- endpoint: EndpointDescriptor,
602
- ): WorkflowPaymentRequirementSpec {
603
- const priceUsd = typeof skill.base_price_usd === "number" && skill.base_price_usd > 0
604
- ? String(skill.base_price_usd)
605
- : undefined;
606
- if (priceUsd) {
607
- return {
608
- status: "x402_required",
609
- price_usd: priceUsd,
610
- currency: "USDC",
611
- wallet_required: true,
612
- provider_hint: "lobster.cash-compatible x402 wallet",
613
- confirmation_field: "payment_verified",
614
- reason: `Published replay for ${skill.skill_id}/${endpoint.endpoint_id} is priced through the marketplace payment lane.`,
615
- };
616
- }
617
- if (skill.owner_compensation_opt_in) {
618
- return {
619
- status: "x402_optional",
620
- wallet_required: false,
621
- provider_hint: "lobster.cash-compatible x402 wallet",
622
- confirmation_field: "payment_verified",
623
- reason: `This skill owner has opted into compensation, so marketplace x402 pricing may apply when this replay is published.`,
624
- };
625
- }
626
- return {
627
- status: "free",
628
- wallet_required: false,
629
- reason: "No x402 payment requirement was recorded for this replay contract at publish time.",
630
- };
631
- }
632
-
633
- function buildReplayContract(
634
- skill: SkillManifest,
635
- endpoint: EndpointDescriptor,
636
- matchedRequests: RawRequest[],
637
- htmlHints: HtmlWorkflowHints,
638
- tokenBindings: TokenBinding[],
639
- finalUrl: string,
640
- actionSequence?: WorkflowActionStep[],
641
- ): WorkflowReplayContract {
642
- const dependencyBindings = Array.from(new Set([
643
- ...templatePathParamNames(endpoint.url_template),
644
- ...Object.keys(endpoint.query ?? {}).map((key) => key),
645
- ...Object.keys(endpoint.body_params ?? {}).map((key) => key),
646
- ...Object.keys(endpoint.body ?? {}).map((key) => key),
647
- ...(endpoint.semantic?.requires ?? []).map((binding) => binding.key),
648
- ...(endpoint.semantic?.provides ?? []).map((binding) => binding.key),
649
- ...tokenBindings.map((binding) => binding.target_name),
650
- ].filter(Boolean))).sort();
651
- const searchTerms = Array.from(new Set([
652
- endpoint.method.toLowerCase(),
653
- ...endpoint.url_template.toLowerCase().split(/[^a-z0-9{}._-]+/).filter((token) => token.length >= 2),
654
- ...dependencyBindings.map((binding) => binding.toLowerCase()),
655
- ...(endpoint.semantic?.example_fields ?? []).map((field) => field.toLowerCase()),
656
- ])).slice(0, 48);
657
- return {
658
- explicit_replay_only: true,
659
- exposure_stage: "publish",
660
- dependency_bindings: dependencyBindings,
661
- search_terms: searchTerms,
662
- parameter_specs: buildParameterSpecs(endpoint, matchedRequests, htmlHints, tokenBindings),
663
- prerequisite_specs: buildPrerequisiteSpecs(skill, endpoint, htmlHints, tokenBindings, actionSequence),
664
- next_state: buildNextStateSpecs(endpoint, finalUrl),
665
- payment_requirement: buildPaymentRequirement(skill, endpoint),
666
- };
667
- }
668
-
669
- function deriveActionSequence(
670
- matchedRequests: RawRequest[],
671
- endpoint: EndpointDescriptor,
672
- ): WorkflowActionStep[] | undefined {
673
- const withProvenance = matchedRequests
674
- .filter((req) => req.triggered_by_action)
675
- .sort((lhs, rhs) => (lhs.triggered_by_step ?? 0) - (rhs.triggered_by_step ?? 0));
676
- if (withProvenance.length > 0) {
677
- return withProvenance.map((req) => ({
678
- action: req.triggered_by_action!,
679
- step_index: req.triggered_by_step,
680
- ref: req.triggered_by_ref,
681
- }));
682
- }
683
- if (endpoint.search_form) {
684
- return [
685
- ...endpoint.search_form.fields.map((field, index) => ({
686
- action: "fill",
687
- selector: field.selector,
688
- value: `{${field.name}}`,
689
- step_index: index,
690
- })),
691
- { action: "submit", selector: endpoint.search_form.submit_selector, step_index: endpoint.search_form.fields.length },
692
- ];
693
- }
694
- return undefined;
695
- }
696
-
697
- function deriveWorkflowSteps(
698
- endpoint: EndpointDescriptor,
699
- matchedRequests: RawRequest[],
700
- ): WorkflowStep[] {
701
- const steps: WorkflowStep[] = [];
702
- const actionSequence = deriveActionSequence(matchedRequests, endpoint);
703
- const addStep = (strategy: WorkflowStepStrategy, provenance: WorkflowStep["provenance"], triggerUrl?: string) => {
704
- if (steps.some((step) => step.strategy === strategy && step.trigger_url === triggerUrl)) return;
705
- steps.push({
706
- step_id: nanoid(),
707
- strategy,
708
- provenance,
709
- ...(triggerUrl ? { trigger_url: triggerUrl } : {}),
710
- ...(strategy === "browser-action" && actionSequence ? { action_sequence: actionSequence } : {}),
711
- });
712
- };
713
-
714
- if (endpoint.exec_strategy) addStep(endpoint.exec_strategy === "browser" ? "browser-fetch" : endpoint.exec_strategy, "learned-runtime", endpoint.trigger_url);
715
- addStep("server", matchedRequests.length > 0 ? "observed-request" : "bundle-inferred");
716
- if (endpoint.trigger_url) addStep("trigger-intercept", matchedRequests.length > 0 ? "observed-request" : "trigger-url", endpoint.trigger_url);
717
- if (actionSequence) addStep("browser-action", endpoint.search_form ? "dom-form" : "observed-request", endpoint.trigger_url);
718
- addStep("browser-fetch", matchedRequests.length > 0 ? "observed-request" : endpoint.search_form ? "dom-form" : "bundle-inferred", endpoint.trigger_url);
719
- return steps;
720
- }
721
-
722
- function parameterMappingConfident(endpoint: EndpointDescriptor): boolean {
723
- return endpoint.method === "GET"
724
- || !!endpoint.search_form
725
- || Object.keys(endpoint.body_params ?? {}).length > 0
726
- || Object.keys(endpoint.path_params ?? {}).length > 0
727
- || Object.keys(endpoint.query ?? {}).length > 0
728
- || !!endpoint.body;
729
- }
730
-
731
- function operationIdForEndpoint(skill: SkillManifest, endpointId: string): string | undefined {
732
- return skill.operation_graph?.operations.find((operation) => operation.endpoint_id === endpointId)?.operation_id;
733
- }
734
-
735
- function buildRecipe(
736
- skill: SkillManifest,
737
- endpoint: EndpointDescriptor,
738
- matchedRequests: RawRequest[],
739
- htmlHints: HtmlWorkflowHints,
740
- finalUrl: string,
741
- ): WorkflowRecipe {
742
- const token_bindings = endpoint.method === "GET" ? [] : matchedRequests.flatMap((req) => inferTokenBindingsForRequest(req, htmlHints));
743
- const provenance_backed = matchedRequests.length > 0 || !!endpoint.trigger_url || !!endpoint.search_form || !!endpoint.dom_extraction;
744
- const auth_required = !!skill.auth_profile_ref || !!endpoint.semantic?.auth_required || token_bindings.length > 0;
745
- const parameter_mapping = parameterMappingConfident(endpoint);
746
- const steps = deriveWorkflowSteps(endpoint, matchedRequests);
747
- const actionSequence = steps.find((step) => step.strategy === "browser-action")?.action_sequence;
748
- return {
749
- recipe_id: nanoid(),
750
- endpoint_id: endpoint.endpoint_id,
751
- operation_id: operationIdForEndpoint(skill, endpoint.endpoint_id),
752
- preferred: false,
753
- provenance_backed,
754
- steps,
755
- token_bindings,
756
- mutation_guard: {
757
- confirm_unsafe_required: endpoint.method !== "GET" && endpoint.idempotency === "unsafe",
758
- provenance_backed,
759
- auth_required,
760
- parameter_mapping_confident: parameter_mapping,
761
- ...(!provenance_backed && endpoint.method !== "GET" ? { block_reason: "mutation requires provenance-backed workflow recipe" } : {}),
762
- },
763
- replay_contract: buildReplayContract(skill, endpoint, matchedRequests, htmlHints, token_bindings, finalUrl, actionSequence),
764
- };
765
- }
766
-
767
- export function buildWorkflowArtifactFromCapture(
768
- skill: SkillManifest,
769
- capture: Pick<CaptureResult, "requests" | "har_lineage_id" | "final_url" | "html" | "js_bundles" | "cookies">,
770
- options?: { authHeaders?: Record<string, string> },
771
- ): WorkflowArtifact {
772
- const htmlHints = extractHtmlWorkflowHints(capture.html);
773
- const recipes = skill.endpoints.map((endpoint) => buildRecipe(
774
- skill,
775
- endpoint,
776
- capture.requests.filter((req) => requestMatchesEndpoint(req, endpoint)),
777
- htmlHints,
778
- capture.final_url,
779
- ));
780
- if (recipes.length > 0) recipes[0].preferred = true;
781
-
782
- return {
783
- artifact_version: "1",
784
- skill_id: skill.skill_id,
785
- domain: skill.domain,
786
- intent_signature: skill.intent_signature,
787
- captured_at: new Date().toISOString(),
788
- final_url: capture.final_url,
789
- auth_state: {
790
- auth_profile_ref: skill.auth_profile_ref,
791
- cookie_names: (capture.cookies ?? []).map((cookie) => cookie.name),
792
- header_names: Object.keys(options?.authHeaders ?? {}),
793
- authenticated: (capture.cookies?.length ?? 0) > 0 || Object.keys(options?.authHeaders ?? {}).length > 0,
794
- },
795
- evidence: {
796
- observed_request_count: capture.requests.length,
797
- observed_request_urls: capture.requests.slice(0, 100).map((req) => req.url),
798
- har_lineage_ids: [capture.har_lineage_id],
799
- trigger_urls: Array.from(new Set(skill.endpoints.map((endpoint) => endpoint.trigger_url).filter((value): value is string => !!value))),
800
- js_bundle_urls: capture.js_bundles ? Array.from(capture.js_bundles.keys()).slice(0, 40) : [],
801
- dom_form_hints: htmlHints.dom_form_hints,
802
- dom_option_hints: htmlHints.dom_option_hints,
803
- meta_hints: htmlHints.meta_hints,
804
- bootstrap_hints: htmlHints.bootstrap_hints,
805
- },
806
- recipes,
807
- };
808
- }