unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/cli.js +79 -33
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +73 -22
  4. package/dist/server.js +25994 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/manifest.json +1 -1
  7. package/runtime-src/agent-outcome.ts +0 -166
  8. package/runtime-src/analytics-session.ts +0 -55
  9. package/runtime-src/api/browse-index.ts +0 -343
  10. package/runtime-src/api/browse-session.ts +0 -572
  11. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  12. package/runtime-src/api/browse-submit.ts +0 -1184
  13. package/runtime-src/api/routes.ts +0 -1833
  14. package/runtime-src/auth/browser-cookies.ts +0 -423
  15. package/runtime-src/auth/index.ts +0 -535
  16. package/runtime-src/auth/runtime.ts +0 -116
  17. package/runtime-src/browser/index.ts +0 -659
  18. package/runtime-src/browser/types.ts +0 -41
  19. package/runtime-src/build-info.generated.ts +0 -6
  20. package/runtime-src/capture/index.ts +0 -1929
  21. package/runtime-src/capture/prefetch.ts +0 -95
  22. package/runtime-src/capture/rsc.ts +0 -45
  23. package/runtime-src/cli/shortcuts.ts +0 -273
  24. package/runtime-src/cli.ts +0 -1734
  25. package/runtime-src/client/graph-client.ts +0 -100
  26. package/runtime-src/client/index.ts +0 -1425
  27. package/runtime-src/debug-trace.ts +0 -18
  28. package/runtime-src/domain.ts +0 -38
  29. package/runtime-src/execution/index.ts +0 -3407
  30. package/runtime-src/execution/retry.ts +0 -46
  31. package/runtime-src/execution/robots.ts +0 -167
  32. package/runtime-src/execution/search-forms.ts +0 -188
  33. package/runtime-src/execution/token-resolver.ts +0 -135
  34. package/runtime-src/extraction/index.ts +0 -1507
  35. package/runtime-src/foundry/publish-bundle.ts +0 -392
  36. package/runtime-src/graph/agent-augment.ts +0 -315
  37. package/runtime-src/graph/index.ts +0 -1532
  38. package/runtime-src/graph/local-fixtures.ts +0 -393
  39. package/runtime-src/graph/local-harness.ts +0 -646
  40. package/runtime-src/graph/planner.ts +0 -411
  41. package/runtime-src/graph/session.ts +0 -294
  42. package/runtime-src/graph/trace-store.ts +0 -136
  43. package/runtime-src/impact-log.ts +0 -227
  44. package/runtime-src/index.ts +0 -24
  45. package/runtime-src/indexer/index.ts +0 -465
  46. package/runtime-src/intent-match.ts +0 -1515
  47. package/runtime-src/kuri/client.ts +0 -1839
  48. package/runtime-src/logger.ts +0 -30
  49. package/runtime-src/marketplace/index.ts +0 -111
  50. package/runtime-src/mcp.ts +0 -1911
  51. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  52. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  53. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  54. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  55. package/runtime-src/orchestrator/index.ts +0 -4480
  56. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  57. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  58. package/runtime-src/payments/cascade.ts +0 -137
  59. package/runtime-src/payments/index.ts +0 -270
  60. package/runtime-src/payments/lobster-pay.ts +0 -182
  61. package/runtime-src/payments/wallet.ts +0 -98
  62. package/runtime-src/publish/review-context.ts +0 -93
  63. package/runtime-src/publish/sanitize.ts +0 -197
  64. package/runtime-src/publish/schema-review.ts +0 -192
  65. package/runtime-src/publish-admission.ts +0 -388
  66. package/runtime-src/ratelimit/index.ts +0 -23
  67. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  68. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  69. package/runtime-src/reverse-engineer/index.ts +0 -1551
  70. package/runtime-src/reverse-engineer/token-sources.ts +0 -379
  71. package/runtime-src/router.ts +0 -17
  72. package/runtime-src/routing-telemetry.ts +0 -395
  73. package/runtime-src/runtime/browser-access.ts +0 -11
  74. package/runtime-src/runtime/browser-auth.ts +0 -12
  75. package/runtime-src/runtime/browser-host.ts +0 -48
  76. package/runtime-src/runtime/lifecycle.ts +0 -17
  77. package/runtime-src/runtime/local-server.ts +0 -311
  78. package/runtime-src/runtime/paths.ts +0 -99
  79. package/runtime-src/runtime/setup.ts +0 -251
  80. package/runtime-src/runtime/supervisor.ts +0 -69
  81. package/runtime-src/runtime/update-hints.ts +0 -351
  82. package/runtime-src/server.ts +0 -100
  83. package/runtime-src/session-logs.ts +0 -142
  84. package/runtime-src/settings.ts +0 -221
  85. package/runtime-src/single-binary.ts +0 -143
  86. package/runtime-src/site-policy.ts +0 -54
  87. package/runtime-src/stale-cleanup-runner.ts +0 -144
  88. package/runtime-src/stale-cleanup.ts +0 -133
  89. package/runtime-src/telemetry-attribution.ts +0 -120
  90. package/runtime-src/telemetry.ts +0 -253
  91. package/runtime-src/template-params.ts +0 -141
  92. package/runtime-src/transform/drift.ts +0 -60
  93. package/runtime-src/transform/index.ts +0 -277
  94. package/runtime-src/types/index.ts +0 -1
  95. package/runtime-src/types/skill.ts +0 -931
  96. package/runtime-src/vault/index.ts +0 -196
  97. package/runtime-src/verification/auth-gate.ts +0 -8
  98. package/runtime-src/verification/candidates.ts +0 -27
  99. package/runtime-src/verification/index.ts +0 -120
  100. package/runtime-src/verification/matrix.ts +0 -30
  101. package/runtime-src/version.ts +0 -148
  102. package/runtime-src/workflow/artifact.ts +0 -161
  103. package/runtime-src/workflow/compile.ts +0 -808
  104. package/runtime-src/workflow/publish.ts +0 -225
  105. package/runtime-src/workflow/runtime.ts +0 -213
@@ -1,127 +0,0 @@
1
- import { getRegistrableDomain } from "../domain.js";
2
- import { log } from "../logger.js";
3
-
4
- export interface BundleRoute {
5
- path: string;
6
- url: string;
7
- source_bundle: string;
8
- match_type: "string_literal" | "fetch_call" | "route_def";
9
- /** Query param names extracted from the JS source (e.g. "/api/search?q=" → ["q"]) */
10
- query_params?: string[];
11
- }
12
-
13
- // String literals containing /api/ paths — primary signal, catches 90% of cases
14
- // Group 1: path, Group 2: query string (without the leading ?)
15
- const API_PATH_PATTERN = /["'`](\/api\/[a-zA-Z0-9/_-]{2,80})(?:\?([^"'`#]*))?["'`]/g;
16
-
17
- // fetch/axios calls with URL strings
18
- const FETCH_PATTERN = /(?:fetch|axios|\.get|\.post|\.put|\.patch|\.delete)\s*\(\s*["'`](\/[a-zA-Z0-9/_-]{3,80})(?:\?([^"'`#]*))?["'`]/g;
19
-
20
- // Versioned API paths: /v1/..., /v2/...
21
- const VERSIONED_API_PATTERN = /["'`](\/v[0-9]+\/[a-zA-Z0-9/_-]{2,80})(?:\?([^"'`#]*))?["'`]/g;
22
-
23
- // Framework internals — NOT API routes
24
- const SKIP_BUNDLE_PATHS = /^\/((_next|__next|__webpack|__vite|static|assets|public|favicon|manifest|service-worker|sw|workbox|chunks?|node_modules|\.well-known)\b)/i;
25
-
26
- // File extension paths — asset references, not APIs
27
- const SKIP_EXTENSIONS = /\.(js|mjs|css|json|html|xml|svg|png|jpg|jpeg|gif|ico|woff2?|ttf|map|ts|tsx|jsx|vue|md|txt|webp|avif)$/i;
28
-
29
- // Too-generic single-segment paths
30
- const TOO_GENERIC = /^\/[a-z]{1,3}$/i;
31
-
32
- // Public-site bundle scans often surface account/settings/bootstrap routes first.
33
- // Skip obviously non-public app-management surfaces so root captures don't learn junk.
34
- const SKIP_NON_PUBLIC_ROUTE_PATHS =
35
- /^(\/manage\/account\/|\/account\/webauthn|\/settings(?:\/|$)|\/org\/create(?:\/|$)|\/login(?:\/|$)|\/logout(?:\/|$)|\/signin(?:\/|$)|\/sign-in(?:\/|$)|\/signup(?:\/|$)|\/register(?:\/|$)|\/oauth(?:\/|$)|\/sso(?:\/|$)|\/session(?:\/|$)|\/admin(?:\/|$)|\/billing(?:\/|$))/i;
36
-
37
- // Sensitive query param names that should not be included
38
- const SENSITIVE_PARAM = /^(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|secret|password|key|token|session[_-]?id|client[_-]?secret|private[_-]?key|bearer)$/i;
39
-
40
- /** Extract param names from a query string found in JS source.
41
- * Handles: "q=", "q=value", "q=${var}", "q", "q=&page=1" */
42
- function extractParamNames(queryString: string): string[] {
43
- if (!queryString) return [];
44
- const names: string[] = [];
45
- for (const part of queryString.split("&")) {
46
- const eqIdx = part.indexOf("=");
47
- const key = (eqIdx >= 0 ? part.slice(0, eqIdx) : part).trim();
48
- // Skip empty keys, template expressions like ${...}, and sensitive params
49
- if (!key || key.startsWith("$") || key.startsWith("{") || SENSITIVE_PARAM.test(key)) continue;
50
- // Skip framework-internal params
51
- if (/^(_rsc|_next|__next|_t|_hash|__cf_chl_tk|nxtP\[.*\])$/i.test(key)) continue;
52
- names.push(key);
53
- }
54
- return names;
55
- }
56
-
57
- function extractMatches(
58
- content: string,
59
- pattern: RegExp,
60
- matchType: BundleRoute["match_type"],
61
- ): Array<{ path: string; query_string?: string; match_type: BundleRoute["match_type"] }> {
62
- const results: Array<{ path: string; query_string?: string; match_type: BundleRoute["match_type"] }> = [];
63
- pattern.lastIndex = 0;
64
- let match: RegExpExecArray | null;
65
- while ((match = pattern.exec(content)) !== null) {
66
- if (match[1]) results.push({ path: match[1], query_string: match[2] || undefined, match_type: matchType });
67
- }
68
- return results;
69
- }
70
-
71
- export function scanBundlesForRoutes(
72
- bundles: Map<string, string>,
73
- pageOrigin: string,
74
- ): BundleRoute[] {
75
- const routes: BundleRoute[] = [];
76
- const seen = new Set<string>();
77
- // Accumulate query params across multiple occurrences of the same path
78
- const queryParamsByPath = new Map<string, Set<string>>();
79
-
80
- for (const [bundleUrl, content] of bundles) {
81
- const matches = [
82
- ...extractMatches(content, API_PATH_PATTERN, "string_literal"),
83
- ...extractMatches(content, FETCH_PATTERN, "fetch_call"),
84
- ...extractMatches(content, VERSIONED_API_PATTERN, "string_literal"),
85
- ];
86
-
87
- for (const { path, query_string, match_type } of matches) {
88
- const normalized = path.replace(/\/+$/, "").replace(/\/\//g, "/");
89
-
90
- if (SKIP_BUNDLE_PATHS.test(normalized)) continue;
91
- if (SKIP_EXTENSIONS.test(normalized)) continue;
92
- if (TOO_GENERIC.test(normalized)) continue;
93
- if (SKIP_NON_PUBLIC_ROUTE_PATHS.test(normalized)) continue;
94
- if (normalized.length < 4) continue;
95
-
96
- // Collect query params from this occurrence
97
- if (query_string) {
98
- const params = extractParamNames(query_string);
99
- if (params.length > 0) {
100
- if (!queryParamsByPath.has(normalized)) queryParamsByPath.set(normalized, new Set());
101
- for (const p of params) queryParamsByPath.get(normalized)!.add(p);
102
- }
103
- }
104
-
105
- if (seen.has(normalized)) continue;
106
- seen.add(normalized);
107
-
108
- routes.push({
109
- path: normalized,
110
- url: `${pageOrigin}${normalized}`,
111
- source_bundle: bundleUrl,
112
- match_type,
113
- });
114
- }
115
- }
116
-
117
- // Attach accumulated query params to routes
118
- for (const route of routes) {
119
- const params = queryParamsByPath.get(route.path);
120
- if (params && params.size > 0) {
121
- route.query_params = [...params];
122
- }
123
- }
124
-
125
- log("bundle-scanner", `found ${routes.length} API routes in ${bundles.size} JS bundles`);
126
- return routes;
127
- }
@@ -1,213 +0,0 @@
1
- import { inferSchema } from "../transform/index.js";
2
- import { sanitizeAgentVisibleText, sanitizeAgentVisibleValue } from "../publish/sanitize.js";
3
-
4
- export interface DescriptionParamContext {
5
- name: string;
6
- in: string;
7
- example?: string;
8
- type?: string;
9
- required?: boolean;
10
- enum_values?: Array<string | number | boolean>;
11
- description?: string;
12
- }
13
-
14
- export interface DescriptionContext {
15
- url_template: string;
16
- method: string;
17
- params: DescriptionParamContext[];
18
- sample_response_keys?: string[];
19
- domain: string;
20
- dependency_bindings?: string[];
21
- search_terms?: string[];
22
- }
23
-
24
- function inferParamDescription(param: DescriptionParamContext): string | undefined {
25
- if (param.description) return sanitizeAgentVisibleText(param.description);
26
- const lower = param.name.toLowerCase();
27
- if (/^(q|query|search|term|keyword)$/.test(lower)) return "Search query text.";
28
- if (/date/.test(lower)) return "Date value used by the request.";
29
- if (/email/.test(lower)) return "Email-style identifier.";
30
- if (/country|residence|region|locale/.test(lower)) return "Audience or geographic selector.";
31
- if (/id|slug|handle|username|urn/.test(lower)) return "Resource identifier.";
32
- if (/token|csrf|session|auth|nonce|secret|key/.test(lower)) return "Derived auth or anti-forgery parameter.";
33
- return undefined;
34
- }
35
-
36
- function renderParamLine(param: DescriptionParamContext): string {
37
- const signatureParts = [param.in];
38
- if (param.type) signatureParts.push(param.type);
39
- if (param.required) signatureParts.push("required");
40
- const suffix: string[] = [];
41
- const description = inferParamDescription(param);
42
- if (description) suffix.push(description);
43
- if (param.enum_values && param.enum_values.length > 0) {
44
- suffix.push(`allowed: ${param.enum_values.join(" | ")}`);
45
- }
46
- if (param.example) {
47
- suffix.push(`example placeholder: "${sanitizeAgentVisibleText(param.example)}"`);
48
- }
49
- return ` - ${param.name} (${signatureParts.join(", ")})${suffix.length > 0 ? ` — ${suffix.join("; ")}` : ""}`;
50
- }
51
-
52
- /**
53
- * Build a grounded prompt for LLM description generation.
54
- * Includes request params and sample response fields to prevent hallucination.
55
- */
56
- export function buildDescriptionPrompt(ctx: DescriptionContext): string {
57
- const parts: string[] = [
58
- `Endpoint: ${ctx.method} ${ctx.url_template}`,
59
- `Domain: ${ctx.domain}`,
60
- ];
61
-
62
- if (ctx.params.length > 0) {
63
- parts.push("Parameters:");
64
- for (const p of ctx.params) {
65
- parts.push(renderParamLine(p));
66
- }
67
- }
68
-
69
- if (ctx.sample_response_keys && ctx.sample_response_keys.length > 0) {
70
- parts.push(`Response fields: ${ctx.sample_response_keys.join(", ")}`);
71
- }
72
-
73
- if (ctx.dependency_bindings && ctx.dependency_bindings.length > 0) {
74
- parts.push(`Dependency vars: ${ctx.dependency_bindings.map((entry) => sanitizeAgentVisibleText(entry)).join(", ")}`);
75
- }
76
-
77
- if (ctx.search_terms && ctx.search_terms.length > 0) {
78
- parts.push(`Search terms: ${ctx.search_terms.map((entry) => sanitizeAgentVisibleText(entry)).join(", ")}`);
79
- }
80
-
81
- parts.push("Do not copy concrete ids, tokens, emails, phone numbers, or user-specific values. Replace concrete values with role-based placeholders when describing parameters.");
82
- parts.push("Write a one-sentence description of what this endpoint does, grounded in the parameters and response fields above.");
83
- return parts.join("\n");
84
- }
85
-
86
- /**
87
- * Extract top-level keys from a sample JSON response for grounding.
88
- */
89
- export function extractResponseKeys(sampleResponse: unknown): string[] {
90
- if (!sampleResponse || typeof sampleResponse !== "object") return [];
91
- const obj = sampleResponse as Record<string, unknown>;
92
-
93
- // If it's an array, look at first element
94
- if (Array.isArray(obj)) {
95
- return obj.length > 0 ? extractResponseKeys(obj[0]) : [];
96
- }
97
-
98
- return Object.keys(obj).slice(0, 20);
99
- }
100
-
101
- /**
102
- * Derive a one-line grounded description from actual endpoint data.
103
- * This replaces pure URL-heuristic descriptions with descriptions that
104
- * name actual params and response fields observed during capture.
105
- */
106
- export function groundedDescription(ctx: DescriptionContext): string {
107
- // Extract path segments from the original template so we can identify
108
- // {param} placeholders before they get mangled by URL parsing.
109
- const url = tryParseUrl(ctx.url_template);
110
- const rawPathname = url
111
- ? extractPathname(ctx.url_template)
112
- : "";
113
- const pathSegments = rawPathname
114
- .split("/")
115
- .filter(Boolean)
116
- .filter((s) => !/^v\d+$/.test(s) && !/^\{[^}]+\}$/.test(s));
117
- const subject =
118
- pathSegments[pathSegments.length - 1] ?? url?.hostname ?? ctx.domain;
119
-
120
- const action = deriveAction(ctx.method, ctx.params, rawPathname);
121
- const parts: string[] = [`${action} ${titleCase(subject)}`];
122
-
123
- if (ctx.sample_response_keys && ctx.sample_response_keys.length > 0) {
124
- parts.push(`with ${ctx.sample_response_keys.slice(0, 6).join(", ")}`);
125
- }
126
-
127
- const paramNames = ctx.params.map((p) => p.name);
128
- if (paramNames.length > 0) {
129
- parts.push(`using ${paramNames.slice(0, 4).join(", ")}`);
130
- }
131
-
132
- return parts.join(" ");
133
- }
134
-
135
- export function inferDescriptionParams(
136
- input: Record<string, unknown>,
137
- locationHints?: Record<string, string>,
138
- ): DescriptionParamContext[] {
139
- const schema = inferSchema([input]);
140
- const requiredKeys = new Set(schema.required ?? []);
141
- const properties = schema.properties ?? {};
142
- return Object.entries(input).map(([name, value]) => {
143
- const propertySchema = properties[name];
144
- const enumValues = Array.isArray(value)
145
- ? value
146
- .filter((entry) => typeof entry === "string" || typeof entry === "number" || typeof entry === "boolean")
147
- .slice(0, 6) as Array<string | number | boolean>
148
- : undefined;
149
- const sanitizedExample = sanitizeAgentVisibleValue(name, value);
150
- return {
151
- name,
152
- in: locationHints?.[name] ?? "body",
153
- ...(propertySchema?.type ? { type: propertySchema.type } : {}),
154
- ...(requiredKeys.has(name) ? { required: true } : {}),
155
- ...(enumValues && enumValues.length > 1 ? { enum_values: enumValues } : {}),
156
- ...(sanitizedExample != null && typeof sanitizedExample !== "object"
157
- ? { example: String(sanitizedExample) }
158
- : {}),
159
- ...(inferParamDescription({ name, in: locationHints?.[name] ?? "body" }) ? { description: inferParamDescription({ name, in: locationHints?.[name] ?? "body" }) } : {}),
160
- };
161
- });
162
- }
163
-
164
- function titleCase(text: string): string {
165
- return text
166
- .replace(/[-_]/g, " ")
167
- .split(/\s+/)
168
- .filter(Boolean)
169
- .map((w) => (w[0] ? w[0].toUpperCase() + w.slice(1) : ""))
170
- .join(" ");
171
- }
172
-
173
- function deriveAction(
174
- method: string,
175
- params: Array<{ name: string; in: string }>,
176
- pathname: string,
177
- ): string {
178
- const hasSearchParam = params.some((p) =>
179
- /^(q|query|search|term|keyword)$/i.test(p.name),
180
- );
181
- if (hasSearchParam || /search|find|lookup/i.test(pathname)) return "Searches";
182
- if (/status|health|incident|maintenance/i.test(pathname))
183
- return "Returns status for";
184
- switch (method.toUpperCase()) {
185
- case "POST":
186
- return "Creates";
187
- case "PUT":
188
- case "PATCH":
189
- return "Updates";
190
- case "DELETE":
191
- return "Deletes";
192
- default:
193
- if (pathname.match(/\{[^}]+\}|\/[0-9A-Za-z_-]{4,}(\/|$)/))
194
- return "Returns details for";
195
- return "Returns";
196
- }
197
- }
198
-
199
- function tryParseUrl(template: string): URL | null {
200
- try {
201
- return new URL(template.replace(/\{[^}]+\}/g, "_"));
202
- } catch {
203
- return null;
204
- }
205
- }
206
-
207
- /** Extract pathname from a URL template string, preserving {param} placeholders. */
208
- function extractPathname(template: string): string {
209
- // Strip scheme + authority, then take everything before ? or #
210
- const afterScheme = template.replace(/^https?:\/\/[^/]*/, "");
211
- const pathOnly = afterScheme.split(/[?#]/)[0] ?? afterScheme;
212
- return pathOnly || "/";
213
- }