unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/cli.js +79 -33
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +73 -22
  4. package/dist/server.js +25994 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/manifest.json +1 -1
  7. package/runtime-src/agent-outcome.ts +0 -166
  8. package/runtime-src/analytics-session.ts +0 -55
  9. package/runtime-src/api/browse-index.ts +0 -343
  10. package/runtime-src/api/browse-session.ts +0 -572
  11. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  12. package/runtime-src/api/browse-submit.ts +0 -1184
  13. package/runtime-src/api/routes.ts +0 -1833
  14. package/runtime-src/auth/browser-cookies.ts +0 -423
  15. package/runtime-src/auth/index.ts +0 -535
  16. package/runtime-src/auth/runtime.ts +0 -116
  17. package/runtime-src/browser/index.ts +0 -659
  18. package/runtime-src/browser/types.ts +0 -41
  19. package/runtime-src/build-info.generated.ts +0 -6
  20. package/runtime-src/capture/index.ts +0 -1929
  21. package/runtime-src/capture/prefetch.ts +0 -95
  22. package/runtime-src/capture/rsc.ts +0 -45
  23. package/runtime-src/cli/shortcuts.ts +0 -273
  24. package/runtime-src/cli.ts +0 -1734
  25. package/runtime-src/client/graph-client.ts +0 -100
  26. package/runtime-src/client/index.ts +0 -1425
  27. package/runtime-src/debug-trace.ts +0 -18
  28. package/runtime-src/domain.ts +0 -38
  29. package/runtime-src/execution/index.ts +0 -3407
  30. package/runtime-src/execution/retry.ts +0 -46
  31. package/runtime-src/execution/robots.ts +0 -167
  32. package/runtime-src/execution/search-forms.ts +0 -188
  33. package/runtime-src/execution/token-resolver.ts +0 -135
  34. package/runtime-src/extraction/index.ts +0 -1507
  35. package/runtime-src/foundry/publish-bundle.ts +0 -392
  36. package/runtime-src/graph/agent-augment.ts +0 -315
  37. package/runtime-src/graph/index.ts +0 -1532
  38. package/runtime-src/graph/local-fixtures.ts +0 -393
  39. package/runtime-src/graph/local-harness.ts +0 -646
  40. package/runtime-src/graph/planner.ts +0 -411
  41. package/runtime-src/graph/session.ts +0 -294
  42. package/runtime-src/graph/trace-store.ts +0 -136
  43. package/runtime-src/impact-log.ts +0 -227
  44. package/runtime-src/index.ts +0 -24
  45. package/runtime-src/indexer/index.ts +0 -465
  46. package/runtime-src/intent-match.ts +0 -1515
  47. package/runtime-src/kuri/client.ts +0 -1839
  48. package/runtime-src/logger.ts +0 -30
  49. package/runtime-src/marketplace/index.ts +0 -111
  50. package/runtime-src/mcp.ts +0 -1911
  51. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  52. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  53. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  54. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  55. package/runtime-src/orchestrator/index.ts +0 -4480
  56. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  57. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  58. package/runtime-src/payments/cascade.ts +0 -137
  59. package/runtime-src/payments/index.ts +0 -270
  60. package/runtime-src/payments/lobster-pay.ts +0 -182
  61. package/runtime-src/payments/wallet.ts +0 -98
  62. package/runtime-src/publish/review-context.ts +0 -93
  63. package/runtime-src/publish/sanitize.ts +0 -197
  64. package/runtime-src/publish/schema-review.ts +0 -192
  65. package/runtime-src/publish-admission.ts +0 -388
  66. package/runtime-src/ratelimit/index.ts +0 -23
  67. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  68. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  69. package/runtime-src/reverse-engineer/index.ts +0 -1551
  70. package/runtime-src/reverse-engineer/token-sources.ts +0 -379
  71. package/runtime-src/router.ts +0 -17
  72. package/runtime-src/routing-telemetry.ts +0 -395
  73. package/runtime-src/runtime/browser-access.ts +0 -11
  74. package/runtime-src/runtime/browser-auth.ts +0 -12
  75. package/runtime-src/runtime/browser-host.ts +0 -48
  76. package/runtime-src/runtime/lifecycle.ts +0 -17
  77. package/runtime-src/runtime/local-server.ts +0 -311
  78. package/runtime-src/runtime/paths.ts +0 -99
  79. package/runtime-src/runtime/setup.ts +0 -251
  80. package/runtime-src/runtime/supervisor.ts +0 -69
  81. package/runtime-src/runtime/update-hints.ts +0 -351
  82. package/runtime-src/server.ts +0 -100
  83. package/runtime-src/session-logs.ts +0 -142
  84. package/runtime-src/settings.ts +0 -221
  85. package/runtime-src/single-binary.ts +0 -143
  86. package/runtime-src/site-policy.ts +0 -54
  87. package/runtime-src/stale-cleanup-runner.ts +0 -144
  88. package/runtime-src/stale-cleanup.ts +0 -133
  89. package/runtime-src/telemetry-attribution.ts +0 -120
  90. package/runtime-src/telemetry.ts +0 -253
  91. package/runtime-src/template-params.ts +0 -141
  92. package/runtime-src/transform/drift.ts +0 -60
  93. package/runtime-src/transform/index.ts +0 -277
  94. package/runtime-src/types/index.ts +0 -1
  95. package/runtime-src/types/skill.ts +0 -931
  96. package/runtime-src/vault/index.ts +0 -196
  97. package/runtime-src/verification/auth-gate.ts +0 -8
  98. package/runtime-src/verification/candidates.ts +0 -27
  99. package/runtime-src/verification/index.ts +0 -120
  100. package/runtime-src/verification/matrix.ts +0 -30
  101. package/runtime-src/version.ts +0 -148
  102. package/runtime-src/workflow/artifact.ts +0 -161
  103. package/runtime-src/workflow/compile.ts +0 -808
  104. package/runtime-src/workflow/publish.ts +0 -225
  105. package/runtime-src/workflow/runtime.ts +0 -213
@@ -1,192 +0,0 @@
1
- import { sanitizeAgentVisibleValue } from "./sanitize.js";
2
- import type {
3
- EndpointParameterReview,
4
- EndpointResponseFieldReview,
5
- EndpointReviewPayload,
6
- OperationBinding,
7
- ResponseSchema,
8
- WorkflowArtifact,
9
- WorkflowParameterConstraint,
10
- WorkflowParameterSpec,
11
- } from "../types/index.js";
12
-
13
- function tokenizeSchemaPath(path: string): string[] {
14
- const tokens: string[] = [];
15
- for (const segment of path.split(".").filter(Boolean)) {
16
- const matches = segment.match(/([^[\]]+)|\[\]/g);
17
- if (matches) tokens.push(...matches);
18
- }
19
- return tokens;
20
- }
21
-
22
- function getSchemaNode(schema: ResponseSchema | undefined, path: string): ResponseSchema | null {
23
- if (!schema) return null;
24
- const tokens = tokenizeSchemaPath(path);
25
- let node: ResponseSchema | undefined = schema;
26
- for (const token of tokens) {
27
- if (!node) return null;
28
- node = token === "[]"
29
- ? node.items
30
- : node.properties?.[token];
31
- }
32
- return node ?? null;
33
- }
34
-
35
- export function flattenResponseSchemaFields(
36
- schema?: ResponseSchema,
37
- prefix = "",
38
- required = false,
39
- ): Array<{
40
- path: string;
41
- type: string;
42
- required: boolean;
43
- description?: string;
44
- enum_values?: Array<string | number | boolean>;
45
- }> {
46
- if (!schema) return [];
47
- const rows: Array<{
48
- path: string;
49
- type: string;
50
- required: boolean;
51
- description?: string;
52
- enum_values?: Array<string | number | boolean>;
53
- }> = [];
54
-
55
- if (prefix) {
56
- rows.push({
57
- path: prefix,
58
- type: schema.type,
59
- required,
60
- ...(schema.description ? { description: schema.description } : {}),
61
- ...(schema.enum_values?.length ? { enum_values: schema.enum_values } : {}),
62
- });
63
- }
64
-
65
- for (const [key, child] of Object.entries(schema.properties ?? {})) {
66
- const childPath = prefix ? `${prefix}.${key}` : key;
67
- rows.push(...flattenResponseSchemaFields(child, childPath, (schema.required ?? []).includes(key)));
68
- }
69
- if (schema.items) {
70
- const childPath = prefix ? `${prefix}[]` : "[]";
71
- rows.push(...flattenResponseSchemaFields(schema.items, childPath, true));
72
- }
73
- return rows;
74
- }
75
-
76
- export function applyResponseSchemaReviews(
77
- schema: ResponseSchema | undefined,
78
- reviews: EndpointResponseFieldReview[] | undefined,
79
- ): ResponseSchema | undefined {
80
- if (!schema || !reviews?.length) return schema;
81
- const next = structuredClone(schema);
82
- for (const review of reviews) {
83
- const node = getSchemaNode(next, review.path);
84
- if (!node) continue;
85
- if (review.description !== undefined) node.description = review.description;
86
- if (review.type !== undefined) node.type = review.type;
87
- if (review.enum_values !== undefined) node.enum_values = review.enum_values;
88
- }
89
- return next;
90
- }
91
-
92
- function mergeConstraint(
93
- constraints: WorkflowParameterConstraint[] | undefined,
94
- nextConstraint: WorkflowParameterConstraint,
95
- ): WorkflowParameterConstraint[] {
96
- const out = [...(constraints ?? []).filter((entry) => entry.kind !== nextConstraint.kind)];
97
- out.push(nextConstraint);
98
- return out;
99
- }
100
-
101
- export function applyParameterReviews(
102
- specs: WorkflowParameterSpec[],
103
- reviews: EndpointParameterReview[] | undefined,
104
- ): WorkflowParameterSpec[] {
105
- if (!reviews?.length) return specs;
106
- const reviewMap = new Map(reviews.map((review) => [`${review.location}:${review.name}`, review]));
107
- return specs.map((spec) => {
108
- const review = reviewMap.get(`${spec.location}:${spec.name}`);
109
- if (!review) return spec;
110
- let constraints = spec.constraints;
111
- if (review.format !== undefined) {
112
- constraints = mergeConstraint(constraints, {
113
- kind: "format",
114
- value: review.format,
115
- });
116
- }
117
- if (review.enum_values !== undefined && review.enum_values.length > 1) {
118
- constraints = mergeConstraint(constraints, {
119
- kind: "enum",
120
- description: `Reviewed allowed values for ${spec.name}.`,
121
- });
122
- }
123
- return {
124
- ...spec,
125
- ...(review.description !== undefined ? { description: review.description } : {}),
126
- ...(review.type !== undefined ? { type: review.type } : {}),
127
- ...(review.required !== undefined ? { required: review.required } : {}),
128
- ...(review.user_supplied !== undefined ? { user_supplied: review.user_supplied } : {}),
129
- ...(review.format !== undefined ? { format: review.format } : {}),
130
- ...(review.enum_values !== undefined ? { enum_values: review.enum_values } : {}),
131
- ...(review.derived_from !== undefined ? { derived_from: review.derived_from } : {}),
132
- ...(constraints?.length ? { constraints } : {}),
133
- };
134
- });
135
- }
136
-
137
- export function applyBindingReviews(
138
- bindings: OperationBinding[] | undefined,
139
- reviews: EndpointParameterReview[] | undefined,
140
- ): OperationBinding[] | undefined {
141
- if (!bindings || !reviews?.length) return bindings;
142
- const reviewMap = new Map(reviews.map((review) => [review.name, review]));
143
- return bindings.map((binding) => {
144
- const review = reviewMap.get(binding.key);
145
- if (!review) return binding;
146
- return {
147
- ...binding,
148
- ...(review.description !== undefined ? { description: review.description } : {}),
149
- ...(review.type !== undefined ? { type: review.type } : {}),
150
- ...(review.required !== undefined ? { required: review.required } : {}),
151
- };
152
- });
153
- }
154
-
155
- export function applyWorkflowSchemaReviews(
156
- artifact: WorkflowArtifact | null,
157
- reviews: EndpointReviewPayload[],
158
- ): WorkflowArtifact | null {
159
- if (!artifact) return artifact;
160
- const reviewMap = new Map(reviews.map((review) => [review.endpoint_id, review]));
161
- return {
162
- ...artifact,
163
- recipes: artifact.recipes.map((recipe) => {
164
- const review = reviewMap.get(recipe.endpoint_id);
165
- if (!review?.parameter_reviews?.length) return recipe;
166
- return {
167
- ...recipe,
168
- replay_contract: {
169
- ...recipe.replay_contract,
170
- parameter_specs: applyParameterReviews(recipe.replay_contract.parameter_specs, review.parameter_reviews),
171
- },
172
- };
173
- }),
174
- };
175
- }
176
-
177
- export function buildSafeParameterSpecView(spec: WorkflowParameterSpec): Record<string, unknown> {
178
- return {
179
- name: spec.name,
180
- location: spec.location,
181
- type: spec.type,
182
- required: spec.required,
183
- user_supplied: spec.user_supplied,
184
- ...(spec.description ? { description: spec.description } : {}),
185
- ...(spec.format ? { format: spec.format } : {}),
186
- ...(spec.enum_values?.length ? { enum_values: spec.enum_values } : {}),
187
- ...(spec.derived_from?.length ? { derived_from: spec.derived_from } : {}),
188
- ...(spec.default_value !== undefined ? { default_value: sanitizeAgentVisibleValue(spec.name, spec.default_value) } : {}),
189
- ...(spec.example_value !== undefined ? { example_value: sanitizeAgentVisibleValue(spec.name, spec.example_value) } : {}),
190
- source_hints: spec.source_hints,
191
- };
192
- }
@@ -1,388 +0,0 @@
1
- import { getRegistrableDomain } from "./domain.js";
2
- import type { EndpointDescriptor, SkillManifest } from "./types/index.js";
3
-
4
- export type PublishAdmissionReason =
5
- | "ws"
6
- | "non_actionable_method"
7
- | "verification_failed"
8
- | "low_reliability"
9
- | "off_domain"
10
- | "noise"
11
- | "fragile_graphql"
12
- | "no_durable_signal"
13
- | "family_dedup"
14
- | "over_limit";
15
-
16
- export type MarketplacePublishSelection = {
17
- endpoints: EndpointDescriptor[];
18
- stats: {
19
- total: number;
20
- kept: number;
21
- by_reason: Record<PublishAdmissionReason, number>;
22
- };
23
- };
24
-
25
- export type MarketplacePublishClosure = MarketplacePublishSelection & {
26
- root_endpoints: EndpointDescriptor[];
27
- root_endpoint_ids: string[];
28
- closure_endpoint_ids: string[];
29
- closure_operation_ids: string[];
30
- closure_edge_count: number;
31
- };
32
-
33
- type PublishSelectionOptions = {
34
- limit?: number;
35
- };
36
-
37
- type ScoredCandidate = {
38
- endpoint: EndpointDescriptor;
39
- familyKey: string;
40
- score: number;
41
- };
42
-
43
- const DEFAULT_PUBLISH_ENDPOINT_LIMIT = readPositiveIntEnv("UNBROWSE_PUBLISH_ENDPOINT_LIMIT", 12);
44
- const MIN_PUBLISH_RELIABILITY = readProbabilityEnv("UNBROWSE_PUBLISH_MIN_RELIABILITY", 0.2);
45
- const CLOSURE_EDGE_KINDS = new Set(["dependency", "auth", "parent_child", "pagination"]);
46
-
47
- function readPositiveIntEnv(name: string, fallback: number): number {
48
- const parsed = Number.parseInt(process.env[name] ?? "", 10);
49
- return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
50
- }
51
-
52
- function readProbabilityEnv(name: string, fallback: number): number {
53
- const parsed = Number.parseFloat(process.env[name] ?? "");
54
- return Number.isFinite(parsed) && parsed >= 0 && parsed <= 1 ? parsed : fallback;
55
- }
56
-
57
- function freshReasonCounts(): Record<PublishAdmissionReason, number> {
58
- return {
59
- ws: 0,
60
- non_actionable_method: 0,
61
- verification_failed: 0,
62
- low_reliability: 0,
63
- off_domain: 0,
64
- noise: 0,
65
- fragile_graphql: 0,
66
- no_durable_signal: 0,
67
- family_dedup: 0,
68
- over_limit: 0,
69
- };
70
- }
71
-
72
- function isApiLikePath(pathname: string): boolean {
73
- return /\/api\/|graphql|\/rest\/|\/rpc\/|voyager/i.test(pathname);
74
- }
75
-
76
- function looksOpaqueIdentifier(value: string): boolean {
77
- if (!value || /^\{[^}]+\}$/.test(value)) return false;
78
- if (/^[0-9a-f]{12,}$/i.test(value)) return true;
79
- if (value.length < 16) return false;
80
- const hasAlpha = /[A-Za-z]/.test(value);
81
- const hasDigit = /\d/.test(value);
82
- return hasAlpha && hasDigit;
83
- }
84
-
85
- function isCanonicalDocumentReplay(endpoint: EndpointDescriptor): boolean {
86
- if (/captured page artifact/i.test(endpoint.description ?? "")) return true;
87
- if (!endpoint.trigger_url) return false;
88
- try {
89
- const endpointUrl = new URL(endpoint.url_template);
90
- const triggerUrl = new URL(endpoint.trigger_url);
91
- return (
92
- endpointUrl.origin === triggerUrl.origin &&
93
- endpointUrl.pathname === triggerUrl.pathname &&
94
- !isApiLikePath(endpointUrl.pathname)
95
- );
96
- } catch {
97
- return false;
98
- }
99
- }
100
-
101
- function hasRichSemanticEvidence(endpoint: EndpointDescriptor): boolean {
102
- return !!(
103
- endpoint.semantic?.example_response_compact ||
104
- endpoint.semantic?.response_summary ||
105
- (endpoint.semantic?.example_fields?.length ?? 0) > 0
106
- );
107
- }
108
-
109
- function hasDurableSignal(endpoint: EndpointDescriptor): boolean {
110
- return !!(
111
- endpoint.response_schema ||
112
- endpoint.dom_extraction ||
113
- endpoint.search_form ||
114
- hasRichSemanticEvidence(endpoint) ||
115
- isCanonicalDocumentReplay(endpoint)
116
- );
117
- }
118
-
119
- function isVerifiedDurable(endpoint: EndpointDescriptor): boolean {
120
- return (
121
- endpoint.verification_status === "verified" &&
122
- endpoint.reliability_score >= 0.9 &&
123
- !!endpoint.response_schema
124
- );
125
- }
126
-
127
- function isSkillDomainEndpoint(skill: SkillManifest, endpoint: EndpointDescriptor): boolean {
128
- try {
129
- const endpointHost = new URL(endpoint.url_template).hostname;
130
- const skillDomain = getRegistrableDomain(skill.domain);
131
- const endpointDomain = getRegistrableDomain(endpointHost);
132
- return endpointDomain === skillDomain;
133
- } catch {
134
- return false;
135
- }
136
- }
137
-
138
- function isLikelyNoiseEndpoint(endpoint: EndpointDescriptor): boolean {
139
- const haystack = [endpoint.url_template, endpoint.description ?? ""].join(" ").toLowerCase();
140
- return /(recaptcha|captcha|csrf|telemetry|analytics|tracking|logging|metrics|beacon|consent|cookie[-_ ]?banner|feature[-_ ]?flag|oauth|session[-_/ ]?refresh|auth[-_/ ]?refresh|header-action|badge)/i.test(
141
- haystack,
142
- );
143
- }
144
-
145
- function isFragileGraphqlEndpoint(endpoint: EndpointDescriptor): boolean {
146
- try {
147
- const url = new URL(endpoint.url_template);
148
- const pathSegments = url.pathname.split("/").filter(Boolean);
149
- const isGraphLike = pathSegments.some((segment) => /graphql|voyager/i.test(segment));
150
- if (!isGraphLike) return false;
151
-
152
- const queryId = url.searchParams.get("queryId") ?? "";
153
- if (queryId) {
154
- const parts = queryId.split(/[.:]/g).filter(Boolean);
155
- if (parts.some(looksOpaqueIdentifier)) return true;
156
- }
157
-
158
- const opaqueGraphSegment = pathSegments.some((segment, index) => {
159
- if (!/graphql|voyager/i.test(pathSegments[index - 1] ?? "")) return false;
160
- return looksOpaqueIdentifier(segment);
161
- });
162
- if (opaqueGraphSegment) return true;
163
-
164
- return pathSegments.some((segment) => looksOpaqueIdentifier(segment) && /graphql|voyager/i.test(url.pathname));
165
- } catch {
166
- return /queryId=/i.test(endpoint.url_template) && /graphql|voyager/i.test(endpoint.url_template);
167
- }
168
- }
169
-
170
- function normalizedPathname(pathname: string): string {
171
- return pathname
172
- .split("/")
173
- .filter(Boolean)
174
- .map((segment) => {
175
- if (/^\{[^}]+\}$/.test(segment)) return "{param}";
176
- if (looksOpaqueIdentifier(segment)) return ":opaque";
177
- return segment.toLowerCase();
178
- })
179
- .join("/");
180
- }
181
-
182
- function endpointFamilyKey(endpoint: EndpointDescriptor): string {
183
- try {
184
- const url = new URL(endpoint.url_template);
185
- const queryKeys = new Set<string>();
186
- for (const key of url.searchParams.keys()) queryKeys.add(key);
187
- for (const key of Object.keys(endpoint.query ?? {})) queryKeys.add(key);
188
- for (const key of Object.keys(endpoint.body_params ?? {})) queryKeys.add(`body:${key}`);
189
- return [
190
- endpoint.method,
191
- url.hostname.toLowerCase(),
192
- normalizedPathname(url.pathname),
193
- [...queryKeys].sort().join(","),
194
- endpoint.dom_extraction ? "dom" : "http",
195
- ].join("|");
196
- } catch {
197
- return `${endpoint.method}|${endpoint.url_template.toLowerCase()}`;
198
- }
199
- }
200
-
201
- function scoreEndpoint(endpoint: EndpointDescriptor): number {
202
- let score = endpoint.reliability_score * 100;
203
- if (endpoint.verification_status === "verified") score += 30;
204
- if (endpoint.response_schema) score += 25;
205
- if (endpoint.dom_extraction) score += 8;
206
- if (endpoint.search_form) score += 10;
207
- if (endpoint.semantic?.action_kind) score += 8;
208
- if (endpoint.semantic?.resource_kind) score += 4;
209
- if (endpoint.method === "GET") score += 8;
210
- if (endpoint.method === "POST") score += 3;
211
- if (endpoint.trigger_url) score -= 6;
212
- if (isCanonicalDocumentReplay(endpoint)) score += 18;
213
- if (isFragileGraphqlEndpoint(endpoint)) score -= 25;
214
- return score;
215
- }
216
-
217
- function rejectionReason(skill: SkillManifest, endpoint: EndpointDescriptor): PublishAdmissionReason | null {
218
- if (endpoint.method === "WS") return "ws";
219
- if (endpoint.method === "HEAD" || endpoint.method === "OPTIONS") return "non_actionable_method";
220
- if (endpoint.verification_status === "failed" || endpoint.verification_status === "disabled") {
221
- return "verification_failed";
222
- }
223
- if (endpoint.reliability_score < MIN_PUBLISH_RELIABILITY) return "low_reliability";
224
- if (!isSkillDomainEndpoint(skill, endpoint)) return "off_domain";
225
- if (isLikelyNoiseEndpoint(endpoint)) return "noise";
226
- if (isFragileGraphqlEndpoint(endpoint) && !isVerifiedDurable(endpoint)) return "fragile_graphql";
227
- if (!hasDurableSignal(endpoint)) return "no_durable_signal";
228
- return null;
229
- }
230
-
231
- export function selectMarketplacePublishEndpoints(
232
- skill: SkillManifest,
233
- options: PublishSelectionOptions = {},
234
- ): MarketplacePublishSelection {
235
- const reasons = freshReasonCounts();
236
- const candidates: ScoredCandidate[] = [];
237
-
238
- for (const endpoint of skill.endpoints ?? []) {
239
- const reason = rejectionReason(skill, endpoint);
240
- if (reason) {
241
- reasons[reason] += 1;
242
- continue;
243
- }
244
- candidates.push({
245
- endpoint,
246
- familyKey: endpointFamilyKey(endpoint),
247
- score: scoreEndpoint(endpoint),
248
- });
249
- }
250
-
251
- candidates.sort((a, b) => {
252
- if (b.score !== a.score) return b.score - a.score;
253
- return a.endpoint.endpoint_id.localeCompare(b.endpoint.endpoint_id);
254
- });
255
-
256
- const selected: ScoredCandidate[] = [];
257
- const seenFamilies = new Set<string>();
258
- for (const candidate of candidates) {
259
- if (seenFamilies.has(candidate.familyKey)) {
260
- reasons.family_dedup += 1;
261
- continue;
262
- }
263
- seenFamilies.add(candidate.familyKey);
264
- selected.push(candidate);
265
- }
266
-
267
- const limit = options.limit ?? DEFAULT_PUBLISH_ENDPOINT_LIMIT;
268
- if (selected.length > limit) {
269
- reasons.over_limit += selected.length - limit;
270
- }
271
-
272
- const endpoints = selected.slice(0, limit).map((candidate) => candidate.endpoint);
273
- return {
274
- endpoints,
275
- stats: {
276
- total: skill.endpoints?.length ?? 0,
277
- kept: endpoints.length,
278
- by_reason: reasons,
279
- },
280
- };
281
- }
282
-
283
- export function selectMarketplacePublishClosure(
284
- skill: SkillManifest,
285
- options: PublishSelectionOptions = {},
286
- ): MarketplacePublishClosure {
287
- const roots = selectMarketplacePublishEndpoints(skill, options);
288
- const rootEndpointIds = roots.endpoints.map((endpoint) => endpoint.endpoint_id);
289
-
290
- if (rootEndpointIds.length === 0) {
291
- return {
292
- ...roots,
293
- root_endpoints: roots.endpoints,
294
- root_endpoint_ids: [],
295
- closure_endpoint_ids: [],
296
- closure_operation_ids: [],
297
- closure_edge_count: 0,
298
- };
299
- }
300
-
301
- const graph = skill.operation_graph;
302
- if (!graph?.operations?.length || !graph.edges?.length) {
303
- return {
304
- ...roots,
305
- root_endpoints: roots.endpoints,
306
- root_endpoint_ids: rootEndpointIds,
307
- closure_endpoint_ids: rootEndpointIds,
308
- closure_operation_ids: rootEndpointIds,
309
- closure_edge_count: 0,
310
- };
311
- }
312
-
313
- const endpointById = new Map(skill.endpoints.map((endpoint) => [endpoint.endpoint_id, endpoint]));
314
- const operationById = new Map(graph.operations.map((operation) => [operation.operation_id, operation]));
315
- const operationByEndpointId = new Map(graph.operations.map((operation) => [operation.endpoint_id, operation]));
316
- const adjacency = new Map<string, Set<string>>();
317
-
318
- for (const edge of graph.edges) {
319
- if (!CLOSURE_EDGE_KINDS.has(edge.kind)) continue;
320
- const fromSet = adjacency.get(edge.from_operation_id) ?? new Set<string>();
321
- fromSet.add(edge.to_operation_id);
322
- adjacency.set(edge.from_operation_id, fromSet);
323
- const toSet = adjacency.get(edge.to_operation_id) ?? new Set<string>();
324
- toSet.add(edge.from_operation_id);
325
- adjacency.set(edge.to_operation_id, toSet);
326
- }
327
-
328
- const seenOperationIds = new Set<string>();
329
- const queue: string[] = [];
330
- for (const endpointId of rootEndpointIds) {
331
- const operationId = operationByEndpointId.get(endpointId)?.operation_id ?? endpointId;
332
- if (seenOperationIds.has(operationId)) continue;
333
- seenOperationIds.add(operationId);
334
- queue.push(operationId);
335
- }
336
-
337
- while (queue.length > 0) {
338
- const current = queue.shift();
339
- if (!current) continue;
340
- for (const neighborId of adjacency.get(current) ?? []) {
341
- if (seenOperationIds.has(neighborId)) continue;
342
- const neighborOperation = operationById.get(neighborId);
343
- if (!neighborOperation) continue;
344
- const neighborEndpoint = endpointById.get(neighborOperation.endpoint_id);
345
- if (!neighborEndpoint) continue;
346
- if (rejectionReason(skill, neighborEndpoint)) continue;
347
- seenOperationIds.add(neighborId);
348
- queue.push(neighborId);
349
- }
350
- }
351
-
352
- const includedEndpointIds = new Set<string>(rootEndpointIds);
353
- for (const operationId of seenOperationIds) {
354
- const operation = operationById.get(operationId);
355
- if (!operation) continue;
356
- includedEndpointIds.add(operation.endpoint_id);
357
- }
358
-
359
- const orderedEndpoints = [
360
- ...roots.endpoints,
361
- ...skill.endpoints.filter(
362
- (endpoint) => includedEndpointIds.has(endpoint.endpoint_id) && !rootEndpointIds.includes(endpoint.endpoint_id),
363
- ),
364
- ];
365
-
366
- const closureEdgeCount = graph.edges.filter((edge) =>
367
- CLOSURE_EDGE_KINDS.has(edge.kind) &&
368
- seenOperationIds.has(edge.from_operation_id) &&
369
- seenOperationIds.has(edge.to_operation_id),
370
- ).length;
371
-
372
- return {
373
- ...roots,
374
- endpoints: orderedEndpoints,
375
- root_endpoints: roots.endpoints,
376
- root_endpoint_ids: rootEndpointIds,
377
- closure_endpoint_ids: orderedEndpoints.map((endpoint) => endpoint.endpoint_id),
378
- closure_operation_ids: [...seenOperationIds],
379
- closure_edge_count: closureEdgeCount,
380
- };
381
- }
382
-
383
- export function formatMarketplacePublishSelection(selection: MarketplacePublishSelection): string {
384
- const parts = Object.entries(selection.stats.by_reason)
385
- .filter(([, count]) => count > 0)
386
- .map(([reason, count]) => `${reason}=${count}`);
387
- return parts.length > 0 ? parts.join(", ") : "no drops";
388
- }
@@ -1,23 +0,0 @@
1
- import type { FastifyInstance } from "fastify";
2
- import rateLimit from "@fastify/rate-limit";
3
-
4
- export async function registerRateLimiter(app: FastifyInstance): Promise<void> {
5
- await app.register(rateLimit, {
6
- max: 100,
7
- timeWindow: "1 minute",
8
- });
9
- }
10
-
11
- /** Per-route rate limit configs. Apply via route options in Fastify. */
12
- export const ROUTE_LIMITS = {
13
- "/v1/intent/resolve": { max: 20, timeWindow: "1 minute" },
14
- "/v1/skills/:skill_id/execute": { max: 30, timeWindow: "1 minute" },
15
- "/v1/skills": { max: 5, timeWindow: "1 minute" }, // POST only
16
- "/v1/auth/login": { max: 3, timeWindow: "5 minutes" },
17
- "/v1/feedback": { max: 60, timeWindow: "1 minute" },
18
- } as const;
19
-
20
- export function routeRateLimit(path: keyof typeof ROUTE_LIMITS) {
21
- const cfg = ROUTE_LIMITS[path];
22
- return { config: { rateLimit: { max: cfg.max, timeWindow: cfg.timeWindow } } };
23
- }