unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/cli.js +79 -33
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +73 -22
  4. package/dist/server.js +25994 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/manifest.json +1 -1
  7. package/runtime-src/agent-outcome.ts +0 -166
  8. package/runtime-src/analytics-session.ts +0 -55
  9. package/runtime-src/api/browse-index.ts +0 -343
  10. package/runtime-src/api/browse-session.ts +0 -572
  11. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  12. package/runtime-src/api/browse-submit.ts +0 -1184
  13. package/runtime-src/api/routes.ts +0 -1833
  14. package/runtime-src/auth/browser-cookies.ts +0 -423
  15. package/runtime-src/auth/index.ts +0 -535
  16. package/runtime-src/auth/runtime.ts +0 -116
  17. package/runtime-src/browser/index.ts +0 -659
  18. package/runtime-src/browser/types.ts +0 -41
  19. package/runtime-src/build-info.generated.ts +0 -6
  20. package/runtime-src/capture/index.ts +0 -1929
  21. package/runtime-src/capture/prefetch.ts +0 -95
  22. package/runtime-src/capture/rsc.ts +0 -45
  23. package/runtime-src/cli/shortcuts.ts +0 -273
  24. package/runtime-src/cli.ts +0 -1734
  25. package/runtime-src/client/graph-client.ts +0 -100
  26. package/runtime-src/client/index.ts +0 -1425
  27. package/runtime-src/debug-trace.ts +0 -18
  28. package/runtime-src/domain.ts +0 -38
  29. package/runtime-src/execution/index.ts +0 -3407
  30. package/runtime-src/execution/retry.ts +0 -46
  31. package/runtime-src/execution/robots.ts +0 -167
  32. package/runtime-src/execution/search-forms.ts +0 -188
  33. package/runtime-src/execution/token-resolver.ts +0 -135
  34. package/runtime-src/extraction/index.ts +0 -1507
  35. package/runtime-src/foundry/publish-bundle.ts +0 -392
  36. package/runtime-src/graph/agent-augment.ts +0 -315
  37. package/runtime-src/graph/index.ts +0 -1532
  38. package/runtime-src/graph/local-fixtures.ts +0 -393
  39. package/runtime-src/graph/local-harness.ts +0 -646
  40. package/runtime-src/graph/planner.ts +0 -411
  41. package/runtime-src/graph/session.ts +0 -294
  42. package/runtime-src/graph/trace-store.ts +0 -136
  43. package/runtime-src/impact-log.ts +0 -227
  44. package/runtime-src/index.ts +0 -24
  45. package/runtime-src/indexer/index.ts +0 -465
  46. package/runtime-src/intent-match.ts +0 -1515
  47. package/runtime-src/kuri/client.ts +0 -1839
  48. package/runtime-src/logger.ts +0 -30
  49. package/runtime-src/marketplace/index.ts +0 -111
  50. package/runtime-src/mcp.ts +0 -1911
  51. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  52. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  53. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  54. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  55. package/runtime-src/orchestrator/index.ts +0 -4480
  56. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  57. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  58. package/runtime-src/payments/cascade.ts +0 -137
  59. package/runtime-src/payments/index.ts +0 -270
  60. package/runtime-src/payments/lobster-pay.ts +0 -182
  61. package/runtime-src/payments/wallet.ts +0 -98
  62. package/runtime-src/publish/review-context.ts +0 -93
  63. package/runtime-src/publish/sanitize.ts +0 -197
  64. package/runtime-src/publish/schema-review.ts +0 -192
  65. package/runtime-src/publish-admission.ts +0 -388
  66. package/runtime-src/ratelimit/index.ts +0 -23
  67. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  68. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  69. package/runtime-src/reverse-engineer/index.ts +0 -1551
  70. package/runtime-src/reverse-engineer/token-sources.ts +0 -379
  71. package/runtime-src/router.ts +0 -17
  72. package/runtime-src/routing-telemetry.ts +0 -395
  73. package/runtime-src/runtime/browser-access.ts +0 -11
  74. package/runtime-src/runtime/browser-auth.ts +0 -12
  75. package/runtime-src/runtime/browser-host.ts +0 -48
  76. package/runtime-src/runtime/lifecycle.ts +0 -17
  77. package/runtime-src/runtime/local-server.ts +0 -311
  78. package/runtime-src/runtime/paths.ts +0 -99
  79. package/runtime-src/runtime/setup.ts +0 -251
  80. package/runtime-src/runtime/supervisor.ts +0 -69
  81. package/runtime-src/runtime/update-hints.ts +0 -351
  82. package/runtime-src/server.ts +0 -100
  83. package/runtime-src/session-logs.ts +0 -142
  84. package/runtime-src/settings.ts +0 -221
  85. package/runtime-src/single-binary.ts +0 -143
  86. package/runtime-src/site-policy.ts +0 -54
  87. package/runtime-src/stale-cleanup-runner.ts +0 -144
  88. package/runtime-src/stale-cleanup.ts +0 -133
  89. package/runtime-src/telemetry-attribution.ts +0 -120
  90. package/runtime-src/telemetry.ts +0 -253
  91. package/runtime-src/template-params.ts +0 -141
  92. package/runtime-src/transform/drift.ts +0 -60
  93. package/runtime-src/transform/index.ts +0 -277
  94. package/runtime-src/types/index.ts +0 -1
  95. package/runtime-src/types/skill.ts +0 -931
  96. package/runtime-src/vault/index.ts +0 -196
  97. package/runtime-src/verification/auth-gate.ts +0 -8
  98. package/runtime-src/verification/candidates.ts +0 -27
  99. package/runtime-src/verification/index.ts +0 -120
  100. package/runtime-src/verification/matrix.ts +0 -30
  101. package/runtime-src/version.ts +0 -148
  102. package/runtime-src/workflow/artifact.ts +0 -161
  103. package/runtime-src/workflow/compile.ts +0 -808
  104. package/runtime-src/workflow/publish.ts +0 -225
  105. package/runtime-src/workflow/runtime.ts +0 -213
@@ -1,646 +0,0 @@
1
- import { existsSync, readdirSync, readFileSync } from "fs";
2
- import { homedir } from "os";
3
- import { performance } from "perf_hooks";
4
- import { join } from "path";
5
- import { getSkillChunk, isOperationHardExcluded, knownBindingsFromInputs, operationSoftPenalty } from "./index.js";
6
- import type { SkillChunk, SkillManifest, SkillOperationNode } from "../types/index.js";
7
- import type { LocalHarnessCase } from "./local-fixtures.js";
8
-
9
- export interface HarnessOperationHit {
10
- skill_id: string;
11
- skill_domain: string;
12
- operation_id: string;
13
- endpoint_id: string;
14
- score: number;
15
- runnable: boolean;
16
- action_kind: string;
17
- resource_kind: string;
18
- negative_tags: string[];
19
- auth_required: boolean;
20
- }
21
-
22
- export interface HarnessCaseResult {
23
- id: string;
24
- pass: boolean;
25
- selected?: HarnessOperationHit;
26
- top_hits: HarnessOperationHit[];
27
- chunk: SkillChunk;
28
- expected_skill_id: string;
29
- expected_operation_id: string;
30
- selection_ms: number;
31
- expected_rank: number;
32
- wrong_top1_before_correct: number;
33
- failure_reason?: string;
34
- }
35
-
36
- export interface HarnessSummary {
37
- total: number;
38
- passed: number;
39
- failed: number;
40
- hit_rate: number;
41
- runnable_top1_rate: number;
42
- public_total: number;
43
- public_passed: number;
44
- auth_total: number;
45
- auth_passed: number;
46
- avg_selection_ms: number;
47
- avg_time_to_correct_ms: number;
48
- avg_wrong_top1_before_correct: number;
49
- }
50
-
51
- export interface DependencyWalkCase {
52
- id: string;
53
- skill_id: string;
54
- intent: string;
55
- authenticated?: boolean;
56
- initial_params?: Record<string, unknown>;
57
- contextUrl?: string;
58
- target_operation_id: string;
59
- expected_path?: string[];
60
- }
61
-
62
- export interface DependencyWalkResult {
63
- id: string;
64
- skill_id: string;
65
- pass: boolean;
66
- target_operation_id: string;
67
- selected_path: string[];
68
- final_bindings: Record<string, unknown>;
69
- time_to_target_ms: number;
70
- wrong_before_target: number;
71
- failure_reason?: string;
72
- }
73
-
74
- export interface DependencyWalkSummary {
75
- total: number;
76
- passed: number;
77
- failed: number;
78
- hit_rate: number;
79
- avg_time_to_target_ms: number;
80
- avg_wrong_before_target: number;
81
- }
82
-
83
- export interface GeneratedCaseSet {
84
- selection_cases: LocalHarnessCase[];
85
- walk_cases: DependencyWalkCase[];
86
- }
87
-
88
- type SparseVector = Map<string, number>;
89
-
90
- function unique<T>(items: T[]): T[] {
91
- return [...new Set(items)];
92
- }
93
-
94
- function average(items: number[]): number {
95
- return items.length > 0 ? items.reduce((sum, value) => sum + value, 0) / items.length : 0;
96
- }
97
-
98
- function tokenize(text: string): string[] {
99
- return text
100
- .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
101
- .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
102
- .replace(/([a-zA-Z])(\d)/g, "$1 $2")
103
- .replace(/(\d)([a-zA-Z])/g, "$1 $2")
104
- .toLowerCase()
105
- .replace(/[^a-z0-9]+/g, " ")
106
- .split(/\s+/)
107
- .filter(Boolean);
108
- }
109
-
110
- function singularize(word: string): string {
111
- if (word.endsWith("ies") && word.length > 4) return `${word.slice(0, -3)}y`;
112
- if (word.endsWith("s") && !word.endsWith("ss") && word.length > 3) return word.slice(0, -1);
113
- return word;
114
- }
115
-
116
- function toVector(text: string): SparseVector {
117
- const vec = new Map<string, number>();
118
- for (const token of tokenize(text)) {
119
- const normalized = singularize(token);
120
- vec.set(normalized, (vec.get(normalized) ?? 0) + 1);
121
- }
122
- return vec;
123
- }
124
-
125
- function cosine(lhs: SparseVector, rhs: SparseVector): number {
126
- let dot = 0;
127
- let lhsNorm = 0;
128
- let rhsNorm = 0;
129
- for (const value of lhs.values()) lhsNorm += value * value;
130
- for (const value of rhs.values()) rhsNorm += value * value;
131
- for (const [key, value] of lhs) {
132
- const rhsValue = rhs.get(key) ?? 0;
133
- dot += value * rhsValue;
134
- }
135
- if (lhsNorm === 0 || rhsNorm === 0) return 0;
136
- return dot / (Math.sqrt(lhsNorm) * Math.sqrt(rhsNorm));
137
- }
138
-
139
- function operationText(skill: SkillManifest, operation: SkillOperationNode): string {
140
- return [
141
- skill.domain,
142
- operation.action_kind,
143
- operation.resource_kind,
144
- operation.description_in ?? "",
145
- operation.description_out ?? "",
146
- operation.response_summary ?? "",
147
- operation.url_template,
148
- ...(operation.example_fields ?? []),
149
- ...operation.requires.map((binding) => binding.key),
150
- ...operation.provides.map((binding) => binding.key),
151
- ...(operation.negative_tags ?? []),
152
- ].join(" ");
153
- }
154
-
155
- function queryText(intent: string, params?: Record<string, unknown>, contextUrl?: string): string {
156
- const bindings = knownBindingsFromInputs(params ?? {}, contextUrl);
157
- return [
158
- intent,
159
- ...Object.keys(bindings),
160
- ...Object.values(bindings).map((value) => String(value)),
161
- ].join(" ");
162
- }
163
-
164
- function pluralize(resource: string): string {
165
- if (resource.endsWith("y")) return `${resource.slice(0, -1)}ies`;
166
- if (resource.endsWith("s")) return resource;
167
- return `${resource}s`;
168
- }
169
-
170
- function intentForOperation(operation: SkillOperationNode): string {
171
- switch (operation.action_kind) {
172
- case "search":
173
- return `search ${pluralize(operation.resource_kind)}`;
174
- case "list":
175
- return `get ${pluralize(operation.resource_kind)}`;
176
- case "detail":
177
- return `get ${operation.resource_kind} details`;
178
- case "timeline":
179
- return `get ${operation.resource_kind} timeline`;
180
- case "trending":
181
- return `get trending ${pluralize(operation.resource_kind)}`;
182
- case "status":
183
- return `get ${operation.resource_kind} status`;
184
- default:
185
- return `${operation.action_kind} ${pluralize(operation.resource_kind)}`;
186
- }
187
- }
188
-
189
- function collectNeighborIds(chunk: SkillChunk, operationId: string): string[] {
190
- return unique(
191
- chunk.edges.flatMap((edge) => {
192
- if (edge.from_operation_id === operationId) return [edge.to_operation_id];
193
- if (edge.to_operation_id === operationId) return [edge.from_operation_id];
194
- return [];
195
- }),
196
- );
197
- }
198
-
199
- function objectParams(value: unknown): Record<string, unknown> {
200
- return value && typeof value === "object" && !Array.isArray(value) ? { ...(value as Record<string, unknown>) } : {};
201
- }
202
-
203
- function findPrimitiveByKey(value: unknown, key: string): unknown {
204
- if (value == null) return undefined;
205
- if (Array.isArray(value)) {
206
- for (const item of value) {
207
- const found = findPrimitiveByKey(item, key);
208
- if (found != null) return found;
209
- }
210
- return undefined;
211
- }
212
- if (typeof value !== "object") return undefined;
213
- for (const [entryKey, entryValue] of Object.entries(value as Record<string, unknown>)) {
214
- if (entryKey === key && (entryValue == null || typeof entryValue !== "object")) return entryValue;
215
- if (entryKey === key && Array.isArray(entryValue) && entryValue.length > 0 && typeof entryValue[0] !== "object") return entryValue[0];
216
- const found = findPrimitiveByKey(entryValue, key);
217
- if (found != null) return found;
218
- }
219
- return undefined;
220
- }
221
-
222
- function bindingsFromOperation(operation: SkillOperationNode): Record<string, unknown> {
223
- const out: Record<string, unknown> = {};
224
- const request = objectParams(operation.example_request);
225
- for (const binding of operation.provides) {
226
- if (binding.example_value != null && binding.example_value !== "") {
227
- out[binding.key] = binding.example_value;
228
- continue;
229
- }
230
- if (request[binding.key] != null && request[binding.key] !== "") {
231
- out[binding.key] = request[binding.key];
232
- continue;
233
- }
234
- const example = operation.example_response_compact;
235
- const direct = example ? findPrimitiveByKey(example, binding.key) : undefined;
236
- if (direct != null && direct !== "") {
237
- out[binding.key] = direct;
238
- continue;
239
- }
240
- if (binding.key.endsWith("_id") && example) {
241
- const fallback = findPrimitiveByKey(example, "id");
242
- if (fallback != null && fallback !== "") out[binding.key] = fallback;
243
- }
244
- }
245
- return out;
246
- }
247
-
248
- function findOperation(skill: SkillManifest, operationId: string): SkillOperationNode | undefined {
249
- const chunk = getSkillChunk(skill, { max_operations: skill.operation_graph?.operations.length ?? skill.endpoints.length });
250
- return chunk.operations.find((operation) => operation.operation_id === operationId);
251
- }
252
-
253
- function shortestDependencyPath(skill: SkillManifest, targetOperationId: string): string[] {
254
- const chunk = getSkillChunk(skill, { max_operations: skill.operation_graph?.operations.length ?? skill.endpoints.length });
255
- const entries = skill.operation_graph?.entry_operation_ids?.length
256
- ? skill.operation_graph.entry_operation_ids
257
- : chunk.operations
258
- .filter((operation) => operation.requires.length === 0 || operation.requires.every((binding) => !binding.required))
259
- .map((operation) => operation.operation_id);
260
- const queue = entries.map((operationId) => [operationId]);
261
- const seen = new Set(entries);
262
- while (queue.length > 0) {
263
- const path = queue.shift()!;
264
- const current = path[path.length - 1]!;
265
- if (current === targetOperationId) return path;
266
- for (const edge of chunk.edges) {
267
- if (edge.from_operation_id !== current) continue;
268
- if (seen.has(edge.to_operation_id)) continue;
269
- seen.add(edge.to_operation_id);
270
- queue.push([...path, edge.to_operation_id]);
271
- }
272
- }
273
- return [];
274
- }
275
-
276
- function canBuildCaseFromOperation(skill: SkillManifest, operation: SkillOperationNode): boolean {
277
- if ((operation.negative_tags ?? []).length > 0) return false;
278
- if (["fetch", "status"].includes(operation.action_kind)) return false;
279
- const request = objectParams(operation.example_request);
280
- return operation.requires.every((binding) => !binding.required || request[binding.key] != null || binding.example_value != null);
281
- }
282
-
283
- function buildCaseForOperation(skill: SkillManifest, operation: SkillOperationNode): LocalHarnessCase {
284
- const request = objectParams(operation.example_request);
285
- const chunk = getSkillChunk(skill, {
286
- intent: intentForOperation(operation),
287
- known_bindings: knownBindingsFromInputs(request, operation.trigger_url),
288
- max_operations: 8,
289
- });
290
- return {
291
- id: `${skill.skill_id}:${operation.operation_id}`,
292
- intent: intentForOperation(operation),
293
- params: request,
294
- contextUrl: operation.trigger_url,
295
- authenticated: operation.auth_required,
296
- expected_skill_id: skill.skill_id,
297
- expected_operation_id: operation.operation_id,
298
- expected_chunk_contains: collectNeighborIds(chunk, operation.operation_id),
299
- };
300
- }
301
-
302
- export function buildGeneratedCasesFromSkills(
303
- skills: SkillManifest[],
304
- opts?: { per_bucket?: number },
305
- ): GeneratedCaseSet {
306
- const perBucket = Math.max(1, opts?.per_bucket ?? 3);
307
- const publicSelections: LocalHarnessCase[] = [];
308
- const authSelections: LocalHarnessCase[] = [];
309
- const publicWalks: DependencyWalkCase[] = [];
310
- const authWalks: DependencyWalkCase[] = [];
311
-
312
- for (const skill of skills) {
313
- const chunk = getSkillChunk(skill, { max_operations: skill.operation_graph?.operations.length ?? skill.endpoints.length });
314
- for (const operation of chunk.operations) {
315
- if (!canBuildCaseFromOperation(skill, operation)) continue;
316
- const target = operation.auth_required ? authSelections : publicSelections;
317
- if (target.length >= perBucket) continue;
318
- target.push(buildCaseForOperation(skill, operation));
319
- }
320
-
321
- for (const operation of chunk.operations) {
322
- const path = shortestDependencyPath(skill, operation.operation_id);
323
- if (path.length < 2) continue;
324
- const pathOps = path.map((operationId) => chunk.operations.find((candidate) => candidate.operation_id === operationId)).filter(Boolean) as SkillOperationNode[];
325
- if (pathOps.length !== path.length) continue;
326
- const entry = pathOps[0]!;
327
- if (!canBuildCaseFromOperation(skill, entry)) continue;
328
- const target = operation.auth_required ? authWalks : publicWalks;
329
- if (target.length >= perBucket) continue;
330
- target.push({
331
- id: `${skill.skill_id}:walk:${operation.operation_id}`,
332
- skill_id: skill.skill_id,
333
- intent: intentForOperation(operation),
334
- authenticated: operation.auth_required,
335
- initial_params: objectParams(entry.example_request),
336
- contextUrl: entry.trigger_url,
337
- target_operation_id: operation.operation_id,
338
- });
339
- }
340
-
341
- for (const edge of chunk.edges) {
342
- const source = chunk.operations.find((operation) => operation.operation_id === edge.from_operation_id);
343
- const target = chunk.operations.find((operation) => operation.operation_id === edge.to_operation_id);
344
- if (!source || !target) continue;
345
- if (!canBuildCaseFromOperation(skill, source)) continue;
346
- const bucket = target.auth_required ? authWalks : publicWalks;
347
- if (bucket.length >= perBucket) continue;
348
- bucket.push({
349
- id: `${skill.skill_id}:edge:${target.operation_id}`,
350
- skill_id: skill.skill_id,
351
- intent: intentForOperation(target),
352
- authenticated: target.auth_required,
353
- initial_params: objectParams(source.example_request),
354
- contextUrl: source.trigger_url,
355
- target_operation_id: target.operation_id,
356
- });
357
- }
358
- }
359
-
360
- return {
361
- selection_cases: [...publicSelections, ...authSelections],
362
- walk_cases: [...publicWalks, ...authWalks],
363
- };
364
- }
365
-
366
- function scoreOperation(
367
- skill: SkillManifest,
368
- operation: SkillOperationNode,
369
- intent: string,
370
- params?: Record<string, unknown>,
371
- contextUrl?: string,
372
- ): number {
373
- const queryVec = toVector(queryText(intent, params, contextUrl));
374
- const opVec = toVector(operationText(skill, operation));
375
- const bindings = knownBindingsFromInputs(params ?? {}, contextUrl);
376
- let score = cosine(queryVec, opVec) * 100;
377
-
378
- const runnable = operation.requires.every((binding) => !binding.required || (bindings[binding.key] != null && bindings[binding.key] !== ""));
379
- if (runnable) score += 18;
380
- const satisfiedRequired = operation.requires.filter((binding) => binding.required && bindings[binding.key] != null && bindings[binding.key] !== "").length;
381
- if (satisfiedRequired > 0) score += satisfiedRequired * 14;
382
-
383
- const intentTokens = new Set(tokenize(intent).map(singularize));
384
- if (intentTokens.has(operation.action_kind)) score += 8;
385
- if (intentTokens.has(operation.resource_kind)) score += 12;
386
-
387
- const negativePenalty = (operation.negative_tags ?? []).reduce((sum, tag) => {
388
- if (intentTokens.has("status") && tag === "status") return sum;
389
- return sum + 12;
390
- }, 0);
391
- score -= negativePenalty;
392
-
393
- const requiresSatisfied = operation.requires.filter((binding) => binding.required).every((binding) => bindings[binding.key] != null && bindings[binding.key] !== "");
394
- if (operation.action_kind === "detail" && !requiresSatisfied) score -= 10;
395
- if (Object.keys(bindings).length > 0 && operation.requires.length === 0) {
396
- const redundantBootstrap = operation.provides.some((binding) => bindings[binding.key] != null && bindings[binding.key] !== "");
397
- if (redundantBootstrap) score -= 15;
398
- }
399
- score -= operationSoftPenalty(operation, intent);
400
-
401
- return score;
402
- }
403
-
404
- export function loadSkillsFromLocalCache(cacheDir = join(homedir(), ".unbrowse", "skill-cache")): SkillManifest[] {
405
- if (!existsSync(cacheDir)) return [];
406
- return readdirSync(cacheDir)
407
- .filter((file) => file.endsWith(".json"))
408
- .flatMap((file) => {
409
- try {
410
- return [JSON.parse(readFileSync(join(cacheDir, file), "utf-8")) as SkillManifest];
411
- } catch {
412
- return [];
413
- }
414
- });
415
- }
416
-
417
- export function selectOperationsLocally(
418
- skills: SkillManifest[],
419
- intent: string,
420
- opts?: {
421
- params?: Record<string, unknown>;
422
- contextUrl?: string;
423
- topK?: number;
424
- domain?: string;
425
- authenticated?: boolean;
426
- },
427
- ): HarnessOperationHit[] {
428
- const scored: HarnessOperationHit[] = [];
429
- for (const skill of skills) {
430
- if (opts?.domain && skill.domain !== opts.domain) continue;
431
- const chunk = getSkillChunk(skill, {
432
- intent,
433
- known_bindings: knownBindingsFromInputs(opts?.params ?? {}, opts?.contextUrl),
434
- max_operations: 8,
435
- });
436
- const allowed = new Set(chunk.operations.map((operation) => operation.operation_id));
437
- for (const operation of chunk.operations) {
438
- if (!allowed.has(operation.operation_id)) continue;
439
- if (isOperationHardExcluded(operation, intent)) continue;
440
- const authRequired = operation.auth_required ?? false;
441
- if (authRequired && !opts?.authenticated) continue;
442
- const runnable = chunk.available_operation_ids.includes(operation.operation_id);
443
- scored.push({
444
- skill_id: skill.skill_id,
445
- skill_domain: skill.domain,
446
- operation_id: operation.operation_id,
447
- endpoint_id: operation.endpoint_id,
448
- score: scoreOperation(skill, operation, intent, opts?.params, opts?.contextUrl),
449
- runnable,
450
- action_kind: operation.action_kind,
451
- resource_kind: operation.resource_kind,
452
- negative_tags: operation.negative_tags ?? [],
453
- auth_required: authRequired,
454
- });
455
- }
456
- }
457
- return scored.sort((a, b) => b.score - a.score).slice(0, opts?.topK ?? 5);
458
- }
459
-
460
- export function evaluateLocalHarness(
461
- skills: SkillManifest[],
462
- cases: LocalHarnessCase[],
463
- ): { summary: HarnessSummary; results: HarnessCaseResult[] } {
464
- const results: HarnessCaseResult[] = cases.map((testCase) => {
465
- const started = performance.now();
466
- const hits = selectOperationsLocally(skills, testCase.intent, {
467
- params: testCase.params,
468
- contextUrl: testCase.contextUrl,
469
- authenticated: testCase.authenticated,
470
- topK: 5,
471
- });
472
- const selectionMs = performance.now() - started;
473
- const selected = hits[0];
474
- const skill = skills.find((candidate) => candidate.skill_id === (selected?.skill_id ?? testCase.expected_skill_id))
475
- ?? skills.find((candidate) => candidate.skill_id === testCase.expected_skill_id)!;
476
- const chunk = getSkillChunk(skill, {
477
- intent: testCase.intent,
478
- known_bindings: knownBindingsFromInputs(testCase.params ?? {}, testCase.contextUrl),
479
- max_operations: 8,
480
- });
481
-
482
- const pass = !!selected &&
483
- selected.skill_id === testCase.expected_skill_id &&
484
- selected.operation_id === testCase.expected_operation_id &&
485
- (testCase.expected_chunk_contains ?? []).every((operationId) => chunk.operations.some((operation) => operation.operation_id === operationId));
486
- const expectedRank = hits.findIndex((hit) =>
487
- hit.skill_id === testCase.expected_skill_id && hit.operation_id === testCase.expected_operation_id);
488
- const wrongTop1BeforeCorrect = expectedRank >= 0 ? expectedRank : hits.length;
489
-
490
- return {
491
- id: testCase.id,
492
- pass,
493
- selected,
494
- top_hits: hits,
495
- chunk,
496
- expected_skill_id: testCase.expected_skill_id,
497
- expected_operation_id: testCase.expected_operation_id,
498
- selection_ms: selectionMs,
499
- expected_rank: expectedRank,
500
- wrong_top1_before_correct: wrongTop1BeforeCorrect,
501
- failure_reason: pass
502
- ? undefined
503
- : selected
504
- ? `selected ${selected.skill_id}:${selected.operation_id}`
505
- : "no_selection",
506
- };
507
- });
508
-
509
- const passed = results.filter((result) => result.pass).length;
510
- const runnableTop1 = results.filter((result) => result.selected?.runnable).length;
511
- const authResults = results.filter((result) => cases.find((testCase) => testCase.id === result.id)?.authenticated);
512
- const publicResults = results.filter((result) => !cases.find((testCase) => testCase.id === result.id)?.authenticated);
513
- const selectionTimings = results.map((result) => result.selection_ms);
514
- const correctnessTimings = results
515
- .filter((result) => result.expected_rank >= 0)
516
- .map((result) => result.selection_ms);
517
- return {
518
- summary: {
519
- total: results.length,
520
- passed,
521
- failed: results.length - passed,
522
- hit_rate: results.length > 0 ? passed / results.length : 0,
523
- runnable_top1_rate: results.length > 0 ? runnableTop1 / results.length : 0,
524
- public_total: publicResults.length,
525
- public_passed: publicResults.filter((result) => result.pass).length,
526
- auth_total: authResults.length,
527
- auth_passed: authResults.filter((result) => result.pass).length,
528
- avg_selection_ms: average(selectionTimings),
529
- avg_time_to_correct_ms: average(correctnessTimings),
530
- avg_wrong_top1_before_correct: average(results.map((result) => result.wrong_top1_before_correct)),
531
- },
532
- results,
533
- };
534
- }
535
-
536
- export function evaluateDependencyWalks(
537
- skills: SkillManifest[],
538
- cases: DependencyWalkCase[],
539
- ): { summary: DependencyWalkSummary; results: DependencyWalkResult[] } {
540
- const results = cases.map((testCase) => {
541
- const skill = skills.find((candidate) => candidate.skill_id === testCase.skill_id);
542
- if (!skill) {
543
- return {
544
- id: testCase.id,
545
- skill_id: testCase.skill_id,
546
- pass: false,
547
- target_operation_id: testCase.target_operation_id,
548
- selected_path: [],
549
- final_bindings: {},
550
- time_to_target_ms: 0,
551
- wrong_before_target: 0,
552
- failure_reason: "missing_skill",
553
- } satisfies DependencyWalkResult;
554
- }
555
-
556
- const selectedPath: string[] = [];
557
- const visited = new Set<string>();
558
- const bindings = knownBindingsFromInputs(testCase.initial_params ?? {}, testCase.contextUrl);
559
- const started = performance.now();
560
- let failureReason = "max_steps";
561
-
562
- for (let step = 0; step < 6; step++) {
563
- const hits = selectOperationsLocally([skill], testCase.intent, {
564
- params: bindings,
565
- contextUrl: testCase.contextUrl,
566
- authenticated: testCase.authenticated,
567
- topK: 6,
568
- });
569
- const next = hits.find((hit) => hit.runnable && !visited.has(hit.operation_id))
570
- ?? hits.find((hit) => !visited.has(hit.operation_id));
571
- if (!next) {
572
- failureReason = "no_candidate";
573
- break;
574
- }
575
-
576
- visited.add(next.operation_id);
577
- selectedPath.push(next.operation_id);
578
- const operation = findOperation(skill, next.operation_id);
579
- if (operation) Object.assign(bindings, bindingsFromOperation(operation));
580
- if (next.operation_id === testCase.target_operation_id) {
581
- failureReason = "";
582
- break;
583
- }
584
- }
585
-
586
- const timeToTargetMs = performance.now() - started;
587
- const pass = selectedPath[selectedPath.length - 1] === testCase.target_operation_id &&
588
- (!testCase.expected_path || testCase.expected_path.every((operationId, index) => selectedPath[index] === operationId));
589
- return {
590
- id: testCase.id,
591
- skill_id: testCase.skill_id,
592
- pass,
593
- target_operation_id: testCase.target_operation_id,
594
- selected_path: selectedPath,
595
- final_bindings: bindings,
596
- time_to_target_ms: timeToTargetMs,
597
- wrong_before_target: Math.max(0, selectedPath.length - 1),
598
- failure_reason: pass ? undefined : failureReason || "wrong_path",
599
- } satisfies DependencyWalkResult;
600
- });
601
-
602
- const passed = results.filter((result) => result.pass).length;
603
- return {
604
- summary: {
605
- total: results.length,
606
- passed,
607
- failed: results.length - passed,
608
- hit_rate: results.length > 0 ? passed / results.length : 0,
609
- avg_time_to_target_ms: average(results.map((result) => result.time_to_target_ms)),
610
- avg_wrong_before_target: average(results.map((result) => result.wrong_before_target)),
611
- },
612
- results,
613
- };
614
- }
615
-
616
- export function formatHarnessReport(summary: HarnessSummary, results: HarnessCaseResult[]): string {
617
- const lines = [
618
- `Local graph harness`,
619
- `total=${summary.total} passed=${summary.passed} failed=${summary.failed} hit_rate=${Math.round(summary.hit_rate * 100)}% runnable_top1=${Math.round(summary.runnable_top1_rate * 100)}% public=${summary.public_passed}/${summary.public_total} auth=${summary.auth_passed}/${summary.auth_total} avg_select_ms=${summary.avg_selection_ms.toFixed(2)} avg_correct_ms=${summary.avg_time_to_correct_ms.toFixed(2)} avg_wrong_before_correct=${summary.avg_wrong_top1_before_correct.toFixed(2)}`,
620
- "",
621
- ];
622
- for (const result of results) {
623
- const head = result.pass ? "PASS" : "FAIL";
624
- const selected = result.selected ? `${result.selected.skill_id}:${result.selected.operation_id}` : "none";
625
- lines.push(`${head} ${result.id} -> ${selected}`);
626
- if (!result.pass) {
627
- lines.push(` expected ${result.expected_skill_id}:${result.expected_operation_id}`);
628
- lines.push(` chunk ${unique(result.chunk.operations.map((operation) => operation.operation_id)).join(", ")}`);
629
- }
630
- }
631
- return lines.join("\n");
632
- }
633
-
634
- export function formatDependencyWalkReport(summary: DependencyWalkSummary, results: DependencyWalkResult[]): string {
635
- const lines = [
636
- `Dependency walk harness`,
637
- `total=${summary.total} passed=${summary.passed} failed=${summary.failed} hit_rate=${Math.round(summary.hit_rate * 100)}% avg_target_ms=${summary.avg_time_to_target_ms.toFixed(2)} avg_wrong_before_target=${summary.avg_wrong_before_target.toFixed(2)}`,
638
- "",
639
- ];
640
- for (const result of results) {
641
- const head = result.pass ? "PASS" : "FAIL";
642
- lines.push(`${head} ${result.id} -> ${result.selected_path.join(" -> ") || "none"}`);
643
- if (!result.pass) lines.push(` target ${result.target_operation_id} reason=${result.failure_reason ?? "unknown"}`);
644
- }
645
- return lines.join("\n");
646
- }