unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +79 -33
- package/dist/index.js +2 -6
- package/dist/mcp.js +73 -22
- package/dist/server.js +25994 -0
- package/package.json +1 -2
- package/vendor/kuri/manifest.json +1 -1
- package/runtime-src/agent-outcome.ts +0 -166
- package/runtime-src/analytics-session.ts +0 -55
- package/runtime-src/api/browse-index.ts +0 -343
- package/runtime-src/api/browse-session.ts +0 -572
- package/runtime-src/api/browse-submit-prereqs.ts +0 -48
- package/runtime-src/api/browse-submit.ts +0 -1184
- package/runtime-src/api/routes.ts +0 -1833
- package/runtime-src/auth/browser-cookies.ts +0 -423
- package/runtime-src/auth/index.ts +0 -535
- package/runtime-src/auth/runtime.ts +0 -116
- package/runtime-src/browser/index.ts +0 -659
- package/runtime-src/browser/types.ts +0 -41
- package/runtime-src/build-info.generated.ts +0 -6
- package/runtime-src/capture/index.ts +0 -1929
- package/runtime-src/capture/prefetch.ts +0 -95
- package/runtime-src/capture/rsc.ts +0 -45
- package/runtime-src/cli/shortcuts.ts +0 -273
- package/runtime-src/cli.ts +0 -1734
- package/runtime-src/client/graph-client.ts +0 -100
- package/runtime-src/client/index.ts +0 -1425
- package/runtime-src/debug-trace.ts +0 -18
- package/runtime-src/domain.ts +0 -38
- package/runtime-src/execution/index.ts +0 -3407
- package/runtime-src/execution/retry.ts +0 -46
- package/runtime-src/execution/robots.ts +0 -167
- package/runtime-src/execution/search-forms.ts +0 -188
- package/runtime-src/execution/token-resolver.ts +0 -135
- package/runtime-src/extraction/index.ts +0 -1507
- package/runtime-src/foundry/publish-bundle.ts +0 -392
- package/runtime-src/graph/agent-augment.ts +0 -315
- package/runtime-src/graph/index.ts +0 -1532
- package/runtime-src/graph/local-fixtures.ts +0 -393
- package/runtime-src/graph/local-harness.ts +0 -646
- package/runtime-src/graph/planner.ts +0 -411
- package/runtime-src/graph/session.ts +0 -294
- package/runtime-src/graph/trace-store.ts +0 -136
- package/runtime-src/impact-log.ts +0 -227
- package/runtime-src/index.ts +0 -24
- package/runtime-src/indexer/index.ts +0 -465
- package/runtime-src/intent-match.ts +0 -1515
- package/runtime-src/kuri/client.ts +0 -1839
- package/runtime-src/logger.ts +0 -30
- package/runtime-src/marketplace/index.ts +0 -111
- package/runtime-src/mcp.ts +0 -1911
- package/runtime-src/orchestrator/browser-agent.ts +0 -374
- package/runtime-src/orchestrator/dag-advisor.ts +0 -59
- package/runtime-src/orchestrator/dag-feedback.ts +0 -257
- package/runtime-src/orchestrator/first-pass-action.ts +0 -403
- package/runtime-src/orchestrator/index.ts +0 -4480
- package/runtime-src/orchestrator/passive-publish.ts +0 -187
- package/runtime-src/orchestrator/timing-economics.ts +0 -80
- package/runtime-src/payments/cascade.ts +0 -137
- package/runtime-src/payments/index.ts +0 -270
- package/runtime-src/payments/lobster-pay.ts +0 -182
- package/runtime-src/payments/wallet.ts +0 -98
- package/runtime-src/publish/review-context.ts +0 -93
- package/runtime-src/publish/sanitize.ts +0 -197
- package/runtime-src/publish/schema-review.ts +0 -192
- package/runtime-src/publish-admission.ts +0 -388
- package/runtime-src/ratelimit/index.ts +0 -23
- package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
- package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
- package/runtime-src/reverse-engineer/index.ts +0 -1551
- package/runtime-src/reverse-engineer/token-sources.ts +0 -379
- package/runtime-src/router.ts +0 -17
- package/runtime-src/routing-telemetry.ts +0 -395
- package/runtime-src/runtime/browser-access.ts +0 -11
- package/runtime-src/runtime/browser-auth.ts +0 -12
- package/runtime-src/runtime/browser-host.ts +0 -48
- package/runtime-src/runtime/lifecycle.ts +0 -17
- package/runtime-src/runtime/local-server.ts +0 -311
- package/runtime-src/runtime/paths.ts +0 -99
- package/runtime-src/runtime/setup.ts +0 -251
- package/runtime-src/runtime/supervisor.ts +0 -69
- package/runtime-src/runtime/update-hints.ts +0 -351
- package/runtime-src/server.ts +0 -100
- package/runtime-src/session-logs.ts +0 -142
- package/runtime-src/settings.ts +0 -221
- package/runtime-src/single-binary.ts +0 -143
- package/runtime-src/site-policy.ts +0 -54
- package/runtime-src/stale-cleanup-runner.ts +0 -144
- package/runtime-src/stale-cleanup.ts +0 -133
- package/runtime-src/telemetry-attribution.ts +0 -120
- package/runtime-src/telemetry.ts +0 -253
- package/runtime-src/template-params.ts +0 -141
- package/runtime-src/transform/drift.ts +0 -60
- package/runtime-src/transform/index.ts +0 -277
- package/runtime-src/types/index.ts +0 -1
- package/runtime-src/types/skill.ts +0 -931
- package/runtime-src/vault/index.ts +0 -196
- package/runtime-src/verification/auth-gate.ts +0 -8
- package/runtime-src/verification/candidates.ts +0 -27
- package/runtime-src/verification/index.ts +0 -120
- package/runtime-src/verification/matrix.ts +0 -30
- package/runtime-src/version.ts +0 -148
- package/runtime-src/workflow/artifact.ts +0 -161
- package/runtime-src/workflow/compile.ts +0 -808
- package/runtime-src/workflow/publish.ts +0 -225
- package/runtime-src/workflow/runtime.ts +0 -213
|
@@ -1,646 +0,0 @@
|
|
|
1
|
-
import { existsSync, readdirSync, readFileSync } from "fs";
|
|
2
|
-
import { homedir } from "os";
|
|
3
|
-
import { performance } from "perf_hooks";
|
|
4
|
-
import { join } from "path";
|
|
5
|
-
import { getSkillChunk, isOperationHardExcluded, knownBindingsFromInputs, operationSoftPenalty } from "./index.js";
|
|
6
|
-
import type { SkillChunk, SkillManifest, SkillOperationNode } from "../types/index.js";
|
|
7
|
-
import type { LocalHarnessCase } from "./local-fixtures.js";
|
|
8
|
-
|
|
9
|
-
export interface HarnessOperationHit {
|
|
10
|
-
skill_id: string;
|
|
11
|
-
skill_domain: string;
|
|
12
|
-
operation_id: string;
|
|
13
|
-
endpoint_id: string;
|
|
14
|
-
score: number;
|
|
15
|
-
runnable: boolean;
|
|
16
|
-
action_kind: string;
|
|
17
|
-
resource_kind: string;
|
|
18
|
-
negative_tags: string[];
|
|
19
|
-
auth_required: boolean;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export interface HarnessCaseResult {
|
|
23
|
-
id: string;
|
|
24
|
-
pass: boolean;
|
|
25
|
-
selected?: HarnessOperationHit;
|
|
26
|
-
top_hits: HarnessOperationHit[];
|
|
27
|
-
chunk: SkillChunk;
|
|
28
|
-
expected_skill_id: string;
|
|
29
|
-
expected_operation_id: string;
|
|
30
|
-
selection_ms: number;
|
|
31
|
-
expected_rank: number;
|
|
32
|
-
wrong_top1_before_correct: number;
|
|
33
|
-
failure_reason?: string;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
export interface HarnessSummary {
|
|
37
|
-
total: number;
|
|
38
|
-
passed: number;
|
|
39
|
-
failed: number;
|
|
40
|
-
hit_rate: number;
|
|
41
|
-
runnable_top1_rate: number;
|
|
42
|
-
public_total: number;
|
|
43
|
-
public_passed: number;
|
|
44
|
-
auth_total: number;
|
|
45
|
-
auth_passed: number;
|
|
46
|
-
avg_selection_ms: number;
|
|
47
|
-
avg_time_to_correct_ms: number;
|
|
48
|
-
avg_wrong_top1_before_correct: number;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
export interface DependencyWalkCase {
|
|
52
|
-
id: string;
|
|
53
|
-
skill_id: string;
|
|
54
|
-
intent: string;
|
|
55
|
-
authenticated?: boolean;
|
|
56
|
-
initial_params?: Record<string, unknown>;
|
|
57
|
-
contextUrl?: string;
|
|
58
|
-
target_operation_id: string;
|
|
59
|
-
expected_path?: string[];
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
export interface DependencyWalkResult {
|
|
63
|
-
id: string;
|
|
64
|
-
skill_id: string;
|
|
65
|
-
pass: boolean;
|
|
66
|
-
target_operation_id: string;
|
|
67
|
-
selected_path: string[];
|
|
68
|
-
final_bindings: Record<string, unknown>;
|
|
69
|
-
time_to_target_ms: number;
|
|
70
|
-
wrong_before_target: number;
|
|
71
|
-
failure_reason?: string;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
export interface DependencyWalkSummary {
|
|
75
|
-
total: number;
|
|
76
|
-
passed: number;
|
|
77
|
-
failed: number;
|
|
78
|
-
hit_rate: number;
|
|
79
|
-
avg_time_to_target_ms: number;
|
|
80
|
-
avg_wrong_before_target: number;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export interface GeneratedCaseSet {
|
|
84
|
-
selection_cases: LocalHarnessCase[];
|
|
85
|
-
walk_cases: DependencyWalkCase[];
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
type SparseVector = Map<string, number>;
|
|
89
|
-
|
|
90
|
-
function unique<T>(items: T[]): T[] {
|
|
91
|
-
return [...new Set(items)];
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
function average(items: number[]): number {
|
|
95
|
-
return items.length > 0 ? items.reduce((sum, value) => sum + value, 0) / items.length : 0;
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
function tokenize(text: string): string[] {
|
|
99
|
-
return text
|
|
100
|
-
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
101
|
-
.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
|
|
102
|
-
.replace(/([a-zA-Z])(\d)/g, "$1 $2")
|
|
103
|
-
.replace(/(\d)([a-zA-Z])/g, "$1 $2")
|
|
104
|
-
.toLowerCase()
|
|
105
|
-
.replace(/[^a-z0-9]+/g, " ")
|
|
106
|
-
.split(/\s+/)
|
|
107
|
-
.filter(Boolean);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
function singularize(word: string): string {
|
|
111
|
-
if (word.endsWith("ies") && word.length > 4) return `${word.slice(0, -3)}y`;
|
|
112
|
-
if (word.endsWith("s") && !word.endsWith("ss") && word.length > 3) return word.slice(0, -1);
|
|
113
|
-
return word;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
function toVector(text: string): SparseVector {
|
|
117
|
-
const vec = new Map<string, number>();
|
|
118
|
-
for (const token of tokenize(text)) {
|
|
119
|
-
const normalized = singularize(token);
|
|
120
|
-
vec.set(normalized, (vec.get(normalized) ?? 0) + 1);
|
|
121
|
-
}
|
|
122
|
-
return vec;
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
function cosine(lhs: SparseVector, rhs: SparseVector): number {
|
|
126
|
-
let dot = 0;
|
|
127
|
-
let lhsNorm = 0;
|
|
128
|
-
let rhsNorm = 0;
|
|
129
|
-
for (const value of lhs.values()) lhsNorm += value * value;
|
|
130
|
-
for (const value of rhs.values()) rhsNorm += value * value;
|
|
131
|
-
for (const [key, value] of lhs) {
|
|
132
|
-
const rhsValue = rhs.get(key) ?? 0;
|
|
133
|
-
dot += value * rhsValue;
|
|
134
|
-
}
|
|
135
|
-
if (lhsNorm === 0 || rhsNorm === 0) return 0;
|
|
136
|
-
return dot / (Math.sqrt(lhsNorm) * Math.sqrt(rhsNorm));
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
function operationText(skill: SkillManifest, operation: SkillOperationNode): string {
|
|
140
|
-
return [
|
|
141
|
-
skill.domain,
|
|
142
|
-
operation.action_kind,
|
|
143
|
-
operation.resource_kind,
|
|
144
|
-
operation.description_in ?? "",
|
|
145
|
-
operation.description_out ?? "",
|
|
146
|
-
operation.response_summary ?? "",
|
|
147
|
-
operation.url_template,
|
|
148
|
-
...(operation.example_fields ?? []),
|
|
149
|
-
...operation.requires.map((binding) => binding.key),
|
|
150
|
-
...operation.provides.map((binding) => binding.key),
|
|
151
|
-
...(operation.negative_tags ?? []),
|
|
152
|
-
].join(" ");
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
function queryText(intent: string, params?: Record<string, unknown>, contextUrl?: string): string {
|
|
156
|
-
const bindings = knownBindingsFromInputs(params ?? {}, contextUrl);
|
|
157
|
-
return [
|
|
158
|
-
intent,
|
|
159
|
-
...Object.keys(bindings),
|
|
160
|
-
...Object.values(bindings).map((value) => String(value)),
|
|
161
|
-
].join(" ");
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
function pluralize(resource: string): string {
|
|
165
|
-
if (resource.endsWith("y")) return `${resource.slice(0, -1)}ies`;
|
|
166
|
-
if (resource.endsWith("s")) return resource;
|
|
167
|
-
return `${resource}s`;
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
function intentForOperation(operation: SkillOperationNode): string {
|
|
171
|
-
switch (operation.action_kind) {
|
|
172
|
-
case "search":
|
|
173
|
-
return `search ${pluralize(operation.resource_kind)}`;
|
|
174
|
-
case "list":
|
|
175
|
-
return `get ${pluralize(operation.resource_kind)}`;
|
|
176
|
-
case "detail":
|
|
177
|
-
return `get ${operation.resource_kind} details`;
|
|
178
|
-
case "timeline":
|
|
179
|
-
return `get ${operation.resource_kind} timeline`;
|
|
180
|
-
case "trending":
|
|
181
|
-
return `get trending ${pluralize(operation.resource_kind)}`;
|
|
182
|
-
case "status":
|
|
183
|
-
return `get ${operation.resource_kind} status`;
|
|
184
|
-
default:
|
|
185
|
-
return `${operation.action_kind} ${pluralize(operation.resource_kind)}`;
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
function collectNeighborIds(chunk: SkillChunk, operationId: string): string[] {
|
|
190
|
-
return unique(
|
|
191
|
-
chunk.edges.flatMap((edge) => {
|
|
192
|
-
if (edge.from_operation_id === operationId) return [edge.to_operation_id];
|
|
193
|
-
if (edge.to_operation_id === operationId) return [edge.from_operation_id];
|
|
194
|
-
return [];
|
|
195
|
-
}),
|
|
196
|
-
);
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
function objectParams(value: unknown): Record<string, unknown> {
|
|
200
|
-
return value && typeof value === "object" && !Array.isArray(value) ? { ...(value as Record<string, unknown>) } : {};
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
function findPrimitiveByKey(value: unknown, key: string): unknown {
|
|
204
|
-
if (value == null) return undefined;
|
|
205
|
-
if (Array.isArray(value)) {
|
|
206
|
-
for (const item of value) {
|
|
207
|
-
const found = findPrimitiveByKey(item, key);
|
|
208
|
-
if (found != null) return found;
|
|
209
|
-
}
|
|
210
|
-
return undefined;
|
|
211
|
-
}
|
|
212
|
-
if (typeof value !== "object") return undefined;
|
|
213
|
-
for (const [entryKey, entryValue] of Object.entries(value as Record<string, unknown>)) {
|
|
214
|
-
if (entryKey === key && (entryValue == null || typeof entryValue !== "object")) return entryValue;
|
|
215
|
-
if (entryKey === key && Array.isArray(entryValue) && entryValue.length > 0 && typeof entryValue[0] !== "object") return entryValue[0];
|
|
216
|
-
const found = findPrimitiveByKey(entryValue, key);
|
|
217
|
-
if (found != null) return found;
|
|
218
|
-
}
|
|
219
|
-
return undefined;
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
function bindingsFromOperation(operation: SkillOperationNode): Record<string, unknown> {
|
|
223
|
-
const out: Record<string, unknown> = {};
|
|
224
|
-
const request = objectParams(operation.example_request);
|
|
225
|
-
for (const binding of operation.provides) {
|
|
226
|
-
if (binding.example_value != null && binding.example_value !== "") {
|
|
227
|
-
out[binding.key] = binding.example_value;
|
|
228
|
-
continue;
|
|
229
|
-
}
|
|
230
|
-
if (request[binding.key] != null && request[binding.key] !== "") {
|
|
231
|
-
out[binding.key] = request[binding.key];
|
|
232
|
-
continue;
|
|
233
|
-
}
|
|
234
|
-
const example = operation.example_response_compact;
|
|
235
|
-
const direct = example ? findPrimitiveByKey(example, binding.key) : undefined;
|
|
236
|
-
if (direct != null && direct !== "") {
|
|
237
|
-
out[binding.key] = direct;
|
|
238
|
-
continue;
|
|
239
|
-
}
|
|
240
|
-
if (binding.key.endsWith("_id") && example) {
|
|
241
|
-
const fallback = findPrimitiveByKey(example, "id");
|
|
242
|
-
if (fallback != null && fallback !== "") out[binding.key] = fallback;
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
return out;
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
function findOperation(skill: SkillManifest, operationId: string): SkillOperationNode | undefined {
|
|
249
|
-
const chunk = getSkillChunk(skill, { max_operations: skill.operation_graph?.operations.length ?? skill.endpoints.length });
|
|
250
|
-
return chunk.operations.find((operation) => operation.operation_id === operationId);
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
function shortestDependencyPath(skill: SkillManifest, targetOperationId: string): string[] {
|
|
254
|
-
const chunk = getSkillChunk(skill, { max_operations: skill.operation_graph?.operations.length ?? skill.endpoints.length });
|
|
255
|
-
const entries = skill.operation_graph?.entry_operation_ids?.length
|
|
256
|
-
? skill.operation_graph.entry_operation_ids
|
|
257
|
-
: chunk.operations
|
|
258
|
-
.filter((operation) => operation.requires.length === 0 || operation.requires.every((binding) => !binding.required))
|
|
259
|
-
.map((operation) => operation.operation_id);
|
|
260
|
-
const queue = entries.map((operationId) => [operationId]);
|
|
261
|
-
const seen = new Set(entries);
|
|
262
|
-
while (queue.length > 0) {
|
|
263
|
-
const path = queue.shift()!;
|
|
264
|
-
const current = path[path.length - 1]!;
|
|
265
|
-
if (current === targetOperationId) return path;
|
|
266
|
-
for (const edge of chunk.edges) {
|
|
267
|
-
if (edge.from_operation_id !== current) continue;
|
|
268
|
-
if (seen.has(edge.to_operation_id)) continue;
|
|
269
|
-
seen.add(edge.to_operation_id);
|
|
270
|
-
queue.push([...path, edge.to_operation_id]);
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
return [];
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
function canBuildCaseFromOperation(skill: SkillManifest, operation: SkillOperationNode): boolean {
|
|
277
|
-
if ((operation.negative_tags ?? []).length > 0) return false;
|
|
278
|
-
if (["fetch", "status"].includes(operation.action_kind)) return false;
|
|
279
|
-
const request = objectParams(operation.example_request);
|
|
280
|
-
return operation.requires.every((binding) => !binding.required || request[binding.key] != null || binding.example_value != null);
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
function buildCaseForOperation(skill: SkillManifest, operation: SkillOperationNode): LocalHarnessCase {
|
|
284
|
-
const request = objectParams(operation.example_request);
|
|
285
|
-
const chunk = getSkillChunk(skill, {
|
|
286
|
-
intent: intentForOperation(operation),
|
|
287
|
-
known_bindings: knownBindingsFromInputs(request, operation.trigger_url),
|
|
288
|
-
max_operations: 8,
|
|
289
|
-
});
|
|
290
|
-
return {
|
|
291
|
-
id: `${skill.skill_id}:${operation.operation_id}`,
|
|
292
|
-
intent: intentForOperation(operation),
|
|
293
|
-
params: request,
|
|
294
|
-
contextUrl: operation.trigger_url,
|
|
295
|
-
authenticated: operation.auth_required,
|
|
296
|
-
expected_skill_id: skill.skill_id,
|
|
297
|
-
expected_operation_id: operation.operation_id,
|
|
298
|
-
expected_chunk_contains: collectNeighborIds(chunk, operation.operation_id),
|
|
299
|
-
};
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
export function buildGeneratedCasesFromSkills(
|
|
303
|
-
skills: SkillManifest[],
|
|
304
|
-
opts?: { per_bucket?: number },
|
|
305
|
-
): GeneratedCaseSet {
|
|
306
|
-
const perBucket = Math.max(1, opts?.per_bucket ?? 3);
|
|
307
|
-
const publicSelections: LocalHarnessCase[] = [];
|
|
308
|
-
const authSelections: LocalHarnessCase[] = [];
|
|
309
|
-
const publicWalks: DependencyWalkCase[] = [];
|
|
310
|
-
const authWalks: DependencyWalkCase[] = [];
|
|
311
|
-
|
|
312
|
-
for (const skill of skills) {
|
|
313
|
-
const chunk = getSkillChunk(skill, { max_operations: skill.operation_graph?.operations.length ?? skill.endpoints.length });
|
|
314
|
-
for (const operation of chunk.operations) {
|
|
315
|
-
if (!canBuildCaseFromOperation(skill, operation)) continue;
|
|
316
|
-
const target = operation.auth_required ? authSelections : publicSelections;
|
|
317
|
-
if (target.length >= perBucket) continue;
|
|
318
|
-
target.push(buildCaseForOperation(skill, operation));
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
for (const operation of chunk.operations) {
|
|
322
|
-
const path = shortestDependencyPath(skill, operation.operation_id);
|
|
323
|
-
if (path.length < 2) continue;
|
|
324
|
-
const pathOps = path.map((operationId) => chunk.operations.find((candidate) => candidate.operation_id === operationId)).filter(Boolean) as SkillOperationNode[];
|
|
325
|
-
if (pathOps.length !== path.length) continue;
|
|
326
|
-
const entry = pathOps[0]!;
|
|
327
|
-
if (!canBuildCaseFromOperation(skill, entry)) continue;
|
|
328
|
-
const target = operation.auth_required ? authWalks : publicWalks;
|
|
329
|
-
if (target.length >= perBucket) continue;
|
|
330
|
-
target.push({
|
|
331
|
-
id: `${skill.skill_id}:walk:${operation.operation_id}`,
|
|
332
|
-
skill_id: skill.skill_id,
|
|
333
|
-
intent: intentForOperation(operation),
|
|
334
|
-
authenticated: operation.auth_required,
|
|
335
|
-
initial_params: objectParams(entry.example_request),
|
|
336
|
-
contextUrl: entry.trigger_url,
|
|
337
|
-
target_operation_id: operation.operation_id,
|
|
338
|
-
});
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
for (const edge of chunk.edges) {
|
|
342
|
-
const source = chunk.operations.find((operation) => operation.operation_id === edge.from_operation_id);
|
|
343
|
-
const target = chunk.operations.find((operation) => operation.operation_id === edge.to_operation_id);
|
|
344
|
-
if (!source || !target) continue;
|
|
345
|
-
if (!canBuildCaseFromOperation(skill, source)) continue;
|
|
346
|
-
const bucket = target.auth_required ? authWalks : publicWalks;
|
|
347
|
-
if (bucket.length >= perBucket) continue;
|
|
348
|
-
bucket.push({
|
|
349
|
-
id: `${skill.skill_id}:edge:${target.operation_id}`,
|
|
350
|
-
skill_id: skill.skill_id,
|
|
351
|
-
intent: intentForOperation(target),
|
|
352
|
-
authenticated: target.auth_required,
|
|
353
|
-
initial_params: objectParams(source.example_request),
|
|
354
|
-
contextUrl: source.trigger_url,
|
|
355
|
-
target_operation_id: target.operation_id,
|
|
356
|
-
});
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
return {
|
|
361
|
-
selection_cases: [...publicSelections, ...authSelections],
|
|
362
|
-
walk_cases: [...publicWalks, ...authWalks],
|
|
363
|
-
};
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
function scoreOperation(
|
|
367
|
-
skill: SkillManifest,
|
|
368
|
-
operation: SkillOperationNode,
|
|
369
|
-
intent: string,
|
|
370
|
-
params?: Record<string, unknown>,
|
|
371
|
-
contextUrl?: string,
|
|
372
|
-
): number {
|
|
373
|
-
const queryVec = toVector(queryText(intent, params, contextUrl));
|
|
374
|
-
const opVec = toVector(operationText(skill, operation));
|
|
375
|
-
const bindings = knownBindingsFromInputs(params ?? {}, contextUrl);
|
|
376
|
-
let score = cosine(queryVec, opVec) * 100;
|
|
377
|
-
|
|
378
|
-
const runnable = operation.requires.every((binding) => !binding.required || (bindings[binding.key] != null && bindings[binding.key] !== ""));
|
|
379
|
-
if (runnable) score += 18;
|
|
380
|
-
const satisfiedRequired = operation.requires.filter((binding) => binding.required && bindings[binding.key] != null && bindings[binding.key] !== "").length;
|
|
381
|
-
if (satisfiedRequired > 0) score += satisfiedRequired * 14;
|
|
382
|
-
|
|
383
|
-
const intentTokens = new Set(tokenize(intent).map(singularize));
|
|
384
|
-
if (intentTokens.has(operation.action_kind)) score += 8;
|
|
385
|
-
if (intentTokens.has(operation.resource_kind)) score += 12;
|
|
386
|
-
|
|
387
|
-
const negativePenalty = (operation.negative_tags ?? []).reduce((sum, tag) => {
|
|
388
|
-
if (intentTokens.has("status") && tag === "status") return sum;
|
|
389
|
-
return sum + 12;
|
|
390
|
-
}, 0);
|
|
391
|
-
score -= negativePenalty;
|
|
392
|
-
|
|
393
|
-
const requiresSatisfied = operation.requires.filter((binding) => binding.required).every((binding) => bindings[binding.key] != null && bindings[binding.key] !== "");
|
|
394
|
-
if (operation.action_kind === "detail" && !requiresSatisfied) score -= 10;
|
|
395
|
-
if (Object.keys(bindings).length > 0 && operation.requires.length === 0) {
|
|
396
|
-
const redundantBootstrap = operation.provides.some((binding) => bindings[binding.key] != null && bindings[binding.key] !== "");
|
|
397
|
-
if (redundantBootstrap) score -= 15;
|
|
398
|
-
}
|
|
399
|
-
score -= operationSoftPenalty(operation, intent);
|
|
400
|
-
|
|
401
|
-
return score;
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
export function loadSkillsFromLocalCache(cacheDir = join(homedir(), ".unbrowse", "skill-cache")): SkillManifest[] {
|
|
405
|
-
if (!existsSync(cacheDir)) return [];
|
|
406
|
-
return readdirSync(cacheDir)
|
|
407
|
-
.filter((file) => file.endsWith(".json"))
|
|
408
|
-
.flatMap((file) => {
|
|
409
|
-
try {
|
|
410
|
-
return [JSON.parse(readFileSync(join(cacheDir, file), "utf-8")) as SkillManifest];
|
|
411
|
-
} catch {
|
|
412
|
-
return [];
|
|
413
|
-
}
|
|
414
|
-
});
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
export function selectOperationsLocally(
|
|
418
|
-
skills: SkillManifest[],
|
|
419
|
-
intent: string,
|
|
420
|
-
opts?: {
|
|
421
|
-
params?: Record<string, unknown>;
|
|
422
|
-
contextUrl?: string;
|
|
423
|
-
topK?: number;
|
|
424
|
-
domain?: string;
|
|
425
|
-
authenticated?: boolean;
|
|
426
|
-
},
|
|
427
|
-
): HarnessOperationHit[] {
|
|
428
|
-
const scored: HarnessOperationHit[] = [];
|
|
429
|
-
for (const skill of skills) {
|
|
430
|
-
if (opts?.domain && skill.domain !== opts.domain) continue;
|
|
431
|
-
const chunk = getSkillChunk(skill, {
|
|
432
|
-
intent,
|
|
433
|
-
known_bindings: knownBindingsFromInputs(opts?.params ?? {}, opts?.contextUrl),
|
|
434
|
-
max_operations: 8,
|
|
435
|
-
});
|
|
436
|
-
const allowed = new Set(chunk.operations.map((operation) => operation.operation_id));
|
|
437
|
-
for (const operation of chunk.operations) {
|
|
438
|
-
if (!allowed.has(operation.operation_id)) continue;
|
|
439
|
-
if (isOperationHardExcluded(operation, intent)) continue;
|
|
440
|
-
const authRequired = operation.auth_required ?? false;
|
|
441
|
-
if (authRequired && !opts?.authenticated) continue;
|
|
442
|
-
const runnable = chunk.available_operation_ids.includes(operation.operation_id);
|
|
443
|
-
scored.push({
|
|
444
|
-
skill_id: skill.skill_id,
|
|
445
|
-
skill_domain: skill.domain,
|
|
446
|
-
operation_id: operation.operation_id,
|
|
447
|
-
endpoint_id: operation.endpoint_id,
|
|
448
|
-
score: scoreOperation(skill, operation, intent, opts?.params, opts?.contextUrl),
|
|
449
|
-
runnable,
|
|
450
|
-
action_kind: operation.action_kind,
|
|
451
|
-
resource_kind: operation.resource_kind,
|
|
452
|
-
negative_tags: operation.negative_tags ?? [],
|
|
453
|
-
auth_required: authRequired,
|
|
454
|
-
});
|
|
455
|
-
}
|
|
456
|
-
}
|
|
457
|
-
return scored.sort((a, b) => b.score - a.score).slice(0, opts?.topK ?? 5);
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
export function evaluateLocalHarness(
|
|
461
|
-
skills: SkillManifest[],
|
|
462
|
-
cases: LocalHarnessCase[],
|
|
463
|
-
): { summary: HarnessSummary; results: HarnessCaseResult[] } {
|
|
464
|
-
const results: HarnessCaseResult[] = cases.map((testCase) => {
|
|
465
|
-
const started = performance.now();
|
|
466
|
-
const hits = selectOperationsLocally(skills, testCase.intent, {
|
|
467
|
-
params: testCase.params,
|
|
468
|
-
contextUrl: testCase.contextUrl,
|
|
469
|
-
authenticated: testCase.authenticated,
|
|
470
|
-
topK: 5,
|
|
471
|
-
});
|
|
472
|
-
const selectionMs = performance.now() - started;
|
|
473
|
-
const selected = hits[0];
|
|
474
|
-
const skill = skills.find((candidate) => candidate.skill_id === (selected?.skill_id ?? testCase.expected_skill_id))
|
|
475
|
-
?? skills.find((candidate) => candidate.skill_id === testCase.expected_skill_id)!;
|
|
476
|
-
const chunk = getSkillChunk(skill, {
|
|
477
|
-
intent: testCase.intent,
|
|
478
|
-
known_bindings: knownBindingsFromInputs(testCase.params ?? {}, testCase.contextUrl),
|
|
479
|
-
max_operations: 8,
|
|
480
|
-
});
|
|
481
|
-
|
|
482
|
-
const pass = !!selected &&
|
|
483
|
-
selected.skill_id === testCase.expected_skill_id &&
|
|
484
|
-
selected.operation_id === testCase.expected_operation_id &&
|
|
485
|
-
(testCase.expected_chunk_contains ?? []).every((operationId) => chunk.operations.some((operation) => operation.operation_id === operationId));
|
|
486
|
-
const expectedRank = hits.findIndex((hit) =>
|
|
487
|
-
hit.skill_id === testCase.expected_skill_id && hit.operation_id === testCase.expected_operation_id);
|
|
488
|
-
const wrongTop1BeforeCorrect = expectedRank >= 0 ? expectedRank : hits.length;
|
|
489
|
-
|
|
490
|
-
return {
|
|
491
|
-
id: testCase.id,
|
|
492
|
-
pass,
|
|
493
|
-
selected,
|
|
494
|
-
top_hits: hits,
|
|
495
|
-
chunk,
|
|
496
|
-
expected_skill_id: testCase.expected_skill_id,
|
|
497
|
-
expected_operation_id: testCase.expected_operation_id,
|
|
498
|
-
selection_ms: selectionMs,
|
|
499
|
-
expected_rank: expectedRank,
|
|
500
|
-
wrong_top1_before_correct: wrongTop1BeforeCorrect,
|
|
501
|
-
failure_reason: pass
|
|
502
|
-
? undefined
|
|
503
|
-
: selected
|
|
504
|
-
? `selected ${selected.skill_id}:${selected.operation_id}`
|
|
505
|
-
: "no_selection",
|
|
506
|
-
};
|
|
507
|
-
});
|
|
508
|
-
|
|
509
|
-
const passed = results.filter((result) => result.pass).length;
|
|
510
|
-
const runnableTop1 = results.filter((result) => result.selected?.runnable).length;
|
|
511
|
-
const authResults = results.filter((result) => cases.find((testCase) => testCase.id === result.id)?.authenticated);
|
|
512
|
-
const publicResults = results.filter((result) => !cases.find((testCase) => testCase.id === result.id)?.authenticated);
|
|
513
|
-
const selectionTimings = results.map((result) => result.selection_ms);
|
|
514
|
-
const correctnessTimings = results
|
|
515
|
-
.filter((result) => result.expected_rank >= 0)
|
|
516
|
-
.map((result) => result.selection_ms);
|
|
517
|
-
return {
|
|
518
|
-
summary: {
|
|
519
|
-
total: results.length,
|
|
520
|
-
passed,
|
|
521
|
-
failed: results.length - passed,
|
|
522
|
-
hit_rate: results.length > 0 ? passed / results.length : 0,
|
|
523
|
-
runnable_top1_rate: results.length > 0 ? runnableTop1 / results.length : 0,
|
|
524
|
-
public_total: publicResults.length,
|
|
525
|
-
public_passed: publicResults.filter((result) => result.pass).length,
|
|
526
|
-
auth_total: authResults.length,
|
|
527
|
-
auth_passed: authResults.filter((result) => result.pass).length,
|
|
528
|
-
avg_selection_ms: average(selectionTimings),
|
|
529
|
-
avg_time_to_correct_ms: average(correctnessTimings),
|
|
530
|
-
avg_wrong_top1_before_correct: average(results.map((result) => result.wrong_top1_before_correct)),
|
|
531
|
-
},
|
|
532
|
-
results,
|
|
533
|
-
};
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
export function evaluateDependencyWalks(
|
|
537
|
-
skills: SkillManifest[],
|
|
538
|
-
cases: DependencyWalkCase[],
|
|
539
|
-
): { summary: DependencyWalkSummary; results: DependencyWalkResult[] } {
|
|
540
|
-
const results = cases.map((testCase) => {
|
|
541
|
-
const skill = skills.find((candidate) => candidate.skill_id === testCase.skill_id);
|
|
542
|
-
if (!skill) {
|
|
543
|
-
return {
|
|
544
|
-
id: testCase.id,
|
|
545
|
-
skill_id: testCase.skill_id,
|
|
546
|
-
pass: false,
|
|
547
|
-
target_operation_id: testCase.target_operation_id,
|
|
548
|
-
selected_path: [],
|
|
549
|
-
final_bindings: {},
|
|
550
|
-
time_to_target_ms: 0,
|
|
551
|
-
wrong_before_target: 0,
|
|
552
|
-
failure_reason: "missing_skill",
|
|
553
|
-
} satisfies DependencyWalkResult;
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
const selectedPath: string[] = [];
|
|
557
|
-
const visited = new Set<string>();
|
|
558
|
-
const bindings = knownBindingsFromInputs(testCase.initial_params ?? {}, testCase.contextUrl);
|
|
559
|
-
const started = performance.now();
|
|
560
|
-
let failureReason = "max_steps";
|
|
561
|
-
|
|
562
|
-
for (let step = 0; step < 6; step++) {
|
|
563
|
-
const hits = selectOperationsLocally([skill], testCase.intent, {
|
|
564
|
-
params: bindings,
|
|
565
|
-
contextUrl: testCase.contextUrl,
|
|
566
|
-
authenticated: testCase.authenticated,
|
|
567
|
-
topK: 6,
|
|
568
|
-
});
|
|
569
|
-
const next = hits.find((hit) => hit.runnable && !visited.has(hit.operation_id))
|
|
570
|
-
?? hits.find((hit) => !visited.has(hit.operation_id));
|
|
571
|
-
if (!next) {
|
|
572
|
-
failureReason = "no_candidate";
|
|
573
|
-
break;
|
|
574
|
-
}
|
|
575
|
-
|
|
576
|
-
visited.add(next.operation_id);
|
|
577
|
-
selectedPath.push(next.operation_id);
|
|
578
|
-
const operation = findOperation(skill, next.operation_id);
|
|
579
|
-
if (operation) Object.assign(bindings, bindingsFromOperation(operation));
|
|
580
|
-
if (next.operation_id === testCase.target_operation_id) {
|
|
581
|
-
failureReason = "";
|
|
582
|
-
break;
|
|
583
|
-
}
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
const timeToTargetMs = performance.now() - started;
|
|
587
|
-
const pass = selectedPath[selectedPath.length - 1] === testCase.target_operation_id &&
|
|
588
|
-
(!testCase.expected_path || testCase.expected_path.every((operationId, index) => selectedPath[index] === operationId));
|
|
589
|
-
return {
|
|
590
|
-
id: testCase.id,
|
|
591
|
-
skill_id: testCase.skill_id,
|
|
592
|
-
pass,
|
|
593
|
-
target_operation_id: testCase.target_operation_id,
|
|
594
|
-
selected_path: selectedPath,
|
|
595
|
-
final_bindings: bindings,
|
|
596
|
-
time_to_target_ms: timeToTargetMs,
|
|
597
|
-
wrong_before_target: Math.max(0, selectedPath.length - 1),
|
|
598
|
-
failure_reason: pass ? undefined : failureReason || "wrong_path",
|
|
599
|
-
} satisfies DependencyWalkResult;
|
|
600
|
-
});
|
|
601
|
-
|
|
602
|
-
const passed = results.filter((result) => result.pass).length;
|
|
603
|
-
return {
|
|
604
|
-
summary: {
|
|
605
|
-
total: results.length,
|
|
606
|
-
passed,
|
|
607
|
-
failed: results.length - passed,
|
|
608
|
-
hit_rate: results.length > 0 ? passed / results.length : 0,
|
|
609
|
-
avg_time_to_target_ms: average(results.map((result) => result.time_to_target_ms)),
|
|
610
|
-
avg_wrong_before_target: average(results.map((result) => result.wrong_before_target)),
|
|
611
|
-
},
|
|
612
|
-
results,
|
|
613
|
-
};
|
|
614
|
-
}
|
|
615
|
-
|
|
616
|
-
export function formatHarnessReport(summary: HarnessSummary, results: HarnessCaseResult[]): string {
|
|
617
|
-
const lines = [
|
|
618
|
-
`Local graph harness`,
|
|
619
|
-
`total=${summary.total} passed=${summary.passed} failed=${summary.failed} hit_rate=${Math.round(summary.hit_rate * 100)}% runnable_top1=${Math.round(summary.runnable_top1_rate * 100)}% public=${summary.public_passed}/${summary.public_total} auth=${summary.auth_passed}/${summary.auth_total} avg_select_ms=${summary.avg_selection_ms.toFixed(2)} avg_correct_ms=${summary.avg_time_to_correct_ms.toFixed(2)} avg_wrong_before_correct=${summary.avg_wrong_top1_before_correct.toFixed(2)}`,
|
|
620
|
-
"",
|
|
621
|
-
];
|
|
622
|
-
for (const result of results) {
|
|
623
|
-
const head = result.pass ? "PASS" : "FAIL";
|
|
624
|
-
const selected = result.selected ? `${result.selected.skill_id}:${result.selected.operation_id}` : "none";
|
|
625
|
-
lines.push(`${head} ${result.id} -> ${selected}`);
|
|
626
|
-
if (!result.pass) {
|
|
627
|
-
lines.push(` expected ${result.expected_skill_id}:${result.expected_operation_id}`);
|
|
628
|
-
lines.push(` chunk ${unique(result.chunk.operations.map((operation) => operation.operation_id)).join(", ")}`);
|
|
629
|
-
}
|
|
630
|
-
}
|
|
631
|
-
return lines.join("\n");
|
|
632
|
-
}
|
|
633
|
-
|
|
634
|
-
export function formatDependencyWalkReport(summary: DependencyWalkSummary, results: DependencyWalkResult[]): string {
|
|
635
|
-
const lines = [
|
|
636
|
-
`Dependency walk harness`,
|
|
637
|
-
`total=${summary.total} passed=${summary.passed} failed=${summary.failed} hit_rate=${Math.round(summary.hit_rate * 100)}% avg_target_ms=${summary.avg_time_to_target_ms.toFixed(2)} avg_wrong_before_target=${summary.avg_wrong_before_target.toFixed(2)}`,
|
|
638
|
-
"",
|
|
639
|
-
];
|
|
640
|
-
for (const result of results) {
|
|
641
|
-
const head = result.pass ? "PASS" : "FAIL";
|
|
642
|
-
lines.push(`${head} ${result.id} -> ${result.selected_path.join(" -> ") || "none"}`);
|
|
643
|
-
if (!result.pass) lines.push(` target ${result.target_operation_id} reason=${result.failure_reason ?? "unknown"}`);
|
|
644
|
-
}
|
|
645
|
-
return lines.join("\n");
|
|
646
|
-
}
|