unbrowse 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +455 -96
- package/dist/index.js +2 -6
- package/dist/mcp.js +695 -46
- package/dist/server.js +25811 -0
- package/package.json +1 -2
- package/vendor/kuri/darwin-arm64/kuri +0 -0
- package/vendor/kuri/darwin-x64/kuri +0 -0
- package/vendor/kuri/linux-arm64/kuri +0 -0
- package/vendor/kuri/linux-x64/kuri +0 -0
- package/vendor/kuri/manifest.json +7 -10
- package/runtime-src/agent-outcome.ts +0 -166
- package/runtime-src/analytics-session.ts +0 -55
- package/runtime-src/api/browse-index.ts +0 -317
- package/runtime-src/api/browse-session.ts +0 -572
- package/runtime-src/api/browse-submit-prereqs.ts +0 -48
- package/runtime-src/api/browse-submit.ts +0 -1184
- package/runtime-src/api/routes.ts +0 -1823
- package/runtime-src/auth/browser-cookies.ts +0 -423
- package/runtime-src/auth/index.ts +0 -535
- package/runtime-src/auth/runtime.ts +0 -116
- package/runtime-src/browser/index.ts +0 -659
- package/runtime-src/browser/types.ts +0 -41
- package/runtime-src/build-info.generated.ts +0 -6
- package/runtime-src/capture/index.ts +0 -1794
- package/runtime-src/capture/prefetch.ts +0 -95
- package/runtime-src/capture/rsc.ts +0 -45
- package/runtime-src/cli/shortcuts.ts +0 -273
- package/runtime-src/cli.ts +0 -1572
- package/runtime-src/client/graph-client.ts +0 -100
- package/runtime-src/client/index.ts +0 -1425
- package/runtime-src/debug-trace.ts +0 -18
- package/runtime-src/domain.ts +0 -38
- package/runtime-src/execution/index.ts +0 -3397
- package/runtime-src/execution/retry.ts +0 -46
- package/runtime-src/execution/robots.ts +0 -167
- package/runtime-src/execution/search-forms.ts +0 -188
- package/runtime-src/extraction/index.ts +0 -1507
- package/runtime-src/foundry/publish-bundle.ts +0 -392
- package/runtime-src/graph/agent-augment.ts +0 -315
- package/runtime-src/graph/index.ts +0 -1524
- package/runtime-src/graph/local-fixtures.ts +0 -393
- package/runtime-src/graph/local-harness.ts +0 -646
- package/runtime-src/graph/planner.ts +0 -411
- package/runtime-src/graph/session.ts +0 -294
- package/runtime-src/graph/trace-store.ts +0 -136
- package/runtime-src/index.ts +0 -24
- package/runtime-src/indexer/index.ts +0 -465
- package/runtime-src/intent-match.ts +0 -1515
- package/runtime-src/kuri/client.ts +0 -1839
- package/runtime-src/logger.ts +0 -30
- package/runtime-src/marketplace/index.ts +0 -103
- package/runtime-src/mcp.ts +0 -1747
- package/runtime-src/orchestrator/browser-agent.ts +0 -374
- package/runtime-src/orchestrator/dag-advisor.ts +0 -59
- package/runtime-src/orchestrator/dag-feedback.ts +0 -257
- package/runtime-src/orchestrator/first-pass-action.ts +0 -403
- package/runtime-src/orchestrator/index.ts +0 -4480
- package/runtime-src/orchestrator/passive-publish.ts +0 -187
- package/runtime-src/orchestrator/timing-economics.ts +0 -80
- package/runtime-src/payments/cascade.ts +0 -137
- package/runtime-src/payments/index.ts +0 -270
- package/runtime-src/payments/lobster-pay.ts +0 -182
- package/runtime-src/payments/wallet.ts +0 -98
- package/runtime-src/publish/review-context.ts +0 -93
- package/runtime-src/publish/sanitize.ts +0 -197
- package/runtime-src/publish/schema-review.ts +0 -192
- package/runtime-src/publish-admission.ts +0 -388
- package/runtime-src/ratelimit/index.ts +0 -23
- package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
- package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
- package/runtime-src/reverse-engineer/index.ts +0 -1551
- package/runtime-src/router.ts +0 -17
- package/runtime-src/routing-telemetry.ts +0 -395
- package/runtime-src/runtime/browser-access.ts +0 -11
- package/runtime-src/runtime/browser-auth.ts +0 -12
- package/runtime-src/runtime/browser-host.ts +0 -48
- package/runtime-src/runtime/lifecycle.ts +0 -17
- package/runtime-src/runtime/local-server.ts +0 -311
- package/runtime-src/runtime/paths.ts +0 -99
- package/runtime-src/runtime/setup.ts +0 -251
- package/runtime-src/runtime/supervisor.ts +0 -69
- package/runtime-src/runtime/update-hints.ts +0 -351
- package/runtime-src/server.ts +0 -100
- package/runtime-src/session-logs.ts +0 -142
- package/runtime-src/settings.ts +0 -221
- package/runtime-src/single-binary.ts +0 -143
- package/runtime-src/site-policy.ts +0 -54
- package/runtime-src/stale-cleanup-runner.ts +0 -144
- package/runtime-src/stale-cleanup.ts +0 -133
- package/runtime-src/telemetry-attribution.ts +0 -120
- package/runtime-src/telemetry.ts +0 -253
- package/runtime-src/template-params.ts +0 -141
- package/runtime-src/transform/drift.ts +0 -60
- package/runtime-src/transform/index.ts +0 -277
- package/runtime-src/types/index.ts +0 -1
- package/runtime-src/types/skill.ts +0 -912
- package/runtime-src/vault/index.ts +0 -196
- package/runtime-src/verification/auth-gate.ts +0 -8
- package/runtime-src/verification/candidates.ts +0 -27
- package/runtime-src/verification/index.ts +0 -120
- package/runtime-src/verification/matrix.ts +0 -30
- package/runtime-src/version.ts +0 -148
- package/runtime-src/workflow/artifact.ts +0 -161
- package/runtime-src/workflow/compile.ts +0 -808
- package/runtime-src/workflow/publish.ts +0 -225
- package/runtime-src/workflow/runtime.ts +0 -213
- package/vendor/kuri/win-x64/kuri.exe +0 -0
|
@@ -1,374 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Phase 4: Agentic Browser Loop
|
|
3
|
-
*
|
|
4
|
-
* When no cached API exists, autonomously drives the browser to achieve
|
|
5
|
-
* the intent. Snapshots the page, asks an LLM what action to take,
|
|
6
|
-
* executes it via Kuri, and checks for API calls after each step.
|
|
7
|
-
* All captured traffic is passively indexed for future reuse.
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
import * as kuri from "../kuri/client.js";
|
|
11
|
-
import type { KuriHarEntry } from "../kuri/client.js";
|
|
12
|
-
import { INTERCEPTOR_SCRIPT, collectInterceptedRequests, type RawRequest } from "../capture/index.js";
|
|
13
|
-
import { extractEndpoints, extractAuthHeaders } from "../reverse-engineer/index.js";
|
|
14
|
-
import { extractBrowserCookies } from "../auth/browser-cookies.js";
|
|
15
|
-
import { queueBackgroundIndex } from "../indexer/index.js";
|
|
16
|
-
import { mergeEndpoints } from "../marketplace/index.js";
|
|
17
|
-
import { buildSkillOperationGraph } from "../graph/index.js";
|
|
18
|
-
import { augmentEndpointsWithAgent } from "../graph/agent-augment.js";
|
|
19
|
-
import { findExistingSkillForDomain, cachePublishedSkill } from "../client/index.js";
|
|
20
|
-
import { storeCredential } from "../vault/index.js";
|
|
21
|
-
import { generateLocalDescription } from "./index.js";
|
|
22
|
-
import { nanoid } from "nanoid";
|
|
23
|
-
import type { EndpointDescriptor, SkillManifest } from "../types/index.js";
|
|
24
|
-
|
|
25
|
-
const MAX_STEPS = 5;
|
|
26
|
-
const ACTION_SETTLE_MS = 2000;
|
|
27
|
-
|
|
28
|
-
// ── LLM-based action planning ─────────────────────────────────────────
|
|
29
|
-
|
|
30
|
-
interface PlannedAction {
|
|
31
|
-
action: "click" | "fill" | "scroll" | "press" | "done";
|
|
32
|
-
ref?: string;
|
|
33
|
-
value?: string;
|
|
34
|
-
reason: string;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Ask the LLM: given this intent and page snapshot, what's the single
|
|
39
|
-
* best action to take? Lightweight structured output (~500 tokens).
|
|
40
|
-
*/
|
|
41
|
-
async function planNextAction(
|
|
42
|
-
intent: string,
|
|
43
|
-
params: Record<string, unknown>,
|
|
44
|
-
snapshot: string,
|
|
45
|
-
currentUrl: string,
|
|
46
|
-
): Promise<PlannedAction> {
|
|
47
|
-
// Trim snapshot to keep prompt small
|
|
48
|
-
const trimmedSnapshot = snapshot.length > 3000 ? snapshot.substring(0, 3000) + "\n..." : snapshot;
|
|
49
|
-
|
|
50
|
-
const prompt = `You are a browser agent. Given a user intent and the interactive elements on the page, decide the SINGLE best action to take to achieve the intent. Return ONLY valid JSON.
|
|
51
|
-
|
|
52
|
-
Intent: "${intent}"
|
|
53
|
-
${Object.keys(params).length > 0 ? `Params: ${JSON.stringify(params)}` : ""}
|
|
54
|
-
Current URL: ${currentUrl}
|
|
55
|
-
|
|
56
|
-
Interactive elements:
|
|
57
|
-
${trimmedSnapshot}
|
|
58
|
-
|
|
59
|
-
Return JSON: {"action":"click"|"fill"|"scroll"|"press"|"done", "ref":"eN", "value":"text for fill/press", "reason":"why"}
|
|
60
|
-
- "click" a link/button/tab that will load the data the intent asks for
|
|
61
|
-
- "fill" a search box then the next action should press Enter
|
|
62
|
-
- "scroll" down if the page needs to load more content
|
|
63
|
-
- "press" a key like Enter after filling a search box
|
|
64
|
-
- "done" if the page already shows the data or no useful action exists
|
|
65
|
-
|
|
66
|
-
JSON:`;
|
|
67
|
-
|
|
68
|
-
try {
|
|
69
|
-
// Use kuri's evaluate to call a local inference endpoint, or fall back to
|
|
70
|
-
// simple heuristic-based planning if no LLM is available
|
|
71
|
-
const result = await heuristicPlan(intent, params, snapshot, currentUrl);
|
|
72
|
-
return result;
|
|
73
|
-
} catch {
|
|
74
|
-
return { action: "done", reason: "planning failed" };
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
/**
|
|
79
|
-
* Heuristic-based action planning — no LLM needed.
|
|
80
|
-
* Handles common patterns: search intents, feed/timeline intents, navigation.
|
|
81
|
-
*/
|
|
82
|
-
function heuristicPlan(
|
|
83
|
-
intent: string,
|
|
84
|
-
params: Record<string, unknown>,
|
|
85
|
-
snapshot: string,
|
|
86
|
-
_currentUrl: string,
|
|
87
|
-
): PlannedAction {
|
|
88
|
-
const intentLower = intent.toLowerCase();
|
|
89
|
-
const lines = snapshot.split("\n").filter(l => l.trim());
|
|
90
|
-
|
|
91
|
-
// Parse refs from snapshot
|
|
92
|
-
const elements = lines.map(l => {
|
|
93
|
-
const match = l.match(/^\[(\w+)\]\s+(\w+)\s+"?(.+?)"?\s*$/);
|
|
94
|
-
if (!match) return null;
|
|
95
|
-
return { ref: match[1], role: match[2], name: match[3].replace(/"$/, "") };
|
|
96
|
-
}).filter(Boolean) as Array<{ ref: string; role: string; name: string }>;
|
|
97
|
-
|
|
98
|
-
// Search intent: find a textbox and fill it
|
|
99
|
-
const isSearch = /search|find|look\s*up|query/i.test(intentLower);
|
|
100
|
-
if (isSearch) {
|
|
101
|
-
const searchTerm = (params.q ?? params.query ?? params.keywords ?? extractSearchTerm(intent)) as string;
|
|
102
|
-
const textbox = elements.find(e => e.role === "textbox" || e.role === "searchbox");
|
|
103
|
-
if (textbox && searchTerm) {
|
|
104
|
-
return { action: "fill", ref: textbox.ref, value: searchTerm, reason: `Fill search box with "${searchTerm}"` };
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
// Feed/timeline intent: look for relevant tabs
|
|
109
|
-
const isFeed = /timeline|feed|home|for\s*you|following|trending/i.test(intentLower);
|
|
110
|
-
if (isFeed) {
|
|
111
|
-
const feedTab = elements.find(e =>
|
|
112
|
-
(e.role === "tab" || e.role === "link") &&
|
|
113
|
-
/for you|following|trending|home|latest/i.test(e.name)
|
|
114
|
-
);
|
|
115
|
-
if (feedTab) {
|
|
116
|
-
return { action: "click", ref: feedTab.ref, reason: `Click "${feedTab.name}" to load feed data` };
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
// Look for a button/link that semantically matches the intent
|
|
121
|
-
const intentWords = intentLower.split(/\s+/).filter(w => w.length > 3);
|
|
122
|
-
for (const el of elements) {
|
|
123
|
-
if (el.role !== "link" && el.role !== "button" && el.role !== "tab") continue;
|
|
124
|
-
const nameLower = el.name.toLowerCase();
|
|
125
|
-
const matchScore = intentWords.filter(w => nameLower.includes(w)).length;
|
|
126
|
-
if (matchScore >= 2 || (matchScore >= 1 && intentWords.length <= 3)) {
|
|
127
|
-
return { action: "click", ref: el.ref, reason: `"${el.name}" matches intent` };
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
// If we just filled a search, press Enter
|
|
132
|
-
if (isSearch) {
|
|
133
|
-
return { action: "press", value: "Enter", reason: "Submit search" };
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
// Scroll down to trigger lazy-loaded content
|
|
137
|
-
return { action: "scroll", reason: "Scroll to trigger lazy-loaded API calls" };
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
function extractSearchTerm(intent: string): string {
|
|
141
|
-
// "search people named minh" → "minh"
|
|
142
|
-
// "search for react components" → "react components"
|
|
143
|
-
const match = intent.match(/(?:search|find|look\s*up)\s+(?:for\s+)?(?:people\s+(?:named|called)\s+)?(.+)/i);
|
|
144
|
-
return match?.[1]?.trim() ?? intent;
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
// ── Agentic browser loop ──────────────────────────────────────────────
|
|
148
|
-
|
|
149
|
-
export interface AgenticBrowseResult {
|
|
150
|
-
endpoints: EndpointDescriptor[];
|
|
151
|
-
skill?: SkillManifest;
|
|
152
|
-
result?: unknown;
|
|
153
|
-
stepsExecuted: number;
|
|
154
|
-
requestsCaptured: number;
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
/**
|
|
158
|
-
* Autonomously drive the browser to achieve an intent.
|
|
159
|
-
* Snapshots → plans → acts → captures APIs → repeats.
|
|
160
|
-
* All traffic is passively indexed regardless of success.
|
|
161
|
-
*/
|
|
162
|
-
export async function agenticBrowserResolve(
|
|
163
|
-
tabId: string,
|
|
164
|
-
intent: string,
|
|
165
|
-
params: Record<string, unknown>,
|
|
166
|
-
url: string,
|
|
167
|
-
): Promise<AgenticBrowseResult> {
|
|
168
|
-
const domain = new URL(url).hostname;
|
|
169
|
-
|
|
170
|
-
// Inject cookies from user's Chrome
|
|
171
|
-
try {
|
|
172
|
-
const { cookies } = extractBrowserCookies(domain);
|
|
173
|
-
for (const c of cookies) await kuri.setCookie(tabId, c).catch(() => {});
|
|
174
|
-
} catch { /* non-fatal */ }
|
|
175
|
-
|
|
176
|
-
// Inject fetch/XHR interceptor
|
|
177
|
-
await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT).catch(() => {});
|
|
178
|
-
|
|
179
|
-
// Start HAR recording
|
|
180
|
-
await kuri.harStart(tabId).catch(() => {});
|
|
181
|
-
|
|
182
|
-
// Navigate if not already on the page
|
|
183
|
-
const currentUrl = await kuri.getCurrentUrl(tabId).catch(() => "");
|
|
184
|
-
if (!currentUrl || !currentUrl.startsWith("http") || new URL(currentUrl).hostname !== domain) {
|
|
185
|
-
await kuri.navigate(tabId, url);
|
|
186
|
-
await new Promise(r => setTimeout(r, 2000));
|
|
187
|
-
// Re-inject interceptor after navigation
|
|
188
|
-
await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT).catch(() => {});
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
let allRequests: RawRequest[] = [];
|
|
192
|
-
let allEndpoints: EndpointDescriptor[] = [];
|
|
193
|
-
let stepsExecuted = 0;
|
|
194
|
-
let lastFillRef: string | undefined;
|
|
195
|
-
|
|
196
|
-
for (let step = 0; step < MAX_STEPS; step++) {
|
|
197
|
-
// 1. Snapshot the page
|
|
198
|
-
let snapshot: string;
|
|
199
|
-
try {
|
|
200
|
-
snapshot = await kuri.snapshot(tabId, "interactive");
|
|
201
|
-
} catch {
|
|
202
|
-
break;
|
|
203
|
-
}
|
|
204
|
-
if (!snapshot || snapshot.length < 10) {
|
|
205
|
-
await new Promise(r => setTimeout(r, 1500));
|
|
206
|
-
continue;
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
// 2. Plan next action
|
|
210
|
-
const decision = planNextAction(intent, params, snapshot, url);
|
|
211
|
-
console.log(`[agentic-browse] step ${step}: ${decision.action} ${decision.ref ?? ""} — ${decision.reason}`);
|
|
212
|
-
|
|
213
|
-
if (decision.action === "done") break;
|
|
214
|
-
|
|
215
|
-
// 3. Execute action
|
|
216
|
-
try {
|
|
217
|
-
switch (decision.action) {
|
|
218
|
-
case "click":
|
|
219
|
-
if (decision.ref) await kuri.click(tabId, decision.ref);
|
|
220
|
-
break;
|
|
221
|
-
case "fill":
|
|
222
|
-
if (decision.ref && decision.value) {
|
|
223
|
-
await kuri.fill(tabId, decision.ref, decision.value);
|
|
224
|
-
lastFillRef = decision.ref;
|
|
225
|
-
}
|
|
226
|
-
break;
|
|
227
|
-
case "scroll":
|
|
228
|
-
await kuri.scroll(tabId, "down");
|
|
229
|
-
break;
|
|
230
|
-
case "press":
|
|
231
|
-
if (decision.value) await kuri.press(tabId, decision.value);
|
|
232
|
-
break;
|
|
233
|
-
}
|
|
234
|
-
} catch (err) {
|
|
235
|
-
console.log(`[agentic-browse] action failed: ${err instanceof Error ? err.message : err}`);
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
stepsExecuted++;
|
|
239
|
-
|
|
240
|
-
// 4. Wait for API calls to settle
|
|
241
|
-
await new Promise(r => setTimeout(r, ACTION_SETTLE_MS));
|
|
242
|
-
|
|
243
|
-
// Re-inject interceptor in case navigation happened
|
|
244
|
-
await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT).catch(() => {});
|
|
245
|
-
|
|
246
|
-
// 5. Check intercepted requests
|
|
247
|
-
try {
|
|
248
|
-
const intercepted = await collectInterceptedRequests(tabId);
|
|
249
|
-
const newRequests: RawRequest[] = intercepted
|
|
250
|
-
.filter(r => !allRequests.some(e => e.url === r.url && e.method === r.method))
|
|
251
|
-
.map(r => ({
|
|
252
|
-
url: r.url,
|
|
253
|
-
method: r.method,
|
|
254
|
-
request_headers: r.request_headers ?? {},
|
|
255
|
-
request_body: r.request_body,
|
|
256
|
-
response_status: r.response_status,
|
|
257
|
-
response_headers: r.response_headers ?? {},
|
|
258
|
-
response_body: r.response_body,
|
|
259
|
-
timestamp: r.timestamp,
|
|
260
|
-
}));
|
|
261
|
-
|
|
262
|
-
if (newRequests.length > 0) {
|
|
263
|
-
allRequests.push(...newRequests);
|
|
264
|
-
const newEndpoints = extractEndpoints(newRequests, undefined, { pageUrl: url, finalUrl: url });
|
|
265
|
-
if (newEndpoints.length > 0) {
|
|
266
|
-
allEndpoints.push(...newEndpoints);
|
|
267
|
-
console.log(`[agentic-browse] step ${step}: found ${newEndpoints.length} new endpoints`);
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
} catch { /* non-fatal */ }
|
|
271
|
-
|
|
272
|
-
// If after a fill we should press Enter
|
|
273
|
-
if (decision.action === "fill" && lastFillRef) {
|
|
274
|
-
// Next iteration will handle pressing Enter via heuristic
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
// Also collect HAR entries
|
|
279
|
-
try {
|
|
280
|
-
const { entries } = await kuri.harStop(tabId);
|
|
281
|
-
const harRequests = entries
|
|
282
|
-
.filter((e: KuriHarEntry) => e.request && e.response)
|
|
283
|
-
.map((e: KuriHarEntry) => ({
|
|
284
|
-
url: e.request.url,
|
|
285
|
-
method: e.request.method,
|
|
286
|
-
request_headers: Object.fromEntries((e.request.headers ?? []).map(h => [h.name.toLowerCase(), h.value])),
|
|
287
|
-
request_body: e.request.postData?.text,
|
|
288
|
-
response_status: e.response.status,
|
|
289
|
-
response_headers: Object.fromEntries((e.response.headers ?? []).map(h => [h.name.toLowerCase(), h.value])),
|
|
290
|
-
response_body: e.response.content?.text,
|
|
291
|
-
timestamp: e.startedDateTime ?? new Date().toISOString(),
|
|
292
|
-
}));
|
|
293
|
-
// Merge HAR requests that interceptor missed
|
|
294
|
-
for (const r of harRequests) {
|
|
295
|
-
if (!allRequests.some(e => e.url === r.url && e.method === r.method)) {
|
|
296
|
-
allRequests.push(r);
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
} catch { /* non-fatal */ }
|
|
300
|
-
|
|
301
|
-
// Merge all captured endpoints
|
|
302
|
-
if (allEndpoints.length === 0 && allRequests.length > 0) {
|
|
303
|
-
allEndpoints = extractEndpoints(allRequests, undefined, { pageUrl: url, finalUrl: url });
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
// ── Full passive indexing pipeline (same as passiveIndexFromRequests) ──
|
|
307
|
-
|
|
308
|
-
let skill: SkillManifest | undefined;
|
|
309
|
-
|
|
310
|
-
if (allEndpoints.length > 0) {
|
|
311
|
-
// Auth extraction + vault storage
|
|
312
|
-
const capturedAuthHeaders = extractAuthHeaders(allRequests);
|
|
313
|
-
if (Object.keys(capturedAuthHeaders).length > 0) {
|
|
314
|
-
await storeCredential(`${domain}-session`, JSON.stringify({ headers: capturedAuthHeaders })).catch(() => {});
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
// Merge with existing skill
|
|
318
|
-
const existingSkill = findExistingSkillForDomain(domain, intent);
|
|
319
|
-
const mergedEndpoints = existingSkill
|
|
320
|
-
? mergeEndpoints(existingSkill.endpoints, allEndpoints)
|
|
321
|
-
: allEndpoints;
|
|
322
|
-
|
|
323
|
-
// Generate descriptions
|
|
324
|
-
for (const ep of mergedEndpoints) {
|
|
325
|
-
if (!ep.description) ep.description = generateLocalDescription(ep);
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
const enrichedEndpoints = mergedEndpoints;
|
|
329
|
-
|
|
330
|
-
// Build operation graph
|
|
331
|
-
const operationGraph = buildSkillOperationGraph(enrichedEndpoints);
|
|
332
|
-
|
|
333
|
-
skill = {
|
|
334
|
-
skill_id: existingSkill?.skill_id ?? nanoid(),
|
|
335
|
-
version: "1.0.0",
|
|
336
|
-
schema_version: "1",
|
|
337
|
-
lifecycle: "active" as const,
|
|
338
|
-
execution_type: "http" as const,
|
|
339
|
-
created_at: existingSkill?.created_at ?? new Date().toISOString(),
|
|
340
|
-
updated_at: new Date().toISOString(),
|
|
341
|
-
name: domain,
|
|
342
|
-
intent_signature: `browse ${domain}`,
|
|
343
|
-
domain,
|
|
344
|
-
description: `API skill for ${domain}`,
|
|
345
|
-
owner_type: "agent" as const,
|
|
346
|
-
endpoints: enrichedEndpoints,
|
|
347
|
-
operation_graph: operationGraph,
|
|
348
|
-
intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
|
|
349
|
-
};
|
|
350
|
-
|
|
351
|
-
// Cache locally
|
|
352
|
-
try { cachePublishedSkill(skill); } catch { /* best-effort */ }
|
|
353
|
-
|
|
354
|
-
// Queue background publish
|
|
355
|
-
queueBackgroundIndex({
|
|
356
|
-
skill,
|
|
357
|
-
domain,
|
|
358
|
-
intent,
|
|
359
|
-
contextUrl: url,
|
|
360
|
-
cacheKey: `agentic:${domain}:${Date.now()}`,
|
|
361
|
-
});
|
|
362
|
-
|
|
363
|
-
console.log(`[agentic-browse] ${domain}: ${enrichedEndpoints.length} endpoints indexed from ${allRequests.length} requests across ${stepsExecuted} steps`);
|
|
364
|
-
} else {
|
|
365
|
-
console.log(`[agentic-browse] ${domain}: 0 endpoints from ${allRequests.length} requests across ${stepsExecuted} steps`);
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
return {
|
|
369
|
-
endpoints: allEndpoints,
|
|
370
|
-
skill,
|
|
371
|
-
stepsExecuted,
|
|
372
|
-
requestsCaptured: allRequests.length,
|
|
373
|
-
};
|
|
374
|
-
}
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* DAG advisory planner bridge — backend-first with local fallback (Issue #218).
|
|
3
|
-
*
|
|
4
|
-
* The orchestrator calls fetchDagAdvisoryPlan with a full SkillManifest.
|
|
5
|
-
* This module tries the backend graph (via fetchChain) first for cross-session
|
|
6
|
-
* intelligence, then falls back to the local planner when the backend is
|
|
7
|
-
* unavailable or returns no data.
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
import type { SkillManifest } from "../types/index.js";
|
|
11
|
-
import {
|
|
12
|
-
fetchDagAdvisoryPlan as localFetchDagAdvisoryPlan,
|
|
13
|
-
applyDagAdvisoryBoosts,
|
|
14
|
-
} from "../graph/planner.js";
|
|
15
|
-
import type { DagAdvisoryPlan } from "../graph/planner.js";
|
|
16
|
-
import { fetchChain } from "../client/graph-client.js";
|
|
17
|
-
|
|
18
|
-
export { applyDagAdvisoryBoosts };
|
|
19
|
-
export type { DagAdvisoryPlan };
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* Fetch a DAG advisory plan — tries backend graph first, falls back to local.
|
|
23
|
-
*
|
|
24
|
-
* @param skill The full skill manifest (used for local fallback planning).
|
|
25
|
-
* @param targetEndpointId The endpoint we want to execute.
|
|
26
|
-
* @param knownBindingKeys Binding keys already available from context/params.
|
|
27
|
-
*/
|
|
28
|
-
export async function fetchDagAdvisoryPlan(
|
|
29
|
-
skill: SkillManifest,
|
|
30
|
-
targetEndpointId: string,
|
|
31
|
-
knownBindingKeys: string[],
|
|
32
|
-
): Promise<DagAdvisoryPlan> {
|
|
33
|
-
// Try backend graph first — provides cross-session intelligence
|
|
34
|
-
if (skill.domain) {
|
|
35
|
-
try {
|
|
36
|
-
const chain = await fetchChain(skill.domain, targetEndpointId, knownBindingKeys);
|
|
37
|
-
if (chain && Array.isArray(chain.chain) && chain.chain.length > 0) {
|
|
38
|
-
// Validate that chain endpoints exist in the local skill — discard stale backend data
|
|
39
|
-
const localEndpointIds = new Set(skill.endpoints.map((ep) => ep.endpoint_id));
|
|
40
|
-
const validPrereqs = chain.chain.filter(
|
|
41
|
-
(link) => link.endpoint_id !== targetEndpointId && localEndpointIds.has(link.endpoint_id),
|
|
42
|
-
);
|
|
43
|
-
if (validPrereqs.length > 0) {
|
|
44
|
-
return {
|
|
45
|
-
chain_ready: chain.resolved ?? true,
|
|
46
|
-
prerequisite_order: validPrereqs.map((link) => link.endpoint_id),
|
|
47
|
-
predicted_next: [],
|
|
48
|
-
skippable: [],
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
} catch {
|
|
53
|
-
// Backend unavailable — fall through to local planner
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
// Local fallback — uses the in-memory operation graph
|
|
58
|
-
return localFetchDagAdvisoryPlan(skill, targetEndpointId, knownBindingKeys);
|
|
59
|
-
}
|
|
@@ -1,257 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* DAG learning loop (Issue #102).
|
|
3
|
-
*
|
|
4
|
-
* Fire-and-forget helpers that reinforce or penalise operation-graph edges
|
|
5
|
-
* based on observed auto-exec outcomes. All writes are debounced per skill to
|
|
6
|
-
* avoid flooding the local skill cache.
|
|
7
|
-
*
|
|
8
|
-
* Session actions and negatives are also forwarded to the backend graph API
|
|
9
|
-
* (fire-and-forget, never blocking) so cross-session intelligence can be
|
|
10
|
-
* aggregated.
|
|
11
|
-
*/
|
|
12
|
-
|
|
13
|
-
import { nanoid } from "nanoid";
|
|
14
|
-
import { cachePublishedSkill, getApiKey } from "../client/index.js";
|
|
15
|
-
import { recordSession, recordNegative } from "../client/graph-client.js";
|
|
16
|
-
import { buildSkillOperationGraph } from "../graph/index.js";
|
|
17
|
-
import type { SkillManifest, SkillOperationEdge, SkillOperationNode } from "../types/index.js";
|
|
18
|
-
import { DEFAULT_BACKEND_URL } from "../version.js";
|
|
19
|
-
|
|
20
|
-
/** Stable session ID — one per process lifetime. */
|
|
21
|
-
const SESSION_ID = nanoid();
|
|
22
|
-
|
|
23
|
-
/** Expose session ID for test verification only. */
|
|
24
|
-
export function _getSessionIdForTesting(): string {
|
|
25
|
-
return SESSION_ID;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
// ---------------------------------------------------------------------------
|
|
29
|
-
// Rate-limit / debounce state
|
|
30
|
-
// ---------------------------------------------------------------------------
|
|
31
|
-
|
|
32
|
-
/** Minimum ms between graph writes for the same skill. */
|
|
33
|
-
export let DAG_WRITE_DEBOUNCE_MS = 5_000;
|
|
34
|
-
|
|
35
|
-
/** Maximum number of pending timers at once (global back-pressure). */
|
|
36
|
-
export const MAX_PENDING_WRITES = 50;
|
|
37
|
-
|
|
38
|
-
const lastWriteAt = new Map<string, number>();
|
|
39
|
-
const pendingTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
/** For tests only: reset all debounce/rate-limit state and optionally override the delay. */
|
|
43
|
-
export function _resetForTesting(debounceMs = 5_000): void {
|
|
44
|
-
for (const t of pendingTimers.values()) clearTimeout(t);
|
|
45
|
-
pendingTimers.clear();
|
|
46
|
-
lastWriteAt.clear();
|
|
47
|
-
DAG_WRITE_DEBOUNCE_MS = debounceMs;
|
|
48
|
-
}
|
|
49
|
-
/** Retrieve the API key for backend edge publishing. Reuses graph-client auth. */
|
|
50
|
-
function _getApiKeyForPublish(): string {
|
|
51
|
-
try {
|
|
52
|
-
return getApiKey();
|
|
53
|
-
} catch {
|
|
54
|
-
return "";
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
function scheduleWrite(skillId: string, fn: () => void): void {
|
|
60
|
-
if (pendingTimers.size >= MAX_PENDING_WRITES) return; // back-pressure: drop silently
|
|
61
|
-
|
|
62
|
-
const existing = pendingTimers.get(skillId);
|
|
63
|
-
if (existing) clearTimeout(existing);
|
|
64
|
-
|
|
65
|
-
const last = lastWriteAt.get(skillId) ?? 0;
|
|
66
|
-
const now = Date.now();
|
|
67
|
-
const delay = Math.max(0, DAG_WRITE_DEBOUNCE_MS - (now - last));
|
|
68
|
-
|
|
69
|
-
const timer = setTimeout(() => {
|
|
70
|
-
pendingTimers.delete(skillId);
|
|
71
|
-
lastWriteAt.set(skillId, Date.now());
|
|
72
|
-
try {
|
|
73
|
-
fn();
|
|
74
|
-
} catch {
|
|
75
|
-
/* non-critical — best effort */
|
|
76
|
-
}
|
|
77
|
-
}, delay);
|
|
78
|
-
|
|
79
|
-
pendingTimers.set(skillId, timer);
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
// ---------------------------------------------------------------------------
|
|
83
|
-
// Confidence adjustment helpers
|
|
84
|
-
// ---------------------------------------------------------------------------
|
|
85
|
-
|
|
86
|
-
const BOOST_STEP = 0.05;
|
|
87
|
-
const PENALTY_STEP = 0.08;
|
|
88
|
-
const CONFIDENCE_MIN = 0.1;
|
|
89
|
-
const CONFIDENCE_MAX = 1.0;
|
|
90
|
-
|
|
91
|
-
function clamp(v: number, lo: number, hi: number): number {
|
|
92
|
-
return Math.max(lo, Math.min(hi, v));
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
function adjustEdgeConfidences(
|
|
96
|
-
edges: SkillOperationEdge[],
|
|
97
|
-
operationId: string,
|
|
98
|
-
delta: number,
|
|
99
|
-
): SkillOperationEdge[] {
|
|
100
|
-
return edges.map((edge) => {
|
|
101
|
-
if (edge.to_operation_id === operationId || edge.from_operation_id === operationId) {
|
|
102
|
-
return {
|
|
103
|
-
...edge,
|
|
104
|
-
confidence: clamp(edge.confidence + delta, CONFIDENCE_MIN, CONFIDENCE_MAX),
|
|
105
|
-
};
|
|
106
|
-
}
|
|
107
|
-
return edge;
|
|
108
|
-
});
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
function operationIdForEndpoint(skill: SkillManifest, endpointId: string): string | undefined {
|
|
112
|
-
return skill.operation_graph?.operations.find((op) => op.endpoint_id === endpointId)
|
|
113
|
-
?.operation_id;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
// ---------------------------------------------------------------------------
|
|
117
|
-
// Public API
|
|
118
|
-
// ---------------------------------------------------------------------------
|
|
119
|
-
|
|
120
|
-
/**
|
|
121
|
-
* Record a successful or failed execution attempt for a given endpoint.
|
|
122
|
-
* Adjusts edge confidences attached to the endpoint's operation node and
|
|
123
|
-
* persists the updated graph to the local skill cache (debounced).
|
|
124
|
-
*/
|
|
125
|
-
export function recordDagSessionAction(
|
|
126
|
-
skill: SkillManifest,
|
|
127
|
-
endpointId: string,
|
|
128
|
-
succeeded: boolean,
|
|
129
|
-
): void {
|
|
130
|
-
if (!skill.operation_graph) return;
|
|
131
|
-
|
|
132
|
-
const opId = operationIdForEndpoint(skill, endpointId);
|
|
133
|
-
if (!opId) return;
|
|
134
|
-
|
|
135
|
-
const delta = succeeded ? BOOST_STEP : -PENALTY_STEP;
|
|
136
|
-
|
|
137
|
-
scheduleWrite(skill.skill_id, () => {
|
|
138
|
-
const updated: SkillManifest = {
|
|
139
|
-
...skill,
|
|
140
|
-
operation_graph: {
|
|
141
|
-
...skill.operation_graph!,
|
|
142
|
-
edges: adjustEdgeConfidences(skill.operation_graph!.edges, opId, delta),
|
|
143
|
-
generated_at: new Date().toISOString(),
|
|
144
|
-
},
|
|
145
|
-
};
|
|
146
|
-
cachePublishedSkill(updated);
|
|
147
|
-
});
|
|
148
|
-
|
|
149
|
-
// Fire-and-forget to backend — never block, never throw
|
|
150
|
-
if (skill.domain) {
|
|
151
|
-
recordSession(
|
|
152
|
-
skill.domain,
|
|
153
|
-
SESSION_ID,
|
|
154
|
-
endpointId,
|
|
155
|
-
skill.intent_signature ?? "",
|
|
156
|
-
succeeded ? "success" : "failure",
|
|
157
|
-
).catch(() => {});
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
/**
|
|
162
|
-
* Record an explicit negative signal for an endpoint (e.g. judge rejected the
|
|
163
|
-
* result). Applies a larger confidence penalty than a plain failure.
|
|
164
|
-
*/
|
|
165
|
-
export function recordDagNegative(skill: SkillManifest, endpointId: string): void {
|
|
166
|
-
if (!skill.operation_graph) return;
|
|
167
|
-
|
|
168
|
-
const opId = operationIdForEndpoint(skill, endpointId);
|
|
169
|
-
if (!opId) return;
|
|
170
|
-
|
|
171
|
-
scheduleWrite(skill.skill_id, () => {
|
|
172
|
-
const updated: SkillManifest = {
|
|
173
|
-
...skill,
|
|
174
|
-
operation_graph: {
|
|
175
|
-
...skill.operation_graph!,
|
|
176
|
-
edges: adjustEdgeConfidences(skill.operation_graph!.edges, opId, -(PENALTY_STEP * 2)),
|
|
177
|
-
generated_at: new Date().toISOString(),
|
|
178
|
-
},
|
|
179
|
-
};
|
|
180
|
-
cachePublishedSkill(updated);
|
|
181
|
-
});
|
|
182
|
-
|
|
183
|
-
// Fire-and-forget to backend — never block, never throw
|
|
184
|
-
if (skill.domain) {
|
|
185
|
-
recordNegative(
|
|
186
|
-
skill.domain,
|
|
187
|
-
skill.intent_signature ?? "",
|
|
188
|
-
endpointId,
|
|
189
|
-
).catch(() => {});
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
/**
|
|
194
|
-
* Rebuild the operation graph from the skill's current endpoints and persist
|
|
195
|
-
* it to the local cache. Call this after live-capture adds or updates endpoints
|
|
196
|
-
* so that future chunk queries reflect the latest topology.
|
|
197
|
-
*/
|
|
198
|
-
export function upsertDagEdgesFromOperationGraph(skill: SkillManifest): void {
|
|
199
|
-
scheduleWrite(skill.skill_id, () => {
|
|
200
|
-
const freshGraph = buildSkillOperationGraph(skill.endpoints);
|
|
201
|
-
// Preserve learned edge confidences if the edge already exists.
|
|
202
|
-
const existing = skill.operation_graph;
|
|
203
|
-
const mergedEdges = freshGraph.edges.map((edge) => {
|
|
204
|
-
const prior = existing?.edges.find((e) => e.edge_id === edge.edge_id);
|
|
205
|
-
return prior ? { ...edge, confidence: prior.confidence } : edge;
|
|
206
|
-
});
|
|
207
|
-
const updated: SkillManifest = {
|
|
208
|
-
...skill,
|
|
209
|
-
operation_graph: {
|
|
210
|
-
...freshGraph,
|
|
211
|
-
edges: mergedEdges,
|
|
212
|
-
},
|
|
213
|
-
};
|
|
214
|
-
cachePublishedSkill(updated);
|
|
215
|
-
|
|
216
|
-
// Fire-and-forget: publish edges to the backend for each operation node
|
|
217
|
-
publishEdgesToBackend(skill.domain, freshGraph);
|
|
218
|
-
});
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
/**
|
|
222
|
-
* Fire-and-forget: POST each operation node's edges to the backend API.
|
|
223
|
-
* Failures are silently ignored — this must never block skill publish.
|
|
224
|
-
*/
|
|
225
|
-
export function publishEdgesToBackend(
|
|
226
|
-
domain: string,
|
|
227
|
-
graph: { operations: SkillOperationNode[]; edges: SkillOperationEdge[] },
|
|
228
|
-
): void {
|
|
229
|
-
const backendUrl = process.env.UNBROWSE_BACKEND_URL || DEFAULT_BACKEND_URL;
|
|
230
|
-
|
|
231
|
-
const apiKey = _getApiKeyForPublish();
|
|
232
|
-
|
|
233
|
-
for (const op of graph.operations) {
|
|
234
|
-
const node = {
|
|
235
|
-
endpoint_id: op.endpoint_id,
|
|
236
|
-
requires: op.requires.map((b) => b.key),
|
|
237
|
-
provides: op.provides.map((b) => b.key),
|
|
238
|
-
action_kind: op.action_kind,
|
|
239
|
-
resource_kind: op.resource_kind,
|
|
240
|
-
};
|
|
241
|
-
|
|
242
|
-
const edges = graph.edges
|
|
243
|
-
.filter((e) => e.from_operation_id === op.operation_id)
|
|
244
|
-
.map((e) => ({ to: e.to_operation_id, binding: e.binding_key }));
|
|
245
|
-
|
|
246
|
-
fetch(`${backendUrl}/v1/graph/edges`, {
|
|
247
|
-
method: "POST",
|
|
248
|
-
headers: {
|
|
249
|
-
"Content-Type": "application/json",
|
|
250
|
-
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
|
|
251
|
-
},
|
|
252
|
-
body: JSON.stringify({ domain, node, edges }),
|
|
253
|
-
}).catch(() => {
|
|
254
|
-
/* fire-and-forget — never block skill publish */
|
|
255
|
-
});
|
|
256
|
-
}
|
|
257
|
-
}
|