unbrowse 2.12.2 → 2.12.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -44
- package/dist/cli.js +514 -20723
- package/package.json +4 -10
- package/runtime-src/api/routes.ts +15 -801
- package/runtime-src/auth/index.ts +32 -142
- package/runtime-src/capture/index.ts +101 -436
- package/runtime-src/cli.ts +371 -956
- package/runtime-src/client/index.ts +29 -622
- package/runtime-src/execution/index.ts +85 -345
- package/runtime-src/graph/index.ts +10 -128
- package/runtime-src/intent-match.ts +27 -27
- package/runtime-src/kuri/client.ts +82 -543
- package/runtime-src/orchestrator/index.ts +462 -2246
- package/runtime-src/reverse-engineer/index.ts +22 -220
- package/runtime-src/runtime/local-server.ts +16 -149
- package/runtime-src/runtime/paths.ts +5 -9
- package/runtime-src/runtime/setup.ts +1 -52
- package/runtime-src/server.ts +11 -6
- package/runtime-src/transform/schema-hints.ts +358 -0
- package/runtime-src/types/skill.ts +2 -49
- package/runtime-src/verification/index.ts +0 -15
- package/runtime-src/version.ts +13 -13
- package/vendor/kuri/darwin-arm64/kuri +0 -0
- package/vendor/kuri/darwin-x64/kuri +0 -0
- package/vendor/kuri/linux-arm64/kuri +0 -0
- package/vendor/kuri/linux-x64/kuri +0 -0
- package/bin/unbrowse-wrapper.mjs +0 -39
- package/bin/unbrowse.js +0 -38
- package/runtime-src/analytics-session.ts +0 -33
- package/runtime-src/api/browse-index.ts +0 -254
- package/runtime-src/api/browse-session.ts +0 -179
- package/runtime-src/api/browse-submit.ts +0 -455
- package/runtime-src/auth/runtime.ts +0 -116
- package/runtime-src/browser/index.ts +0 -635
- package/runtime-src/browser/types.ts +0 -41
- package/runtime-src/capture/prefetch.ts +0 -122
- package/runtime-src/capture/rsc.ts +0 -45
- package/runtime-src/cli/shortcuts.ts +0 -273
- package/runtime-src/client/graph-client.ts +0 -99
- package/runtime-src/execution/robots.ts +0 -167
- package/runtime-src/execution/search-forms.ts +0 -188
- package/runtime-src/graph/planner.ts +0 -411
- package/runtime-src/graph/session.ts +0 -294
- package/runtime-src/graph/trace-store.ts +0 -136
- package/runtime-src/indexer/index.ts +0 -480
- package/runtime-src/orchestrator/browser-agent.ts +0 -374
- package/runtime-src/orchestrator/dag-advisor.ts +0 -59
- package/runtime-src/orchestrator/dag-feedback.ts +0 -256
- package/runtime-src/orchestrator/first-pass-action.ts +0 -362
- package/runtime-src/orchestrator/passive-publish.ts +0 -152
- package/runtime-src/orchestrator/timing-economics.ts +0 -80
- package/runtime-src/payments/cascade.ts +0 -137
- package/runtime-src/payments/index.ts +0 -268
- package/runtime-src/payments/wallet.ts +0 -33
- package/runtime-src/reverse-engineer/description-prompt.ts +0 -132
- package/runtime-src/router.ts +0 -17
- package/runtime-src/runtime/browser-access.ts +0 -11
- package/runtime-src/runtime/browser-host.ts +0 -48
- package/runtime-src/runtime/lifecycle.ts +0 -17
- package/runtime-src/runtime/supervisor.ts +0 -69
- package/runtime-src/single-binary.ts +0 -141
- package/runtime-src/telemetry.ts +0 -253
- package/runtime-src/verification/matrix.ts +0 -30
- package/scripts/postinstall.mjs +0 -81
|
@@ -6,13 +6,10 @@ import { log } from "../logger.js";
|
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import os from "node:os";
|
|
8
8
|
import fs from "node:fs";
|
|
9
|
-
import { getDefaultLoginConfig } from "../runtime/supervisor.js";
|
|
10
9
|
|
|
11
10
|
const LOGIN_TIMEOUT_MS = 300_000;
|
|
12
11
|
const POLL_INTERVAL_MS = 2_000;
|
|
13
12
|
const MIN_WAIT_MS = 15_000;
|
|
14
|
-
const LOGIN_PATHS = /\/(login|signin|sign-in|sso|auth|oauth|uas\/login|checkpoint)/i;
|
|
15
|
-
const CLOUDFLARE_TEXT = /just a moment|attention required|verify you are human|cloudflare/i;
|
|
16
13
|
|
|
17
14
|
/**
|
|
18
15
|
* Returns the persistent profile directory for a given domain.
|
|
@@ -29,68 +26,6 @@ export interface LoginResult {
|
|
|
29
26
|
error?: string;
|
|
30
27
|
}
|
|
31
28
|
|
|
32
|
-
export interface BrowserAuthSourceMeta {
|
|
33
|
-
family?: string;
|
|
34
|
-
userDataDir?: string;
|
|
35
|
-
cookieDbPath?: string;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export interface StoredAuthBundle {
|
|
39
|
-
cookies: AuthCookie[];
|
|
40
|
-
headers: Record<string, string>;
|
|
41
|
-
source_keys: string[];
|
|
42
|
-
source_meta?: BrowserAuthSourceMeta | null;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
export type InteractiveLoginAssessment =
|
|
46
|
-
| { status: "pending"; reason: string }
|
|
47
|
-
| { status: "authenticated"; reason: string }
|
|
48
|
-
| { status: "blocked"; reason: string };
|
|
49
|
-
|
|
50
|
-
export function assessInteractiveLoginState(input: {
|
|
51
|
-
currentUrl: string;
|
|
52
|
-
targetDomain: string;
|
|
53
|
-
initialCookieCount: number;
|
|
54
|
-
currentCookieCount: number;
|
|
55
|
-
hasCloudflareChallenge?: boolean;
|
|
56
|
-
pageText?: string;
|
|
57
|
-
}): InteractiveLoginAssessment {
|
|
58
|
-
let parsed: URL;
|
|
59
|
-
try {
|
|
60
|
-
parsed = new URL(input.currentUrl);
|
|
61
|
-
} catch {
|
|
62
|
-
return { status: "pending", reason: "invalid_url" };
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
const currentDomain = parsed.hostname.toLowerCase();
|
|
66
|
-
const targetNorm = input.targetDomain.toLowerCase();
|
|
67
|
-
const isOnTarget = currentDomain === targetNorm || currentDomain.endsWith(`.${targetNorm}`);
|
|
68
|
-
if (!isOnTarget) return { status: "pending", reason: "off_target_domain" };
|
|
69
|
-
|
|
70
|
-
if (input.hasCloudflareChallenge) return { status: "blocked", reason: "cloudflare_challenge" };
|
|
71
|
-
if (input.pageText && CLOUDFLARE_TEXT.test(input.pageText)) return { status: "blocked", reason: "cloudflare_text" };
|
|
72
|
-
if (LOGIN_PATHS.test(parsed.pathname)) return { status: "pending", reason: "still_on_login_path" };
|
|
73
|
-
|
|
74
|
-
if (input.currentCookieCount > input.initialCookieCount) {
|
|
75
|
-
return { status: "authenticated", reason: "new_cookies_on_target" };
|
|
76
|
-
}
|
|
77
|
-
if (input.currentCookieCount > 0) {
|
|
78
|
-
return { status: "authenticated", reason: "cookies_present_on_target" };
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
return { status: "pending", reason: "no_session_cookies" };
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
export function storedAuthNeedsBrowserRefresh(bundle: StoredAuthBundle | null | undefined): boolean {
|
|
85
|
-
if (!bundle) return true;
|
|
86
|
-
if (bundle.cookies.length === 0 && Object.keys(bundle.headers).length === 0) return true;
|
|
87
|
-
const sourceMeta = bundle.source_meta;
|
|
88
|
-
if (!sourceMeta) return true;
|
|
89
|
-
if (sourceMeta.family === "chromium" && !sourceMeta.userDataDir && !sourceMeta.cookieDbPath) {
|
|
90
|
-
return true;
|
|
91
|
-
}
|
|
92
|
-
return false;
|
|
93
|
-
}
|
|
94
29
|
/**
|
|
95
30
|
* Open a visible browser for the user to complete login.
|
|
96
31
|
* Uses Kuri to manage the browser tab, polls for login completion via cookies.
|
|
@@ -105,20 +40,11 @@ export async function interactiveLogin(
|
|
|
105
40
|
const targetDomain = domain ?? new URL(url).hostname;
|
|
106
41
|
const profileDir = getProfilePath(targetDomain);
|
|
107
42
|
|
|
108
|
-
|
|
109
|
-
const loginConfig = getDefaultLoginConfig(isHeadless);
|
|
110
|
-
log("auth", `interactiveLogin — url: ${url}, domain: ${targetDomain}, interactive: ${loginConfig.interactive}, timeout: ${loginConfig.timeout_ms}ms`);
|
|
111
|
-
|
|
112
|
-
// Login requires a visible browser — disable headless for this flow
|
|
113
|
-
const prevHeadless = process.env.HEADLESS;
|
|
114
|
-
process.env.HEADLESS = "false";
|
|
43
|
+
log("auth", `interactiveLogin — url: ${url}, domain: ${targetDomain}`);
|
|
115
44
|
|
|
116
45
|
try {
|
|
117
46
|
fs.mkdirSync(profileDir, { recursive: true });
|
|
118
47
|
|
|
119
|
-
// Stop any existing headless Kuri so it restarts with HEADLESS=false
|
|
120
|
-
try { await kuri.stop(); } catch { /* may not be running */ }
|
|
121
|
-
|
|
122
48
|
// Start Kuri and get a tab
|
|
123
49
|
await kuri.start();
|
|
124
50
|
const tabId = await kuri.getDefaultTab();
|
|
@@ -136,15 +62,16 @@ export async function interactiveLogin(
|
|
|
136
62
|
|
|
137
63
|
// Wait for user to complete login — detect via cookie changes + URL change
|
|
138
64
|
let loggedIn = false;
|
|
139
|
-
let blockedReason: string | null = null;
|
|
140
65
|
let lastLoggedUrl = "";
|
|
141
|
-
|
|
142
|
-
while (Date.now() - startTime < effectiveTimeout) {
|
|
66
|
+
while (Date.now() - startTime < LOGIN_TIMEOUT_MS) {
|
|
143
67
|
await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
|
|
144
68
|
const elapsed = Date.now() - startTime;
|
|
145
69
|
|
|
146
70
|
try {
|
|
147
71
|
const currentUrl = await kuri.getCurrentUrl(tabId);
|
|
72
|
+
const currentDomain = new URL(currentUrl).hostname.toLowerCase();
|
|
73
|
+
const targetNorm = targetDomain.toLowerCase();
|
|
74
|
+
|
|
148
75
|
if (currentUrl !== lastLoggedUrl) {
|
|
149
76
|
log("auth", `navigated to: ${currentUrl}`);
|
|
150
77
|
lastLoggedUrl = currentUrl;
|
|
@@ -152,45 +79,31 @@ export async function interactiveLogin(
|
|
|
152
79
|
|
|
153
80
|
if (elapsed < MIN_WAIT_MS) continue;
|
|
154
81
|
|
|
155
|
-
const
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
if (assessment.status === "blocked") {
|
|
175
|
-
blockedReason = assessment.reason;
|
|
176
|
-
log("auth", `login blocked — ${currentUrl} (${assessment.reason})`);
|
|
82
|
+
const isOnTarget = currentDomain === targetNorm || currentDomain.endsWith("." + targetNorm);
|
|
83
|
+
if (isOnTarget) {
|
|
84
|
+
const isStillLogin = /\/(login|signin|sign-in|sso|auth|oauth|uas\/login|checkpoint)/.test(new URL(currentUrl).pathname);
|
|
85
|
+
|
|
86
|
+
const currentCookies = await kuri.getCookies(tabId);
|
|
87
|
+
const currentCookieCount = currentCookies.filter((c) => isDomainMatch(c.domain, targetDomain)).length;
|
|
88
|
+
const gotNewCookies = currentCookieCount > initialCookieCount;
|
|
89
|
+
|
|
90
|
+
if (!isStillLogin && gotNewCookies) {
|
|
91
|
+
loggedIn = true;
|
|
92
|
+
log("auth", `login complete — ${currentUrl} (cookies: ${initialCookieCount} → ${currentCookieCount})`);
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (!isStillLogin && currentCookieCount > 0) {
|
|
97
|
+
loggedIn = true;
|
|
98
|
+
log("auth", `already logged in — ${currentUrl} (${currentCookieCount} cookies present)`);
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
177
101
|
}
|
|
178
102
|
} catch { /* page navigating */ }
|
|
179
103
|
}
|
|
180
104
|
|
|
181
105
|
if (!loggedIn) {
|
|
182
|
-
log("auth", `login wait ended after ${Math.round((Date.now() - startTime) / 1000)}s —
|
|
183
|
-
if (loginConfig.fallback_strategy === "fail") {
|
|
184
|
-
const error = blockedReason
|
|
185
|
-
? `Login blocked (${blockedReason})`
|
|
186
|
-
: "Login timed out (fallback: fail)";
|
|
187
|
-
return { success: false, domain: targetDomain, cookies_stored: 0, error };
|
|
188
|
-
}
|
|
189
|
-
if (loginConfig.fallback_strategy === "skip") {
|
|
190
|
-
log("auth", `skipping cookie capture per fallback_strategy`);
|
|
191
|
-
return { success: false, domain: targetDomain, cookies_stored: 0, error: "Login skipped (headless)" };
|
|
192
|
-
}
|
|
193
|
-
// fallback_strategy === "prompt" — continue to capture cookies anyway
|
|
106
|
+
log("auth", `login wait ended after ${Math.round((Date.now() - startTime) / 1000)}s — capturing cookies anyway`);
|
|
194
107
|
}
|
|
195
108
|
|
|
196
109
|
// Extract and store cookies
|
|
@@ -210,17 +123,9 @@ export async function interactiveLogin(
|
|
|
210
123
|
await storeCredential(vaultKey, JSON.stringify({ cookies: storableCookies }));
|
|
211
124
|
log("auth", `stored ${storableCookies.length} cookies under ${vaultKey}`);
|
|
212
125
|
|
|
213
|
-
// Also save as Kuri auth profile so browse commands (go/snap/click) have auth
|
|
214
|
-
try {
|
|
215
|
-
await kuri.authProfileSave(tabId, targetDomain.replace(/^www\./, ""));
|
|
216
|
-
log("auth", `saved Kuri auth profile for ${targetDomain}`);
|
|
217
|
-
} catch { /* non-fatal — Kuri auth profile save is best-effort */ }
|
|
218
|
-
|
|
219
126
|
return { success: true, domain: targetDomain, cookies_stored: storableCookies.length };
|
|
220
127
|
} finally {
|
|
221
|
-
//
|
|
222
|
-
if (prevHeadless !== undefined) process.env.HEADLESS = prevHeadless;
|
|
223
|
-
else delete process.env.HEADLESS;
|
|
128
|
+
// Cleanup handled by Kuri's tab management
|
|
224
129
|
}
|
|
225
130
|
}
|
|
226
131
|
|
|
@@ -293,17 +198,6 @@ function filterExpired(cookies: AuthCookie[]): AuthCookie[] {
|
|
|
293
198
|
export async function getStoredAuth(
|
|
294
199
|
domain: string
|
|
295
200
|
): Promise<AuthCookie[] | null> {
|
|
296
|
-
const bundle = await getStoredAuthBundle(domain);
|
|
297
|
-
return bundle?.cookies?.length ? bundle.cookies : null;
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
/**
|
|
301
|
-
* Retrieve the stored auth bundle for a domain from the vault.
|
|
302
|
-
* Preserves headers/source metadata while filtering expired cookies.
|
|
303
|
-
*/
|
|
304
|
-
export async function getStoredAuthBundle(
|
|
305
|
-
domain: string
|
|
306
|
-
): Promise<StoredAuthBundle | null> {
|
|
307
201
|
const regDomain = getRegistrableDomain(domain);
|
|
308
202
|
const keysToTry = [`auth:${regDomain}`];
|
|
309
203
|
if (domain !== regDomain) keysToTry.push(`auth:${domain}`);
|
|
@@ -312,10 +206,12 @@ export async function getStoredAuthBundle(
|
|
|
312
206
|
const stored = await getCredential(key);
|
|
313
207
|
if (!stored) continue;
|
|
314
208
|
try {
|
|
315
|
-
const parsed = JSON.parse(stored) as
|
|
316
|
-
const cookies = parsed.cookies
|
|
209
|
+
const parsed = JSON.parse(stored) as { cookies?: AuthCookie[] };
|
|
210
|
+
const cookies = parsed.cookies;
|
|
211
|
+
if (!cookies || cookies.length === 0) continue;
|
|
212
|
+
|
|
317
213
|
const valid = filterExpired(cookies);
|
|
318
|
-
if (
|
|
214
|
+
if (valid.length === 0) {
|
|
319
215
|
log("auth", `all ${cookies.length} cookies for ${domain} (key: ${key}) are expired — deleting`);
|
|
320
216
|
await deleteCredential(key);
|
|
321
217
|
continue;
|
|
@@ -323,17 +219,11 @@ export async function getStoredAuthBundle(
|
|
|
323
219
|
if (valid.length < cookies.length) {
|
|
324
220
|
log("auth", `filtered ${cookies.length - valid.length} expired cookies for ${domain}`);
|
|
325
221
|
}
|
|
326
|
-
return
|
|
327
|
-
cookies: valid,
|
|
328
|
-
headers: parsed.headers ?? {},
|
|
329
|
-
source_keys: parsed.source_keys ?? [],
|
|
330
|
-
source_meta: parsed.source_meta ?? null,
|
|
331
|
-
};
|
|
222
|
+
return valid;
|
|
332
223
|
} catch {
|
|
333
224
|
continue;
|
|
334
225
|
}
|
|
335
226
|
}
|
|
336
|
-
|
|
337
227
|
return null;
|
|
338
228
|
}
|
|
339
229
|
|