unbrowse 2.12.2 → 2.12.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/README.md +8 -44
  2. package/dist/cli.js +514 -20723
  3. package/package.json +4 -10
  4. package/runtime-src/api/routes.ts +15 -801
  5. package/runtime-src/auth/index.ts +32 -142
  6. package/runtime-src/capture/index.ts +101 -436
  7. package/runtime-src/cli.ts +371 -956
  8. package/runtime-src/client/index.ts +29 -622
  9. package/runtime-src/execution/index.ts +85 -345
  10. package/runtime-src/graph/index.ts +10 -128
  11. package/runtime-src/intent-match.ts +27 -27
  12. package/runtime-src/kuri/client.ts +82 -543
  13. package/runtime-src/orchestrator/index.ts +462 -2246
  14. package/runtime-src/reverse-engineer/index.ts +22 -220
  15. package/runtime-src/runtime/local-server.ts +16 -149
  16. package/runtime-src/runtime/paths.ts +5 -9
  17. package/runtime-src/runtime/setup.ts +1 -52
  18. package/runtime-src/server.ts +11 -6
  19. package/runtime-src/transform/schema-hints.ts +358 -0
  20. package/runtime-src/types/skill.ts +2 -49
  21. package/runtime-src/verification/index.ts +0 -15
  22. package/runtime-src/version.ts +13 -13
  23. package/vendor/kuri/darwin-arm64/kuri +0 -0
  24. package/vendor/kuri/darwin-x64/kuri +0 -0
  25. package/vendor/kuri/linux-arm64/kuri +0 -0
  26. package/vendor/kuri/linux-x64/kuri +0 -0
  27. package/bin/unbrowse-wrapper.mjs +0 -39
  28. package/bin/unbrowse.js +0 -38
  29. package/runtime-src/analytics-session.ts +0 -33
  30. package/runtime-src/api/browse-index.ts +0 -254
  31. package/runtime-src/api/browse-session.ts +0 -179
  32. package/runtime-src/api/browse-submit.ts +0 -455
  33. package/runtime-src/auth/runtime.ts +0 -116
  34. package/runtime-src/browser/index.ts +0 -635
  35. package/runtime-src/browser/types.ts +0 -41
  36. package/runtime-src/capture/prefetch.ts +0 -122
  37. package/runtime-src/capture/rsc.ts +0 -45
  38. package/runtime-src/cli/shortcuts.ts +0 -273
  39. package/runtime-src/client/graph-client.ts +0 -99
  40. package/runtime-src/execution/robots.ts +0 -167
  41. package/runtime-src/execution/search-forms.ts +0 -188
  42. package/runtime-src/graph/planner.ts +0 -411
  43. package/runtime-src/graph/session.ts +0 -294
  44. package/runtime-src/graph/trace-store.ts +0 -136
  45. package/runtime-src/indexer/index.ts +0 -480
  46. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  47. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  48. package/runtime-src/orchestrator/dag-feedback.ts +0 -256
  49. package/runtime-src/orchestrator/first-pass-action.ts +0 -362
  50. package/runtime-src/orchestrator/passive-publish.ts +0 -152
  51. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  52. package/runtime-src/payments/cascade.ts +0 -137
  53. package/runtime-src/payments/index.ts +0 -268
  54. package/runtime-src/payments/wallet.ts +0 -33
  55. package/runtime-src/reverse-engineer/description-prompt.ts +0 -132
  56. package/runtime-src/router.ts +0 -17
  57. package/runtime-src/runtime/browser-access.ts +0 -11
  58. package/runtime-src/runtime/browser-host.ts +0 -48
  59. package/runtime-src/runtime/lifecycle.ts +0 -17
  60. package/runtime-src/runtime/supervisor.ts +0 -69
  61. package/runtime-src/single-binary.ts +0 -141
  62. package/runtime-src/telemetry.ts +0 -253
  63. package/runtime-src/verification/matrix.ts +0 -30
  64. package/scripts/postinstall.mjs +0 -81
@@ -6,13 +6,10 @@ import { log } from "../logger.js";
6
6
  import path from "node:path";
7
7
  import os from "node:os";
8
8
  import fs from "node:fs";
9
- import { getDefaultLoginConfig } from "../runtime/supervisor.js";
10
9
 
11
10
  const LOGIN_TIMEOUT_MS = 300_000;
12
11
  const POLL_INTERVAL_MS = 2_000;
13
12
  const MIN_WAIT_MS = 15_000;
14
- const LOGIN_PATHS = /\/(login|signin|sign-in|sso|auth|oauth|uas\/login|checkpoint)/i;
15
- const CLOUDFLARE_TEXT = /just a moment|attention required|verify you are human|cloudflare/i;
16
13
 
17
14
  /**
18
15
  * Returns the persistent profile directory for a given domain.
@@ -29,68 +26,6 @@ export interface LoginResult {
29
26
  error?: string;
30
27
  }
31
28
 
32
- export interface BrowserAuthSourceMeta {
33
- family?: string;
34
- userDataDir?: string;
35
- cookieDbPath?: string;
36
- }
37
-
38
- export interface StoredAuthBundle {
39
- cookies: AuthCookie[];
40
- headers: Record<string, string>;
41
- source_keys: string[];
42
- source_meta?: BrowserAuthSourceMeta | null;
43
- }
44
-
45
- export type InteractiveLoginAssessment =
46
- | { status: "pending"; reason: string }
47
- | { status: "authenticated"; reason: string }
48
- | { status: "blocked"; reason: string };
49
-
50
- export function assessInteractiveLoginState(input: {
51
- currentUrl: string;
52
- targetDomain: string;
53
- initialCookieCount: number;
54
- currentCookieCount: number;
55
- hasCloudflareChallenge?: boolean;
56
- pageText?: string;
57
- }): InteractiveLoginAssessment {
58
- let parsed: URL;
59
- try {
60
- parsed = new URL(input.currentUrl);
61
- } catch {
62
- return { status: "pending", reason: "invalid_url" };
63
- }
64
-
65
- const currentDomain = parsed.hostname.toLowerCase();
66
- const targetNorm = input.targetDomain.toLowerCase();
67
- const isOnTarget = currentDomain === targetNorm || currentDomain.endsWith(`.${targetNorm}`);
68
- if (!isOnTarget) return { status: "pending", reason: "off_target_domain" };
69
-
70
- if (input.hasCloudflareChallenge) return { status: "blocked", reason: "cloudflare_challenge" };
71
- if (input.pageText && CLOUDFLARE_TEXT.test(input.pageText)) return { status: "blocked", reason: "cloudflare_text" };
72
- if (LOGIN_PATHS.test(parsed.pathname)) return { status: "pending", reason: "still_on_login_path" };
73
-
74
- if (input.currentCookieCount > input.initialCookieCount) {
75
- return { status: "authenticated", reason: "new_cookies_on_target" };
76
- }
77
- if (input.currentCookieCount > 0) {
78
- return { status: "authenticated", reason: "cookies_present_on_target" };
79
- }
80
-
81
- return { status: "pending", reason: "no_session_cookies" };
82
- }
83
-
84
- export function storedAuthNeedsBrowserRefresh(bundle: StoredAuthBundle | null | undefined): boolean {
85
- if (!bundle) return true;
86
- if (bundle.cookies.length === 0 && Object.keys(bundle.headers).length === 0) return true;
87
- const sourceMeta = bundle.source_meta;
88
- if (!sourceMeta) return true;
89
- if (sourceMeta.family === "chromium" && !sourceMeta.userDataDir && !sourceMeta.cookieDbPath) {
90
- return true;
91
- }
92
- return false;
93
- }
94
29
  /**
95
30
  * Open a visible browser for the user to complete login.
96
31
  * Uses Kuri to manage the browser tab, polls for login completion via cookies.
@@ -105,20 +40,11 @@ export async function interactiveLogin(
105
40
  const targetDomain = domain ?? new URL(url).hostname;
106
41
  const profileDir = getProfilePath(targetDomain);
107
42
 
108
- const isHeadless = process.env.HEADLESS === "true" || process.env.HEADLESS === "1";
109
- const loginConfig = getDefaultLoginConfig(isHeadless);
110
- log("auth", `interactiveLogin — url: ${url}, domain: ${targetDomain}, interactive: ${loginConfig.interactive}, timeout: ${loginConfig.timeout_ms}ms`);
111
-
112
- // Login requires a visible browser — disable headless for this flow
113
- const prevHeadless = process.env.HEADLESS;
114
- process.env.HEADLESS = "false";
43
+ log("auth", `interactiveLogin url: ${url}, domain: ${targetDomain}`);
115
44
 
116
45
  try {
117
46
  fs.mkdirSync(profileDir, { recursive: true });
118
47
 
119
- // Stop any existing headless Kuri so it restarts with HEADLESS=false
120
- try { await kuri.stop(); } catch { /* may not be running */ }
121
-
122
48
  // Start Kuri and get a tab
123
49
  await kuri.start();
124
50
  const tabId = await kuri.getDefaultTab();
@@ -136,15 +62,16 @@ export async function interactiveLogin(
136
62
 
137
63
  // Wait for user to complete login — detect via cookie changes + URL change
138
64
  let loggedIn = false;
139
- let blockedReason: string | null = null;
140
65
  let lastLoggedUrl = "";
141
- const effectiveTimeout = loginConfig.interactive ? LOGIN_TIMEOUT_MS : loginConfig.timeout_ms;
142
- while (Date.now() - startTime < effectiveTimeout) {
66
+ while (Date.now() - startTime < LOGIN_TIMEOUT_MS) {
143
67
  await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
144
68
  const elapsed = Date.now() - startTime;
145
69
 
146
70
  try {
147
71
  const currentUrl = await kuri.getCurrentUrl(tabId);
72
+ const currentDomain = new URL(currentUrl).hostname.toLowerCase();
73
+ const targetNorm = targetDomain.toLowerCase();
74
+
148
75
  if (currentUrl !== lastLoggedUrl) {
149
76
  log("auth", `navigated to: ${currentUrl}`);
150
77
  lastLoggedUrl = currentUrl;
@@ -152,45 +79,31 @@ export async function interactiveLogin(
152
79
 
153
80
  if (elapsed < MIN_WAIT_MS) continue;
154
81
 
155
- const currentCookies = await kuri.getCookies(tabId);
156
- const currentCookieCount = currentCookies.filter((c) => isDomainMatch(c.domain, targetDomain)).length;
157
- const hasCloudflareChallenge = await kuri.hasCloudflareChallenge(tabId).catch(() => false);
158
- const pageText = hasCloudflareChallenge ? await kuri.getText(tabId).catch(() => "") : "";
159
- const assessment = assessInteractiveLoginState({
160
- currentUrl,
161
- targetDomain,
162
- initialCookieCount,
163
- currentCookieCount,
164
- hasCloudflareChallenge,
165
- pageText,
166
- });
167
-
168
- if (assessment.status === "authenticated") {
169
- loggedIn = true;
170
- log("auth", `login complete — ${currentUrl} (cookies: ${initialCookieCount} → ${currentCookieCount}; ${assessment.reason})`);
171
- break;
172
- }
173
-
174
- if (assessment.status === "blocked") {
175
- blockedReason = assessment.reason;
176
- log("auth", `login blocked — ${currentUrl} (${assessment.reason})`);
82
+ const isOnTarget = currentDomain === targetNorm || currentDomain.endsWith("." + targetNorm);
83
+ if (isOnTarget) {
84
+ const isStillLogin = /\/(login|signin|sign-in|sso|auth|oauth|uas\/login|checkpoint)/.test(new URL(currentUrl).pathname);
85
+
86
+ const currentCookies = await kuri.getCookies(tabId);
87
+ const currentCookieCount = currentCookies.filter((c) => isDomainMatch(c.domain, targetDomain)).length;
88
+ const gotNewCookies = currentCookieCount > initialCookieCount;
89
+
90
+ if (!isStillLogin && gotNewCookies) {
91
+ loggedIn = true;
92
+ log("auth", `login complete — ${currentUrl} (cookies: ${initialCookieCount} → ${currentCookieCount})`);
93
+ break;
94
+ }
95
+
96
+ if (!isStillLogin && currentCookieCount > 0) {
97
+ loggedIn = true;
98
+ log("auth", `already logged in — ${currentUrl} (${currentCookieCount} cookies present)`);
99
+ break;
100
+ }
177
101
  }
178
102
  } catch { /* page navigating */ }
179
103
  }
180
104
 
181
105
  if (!loggedIn) {
182
- log("auth", `login wait ended after ${Math.round((Date.now() - startTime) / 1000)}s — fallback: ${loginConfig.fallback_strategy}`);
183
- if (loginConfig.fallback_strategy === "fail") {
184
- const error = blockedReason
185
- ? `Login blocked (${blockedReason})`
186
- : "Login timed out (fallback: fail)";
187
- return { success: false, domain: targetDomain, cookies_stored: 0, error };
188
- }
189
- if (loginConfig.fallback_strategy === "skip") {
190
- log("auth", `skipping cookie capture per fallback_strategy`);
191
- return { success: false, domain: targetDomain, cookies_stored: 0, error: "Login skipped (headless)" };
192
- }
193
- // fallback_strategy === "prompt" — continue to capture cookies anyway
106
+ log("auth", `login wait ended after ${Math.round((Date.now() - startTime) / 1000)}s — capturing cookies anyway`);
194
107
  }
195
108
 
196
109
  // Extract and store cookies
@@ -210,17 +123,9 @@ export async function interactiveLogin(
210
123
  await storeCredential(vaultKey, JSON.stringify({ cookies: storableCookies }));
211
124
  log("auth", `stored ${storableCookies.length} cookies under ${vaultKey}`);
212
125
 
213
- // Also save as Kuri auth profile so browse commands (go/snap/click) have auth
214
- try {
215
- await kuri.authProfileSave(tabId, targetDomain.replace(/^www\./, ""));
216
- log("auth", `saved Kuri auth profile for ${targetDomain}`);
217
- } catch { /* non-fatal — Kuri auth profile save is best-effort */ }
218
-
219
126
  return { success: true, domain: targetDomain, cookies_stored: storableCookies.length };
220
127
  } finally {
221
- // Restore headless setting so subsequent captures run headless
222
- if (prevHeadless !== undefined) process.env.HEADLESS = prevHeadless;
223
- else delete process.env.HEADLESS;
128
+ // Cleanup handled by Kuri's tab management
224
129
  }
225
130
  }
226
131
 
@@ -293,17 +198,6 @@ function filterExpired(cookies: AuthCookie[]): AuthCookie[] {
293
198
  export async function getStoredAuth(
294
199
  domain: string
295
200
  ): Promise<AuthCookie[] | null> {
296
- const bundle = await getStoredAuthBundle(domain);
297
- return bundle?.cookies?.length ? bundle.cookies : null;
298
- }
299
-
300
- /**
301
- * Retrieve the stored auth bundle for a domain from the vault.
302
- * Preserves headers/source metadata while filtering expired cookies.
303
- */
304
- export async function getStoredAuthBundle(
305
- domain: string
306
- ): Promise<StoredAuthBundle | null> {
307
201
  const regDomain = getRegistrableDomain(domain);
308
202
  const keysToTry = [`auth:${regDomain}`];
309
203
  if (domain !== regDomain) keysToTry.push(`auth:${domain}`);
@@ -312,10 +206,12 @@ export async function getStoredAuthBundle(
312
206
  const stored = await getCredential(key);
313
207
  if (!stored) continue;
314
208
  try {
315
- const parsed = JSON.parse(stored) as Partial<StoredAuthBundle> & { cookies?: AuthCookie[] };
316
- const cookies = parsed.cookies ?? [];
209
+ const parsed = JSON.parse(stored) as { cookies?: AuthCookie[] };
210
+ const cookies = parsed.cookies;
211
+ if (!cookies || cookies.length === 0) continue;
212
+
317
213
  const valid = filterExpired(cookies);
318
- if (cookies.length > 0 && valid.length === 0 && Object.keys(parsed.headers ?? {}).length === 0) {
214
+ if (valid.length === 0) {
319
215
  log("auth", `all ${cookies.length} cookies for ${domain} (key: ${key}) are expired — deleting`);
320
216
  await deleteCredential(key);
321
217
  continue;
@@ -323,17 +219,11 @@ export async function getStoredAuthBundle(
323
219
  if (valid.length < cookies.length) {
324
220
  log("auth", `filtered ${cookies.length - valid.length} expired cookies for ${domain}`);
325
221
  }
326
- return {
327
- cookies: valid,
328
- headers: parsed.headers ?? {},
329
- source_keys: parsed.source_keys ?? [],
330
- source_meta: parsed.source_meta ?? null,
331
- };
222
+ return valid;
332
223
  } catch {
333
224
  continue;
334
225
  }
335
226
  }
336
-
337
227
  return null;
338
228
  }
339
229