unbrowse 3.1.0 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/cli.js +455 -96
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +695 -46
  4. package/dist/server.js +25811 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/darwin-arm64/kuri +0 -0
  7. package/vendor/kuri/darwin-x64/kuri +0 -0
  8. package/vendor/kuri/linux-arm64/kuri +0 -0
  9. package/vendor/kuri/linux-x64/kuri +0 -0
  10. package/vendor/kuri/manifest.json +7 -10
  11. package/runtime-src/agent-outcome.ts +0 -166
  12. package/runtime-src/analytics-session.ts +0 -55
  13. package/runtime-src/api/browse-index.ts +0 -317
  14. package/runtime-src/api/browse-session.ts +0 -572
  15. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  16. package/runtime-src/api/browse-submit.ts +0 -1184
  17. package/runtime-src/api/routes.ts +0 -1823
  18. package/runtime-src/auth/browser-cookies.ts +0 -423
  19. package/runtime-src/auth/index.ts +0 -535
  20. package/runtime-src/auth/runtime.ts +0 -116
  21. package/runtime-src/browser/index.ts +0 -659
  22. package/runtime-src/browser/types.ts +0 -41
  23. package/runtime-src/build-info.generated.ts +0 -6
  24. package/runtime-src/capture/index.ts +0 -1794
  25. package/runtime-src/capture/prefetch.ts +0 -95
  26. package/runtime-src/capture/rsc.ts +0 -45
  27. package/runtime-src/cli/shortcuts.ts +0 -273
  28. package/runtime-src/cli.ts +0 -1572
  29. package/runtime-src/client/graph-client.ts +0 -100
  30. package/runtime-src/client/index.ts +0 -1425
  31. package/runtime-src/debug-trace.ts +0 -18
  32. package/runtime-src/domain.ts +0 -38
  33. package/runtime-src/execution/index.ts +0 -3397
  34. package/runtime-src/execution/retry.ts +0 -46
  35. package/runtime-src/execution/robots.ts +0 -167
  36. package/runtime-src/execution/search-forms.ts +0 -188
  37. package/runtime-src/extraction/index.ts +0 -1507
  38. package/runtime-src/foundry/publish-bundle.ts +0 -392
  39. package/runtime-src/graph/agent-augment.ts +0 -315
  40. package/runtime-src/graph/index.ts +0 -1524
  41. package/runtime-src/graph/local-fixtures.ts +0 -393
  42. package/runtime-src/graph/local-harness.ts +0 -646
  43. package/runtime-src/graph/planner.ts +0 -411
  44. package/runtime-src/graph/session.ts +0 -294
  45. package/runtime-src/graph/trace-store.ts +0 -136
  46. package/runtime-src/index.ts +0 -24
  47. package/runtime-src/indexer/index.ts +0 -465
  48. package/runtime-src/intent-match.ts +0 -1515
  49. package/runtime-src/kuri/client.ts +0 -1839
  50. package/runtime-src/logger.ts +0 -30
  51. package/runtime-src/marketplace/index.ts +0 -103
  52. package/runtime-src/mcp.ts +0 -1747
  53. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  54. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  55. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  56. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  57. package/runtime-src/orchestrator/index.ts +0 -4480
  58. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  59. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  60. package/runtime-src/payments/cascade.ts +0 -137
  61. package/runtime-src/payments/index.ts +0 -270
  62. package/runtime-src/payments/lobster-pay.ts +0 -182
  63. package/runtime-src/payments/wallet.ts +0 -98
  64. package/runtime-src/publish/review-context.ts +0 -93
  65. package/runtime-src/publish/sanitize.ts +0 -197
  66. package/runtime-src/publish/schema-review.ts +0 -192
  67. package/runtime-src/publish-admission.ts +0 -388
  68. package/runtime-src/ratelimit/index.ts +0 -23
  69. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  70. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  71. package/runtime-src/reverse-engineer/index.ts +0 -1551
  72. package/runtime-src/router.ts +0 -17
  73. package/runtime-src/routing-telemetry.ts +0 -395
  74. package/runtime-src/runtime/browser-access.ts +0 -11
  75. package/runtime-src/runtime/browser-auth.ts +0 -12
  76. package/runtime-src/runtime/browser-host.ts +0 -48
  77. package/runtime-src/runtime/lifecycle.ts +0 -17
  78. package/runtime-src/runtime/local-server.ts +0 -311
  79. package/runtime-src/runtime/paths.ts +0 -99
  80. package/runtime-src/runtime/setup.ts +0 -251
  81. package/runtime-src/runtime/supervisor.ts +0 -69
  82. package/runtime-src/runtime/update-hints.ts +0 -351
  83. package/runtime-src/server.ts +0 -100
  84. package/runtime-src/session-logs.ts +0 -142
  85. package/runtime-src/settings.ts +0 -221
  86. package/runtime-src/single-binary.ts +0 -143
  87. package/runtime-src/site-policy.ts +0 -54
  88. package/runtime-src/stale-cleanup-runner.ts +0 -144
  89. package/runtime-src/stale-cleanup.ts +0 -133
  90. package/runtime-src/telemetry-attribution.ts +0 -120
  91. package/runtime-src/telemetry.ts +0 -253
  92. package/runtime-src/template-params.ts +0 -141
  93. package/runtime-src/transform/drift.ts +0 -60
  94. package/runtime-src/transform/index.ts +0 -277
  95. package/runtime-src/types/index.ts +0 -1
  96. package/runtime-src/types/skill.ts +0 -912
  97. package/runtime-src/vault/index.ts +0 -196
  98. package/runtime-src/verification/auth-gate.ts +0 -8
  99. package/runtime-src/verification/candidates.ts +0 -27
  100. package/runtime-src/verification/index.ts +0 -120
  101. package/runtime-src/verification/matrix.ts +0 -30
  102. package/runtime-src/version.ts +0 -148
  103. package/runtime-src/workflow/artifact.ts +0 -161
  104. package/runtime-src/workflow/compile.ts +0 -808
  105. package/runtime-src/workflow/publish.ts +0 -225
  106. package/runtime-src/workflow/runtime.ts +0 -213
  107. package/vendor/kuri/win-x64/kuri.exe +0 -0
@@ -1,423 +0,0 @@
1
- /**
2
- * Extract cookies directly from Chrome/Firefox SQLite databases.
3
- * Adapted from github.com/jawond/bird — generalized for any domain.
4
- *
5
- * Chrome cookies are AES-128-CBC encrypted with a key from the macOS keychain.
6
- * Firefox cookies are stored unencrypted.
7
- *
8
- * This avoids needing to launch a browser or close Chrome (reads a copy of the DB).
9
- */
10
-
11
- import { execSync, execFileSync } from "node:child_process";
12
- import { createDecipheriv, pbkdf2Sync } from "node:crypto";
13
- import { copyFileSync, existsSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
14
- import { tmpdir, homedir, platform } from "node:os";
15
- import { join } from "node:path";
16
- import { log } from "../logger.js";
17
- import { getRegistrableDomain, isDomainMatch } from "../domain.js";
18
-
19
- export interface BrowserCookie {
20
- name: string;
21
- value: string;
22
- domain: string;
23
- path: string;
24
- secure: boolean;
25
- httpOnly: boolean;
26
- sameSite: string;
27
- expires: number;
28
- }
29
-
30
- export interface ExtractionResult {
31
- cookies: BrowserCookie[];
32
- source: string | null;
33
- warnings: string[];
34
- }
35
-
36
- export type BrowserSource = "auto" | "firefox" | "chrome" | "chromium";
37
-
38
- export interface ChromiumCookieSourceOptions {
39
- profile?: string;
40
- userDataDir?: string;
41
- cookieDbPath?: string;
42
- safeStorageService?: string;
43
- browserName?: string;
44
- }
45
-
46
- export interface ExtractBrowserCookiesOptions {
47
- browser?: BrowserSource;
48
- chromeProfile?: string;
49
- firefoxProfile?: string;
50
- chromium?: ChromiumCookieSourceOptions;
51
- }
52
-
53
- // ---------------------------------------------------------------------------
54
- // Path helpers
55
- // ---------------------------------------------------------------------------
56
-
57
- function getChromeUserDataDir(): string {
58
- const home = homedir();
59
- if (platform() === "darwin") {
60
- return join(home, "Library", "Application Support", "Google", "Chrome");
61
- }
62
- if (platform() === "win32") {
63
- const appData = process.env.LOCALAPPDATA ?? join(home, "AppData", "Local");
64
- return join(appData, "Google", "Chrome", "User Data");
65
- }
66
- return join(home, ".config", "google-chrome");
67
- }
68
-
69
- export function resolveChromiumCookiesPath(opts?: ChromiumCookieSourceOptions): string | null {
70
- if (opts?.cookieDbPath) {
71
- return opts.cookieDbPath.replace(/^~\//, homedir() + "/");
72
- }
73
-
74
- const profileDir = opts?.profile || "Default";
75
- const userDataDir = (opts?.userDataDir || getChromeUserDataDir()).replace(/^~\//, homedir() + "/");
76
- const candidates = [
77
- join(userDataDir, profileDir, "Network", "Cookies"),
78
- join(userDataDir, profileDir, "Cookies"),
79
- join(userDataDir, "Network", "Cookies"),
80
- join(userDataDir, "Cookies"),
81
- ];
82
-
83
- return candidates.find((candidate) => existsSync(candidate)) ?? candidates[0] ?? null;
84
- }
85
-
86
- function getFirefoxProfilesRoot(): string | null {
87
- const home = homedir();
88
- if (platform() === "darwin") {
89
- return join(home, "Library", "Application Support", "Firefox", "Profiles");
90
- }
91
- if (platform() === "linux") {
92
- return join(home, ".mozilla", "firefox");
93
- }
94
- if (platform() === "win32") {
95
- const appData = process.env.APPDATA;
96
- if (!appData) return null;
97
- return join(appData, "Mozilla", "Firefox", "Profiles");
98
- }
99
- return null;
100
- }
101
-
102
- function pickFirefoxProfile(profilesRoot: string, profile?: string): string | null {
103
- if (profile) {
104
- const candidate = join(profilesRoot, profile, "cookies.sqlite");
105
- return existsSync(candidate) ? candidate : null;
106
- }
107
- const entries = readdirSync(profilesRoot, { withFileTypes: true });
108
- const defaultRelease = entries.find((e) => e.isDirectory() && e.name.includes("default-release"));
109
- const targetDir = defaultRelease?.name ?? entries.find((e) => e.isDirectory())?.name;
110
- if (!targetDir) return null;
111
- const candidate = join(profilesRoot, targetDir, "cookies.sqlite");
112
- return existsSync(candidate) ? candidate : null;
113
- }
114
-
115
- function getFirefoxCookiesPath(profile?: string): string | null {
116
- const profilesRoot = getFirefoxProfilesRoot();
117
- if (!profilesRoot || !existsSync(profilesRoot)) return null;
118
- return pickFirefoxProfile(profilesRoot, profile);
119
- }
120
-
121
- // ---------------------------------------------------------------------------
122
- // Chrome decryption (macOS — uses keychain + PBKDF2 + AES-128-CBC)
123
- // ---------------------------------------------------------------------------
124
-
125
- const _chromiumKeyCache = new Map<string, Buffer>();
126
-
127
- function getChromiumKeychainServiceName(opts?: ChromiumCookieSourceOptions): string {
128
- if (opts?.safeStorageService) return opts.safeStorageService;
129
- return `${opts?.browserName || "Chrome"} Safe Storage`;
130
- }
131
-
132
- function getChromiumDecryptionKey(opts?: ChromiumCookieSourceOptions): Buffer | null {
133
- const service = getChromiumKeychainServiceName(opts);
134
- const cached = _chromiumKeyCache.get(service);
135
- if (cached) return cached;
136
- if (platform() !== "darwin") return null; // TODO: Linux/Windows support
137
-
138
- try {
139
- const keyOutput = execFileSync(
140
- "security",
141
- ["find-generic-password", "-s", service, "-w"],
142
- { encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] },
143
- ).trim();
144
- if (!keyOutput) return null;
145
-
146
- const derived = pbkdf2Sync(keyOutput, "saltysalt", 1003, 16, "sha1");
147
- _chromiumKeyCache.set(service, derived);
148
- return derived;
149
- } catch {
150
- return null;
151
- }
152
- }
153
-
154
- function decryptChromiumValue(encryptedHex: string, opts?: ChromiumCookieSourceOptions): string | null {
155
- try {
156
- const buf = Buffer.from(encryptedHex, "hex");
157
- if (buf.length < 4) return null;
158
-
159
- const version = buf.subarray(0, 3).toString("utf8");
160
- if (version !== "v10" && version !== "v11") {
161
- // Not encrypted
162
- return buf.toString("utf8");
163
- }
164
-
165
- const key = getChromiumDecryptionKey(opts);
166
- if (!key) return null;
167
-
168
- const payload = buf.subarray(3);
169
-
170
- // Modern Chrome (v131+) prepends a 32-byte header (key derivation nonce)
171
- // before the actual AES-128-CBC ciphertext. The second 16-byte block of
172
- // the raw payload acts as the CBC IV for the remaining ciphertext.
173
- // Fallback: legacy format has no header (IV = 16 × 0x20 space bytes).
174
- if (payload.length >= 48) {
175
- try {
176
- const iv = payload.subarray(16, 32);
177
- const encrypted = payload.subarray(32);
178
- const decipher = createDecipheriv("aes-128-cbc", key, iv);
179
- decipher.setAutoPadding(true);
180
- const decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]);
181
- const val = decrypted.toString("utf8").replace(/[^\x20-\x7E]/g, "");
182
- if (val.length > 0) return val;
183
- } catch { /* fall through to legacy */ }
184
- }
185
-
186
- // Legacy format: IV = 16 bytes of space, ciphertext starts right after version
187
- const iv = Buffer.alloc(16, 0x20);
188
- const decipher = createDecipheriv("aes-128-cbc", key, iv);
189
- decipher.setAutoPadding(true);
190
- const decrypted = Buffer.concat([decipher.update(payload), decipher.final()]);
191
- return decrypted.toString("utf8").replace(/[^\x20-\x7E]/g, "");
192
- } catch {
193
- return null;
194
- }
195
- }
196
-
197
- export function decodeChromiumCookieValue(rawValue: string, encryptedHex: string, opts?: ChromiumCookieSourceOptions): string | null {
198
- if (rawValue) return rawValue;
199
- if (!encryptedHex) return null;
200
- return decryptChromiumValue(encryptedHex, opts);
201
- }
202
-
203
- // ---------------------------------------------------------------------------
204
- // SQLite helpers — copy DB to temp dir, query, cleanup
205
- // ---------------------------------------------------------------------------
206
-
207
- function withTempCopy<T>(dbPath: string, fn: (tempPath: string) => T): T {
208
- const tempDir = mkdtempSync(join(tmpdir(), "unbrowse-cookies-"));
209
- const tempDb = join(tempDir, "cookies.db");
210
- try {
211
- copyFileSync(dbPath, tempDb);
212
- // Copy WAL/SHM so we get the latest committed state even while Chrome is open
213
- for (const ext of ["-wal", "-shm"]) {
214
- const src = dbPath + ext;
215
- if (existsSync(src)) copyFileSync(src, tempDb + ext);
216
- }
217
- return fn(tempDb);
218
- } finally {
219
- try { rmSync(tempDir, { recursive: true, force: true }); } catch { /* ignore */ }
220
- }
221
- }
222
-
223
- function sqliteQuery(dbPath: string, sql: string): string {
224
- return execFileSync("sqlite3", ["-separator", "|", dbPath, sql], {
225
- encoding: "utf8",
226
- maxBuffer: 4 * 1024 * 1024,
227
- }).trim();
228
- }
229
-
230
- // ---------------------------------------------------------------------------
231
- // Domain matching helpers for SQL WHERE clauses
232
- // ---------------------------------------------------------------------------
233
-
234
- function buildDomainWhereClause(domain: string, column: string): string {
235
- const reg = getRegistrableDomain(domain);
236
- // Match exact domains: .example.com, example.com, plus common subdomains
237
- const variants = new Set([
238
- reg,
239
- `.${reg}`,
240
- domain,
241
- `.${domain}`,
242
- `www.${reg}`,
243
- `.www.${reg}`,
244
- ]);
245
- // Use parameterized-safe quoting: reject any domain containing single quotes
246
- for (const d of variants) {
247
- if (d.includes("'")) throw new Error(`Invalid domain for cookie query: ${d}`);
248
- }
249
- const escaped = [...variants].map((d) => `'${d}'`);
250
- const likeReg = reg.includes("'") ? reg : reg;
251
- const likePattern = `'%.${likeReg}'`;
252
- return `(${column} IN (${escaped.join(", ")}) OR ${column} LIKE ${likePattern})`;
253
- }
254
-
255
- // ---------------------------------------------------------------------------
256
- // Chrome extraction
257
- // ---------------------------------------------------------------------------
258
-
259
- export function extractFromChrome(
260
- domain: string,
261
- opts?: { profile?: string },
262
- ): ExtractionResult {
263
- return extractFromChromium(domain, {
264
- profile: opts?.profile,
265
- browserName: "Chrome",
266
- });
267
- }
268
-
269
- export function extractFromChromium(
270
- domain: string,
271
- opts?: ChromiumCookieSourceOptions,
272
- ): ExtractionResult {
273
- const warnings: string[] = [];
274
- const dbPath = resolveChromiumCookiesPath(opts);
275
- const sourceLabel = opts?.browserName || "Chromium";
276
-
277
- if (!dbPath || !existsSync(dbPath)) {
278
- warnings.push(`${sourceLabel} cookies DB not found${dbPath ? ` at ${dbPath}` : ""}`);
279
- return { cookies: [], source: null, warnings };
280
- }
281
-
282
- try {
283
- const cookies = withTempCopy(dbPath, (tempDb) => {
284
- const where = buildDomainWhereClause(domain, "host_key");
285
- const sql = `SELECT name, value, hex(encrypted_value) as ev, host_key, path, is_secure, is_httponly, samesite, expires_utc FROM cookies WHERE ${where};`;
286
- const rows = sqliteQuery(tempDb, sql);
287
- if (!rows) return [];
288
-
289
- const results: BrowserCookie[] = [];
290
- for (const line of rows.split("\n")) {
291
- const parts = line.split("|");
292
- if (parts.length < 9) continue;
293
- const [name, rawValue, encHex, host, cookiePath, secure, httpOnly, sameSite, expiresUtc] = parts;
294
- const value = decodeChromiumCookieValue(rawValue, encHex, opts);
295
- if (!value) continue;
296
-
297
- results.push({
298
- name,
299
- value,
300
- domain: host,
301
- path: cookiePath || "/",
302
- secure: secure === "1",
303
- httpOnly: httpOnly === "1",
304
- sameSite: sameSite === "0" ? "None" : sameSite === "1" ? "Lax" : "Strict",
305
- // Chrome stores expiry as microseconds since 1601-01-01
306
- expires: expiresUtc === "0" ? -1 : Math.floor(
307
- (Number(expiresUtc) - 11644473600000000) / 1000000
308
- ),
309
- });
310
- }
311
- return results;
312
- });
313
-
314
- const source = opts?.cookieDbPath
315
- ? `${sourceLabel} cookie DB "${dbPath}"`
316
- : opts?.userDataDir
317
- ? `${sourceLabel} user data "${opts.userDataDir}"${opts.profile ? ` profile "${opts.profile}"` : ""}`
318
- : opts?.profile
319
- ? `${sourceLabel} profile "${opts.profile}"`
320
- : `${sourceLabel} default profile`;
321
- if (cookies.length === 0) {
322
- warnings.push(`No cookies for ${domain} found in ${source}`);
323
- }
324
- log("auth", `extracted ${cookies.length} cookies for ${domain} from ${source}`);
325
- return { cookies, source: cookies.length > 0 ? source : null, warnings };
326
- } catch (err) {
327
- warnings.push(`${sourceLabel} extraction failed: ${err instanceof Error ? err.message : err}`);
328
- return { cookies: [], source: null, warnings };
329
- }
330
- }
331
-
332
- // ---------------------------------------------------------------------------
333
- // Firefox extraction
334
- // ---------------------------------------------------------------------------
335
-
336
- export function extractFromFirefox(
337
- domain: string,
338
- opts?: { profile?: string },
339
- ): ExtractionResult {
340
- const warnings: string[] = [];
341
- const dbPath = getFirefoxCookiesPath(opts?.profile);
342
-
343
- if (!dbPath) {
344
- warnings.push("Firefox cookies DB not found");
345
- return { cookies: [], source: null, warnings };
346
- }
347
-
348
- try {
349
- const cookies = withTempCopy(dbPath, (tempDb) => {
350
- const where = buildDomainWhereClause(domain, "host");
351
- const sql = `SELECT name, value, host, path, isSecure, isHttpOnly, sameSite, expiry FROM moz_cookies WHERE ${where};`;
352
- const rows = sqliteQuery(tempDb, sql);
353
- if (!rows) return [];
354
-
355
- const results: BrowserCookie[] = [];
356
- for (const line of rows.split("\n")) {
357
- const parts = line.split("|");
358
- if (parts.length < 8) continue;
359
- const [name, value, host, cookiePath, secure, httpOnly, sameSite, expiry] = parts;
360
- if (!name || !value) continue;
361
-
362
- results.push({
363
- name,
364
- value,
365
- domain: host,
366
- path: cookiePath || "/",
367
- secure: secure === "1",
368
- httpOnly: httpOnly === "1",
369
- sameSite: sameSite === "0" ? "None" : sameSite === "1" ? "Lax" : "Strict",
370
- expires: Number(expiry) || -1,
371
- });
372
- }
373
- return results;
374
- });
375
-
376
- const source = opts?.profile ? `Firefox profile "${opts.profile}"` : "Firefox default profile";
377
- if (cookies.length === 0) {
378
- warnings.push(`No cookies for ${domain} found in ${source}`);
379
- }
380
- log("auth", `extracted ${cookies.length} cookies for ${domain} from ${source}`);
381
- return { cookies, source: cookies.length > 0 ? source : null, warnings };
382
- } catch (err) {
383
- warnings.push(`Firefox extraction failed: ${err instanceof Error ? err.message : err}`);
384
- return { cookies: [], source: null, warnings };
385
- }
386
- }
387
-
388
- // ---------------------------------------------------------------------------
389
- // Unified extraction — tries Firefox first, then Chrome (bird's priority)
390
- // ---------------------------------------------------------------------------
391
-
392
- export function extractBrowserCookies(
393
- domain: string,
394
- opts?: ExtractBrowserCookiesOptions,
395
- ): ExtractionResult {
396
- if (opts?.browser === "firefox") {
397
- return extractFromFirefox(domain, { profile: opts.firefoxProfile });
398
- }
399
-
400
- if (opts?.browser === "chrome") {
401
- return extractFromChrome(domain, { profile: opts.chromeProfile });
402
- }
403
-
404
- if (opts?.browser === "chromium") {
405
- return extractFromChromium(domain, opts.chromium);
406
- }
407
-
408
- // Try Firefox first (no decryption needed, more reliable)
409
- const ff = extractFromFirefox(domain, { profile: opts?.firefoxProfile });
410
- if (ff.cookies.length > 0) return ff;
411
-
412
- // If caller provided an explicit Chromium-family source, try that next.
413
- if (opts?.chromium?.cookieDbPath || opts?.chromium?.userDataDir) {
414
- const chromium = extractFromChromium(domain, opts.chromium);
415
- chromium.warnings.push(...ff.warnings);
416
- return chromium;
417
- }
418
-
419
- // Fall back to Chrome
420
- const chrome = extractFromChrome(domain, { profile: opts?.chromeProfile });
421
- chrome.warnings.push(...ff.warnings);
422
- return chrome;
423
- }