unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/cli.js +79 -33
  2. package/dist/index.js +2 -6
  3. package/dist/mcp.js +73 -22
  4. package/dist/server.js +25994 -0
  5. package/package.json +1 -2
  6. package/vendor/kuri/manifest.json +1 -1
  7. package/runtime-src/agent-outcome.ts +0 -166
  8. package/runtime-src/analytics-session.ts +0 -55
  9. package/runtime-src/api/browse-index.ts +0 -343
  10. package/runtime-src/api/browse-session.ts +0 -572
  11. package/runtime-src/api/browse-submit-prereqs.ts +0 -48
  12. package/runtime-src/api/browse-submit.ts +0 -1184
  13. package/runtime-src/api/routes.ts +0 -1833
  14. package/runtime-src/auth/browser-cookies.ts +0 -423
  15. package/runtime-src/auth/index.ts +0 -535
  16. package/runtime-src/auth/runtime.ts +0 -116
  17. package/runtime-src/browser/index.ts +0 -659
  18. package/runtime-src/browser/types.ts +0 -41
  19. package/runtime-src/build-info.generated.ts +0 -6
  20. package/runtime-src/capture/index.ts +0 -1929
  21. package/runtime-src/capture/prefetch.ts +0 -95
  22. package/runtime-src/capture/rsc.ts +0 -45
  23. package/runtime-src/cli/shortcuts.ts +0 -273
  24. package/runtime-src/cli.ts +0 -1734
  25. package/runtime-src/client/graph-client.ts +0 -100
  26. package/runtime-src/client/index.ts +0 -1425
  27. package/runtime-src/debug-trace.ts +0 -18
  28. package/runtime-src/domain.ts +0 -38
  29. package/runtime-src/execution/index.ts +0 -3407
  30. package/runtime-src/execution/retry.ts +0 -46
  31. package/runtime-src/execution/robots.ts +0 -167
  32. package/runtime-src/execution/search-forms.ts +0 -188
  33. package/runtime-src/execution/token-resolver.ts +0 -135
  34. package/runtime-src/extraction/index.ts +0 -1507
  35. package/runtime-src/foundry/publish-bundle.ts +0 -392
  36. package/runtime-src/graph/agent-augment.ts +0 -315
  37. package/runtime-src/graph/index.ts +0 -1532
  38. package/runtime-src/graph/local-fixtures.ts +0 -393
  39. package/runtime-src/graph/local-harness.ts +0 -646
  40. package/runtime-src/graph/planner.ts +0 -411
  41. package/runtime-src/graph/session.ts +0 -294
  42. package/runtime-src/graph/trace-store.ts +0 -136
  43. package/runtime-src/impact-log.ts +0 -227
  44. package/runtime-src/index.ts +0 -24
  45. package/runtime-src/indexer/index.ts +0 -465
  46. package/runtime-src/intent-match.ts +0 -1515
  47. package/runtime-src/kuri/client.ts +0 -1839
  48. package/runtime-src/logger.ts +0 -30
  49. package/runtime-src/marketplace/index.ts +0 -111
  50. package/runtime-src/mcp.ts +0 -1911
  51. package/runtime-src/orchestrator/browser-agent.ts +0 -374
  52. package/runtime-src/orchestrator/dag-advisor.ts +0 -59
  53. package/runtime-src/orchestrator/dag-feedback.ts +0 -257
  54. package/runtime-src/orchestrator/first-pass-action.ts +0 -403
  55. package/runtime-src/orchestrator/index.ts +0 -4480
  56. package/runtime-src/orchestrator/passive-publish.ts +0 -187
  57. package/runtime-src/orchestrator/timing-economics.ts +0 -80
  58. package/runtime-src/payments/cascade.ts +0 -137
  59. package/runtime-src/payments/index.ts +0 -270
  60. package/runtime-src/payments/lobster-pay.ts +0 -182
  61. package/runtime-src/payments/wallet.ts +0 -98
  62. package/runtime-src/publish/review-context.ts +0 -93
  63. package/runtime-src/publish/sanitize.ts +0 -197
  64. package/runtime-src/publish/schema-review.ts +0 -192
  65. package/runtime-src/publish-admission.ts +0 -388
  66. package/runtime-src/ratelimit/index.ts +0 -23
  67. package/runtime-src/reverse-engineer/bundle-scanner.ts +0 -127
  68. package/runtime-src/reverse-engineer/description-prompt.ts +0 -213
  69. package/runtime-src/reverse-engineer/index.ts +0 -1551
  70. package/runtime-src/reverse-engineer/token-sources.ts +0 -379
  71. package/runtime-src/router.ts +0 -17
  72. package/runtime-src/routing-telemetry.ts +0 -395
  73. package/runtime-src/runtime/browser-access.ts +0 -11
  74. package/runtime-src/runtime/browser-auth.ts +0 -12
  75. package/runtime-src/runtime/browser-host.ts +0 -48
  76. package/runtime-src/runtime/lifecycle.ts +0 -17
  77. package/runtime-src/runtime/local-server.ts +0 -311
  78. package/runtime-src/runtime/paths.ts +0 -99
  79. package/runtime-src/runtime/setup.ts +0 -251
  80. package/runtime-src/runtime/supervisor.ts +0 -69
  81. package/runtime-src/runtime/update-hints.ts +0 -351
  82. package/runtime-src/server.ts +0 -100
  83. package/runtime-src/session-logs.ts +0 -142
  84. package/runtime-src/settings.ts +0 -221
  85. package/runtime-src/single-binary.ts +0 -143
  86. package/runtime-src/site-policy.ts +0 -54
  87. package/runtime-src/stale-cleanup-runner.ts +0 -144
  88. package/runtime-src/stale-cleanup.ts +0 -133
  89. package/runtime-src/telemetry-attribution.ts +0 -120
  90. package/runtime-src/telemetry.ts +0 -253
  91. package/runtime-src/template-params.ts +0 -141
  92. package/runtime-src/transform/drift.ts +0 -60
  93. package/runtime-src/transform/index.ts +0 -277
  94. package/runtime-src/types/index.ts +0 -1
  95. package/runtime-src/types/skill.ts +0 -931
  96. package/runtime-src/vault/index.ts +0 -196
  97. package/runtime-src/verification/auth-gate.ts +0 -8
  98. package/runtime-src/verification/candidates.ts +0 -27
  99. package/runtime-src/verification/index.ts +0 -120
  100. package/runtime-src/verification/matrix.ts +0 -30
  101. package/runtime-src/version.ts +0 -148
  102. package/runtime-src/workflow/artifact.ts +0 -161
  103. package/runtime-src/workflow/compile.ts +0 -808
  104. package/runtime-src/workflow/publish.ts +0 -225
  105. package/runtime-src/workflow/runtime.ts +0 -213
@@ -1,423 +0,0 @@
1
- /**
2
- * Extract cookies directly from Chrome/Firefox SQLite databases.
3
- * Adapted from github.com/jawond/bird — generalized for any domain.
4
- *
5
- * Chrome cookies are AES-128-CBC encrypted with a key from the macOS keychain.
6
- * Firefox cookies are stored unencrypted.
7
- *
8
- * This avoids needing to launch a browser or close Chrome (reads a copy of the DB).
9
- */
10
-
11
- import { execSync, execFileSync } from "node:child_process";
12
- import { createDecipheriv, pbkdf2Sync } from "node:crypto";
13
- import { copyFileSync, existsSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
14
- import { tmpdir, homedir, platform } from "node:os";
15
- import { join } from "node:path";
16
- import { log } from "../logger.js";
17
- import { getRegistrableDomain, isDomainMatch } from "../domain.js";
18
-
19
- export interface BrowserCookie {
20
- name: string;
21
- value: string;
22
- domain: string;
23
- path: string;
24
- secure: boolean;
25
- httpOnly: boolean;
26
- sameSite: string;
27
- expires: number;
28
- }
29
-
30
- export interface ExtractionResult {
31
- cookies: BrowserCookie[];
32
- source: string | null;
33
- warnings: string[];
34
- }
35
-
36
- export type BrowserSource = "auto" | "firefox" | "chrome" | "chromium";
37
-
38
- export interface ChromiumCookieSourceOptions {
39
- profile?: string;
40
- userDataDir?: string;
41
- cookieDbPath?: string;
42
- safeStorageService?: string;
43
- browserName?: string;
44
- }
45
-
46
- export interface ExtractBrowserCookiesOptions {
47
- browser?: BrowserSource;
48
- chromeProfile?: string;
49
- firefoxProfile?: string;
50
- chromium?: ChromiumCookieSourceOptions;
51
- }
52
-
53
- // ---------------------------------------------------------------------------
54
- // Path helpers
55
- // ---------------------------------------------------------------------------
56
-
57
- function getChromeUserDataDir(): string {
58
- const home = homedir();
59
- if (platform() === "darwin") {
60
- return join(home, "Library", "Application Support", "Google", "Chrome");
61
- }
62
- if (platform() === "win32") {
63
- const appData = process.env.LOCALAPPDATA ?? join(home, "AppData", "Local");
64
- return join(appData, "Google", "Chrome", "User Data");
65
- }
66
- return join(home, ".config", "google-chrome");
67
- }
68
-
69
- export function resolveChromiumCookiesPath(opts?: ChromiumCookieSourceOptions): string | null {
70
- if (opts?.cookieDbPath) {
71
- return opts.cookieDbPath.replace(/^~\//, homedir() + "/");
72
- }
73
-
74
- const profileDir = opts?.profile || "Default";
75
- const userDataDir = (opts?.userDataDir || getChromeUserDataDir()).replace(/^~\//, homedir() + "/");
76
- const candidates = [
77
- join(userDataDir, profileDir, "Network", "Cookies"),
78
- join(userDataDir, profileDir, "Cookies"),
79
- join(userDataDir, "Network", "Cookies"),
80
- join(userDataDir, "Cookies"),
81
- ];
82
-
83
- return candidates.find((candidate) => existsSync(candidate)) ?? candidates[0] ?? null;
84
- }
85
-
86
- function getFirefoxProfilesRoot(): string | null {
87
- const home = homedir();
88
- if (platform() === "darwin") {
89
- return join(home, "Library", "Application Support", "Firefox", "Profiles");
90
- }
91
- if (platform() === "linux") {
92
- return join(home, ".mozilla", "firefox");
93
- }
94
- if (platform() === "win32") {
95
- const appData = process.env.APPDATA;
96
- if (!appData) return null;
97
- return join(appData, "Mozilla", "Firefox", "Profiles");
98
- }
99
- return null;
100
- }
101
-
102
- function pickFirefoxProfile(profilesRoot: string, profile?: string): string | null {
103
- if (profile) {
104
- const candidate = join(profilesRoot, profile, "cookies.sqlite");
105
- return existsSync(candidate) ? candidate : null;
106
- }
107
- const entries = readdirSync(profilesRoot, { withFileTypes: true });
108
- const defaultRelease = entries.find((e) => e.isDirectory() && e.name.includes("default-release"));
109
- const targetDir = defaultRelease?.name ?? entries.find((e) => e.isDirectory())?.name;
110
- if (!targetDir) return null;
111
- const candidate = join(profilesRoot, targetDir, "cookies.sqlite");
112
- return existsSync(candidate) ? candidate : null;
113
- }
114
-
115
- function getFirefoxCookiesPath(profile?: string): string | null {
116
- const profilesRoot = getFirefoxProfilesRoot();
117
- if (!profilesRoot || !existsSync(profilesRoot)) return null;
118
- return pickFirefoxProfile(profilesRoot, profile);
119
- }
120
-
121
- // ---------------------------------------------------------------------------
122
- // Chrome decryption (macOS — uses keychain + PBKDF2 + AES-128-CBC)
123
- // ---------------------------------------------------------------------------
124
-
125
- const _chromiumKeyCache = new Map<string, Buffer>();
126
-
127
- function getChromiumKeychainServiceName(opts?: ChromiumCookieSourceOptions): string {
128
- if (opts?.safeStorageService) return opts.safeStorageService;
129
- return `${opts?.browserName || "Chrome"} Safe Storage`;
130
- }
131
-
132
- function getChromiumDecryptionKey(opts?: ChromiumCookieSourceOptions): Buffer | null {
133
- const service = getChromiumKeychainServiceName(opts);
134
- const cached = _chromiumKeyCache.get(service);
135
- if (cached) return cached;
136
- if (platform() !== "darwin") return null; // TODO: Linux/Windows support
137
-
138
- try {
139
- const keyOutput = execFileSync(
140
- "security",
141
- ["find-generic-password", "-s", service, "-w"],
142
- { encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] },
143
- ).trim();
144
- if (!keyOutput) return null;
145
-
146
- const derived = pbkdf2Sync(keyOutput, "saltysalt", 1003, 16, "sha1");
147
- _chromiumKeyCache.set(service, derived);
148
- return derived;
149
- } catch {
150
- return null;
151
- }
152
- }
153
-
154
- function decryptChromiumValue(encryptedHex: string, opts?: ChromiumCookieSourceOptions): string | null {
155
- try {
156
- const buf = Buffer.from(encryptedHex, "hex");
157
- if (buf.length < 4) return null;
158
-
159
- const version = buf.subarray(0, 3).toString("utf8");
160
- if (version !== "v10" && version !== "v11") {
161
- // Not encrypted
162
- return buf.toString("utf8");
163
- }
164
-
165
- const key = getChromiumDecryptionKey(opts);
166
- if (!key) return null;
167
-
168
- const payload = buf.subarray(3);
169
-
170
- // Modern Chrome (v131+) prepends a 32-byte header (key derivation nonce)
171
- // before the actual AES-128-CBC ciphertext. The second 16-byte block of
172
- // the raw payload acts as the CBC IV for the remaining ciphertext.
173
- // Fallback: legacy format has no header (IV = 16 × 0x20 space bytes).
174
- if (payload.length >= 48) {
175
- try {
176
- const iv = payload.subarray(16, 32);
177
- const encrypted = payload.subarray(32);
178
- const decipher = createDecipheriv("aes-128-cbc", key, iv);
179
- decipher.setAutoPadding(true);
180
- const decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]);
181
- const val = decrypted.toString("utf8").replace(/[^\x20-\x7E]/g, "");
182
- if (val.length > 0) return val;
183
- } catch { /* fall through to legacy */ }
184
- }
185
-
186
- // Legacy format: IV = 16 bytes of space, ciphertext starts right after version
187
- const iv = Buffer.alloc(16, 0x20);
188
- const decipher = createDecipheriv("aes-128-cbc", key, iv);
189
- decipher.setAutoPadding(true);
190
- const decrypted = Buffer.concat([decipher.update(payload), decipher.final()]);
191
- return decrypted.toString("utf8").replace(/[^\x20-\x7E]/g, "");
192
- } catch {
193
- return null;
194
- }
195
- }
196
-
197
- export function decodeChromiumCookieValue(rawValue: string, encryptedHex: string, opts?: ChromiumCookieSourceOptions): string | null {
198
- if (rawValue) return rawValue;
199
- if (!encryptedHex) return null;
200
- return decryptChromiumValue(encryptedHex, opts);
201
- }
202
-
203
- // ---------------------------------------------------------------------------
204
- // SQLite helpers — copy DB to temp dir, query, cleanup
205
- // ---------------------------------------------------------------------------
206
-
207
- function withTempCopy<T>(dbPath: string, fn: (tempPath: string) => T): T {
208
- const tempDir = mkdtempSync(join(tmpdir(), "unbrowse-cookies-"));
209
- const tempDb = join(tempDir, "cookies.db");
210
- try {
211
- copyFileSync(dbPath, tempDb);
212
- // Copy WAL/SHM so we get the latest committed state even while Chrome is open
213
- for (const ext of ["-wal", "-shm"]) {
214
- const src = dbPath + ext;
215
- if (existsSync(src)) copyFileSync(src, tempDb + ext);
216
- }
217
- return fn(tempDb);
218
- } finally {
219
- try { rmSync(tempDir, { recursive: true, force: true }); } catch { /* ignore */ }
220
- }
221
- }
222
-
223
- function sqliteQuery(dbPath: string, sql: string): string {
224
- return execFileSync("sqlite3", ["-separator", "|", dbPath, sql], {
225
- encoding: "utf8",
226
- maxBuffer: 4 * 1024 * 1024,
227
- }).trim();
228
- }
229
-
230
- // ---------------------------------------------------------------------------
231
- // Domain matching helpers for SQL WHERE clauses
232
- // ---------------------------------------------------------------------------
233
-
234
- function buildDomainWhereClause(domain: string, column: string): string {
235
- const reg = getRegistrableDomain(domain);
236
- // Match exact domains: .example.com, example.com, plus common subdomains
237
- const variants = new Set([
238
- reg,
239
- `.${reg}`,
240
- domain,
241
- `.${domain}`,
242
- `www.${reg}`,
243
- `.www.${reg}`,
244
- ]);
245
- // Use parameterized-safe quoting: reject any domain containing single quotes
246
- for (const d of variants) {
247
- if (d.includes("'")) throw new Error(`Invalid domain for cookie query: ${d}`);
248
- }
249
- const escaped = [...variants].map((d) => `'${d}'`);
250
- const likeReg = reg.includes("'") ? reg : reg;
251
- const likePattern = `'%.${likeReg}'`;
252
- return `(${column} IN (${escaped.join(", ")}) OR ${column} LIKE ${likePattern})`;
253
- }
254
-
255
- // ---------------------------------------------------------------------------
256
- // Chrome extraction
257
- // ---------------------------------------------------------------------------
258
-
259
- export function extractFromChrome(
260
- domain: string,
261
- opts?: { profile?: string },
262
- ): ExtractionResult {
263
- return extractFromChromium(domain, {
264
- profile: opts?.profile,
265
- browserName: "Chrome",
266
- });
267
- }
268
-
269
- export function extractFromChromium(
270
- domain: string,
271
- opts?: ChromiumCookieSourceOptions,
272
- ): ExtractionResult {
273
- const warnings: string[] = [];
274
- const dbPath = resolveChromiumCookiesPath(opts);
275
- const sourceLabel = opts?.browserName || "Chromium";
276
-
277
- if (!dbPath || !existsSync(dbPath)) {
278
- warnings.push(`${sourceLabel} cookies DB not found${dbPath ? ` at ${dbPath}` : ""}`);
279
- return { cookies: [], source: null, warnings };
280
- }
281
-
282
- try {
283
- const cookies = withTempCopy(dbPath, (tempDb) => {
284
- const where = buildDomainWhereClause(domain, "host_key");
285
- const sql = `SELECT name, value, hex(encrypted_value) as ev, host_key, path, is_secure, is_httponly, samesite, expires_utc FROM cookies WHERE ${where};`;
286
- const rows = sqliteQuery(tempDb, sql);
287
- if (!rows) return [];
288
-
289
- const results: BrowserCookie[] = [];
290
- for (const line of rows.split("\n")) {
291
- const parts = line.split("|");
292
- if (parts.length < 9) continue;
293
- const [name, rawValue, encHex, host, cookiePath, secure, httpOnly, sameSite, expiresUtc] = parts;
294
- const value = decodeChromiumCookieValue(rawValue, encHex, opts);
295
- if (!value) continue;
296
-
297
- results.push({
298
- name,
299
- value,
300
- domain: host,
301
- path: cookiePath || "/",
302
- secure: secure === "1",
303
- httpOnly: httpOnly === "1",
304
- sameSite: sameSite === "0" ? "None" : sameSite === "1" ? "Lax" : "Strict",
305
- // Chrome stores expiry as microseconds since 1601-01-01
306
- expires: expiresUtc === "0" ? -1 : Math.floor(
307
- (Number(expiresUtc) - 11644473600000000) / 1000000
308
- ),
309
- });
310
- }
311
- return results;
312
- });
313
-
314
- const source = opts?.cookieDbPath
315
- ? `${sourceLabel} cookie DB "${dbPath}"`
316
- : opts?.userDataDir
317
- ? `${sourceLabel} user data "${opts.userDataDir}"${opts.profile ? ` profile "${opts.profile}"` : ""}`
318
- : opts?.profile
319
- ? `${sourceLabel} profile "${opts.profile}"`
320
- : `${sourceLabel} default profile`;
321
- if (cookies.length === 0) {
322
- warnings.push(`No cookies for ${domain} found in ${source}`);
323
- }
324
- log("auth", `extracted ${cookies.length} cookies for ${domain} from ${source}`);
325
- return { cookies, source: cookies.length > 0 ? source : null, warnings };
326
- } catch (err) {
327
- warnings.push(`${sourceLabel} extraction failed: ${err instanceof Error ? err.message : err}`);
328
- return { cookies: [], source: null, warnings };
329
- }
330
- }
331
-
332
- // ---------------------------------------------------------------------------
333
- // Firefox extraction
334
- // ---------------------------------------------------------------------------
335
-
336
- export function extractFromFirefox(
337
- domain: string,
338
- opts?: { profile?: string },
339
- ): ExtractionResult {
340
- const warnings: string[] = [];
341
- const dbPath = getFirefoxCookiesPath(opts?.profile);
342
-
343
- if (!dbPath) {
344
- warnings.push("Firefox cookies DB not found");
345
- return { cookies: [], source: null, warnings };
346
- }
347
-
348
- try {
349
- const cookies = withTempCopy(dbPath, (tempDb) => {
350
- const where = buildDomainWhereClause(domain, "host");
351
- const sql = `SELECT name, value, host, path, isSecure, isHttpOnly, sameSite, expiry FROM moz_cookies WHERE ${where};`;
352
- const rows = sqliteQuery(tempDb, sql);
353
- if (!rows) return [];
354
-
355
- const results: BrowserCookie[] = [];
356
- for (const line of rows.split("\n")) {
357
- const parts = line.split("|");
358
- if (parts.length < 8) continue;
359
- const [name, value, host, cookiePath, secure, httpOnly, sameSite, expiry] = parts;
360
- if (!name || !value) continue;
361
-
362
- results.push({
363
- name,
364
- value,
365
- domain: host,
366
- path: cookiePath || "/",
367
- secure: secure === "1",
368
- httpOnly: httpOnly === "1",
369
- sameSite: sameSite === "0" ? "None" : sameSite === "1" ? "Lax" : "Strict",
370
- expires: Number(expiry) || -1,
371
- });
372
- }
373
- return results;
374
- });
375
-
376
- const source = opts?.profile ? `Firefox profile "${opts.profile}"` : "Firefox default profile";
377
- if (cookies.length === 0) {
378
- warnings.push(`No cookies for ${domain} found in ${source}`);
379
- }
380
- log("auth", `extracted ${cookies.length} cookies for ${domain} from ${source}`);
381
- return { cookies, source: cookies.length > 0 ? source : null, warnings };
382
- } catch (err) {
383
- warnings.push(`Firefox extraction failed: ${err instanceof Error ? err.message : err}`);
384
- return { cookies: [], source: null, warnings };
385
- }
386
- }
387
-
388
- // ---------------------------------------------------------------------------
389
- // Unified extraction — tries Firefox first, then Chrome (bird's priority)
390
- // ---------------------------------------------------------------------------
391
-
392
- export function extractBrowserCookies(
393
- domain: string,
394
- opts?: ExtractBrowserCookiesOptions,
395
- ): ExtractionResult {
396
- if (opts?.browser === "firefox") {
397
- return extractFromFirefox(domain, { profile: opts.firefoxProfile });
398
- }
399
-
400
- if (opts?.browser === "chrome") {
401
- return extractFromChrome(domain, { profile: opts.chromeProfile });
402
- }
403
-
404
- if (opts?.browser === "chromium") {
405
- return extractFromChromium(domain, opts.chromium);
406
- }
407
-
408
- // Try Firefox first (no decryption needed, more reliable)
409
- const ff = extractFromFirefox(domain, { profile: opts?.firefoxProfile });
410
- if (ff.cookies.length > 0) return ff;
411
-
412
- // If caller provided an explicit Chromium-family source, try that next.
413
- if (opts?.chromium?.cookieDbPath || opts?.chromium?.userDataDir) {
414
- const chromium = extractFromChromium(domain, opts.chromium);
415
- chromium.warnings.push(...ff.warnings);
416
- return chromium;
417
- }
418
-
419
- // Fall back to Chrome
420
- const chrome = extractFromChrome(domain, { profile: opts?.chromeProfile });
421
- chrome.warnings.push(...ff.warnings);
422
- return chrome;
423
- }