pi-lean-dimension 0.1.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/README.md +96 -36
  2. package/node_modules/pi-lean-portal/AGENTS.md +146 -4
  3. package/node_modules/pi-lean-portal/README.md +112 -50
  4. package/node_modules/pi-lean-portal/__tests__/browser-data.test.ts +124 -0
  5. package/node_modules/pi-lean-portal/__tests__/browser-inspect.test.ts +192 -16
  6. package/node_modules/pi-lean-portal/__tests__/browser-toggle-profile.test.ts +3 -3
  7. package/node_modules/pi-lean-portal/__tests__/browser-toggle.test.ts +49 -35
  8. package/node_modules/pi-lean-portal/__tests__/chromium-py-persistence.test.ts +21 -195
  9. package/node_modules/pi-lean-portal/__tests__/chromium-py.test.ts +17 -81
  10. package/node_modules/pi-lean-portal/__tests__/chromium.test.ts +25 -0
  11. package/node_modules/pi-lean-portal/__tests__/contributed/invisible-py/invisible-py.test.ts +299 -0
  12. package/node_modules/pi-lean-portal/__tests__/cookie-persistence.test.ts +22 -182
  13. package/node_modules/pi-lean-portal/__tests__/fetch-backend.test.ts +1 -1
  14. package/node_modules/pi-lean-portal/__tests__/firefox-py-persistence.test.ts +21 -184
  15. package/node_modules/pi-lean-portal/__tests__/firefox-py.test.ts +17 -101
  16. package/node_modules/pi-lean-portal/__tests__/firefox.test.ts +2 -18
  17. package/node_modules/pi-lean-portal/__tests__/helpers/__pycache__/mock-python-bridge.cpython-313.pyc +0 -0
  18. package/node_modules/pi-lean-portal/__tests__/helpers/create-py-backend-harness.ts +105 -0
  19. package/node_modules/pi-lean-portal/__tests__/helpers/load-plugin-config-from-file.ts +53 -0
  20. package/node_modules/pi-lean-portal/__tests__/helpers/mock-plugin.ts +11 -7
  21. package/node_modules/pi-lean-portal/__tests__/helpers/mock-python-bridge.py +4 -0
  22. package/node_modules/pi-lean-portal/__tests__/helpers/persistence-suite.ts +218 -0
  23. package/node_modules/pi-lean-portal/__tests__/helpers/plugin-contract.ts +198 -318
  24. package/node_modules/pi-lean-portal/__tests__/helpers/probe-user-backend.ts +198 -0
  25. package/node_modules/pi-lean-portal/__tests__/helpers/test-server.ts +14 -0
  26. package/node_modules/pi-lean-portal/__tests__/plugin-config-browser.test.ts +150 -15
  27. package/node_modules/pi-lean-portal/__tests__/plugin-loading.test.ts +120 -18
  28. package/node_modules/pi-lean-portal/__tests__/plugin-registry.test.ts +6 -67
  29. package/node_modules/pi-lean-portal/__tests__/probe-user-backend.test.ts +236 -0
  30. package/node_modules/pi-lean-portal/__tests__/python-adapter.test.ts +401 -11
  31. package/node_modules/pi-lean-portal/__tests__/router-session.test.ts +4 -1
  32. package/node_modules/pi-lean-portal/__tests__/run-contributed-suites.test.ts +318 -0
  33. package/node_modules/pi-lean-portal/__tests__/session-manager.test.ts +50 -0
  34. package/node_modules/pi-lean-portal/__tests__/snapshot-cache.test.ts +2 -2
  35. package/node_modules/pi-lean-portal/__tests__/url-safety.test.ts +1 -1
  36. package/node_modules/pi-lean-portal/backends/chromium/index.ts +5 -13
  37. package/node_modules/pi-lean-portal/backends/chromium-py/__pycache__/bridge.cpython-313.pyc +0 -0
  38. package/node_modules/pi-lean-portal/backends/chromium-py/bridge.py +0 -2
  39. package/node_modules/pi-lean-portal/backends/firefox/index.ts +6 -5
  40. package/node_modules/pi-lean-portal/backends/firefox-py/__pycache__/bridge.cpython-313.pyc +0 -0
  41. package/node_modules/pi-lean-portal/backends/firefox-py/bridge.py +7 -6
  42. package/node_modules/pi-lean-portal/backends/playwright-base/playwright-plugin.ts +241 -398
  43. package/node_modules/pi-lean-portal/backends/python-adapter.ts +182 -83
  44. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__init__.py +1 -42
  45. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/__init__.cpython-312.pyc +0 -0
  46. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/__init__.cpython-313.pyc +0 -0
  47. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/accessibility.cpython-312.pyc +0 -0
  48. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/accessibility.cpython-313.pyc +0 -0
  49. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/bot_detection.cpython-312.pyc +0 -0
  50. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/bot_detection.cpython-313.pyc +0 -0
  51. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/bridge.cpython-312.pyc +0 -0
  52. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/bridge.cpython-313.pyc +0 -0
  53. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/browser_data.cpython-312.pyc +0 -0
  54. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/browser_data.cpython-313.pyc +0 -0
  55. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/patch_playwright.cpython-312.pyc +0 -0
  56. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/patch_playwright.cpython-313.pyc +0 -0
  57. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/playwright_base.cpython-312.pyc +0 -0
  58. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/playwright_base.cpython-313.pyc +0 -0
  59. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/transport.cpython-312.pyc +0 -0
  60. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/transport.cpython-313.pyc +0 -0
  61. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/accessibility.py +12 -147
  62. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/bot_detection.py +12 -37
  63. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/bridge.py +249 -322
  64. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/browser_data.py +92 -0
  65. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/patch_playwright.py +321 -0
  66. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/playwright_base.py +511 -299
  67. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/conftest.cpython-313-pytest-9.1.1.pyc +0 -0
  68. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/{test_accessibility.cpython-313-pytest-9.1.0.pyc → test_accessibility.cpython-313-pytest-9.1.1.pyc} +0 -0
  69. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/test_accessibility.cpython-313.pyc +0 -0
  70. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/test_bot_detection.cpython-313-pytest-9.1.1.pyc +0 -0
  71. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/test_bot_detection.cpython-313.pyc +0 -0
  72. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/test_browser_data.cpython-313-pytest-9.1.1.pyc +0 -0
  73. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/test_browser_data.cpython-313.pyc +0 -0
  74. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/{test_chromium_py_bridge.cpython-313-pytest-9.1.0.pyc → test_chromium_py_bridge.cpython-313-pytest-9.1.1.pyc} +0 -0
  75. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/{test_firefox_py_bridge.cpython-313-pytest-9.1.0.pyc → test_firefox_py_bridge.cpython-313-pytest-9.1.1.pyc} +0 -0
  76. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/test_playwright_base_quirks.cpython-313-pytest-9.1.1.pyc +0 -0
  77. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/test_playwright_base_quirks.cpython-313.pyc +0 -0
  78. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/test_py_bridges.cpython-313-pytest-9.1.1.pyc +0 -0
  79. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/{test_transport.cpython-313-pytest-9.1.0.pyc → test_transport.cpython-313-pytest-9.1.1.pyc} +0 -0
  80. package/node_modules/pi-lean-portal/backends/python-base/tests/conftest.py +95 -0
  81. package/node_modules/pi-lean-portal/backends/python-base/tests/test_accessibility.py +8 -132
  82. package/node_modules/pi-lean-portal/backends/python-base/tests/test_bot_detection.py +8 -147
  83. package/node_modules/pi-lean-portal/backends/python-base/tests/test_browser_data.py +131 -0
  84. package/node_modules/pi-lean-portal/backends/python-base/tests/test_playwright_base_quirks.py +768 -0
  85. package/node_modules/pi-lean-portal/backends/python-base/tests/test_py_bridges.py +198 -0
  86. package/node_modules/pi-lean-portal/browser-toggle.ts +33 -69
  87. package/node_modules/pi-lean-portal/contributed/CHOOSING.md +126 -0
  88. package/node_modules/pi-lean-portal/contributed/README.md +304 -0
  89. package/node_modules/pi-lean-portal/contributed/camoufox-py/bridge.py +216 -0
  90. package/node_modules/pi-lean-portal/contributed/invisible-py/__pycache__/bridge.cpython-313.pyc +0 -0
  91. package/node_modules/pi-lean-portal/contributed/invisible-py/bridge.py +434 -0
  92. package/node_modules/pi-lean-portal/core/fetch-backend.ts +0 -5
  93. package/node_modules/pi-lean-portal/core/plugin-api.ts +6 -33
  94. package/node_modules/pi-lean-portal/core/plugin-config.ts +75 -58
  95. package/node_modules/pi-lean-portal/core/plugin-registry.ts +11 -49
  96. package/node_modules/pi-lean-portal/core/router.ts +59 -98
  97. package/node_modules/pi-lean-portal/core/shared/accessibility-tree.ts +10 -143
  98. package/node_modules/pi-lean-portal/core/shared/bot-detection.ts +31 -77
  99. package/node_modules/pi-lean-portal/core/shared/browser-data.json +183 -0
  100. package/node_modules/pi-lean-portal/core/shared/browser-data.ts +50 -0
  101. package/node_modules/pi-lean-portal/core/shared/browser-events.ts +6 -6
  102. package/node_modules/pi-lean-portal/core/shared/dom-extractor.ts +97 -36
  103. package/node_modules/pi-lean-portal/core/shared/nav-settle.ts +12 -15
  104. package/node_modules/pi-lean-portal/core/shared/paths.ts +3 -0
  105. package/node_modules/pi-lean-portal/core/shared/session-manager.ts +7 -21
  106. package/node_modules/pi-lean-portal/{verify-ship-manifest.ts → core/shared/ship-manifest.ts} +34 -9
  107. package/node_modules/pi-lean-portal/core/shared/snapshot-cache.ts +7 -6
  108. package/node_modules/pi-lean-portal/core/shared/storage-state.ts +40 -9
  109. package/node_modules/pi-lean-portal/index.ts +42 -7
  110. package/node_modules/pi-lean-portal/package.json +8 -3
  111. package/node_modules/pi-lean-portal/ship-manifest.test.ts +8 -3
  112. package/node_modules/pi-lean-portal/tools/browser-inspect.ts +2 -6
  113. package/node_modules/pi-lean-portal/tools/browser-navigate.ts +7 -5
  114. package/node_modules/pi-lean-portal/tools/browser-snapshot.ts +2 -5
  115. package/node_modules/pi-lean-portal/tools/utils.ts +22 -4
  116. package/node_modules/pi-lean-portal/tools/web-fetch.ts +3 -4
  117. package/node_modules/pi-lean-search/README.md +6 -2
  118. package/node_modules/pi-lean-search/__tests__/web-search.test.ts +11 -0
  119. package/node_modules/pi-lean-search/index.ts +4 -21
  120. package/node_modules/pi-lean-search/package.json +2 -2
  121. package/node_modules/pi-lean-search/verify-ship-manifest.ts +6 -92
  122. package/node_modules/pi-lean-search/web-search-tool.ts +19 -13
  123. package/package.json +4 -4
  124. package/node_modules/pi-lean-portal/__tests__/helpers/reddit-fixture.ts +0 -264
  125. package/node_modules/pi-lean-portal/__tests__/helpers/toggle-test-utils.ts +0 -31
  126. package/node_modules/pi-lean-portal/__tests__/reddit-dialog.test.ts +0 -302
  127. package/node_modules/pi-lean-portal/backends/python-base/pi_browser_bridge/__pycache__/occlusion.cpython-313.pyc +0 -0
  128. package/node_modules/pi-lean-portal/backends/python-base/tests/__pycache__/test_bot_detection.cpython-313-pytest-9.1.0.pyc +0 -0
  129. package/node_modules/pi-lean-portal/backends/python-base/tests/test_chromium_py_bridge.py +0 -281
  130. package/node_modules/pi-lean-portal/backends/python-base/tests/test_firefox_py_bridge.py +0 -212
@@ -7,7 +7,7 @@
7
7
  * - runExtractor() — calls the script and validates the result
8
8
  * - correlateElements() — matches extracted content against the element cache
9
9
  * - queryElementCache() — synchronous cache filtering
10
- * - formatCorrelatedOutput(), formatElementList(), formatRoleCountSummary()
10
+ * - correlateElements(), formatElementList(), formatRoleCountSummary()
11
11
  * - Boilerplate filtering constants
12
12
  */
13
13
 
@@ -49,7 +49,7 @@ export interface ExtractResult {
49
49
  }
50
50
 
51
51
  /** Output of the correlation step — text with @e annotations. */
52
- export interface CorrelatedResult {
52
+ interface CorrelatedResult {
53
53
  /** Formatted text output with @e annotations */
54
54
  text: string;
55
55
  /** Number of matched refs */
@@ -58,14 +58,33 @@ export interface CorrelatedResult {
58
58
  staleCache: boolean;
59
59
  }
60
60
 
61
+ /** Outcome of runExtractor — discriminated union. */
62
+ type ExtractorOutcome =
63
+ | { ok: true; result: ExtractResult }
64
+ | { ok: false; error: string };
65
+
61
66
  /** Parameters for queryElementCache. */
62
- export interface ElementCacheQuery {
67
+ interface ElementCacheQuery {
63
68
  role?: string;
64
69
  name?: string;
65
70
  ref?: string;
66
71
  subtree?: string;
67
72
  }
68
73
 
74
+ /**
75
+ * Optional out-param for queryElementCache to signal when a ref was found
76
+ * but an extra role/name/subtree filter rejected it.
77
+ */
78
+ export interface QueryStatus {
79
+ /** Set when filters.ref resolved to a cache node but an extra
80
+ * role/name/subtree filter rejected it. */
81
+ refFilteredOut?: {
82
+ node: AriaCachedNode;
83
+ filter: "role" | "name" | "subtree";
84
+ value: string;
85
+ };
86
+ }
87
+
69
88
  // ─── EXTRACTOR_SCRIPT — runs in browser page context ──────────────
70
89
 
71
90
  /**
@@ -75,8 +94,14 @@ export interface ElementCacheQuery {
75
94
  * Playwright injects evaluate() at the DevTools protocol level,
76
95
  * bypassing page CSP. Sandboxed iframes without allow-scripts
77
96
  * are the only case that fails — the caller handles that.
97
+ *
98
+ * INVARIANT: EXTRACTOR_SCRIPT is pure DOM reads (no fetch/XHR/eval/
99
+ * writes). runExtractor() relies on this to pass read_only=true so
100
+ * the bridge routes it through the stable isolated-world context on
101
+ * Camoufox. If you add a write here, drop the `true` arg in
102
+ * runExtractor().
78
103
  */
79
- const EXTRACTOR_SCRIPT = `(() => {
104
+ export const EXTRACTOR_SCRIPT = `(() => {
80
105
  'use strict';
81
106
  try {
82
107
  const result = {
@@ -182,18 +207,21 @@ const EXTRACTOR_SCRIPT = `(() => {
182
207
  * Run the DOM extractor in the page context via plugin.evaluate().
183
208
  *
184
209
  * The extractor runs purely on DOM properties — no fetch, no XHR, no eval.
185
- * Returns a parsed ExtractResult, or null on failure (including error
186
- * caught by the script itself, evaluate rejection, or invalid JSON).
210
+ * Returns an ExtractorOutcome with the parsed result on success or an
211
+ * error string on failure.
187
212
  */
188
213
  export async function runExtractor(
189
214
  taskId: string,
190
215
  plugin: BrowserPlugin,
191
- ): Promise<ExtractResult | null> {
216
+ ): Promise<ExtractorOutcome> {
192
217
  try {
193
- const evalResult = await plugin.evaluate(taskId, EXTRACTOR_SCRIPT);
218
+ const evalResult = await plugin.evaluate(taskId, EXTRACTOR_SCRIPT, true);
194
219
 
195
220
  if (!evalResult.success) {
196
- return null;
221
+ return {
222
+ ok: false,
223
+ error: evalResult.error ?? "evaluate failed (no error detail)",
224
+ };
197
225
  }
198
226
 
199
227
  // The script returns a JSON string inside result.result
@@ -202,12 +230,23 @@ export async function runExtractor(
202
230
  ? evalResult.result
203
231
  : JSON.stringify(evalResult.result);
204
232
 
205
- const parsed = JSON.parse(rawJson) as ExtractResult;
233
+ let parsed: ExtractResult;
234
+ try {
235
+ parsed = JSON.parse(rawJson) as ExtractResult;
236
+ } catch (e: unknown) {
237
+ return {
238
+ ok: false,
239
+ error: `extractor returned invalid JSON: ${e instanceof Error ? e.message : String(e)}`,
240
+ };
241
+ }
206
242
 
207
243
  // Check for script-level error
208
244
  if (parsed.error) {
209
- console.warn("[pi-lean-portal] DOM extractor script error:", parsed.error);
210
- return null;
245
+ console.warn(
246
+ "[pi-lean-portal] DOM extractor script error:",
247
+ parsed.error,
248
+ );
249
+ return { ok: false, error: parsed.error };
211
250
  }
212
251
 
213
252
  // Validate shape (basic structural check)
@@ -219,12 +258,15 @@ export async function runExtractor(
219
258
  !Array.isArray(parsed.images) ||
220
259
  !Array.isArray(parsed.interactive)
221
260
  ) {
222
- return null;
261
+ return { ok: false, error: "extractor returned malformed result" };
223
262
  }
224
263
 
225
- return parsed;
264
+ return { ok: true, result: parsed };
226
265
  } catch {
227
- return null;
266
+ return {
267
+ ok: false,
268
+ error: "evaluate failed (no error detail)",
269
+ };
228
270
  }
229
271
  }
230
272
 
@@ -353,8 +395,18 @@ export function correlateElements(
353
395
  );
354
396
  }
355
397
 
398
+ // ── Empty-output notice ──
399
+ let text = lines.join("\n").trim();
400
+ if (text.length === 0) {
401
+ text =
402
+ "⚠ No extractable content found on the page. " +
403
+ "The page may be empty, render behind a challenge, or block DOM queries. " +
404
+ "Use browser-inspect { role/name/ref } against the element cache, " +
405
+ "or browser-snapshot to inspect visually.";
406
+ }
407
+
356
408
  return {
357
- text: lines.join("\n").trim(),
409
+ text,
358
410
  matchedRefs: matchedRefs.size,
359
411
  staleCache,
360
412
  };
@@ -390,9 +442,15 @@ function annotateRefs(refs: string[], matchedRefs: Set<string>): string {
390
442
  export function queryElementCache(
391
443
  cache: Map<string, AriaCachedNode>,
392
444
  filters: ElementCacheQuery,
445
+ status?: QueryStatus,
393
446
  ): AriaCachedNode[] {
394
447
  const results: AriaCachedNode[] = [];
395
448
 
449
+ // Pre-compute subtree ancestry map once (shared by ref + loop paths)
450
+ const subtreeAncestors = filters.subtree
451
+ ? computeSubtreeAncestors(cache, filters.subtree)
452
+ : null;
453
+
396
454
  // ref lookup: direct map access (cache keys are "e5" not "@e5")
397
455
  if (filters.ref) {
398
456
  const key = filters.ref.startsWith("@")
@@ -401,26 +459,38 @@ export function queryElementCache(
401
459
  const node = cache.get(key);
402
460
  if (node) {
403
461
  // Apply additional filters if any
404
- if (filters.role && !matchesRole(node, filters.role)) return [];
405
- if (filters.name && !matchesName(node, filters.name)) return [];
406
- if (filters.subtree && !matchesSubtree(node, cache, filters.subtree))
462
+ if (filters.role && !matchesRole(node, filters.role)) {
463
+ if (status)
464
+ status.refFilteredOut = { node, filter: "role", value: filters.role };
407
465
  return [];
466
+ }
467
+ if (filters.name && !matchesName(node, filters.name)) {
468
+ if (status)
469
+ status.refFilteredOut = { node, filter: "name", value: filters.name };
470
+ return [];
471
+ }
472
+ if (
473
+ filters.subtree &&
474
+ subtreeAncestors &&
475
+ !subtreeAncestors.has(node.ref)
476
+ ) {
477
+ if (status)
478
+ status.refFilteredOut = {
479
+ node,
480
+ filter: "subtree",
481
+ value: filters.subtree,
482
+ };
483
+ return [];
484
+ }
408
485
  return [node];
409
486
  }
410
487
  return [];
411
488
  }
412
489
 
413
- // Pre-compute subtree ancestry map for subtree filter
414
- const subtreeAncestors = filters.subtree
415
- ? computeSubtreeAncestors(cache, filters.subtree)
416
- : null;
417
-
418
490
  for (const [, node] of cache) {
419
491
  if (filters.role && !matchesRole(node, filters.role)) continue;
420
492
  if (filters.name && !matchesName(node, filters.name)) continue;
421
- if (filters.subtree && subtreeAncestors) {
422
- if (!subtreeAncestors.has(node.ref)) continue;
423
- }
493
+ if (subtreeAncestors && !subtreeAncestors.has(node.ref)) continue;
424
494
  results.push(node);
425
495
  }
426
496
 
@@ -478,15 +548,6 @@ function computeSubtreeAncestors(
478
548
  return insideRefs;
479
549
  }
480
550
 
481
- function matchesSubtree(
482
- node: AriaCachedNode,
483
- cache: Map<string, AriaCachedNode>,
484
- containerRole: string,
485
- ): boolean {
486
- const ancestors = computeSubtreeAncestors(cache, containerRole);
487
- return ancestors.has(node.ref);
488
- }
489
-
490
551
  // ─── Output formatting ────────────────────────────────────────────
491
552
 
492
553
  /**
@@ -1,19 +1,13 @@
1
1
  /**
2
- * Navigation settle detection for the browser extension.
2
+ * Navigation settle detection — waits for framenavigated event or DOM
3
+ * stabilisation after interactions, eliminating stale-@e-ref bugs from fixed sleeps.
3
4
  *
4
- * After a user interaction (click, press) that may or may not trigger
5
- * a page navigation, this module provides a reliable way to wait for
6
- * the page to settle before reading URL / title / snapshot.
7
- *
8
- * The core insight: instead of a fixed sleep (e.g. `waitForTimeout(300)`)
9
- * that races against navigation commit, we listen for the actual
10
- * `framenavigated` event and wait for page readiness (load + networkidle)
11
- * only when navigation has actually started.
12
- *
13
- * This eliminates the URL / DOM mismatch that causes stale-@e-ref and
14
- * mismatched URL/content bugs.
5
+ * Constants (timeouts, race window) are sourced from the shared
6
+ * ``browser-data.json`` so the TypeScript and Python sides never drift.
15
7
  */
16
8
 
9
+ import { NAV_SETTLE } from "./browser-data.js";
10
+
17
11
  // ─── Public types ───────────────────────────────────────────────────
18
12
 
19
13
  /**
@@ -129,8 +123,8 @@ export async function waitForNavigationSettle(
129
123
  urlBefore: string,
130
124
  opts?: NavigationSettleOptions,
131
125
  ): Promise<NavigationSettleResult> {
132
- const navTimeout = opts?.navTimeoutMs ?? 5000;
133
- const settleTimeout = opts?.settleTimeoutMs ?? 400;
126
+ const navTimeout = opts?.navTimeoutMs ?? NAV_SETTLE.navTimeoutMs;
127
+ const settleTimeout = opts?.settleTimeoutMs ?? NAV_SETTLE.settleTimeoutMs;
134
128
 
135
129
  let navigated = false;
136
130
 
@@ -156,7 +150,10 @@ export async function waitForNavigationSettle(
156
150
  // Most link clicks and Enter presses trigger navigation within one
157
151
  // event-loop tick; the race means we don't waste time when nav
158
152
  // starts immediately.
159
- await Promise.race([page.waitForTimeout(150), navStarted]);
153
+ await Promise.race([
154
+ page.waitForTimeout(NAV_SETTLE.settleRaceMs),
155
+ navStarted,
156
+ ]);
160
157
 
161
158
  let waitedForLoad = false;
162
159
  if (navigated) {
@@ -29,6 +29,9 @@ export const PORTAL_DATA_DIR = join(
29
29
  "pi-lean-portal",
30
30
  );
31
31
 
32
+ /** User-installed Python backend root; sibling of web-guides/ and browser-state/. */
33
+ export const USER_BACKENDS_DIR = join(PORTAL_DATA_DIR, "user-backends");
34
+
32
35
  /**
33
36
  * Sanitize a taskId (or any string) for use in filenames.
34
37
  * Replaces any character that is not alphanumeric or hyphen with `_`.
@@ -90,28 +90,14 @@ class SessionManager {
90
90
  >,
91
91
  ): void {
92
92
  const session = this.#sessions.get(taskId);
93
- if (session) {
94
- if (updates.currentUrl !== undefined)
95
- session.currentUrl = updates.currentUrl;
96
- if (updates.currentTitle !== undefined)
97
- session.currentTitle = updates.currentTitle;
98
- if (updates.pluginName !== undefined)
99
- session.pluginName = updates.pluginName;
100
- if (updates.crashed !== undefined) session.crashed = updates.crashed;
101
- if (updates.currentSnapshotFingerprint !== undefined)
102
- session.currentSnapshotFingerprint = updates.currentSnapshotFingerprint;
103
- if (updates.cachePopulatedAt !== undefined)
104
- session.cachePopulatedAt = updates.cachePopulatedAt;
105
- if (updates.lastInteractionAt !== undefined)
106
- session.lastInteractionAt = updates.lastInteractionAt;
107
- if (updates.persistState !== undefined)
108
- session.persistState = updates.persistState;
109
- if (updates.profileName !== undefined)
110
- session.profileName = updates.profileName;
111
- if (updates.piSessionId !== undefined)
112
- session.piSessionId = updates.piSessionId;
113
- session.lastActive = Date.now();
93
+ if (!session) return;
94
+ // ponytail: the `as any` is bounded by the Partial<Pick<BrowserSession, …>>
95
+ // type above — only declared session fields can land here. Skip-undefined
96
+ // preserves existing fields (matches the prior per-field guards).
97
+ for (const [k, v] of Object.entries(updates)) {
98
+ if (v !== undefined) (session as any)[k] = v;
114
99
  }
100
+ session.lastActive = Date.now();
115
101
  }
116
102
 
117
103
  // ─── Last navigation storage (for session auto-recovery) ───
@@ -13,7 +13,7 @@ import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
13
13
  import { dirname, relative, resolve } from "node:path";
14
14
  import { fileURLToPath } from "node:url";
15
15
 
16
- const SKIP_DIRS = new Set(["node_modules", "docs", "__tests__"]);
16
+ const DEFAULT_SKIP_DIRS = ["node_modules", "docs", "__tests__"];
17
17
  const SKIP_FILES = new Set(["test-fixtures.ts"]);
18
18
  /** Entries that won't exist on disk at rest but are valid (generated at pack time, e.g. by prepack). */
19
19
  const SKIP_STALE = new Set(["LICENSE"]);
@@ -44,15 +44,38 @@ export interface ShipManifestResult {
44
44
  * would omit; `stale` flags `files` entries that point at nothing on disk
45
45
  * (asset entries like `README.md` count as present — staleness is plain
46
46
  * existence, not the production-`.ts` walk).
47
+ *
48
+ * @param opts.skipDirs - Additional directory names to skip beyond the defaults
49
+ * (`node_modules`, `docs`, `__tests__`).
47
50
  */
48
51
  export function verifyShipManifest(
49
52
  packageDirOrUrl: string,
53
+ opts?: { skipDirs?: readonly string[] },
50
54
  ): ShipManifestResult {
51
55
  const packageDir = packageDirOrUrl.startsWith("file:")
52
56
  ? dirname(fileURLToPath(packageDirOrUrl))
53
57
  : packageDirOrUrl;
54
- const pkgRaw = readFileSync(resolve(packageDir, "package.json"), "utf8");
55
- const pkg = JSON.parse(pkgRaw) as { files?: string[] };
58
+ const skipDirs = new Set([...DEFAULT_SKIP_DIRS, ...(opts?.skipDirs ?? [])]);
59
+ let pkgRaw: string;
60
+ try {
61
+ pkgRaw = readFileSync(resolve(packageDir, "package.json"), "utf8");
62
+ } catch (err) {
63
+ throw new Error(
64
+ `verifyShipManifest: could not read package.json under "${packageDir}": ${
65
+ err instanceof Error ? err.message : String(err)
66
+ }`,
67
+ );
68
+ }
69
+ let pkg: { files?: string[] };
70
+ try {
71
+ pkg = JSON.parse(pkgRaw) as { files?: string[] };
72
+ } catch (err) {
73
+ throw new Error(
74
+ `verifyShipManifest: package.json under "${packageDir}" is not valid JSON: ${
75
+ err instanceof Error ? err.message : String(err)
76
+ }`,
77
+ );
78
+ }
56
79
  const declared = pkg.files ?? [];
57
80
  const exactFiles = new Set<string>();
58
81
  const dirPrefixes: string[] = [];
@@ -72,7 +95,7 @@ export function verifyShipManifest(
72
95
  else exactFiles.add(entry);
73
96
  }
74
97
 
75
- const onDisk = walkProductionTs(packageDir, packageDir);
98
+ const onDisk = walkProductionTs(packageDir, packageDir, skipDirs);
76
99
  const missing = onDisk.filter((f) => !isCovered(f, exactFiles, dirPrefixes));
77
100
  // Staleness: check only non-negation entries. Negation patterns (`!foo/`)
78
101
  // have no on-disk counterpart.
@@ -90,8 +113,6 @@ function isDirOnDisk(packageDir: string, entry: string): boolean {
90
113
  try {
91
114
  return statSync(resolve(packageDir, entry)).isDirectory();
92
115
  } catch {
93
- // Entry not present on disk (e.g. an asset/extraneous `files` entry) —
94
- // not a directory we can recurse into; the caller treats it as an exact file.
95
116
  return false;
96
117
  }
97
118
  }
@@ -108,14 +129,18 @@ function isCovered(
108
129
  return false;
109
130
  }
110
131
 
111
- function walkProductionTs(root: string, dir: string): string[] {
132
+ function walkProductionTs(
133
+ root: string,
134
+ dir: string,
135
+ skipDirs: Set<string>,
136
+ ): string[] {
112
137
  const out: string[] = [];
113
138
  for (const entry of readdirSync(dir, { withFileTypes: true })) {
114
139
  if (entry.name.startsWith(".")) continue;
115
- if (entry.isDirectory() && SKIP_DIRS.has(entry.name)) continue;
140
+ if (entry.isDirectory() && skipDirs.has(entry.name)) continue;
116
141
  const abs = resolve(dir, entry.name);
117
142
  if (entry.isDirectory()) {
118
- out.push(...walkProductionTs(root, abs));
143
+ out.push(...walkProductionTs(root, abs, skipDirs));
119
144
  continue;
120
145
  }
121
146
  if (!entry.isFile() || !entry.name.endsWith(".ts")) continue;
@@ -55,6 +55,9 @@ interface CacheEntry {
55
55
  */
56
56
  const activeSnapshotFiles = new Map<string, CacheEntry[]>();
57
57
 
58
+ /** Monotonic file-index counter — guarantees unique filenames across writes. */
59
+ let _snapshotIndexCounter = 0;
60
+
58
61
  // ─── Helpers ────────────────────────────────────────────────────────────
59
62
 
60
63
  /**
@@ -107,12 +110,10 @@ export function cacheSnapshot(
107
110
  const digest = sha256Prefix(snapshot);
108
111
  const existingEntries = activeSnapshotFiles.get(taskId) ?? [];
109
112
 
110
- // Determine index: next sequential index based on existing files
111
- const nextIndex = existingEntries.reduce((max, entry) => {
112
- const match = entry.path.match(/-(\d+)\.txt$/);
113
- const idx = match ? parseInt(match[1]!, 10) : -1;
114
- return Math.max(max, idx + 1);
115
- }, 0);
113
+ // Monotonic index — avoids filename collisions with currently-tracked
114
+ // entries (length would collide with surviving higher-index files
115
+ // after eviction; Date.now() can collide within the same ms).
116
+ const nextIndex = _snapshotIndexCounter++;
116
117
 
117
118
  const filePath = buildCacheFilePath(taskId, digest, nextIndex);
118
119
  writeFileSync(filePath, snapshot, "utf-8");
@@ -26,7 +26,6 @@ import { PORTAL_DATA_DIR } from "./paths.js";
26
26
 
27
27
  // ─── Constants ────────────────────────────────────────────────────────
28
28
 
29
- /** Root directory for all browser profiles. */
30
29
  export const PROFILE_DIR = join(PORTAL_DATA_DIR, "browser-state");
31
30
 
32
31
  /** Current storage state version. Increment on breaking format changes. */
@@ -38,7 +37,6 @@ const DEFAULT_MAX_STORAGE_STATE_SIZE = 10 * 1024 * 1024;
38
37
  /** Profile name validation regex. */
39
38
  const PROFILE_NAME_RE = /^[a-zA-Z0-9_-]{1,64}$/;
40
39
 
41
- /** Reserved keywords that cannot be used as profile names. */
42
40
  const RESERVED_PROFILE_NAMES = new Set([
43
41
  "none",
44
42
  "session", // profile modes
@@ -52,7 +50,6 @@ const RESERVED_PROFILE_NAMES = new Set([
52
50
  /** Prefix for auto-generated session-scoped profiles. */
53
51
  const SESSION_PROFILE_PREFIX = "_session-";
54
52
 
55
- /** Directory where pi stores active session tracking files. */
56
53
  export const SESSIONS_DIR = join(homedir(), ".pi", "agent", "sessions");
57
54
 
58
55
  // ─── Types ────────────────────────────────────────────────────────────
@@ -69,13 +66,11 @@ interface StoredCookie {
69
66
  sameSite: "Strict" | "Lax" | "None";
70
67
  }
71
68
 
72
- /** A localStorage entry for a given origin. */
73
69
  interface StoredLocalStorageEntry {
74
70
  name: string;
75
71
  value: string;
76
72
  }
77
73
 
78
- /** An origin with its localStorage data. */
79
74
  interface StoredOrigin {
80
75
  origin: string;
81
76
  localStorage: StoredLocalStorageEntry[];
@@ -153,9 +148,6 @@ export function profileDir(profileName: string): string {
153
148
  return join(PROFILE_DIR, safe);
154
149
  }
155
150
 
156
- /**
157
- * Get the filesystem path to a profile's storage state file.
158
- */
159
151
  export function profileFilePath(profileName: string): string {
160
152
  return join(profileDir(profileName), "storage-state.json");
161
153
  }
@@ -212,7 +204,6 @@ export function loadStorageState(
212
204
 
213
205
  // ─── Private Helpers ──────────────────────────────────────────────────
214
206
 
215
- /** Temp file prefix for atomic writes. */
216
207
  const TEMP_FILE_PREFIX = ".storage-state.";
217
208
  const TEMP_FILE_SUFFIX = ".tmp";
218
209
 
@@ -462,6 +453,46 @@ export function deleteStorageState(profileName: string): void {
462
453
 
463
454
  // ─── Session Profile Helpers ───────────────────────────────────────
464
455
 
456
+ /**
457
+ * Persist storage state for a session if it is marked persistent.
458
+ *
459
+ * Shared by `PlaywrightPluginBase._persistState` (which reads state directly
460
+ * from a Playwright `BrowserContext`) and `PythonPluginAdapter._persistState`
461
+ * (which fetches state over JSON-RPC). Both supply a `getStorageState`
462
+ * callback returning the raw `{ cookies, origins }` state; this helper owns
463
+ * the `persistState` gate, the `saveStorageState` call, and the
464
+ * warn-and-swallow error path so the two backends can't drift.
465
+ *
466
+ * @param session - The session-manager entry, or null/undefined. Only
467
+ * sessions with `persistState` true trigger a save.
468
+ * @param getStorageState - Callback that produces the raw storage state.
469
+ * @param viaLabel - Optional backend label inserted into the warning
470
+ * (e.g. `"via Python bridge"`) for diagnostic output.
471
+ * @returns The raw state object, or `undefined` if the session is
472
+ * non-persistent or the save failed.
473
+ */
474
+ export async function persistSessionState(
475
+ session: { persistState?: boolean; profileName?: string } | null | undefined,
476
+ getStorageState: () => Promise<{ cookies: unknown[]; origins: unknown[] }>,
477
+ viaLabel: string = "",
478
+ ): Promise<{ cookies: unknown[]; origins: unknown[] } | undefined> {
479
+ if (!session?.persistState) return undefined;
480
+ const name = session.profileName ?? "default";
481
+ try {
482
+ const state = await getStorageState();
483
+ saveStorageState(name, state);
484
+ return state;
485
+ } catch (err) {
486
+ console.warn(
487
+ `[pi-lean-portal] Failed to auto-save storage state for profile ` +
488
+ `'${name}'${viaLabel ? ` ${viaLabel}` : ""}: ` +
489
+ `${err instanceof Error ? err.message : String(err)}. ` +
490
+ "Session state may be lost.",
491
+ );
492
+ return undefined;
493
+ }
494
+ }
495
+
465
496
  /**
466
497
  * Check whether a profile name follows the session-scoped naming convention.
467
498
  *
@@ -4,9 +4,9 @@ import * as router from "./core/router.js";
4
4
  import { cleanupFetchTempFiles } from "./core/fetch-backend.js";
5
5
  import { pluginRegistry } from "./core/plugin-registry.js";
6
6
  import {
7
- loadPluginConfig,
7
+ loadFullConfig,
8
8
  detectPluginType,
9
- DEFAULT_BACKENDS_ROOT,
9
+ DEFAULT_BACKEND_ROOTS,
10
10
  invalidateConfigCache,
11
11
  } from "./core/plugin-config.js";
12
12
  import { ChromiumPlugin } from "./backends/chromium/index.js";
@@ -51,7 +51,12 @@ export default function (pi: ExtensionAPI) {
51
51
  resetToggleModuleState();
52
52
 
53
53
  // --- Plugin registration ----------------------------------------
54
- const { plugins: pluginConfigs, errors: configErrors } = loadPluginConfig();
54
+ // Resolve plugin `dir` values against the shipped backends root first,
55
+ // then the user-writable `~/.pi/agent/pi-lean-portal/user-backends/`
56
+ // tree (Phase 0b). An absolute `dir` short-circuits both roots.
57
+ const { plugins: pluginConfigs, errors: configErrors } = loadFullConfig(
58
+ DEFAULT_BACKEND_ROOTS,
59
+ ).plugins;
55
60
 
56
61
  // Log config errors
57
62
  for (const err of configErrors) {
@@ -68,7 +73,7 @@ export default function (pi: ExtensionAPI) {
68
73
  for (const config of pluginConfigs) {
69
74
  let detection;
70
75
  try {
71
- detection = detectPluginType(config.dir, DEFAULT_BACKENDS_ROOT);
76
+ detection = detectPluginType(config.dir, DEFAULT_BACKEND_ROOTS);
72
77
  } catch (err) {
73
78
  console.error(
74
79
  `[pi-lean-portal] Plugin '${config.name}' (dir: '${config.dir}'): ${err instanceof Error ? err.message : String(err)}`,
@@ -111,8 +116,8 @@ export default function (pi: ExtensionAPI) {
111
116
  // ── Second pass: load and register plugins ───────────────────
112
117
  // Node plugins register asynchronously (after dynamic import resolves).
113
118
  // Python plugins register synchronously here.
114
- // The pre-seeded ordering ensures all plugins get the correct priority level
115
- // regardless of when register() is called.
119
+ // The pre-seeded ordering ensures all plugins keep their configured
120
+ // position regardless of when register() is called.
116
121
  for (const { config, detection } of validConfigs) {
117
122
  if (detection.type === "node") {
118
123
  // Node-based backend — dynamically import the detected plugin
@@ -182,6 +187,36 @@ export default function (pi: ExtensionAPI) {
182
187
  }
183
188
 
184
189
  // --- Register tools ---------------------------------------------
190
+ // Patch the browser-navigate strategy description with the actually
191
+ // configured plugin names so the agent doesn't second-guess which
192
+ // strategies exist (matches what /web status reports).
193
+ const strategyPlugins =
194
+ validConfigs.length > 0
195
+ ? validConfigs.map(({ config }) => ({
196
+ name: config.name,
197
+ enabled: config.enabled,
198
+ }))
199
+ : [{ name: "chromium", enabled: true }]; // fallback path
200
+ const enabledNames = strategyPlugins
201
+ .filter((p) => p.enabled)
202
+ .map((p) => p.name);
203
+ const disabledNames = strategyPlugins
204
+ .filter((p) => !p.enabled)
205
+ .map((p) => p.name);
206
+ const availList =
207
+ enabledNames.length > 0 ? enabledNames.join(", ") : "(none)";
208
+ const disabledClause =
209
+ disabledNames.length > 0 ? ` Disabled: ${disabledNames.join(", ")}.` : "";
210
+ (
211
+ browserNavigateTool as unknown as {
212
+ parameters: { properties: { strategy: { description: string } } };
213
+ }
214
+ ).parameters.properties.strategy.description =
215
+ `Backend strategy: "auto" (default) uses the first available plugin; ` +
216
+ `specify a registered plugin name to use that backend. ` +
217
+ `Available: ${availList}.${disabledClause} ` +
218
+ `For stateless HTTP fetches, use web-fetch instead.`;
219
+
185
220
  pi.registerTool(webFetchTool);
186
221
  pi.registerTool(browserNavigateTool);
187
222
  pi.registerTool(browserSnapshotTool);
@@ -238,7 +273,7 @@ export default function (pi: ExtensionAPI) {
238
273
 
239
274
  // Clean up all registered plugins
240
275
  const ordered = pluginRegistry.getOrdered();
241
- for (const { plugin } of ordered) {
276
+ for (const plugin of ordered) {
242
277
  await plugin.cleanupAll().catch(() => {});
243
278
  }
244
279
 
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "pi-lean-portal",
3
- "version": "0.1.0",
4
- "description": "Pi extension. Interactive web browsing for Pi — Playwright Chromium/Firefox, accessibility-tree snapshots, profiles, cookies, guides. Owns the /web command.",
3
+ "version": "0.2.1",
4
+ "description": "Pi extension. Interactive web browsing for Pi — a /web toggle removes the tools from context when switched off, Playwright Chromium/Firefox deliver accessibility-tree snapshots, persistent profiles, cookies, and domain-matched guides; custom/stealth backends (Camoufox) plug in when a site blocks the shipped browsers.",
5
5
  "keywords": [
6
6
  "pi-package",
7
7
  "pi-extension",
@@ -20,6 +20,12 @@
20
20
  "publishConfig": {
21
21
  "access": "public"
22
22
  },
23
+ "exports": {
24
+ ".": {
25
+ "types": "./core/plugin-api.ts",
26
+ "default": "./index.ts"
27
+ }
28
+ },
23
29
  "files": [
24
30
  "index.ts",
25
31
  "browser-toggle.ts",
@@ -35,7 +41,6 @@
35
41
  "!backends/python-base/**/*.egg-info/",
36
42
  "core/",
37
43
  "tools/",
38
- "verify-ship-manifest.ts",
39
44
  ".npmrc",
40
45
  "README.md",
41
46
  "ship-manifest.test.ts"