fable-engine 1.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. fable_compressor.py +356 -0
  2. fable_engine/__init__.py +1 -0
  3. fable_engine/actions/__init__.py +291 -0
  4. fable_engine/actions/cas.py +182 -0
  5. fable_engine/actions/deliberation.py +523 -0
  6. fable_engine/actions/fleet.py +807 -0
  7. fable_engine/actions/lifecycle.py +298 -0
  8. fable_engine/actions/scrapers.py +116 -0
  9. fable_engine/actions/system3.py +815 -0
  10. fable_engine/browser.py +824 -0
  11. fable_engine/cas.py +974 -0
  12. fable_engine/fable_session.json +510 -0
  13. fable_engine/guards.py +283 -0
  14. fable_engine/schema.py +714 -0
  15. fable_engine/scrapers/__init__.py +32 -0
  16. fable_engine/scrapers/arxiv.py +115 -0
  17. fable_engine/scrapers/base.py +386 -0
  18. fable_engine/scrapers/github.py +129 -0
  19. fable_engine/scrapers/reddit.py +154 -0
  20. fable_engine/scrapers/web.py +120 -0
  21. fable_engine/scrapers/x.py +125 -0
  22. fable_engine/scrapers/youtube.py +132 -0
  23. fable_engine/server.py +414 -0
  24. fable_engine/session.py +1819 -0
  25. fable_engine/test_server.py +1362 -0
  26. fable_engine/updater.py +541 -0
  27. fable_engine-1.3.1.dist-info/LICENSE +22 -0
  28. fable_engine-1.3.1.dist-info/METADATA +173 -0
  29. fable_engine-1.3.1.dist-info/RECORD +104 -0
  30. fable_engine-1.3.1.dist-info/WHEEL +5 -0
  31. fable_engine-1.3.1.dist-info/entry_points.txt +5 -0
  32. fable_engine-1.3.1.dist-info/top_level.txt +6 -0
  33. fable_mode/__init__.py +3 -0
  34. fable_mode/__main__.py +4 -0
  35. fable_mode/adapters.py +1014 -0
  36. fable_mode/installer.py +553 -0
  37. fable_mode/launcher.py +437 -0
  38. fable_mode/manifest.py +142 -0
  39. fable_mode/resources.json +114 -0
  40. fable_mode/safety.py +103 -0
  41. fable_mode_entry.py +10 -0
  42. fable_v2/__init__.py +146 -0
  43. fable_v2/adapters.py +151 -0
  44. fable_v2/coder_fleet/__init__.py +100 -0
  45. fable_v2/coder_fleet/ast_tools.py +158 -0
  46. fable_v2/coder_fleet/compute.py +199 -0
  47. fable_v2/coder_fleet/design_engine.py +1316 -0
  48. fable_v2/coder_fleet/diagnostics.py +293 -0
  49. fable_v2/coder_fleet/fleet_dispatcher.py +214 -0
  50. fable_v2/coder_fleet/mock_auditor.py +306 -0
  51. fable_v2/coder_fleet/mutation.py +216 -0
  52. fable_v2/coder_fleet/property_oracle.py +260 -0
  53. fable_v2/coder_fleet/receipt_attestor.py +122 -0
  54. fable_v2/coder_fleet/red_team_swarm.py +908 -0
  55. fable_v2/coder_fleet/test_harness.py +198 -0
  56. fable_v2/coder_fleet/vector_engine.py +1287 -0
  57. fable_v2/coder_fleet/visual.py +357 -0
  58. fable_v2/coder_fleet/workspace.py +153 -0
  59. fable_v2/cortical/__init__.py +20 -0
  60. fable_v2/cortical/plasticity_engine.py +992 -0
  61. fable_v2/execution_broker.py +811 -0
  62. fable_v2/proof_engine.py +1141 -0
  63. fable_v2/protocol.py +485 -0
  64. fable_v2/runtime.py +1010 -0
  65. fable_v2/system3/__init__.py +204 -0
  66. fable_v2/system3/causal.py +558 -0
  67. fable_v2/system3/dialectical.py +577 -0
  68. fable_v2/system3/evolution.py +503 -0
  69. fable_v2/system3/executive.py +338 -0
  70. fable_v2/system3/free_energy.py +479 -0
  71. fable_v2/system3/hyperbolic.py +555 -0
  72. fable_v2/system3/induction.py +336 -0
  73. fable_v2/system3/kripke.py +548 -0
  74. fable_v2/system3/oracle.py +745 -0
  75. fable_v2/verifiers.py +72 -0
  76. tests/__init__.py +1 -0
  77. tests/test_anti_loop_circuit_breaker.py +64 -0
  78. tests/test_auto_updater.py +407 -0
  79. tests/test_coder_fleet.py +535 -0
  80. tests/test_delegation_compiler.py +54 -0
  81. tests/test_descriptor_boundaries.py +126 -0
  82. tests/test_design_engine.py +603 -0
  83. tests/test_epistemic_evidence_validator.py +66 -0
  84. tests/test_execution_broker.py +233 -0
  85. tests/test_fable_v2.py +406 -0
  86. tests/test_fleet_transitions.py +116 -0
  87. tests/test_fsm_redteam_evolution.py +406 -0
  88. tests/test_goal_rubric_and_pipeline.py +367 -0
  89. tests/test_hebbian_plasticity.py +585 -0
  90. tests/test_packaging_runtime.py +194 -0
  91. tests/test_proof_engine.py +259 -0
  92. tests/test_red_team_swarm.py +645 -0
  93. tests/test_redteam_remediation.py +169 -0
  94. tests/test_registration_transaction.py +375 -0
  95. tests/test_requested_regressions.py +467 -0
  96. tests/test_scrapers.py +370 -0
  97. tests/test_server_actions.py +93 -0
  98. tests/test_server_frontier_actions.py +269 -0
  99. tests/test_server_protocol.py +88 -0
  100. tests/test_stealth_browser.py +970 -0
  101. tests/test_system3.py +381 -0
  102. tests/test_system3_deep_integration.py +385 -0
  103. tests/test_system3_frontier.py +436 -0
  104. tests/test_vector_engine.py +608 -0
@@ -0,0 +1,824 @@
1
+ """
2
+ Stealth Agent Browser Engine for Fable Engine.
3
+ A lightweight, agent-first browser engine designed for low memory footprint (<= 20 MB RSS target),
4
+ persistent local logins/profiles (~/.fable/browser-profile/), localhost support across any language server,
5
+ element indexing, navigation, and layered PNG screenshot generation.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import base64
11
+ import http.cookiejar
12
+ from html.parser import HTMLParser
13
+ import ipaddress
14
+ import json
15
+ import logging
16
+ import math
17
+ import os
18
+ import pathlib
19
+ import re
20
+ import struct
21
+ import sys
22
+ import threading
23
+ import time
24
+ import urllib.parse
25
+ import urllib.request
26
+ import zlib
27
+ from typing import Any, Dict, List, Optional, Tuple
28
+
29
+ logger = logging.getLogger("fable-engine.browser")
30
+
31
+ DEFAULT_PROFILE_DIR = pathlib.Path.home() / ".fable" / "browser-profile"
32
+ DEFAULT_VIEWPORT_WIDTH = 1280
33
+ DEFAULT_VIEWPORT_HEIGHT = 800
34
+ MAX_BROWSER_SESSIONS = 32
35
+ MAX_RETAINED_PAGE_HTML_BYTES = 16 * 1024 * 1024
36
+ DEFAULT_MAX_RESPONSE_BYTES = MAX_RETAINED_PAGE_HTML_BYTES // MAX_BROWSER_SESSIONS
37
+ DEFAULT_BROWSER_OPEN_TIMEOUT_SECONDS = 15.0
38
+ MAX_BROWSER_OPEN_TIMEOUT_SECONDS = 30.0
39
+ MAX_SCREENSHOT_LAYERS = 10
40
+ MAX_DOM_NODES = 10_000
41
+ MAX_HISTORY_ENTRIES = 100
42
+
43
+
44
+ def bound_browser_timeout(timeout: float) -> float:
45
+ """Return a finite, non-negative browser timeout capped at the server limit."""
46
+ value = float(timeout)
47
+ if not math.isfinite(value):
48
+ raise ValueError("Browser timeout must be finite")
49
+ if value < 0:
50
+ raise ValueError("Browser timeout must be non-negative")
51
+ return min(value, MAX_BROWSER_OPEN_TIMEOUT_SECONDS)
52
+
53
+
54
+ class DOMElement:
55
+ """Represents a node in the rendered DOM tree with stable element ID and layout box."""
56
+
57
+ def __init__(
58
+ self,
59
+ element_id: str,
60
+ tag: str,
61
+ attrs: Dict[str, str],
62
+ text: str = "",
63
+ children: Optional[List[DOMElement]] = None,
64
+ ):
65
+ self.element_id = element_id
66
+ self.tag = tag.lower()
67
+ self.attrs = attrs
68
+ self.text = text
69
+ self.children = children or []
70
+ self.parent: Optional[DOMElement] = None
71
+ self.x = 0
72
+ self.y = 0
73
+ self.width = 0
74
+ self.height = 0
75
+
76
+ def to_dict(self) -> Dict[str, Any]:
77
+ attrs = self.attrs
78
+ if self.attrs.get("type", "").lower() == "password" and "value" in self.attrs:
79
+ attrs = dict(self.attrs)
80
+ attrs["value"] = "[REDACTED]"
81
+ return {
82
+ "element_id": self.element_id,
83
+ "tag": self.tag,
84
+ "attrs": attrs,
85
+ "text": self.text.strip(),
86
+ "bbox": [self.x, self.y, self.width, self.height],
87
+ "children_count": len(self.children),
88
+ }
89
+
90
+
91
+ class BrowserDocumentTooLargeError(ValueError):
92
+ """Raised when a page exceeds the browser's bounded DOM size."""
93
+
94
+
95
+ class NoHTTPSDowngradeRedirectHandler(urllib.request.HTTPRedirectHandler):
96
+ """Follow normal HTTP redirects except HTTPS-to-HTTP downgrades."""
97
+
98
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
99
+ source_scheme = urllib.parse.urlsplit(req.full_url).scheme.lower()
100
+ target_scheme = urllib.parse.urlsplit(
101
+ urllib.parse.urljoin(req.full_url, newurl)
102
+ ).scheme.lower()
103
+ if source_scheme == "https" and target_scheme == "http":
104
+ return None
105
+ redirected = super().redirect_request(req, fp, code, msg, headers, newurl)
106
+ if redirected is not None:
107
+ for attr in ("_fable_navigation_deadline", "_fable_navigation_timeout"):
108
+ if hasattr(req, attr):
109
+ setattr(redirected, attr, getattr(req, attr))
110
+ return redirected
111
+
112
+ def http_error_302(self, req, fp, code, msg, headers):
113
+ deadline = getattr(req, "_fable_navigation_deadline", None)
114
+ if deadline is not None:
115
+ timeout = getattr(req, "_fable_navigation_timeout", req.timeout)
116
+ remaining = deadline - time.monotonic()
117
+ if remaining <= 0:
118
+ raise TimeoutError(
119
+ f"Browser navigation timed out after {timeout:g} seconds"
120
+ )
121
+ req.timeout = remaining
122
+ return super().http_error_302(req, fp, code, msg, headers)
123
+
124
+ http_error_301 = http_error_303 = http_error_307 = http_error_308 = http_error_302
125
+
126
+
127
+ def _is_loopback_target(url: str) -> bool:
128
+ """Return whether a URL or scheme-less target names the local machine."""
129
+ try:
130
+ parsed = urllib.parse.urlsplit(url)
131
+ if parsed.scheme.lower() in ("http", "https"):
132
+ hostname = parsed.hostname
133
+ else:
134
+ hostname = urllib.parse.urlsplit(f"//{url}").hostname
135
+ except ValueError:
136
+ return False
137
+ if not hostname:
138
+ return False
139
+ hostname = hostname.rstrip(".").lower()
140
+ if hostname == "localhost":
141
+ return True
142
+ try:
143
+ return ipaddress.ip_address(hostname).is_loopback
144
+ except ValueError:
145
+ return False
146
+
147
+
148
+ class NoRemoteHTTPCookiePolicy(http.cookiejar.DefaultCookiePolicy):
149
+ """Never return persistent cookies over remote cleartext HTTP."""
150
+
151
+ def return_ok(self, cookie, request):
152
+ parsed = urllib.parse.urlsplit(request.full_url)
153
+ if parsed.scheme.lower() == "http" and not _is_loopback_target(request.full_url):
154
+ return False
155
+ return super().return_ok(cookie, request)
156
+
157
+
158
+ class SimpleDOMParser(HTMLParser):
159
+ """HTML parser that constructs a clean DOM tree with stable element IDs."""
160
+
161
+ def __init__(self, max_nodes: int = MAX_DOM_NODES):
162
+ super().__init__()
163
+ self.root = DOMElement("elem_0", "root", {})
164
+ self.stack: List[DOMElement] = [self.root]
165
+ self.counter = 1
166
+ self.assigned_ids = {self.root.element_id}
167
+ self.max_nodes = max(1, min(int(max_nodes), MAX_DOM_NODES))
168
+ self.node_count = 0
169
+
170
+ def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]):
171
+ if self.node_count >= self.max_nodes:
172
+ raise BrowserDocumentTooLargeError(
173
+ f"Document exceeds the {self.max_nodes} DOM node limit"
174
+ )
175
+ attr_dict = {k: (v or "") for k, v in attrs}
176
+ base_id = attr_dict.get("id") or f"elem_{self.counter}"
177
+ self.counter += 1
178
+ elem_id = base_id
179
+ suffix = 2
180
+ while elem_id in self.assigned_ids:
181
+ elem_id = f"{base_id}_{suffix}"
182
+ suffix += 1
183
+ self.assigned_ids.add(elem_id)
184
+ elem = DOMElement(elem_id, tag, attr_dict)
185
+ self.node_count += 1
186
+ elem.parent = self.stack[-1]
187
+ self.stack[-1].children.append(elem)
188
+ if tag.lower() not in ("img", "br", "hr", "input", "meta", "link"):
189
+ self.stack.append(elem)
190
+
191
+ def handle_endtag(self, tag: str):
192
+ if len(self.stack) > 1 and self.stack[-1].tag == tag.lower():
193
+ self.stack.pop()
194
+
195
+ def handle_data(self, data: str):
196
+ if self.stack:
197
+ clean_text = data.strip()
198
+ if clean_text:
199
+ if self.stack[-1].text:
200
+ self.stack[-1].text += " " + clean_text
201
+ else:
202
+ self.stack[-1].text = clean_text
203
+
204
+
205
+ def generate_minimal_png(width: int, height: int, elements: List[DOMElement], bg_color: Tuple[int, int, int] = (245, 247, 250)) -> bytes:
206
+ """
207
+ Generates a valid 24-bit RGB PNG image buffer without external image dependencies (pure stdlib struct + zlib).
208
+ Renders background canvas and simple color-coded layout boxes for DOM elements.
209
+ """
210
+ # Clamp maximum raster dimensions to avoid excessive memory during snapshot
211
+ w = min(max(width, 100), 1280)
212
+ h = min(max(height, 100), 2400)
213
+
214
+ # 3 bytes per pixel RGB
215
+ row_bytes = w * 3
216
+ bg_r, bg_g, bg_b = bg_color
217
+ # Prefix each scanline with filter type 0, then repeat the RGB background.
218
+ scanline = b"\x00" + bytes((bg_r, bg_g, bg_b)) * w
219
+ raw_data = bytearray(scanline * h)
220
+
221
+ # Simple element box rasterization
222
+ for elem in elements:
223
+ if elem.width <= 0 or elem.height <= 0:
224
+ continue
225
+ # Color coding by tag
226
+ if elem.tag in ("a", "button", "input"):
227
+ box_r, box_g, box_b = (59, 130, 246) # Blue
228
+ elif elem.tag in ("h1", "h2", "h3", "header"):
229
+ box_r, box_g, box_b = (30, 41, 59) # Dark Slate
230
+ else:
231
+ box_r, box_g, box_b = (203, 213, 225) # Light Slate Border
232
+
233
+ x1 = max(0, min(elem.x, w - 1))
234
+ y1 = max(0, min(elem.y, h - 1))
235
+ x2 = max(0, min(elem.x + elem.width, w - 1))
236
+ y2 = max(0, min(elem.y + elem.height, h - 1))
237
+
238
+ # Fill/Border
239
+ for py in range(y1, y2 + 1):
240
+ row_offset = py * (row_bytes + 1)
241
+ for px_x in range(x1, x2 + 1):
242
+ is_border = (px_x == x1 or px_x == x2 or py == y1 or py == y2)
243
+ if is_border or elem.tag in ("button", "input"):
244
+ idx = row_offset + 1 + px_x * 3
245
+ raw_data[idx] = box_r
246
+ raw_data[idx + 1] = box_g
247
+ raw_data[idx + 2] = box_b
248
+
249
+ compressed = zlib.compress(bytes(raw_data), level=6)
250
+
251
+ # Build PNG chunks
252
+ ihdr_data = struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)
253
+ ihdr_crc = zlib.crc32(b"IHDR" + ihdr_data)
254
+ ihdr_chunk = struct.pack(">I", 13) + b"IHDR" + ihdr_data + struct.pack(">I", ihdr_crc)
255
+
256
+ idat_len = len(compressed)
257
+ idat_crc = zlib.crc32(b"IDAT" + compressed)
258
+ idat_chunk = struct.pack(">I", idat_len) + b"IDAT" + compressed + struct.pack(">I", idat_crc)
259
+
260
+ iend_crc = zlib.crc32(b"IEND")
261
+ iend_chunk = struct.pack(">I", 0) + b"IEND" + struct.pack(">I", iend_crc)
262
+
263
+ png_header = b"\x89PNG\r\n\x1a\n"
264
+ return png_header + ihdr_chunk + idat_chunk + iend_chunk
265
+
266
+
267
+ class ProfileManager:
268
+ """Manages persistent cookies, session data, and login state on local user machine."""
269
+
270
+ def __init__(self, profile_dir: Optional[pathlib.Path | str] = None):
271
+ if profile_dir is None:
272
+ self.profile_dir = DEFAULT_PROFILE_DIR
273
+ else:
274
+ self.profile_dir = pathlib.Path(profile_dir)
275
+ self.profile_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
276
+ self.profile_dir.chmod(0o700)
277
+ self.cookie_file = self.profile_dir / "cookies.txt"
278
+ self.storage_file = self.profile_dir / "local_storage.json"
279
+ self.cookies = http.cookiejar.MozillaCookieJar(
280
+ str(self.cookie_file), policy=NoRemoteHTTPCookiePolicy()
281
+ )
282
+ self.local_storage: Dict[str, Any] = {}
283
+ self._load()
284
+
285
+ @staticmethod
286
+ def _ensure_private_file(path: pathlib.Path) -> None:
287
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0o600)
288
+ try:
289
+ if hasattr(os, "fchmod"):
290
+ os.fchmod(fd, 0o600)
291
+ finally:
292
+ os.close(fd)
293
+
294
+ def _load(self):
295
+ if self.cookie_file.exists():
296
+ try:
297
+ try:
298
+ self.cookie_file.chmod(0o600)
299
+ except OSError:
300
+ pass
301
+ self.cookies.load(ignore_discard=True, ignore_expires=True)
302
+ except Exception as e:
303
+ logger.warning(f"Could not load cookies from {self.cookie_file}: {e}")
304
+ self.cookies.clear()
305
+ if self.storage_file.exists():
306
+ try:
307
+ try:
308
+ self.storage_file.chmod(0o600)
309
+ except OSError:
310
+ pass
311
+ with open(self.storage_file, "r", encoding="utf-8") as f:
312
+ self.local_storage = json.load(f)
313
+ except Exception as e:
314
+ logger.warning(f"Could not load local_storage from {self.storage_file}: {e}")
315
+ self.local_storage = {}
316
+
317
+ def save(self):
318
+ try:
319
+ self._ensure_private_file(self.cookie_file)
320
+ self.cookies.save(ignore_discard=True, ignore_expires=True)
321
+ try:
322
+ self.cookie_file.chmod(0o600)
323
+ except OSError:
324
+ pass
325
+ storage_fd = os.open(self.storage_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
326
+ if hasattr(os, "fchmod"):
327
+ os.fchmod(storage_fd, 0o600)
328
+ with os.fdopen(storage_fd, "w", encoding="utf-8") as f:
329
+ json.dump(self.local_storage, f, indent=2)
330
+ try:
331
+ self.storage_file.chmod(0o600)
332
+ except OSError:
333
+ pass
334
+ except Exception as e:
335
+ logger.warning(f"Failed to save profile state: {e}")
336
+
337
+
338
+ class StealthBrowserSession:
339
+ """
340
+ Active browser tab session.
341
+ Parses HTML, layouts elements into coordinates, handles inputs, and captures layered PNG screenshots.
342
+ """
343
+
344
+ def __init__(
345
+ self,
346
+ session_id: str,
347
+ profile_manager: ProfileManager,
348
+ viewport_width: int = DEFAULT_VIEWPORT_WIDTH,
349
+ viewport_height: int = DEFAULT_VIEWPORT_HEIGHT,
350
+ max_response_bytes: int = DEFAULT_MAX_RESPONSE_BYTES,
351
+ ):
352
+ self.session_id = session_id
353
+ self.profile_manager = profile_manager
354
+ self.viewport_width = viewport_width
355
+ self.viewport_height = viewport_height
356
+ self.max_response_bytes = max(1, int(max_response_bytes))
357
+ self.opener = urllib.request.build_opener(
358
+ urllib.request.HTTPCookieProcessor(self.profile_manager.cookies),
359
+ NoHTTPSDowngradeRedirectHandler(),
360
+ )
361
+ self.credential_free_opener = urllib.request.build_opener(
362
+ NoHTTPSDowngradeRedirectHandler()
363
+ )
364
+ self.url = "about:blank"
365
+ self.history: List[str] = []
366
+ self.history_index = -1
367
+ self.dom_root: Optional[DOMElement] = None
368
+ self.elements_by_id: Dict[str, DOMElement] = {}
369
+ self.scroll_y = 0
370
+ self.document_height = DEFAULT_VIEWPORT_HEIGHT
371
+ self.page_title = ""
372
+ self.page_html = ""
373
+ self.active_element_id: Optional[str] = None
374
+
375
+ def open(
376
+ self,
377
+ url: str,
378
+ timeout: float = DEFAULT_BROWSER_OPEN_TIMEOUT_SECONDS,
379
+ *,
380
+ record_history: bool = True,
381
+ ) -> Dict[str, Any]:
382
+ """Opens a URL (including localhost) using urllib with persistent cookies and standard headers."""
383
+ timeout = bound_browser_timeout(timeout)
384
+ scheme = urllib.parse.urlsplit(url).scheme.lower()
385
+ if scheme not in ("http", "https", "about"):
386
+ default_scheme = "http" if self._is_loopback_target(url) else "https"
387
+ url = f"{default_scheme}://{url}"
388
+ scheme = default_scheme
389
+
390
+ if scheme == "about":
391
+ page_html = "<html><head><title>Blank Page</title></head><body><h1>Blank Page</h1></body></html>"
392
+ self._commit_navigation(url, page_html, record_history)
393
+ return self._build_status()
394
+
395
+ req = urllib.request.Request(url)
396
+ req.add_header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
397
+ req.add_header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
398
+ req.add_header("Accept-Language", "en-US,en;q=0.9")
399
+
400
+ deadline = time.monotonic() + timeout
401
+ req._fable_navigation_deadline = deadline
402
+ req._fable_navigation_timeout = timeout
403
+ try:
404
+ remaining = deadline - time.monotonic()
405
+ if remaining <= 0:
406
+ raise TimeoutError(f"Browser navigation timed out after {timeout:g} seconds")
407
+ opener = self.opener
408
+ if scheme == "http" and not self._is_loopback_target(url):
409
+ opener = self.credential_free_opener
410
+ with opener.open(req, timeout=remaining) as resp:
411
+ body_bytes = self._read_response_body(resp, deadline, timeout)
412
+ self.profile_manager.save()
413
+ if len(body_bytes) > self.max_response_bytes:
414
+ return {
415
+ "status": "error",
416
+ "error": f"Response body exceeds {self.max_response_bytes} byte limit",
417
+ "url": url,
418
+ }
419
+ body = body_bytes.decode("utf-8", errors="replace")
420
+ response_url = resp.geturl()
421
+ self._commit_navigation(response_url, body, record_history)
422
+ return self._build_status()
423
+ except Exception as e:
424
+ logger.error(f"Browser navigation error to {url}: {e}")
425
+ return {"status": "error", "error": str(e), "url": url}
426
+
427
+ @staticmethod
428
+ def _is_loopback_target(url: str) -> bool:
429
+ return _is_loopback_target(url)
430
+
431
+ def _commit_navigation(
432
+ self,
433
+ url: str,
434
+ page_html: str,
435
+ record_history: bool,
436
+ ) -> None:
437
+ """Parse a candidate document before publishing any navigation state."""
438
+ dom_root, elements_by_id, document_height, page_title = self._layout_document(
439
+ url, page_html
440
+ )
441
+ self.url = url
442
+ self.page_html = page_html
443
+ self.dom_root = dom_root
444
+ self.elements_by_id = elements_by_id
445
+ self.document_height = document_height
446
+ self.page_title = page_title
447
+ self.active_element_id = None
448
+ self.scroll_y = 0
449
+ if record_history:
450
+ self._record_history(url)
451
+
452
+ def _read_response_body(self, response: Any, deadline: float, timeout: float) -> bytes:
453
+ """Read a bounded response while enforcing the navigation's monotonic deadline."""
454
+ body = bytearray()
455
+ read_chunk = getattr(response, "read1", response.read)
456
+ while len(body) <= self.max_response_bytes:
457
+ remaining = deadline - time.monotonic()
458
+ if remaining <= 0:
459
+ raise TimeoutError(f"Browser navigation timed out after {timeout:g} seconds")
460
+ self._set_response_socket_timeout(response, remaining)
461
+ chunk = read_chunk(min(64 * 1024, self.max_response_bytes + 1 - len(body)))
462
+ if not chunk:
463
+ break
464
+ body.extend(chunk)
465
+ return bytes(body)
466
+
467
+ @staticmethod
468
+ def _set_response_socket_timeout(response: Any, timeout: float) -> None:
469
+ """Apply the remaining deadline to urllib's underlying response socket."""
470
+ stream = response
471
+ for _ in range(4):
472
+ sock = getattr(stream, "_sock", None)
473
+ if sock is not None and hasattr(sock, "settimeout"):
474
+ sock.settimeout(timeout)
475
+ return
476
+ stream = getattr(stream, "raw", None) or getattr(stream, "fp", None)
477
+ if stream is None:
478
+ return
479
+
480
+ def _record_history(self, url: str):
481
+ if self.history_index >= 0 and self.history_index < len(self.history):
482
+ self.history = self.history[: self.history_index + 1]
483
+ if self.history[self.history_index] == url:
484
+ return
485
+ self.history.append(url)
486
+ self.history_index = len(self.history) - 1
487
+ overflow = len(self.history) - MAX_HISTORY_ENTRIES
488
+ if overflow > 0:
489
+ del self.history[:overflow]
490
+ self.history_index -= overflow
491
+
492
+ def back(self) -> Dict[str, Any]:
493
+ if self.history_index > 0:
494
+ target_index = self.history_index - 1
495
+ result = self.open(self.history[target_index], record_history=False)
496
+ if result.get("status") != "error":
497
+ self.history_index = target_index
498
+ return result
499
+ return self._build_status()
500
+
501
+ def forward(self) -> Dict[str, Any]:
502
+ if self.history_index < len(self.history) - 1:
503
+ target_index = self.history_index + 1
504
+ result = self.open(self.history[target_index], record_history=False)
505
+ if result.get("status") != "error":
506
+ self.history_index = target_index
507
+ return result
508
+ return self._build_status()
509
+
510
+ def reload(self) -> Dict[str, Any]:
511
+ if self.url and not self.url.startswith("about:"):
512
+ return self.open(self.url, record_history=False)
513
+ return self._build_status()
514
+
515
+ def _parse_and_layout(self):
516
+ dom_root, elements_by_id, document_height, page_title = self._layout_document(
517
+ self.url, self.page_html
518
+ )
519
+ self.dom_root = dom_root
520
+ self.elements_by_id = elements_by_id
521
+ self.document_height = document_height
522
+ self.page_title = page_title
523
+ self.active_element_id = None
524
+
525
+ def _layout_document(
526
+ self,
527
+ url: str,
528
+ page_html: str,
529
+ ) -> Tuple[DOMElement, Dict[str, DOMElement], int, str]:
530
+ parser = SimpleDOMParser()
531
+ try:
532
+ parser.feed(page_html)
533
+ except BrowserDocumentTooLargeError:
534
+ raise
535
+ except Exception:
536
+ pass
537
+
538
+ dom_root = parser.root
539
+ elements_by_id: Dict[str, DOMElement] = {}
540
+
541
+ # Extract title
542
+ title_match = re.search(r"<title>(.*?)</title>", page_html, re.IGNORECASE | re.DOTALL)
543
+ page_title = title_match.group(1).strip() if title_match else url
544
+
545
+ # Compute layout boxes
546
+ current_y = 20
547
+ all_elems: List[DOMElement] = []
548
+
549
+ def traverse(root: DOMElement):
550
+ nonlocal current_y
551
+ stack = [root]
552
+ while stack:
553
+ node = stack.pop()
554
+ elements_by_id[node.element_id] = node
555
+ all_elems.append(node)
556
+
557
+ if node.tag in ("h1", "h2", "h3", "p", "div", "button", "input", "a", "section"):
558
+ node.x = 40
559
+ node.y = current_y
560
+ node.width = self.viewport_width - 80
561
+ node.height = 36 if node.tag in ("button", "input") else 24
562
+ current_y += node.height + 12
563
+
564
+ stack.extend(reversed(node.children))
565
+
566
+ traverse(dom_root)
567
+ document_height = max(self.viewport_height, current_y + 40)
568
+ return dom_root, elements_by_id, document_height, page_title
569
+
570
+ def scroll(self, delta_y: int) -> Dict[str, Any]:
571
+ self.scroll_y = max(0, min(self.scroll_y + delta_y, self.document_height - self.viewport_height))
572
+ return self._build_status()
573
+
574
+ def click(self, element_id: str) -> Dict[str, Any]:
575
+ """Follow an element's link target; non-link controls are unsupported."""
576
+ elem = self.elements_by_id.get(element_id)
577
+ if not elem:
578
+ return {"error": f"Element '{element_id}' not found"}
579
+ href = elem.attrs.get("href")
580
+ if href is not None:
581
+ target_url = urllib.parse.urljoin(self.url, href)
582
+ return self.open(target_url)
583
+ return {
584
+ "status": "error",
585
+ "error": f"Click is unsupported for <{elem.tag}> elements without a link target",
586
+ "element_id": element_id,
587
+ }
588
+
589
+ def type_text(self, element_id: str, text: str) -> Dict[str, Any]:
590
+ elem = self.elements_by_id.get(element_id)
591
+ if not elem:
592
+ return {"error": f"Element '{element_id}' not found"}
593
+ input_type = elem.attrs.get("type", "text").lower()
594
+ editable_input_types = {
595
+ "text", "search", "email", "url", "tel", "password", "number",
596
+ "date", "datetime-local", "month", "time", "week",
597
+ }
598
+ if elem.tag != "textarea" and not (
599
+ elem.tag == "input" and input_type in editable_input_types
600
+ ):
601
+ return {
602
+ "status": "error",
603
+ "error": f"Typing is unsupported for non-editable <{elem.tag}> elements",
604
+ "element_id": element_id,
605
+ }
606
+ elem.attrs["value"] = text
607
+ return {"status": "typed", "element_id": element_id, "text": text}
608
+
609
+ def close(self) -> None:
610
+ """Release per-session resources and discard retained page state."""
611
+ try:
612
+ self.opener.close()
613
+ finally:
614
+ try:
615
+ self.credential_free_opener.close()
616
+ finally:
617
+ self.history.clear()
618
+ self.history_index = -1
619
+ self.dom_root = None
620
+ self.elements_by_id.clear()
621
+ self.page_html = ""
622
+ self.page_title = ""
623
+ self.active_element_id = None
624
+
625
+ def press_key(self, key: str, element_id: Optional[str] = None) -> Dict[str, Any]:
626
+ if not key:
627
+ return {"status": "error", "error": "Keyboard key must not be empty"}
628
+
629
+ if element_id is not None:
630
+ if element_id not in self.elements_by_id:
631
+ return {"status": "error", "error": f"Element '{element_id}' not found"}
632
+ self.active_element_id = element_id
633
+
634
+ if key == "Tab":
635
+ focusable = [
636
+ elem.element_id for elem in self.elements_by_id.values()
637
+ if elem.tag in ("a", "button", "input", "textarea")
638
+ ]
639
+ if not focusable:
640
+ return {
641
+ "status": "error",
642
+ "error": "Tab is unsupported because the page has no focusable elements",
643
+ }
644
+ try:
645
+ current_index = focusable.index(self.active_element_id)
646
+ except ValueError:
647
+ current_index = -1
648
+ self.active_element_id = focusable[(current_index + 1) % len(focusable)]
649
+ return {
650
+ "status": "pressed",
651
+ "key": key,
652
+ "element_id": self.active_element_id,
653
+ }
654
+
655
+ target_id = element_id or self.active_element_id
656
+ target = self.elements_by_id.get(target_id) if target_id else None
657
+ if target is not None:
658
+ input_type = target.attrs.get("type", "text").lower()
659
+ is_editable = target.tag == "textarea" or (
660
+ target.tag == "input"
661
+ and input_type in {
662
+ "text", "search", "email", "url", "tel", "password", "number",
663
+ "date", "datetime-local", "month", "time", "week",
664
+ }
665
+ )
666
+ if is_editable:
667
+ current_value = target.attrs.get("value", "")
668
+ updated_value: Optional[str] = None
669
+ if key == "Backspace":
670
+ updated_value = current_value[:-1]
671
+ elif key == "Space":
672
+ updated_value = current_value + " "
673
+ elif key == "Enter" and target.tag == "textarea":
674
+ updated_value = current_value + "\n"
675
+ elif len(key) == 1:
676
+ updated_value = current_value + key
677
+ if updated_value is not None and updated_value != current_value:
678
+ target.attrs["value"] = updated_value
679
+ return {"status": "pressed", "key": key, "element_id": target_id}
680
+ elif key == "Enter" and target.attrs.get("href") is not None:
681
+ return self.click(target.element_id)
682
+
683
+ target_description = f"<{target.tag}>" if target is not None else "the active window"
684
+ return {
685
+ "status": "error",
686
+ "error": f"Key '{key}' is unsupported for {target_description}",
687
+ "element_id": target_id,
688
+ }
689
+
690
+ def snapshot_layers(self, max_layers: int = 3) -> Dict[str, Any]:
691
+ """Captures N viewport-sized sequential screenshots (1 layer = 1 viewport height)."""
692
+ requested_layers = min(max(1, max_layers), MAX_SCREENSHOT_LAYERS)
693
+ document_layers = (self.document_height + self.viewport_height - 1) // self.viewport_height
694
+ total_layers = max(1, min(requested_layers, document_layers))
695
+
696
+ return self._capture_layers(total_layers, start_top=0)
697
+
698
+ def snapshot_viewport(self) -> Dict[str, Any]:
699
+ """Captures one viewport at the session's current scroll offset."""
700
+ return self._capture_layers(1, start_top=self.scroll_y)
701
+
702
+ def _capture_layers(self, total_layers: int, start_top: int) -> Dict[str, Any]:
703
+ layers = []
704
+ all_elements = list(self.elements_by_id.values())
705
+
706
+ for layer_idx in range(total_layers):
707
+ layer_top = start_top + layer_idx * self.viewport_height
708
+
709
+ visible_elements = [
710
+ e for e in all_elements
711
+ if e.y + e.height >= layer_top and e.y <= layer_top + self.viewport_height
712
+ ]
713
+
714
+ layer_elements = []
715
+ for e in visible_elements:
716
+ adjusted = DOMElement(e.element_id, e.tag, e.attrs, e.text)
717
+ adjusted.x = e.x
718
+ adjusted.y = e.y - layer_top
719
+ adjusted.width = e.width
720
+ adjusted.height = e.height
721
+ layer_elements.append(adjusted)
722
+
723
+ png_bytes = generate_minimal_png(self.viewport_width, self.viewport_height, layer_elements)
724
+ b64_png = base64.b64encode(png_bytes).decode("ascii")
725
+
726
+ layers.append({
727
+ "layer_index": layer_idx + 1,
728
+ "viewport": [self.viewport_width, self.viewport_height],
729
+ "scroll_top": layer_top,
730
+ "b64_png": b64_png,
731
+ "elements_count": len(visible_elements),
732
+ })
733
+
734
+ return {
735
+ "session_id": self.session_id,
736
+ "url": self.url,
737
+ "title": self.page_title,
738
+ "total_layers": len(layers),
739
+ "layers": layers,
740
+ }
741
+
742
+ def _build_status(self) -> Dict[str, Any]:
743
+ actionable_tags = ("a", "button", "input", "textarea")
744
+ informational_tags = ("h1", "h2", "h3", "form")
745
+ actionable_elements = [
746
+ elem.to_dict() for elem in self.elements_by_id.values()
747
+ if elem.tag in actionable_tags
748
+ ]
749
+ informational_elements = [
750
+ elem.to_dict() for elem in self.elements_by_id.values()
751
+ if elem.tag in informational_tags
752
+ ]
753
+ return {
754
+ "session_id": self.session_id,
755
+ "url": self.url,
756
+ "title": self.page_title,
757
+ "viewport": [self.viewport_width, self.viewport_height],
758
+ "scroll_y": self.scroll_y,
759
+ "document_height": self.document_height,
760
+ "interactive_elements": (actionable_elements + informational_elements)[:50],
761
+ }
762
+
763
+
764
+ class StealthBrowserEngine:
765
+ """Master Stealth Browser Manager managing active tab sessions and profiles."""
766
+
767
+ def __init__(
768
+ self,
769
+ profile_dir: Optional[pathlib.Path] = None,
770
+ max_sessions: int = MAX_BROWSER_SESSIONS,
771
+ ):
772
+ self.profile_manager = ProfileManager(profile_dir)
773
+ self.sessions: Dict[str, StealthBrowserSession] = {}
774
+ self.active_session_id: Optional[str] = None
775
+ self.max_sessions = max(1, min(int(max_sessions), MAX_BROWSER_SESSIONS))
776
+
777
+ def get_or_create_session(self, session_id: Optional[str] = None) -> StealthBrowserSession:
778
+ sid = session_id or self.active_session_id or "default_session"
779
+ if sid not in self.sessions:
780
+ if len(self.sessions) >= self.max_sessions:
781
+ oldest_sid = next(iter(self.sessions))
782
+ self.close_session(oldest_sid)
783
+ self.sessions[sid] = StealthBrowserSession(sid, self.profile_manager)
784
+ self.active_session_id = sid
785
+ return self.sessions[sid]
786
+
787
+ def close_session(self, session_id: Optional[str] = None) -> Dict[str, Any]:
788
+ sid = session_id or self.active_session_id
789
+ if sid and sid in self.sessions:
790
+ session = self.sessions.pop(sid)
791
+ session.close()
792
+ if self.active_session_id == sid:
793
+ self.active_session_id = next(iter(self.sessions.keys())) if self.sessions else None
794
+ return {"status": "closed", "session_id": sid}
795
+ return {"status": "not_found", "session_id": sid}
796
+
797
+
798
+ class _LazyBrowserEngine:
799
+ """Compatibility proxy that defers profile creation until first use."""
800
+
801
+ def __init__(self):
802
+ self._engine: Optional[StealthBrowserEngine] = None
803
+ self._lock = threading.Lock()
804
+
805
+ def _get(self) -> StealthBrowserEngine:
806
+ if self._engine is None:
807
+ with self._lock:
808
+ if self._engine is None:
809
+ self._engine = StealthBrowserEngine()
810
+ return self._engine
811
+
812
+ def get_or_create_session(self, session_id: Optional[str] = None) -> StealthBrowserSession:
813
+ return self._get().get_or_create_session(session_id)
814
+
815
+ def close_session(self, session_id: Optional[str] = None) -> Dict[str, Any]:
816
+ if self._engine is None:
817
+ return {"status": "not_found", "session_id": session_id}
818
+ return self._engine.close_session(session_id)
819
+
820
+ def __getattr__(self, name: str) -> Any:
821
+ return getattr(self._get(), name)
822
+
823
+
824
+ GLOBAL_BROWSER_ENGINE = _LazyBrowserEngine()