phantomfetch 0.4.4__tar.gz → 0.4.6__tar.gz

{phantomfetch-0.4.4 → phantomfetch-0.4.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: phantomfetch
-Version: 0.4.4
+Version: 0.4.6
 Summary: High-performance agentic web scraping library combining curl-cffi speed with Playwright browser capabilities
 Keywords: web-scraping,playwright,curl-cffi,async,browser-automation,http-client,agentic,anti-detection
 Author: CosmicBull

{phantomfetch-0.4.4 → phantomfetch-0.4.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "phantomfetch"
-version = "0.4.4"
+version = "0.4.6"
 description = "High-performance agentic web scraping library combining curl-cffi speed with Playwright browser capabilities"
 readme = "README.md"
 requires-python = ">=3.13"

{phantomfetch-0.4.4 → phantomfetch-0.4.6}/src/phantomfetch/engines/browser/actions.py

@@ -5,7 +5,7 @@ import time
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
-    from rebrowser_playwright.async_api import Locator, Page
+    from playwright.async_api import Locator, Page
 
     from ...types import ActionResult
 
@@ -71,7 +71,7 @@ async def execute_actions(
     Returns:
         List of ActionResult objects
     """
-    from rebrowser_playwright.async_api import Page
+    from playwright.async_api import Page
 
     from ...types import ActionResult
 
@@ -172,23 +172,138 @@ async def execute_actions(
                             result.success = False
                             result.error = "Loop requires selector and child actions"
                         else:
-                            # 1. Find all elements on the resolved context
-                            elements = await ctx.locator(action.selector).all()
-                            loop_results = []
-
                             limit = action.max_iterations or 100
                             logger.debug(
-                                f"[browser] Looping over {len(elements)} elements (limit={limit})"
+                                f"[browser] Compiling loop for '{action.selector}' to native JS (limit={limit})"
                             )
 
-                            for i, el in enumerate(elements):
-                                if i >= limit:
-                                    break
-                                # 2. Execute child actions on EACH element locator
-                                sub_res = await execute_actions(el, action.actions)
-                                loop_results.append({"index": i, "results": sub_res})
+                            serialized_actions = actions_to_payload(action.actions)
+
+                            js_native_loop = """
+                            async (nodeOrArgs, argsIfNode) => {
+                                let rootNode = document;
+                                let args = nodeOrArgs;
+                                if (nodeOrArgs instanceof Node) {
+                                    rootNode = nodeOrArgs;
+                                    args = argsIfNode;
+                                }
+
+                                const { selector, actions, limit } = args;
+                                const elements = Array.from(rootNode.querySelectorAll(selector)).slice(0, limit);
+                                const results = [];
+
+                                const sleep = ms => new Promise(r => setTimeout(r, ms));
+
+                                const waitFor = async (sel, state, timeout) => {
+                                    const start = Date.now();
+                                    while (Date.now() - start < timeout) {
+                                        const el = document.querySelector(sel);
+                                        const isVis = el && el.offsetParent !== null && window.getComputedStyle(el).display !== 'none' && window.getComputedStyle(el).visibility !== 'hidden';
+                                        if (state === 'hidden' && !isVis) return true;
+                                        if (state !== 'hidden' && isVis) return true;
+                                        await sleep(100);
+                                    }
+                                    throw new Error(`Timeout waiting for ${sel} to be ${state}`);
+                                };
 
-                            result.data = loop_results
+                                const isVisible = (el) => el && el.offsetParent !== null && window.getComputedStyle(el).display !== 'none' && window.getComputedStyle(el).visibility !== 'hidden';
+
+                                const extractSingle = (c, spec) => {
+                                    let sel = spec, op = "text", param = null, visibleOnly = false;
+                                    if (typeof spec === "object" && spec !== null && spec._selector) {
+                                        sel = spec._selector; visibleOnly = !!spec._visible_only;
+                                    } else if (typeof spec === "string") {
+                                        if (spec.includes(" :: ")) {
+                                            const parts = spec.split(" :: ");
+                                            sel = parts[0];
+                                            if (parts[1].startsWith("attr(")) { op = "attr"; param = parts[1].slice(5, -1); }
+                                            else { op = parts[1]; }
+                                        } else if (spec.trim().startsWith("::")) {
+                                            sel = null;
+                                            const parts = spec.trim().substring(2).trim();
+                                            if (parts.startsWith("attr(")) { op = "attr"; param = parts.slice(5, -1); }
+                                            else { op = parts; }
+                                        }
+                                    }
+                                    if (!sel || sel.trim() === "") sel = null;
+                                    let el = visibleOnly ? Array.from(sel ? c.querySelectorAll(sel) : [c]).find(isVisible) : (sel ? c.querySelector(sel) : c);
+                                    if (!el) return null;
+                                    if (op === "text") return el.innerText.trim();
+                                    if (op === "html") return el.outerHTML;
+                                    if (op === "attr" && param) return el.getAttribute(param);
+                                    return null;
+                                };
+
+                                const processSchema = (c, s) => {
+                                    const out = {};
+                                    for (const [key, val] of Object.entries(s)) {
+                                        if (typeof val === "string") out[key] = extractSingle(c, val);
+                                        else if (typeof val === "object" && val !== null) {
+                                            if (val._selector) out[key] = extractSingle(c, val);
+                                            else out[key] = processSchema(c, val);
+                                        }
+                                    }
+                                    return out;
+                                };
+
+                                for (let i = 0; i < elements.length; i++) {
+                                    const el = elements[i];
+                                    const iterResults = [];
+
+                                    for (const act of actions) {
+                                        const scopeCtx = act.scope === 'page' ? document : el;
+                                        const actionResult = { action: act, success: true, data: null };
+
+                                        try {
+                                            if (act.action === 'wait') {
+                                                if (act.selector) {
+                                                    await waitFor(act.selector, act.state || 'visible', act.timeout || 5000);
+                                                } else if (act.timeout) {
+                                                    await sleep(act.timeout);
+                                                }
+                                            } else if (act.action === 'click') {
+                                                const target = act.selector ? scopeCtx.querySelector(act.selector) : scopeCtx;
+                                                if (target) {
+                                                    target.scrollIntoView({ behavior: 'instant', block: 'center' });
+                                                    target.click();
+                                                    await sleep(act.human_like ? Math.floor(Math.random() * 100) + 50 : 10);
+                                                } else throw new Error(`Click target not found: ${act.selector}`);
+                                            } else if (act.action === 'extract') {
+                                                const root = act.selector ? scopeCtx.querySelector(act.selector) : scopeCtx;
+                                                if (root) actionResult.data = processSchema(root, act.schema);
+                                                else throw new Error(`Extract target not found: ${act.selector}`);
+                                            } else if (act.action === 'evaluate') {
+                                                if (act.value) actionResult.data = eval(act.value);
+                                            }
+                                        } catch (err) {
+                                            actionResult.success = false;
+                                            actionResult.error = err.message;
+                                            if (act.fail_on_error) {
+                                                iterResults.push(actionResult);
+                                                throw err;
+                                            }
+                                        }
+                                        iterResults.push(actionResult);
+                                    }
+                                    results.push({ index: i, results: iterResults });
+                                }
+                                return results;
+                            }
+                            """
+
+                            try:
+                                loop_results = await ctx.evaluate(
+                                    js_native_loop,
+                                    {
+                                        "selector": action.selector,
+                                        "actions": serialized_actions,
+                                        "limit": limit,
+                                    },
+                                )
+                                result.data = loop_results
+                            except Exception as e:
+                                result.success = False
+                                result.error = f"Native JS Loop Execution Failed: {e}"
 
                     case "click":
                         if action.selector:
@@ -644,10 +759,12 @@ async def execute_actions(
 
             finally:
                 result.duration = time.perf_counter() - start_time
-                
+
                 # Enhanced OTel Attributes
                 span.set_attribute("phantomfetch.action.success", result.success)
-                span.set_attribute("phantomfetch.action.duration_ms", result.duration * 1000)
+                span.set_attribute(
+                    "phantomfetch.action.duration_ms", result.duration * 1000
+                )
                 if result.error:
                     span.set_attribute("phantomfetch.action.error", str(result.error))
 

{phantomfetch-0.4.4 → phantomfetch-0.4.6}/src/phantomfetch/engines/browser/cdp.py

@@ -1,20 +1,25 @@
+import asyncio
+import re
 import time
 from typing import TYPE_CHECKING, Any, Optional
-import sys
 from urllib.parse import urlparse
 
 from browserforge.fingerprints import FingerprintGenerator, Screen
 from browserforge.injectors.playwright import AsyncNewContext
 from loguru import logger
 from opentelemetry import context
+from undetected_playwright import stealth_async
 
 from ...telemetry import get_tracer
 from ...types import Action, Cookie, Proxy, Response
 from .actions import execute_actions
-from undetected_playwright import stealth_async
 
 tracer = get_tracer()
 
+BLOCK_DOMAINS = re.compile(
+    r"(google-analytics\.com|doubleclick\.net|googletagmanager\.com|facebook\.net|hotjar\.com)"
+)
+
 if TYPE_CHECKING:
     # ... (omitted shared lines) ...
 
@@ -102,7 +107,11 @@ class CDPEngine:
         self.browser_type = browser_type
 
         # Map playwright browser type to BrowserForge constraint
-        if "browser" not in self.fingerprint_options and self.browser_type in ("chromium", "firefox", "webkit"):
+        if "browser" not in self.fingerprint_options and self.browser_type in (
+            "chromium",
+            "firefox",
+            "webkit",
+        ):
             bf_map = {"chromium": "chrome", "webkit": "safari", "firefox": "firefox"}
             self.fingerprint_options["browser"] = bf_map[self.browser_type]
 
@@ -112,7 +121,9 @@ class CDPEngine:
 
         # BrowserForge fingerprint generator — instantiated once, generate() per request
         if fingerprint:
-            screen = Screen(min_width=1280, max_width=1920, min_height=720, max_height=1080)
+            screen = Screen(
+                min_width=1280, max_width=1920, min_height=720, max_height=1080
+            )
             self._fp_gen: FingerprintGenerator | None = FingerprintGenerator(
                 screen=screen,
                 mock_webrtc=True,
@@ -131,22 +142,41 @@ class CDPEngine:
         from playwright.async_api import async_playwright
 
         self._playwright = await async_playwright().start()
-
-        if self.cdp_endpoint:
-            logger.info(f"[cdp] Connecting to: {self.cdp_endpoint}")
-            browser_obj = getattr(self._playwright, self.browser_type)
-            self._browser = await browser_obj.connect_over_cdp(
-                self.cdp_endpoint
-            )
-
-            # Detect existing contexts/pages for recording compatibility
-            if self.use_existing_page and self._browser.contexts:
-                self._existing_context = self._browser.contexts[0]
-                if self._existing_context.pages:
-                    self._existing_page = self._existing_context.pages[0]
-                    logger.info("[cdp] Using existing page for recording compatibility")
-        else:
-            logger.info(f"[cdp] Launching local browser (headless={self.headless}, type={self.browser_type})")
+        try:
+            if self.cdp_endpoint:
+                logger.info(f"[cdp] Connecting to: {self.cdp_endpoint}")
+                browser_obj = getattr(self._playwright, self.browser_type)
+                self._browser = await browser_obj.connect_over_cdp(self.cdp_endpoint)
+
+                if self.use_existing_page and self._browser.contexts:
+                    self._existing_context = self._browser.contexts[0]
+                    if self._existing_context.pages:
+                        self._existing_page = self._existing_context.pages[0]
+                        logger.info(
+                            "[cdp] Using existing page for recording compatibility"
+                        )
+            else:
+                logger.info(
+                    f"[cdp] Launching local browser (headless={self.headless}, type={self.browser_type})"
+                )
+                base_args = []
+                if self.browser_type == "chromium":
+                    base_args = [
+                        "--disable-blink-features=AutomationControlled",
+                        "--no-sandbox",
+                        "--disable-setuid-sandbox",
+                    ]
+                extra = [a for a in self.launch_args if a not in base_args]
+
+                browser_obj = getattr(self._playwright, self.browser_type)
+                self._browser = await browser_obj.launch(
+                    headless=self.headless,
+                    args=base_args + extra,
+                )
+        except Exception as e:
+            await self._playwright.stop()
+            self._playwright = None
+            raise e
             base_args = []
             if self.browser_type == "chromium":
                 # --no-sandbox is required on Linux environments that don't have
@@ -158,7 +188,7 @@ class CDPEngine:
                     "--disable-setuid-sandbox",
                 ]
             extra = [a for a in self.launch_args if a not in base_args]
-            
+
             browser_obj = getattr(self._playwright, self.browser_type)
             self._browser = await browser_obj.launch(
                 headless=self.headless,
@@ -348,13 +378,17 @@ class CDPEngine:
 
             if proxy:
                 parsed = urlparse(proxy.url)
-                proxy_server = f"{parsed.scheme}://{parsed.hostname}:{parsed.port}" if parsed.port else f"{parsed.scheme}://{parsed.hostname}"
+                proxy_server = (
+                    f"{parsed.scheme}://{parsed.hostname}:{parsed.port}"
+                    if parsed.port
+                    else f"{parsed.scheme}://{parsed.hostname}"
+                )
                 proxy_dict = {"server": proxy_server}
                 if parsed.username:
                     proxy_dict["username"] = parsed.username
                 if parsed.password:
                     proxy_dict["password"] = parsed.password
-                
+
                 context_opts["proxy"] = proxy_dict
 
             # If we have basic cookies to set via context creation (simpler than add_cookies sometimes)
@@ -371,47 +405,50 @@ class CDPEngine:
                     page = self._existing_page
                     using_existing = True
                     logger.debug(f"[cdp] Reusing existing page for {url}")
-                else:
-                    # Create new context with BrowserForge fingerprint injection (if enabled)
-                    if self._fp_gen is not None:
-                        # Generate a fingerprint with any per-instance constraints
-                        fp = self._fp_gen.generate(**self.fingerprint_options)
-
-                        # Sync UA and viewport FROM the fingerprint for full consistency.
-                        # Explicit user_agent/viewport overrides take precedence.
-                        if not self.user_agent:
-                            context_opts["user_agent"] = fp.navigator.userAgent
-                        if not self.viewport:
-                            context_opts["viewport"] = {
-                                "width": fp.screen.width,
-                                "height": fp.screen.height,
-                            }
-
-                        logger.debug(
-                            f"[cdp] BrowserForge fingerprint: UA={fp.navigator.userAgent[:60]}..."
-                        )
+                # Create new context with BrowserForge fingerprint injection (if enabled)
+                elif self._fp_gen is not None:
+                    # Generate a fingerprint with any per-instance constraints
+                    fp = self._fp_gen.generate(**self.fingerprint_options)
+
+                    # Sync UA and viewport FROM the fingerprint for full consistency.
+                    # Explicit user_agent/viewport overrides take precedence.
+                    if not self.user_agent:
+                        context_opts["user_agent"] = fp.navigator.userAgent
+                    if not self.viewport:
+                        context_opts["viewport"] = {
+                            "width": fp.screen.width,
+                            "height": fp.screen.height,
+                        }
+
+                    logger.debug(
+                        f"[cdp] BrowserForge fingerprint: UA={fp.navigator.userAgent[:60]}..."
+                    )
 
-                        # AsyncNewContext injects all fingerprint JS overrides
-                        # (Canvas, WebGL, navigator, AudioContext, fonts, etc.)
-                        browser_context = await AsyncNewContext(
-                            self._browser,
-                            fingerprint=fp,
-                            **context_opts,
-                        )
+                    # AsyncNewContext injects all fingerprint JS overrides
+                    # (Canvas, WebGL, navigator, AudioContext, fonts, etc.)
+                    browser_context = await AsyncNewContext(
+                        self._browser,
+                        fingerprint=fp,
+                        **context_opts,
+                    )
 
-                        # Apply matching headers so HTTP-layer headers are consistent
-                        # with the JS navigator (UA, sec-ch-ua*, Accept-Language, etc.)
-                        if fp.headers:
-                            # Merge: fingerprint headers are base; caller's explicit
-                            # headers=... arg will be applied later via set_extra_http_headers.
-                            # Here we only set the fingerprint-derived defaults.
-                            fp_headers = dict(fp.headers)
-                            # Remove headers that Playwright manages internally to avoid conflicts
-                            for managed in ("Host", "Content-Length", "Transfer-Encoding"):
-                                fp_headers.pop(managed, None)
-                            await browser_context.set_extra_http_headers(fp_headers)
-                    else:
-                        browser_context = await self._browser.new_context(**context_opts)
+                    # Apply matching headers so HTTP-layer headers are consistent
+                    # with the JS navigator (UA, sec-ch-ua*, Accept-Language, etc.)
+                    if fp.headers:
+                        # Merge: fingerprint headers are base; caller's explicit
+                        # headers=... arg will be applied later via set_extra_http_headers.
+                        # Here we only set the fingerprint-derived defaults.
+                        fp_headers = dict(fp.headers)
+                        # Remove headers that Playwright manages internally to avoid conflicts
+                        for managed in (
+                            "Host",
+                            "Content-Length",
+                            "Transfer-Encoding",
+                        ):
+                            fp_headers.pop(managed, None)
+                        await browser_context.set_extra_http_headers(fp_headers)
+                else:
+                    browser_context = await self._browser.new_context(**context_opts)
 
             except Exception as e:
                 span.record_exception(e)
@@ -457,23 +494,16 @@ class CDPEngine:
                     # Actually valid Playwright storage_state is best.
                     # If we use context.add_init_script, we can inject it.
                     if "origins" in storage_state:
+                        import json
+
                         for origin_data in storage_state["origins"]:
                             origin = origin_data["origin"]
-                            ls_data = origin_data[
-                                "localStorage"
-                            ]  # list of {name, value}
-                            # Construct JS to set items
-                            js_setter = ""
-                            for item in ls_data:
-                                k = item["name"].replace('"', '\\"')
-                                v = item["value"].replace('"', '\\"')
-                                js_setter += (
-                                    f'window.localStorage.setItem("{k}", "{v}");'
-                                )
-
+                            ls_data = origin_data["localStorage"]
+                            ls_json = json.dumps(ls_data)
                             script = f"""
                             if (window.location.origin === "{origin}") {{
-                                {js_setter}
+                                const data = {ls_json};
+                                data.forEach(item => window.localStorage.setItem(item.name, item.value));
                             }}
                             """
                             await browser_context.add_init_script(script)
@@ -538,12 +568,13 @@ class CDPEngine:
                         try:
                             req = response.request
                             if req.resource_type in ("xhr", "fetch"):
-                                # Capture body safely
                                 resp_body = None
                                 try:
-                                    # Limit body size capture to avoid memory issues
-                                    body_bytes = await response.body()
-                                    if len(body_bytes) < 1024 * 1024:  # 1MB limit
+                                    content_length = int(
+                                        response.headers.get("content-length", 0)
+                                    )
+                                    if content_length < 1024 * 1024:
+                                        body_bytes = await response.body()
                                         resp_body = body_bytes.decode(
                                             "utf-8", errors="replace"
                                         )
@@ -609,10 +640,8 @@ class CDPEngine:
                         context.detach(token)
 
                 async def handle_route_with_context(route: "Route") -> None:
-                    # Attach the captured context
                     token = context.attach(current_ctx)
                     try:
-                        # Handle resource blocking
                         if (
                             block_resources
                             and route.request.resource_type in block_resources
@@ -620,7 +649,19 @@ class CDPEngine:
                             await route.abort()
                             return
 
+                        if BLOCK_DOMAINS.search(route.request.url):
+                            await route.abort()
+                            return
+
                         await self._handle_route(route)
+                    except Exception as e:
+                        logger.error(
+                            f"[cdp] Route handler catastrophically failed: {e}"
+                        )
+                        try:
+                            await route.continue_()
+                        except Exception:
+                            pass
                     finally:
                         context.detach(token)
 
@@ -634,12 +675,18 @@ class CDPEngine:
 
                 # Navigate
                 span.add_event("navigation.start")
+                await browser_context.tracing.start(
+                    screenshots=True, snapshots=True, sources=True
+                )
                 try:
                     response = await page.goto(
                         url, timeout=timeout_ms, wait_until=wait_until
                     )
                 except Exception as e:
                     span.record_exception(e)
+                    await browser_context.tracing.stop(
+                        path=f"/tmp/trace_{int(time.time())}.zip"
+                    )
                     return Response(
                         url=url,
                         status=0,
@@ -660,9 +707,9 @@ class CDPEngine:
                     except Exception as e:
                         logger.warning(f"[cdp] wait_for_url failed: {e}")
                         span.record_exception(e)
-                        # Option 2: Strict Waiting - Return error response
-
-                        # Safely capture content
+                        await browser_context.tracing.stop(
+                            path=f"/tmp/trace_{int(time.time())}.zip"
+                        )
                         body_content = b""
                         try:
                             body_content = (await page.content()).encode("utf-8")
@@ -671,7 +718,7 @@ class CDPEngine:
 
                         return Response(
                             url=page.url,
-                            status=0,  # Or appropriate error code
+                            status=0,
                             body=body_content,
                             engine="browser",
                             error=f"Wait for URL failed: {wait_for_url}. Current URL: {page.url}",
@@ -686,10 +733,11 @@ class CDPEngine:
                     try:
                         action_results = await execute_actions(page, actions)
                     except RuntimeError as e:
-                        # Catch Fail Fast exceptions
                         logger.warning(f"[cdp] Action fail_on_error triggered: {e}")
                         span.record_exception(e)
-                        # Safely capture content
+                        await browser_context.tracing.stop(
+                            path=f"/tmp/trace_{int(time.time())}.zip"
+                        )
                         body_content = b""
                         try:
                             body_content = (await page.content()).encode("utf-8")
@@ -703,11 +751,6 @@ class CDPEngine:
                             engine="browser",
                             error=f"Action Execution Failed: {e}",
                             network_log=network_log,
-                            # We might have partial results if execute_actions appends before raising
-                            # But execute_actions usually returns the list.
-                            # In failure case, it raises, so we don't get the list return value.
-                            # However, we modified execute_actions to raise, so we lose the list unless we modify it to attach results to exception.
-                            # For now, simplistic error return is fine.
                             action_results=[],
                         )
                     span.add_event("actions.end")
@@ -741,7 +784,9 @@ class CDPEngine:
                         break
                     except Exception as e:
                         if "Unable to retrieve content" in str(e) and i < 2:
-                            logger.debug(f"[cdp] Retrying content retrieval due to navigation ({i+1}/3)")
+                            logger.debug(
+                                f"[cdp] Retrying content retrieval due to navigation ({i + 1}/3)"
+                            )
                             await asyncio.sleep(0.5)
                         else:
                             logger.warning(f"[cdp] Failed to retrieve content: {e}")
@@ -796,6 +841,15 @@ class CDPEngine:
                     storage_state=current_storage_state,
                 )
 
+            except asyncio.CancelledError:
+                logger.warning("Scrape cancelled by orchestrator. Shielding teardown.")
+                if not using_existing and browser_context:
+                    try:
+                        await asyncio.shield(browser_context.close())
+                    except Exception:
+                        pass
+                raise
+
             except Exception as e:
                 logger.error(f"[cdp] Error: {e}")
                 span.record_exception(e)
@@ -811,9 +865,8 @@ class CDPEngine:
                 )
 
             finally:
-                # Only close if we created new page/context (not reusing existing)
-                if not using_existing:
-                    if page:
-                        await page.close()
-                    if browser_context:
+                if not using_existing and browser_context:
+                    try:
                         await browser_context.close()
+                    except Exception:
+                        pass