phantomwright 0.1.4__tar.gz → 0.2.0__tar.gz

{phantomwright-0.1.4 → phantomwright-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: phantomwright
-Version: 0.1.4
+Version: 0.2.0
 Summary: Bridging playwright-core patch + extending playwright API for stealth injection & user simulation
 Project-URL: homepage, https://github.com/ai-microsoft/phantom-wright
 Project-URL: changelog, https://github.com/ai-microsoft/phantom-wright/blob/main/CHANGELOG.md
@@ -9,8 +9,12 @@ License-Expression: MIT
 License-File: LICENSE
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.9
-Requires-Dist: phantomwright-driver==1.57.7
+Requires-Python: >=3.10
+Requires-Dist: aiohttp>=3.9.0
+Requires-Dist: click>=8.0.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: markitdown>=0.1.0
+Requires-Dist: phantomwright-driver==1.58.3
 Provides-Extra: black
 Requires-Dist: black>=25.9.0; extra == 'black'
 Provides-Extra: dev

phantomwright-0.2.0/phantomwright/_repo_version.py

@@ -0,0 +1 @@
+version = 'v0.2.0'

{phantomwright-0.1.4 → phantomwright-0.2.0}/phantomwright/captcha/cloudfare/scripts/observer.js

@@ -41,6 +41,14 @@
     }
   }
 
+  // Expose function to trigger rescan from Python side
+  window.__triggerCfRescan = function() {
+    const root = document.body || document.documentElement;
+    if (root) {
+      scan(root);
+    }
+  };
+
   function startObserve() {
     const root = document.body || document.documentElement;
     if (!root) return;

{phantomwright-0.1.4 → phantomwright-0.2.0}/phantomwright/captcha/cloudfare/solver.py

@@ -87,18 +87,27 @@ class CloudflareSolverAsync:
         return self.page_solve_state.setdefault(
             page,
             {
-                "status": "idle",  # idle | solving | done
+                "status": "idle",  # idle | solving
                 "last_url": None,
             },
         )
 
     # ---------------- js rebind ----------------
     async def _rebind(self, page: Page):
-        await page.evaluate("""
-            window.onCloudflareDetected = function(sel, url) {
-                window.__cf_callback(sel, url);
-            };
-        """)
+        try:
+            await page.evaluate("""
+                window.onCloudflareDetected = function(sel, url) {
+                    window.__cf_callback(sel, url);
+                };
+            """)
+            await asyncio.sleep(1)
+            await page.evaluate("""
+                if (typeof window.__triggerCfRescan === 'function') {
+                    window.__triggerCfRescan();
+                }
+            """)
+        except Exception:
+            pass
 
     # ---------------- report helper ----------------
     def _log_final_report(self, report: dict):
@@ -113,7 +122,9 @@ class CloudflareSolverAsync:
             "timestamp": time.time(),
         }
 
-        self.log(json.dumps(data, ensure_ascii=False))
+        if self.log:
+            log_str = json.dumps(data, ensure_ascii=False)
+            self.log(log_str)
 
     # ---------------- core solve ----------------
     async def _auto_solve_cf(self, page: Page):
@@ -140,7 +151,7 @@ class CloudflareSolverAsync:
                     report["challenge_type"] = challenge_type.name
 
                     if challenge_type is ChallengeType.TURNSTILE:
-                        await page.locator("#cf-turnstile").wait_for(10000)
+                        await page.locator("#cf-turnstile").wait_for(timeout=10000)
 
                     cf_iframes = await search_shadow_root_iframes(
                         captcha_container=page,
@@ -174,13 +185,22 @@ class CloudflareSolverAsync:
                         success_elements = await search_shadow_root_elements(
                             iframe, 'div[id="success"]'
                         )
-                        solved = bool(success_elements)
+                        
+                        # Check if success element is actually visible
+                        solved = False
+                        for el in success_elements:
+                            try:
+                                is_visible = await el.is_visible()
+                                if is_visible:
+                                    solved = True
+                                    break
+                            except:
+                                pass
                     else:
                         solved = not await detect_cloudflare_challenge(page)
 
                     if solved:
-                        state["status"] = "done"
-                        state["last_url"] = page.url
+                        state["status"] = "idle"
 
                         report["success"] = True
                         return
@@ -211,15 +231,17 @@ class CloudflareSolverAsync:
                 return
 
             state = self._get_page_state(page)
+            current_url = page.url
+            
+            # Skip if this URL was already attempted
+            if state["last_url"] == current_url:
+                return
 
-            if state["last_url"] != page.url:
-                state["status"] = "idle"
-                state["last_url"] = page.url
-
-            if state["status"] in ("solving", "done"):
+            if state["status"] == "solving":
                 return
 
             state["status"] = "solving"
+            state["last_url"] = current_url
 
             asyncio.create_task(self._auto_solve_cf(page))
 
@@ -234,7 +256,14 @@ class CloudflareSolverAsync:
         await page.add_init_script(shadow_root_js)
 
         await page.expose_function("__cf_callback", self._make_on_cf_detected(page))
+        
+        # Listen to multiple events for more reliable detection
         page.on("load", lambda: asyncio.create_task(self._rebind(page)))
+        page.on("domcontentloaded", lambda: asyncio.create_task(self._rebind(page)))
+        page.on("framenavigated", lambda frame: asyncio.create_task(self._rebind(page)) if frame == page.main_frame else None)
+        
+        # Immediately rebind in case page is already loaded
+        asyncio.create_task(self._rebind(page))
 
     # ---------------- public api ----------------
     def start(self) -> None:

phantomwright-0.2.0/phantomwright/cli/__init__.py

@@ -0,0 +1,5 @@
+"""PhantomWright CLI - Browser automation from the command line."""
+
+from .main import cli
+
+__all__ = ["cli"]

phantomwright-0.2.0/phantomwright/cli/client.py

@@ -0,0 +1,201 @@
+"""Client for communicating with the browser session server."""
+
+import json
+from pathlib import Path
+from typing import Any, Optional
+
+import httpx
+
+from .server import SESSION_FILE, DEFAULT_PORT
+
+
+class SessionClient:
+    """Client for communicating with the browser session server."""
+
+    def __init__(self, port: Optional[int] = None):
+        self._port = port
+        self._base_url: Optional[str] = None
+
+    def _get_base_url(self) -> Optional[str]:
+        """Get the base URL for the server."""
+        if self._base_url:
+            return self._base_url
+
+        # Try to load port from session file
+        port = self._port
+        if port is None:
+            if SESSION_FILE.exists():
+                try:
+                    with open(SESSION_FILE, "r") as f:
+                        session_info = json.load(f)
+                        port = session_info.get("port", DEFAULT_PORT)
+                except (json.JSONDecodeError, IOError):
+                    port = DEFAULT_PORT
+            else:
+                port = DEFAULT_PORT
+
+        self._base_url = f"http://127.0.0.1:{port}"
+        return self._base_url
+
+    def is_server_running(self) -> bool:
+        """Check if the server is running."""
+        base_url = self._get_base_url()
+        if not base_url:
+            return False
+
+        try:
+            with httpx.Client(timeout=2.0) as client:
+                response = client.get(f"{base_url}/status")
+                return response.status_code == 200
+        except httpx.RequestError:
+            return False
+
+    def _request(
+        self,
+        method: str,
+        endpoint: str,
+        data: Optional[dict] = None,
+        timeout: float = 60.0,
+    ) -> dict:
+        """Make a request to the server."""
+        base_url = self._get_base_url()
+        if not base_url:
+            return {"error": "NoSession", "message": "No active browser session. Run 'phantomwright start' first."}
+
+        try:
+            with httpx.Client(timeout=timeout) as client:
+                if method == "GET":
+                    response = client.get(f"{base_url}{endpoint}")
+                else:
+                    response = client.post(f"{base_url}{endpoint}", json=data or {})
+
+                return response.json()
+        except httpx.ConnectError:
+            return {"error": "NoSession", "message": "No active browser session. Run 'phantomwright start' first."}
+        except httpx.RequestError as e:
+            return {"error": "ConnectionError", "message": str(e)}
+        except json.JSONDecodeError:
+            return {"error": "InvalidResponse", "message": "Invalid response from server"}
+
+    def get_status(self) -> dict:
+        """Get server status."""
+        return self._request("GET", "/status")
+
+    def stop(self) -> dict:
+        """Stop the server."""
+        return self._request("POST", "/stop")
+
+    def navigate(
+        self,
+        url: str,
+        simulate: bool = True,
+        cool_down: bool = True,
+        wait_for: Optional[str] = None,
+        timeout: int = 30000,
+    ) -> dict:
+        """Navigate to a URL."""
+        return self._request("POST", "/navigate", {
+            "url": url,
+            "simulate": simulate,
+            "cool_down": cool_down,
+            "wait_for": wait_for,
+            "timeout": timeout,
+        }, timeout=timeout / 1000 + 30)
+
+    def scroll(self, duration: int = 2000) -> dict:
+        """Scroll the page."""
+        return self._request("POST", "/scroll", {"duration": duration}, timeout=duration / 1000 + 10)
+
+    def browse(self, duration: int = 2000) -> dict:
+        """Simulate browsing."""
+        return self._request("POST", "/browse", {"duration": duration}, timeout=duration / 1000 + 10)
+
+    def click(self, selector: str, simulate: bool = True) -> dict:
+        """Click an element."""
+        return self._request("POST", "/click", {
+            "selector": selector,
+            "simulate": simulate,
+        })
+
+    def type_text(self, selector: str, text: str, simulate: bool = True, typos: bool = False) -> dict:
+        """Type text into an element."""
+        return self._request("POST", "/type", {
+            "selector": selector,
+            "text": text,
+            "simulate": simulate,
+            "typos": typos,
+        }, timeout=len(text) * 0.5 + 30)
+
+    def screenshot(
+        self,
+        selector: Optional[str] = None,
+        full_page: bool = False,
+    ) -> dict:
+        """Take a screenshot."""
+        return self._request("POST", "/screenshot", {
+            "selector": selector,
+            "full_page": full_page,
+        })
+
+    def get_html(self, selector: Optional[str] = None, outer: bool = True) -> dict:
+        """Get HTML content."""
+        return self._request("POST", "/html", {
+            "selector": selector,
+            "outer": outer,
+        })
+
+    def get_text(self, selector: str) -> dict:
+        """Get text content."""
+        return self._request("POST", "/text", {"selector": selector})
+
+    def get_attr(self, selector: str, attribute: str) -> dict:
+        """Get attribute value."""
+        return self._request("POST", "/attr", {
+            "selector": selector,
+            "attribute": attribute,
+        })
+
+    def query(
+        self,
+        selector: str,
+        limit: Optional[int] = None,
+        attrs: Optional[str] = None,
+    ) -> dict:
+        """Query elements."""
+        return self._request("POST", "/query", {
+            "selector": selector,
+            "limit": limit,
+            "attrs": attrs,
+        })
+
+    def get_url(self) -> dict:
+        """Get current URL."""
+        return self._request("GET", "/url")
+
+    def get_title(self) -> dict:
+        """Get page title."""
+        return self._request("GET", "/title")
+
+    def wait(self, selector: str, timeout: int = 30000, state: str = "visible") -> dict:
+        """Wait for an element."""
+        return self._request("POST", "/wait", {
+            "selector": selector,
+            "timeout": timeout,
+            "state": state,
+        }, timeout=timeout / 1000 + 10)
+
+    def get_markdown(self, selector: Optional[str] = None) -> dict:
+        """Get page content as markdown with selectors."""
+        return self._request("POST", "/markdown", {"selector": selector})
+
+
+# Global client instance
+_client: Optional[SessionClient] = None
+
+
+def get_client() -> SessionClient:
+    """Get the global session client."""
+    global _client
+    if _client is None:
+        _client = SessionClient()
+    return _client

phantomwright-0.2.0/phantomwright/cli/commands/__init__.py

@@ -0,0 +1 @@
+"""PhantomWright CLI commands."""

phantomwright-0.2.0/phantomwright/cli/commands/extraction.py

@@ -0,0 +1,137 @@
+"""Extraction commands: screenshot, html, text, attr, query."""
+
+import base64
+import os
+import click
+
+from ..client import get_client
+from ..output import output_success, output_error
+
+
+@click.command()
+@click.option("--path", default=None, help="File path to save screenshot")
+@click.option("--selector", default=None, help="CSS selector for element screenshot")
+@click.option("--full-page/--no-full-page", default=False, help="Capture full scrollable page")
+def screenshot(path: str, selector: str, full_page: bool):
+    """Take a screenshot of the page or element."""
+    client = get_client()
+    result = client.screenshot(selector=selector, full_page=full_page)
+
+    if "error" in result:
+        output_error("screenshot", result["error"], result["message"], selector=selector)
+        return
+
+    screenshot_bytes = base64.b64decode(result["base64"])
+
+    if path:
+        # Ensure directory exists
+        dir_path = os.path.dirname(path)
+        if dir_path:
+            os.makedirs(dir_path, exist_ok=True)
+        with open(path, "wb") as f:
+            f.write(screenshot_bytes)
+        output_success("screenshot", {
+            "path": os.path.abspath(path),
+            "size_bytes": result["size_bytes"],
+            "selector": selector,
+            "full_page": full_page,
+        })
+    else:
+        # Return base64 encoded image
+        output_success("screenshot", {
+            "base64": result["base64"],
+            "size_bytes": result["size_bytes"],
+            "selector": selector,
+            "full_page": full_page,
+        })
+
+
+@click.command()
+@click.option("--selector", default=None, help="CSS selector to get HTML from")
+@click.option("--outer/--inner", default=True, help="Get outer HTML (default) or inner HTML")
+def html(selector: str, outer: bool):
+    """Get page or element HTML content."""
+    client = get_client()
+    result = client.get_html(selector=selector, outer=outer)
+
+    if "error" in result:
+        output_error("html", result["error"], result["message"], selector=selector)
+    else:
+        output_success("html", {
+            "html": result["html"],
+            "length": result["length"],
+            "selector": selector,
+            "type": result["type"],
+        })
+
+
+@click.command()
+@click.argument("selector")
+def text(selector: str):
+    """Get text content of an element."""
+    client = get_client()
+    result = client.get_text(selector=selector)
+
+    if "error" in result:
+        output_error("text", result["error"], result["message"], selector=selector)
+    else:
+        output_success("text", {
+            "text": result["text"],
+            "length": result["length"],
+            "selector": selector,
+        })
+
+
+@click.command()
+@click.argument("selector")
+@click.argument("attribute")
+def attr(selector: str, attribute: str):
+    """Get an attribute value from an element."""
+    client = get_client()
+    result = client.get_attr(selector=selector, attribute=attribute)
+
+    if "error" in result:
+        output_error("attr", result["error"], result["message"], selector=selector)
+    else:
+        output_success("attr", {
+            "attribute": result["attribute"],
+            "value": result["value"],
+            "selector": selector,
+        })
+
+
+@click.command()
+@click.argument("selector")
+@click.option("--limit", default=None, type=int, help="Maximum number of elements to return")
+@click.option("--attrs", default=None, help="Comma-separated list of attributes to extract")
+def query(selector: str, limit: int, attrs: str):
+    """Query all elements matching a selector."""
+    client = get_client()
+    result = client.query(selector=selector, limit=limit, attrs=attrs)
+
+    if "error" in result:
+        output_error("query", result["error"], result["message"], selector=selector)
+    else:
+        output_success("query", {
+            "elements": result["elements"],
+            "count": result["count"],
+            "total_matches": result.get("total_matches", result["count"]),
+            "selector": selector,
+        })
+
+
+@click.command()
+@click.option("--selector", default=None, help="CSS selector to convert (default: entire page)")
+def markdown(selector: str):
+    """Get page content as markdown with embedded selectors for agent interaction."""
+    client = get_client()
+    result = client.get_markdown(selector=selector)
+
+    if "error" in result:
+        output_error("markdown", result["error"], result["message"], selector=selector)
+    else:
+        output_success("markdown", {
+            "markdown": result["markdown"],
+            "url": result["url"],
+            "title": result["title"],
+        })

phantomwright-0.2.0/phantomwright/cli/commands/info.py

@@ -0,0 +1,169 @@
+"""Info commands: url, title, wait, status, start, stop."""
+
+import asyncio
+import subprocess
+import sys
+import click
+
+from ..session import load_session, clear_session, is_session_active
+from ..client import get_client
+from ..output import output_success, output_error
+
+
+@click.command()
+def url():
+    """Get the current page URL."""
+    client = get_client()
+    result = client.get_url()
+
+    if "error" in result:
+        output_error("url", result["error"], result["message"])
+    else:
+        output_success("url", {"url": result["url"]})
+
+
+@click.command()
+def title():
+    """Get the current page title."""
+    client = get_client()
+    result = client.get_title()
+
+    if "error" in result:
+        output_error("title", result["error"], result["message"])
+    else:
+        output_success("title", {
+            "title": result["title"],
+            "url": result["url"],
+        })
+
+
+@click.command()
+@click.argument("selector")
+@click.option("--timeout", default=30000, help="Timeout in milliseconds")
+@click.option("--state", default="visible", type=click.Choice(["attached", "detached", "visible", "hidden"]), help="Element state to wait for")
+def wait(selector: str, timeout: int, state: str):
+    """Wait for an element to appear."""
+    client = get_client()
+    result = client.wait(selector, timeout=timeout, state=state)
+
+    if "error" in result:
+        output_error("wait", result["error"], result["message"], selector=selector, timeout=timeout)
+    else:
+        output_success("wait", {
+            "selector": selector,
+            "state": state,
+            "found": result["found"],
+            "count": result["count"],
+        })
+
+
+@click.command()
+def status():
+    """Get the current session status."""
+    client = get_client()
+    result = client.get_status()
+    output_success("status", result)
+
+
+@click.command()
+@click.option("--browser", default="chrome", type=click.Choice(["chrome", "msedge"]), help="Browser to use (default: chrome)")
+@click.option("--headless/--no-headless", default=False, help="Run browser in headless mode")
+@click.option("--stealth/--no-stealth", default=True, help="Enable stealth mode")
+@click.option("--cloudflare-solver/--no-cloudflare-solver", default=True, help="Enable Cloudflare solver")
+@click.option("--visualize-mouse/--no-visualize-mouse", default=None, help="Show visual cursor (default: on when not headless)")
+@click.option("--port", default=9323, help="Server port (default: 9323)")
+@click.option("--foreground/--no-foreground", default=False, help="Run server in foreground (default: background)")
+@click.option("--user-data-dir", default=None, help="Browser user data directory for persistent sessions (preserves logins)")
+def start(browser: str, headless: bool, stealth: bool, cloudflare_solver: bool, visualize_mouse: bool, port: int, foreground: bool, user_data_dir: str):
+    """Start a new browser session."""
+    # Check if already running
+    if is_session_active():
+        output_error("start", "SessionExists", "A browser session is already active. Run 'phantomwright stop' first.")
+        return
+
+    if not foreground:
+        # Start server in background subprocess
+        args = [
+            sys.executable, "-m", "phantomwright.cli.server",
+            "--port", str(port),
+            "--browser", browser,
+        ]
+        if headless:
+            args.append("--headless")
+        if not stealth:
+            args.append("--no-stealth")
+        if not cloudflare_solver:
+            args.append("--no-cloudflare-solver")
+        if visualize_mouse is not None:
+            args.append("--visualize-mouse" if visualize_mouse else "--no-visualize-mouse")
+        if user_data_dir:
+            args.extend(["--user-data-dir", user_data_dir])
+
+        # Use CREATE_NEW_PROCESS_GROUP on Windows, start_new_session on Unix
+        if sys.platform == "win32":
+            subprocess.Popen(
+                args,
+                creationflags=subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.DETACHED_PROCESS,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+        else:
+            subprocess.Popen(
+                args,
+                start_new_session=True,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+
+        # Wait for server to start
+        import time
+        from ..client import SessionClient
+        client = SessionClient(port=port)
+        for _ in range(300):  # Wait up to 30 seconds
+            time.sleep(0.1)
+            if client.is_server_running():
+                result = client.get_status()
+                output_success("start", result)
+                return
+
+        output_error("start", "StartupTimeout", "Server did not start within timeout.")
+    else:
+        # Run server in foreground
+        asyncio.run(_start_foreground(browser, headless, stealth, cloudflare_solver, visualize_mouse, port, user_data_dir))
+
+
+async def _start_foreground(browser: str, headless: bool, stealth: bool, cloudflare_solver: bool, visualize_mouse: bool, port: int, user_data_dir: str):
+    """Start server in foreground."""
+    from ..server import run_server
+    await run_server(
+        port=port,
+        browser=browser,
+        headless=headless,
+        stealth=stealth,
+        cloudflare_solver=cloudflare_solver,
+        visualize_mouse=visualize_mouse,
+        user_data_dir=user_data_dir,
+    )
+
+
+@click.command()
+def stop():
+    """Stop the current browser session."""
+    client = get_client()
+
+    if not client.is_server_running():
+        # Check if there's a stale session file
+        stored_session = load_session()
+        if stored_session:
+            clear_session()
+            output_success("stop", {"message": "Cleaned up stale session file."})
+        else:
+            output_error("stop", "NoSession", "No active browser session to stop.")
+        return
+
+    result = client.stop()
+
+    if "error" in result:
+        output_error("stop", result["error"], result["message"])
+    else:
+        output_success("stop", {"message": "Browser session stopped."})