phantomfetch 0.6.0__tar.gz → 0.6.1__tar.gz

{phantomfetch-0.6.0 → phantomfetch-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: phantomfetch
-Version: 0.6.0
+Version: 0.6.1
 Summary: High-performance agentic web scraping library combining curl-cffi speed with Playwright browser capabilities
 Keywords: web-scraping,playwright,curl-cffi,async,browser-automation,http-client,agentic,anti-detection
 Author: CosmicBull
@@ -26,14 +26,14 @@ Requires-Dist: opentelemetry-sdk>=1.38.0
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: beautifulsoup4>=4.14.3
 Requires-Dist: lmdb>=1.4.1
-Requires-Dist: cloakbrowser[geoip]>=0.3.31 ; extra == 'all'
+Requires-Dist: cloakbrowser>=0.3.31 ; extra == 'all'
 Requires-Dist: camoufox[geoip]>=0.4.11 ; extra == 'all'
 Requires-Dist: rebrowser-playwright>=1.52.0 ; extra == 'all'
 Requires-Dist: patchright>=1.60.1 ; extra == 'all'
 Requires-Dist: maxminddb>=2.0.0 ; extra == 'all'
 Requires-Dist: camoufox[geoip]>=0.4.11 ; extra == 'camoufox'
 Requires-Dist: cloakbrowser>=0.3.31 ; extra == 'cloakbrowser'
-Requires-Dist: cloakbrowser[geoip]>=0.3.31 ; extra == 'geoip'
+Requires-Dist: cloakbrowser>=0.3.31 ; extra == 'geoip'
 Requires-Dist: maxminddb>=2.0.0 ; extra == 'geoip'
 Requires-Dist: patchright>=1.60.1 ; extra == 'patchright'
 Requires-Dist: rebrowser-playwright>=1.52.0 ; extra == 'rebrowser'

{phantomfetch-0.6.0 → phantomfetch-0.6.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "phantomfetch"
-version = "0.6.0"
+version = "0.6.1"
 description = "High-performance agentic web scraping library combining curl-cffi speed with Playwright browser capabilities"
 readme = "README.md"
 requires-python = ">=3.13"
@@ -45,11 +45,11 @@ dependencies = [
 
 [project.optional-dependencies]
 cloakbrowser = ["cloakbrowser>=0.3.31"]
-geoip = ["cloakbrowser[geoip]>=0.3.31", "maxminddb>=2.0.0"]
+geoip = ["cloakbrowser>=0.3.31", "maxminddb>=2.0.0"]
 camoufox = ["camoufox[geoip]>=0.4.11"]
 rebrowser = ["rebrowser-playwright>=1.52.0"]
 patchright = ["patchright>=1.60.1"]
-all = ["cloakbrowser[geoip]>=0.3.31", "camoufox[geoip]>=0.4.11", "rebrowser-playwright>=1.52.0", "patchright>=1.60.1", "maxminddb>=2.0.0"]
+all = ["cloakbrowser>=0.3.31", "camoufox[geoip]>=0.4.11", "rebrowser-playwright>=1.52.0", "patchright>=1.60.1", "maxminddb>=2.0.0"]
 
 [project.urls]
 Homepage = "https://github.com/iristech-systems/PhantomFetch"

{phantomfetch-0.6.0 → phantomfetch-0.6.1}/src/phantomfetch/fetch.py

@@ -403,95 +403,125 @@ class Fetcher:
 
             span.set_attribute("phantomfetch.cache.hit", False)
 
-            # Get proxy
-            # 1. Manual override
-            selected_proxy: Proxy | None = None
-            if proxy:
-                if isinstance(proxy, str):
-                    selected_proxy = Proxy(
-                        url=proxy, metadata={"source": "manual_override"}
-                    )
-                else:
-                    selected_proxy = proxy
+            # Resolve retry configurations
+            actual_max_retries = max_retries if max_retries is not None else self.max_retries
+            attempts = max(1, actual_max_retries)
+            if retry_on is None:
+                retry_status_codes: set[int] = {0, 429, 500, 502, 503, 504}
+            elif isinstance(retry_on, int):
+                retry_status_codes = {retry_on}
+            else:
+                retry_status_codes = set(retry_on)
+                
+            backoff_base = 2.0 if retry_backoff is None else retry_backoff
 
-            # 2. From Pool (if no override)
-            if not selected_proxy:
-                selected_proxy = self.proxy_pool.get(url=url, location=location)
+            last_resp: Response | None = None
 
-            if selected_proxy:
-                span.set_attribute("phantomfetch.proxy", selected_proxy.url)
-                if selected_proxy.vendor:
-                    span.set_attribute(
-                        "phantomfetch.proxy.vendor", selected_proxy.vendor
-                    )
-                if selected_proxy.proxy_type:
-                    span.set_attribute(
-                        "phantomfetch.proxy.type", selected_proxy.proxy_type
-                    )
-                if selected_proxy.location:
-                    span.set_attribute(
-                        "phantomfetch.proxy.location", selected_proxy.location
+            for attempt in range(attempts):
+                # Get proxy
+                # 1. Manual override
+                selected_proxy: Proxy | None = None
+                if proxy:
+                    if isinstance(proxy, str):
+                        selected_proxy = Proxy(
+                            url=proxy, metadata={"source": "manual_override"}
+                        )
+                    else:
+                        selected_proxy = proxy
+    
+                # 2. From Pool (if no override)
+                if not selected_proxy:
+                    selected_proxy = self.proxy_pool.get(url=url, location=location)
+    
+                if selected_proxy:
+                    span.set_attribute("phantomfetch.proxy", selected_proxy.url)
+                    if selected_proxy.vendor:
+                        span.set_attribute(
+                            "phantomfetch.proxy.vendor", selected_proxy.vendor
+                        )
+                    if selected_proxy.proxy_type:
+                        span.set_attribute(
+                            "phantomfetch.proxy.type", selected_proxy.proxy_type
+                        )
+                    if selected_proxy.location:
+                        span.set_attribute(
+                            "phantomfetch.proxy.location", selected_proxy.location
+                        )
+                    if selected_proxy.provider:
+                        span.set_attribute(
+                            "phantomfetch.proxy.provider", selected_proxy.provider
+                        )
+    
+                # Route to engine
+                if engine == "browser":
+                    resp = await self._fetch_browser(
+                        url=url,
+                        proxy=selected_proxy,
+                        headers=headers,
+                        cookies=cookies,
+                        actions=normalized_actions,
+                        timeout=timeout or self.browser_timeout,
+                        location=location,
+                        wait_until=wait_until,
+                        block_resources=block_resources,
+                        wait_for_url=wait_for_url,
+                        storage_state=self.session_data,  # Pass current session
+                        stealth=stealth,
                     )
-                if selected_proxy.provider:
-                    span.set_attribute(
-                        "phantomfetch.proxy.provider", selected_proxy.provider
+                else:
+                    resp = await self._fetch_curl(
+                        url=url,
+                        proxy=selected_proxy,
+                        headers=headers,
+                        cookies=cookies,
+                        timeout=timeout or self.timeout,
+                        max_retries=1,  # Prevent internal retries so we can rotate proxies
+                        retry_on=retry_on,
+                        retry_backoff=retry_backoff,
+                        referer=referer,
+                        allow_redirects=allow_redirects,
                     )
-
-            # Route to engine
-            if engine == "browser":
-                resp = await self._fetch_browser(
-                    url=url,
-                    proxy=selected_proxy,
-                    headers=headers,
-                    cookies=cookies,
-                    actions=normalized_actions,
-                    timeout=timeout or self.browser_timeout,
-                    location=location,
-                    wait_until=wait_until,
-                    block_resources=block_resources,
-                    wait_for_url=wait_for_url,
-                    storage_state=self.session_data,  # Pass current session
-                    stealth=stealth,
-                    max_retries=self.max_retries
-                    if max_retries is None
-                    else max_retries,
-                    retry_on=retry_on,
-                    retry_backoff=retry_backoff,
+                
+                last_resp = resp
+    
+                # Evaluate proxy status
+                if not proxy and selected_proxy:
+                    if resp.ok:
+                        self.proxy_pool.mark_success(selected_proxy)
+                    elif resp.error or resp.status in retry_status_codes:
+                        # Mark proxy as failed so a fresh one is used on retry
+                        self.proxy_pool.mark_failed(selected_proxy)
+
+                # Should we retry?
+                should_retry = (
+                    attempt < attempts - 1 and resp.status in retry_status_codes
                 )
-            else:
-                resp = await self._fetch_curl(
+                if not should_retry:
+                    break
+
+                # Backoff before getting a new proxy
+                wait = backoff_base**attempt * (0.5 + random.random())
+                logger.warning(
+                    f"[{engine}] Retryable status {resp.status}, "
+                    f"attempt {attempt + 1}/{attempts}, retrying in {wait:.2f}s"
+                )
+                await asyncio.sleep(wait)
+
+            if not last_resp:
+                last_resp = Response(
                     url=url,
-                    proxy=selected_proxy,
-                    headers=headers,
-                    cookies=cookies,
-                    timeout=timeout or self.timeout,
-                    max_retries=self.max_retries
-                    if max_retries is None
-                    else max_retries,
-                    retry_on=retry_on,
-                    retry_backoff=retry_backoff,
-                    referer=referer,
-                    allow_redirects=allow_redirects,
+                    status=0,
+                    body=b"",
+                    engine=engine,
+                    error="Max retries exhausted",
                 )
+                
+            resp = last_resp
 
             # Update session data from response if present
             if resp.storage_state:
                 self.session_data = resp.storage_state
 
-            # Update proxy stats (ONLY if it came from the pool, or generally?)
-            # If manual override, we might NOT want to impact the pool stats unless the manual proxy IS in the pool?
-            # For simplicity, if we have a pool, and this proxy matches one in the pool, we could update it.
-            # But the 'selected_proxy' might be a new instance specific to this request (manual override).
-            # The pool.mark_* methods take a Proxy object.
-
-            # Logic: If 'proxy' argument was None, it came from pool -> Update stats.
-            # If 'proxy' argument was set -> Do NOT update pool stats (it's a manual override).
-            if not proxy and selected_proxy:
-                if resp.ok:
-                    self.proxy_pool.mark_success(selected_proxy)
-                elif resp.error:
-                    self.proxy_pool.mark_failed(selected_proxy)
-
             # Cache response
             if self.cache and resp.ok and self.cache.should_cache_request("document"):
                 # Re-generate key (same logic as above)
@@ -523,9 +553,6 @@ class Fetcher:
         wait_for_url: str | None = None,
         storage_state: dict[str, Any] | None = None,
         stealth: bool = False,
-        max_retries: int = 1,
-        retry_on: Collection[int] | int | None = None,
-        retry_backoff: float | None = None,
     ) -> Response:
         async with self._semaphore:
             async with self._browser_semaphore:
@@ -538,54 +565,21 @@ class Fetcher:
                         error="CDP engine not initialized",
                     )
 
-                attempts = max(1, max_retries)
-                if retry_on is None:
-                    retry_status_codes: set[int] = set()
-                elif isinstance(retry_on, int):
-                    retry_status_codes = {retry_on}
-                else:
-                    retry_status_codes = set(retry_on)
-
-                backoff_base = 2.0 if retry_backoff is None else retry_backoff
-                last_response: Response | None = None
-
-                for attempt in range(attempts):
-                    response = await self._cdp_engine.fetch(
-                        url=url,
-                        proxy=proxy,
-                        headers=headers,
-                        cookies=cookies,
-                        actions=actions,
-                        timeout=timeout,
-                        location=location,
-                        wait_until=wait_until,
-                        block_resources=block_resources,
-                        wait_for_url=wait_for_url,
-                        storage_state=storage_state,
-                        stealth=stealth,
-                    )
-                    last_response = response
-
-                    should_retry = (
-                        attempt < attempts - 1 and response.status in retry_status_codes
-                    )
-                    if not should_retry:
-                        return response
-
-                    wait = backoff_base**attempt * (0.5 + random.random())
-                    logger.warning(
-                        f"[browser] Retryable status {response.status}, "
-                        f"attempt {attempt + 1}/{attempts}, retrying in {wait:.2f}s"
-                    )
-                    await asyncio.sleep(wait)
-
-                return last_response or Response(
+                response = await self._cdp_engine.fetch(
                     url=url,
-                    status=0,
-                    body=b"",
-                    engine="browser",
-                    error="Max retries exhausted",
+                    proxy=proxy,
+                    headers=headers,
+                    cookies=cookies,
+                    actions=actions,
+                    timeout=timeout,
+                    location=location,
+                    wait_until=wait_until,
+                    block_resources=block_resources,
+                    wait_for_url=wait_for_url,
+                    storage_state=storage_state,
+                    stealth=stealth,
                 )
+                return response
 
     async def _fetch_curl(
         self,