fraudcrawler 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

fraudcrawler/base/base.py

@@ -9,7 +9,7 @@ from pydantic import (
 from pydantic_settings import BaseSettings
 from urllib.parse import urlparse
 import re
-from typing import Any, Dict, List
+from typing import Any, Dict, List, TYPE_CHECKING
 
 import httpx
 
@@ -23,6 +23,9 @@ from fraudcrawler.settings import (
     DEFAULT_HTTPX_REDIRECTS,
 )
 
+if TYPE_CHECKING:
+    from fraudcrawler.scraping.zyte import ZyteAPI
+
 logger = logging.getLogger(__name__)
 
 # Load google locations and languages
@@ -241,3 +244,35 @@ class DomainUtils:
         if hostname and hostname.startswith("www."):
             hostname = hostname[4:]
         return hostname.lower()
+
+    async def _unblock_url(self, url: str, zyte_api: "ZyteAPI") -> bytes | None:
+        """Attempts to unblock a URL using Zyte proxy mode when direct access fails.
+
+        This method is specifically designed to handle 403 Forbidden errors for domains
+        that may be blocking requests from certain IP ranges (like cloud providers).
+
+        Args:
+            url: The URL to fetch using Zyte proxy mode.
+            zyte_api: An instance of ZyteAPI to use for the request.
+
+        Returns:
+            The HTML content as bytes if successful, None if failed.
+        """
+        try:
+            logger.info(f"Attempting to unblock URL using Zyte proxy: {url}")
+            details = await zyte_api.details(url)
+
+            if details and "httpResponseBody" in details:
+                # Decode the base64 content
+                import base64
+
+                html_content = base64.b64decode(details["httpResponseBody"])
+                logger.info(f"Successfully unblocked URL using Zyte proxy: {url}")
+                return html_content
+            else:
+                logger.warning(f"Zyte proxy request failed for URL: {url}")
+                return None
+
+        except Exception as e:
+            logger.error(f"Error unblocking URL with Zyte proxy: {url}, error: {e}")
+            return None

fraudcrawler/base/orchestrator.py

@@ -114,8 +114,13 @@ class Orchestrator(ABC):
             self._owns_http_client = True
 
         # Setup the clients
+        self._zyteapi = ZyteAPI(
+            http_client=self._http_client, api_key=self._zyteapi_key
+        )
         self._search = Search(
-            http_client=self._http_client, serpapi_key=self._serpapi_key
+            http_client=self._http_client,
+            serpapi_key=self._serpapi_key,
+            zyte_api=self._zyteapi,
         )
         self._enricher = Enricher(
             http_client=self._http_client,
@@ -123,9 +128,6 @@ class Orchestrator(ABC):
             pwd=self._dataforseo_pwd,
         )
         self._url_collector = URLCollector()
-        self._zyteapi = ZyteAPI(
-            http_client=self._http_client, api_key=self._zyteapi_key
-        )
         self._processor = Processor(
             http_client=self._http_client,
             api_key=self._openaiapi_key,

fraudcrawler/scraping/search.py

@@ -376,13 +376,15 @@ class Toppreise(SearchEngine):
         "Upgrade-Insecure-Requests": "1",
     }
 
-    def __init__(self, http_client: httpx.AsyncClient):
+    def __init__(self, http_client: httpx.AsyncClient, zyte_api=None):
         """Initializes the Toppreise client.
 
         Args:
             http_client: An httpx.AsyncClient to use for the async requests.
+            zyte_api: Optional ZyteAPI instance for fallback when direct access fails.
         """
         self._http_client = http_client
+        self._zyte_api = zyte_api
 
     @property
     def _search_engine_name(self) -> str:
@@ -448,16 +450,32 @@ class Toppreise(SearchEngine):
         retry.before_sleep = lambda retry_state: self._log_before_sleep(
             search_string=search_string, retry_state=retry_state
         )
-        async for attempt in retry:
-            with attempt:
-                response = await self._http_client.get(
-                    url=url,
-                    headers=self._headers,
+
+        content = None
+        try:
+            async for attempt in retry:
+                with attempt:
+                    response = await self._http_client.get(
+                        url=url,
+                        headers=self._headers,
+                    )
+                    response.raise_for_status()
+                    content = response.content
+        except httpx.HTTPStatusError as e:
+            if e.response.status_code == 403 and self._zyte_api:
+                logger.warning(
+                    f"Received 403 Forbidden for {url}, attempting to unblock with Zyte proxy"
                 )
-                response.raise_for_status()
+                content = await self._unblock_url(url, self._zyte_api)
+                if content is None:
+                    raise e  # Re-raise if zyte fallback also failed
+            else:
+                raise e
+
+        if content is None:
+            raise httpx.HTTPError("Failed to fetch content")
 
         # Get external product urls from the content
-        content = response.content
         urls = self._get_external_product_urls(content=content)
         urls = urls[:num_results]  # Limit to num_results if needed
 
@@ -491,18 +509,19 @@ class Toppreise(SearchEngine):
 class Search(DomainUtils):
     """Class to perform searches using different search engines."""
 
-    def __init__(self, http_client: httpx.AsyncClient, serpapi_key: str):
+    def __init__(self, http_client: httpx.AsyncClient, serpapi_key: str, zyte_api=None):
         """Initializes the Search class with the given SerpAPI key.
 
         Args:
             http_client: An httpx.AsyncClient to use for the async requests.
             serpapi_key: The API key for SERP API.
+            zyte_api: Optional ZyteAPI instance for fallback when direct access fails.
         """
         self._google = SerpAPIGoogle(http_client=http_client, api_key=serpapi_key)
         self._google_shopping = SerpAPIGoogleShopping(
             http_client=http_client, api_key=serpapi_key
         )
-        self._toppreise = Toppreise(http_client=http_client)
+        self._toppreise = Toppreise(http_client=http_client, zyte_api=zyte_api)
 
     @staticmethod
     def _domain_in_host(domain: str, host: Host) -> bool:

{fraudcrawler-0.5.6.dist-info → fraudcrawler-0.5.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fraudcrawler
-Version: 0.5.6
+Version: 0.5.8
 Summary: Intelligent Market Monitoring
 Home-page: https://github.com/open-veanu/fraudcrawler
 License: MIT

{fraudcrawler-0.5.6.dist-info → fraudcrawler-0.5.8.dist-info}/RECORD

@@ -1,22 +1,22 @@
 fraudcrawler/__init__.py,sha256=Kr19jWhtbC1shVoB9fHvBSeoG1IyQB9re1kCZ4YIAi0,842
 fraudcrawler/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fraudcrawler/base/base.py,sha256=suQMnvLIsZO_R0eHZKDWS4u9qnd1ryzPhjGlwcaMD5A,7295
+fraudcrawler/base/base.py,sha256=NOJC12qw-iSkHScPnxFLfzUvg0w57qGaID6OAzHRXeo,8695
 fraudcrawler/base/client.py,sha256=yhkNrhL2SuJXTknLf-8P81fv01FnFMahREZgem-Z-f0,5832
 fraudcrawler/base/google-languages.json,sha256=z0VtjMCsCcZq11OkCIb9jMDD1p9Ty4lhV7bq4ddYvec,10748
 fraudcrawler/base/google-locations.json,sha256=UtNu0iSStllvFRTQXMobWKmZR1hKmtgtHftLNgaJTT0,9204345
-fraudcrawler/base/orchestrator.py,sha256=AKEETrYwKbMy_6YgTdgc6L-VA1iHYOtj3wIqEN3ngO4,26990
+fraudcrawler/base/orchestrator.py,sha256=28X45XLPlJe2hvff8HTLo-V08LNeS0zMWBHe5W3hk4c,27039
 fraudcrawler/base/retry.py,sha256=9VyVrbYR_0YnfxFhUrvcM3aWCYR6oR4iZE4A3zzVZUs,1015
 fraudcrawler/launch_demo_pipeline.py,sha256=j5lu8lLl8QrkVU1MJH25uKtyYk_6lBSeoouCo30aRXg,4634
 fraudcrawler/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fraudcrawler/processing/processor.py,sha256=Qq8QcTlqfnzFi1t-1KkncXxaIszUO7pGK3LXTdHkDnM,7638
 fraudcrawler/scraping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fraudcrawler/scraping/enrich.py,sha256=M4ErLF7q-5EKfEd-dIDS73mZc-aPFp5pJbgdRKCO3f8,13112
-fraudcrawler/scraping/search.py,sha256=nHMYaSkq9o6Hr4yUDEPguj8IHVcOpws3_XWiAbCVgLg,24062
+fraudcrawler/scraping/search.py,sha256=ZjxOj95ih6o6bOWA0JnBwjFlMzGS-8Sb1P-yvHI5aO0,24957
 fraudcrawler/scraping/url.py,sha256=5Z3hPW73E-TLhM-Zha8OTcUOumc_rcx64R0fT9z2Hi8,1748
 fraudcrawler/scraping/zyte.py,sha256=GqvVWA1AWVoClAwd-hQ9iynsT0dOb7R0ntaLK5XVivM,8340
 fraudcrawler/settings.py,sha256=uwXMOQpuwyWkuMU0asYGtBlL_qJj8F-Xkg4dUaCmDxE,3670
-fraudcrawler-0.5.6.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
-fraudcrawler-0.5.6.dist-info/METADATA,sha256=JXYbk0Qo24eBAByL_alhmDCTXTpzHycJeGnpZrGnOYg,6642
-fraudcrawler-0.5.6.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-fraudcrawler-0.5.6.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
-fraudcrawler-0.5.6.dist-info/RECORD,,
+fraudcrawler-0.5.8.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
+fraudcrawler-0.5.8.dist-info/METADATA,sha256=-e9xqpIk0EjO6fqwhmQZ5gsDrl6eJKU7VQdp8MeN0R4,6642
+fraudcrawler-0.5.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+fraudcrawler-0.5.8.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
+fraudcrawler-0.5.8.dist-info/RECORD,,