fraudcrawler 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

fraudcrawler/base/base.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import base64
 from pydantic import (
     BaseModel,
     Field,
@@ -9,7 +10,7 @@ from pydantic import (
 from pydantic_settings import BaseSettings
 from urllib.parse import urlparse
 import re
-from typing import Any, Dict, List, TYPE_CHECKING
+from typing import Any, Dict, List
 
 import httpx
 
@@ -23,9 +24,6 @@ from fraudcrawler.settings import (
     DEFAULT_HTTPX_REDIRECTS,
 )
 
-if TYPE_CHECKING:
-    from fraudcrawler.scraping.zyte import ZyteAPI
-
 logger = logging.getLogger(__name__)
 
 # Load google locations and languages
@@ -245,33 +243,31 @@ class DomainUtils:
             hostname = hostname[4:]
         return hostname.lower()
 
-    async def _unblock_url(self, url: str, zyte_api: "ZyteAPI") -> bytes | None:
+    async def _unblock_url(self, url: str, zyte_api: Any) -> bytes | None:
         """Attempts to unblock a URL using Zyte proxy mode when direct access fails.
-        
+
         This method is specifically designed to handle 403 Forbidden errors for domains
         that may be blocking requests from certain IP ranges (like cloud providers).
-        
+
         Args:
             url: The URL to fetch using Zyte proxy mode.
             zyte_api: An instance of ZyteAPI to use for the request.
-            
+
         Returns:
             The HTML content as bytes if successful, None if failed.
         """
         try:
             logger.info(f"Attempting to unblock URL using Zyte proxy: {url}")
             details = await zyte_api.details(url)
-            
+
             if details and "httpResponseBody" in details:
-                # Decode the base64 content
-                import base64
                 html_content = base64.b64decode(details["httpResponseBody"])
                 logger.info(f"Successfully unblocked URL using Zyte proxy: {url}")
                 return html_content
             else:
                 logger.warning(f"Zyte proxy request failed for URL: {url}")
                 return None
-                
+
         except Exception as e:
             logger.error(f"Error unblocking URL with Zyte proxy: {url}, error: {e}")
             return None

fraudcrawler/base/orchestrator.py

@@ -114,8 +114,13 @@ class Orchestrator(ABC):
             self._owns_http_client = True
 
         # Setup the clients
+        self._zyteapi = ZyteAPI(
+            http_client=self._http_client, api_key=self._zyteapi_key
+        )
         self._search = Search(
-            http_client=self._http_client, serpapi_key=self._serpapi_key, zyte_api=self._zyteapi
+            http_client=self._http_client,
+            serpapi_key=self._serpapi_key,
+            zyte_api=self._zyteapi,
         )
         self._enricher = Enricher(
             http_client=self._http_client,
@@ -123,9 +128,6 @@ class Orchestrator(ABC):
             pwd=self._dataforseo_pwd,
         )
         self._url_collector = URLCollector()
-        self._zyteapi = ZyteAPI(
-            http_client=self._http_client, api_key=self._zyteapi_key
-        )
         self._processor = Processor(
             http_client=self._http_client,
             api_key=self._openaiapi_key,

fraudcrawler/launch_demo_pipeline.py

@@ -97,4 +97,4 @@ def search(search_term: str):
 
 
 if __name__ == "__main__":
-    search(search_term='Liebherr "TP1410"')
+    search(search_term="electric cigarettes")

fraudcrawler/scraping/search.py

@@ -131,6 +131,17 @@ class SerpAPI(SearchEngine):
             search_string += " site:" + " OR site:".join(s for s in sites)
         return search_string
 
+    @staticmethod
+    def _get_google_domain(location: Location) -> str:
+        """Gets the Google domain for the given location if they do not use the default pattern google.tld"""
+        if location.name == "Brazil":
+            return "google.com.br"
+        elif location.name == "United Kingdom":
+            return "google.co.uk"
+        elif location.name == "Argentina":
+            return "google.com.ar"
+        return f"google.{location.code}"
+
     async def _search(
         self,
         search_string: str,
@@ -169,16 +180,19 @@ class SerpAPI(SearchEngine):
             f"num_results={num_results}."
         )
 
-        # Setup the parameters
+        # Get Google domain and country code
+        google_domain = self._get_google_domain(location)
+        country_code = location.code
+
         params: Dict[str, str | int] = {
             "engine": engine,
             "q": search_string,
-            "google_domain": f"google.{location.code}",
+            "google_domain": google_domain,
             "location_requested": location.name,
             "location_used": location.name,
-            "tbs": f"ctr:{location.code.upper()}",
-            "cr": f"country{location.code.upper()}",
-            "gl": location.code,
+            "tbs": f"ctr:{country_code.upper()}",
+            "cr": f"country{country_code.upper()}",
+            "gl": country_code,
             "hl": language.code,
             "num": num_results,
             "api_key": self._api_key,
@@ -450,7 +464,7 @@ class Toppreise(SearchEngine):
         retry.before_sleep = lambda retry_state: self._log_before_sleep(
             search_string=search_string, retry_state=retry_state
         )
-        
+
         content = None
         try:
             async for attempt in retry:
@@ -463,7 +477,9 @@ class Toppreise(SearchEngine):
                     content = response.content
         except httpx.HTTPStatusError as e:
             if e.response.status_code == 403 and self._zyte_api:
-                logger.warning(f"Received 403 Forbidden for {url}, attempting to unblock with Zyte proxy")
+                logger.warning(
+                    f"Received 403 Forbidden for {url}, attempting to unblock with Zyte proxy"
+                )
                 content = await self._unblock_url(url, self._zyte_api)
                 if content is None:
                     raise e  # Re-raise if zyte fallback also failed
@@ -471,7 +487,7 @@ class Toppreise(SearchEngine):
                 raise e
 
         if content is None:
-            raise httpx.HTTPStatusError("Failed to fetch content", request=None, response=None)
+            raise httpx.HTTPError("Failed to fetch content")
 
         # Get external product urls from the content
         urls = self._get_external_product_urls(content=content)

{fraudcrawler-0.5.7.dist-info → fraudcrawler-0.5.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fraudcrawler
-Version: 0.5.7
+Version: 0.5.9
 Summary: Intelligent Market Monitoring
 Home-page: https://github.com/open-veanu/fraudcrawler
 License: MIT

{fraudcrawler-0.5.7.dist-info → fraudcrawler-0.5.9.dist-info}/RECORD

@@ -1,22 +1,22 @@
 fraudcrawler/__init__.py,sha256=Kr19jWhtbC1shVoB9fHvBSeoG1IyQB9re1kCZ4YIAi0,842
 fraudcrawler/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fraudcrawler/base/base.py,sha256=94HTs8RpdpEics9d6o_uDniTRG1CCSO35LDsjY4hp5E,8750
+fraudcrawler/base/base.py,sha256=S5VP1PD2tymZMmVcVmP4FI0VlCQxDCB2gp0vVD6eN8g,8543
 fraudcrawler/base/client.py,sha256=yhkNrhL2SuJXTknLf-8P81fv01FnFMahREZgem-Z-f0,5832
 fraudcrawler/base/google-languages.json,sha256=z0VtjMCsCcZq11OkCIb9jMDD1p9Ty4lhV7bq4ddYvec,10748
 fraudcrawler/base/google-locations.json,sha256=UtNu0iSStllvFRTQXMobWKmZR1hKmtgtHftLNgaJTT0,9204345
-fraudcrawler/base/orchestrator.py,sha256=lyrdX_pEq2y3VguXMRMmyEJviGEr5-SnqeIxoJmqWKc,27014
+fraudcrawler/base/orchestrator.py,sha256=28X45XLPlJe2hvff8HTLo-V08LNeS0zMWBHe5W3hk4c,27039
 fraudcrawler/base/retry.py,sha256=9VyVrbYR_0YnfxFhUrvcM3aWCYR6oR4iZE4A3zzVZUs,1015
-fraudcrawler/launch_demo_pipeline.py,sha256=j5lu8lLl8QrkVU1MJH25uKtyYk_6lBSeoouCo30aRXg,4634
+fraudcrawler/launch_demo_pipeline.py,sha256=hTzGFQDEwchDSwUx0HgG_TW5h9J7BXM7jn_iB8iI838,4636
 fraudcrawler/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fraudcrawler/processing/processor.py,sha256=Qq8QcTlqfnzFi1t-1KkncXxaIszUO7pGK3LXTdHkDnM,7638
 fraudcrawler/scraping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fraudcrawler/scraping/enrich.py,sha256=M4ErLF7q-5EKfEd-dIDS73mZc-aPFp5pJbgdRKCO3f8,13112
-fraudcrawler/scraping/search.py,sha256=JQ4nbylYdAk65yDDAatv-qGekRRRNy769VHQgzhqN8Y,24962
+fraudcrawler/scraping/search.py,sha256=5fTRo7Tkkz9REU-u50oZu4kbynjIcIigp9HZ0cS_UCg,25510
 fraudcrawler/scraping/url.py,sha256=5Z3hPW73E-TLhM-Zha8OTcUOumc_rcx64R0fT9z2Hi8,1748
 fraudcrawler/scraping/zyte.py,sha256=GqvVWA1AWVoClAwd-hQ9iynsT0dOb7R0ntaLK5XVivM,8340
 fraudcrawler/settings.py,sha256=uwXMOQpuwyWkuMU0asYGtBlL_qJj8F-Xkg4dUaCmDxE,3670
-fraudcrawler-0.5.7.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
-fraudcrawler-0.5.7.dist-info/METADATA,sha256=tMdND63UPo5x2s49o_RMzQzqTSEBdrsv1TqQPL65DaM,6642
-fraudcrawler-0.5.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-fraudcrawler-0.5.7.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
-fraudcrawler-0.5.7.dist-info/RECORD,,
+fraudcrawler-0.5.9.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
+fraudcrawler-0.5.9.dist-info/METADATA,sha256=sMszTwcTVxuMGfRhJJZKLAPShnY4zDU6gShAa6k3tPg,6642
+fraudcrawler-0.5.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+fraudcrawler-0.5.9.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
+fraudcrawler-0.5.9.dist-info/RECORD,,