fraudcrawler 0.4.7__py3-none-any.whl → 0.5.1__py3-none-any.whl

fraudcrawler/scraping/zyte.py

@@ -2,17 +2,17 @@ import logging
 from typing import List
 from base64 import b64decode
 
-import aiohttp
+import httpx
 from tenacity import RetryCallState
 
 from fraudcrawler.settings import ZYTE_DEFALUT_PROBABILITY_THRESHOLD
-from fraudcrawler.base.base import AsyncClient
+from fraudcrawler.base.base import DomainUtils
 from fraudcrawler.base.retry import get_async_retry
 
 logger = logging.getLogger(__name__)
 
 
-class ZyteApi(AsyncClient):
+class ZyteAPI(DomainUtils):
     """A client to interact with the Zyte API for fetching product details."""
 
     _endpoint = "https://api.zyte.com/v1/extract"
@@ -30,14 +30,17 @@ class ZyteApi(AsyncClient):
 
     def __init__(
         self,
+        http_client: httpx.AsyncClient,
         api_key: str,
     ):
         """Initializes the ZyteApiClient with the given API key and retry configurations.
 
         Args:
+            http_client: An httpx.AsyncClient to use for the async requests.
             api_key: The API key for Zyte API.
         """
-        self._aiohttp_basic_auth = aiohttp.BasicAuth(api_key)
+        self._http_client = http_client
+        self._api_key = api_key
 
     def _log_before(self, url: str, retry_state: RetryCallState | None) -> None:
         """Context aware logging before the request is made."""
@@ -58,7 +61,7 @@ class ZyteApi(AsyncClient):
         else:
             logger.debug(f"retry_state is {retry_state}; not logging before_sleep.")
 
-    async def get_details(self, url: str) -> dict:
+    async def details(self, url: str) -> dict:
         """Fetches product details for a single URL.
 
         Args:
@@ -97,16 +100,20 @@ class ZyteApi(AsyncClient):
         )
         async for attempt in retry:
             with attempt:
-                product = await self.post(
+                response = await self._http_client.post(
                     url=self._endpoint,
-                    data={"url": url, **self._config},
-                    auth=self._aiohttp_basic_auth,
+                    json={"url": url, **self._config},
+                    auth=(self._api_key, ""),  # API key as username, empty password
                 )
-        return product
+                response.raise_for_status()
+
+        details = response.json()
+        return details
 
     @staticmethod
     def keep_product(
-        details: dict, threshold: float = ZYTE_DEFALUT_PROBABILITY_THRESHOLD
+        details: dict,
+        threshold: float = ZYTE_DEFALUT_PROBABILITY_THRESHOLD,
     ) -> bool:
         """Determines whether to keep the product based on the probability threshold.
 
@@ -136,6 +143,19 @@ class ZyteApi(AsyncClient):
         """
         return details.get("product", {}).get("name")
 
+    @staticmethod
+    def extract_url_resolved(details: dict) -> str | None:
+        """Extracts the resolved URL from the product data - this is automatically resolved by Zyte.
+
+        The input argument is a dictionary of the following structure:
+            {
+                "product": {
+                    "url": str,
+                }
+            }
+        """
+        return details.get("product", {}).get("url")
+
     @staticmethod
     def extract_product_price(details: dict) -> str | None:
         """Extracts the product price from the product data.
@@ -198,7 +218,9 @@ class ZyteApi(AsyncClient):
                 }
             }
         """
-        return float(details.get("product", {}).get("metadata", {}).get("probability"))
+        return float(
+            details.get("product", {}).get("metadata", {}).get("probability", 0.0)
+        )
 
     @staticmethod
     def extract_html(details: dict) -> str | None:
@@ -209,7 +231,6 @@ class ZyteApi(AsyncClient):
                 "httpResponseBody": base64
             }
         """
-
         # Get the Base64-encoded content
         encoded = details.get("httpResponseBody")
 
@@ -217,6 +238,7 @@ class ZyteApi(AsyncClient):
         if isinstance(encoded, str):
             decoded_bytes = b64decode(encoded)
 
-        # Convert bytes to string (assuming UTF-8 encoding)
-        decoded_string = decoded_bytes.decode("utf-8")
-        return decoded_string
+            # Convert bytes to string (assuming UTF-8 encoding)
+            decoded_string = decoded_bytes.decode("utf-8")
+            return decoded_string
+        return None

fraudcrawler/settings.py

@@ -17,7 +17,7 @@ RETRY_SKIP_IF_CODE = [400, 401, 403]  # Skip retrying on these HTTP status codes
 # Serp settings
 GOOGLE_LOCATIONS_FILENAME = ROOT_DIR / "fraudcrawler" / "base" / "google-locations.json"
 GOOGLE_LANGUAGES_FILENAME = ROOT_DIR / "fraudcrawler" / "base" / "google-languages.json"
-SERP_DEFAULT_COUNTRY_CODES: List[str] = [
+SEARCH_DEFAULT_COUNTRY_CODES: List[str] = [
     # ".com",
 ]
 
@@ -75,7 +75,18 @@ PROCESSOR_EMPTY_TOKEN_COUNT = -1
 PROCESSOR_USER_PROMPT_TEMPLATE = "Product Details:\n{product_details}\n\nRelevance:"
 PROCESSOR_PRODUCT_DETAILS_TEMPLATE = "{field_name}:\n{field_value}"
 
-# Async settings
+# Async workers settings
 DEFAULT_N_SERP_WKRS = 10
 DEFAULT_N_ZYTE_WKRS = 10
 DEFAULT_N_PROC_WKRS = 10
+
+# HTTPX client settings
+DEFAULT_HTTPX_TIMEOUT = {
+    "timeout": 600,
+    "connect": 5.0,
+}
+DEFAULT_HTTPX_LIMITS = {
+    "max_connections": 1000,
+    "max_keepalive_connections": 100,
+}
+DEFAULT_HTTPX_REDIRECTS = True

{fraudcrawler-0.4.7.dist-info → fraudcrawler-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fraudcrawler
-Version: 0.4.7
+Version: 0.5.1
 Summary: Intelligent Market Monitoring
 Home-page: https://github.com/open-veanu/fraudcrawler
 License: MIT
@@ -11,8 +11,8 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: aiohttp (>=3.11.14,<4.0.0)
 Requires-Dist: beautifulsoup4 (>=4.13.4,<5.0.0)
+Requires-Dist: httpx (>=0.28.1,<0.29.0)
 Requires-Dist: openai (>=1.68.2,<2.0.0)
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0)
@@ -158,6 +158,10 @@ client.print_available_results()
 see `CONTRIBUTING.md`
 
 ### Async Setup
+The `Orchestrator` class in `src/base/orchestrator.py` is designed to coordinate multiple services that may have interdependencies, allowing them to run in a semi-iterative manner. This means, for example, that product A can be at stage III of the pipeline while product B is still at stage I.
+
+This behavior is enabled through an asynchronous pipeline setup. The three main steps, `SerpAPI`, `ZyteAPI`, and `Processor`, all utilize `httpx.AsyncClient`. It is both possible and highly recommended to manage a single AsyncClient instance per application for efficiency. We provide a `HttpxAsyncClient` class that you can pass For more details, see the [httpx documentation](https://www.python-httpx.org/api/#asyncclient).
+
 The following image provides a schematic representation of the package's async setup.
 ![Async Setup](https://github.com/open-veanu/fraudcrawler/raw/master/docs/assets/images/Fraudcrawler_Async_Setup.svg)
 

fraudcrawler-0.5.1.dist-info/RECORD

@@ -0,0 +1,22 @@
+fraudcrawler/__init__.py,sha256=Kr19jWhtbC1shVoB9fHvBSeoG1IyQB9re1kCZ4YIAi0,842
+fraudcrawler/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fraudcrawler/base/base.py,sha256=suQMnvLIsZO_R0eHZKDWS4u9qnd1ryzPhjGlwcaMD5A,7295
+fraudcrawler/base/client.py,sha256=yhkNrhL2SuJXTknLf-8P81fv01FnFMahREZgem-Z-f0,5832
+fraudcrawler/base/google-languages.json,sha256=z0VtjMCsCcZq11OkCIb9jMDD1p9Ty4lhV7bq4ddYvec,10748
+fraudcrawler/base/google-locations.json,sha256=UtNu0iSStllvFRTQXMobWKmZR1hKmtgtHftLNgaJTT0,9204345
+fraudcrawler/base/orchestrator.py,sha256=AKEETrYwKbMy_6YgTdgc6L-VA1iHYOtj3wIqEN3ngO4,26990
+fraudcrawler/base/retry.py,sha256=9VyVrbYR_0YnfxFhUrvcM3aWCYR6oR4iZE4A3zzVZUs,1015
+fraudcrawler/launch_demo_pipeline.py,sha256=j5lu8lLl8QrkVU1MJH25uKtyYk_6lBSeoouCo30aRXg,4634
+fraudcrawler/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fraudcrawler/processing/processor.py,sha256=Qq8QcTlqfnzFi1t-1KkncXxaIszUO7pGK3LXTdHkDnM,7638
+fraudcrawler/scraping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fraudcrawler/scraping/enrich.py,sha256=M4ErLF7q-5EKfEd-dIDS73mZc-aPFp5pJbgdRKCO3f8,13112
+fraudcrawler/scraping/search.py,sha256=nNkLgkF12AFS9A5vPs8yCb7irqxn6hiNWhS8Ku28oO0,24062
+fraudcrawler/scraping/url.py,sha256=5Z3hPW73E-TLhM-Zha8OTcUOumc_rcx64R0fT9z2Hi8,1748
+fraudcrawler/scraping/zyte.py,sha256=GqvVWA1AWVoClAwd-hQ9iynsT0dOb7R0ntaLK5XVivM,8340
+fraudcrawler/settings.py,sha256=uwXMOQpuwyWkuMU0asYGtBlL_qJj8F-Xkg4dUaCmDxE,3670
+fraudcrawler-0.5.1.dist-info/LICENSE,sha256=B-3FuHfe3S0fWAlKlceskPcRhzXq81g-rJ-ddUYb4O8,1062
+fraudcrawler-0.5.1.dist-info/METADATA,sha256=QsNxcNbL02EmhHZTpusO5t4uleOW15lEmWc76Kv3BHQ,6642
+fraudcrawler-0.5.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+fraudcrawler-0.5.1.dist-info/entry_points.txt,sha256=1Befm7cM6945y2AA1z9V4gZV63mtSWcAs7ypvgux_Xg,79
+fraudcrawler-0.5.1.dist-info/RECORD,,