datamarket 0.7.99__tar.gz → 0.7.101__tar.gz

{datamarket-0.7.99 → datamarket-0.7.101}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.7.99
+Version: 0.7.101
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.7.99 → datamarket-0.7.101}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.7.99"
+version = "0.7.101"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"

{datamarket-0.7.99 → datamarket-0.7.101}/src/datamarket/exceptions/main.py

@@ -12,3 +12,9 @@ class NotFoundError(Exception):
     def __init__(self, message="Not found!"):
         self.message = message
         super().__init__(self.message)
+
+
+class BadRequestError(Exception):
+    def __init__(self, message="Bad request!"):
+        self.message = message
+        super().__init__(self.message)

{datamarket-0.7.99 → datamarket-0.7.101}/src/datamarket/utils/main.py

@@ -3,6 +3,7 @@
 
 import asyncio
 import configparser
+from datetime import timedelta
 import logging
 import random
 import re
@@ -25,7 +26,7 @@ from tenacity import (
     wait_exponential,
 )
 
-from ..exceptions import RedirectionDetectedError, NotFoundError
+from ..exceptions import RedirectionDetectedError, NotFoundError, BadRequestError
 from ..interfaces.proxy import ProxyInterface
 
 ########################################################################################################################
@@ -131,7 +132,7 @@ def parse_field(dict_struct, field_path, format_method=None):
 
 
 @retry(
-    retry=retry_if_not_exception_type((NotFoundError, RedirectionDetectedError, ProxyError)),
+    retry=retry_if_not_exception_type((NotFoundError, BadRequestError, RedirectionDetectedError, ProxyError)),
     wait=wait_exponential(exp_base=3, multiplier=3, max=60),
     stop=stop_after_attempt(5),
     before_sleep=before_sleep_log(logger, logging.WARNING),
@@ -144,7 +145,7 @@ def get_data(
     sleep: tuple = (6, 3),
     proxy_interface: ProxyInterface = None,
     use_auth_proxies: bool = False,
-    max_proxy_delay: int = 1800,
+    max_proxy_delay: timedelta = timedelta(minutes=10),
     **kwargs,
 ):
     retry_type = retry_if_exception_type(ProxyError)
@@ -175,6 +176,8 @@ def get_data(
 
     if r.status_code == 404:
         raise NotFoundError(f"404 Not Found error for {url}")
+    if r.status_code == 400:
+        raise BadRequestError(f"400 Bad Request error for {url}")
     r.raise_for_status()
     r.encoding = "utf-8"
 

{datamarket-0.7.99 → datamarket-0.7.101}/src/datamarket/utils/playwright/sync_api.py

@@ -53,12 +53,14 @@ def human_press_key(page: Page, key: str, count: int = 1, delay: int = 100, add_
 class PlaywrightCrawler:
     """A robust, proxy-enabled Playwright crawler with captcha bypass and retry logic."""
 
-    def __init__(self, proxy_interface: ProxyInterface):
+    def __init__(self, proxy_interface: Optional[ProxyInterface] = None):
         """
-        Initializes the crawler with a proxy interface.
+        Initializes the crawler.
 
         Args:
-            proxy_interface (ProxyInterface): An object to fetch proxy credentials.
+            proxy_interface (Optional[ProxyInterface], optional): Provider used to fetch
+                proxy credentials. Defaults to None. When None, no proxy is configured and
+                the browser will run without a proxy.
         """
         self.proxy_interface = proxy_interface
         self.pw: Optional[Camoufox] = None
@@ -81,6 +83,25 @@ class PlaywrightCrawler:
         if self.pw:
             self.pw.__exit__(exc_type, exc_val, exc_tb)
 
+    def _build_proxy_config(self) -> Optional[dict]:
+        """Builds the proxy configuration dictionary.
+
+        Returns:
+            Optional[dict]: Proxy configuration if a proxy_interface is provided; otherwise None.
+        """
+        if not self.proxy_interface:
+            logger.info("Starting browser without proxy.")
+            return None
+
+        host, port, user, pwd = self.proxy_interface.get_proxies(raw=True, use_auth=True)
+        proxy_url = f"http://{host}:{port}"
+        proxy_cfg: dict = {"server": proxy_url}
+        if user and pwd:
+            proxy_cfg.update({"username": user, "password": pwd})
+
+        logger.info(f"Starting browser with proxy: {proxy_url}")
+        return proxy_cfg
+
     @retry(
         wait=wait_exponential(exp_base=2, multiplier=3, max=90),
         stop=stop_after_delay(timedelta(minutes=10)),
@@ -88,16 +109,20 @@ class PlaywrightCrawler:
         reraise=True,
     )
     def init_context(self) -> Self:
-        """Initializes a new browser instance and context with a fresh proxy."""
-        try:
-            host, port, user, pwd = self.proxy_interface.get_proxies(raw=True, use_auth=True)
-            proxy_url = f"http://{host}:{port}"
-            proxy_cfg = {"server": proxy_url}
+        """
+        Initializes a new browser instance and context.
+
+        Behavior:
+        - If a proxy_interface is provided, fetches fresh proxy credentials and starts
+          the browser using that proxy.
+        - If proxy_interface is None, starts the browser without any proxy.
 
-            if user and pwd:
-                proxy_cfg.update({"username": user, "password": pwd})
+        Returns:
+            Self: The crawler instance with active browser, context, and page.
+        """
+        try:
+            proxy_cfg: Optional[dict] = self._build_proxy_config()
 
-            logger.info(f"Starting browser with proxy: {proxy_url}")
             self.pw = Camoufox(headless=True, geoip=True, humanize=True, proxy=proxy_cfg)
             self.browser = self.pw.__enter__()
             self.context = self.browser.new_context()
@@ -146,4 +171,4 @@ class PlaywrightCrawler:
         if not self.page:
             logger.info("Browser context not found, initializing now...")
             self.init_context()
-        return self._goto_with_retry(url)
+        return self._goto_with_retry(url)