datamarket 0.9.40__tar.gz → 0.9.41__tar.gz

{datamarket-0.9.40 → datamarket-0.9.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.40
+Version: 0.9.41
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -48,6 +48,7 @@ Provides-Extra: proxy
 Provides-Extra: pyarrow
 Provides-Extra: pydrive2
 Provides-Extra: pymupdf
+Provides-Extra: pyrate-limiter
 Provides-Extra: pysocks
 Provides-Extra: pyspark
 Provides-Extra: pytest
@@ -105,6 +106,7 @@ Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "pyarrow"
 Requires-Dist: pycountry (>=24.0.0,<25.0.0)
 Requires-Dist: pydrive2 (>=1.0.0,<2.0.0) ; extra == "pydrive2" or extra == "drive"
 Requires-Dist: pymupdf (>=1.0.0,<2.0.0) ; extra == "pymupdf"
+Requires-Dist: pyrate-limiter (>=3.0.0,<4.0.0) ; extra == "pyrate-limiter"
 Requires-Dist: pysocks (>=1.0.0,<2.0.0) ; extra == "pysocks"
 Requires-Dist: pyspark (>=3.0.0,<4.0.0) ; extra == "pyspark"
 Requires-Dist: pytest (>=8.0.0,<9.0.0) ; extra == "pytest"

{datamarket-0.9.40 → datamarket-0.9.41}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.40"
+version = "0.9.41"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"
@@ -83,6 +83,7 @@ presidio-anonymizer = { version = "^2.0.0", optional = true }
 spacy = { version = "^3.0.0", optional = true }
 spacy-langdetect = { version = "~0.1.0", optional = true }
 pandarallel = { version = "^1.0.0", optional = true }
+pyrate-limiter = { version = "^3.0.0", optional = true }
 
 [tool.poetry.extras]
 boto3 = ["boto3"]
@@ -128,6 +129,7 @@ openpyxl = ["openpyxl"]
 httpx = ["httpx"]
 camoufox = ["camoufox", "browserforge", "playwright"]
 pandarallel = ["pandarallel"]
+pyrate-limiter = ["pyrate-limiter"]
 
 # Interface groups
 aws = ["boto3"]

datamarket-0.9.41/src/datamarket/exceptions/__init__.py

@@ -0,0 +1 @@
+from .main import *  # noqa: F403

datamarket-0.9.41/src/datamarket/exceptions/main.py

@@ -0,0 +1,14 @@
+########################################################################################################################
+# CLASSES
+
+
+class RedirectionDetectedError(Exception):
+    def __init__(self, message="Redirection detected!"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class NotFoundError(Exception):
+    def __init__(self, message="Not found!"):
+        self.message = message
+        super().__init__(self.message)

datamarket-0.9.41/src/datamarket/utils/__init__.py

@@ -0,0 +1 @@
+from .main import *  # noqa: F403

{datamarket-0.9.40 → datamarket-0.9.41}/src/datamarket/utils/main.py

@@ -13,8 +13,26 @@ from pathlib import Path
 from typing import Any, Literal, Self, Union
 
 import pendulum
+import requests
+from bs4 import BeautifulSoup
 from croniter import croniter
 from dynaconf import Dynaconf, add_converter
+from requests.exceptions import ProxyError
+from tenacity import (
+    before_sleep_log,
+    retry,
+    retry_if_exception_type,
+    retry_if_not_exception_type,
+    stop_after_attempt,
+    stop_after_delay,
+    wait_exponential,
+)
+
+from ..exceptions import NotFoundError, RedirectionDetectedError
+from ..interfaces.proxy import ProxyInterface
+
+########################################################################################################################
+# FUNCTIONS
 
 logger = logging.getLogger(__name__)
 
@@ -26,9 +44,7 @@ class NoProjectFoundError(Exception):
 
 class NoPackageFoundError(Exception):
     def __init__(self):
-        super().__init__(
-            "A project was detected but it has no packages inside the 'src' directory"
-        )
+        super().__init__("A project was detected but it has no packages inside the 'src' directory")
 
 
 ########################################################################################################################
@@ -84,8 +100,8 @@ class Project:
 
         try:
             self.pkg_name = next((self.path / "src").glob("*")).name
-        except StopIteration:
-            raise NoPackageFoundError()
+        except StopIteration as e:
+            raise NoPackageFoundError() from e
 
         self.env_name = f"{self.pkg_name}_env"
         self.config_path = self.path / self.CONFIG_FILE_NAME
@@ -227,3 +243,61 @@ def parse_field(dict_struct, field_path, format_method=None):
         if field_value is None:
             return None
     return format_method(field_value) if format_method else field_value
+
+
+@retry(
+    retry=retry_if_not_exception_type((NotFoundError, RedirectionDetectedError, ProxyError)),
+    wait=wait_exponential(exp_base=3, multiplier=3, max=60),
+    stop=stop_after_attempt(5),
+    before_sleep=before_sleep_log(logger, logging.WARNING),
+    reraise=True,
+)
+def get_data(
+    url: str,
+    method: str = "GET",
+    output: str = "json",
+    sleep: tuple = (6, 3),
+    proxy_interface: ProxyInterface = None,
+    use_auth_proxies: bool = False,
+    max_proxy_delay: int = 1800,
+    **kwargs,
+):
+    retry_type = retry_if_exception_type(ProxyError)
+    wait = wait_exponential(exp_base=3, multiplier=3, max=60)
+    stop = stop_after_delay(max_proxy_delay)
+    before_sleep = before_sleep_log(logger, logging.WARNING)
+
+    @retry(retry=retry_type, wait=wait, stop=stop, before_sleep=before_sleep, reraise=True)
+    def _fetch_with_proxy_retry(url, method, proxy_interface, use_auth, **params):
+        logger.info(f"Fetching data from {url} ...")
+        proxy_cfg = None
+        if proxy_interface:
+            host, port, user, pwd = proxy_interface.get_proxies(raw=True, use_auth=use_auth)
+            if host and port:
+                proxy_url = f"http://{host}:{port}"
+                proxy_auth_url = f"http://{user}:{pwd}@{host}:{port}"
+                proxy_cfg = {"http": proxy_url, "https": proxy_url}
+                if user and pwd:
+                    proxy_cfg = {"http": proxy_auth_url, "https": proxy_auth_url}
+                logger.info(f"Using proxy: {proxy_url}")
+        response = getattr(requests, method.lower())(url, proxies=proxy_cfg, **params)
+        return response
+
+    params = {"timeout": 30} | kwargs
+    r = _fetch_with_proxy_retry(url, method, proxy_interface, use_auth_proxies, **params)
+
+    ban_sleep(*sleep)
+
+    if r.status_code == 404:
+        raise NotFoundError(f"404 Not Found error for {url}")
+    r.raise_for_status()
+    r.encoding = "utf-8"
+
+    if output == "json":
+        return r.json()
+    elif output == "text":
+        return r.text
+    elif output == "soup":
+        return BeautifulSoup(r.content, "html.parser")
+    elif output == "response":
+        return r

datamarket-0.9.40/src/datamarket/utils/__init__.py

@@ -1 +0,0 @@
-from .main import *