datamarket 0.9.40__py3-none-any.whl → 0.9.42__py3-none-any.whl

datamarket/exceptions/__init__.py

@@ -0,0 +1 @@
+from .main import *  # noqa: F403

datamarket/exceptions/main.py

@@ -0,0 +1,14 @@
+########################################################################################################################
+# CLASSES
+
+
+class RedirectionDetectedError(Exception):
+    def __init__(self, message="Redirection detected!"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class NotFoundError(Exception):
+    def __init__(self, message="Not found!"):
+        self.message = message
+        super().__init__(self.message)

datamarket/utils/__init__.py

@@ -1 +1 @@
-from .main import *
+from .main import *  # noqa: F403

datamarket/utils/main.py

@@ -13,8 +13,26 @@ from pathlib import Path
 from typing import Any, Literal, Self, Union
 
 import pendulum
+import requests
+from bs4 import BeautifulSoup
 from croniter import croniter
 from dynaconf import Dynaconf, add_converter
+from requests.exceptions import ProxyError
+from tenacity import (
+    before_sleep_log,
+    retry,
+    retry_if_exception_type,
+    retry_if_not_exception_type,
+    stop_after_attempt,
+    stop_after_delay,
+    wait_exponential,
+)
+
+from ..exceptions import NotFoundError, RedirectionDetectedError
+from ..interfaces.proxy import ProxyInterface
+
+########################################################################################################################
+# FUNCTIONS
 
 logger = logging.getLogger(__name__)
 
@@ -26,9 +44,7 @@ class NoProjectFoundError(Exception):
 
 class NoPackageFoundError(Exception):
     def __init__(self):
-        super().__init__(
-            "A project was detected but it has no packages inside the 'src' directory"
-        )
+        super().__init__("A project was detected but it has no packages inside the 'src' directory")
 
 
 ########################################################################################################################
@@ -84,8 +100,8 @@ class Project:
 
         try:
             self.pkg_name = next((self.path / "src").glob("*")).name
-        except StopIteration:
-            raise NoPackageFoundError()
+        except StopIteration as e:
+            raise NoPackageFoundError() from e
 
         self.env_name = f"{self.pkg_name}_env"
         self.config_path = self.path / self.CONFIG_FILE_NAME
@@ -227,3 +243,61 @@ def parse_field(dict_struct, field_path, format_method=None):
         if field_value is None:
             return None
     return format_method(field_value) if format_method else field_value
+
+
+@retry(
+    retry=retry_if_not_exception_type((NotFoundError, RedirectionDetectedError, ProxyError)),
+    wait=wait_exponential(exp_base=3, multiplier=3, max=60),
+    stop=stop_after_attempt(5),
+    before_sleep=before_sleep_log(logger, logging.WARNING),
+    reraise=True,
+)
+def get_data(
+    url: str,
+    method: str = "GET",
+    output: str = "json",
+    sleep: tuple = (6, 3),
+    proxy_interface: ProxyInterface = None,
+    use_auth_proxies: bool = False,
+    max_proxy_delay: int = 1800,
+    **kwargs,
+):
+    retry_type = retry_if_exception_type(ProxyError)
+    wait = wait_exponential(exp_base=3, multiplier=3, max=60)
+    stop = stop_after_delay(max_proxy_delay)
+    before_sleep = before_sleep_log(logger, logging.WARNING)
+
+    @retry(retry=retry_type, wait=wait, stop=stop, before_sleep=before_sleep, reraise=True)
+    def _fetch_with_proxy_retry(url, method, proxy_interface, use_auth, **params):
+        logger.info(f"Fetching data from {url} ...")
+        proxy_cfg = None
+        if proxy_interface:
+            host, port, user, pwd = proxy_interface.get_proxies(raw=True, use_auth=use_auth)
+            if host and port:
+                proxy_url = f"http://{host}:{port}"
+                proxy_auth_url = f"http://{user}:{pwd}@{host}:{port}"
+                proxy_cfg = {"http": proxy_url, "https": proxy_url}
+                if user and pwd:
+                    proxy_cfg = {"http": proxy_auth_url, "https": proxy_auth_url}
+                logger.info(f"Using proxy: {proxy_url}")
+        response = getattr(requests, method.lower())(url, proxies=proxy_cfg, **params)
+        return response
+
+    params = {"timeout": 30} | kwargs
+    r = _fetch_with_proxy_retry(url, method, proxy_interface, use_auth_proxies, **params)
+
+    ban_sleep(*sleep)
+
+    if r.status_code == 404:
+        raise NotFoundError(f"404 Not Found error for {url}")
+    r.raise_for_status()
+    r.encoding = "utf-8"
+
+    if output == "json":
+        return r.json()
+    elif output == "text":
+        return r.text
+    elif output == "soup":
+        return BeautifulSoup(r.content, "html.parser")
+    elif output == "response":
+        return r

{datamarket-0.9.40.dist-info → datamarket-0.9.42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.40
+Version: 0.9.42
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -48,6 +48,8 @@ Provides-Extra: proxy
 Provides-Extra: pyarrow
 Provides-Extra: pydrive2
 Provides-Extra: pymupdf
+Provides-Extra: pyproj
+Provides-Extra: pyrate-limiter
 Provides-Extra: pysocks
 Provides-Extra: pyspark
 Provides-Extra: pytest
@@ -105,6 +107,8 @@ Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "pyarrow"
 Requires-Dist: pycountry (>=24.0.0,<25.0.0)
 Requires-Dist: pydrive2 (>=1.0.0,<2.0.0) ; extra == "pydrive2" or extra == "drive"
 Requires-Dist: pymupdf (>=1.0.0,<2.0.0) ; extra == "pymupdf"
+Requires-Dist: pyproj (>=3.0.0,<4.0.0) ; extra == "pyproj"
+Requires-Dist: pyrate-limiter (>=3.0.0,<4.0.0) ; extra == "pyrate-limiter"
 Requires-Dist: pysocks (>=1.0.0,<2.0.0) ; extra == "pysocks"
 Requires-Dist: pyspark (>=3.0.0,<4.0.0) ; extra == "pyspark"
 Requires-Dist: pytest (>=8.0.0,<9.0.0) ; extra == "pytest"

{datamarket-0.9.40.dist-info → datamarket-0.9.42.dist-info}/RECORD

@@ -1,4 +1,6 @@
 datamarket/__init__.py,sha256=FHS77P9qNewKMoN-p0FLEUEC60oWIYup1QkbJZP4ays,12
+datamarket/exceptions/__init__.py,sha256=-Vu-RZNKjW6fYCLqbUJTkKNuHeA8Yi_gyR50oZNaA_8,33
+datamarket/exceptions/main.py,sha256=MP5ql6M7DoMbBf-Dg_2ohcUFdWXgzv-dXHntPPit31s,453
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/interfaces/alchemy.py,sha256=mQwjDqBpz1QHRV2JTCALvn5iK_ky69oE2Gw-EtRXsqQ,14664
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
@@ -10,10 +12,10 @@ datamarket/interfaces/proxy.py,sha256=updoOStKd8-nQBbxWbnD9eOt6HksnYi-5dQ0rEySf5
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/params/nominatim.py,sha256=XrCru3yEbs-X3ueOaCeSTBZwi4CWHW7oNhEyexBleMw,1184
-datamarket/utils/__init__.py,sha256=8D5a8oKgqd6WA1RUkiKCn4l_PVemtyuckxQut0vDHXM,20
+datamarket/utils/__init__.py,sha256=FHLh-Qp9XpM4LkAocppCf_llW2CWVVghGorkqxqt1wk,34
 datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,795
 datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
-datamarket/utils/main.py,sha256=j8wnAxeLvijdRU9M4V6HunWH7vgWWHP4u4xamzkWcUU,7009
+datamarket/utils/main.py,sha256=DMMgkQnMS6fNziTru8FM9z2ERfYfkdR9qFPF7s6sp7U,9657
 datamarket/utils/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/utils/playwright/async_api.py,sha256=pWfVj-ItfIeZBxG7WiUHKSeZDcHQFUQ5mrNGyIh1IdA,883
 datamarket/utils/playwright/sync_api.py,sha256=lIGm8mLxhFg04LVNdF8SO_9yjOLBnWe5pPry6ZFMnIg,846
@@ -24,7 +26,7 @@ datamarket/utils/strings/normalization.py,sha256=QLZ-THzjGOK9eWPPR1PrsffwQkSOx_M
 datamarket/utils/strings/obfuscation.py,sha256=8gMepfjPq0N4_IpKR6i2dy_9VJugQ3qJiRiRvKavB3s,5246
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.9.40.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.40.dist-info/METADATA,sha256=odG5B_7jOuXZFBNe260K0HP7vkDntMf8yAUOgSxNOws,6961
-datamarket-0.9.40.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.9.40.dist-info/RECORD,,
+datamarket-0.9.42.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.42.dist-info/METADATA,sha256=psplepRhULPjVtWuUwxCUEcK3mrS923oWPXPh8wPnlw,7149
+datamarket-0.9.42.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.9.42.dist-info/RECORD,,