datamarket 0.7.100__py3-none-any.whl → 0.7.102__py3-none-any.whl

datamarket/exceptions/main.py

@@ -2,6 +2,9 @@
 # CLASSES
 
 
+import requests
+
+
 class RedirectionDetectedError(Exception):
     def __init__(self, message="Redirection detected!"):
         self.message = message
@@ -12,3 +15,19 @@ class NotFoundError(Exception):
     def __init__(self, message="Not found!"):
         self.message = message
         super().__init__(self.message)
+
+
+class BadRequestError(Exception):
+    def __init__(self, message="Bad request!"):
+        self.message = message
+        super().__init__(self.message)
+        
+class ManagedHTTPError(Exception):
+    """Signal that this HTTP status was handled and should not be retried."""
+    def __init__(self, response: requests.Response, *, url: str | None = None, message: str | None = None):
+        self.response = response
+        self.request = getattr(response, "request", None)
+        self.status_code = getattr(response, "status_code", None)
+        self.url = url or (self.request.url if self.request is not None else None)
+        self.message = message
+        super().__init__(message or f"HTTP {self.status_code} for {self.url}")

datamarket/utils/main.py

@@ -3,12 +3,14 @@
 
 import asyncio
 import configparser
+from datetime import timedelta
 import logging
 import random
 import re
 import shlex
 import subprocess
 import time
+from typing import Sequence
 from babel.numbers import parse_decimal
 
 from bs4 import BeautifulSoup
@@ -25,7 +27,9 @@ from tenacity import (
     wait_exponential,
 )
 
-from ..exceptions import RedirectionDetectedError, NotFoundError
+from datamarket.exceptions.main import ManagedHTTPError
+
+from ..exceptions import RedirectionDetectedError, NotFoundError, BadRequestError
 from ..interfaces.proxy import ProxyInterface
 
 ########################################################################################################################
@@ -131,7 +135,9 @@ def parse_field(dict_struct, field_path, format_method=None):
 
 
 @retry(
-    retry=retry_if_not_exception_type((NotFoundError, RedirectionDetectedError, ProxyError)),
+    retry=retry_if_not_exception_type(
+        (NotFoundError, BadRequestError, RedirectionDetectedError, ProxyError, ManagedHTTPError)
+    ),
     wait=wait_exponential(exp_base=3, multiplier=3, max=60),
     stop=stop_after_attempt(5),
     before_sleep=before_sleep_log(logger, logging.WARNING),
@@ -144,9 +150,38 @@ def get_data(
     sleep: tuple = (6, 3),
     proxy_interface: ProxyInterface = None,
     use_auth_proxies: bool = False,
-    max_proxy_delay: int = 1800,
+    max_proxy_delay: timedelta = timedelta(minutes=10),
+    ignored_status_codes: Sequence[int] = (),
     **kwargs,
 ):
+    """
+    Fetches data from a given URL using HTTP requests, with support for proxy configuration, retries, and flexible output formats.
+
+    Args:
+        url (str): The target URL to fetch data from.
+        method (str, optional): HTTP method to use (e.g., 'GET', 'POST'). Defaults to 'GET'.
+        output (str, optional): Output format ('json', 'text', 'soup', 'response'). Defaults to 'json'.
+        sleep (tuple, optional): Tuple specifying max and min sleep times (seconds) after request. Defaults to (6, 3).
+        proxy_interface (ProxyInterface, optional): Proxy provider. If None, no proxy is used. Defaults to None.
+        use_auth_proxies (bool, optional): Whether to use authenticated proxies. Defaults to False.
+        max_proxy_delay (timedelta, optional): Maximum delay for proxy retry logic. Defaults to 10 minutes.
+        ignored_status_codes (Sequence[int], optional): Status codes to ignore and return response for. Defaults to ().
+        **kwargs: Additional arguments passed to the requests method.
+
+    Returns:
+        Depends on the 'output' argument:
+            - 'json': Parsed JSON response.
+            - 'text': Response text.
+            - 'soup': BeautifulSoup-parsed HTML.
+            - 'response': Raw requests.Response object.
+
+    Raises:
+        ManagedHTTPError: If a response status code is in `ignored_status_codes`.
+        NotFoundError: If a 404 status code is returned.
+        BadRequestError: If a 400 status code is returned.
+        RedirectionDetectedError, ProxyError: On specific error conditions.
+        requests.HTTPError: For other HTTP errors if not ignored.
+    """
     retry_type = retry_if_exception_type(ProxyError)
     wait = wait_exponential(exp_base=3, multiplier=3, max=60)
     stop = stop_after_delay(max_proxy_delay)
@@ -173,9 +208,15 @@ def get_data(
 
     ban_sleep(*sleep)
 
+    if r.status_code in ignored_status_codes:
+        raise ManagedHTTPError(r, url=url, message=f"Status {r.status_code} in ignored_status_codes for URL {url}")
     if r.status_code == 404:
         raise NotFoundError(f"404 Not Found error for {url}")
+    if r.status_code == 400:
+        raise BadRequestError(f"400 Bad Request error for {url}")
+
     r.raise_for_status()
+
     r.encoding = "utf-8"
 
     if output == "json":

{datamarket-0.7.100.dist-info → datamarket-0.7.102.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.7.100
+Version: 0.7.102
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.7.100.dist-info → datamarket-0.7.102.dist-info}/RECORD

@@ -1,6 +1,6 @@
 datamarket/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/exceptions/__init__.py,sha256=-Vu-RZNKjW6fYCLqbUJTkKNuHeA8Yi_gyR50oZNaA_8,33
-datamarket/exceptions/main.py,sha256=MP5ql6M7DoMbBf-Dg_2ohcUFdWXgzv-dXHntPPit31s,453
+datamarket/exceptions/main.py,sha256=SuP-ZKZIxJYdnOpNb63Y7BpYGRhLl-4JIyTEqgUoWV4,1205
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/interfaces/alchemy.py,sha256=i2lKLLLy3-jpbzV3-jxfRCXTy7jRoTsNU3063pmSonk,15749
 datamarket/interfaces/aws.py,sha256=co5JkC3iFIp-0FqdYX4eKy3_m71LhZKuJoW6kXwEImc,4780
@@ -16,7 +16,7 @@ datamarket/params/nominatim.py,sha256=RnmYXGoJQCijOsuCavCYcxw98WvOd_vOMK4KaraI0R
 datamarket/utils/__init__.py,sha256=FHLh-Qp9XpM4LkAocppCf_llW2CWVVghGorkqxqt1wk,34
 datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,795
 datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
-datamarket/utils/main.py,sha256=KYHjDOps6_Q3TFV_Jj7MLj-L9Evx05AXELCvp06BARU,5857
+datamarket/utils/main.py,sha256=OORsHggUqa2lKj5AG5LTPzEvXfAtx3ry4rSaAwkuS38,8001
 datamarket/utils/nominatim.py,sha256=IxexKY2KOlDhiKtzsqQfoVUjJXPxJl7tn3iHUaQKg08,5795
 datamarket/utils/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/utils/playwright/async_api.py,sha256=UbA2D4ScBtYeMfrRjly4RO-s8wXIub9c05J1eoOCpsQ,5782
@@ -29,7 +29,7 @@ datamarket/utils/strings/obfuscation.py,sha256=Jo-x3f2Cb75983smmpcdPqUlBrLCTyrnm
 datamarket/utils/strings/standardization.py,sha256=c8CAG6HI3AfK0hB3A3IGwsbnQebZ6R3PrA5PELHRXM0,1492
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.7.100.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.7.100.dist-info/METADATA,sha256=ZzGfCV51bIyPYJVdCSfJDdX8YuC9_BjKR1VCoRtd6yI,7382
-datamarket-0.7.100.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.7.100.dist-info/RECORD,,
+datamarket-0.7.102.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.7.102.dist-info/METADATA,sha256=FQcOMGhkANO_QeIbb9ISfm_MwcRCEo3TqCeVV5PONnI,7382
+datamarket-0.7.102.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.7.102.dist-info/RECORD,,