datamarket 0.7.37__tar.gz → 0.7.112__tar.gz

{datamarket-0.7.37 → datamarket-0.7.112}/PKG-INFO

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: datamarket
-Version: 0.7.37
+Version: 0.7.112
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
+License-File: LICENSE
 Author: DataMarket
 Author-email: techsupport@datamarket.es
 Requires-Python: >=3.12,<4.0
@@ -12,22 +13,22 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: aws
 Provides-Extra: azure-storage-blob
 Provides-Extra: boto3
+Provides-Extra: camoufox
 Provides-Extra: chompjs
 Provides-Extra: click
 Provides-Extra: clickhouse-driver
-Provides-Extra: croniter
 Provides-Extra: datetime
+Provides-Extra: ddgs
 Provides-Extra: demjson3
 Provides-Extra: dnspython
 Provides-Extra: drive
-Provides-Extra: duckduckgo-search
 Provides-Extra: fake-useragent
 Provides-Extra: geoalchemy2
 Provides-Extra: geopandas
-Provides-Extra: geopy
 Provides-Extra: google-api-python-client
 Provides-Extra: google-auth-httplib2
 Provides-Extra: google-auth-oauthlib
@@ -35,83 +36,105 @@ Provides-Extra: html2text
 Provides-Extra: httpx
 Provides-Extra: json5
 Provides-Extra: lxml
+Provides-Extra: matplotlib
 Provides-Extra: nodriver
 Provides-Extra: openpyxl
+Provides-Extra: pandarallel
 Provides-Extra: pandas
 Provides-Extra: pandera
 Provides-Extra: peerdb
-Provides-Extra: pendulum
+Provides-Extra: pii
 Provides-Extra: pillow
 Provides-Extra: playwright
 Provides-Extra: playwright-stealth
-Provides-Extra: proxy
+Provides-Extra: plotly
 Provides-Extra: pyarrow
 Provides-Extra: pydrive2
 Provides-Extra: pymupdf
+Provides-Extra: pyproj
+Provides-Extra: pyrate-limiter
 Provides-Extra: pysocks
 Provides-Extra: pyspark
 Provides-Extra: pytest
-Provides-Extra: rapidfuzz
 Provides-Extra: retry
+Provides-Extra: rnet
 Provides-Extra: shapely
 Provides-Extra: soda-core-mysql
 Provides-Extra: soda-core-postgres
-Provides-Extra: stem
+Provides-Extra: sqlparse
 Provides-Extra: tqdm
 Provides-Extra: undetected-chromedriver
-Provides-Extra: unidecode
 Provides-Extra: xmltodict
 Requires-Dist: SQLAlchemy (>=2.0.0,<3.0.0)
 Requires-Dist: azure-storage-blob (>=12.0.0,<13.0.0) ; extra == "azure-storage-blob"
+Requires-Dist: babel (>=2.0.0,<3.0.0)
 Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
 Requires-Dist: boto3 (>=1.35.0,<1.36.0) ; extra == "boto3" or extra == "aws" or extra == "peerdb"
+Requires-Dist: browserforge (>=1.2.0,<2.0.0) ; extra == "camoufox"
+Requires-Dist: camoufox[geoip] (>=0.4.11,<0.5.0) ; extra == "camoufox"
 Requires-Dist: chompjs (>=1.0.0,<2.0.0) ; extra == "chompjs"
 Requires-Dist: click (>=8.0.0,<9.0.0) ; extra == "click"
 Requires-Dist: clickhouse-driver (>=0.2.0,<0.3.0) ; extra == "clickhouse-driver" or extra == "peerdb"
-Requires-Dist: croniter (>=3.0.0,<4.0.0) ; extra == "croniter"
+Requires-Dist: croniter (>=3.0.0,<4.0.0)
 Requires-Dist: datetime (>=5.0,<6.0) ; extra == "datetime"
+Requires-Dist: ddgs (>=9.0.0,<10.0.0) ; extra == "ddgs"
 Requires-Dist: demjson3 (>=3.0.0,<4.0.0) ; extra == "demjson3"
 Requires-Dist: dnspython (>=2.0.0,<3.0.0) ; extra == "dnspython"
-Requires-Dist: duckduckgo-search (>=7.0.0,<8.0.0) ; extra == "duckduckgo-search"
 Requires-Dist: dynaconf (>=3.0.0,<4.0.0)
 Requires-Dist: fake-useragent (>=2.0.0,<3.0.0) ; extra == "fake-useragent"
 Requires-Dist: geoalchemy2 (>=0.17.0,<0.18.0) ; extra == "geoalchemy2"
 Requires-Dist: geopandas (>=1.0.0,<2.0.0) ; extra == "geopandas"
-Requires-Dist: geopy (>=2.0.0,<3.0.0) ; extra == "geopy"
+Requires-Dist: geopy (>=2.0.0,<3.0.0)
 Requires-Dist: google-api-python-client (>=2.0.0,<3.0.0) ; extra == "google-api-python-client"
 Requires-Dist: google-auth-httplib2 (>=0.2.0,<0.3.0) ; extra == "google-auth-httplib2"
 Requires-Dist: google-auth-oauthlib (>=1.0.0,<2.0.0) ; extra == "google-auth-oauthlib"
 Requires-Dist: html2text (>=2024.0.0,<2025.0.0) ; extra == "html2text"
 Requires-Dist: httpx[http2] (>=0.28.0,<0.29.0) ; extra == "httpx"
+Requires-Dist: inflection (>=0.5.0,<0.6.0)
+Requires-Dist: jellyfish (>=1.0.0,<2.0.0)
 Requires-Dist: jinja2 (>=3.0.0,<4.0.0)
 Requires-Dist: json5 (>=0.10.0,<0.11.0) ; extra == "json5"
 Requires-Dist: lxml[html-clean] (>=5.0.0,<6.0.0) ; extra == "lxml"
-Requires-Dist: nodriver (==0.38.post1) ; extra == "nodriver"
+Requires-Dist: matplotlib (>=3.0.0,<4.0.0) ; extra == "matplotlib"
+Requires-Dist: nodriver (>=0.44,<0.45) ; extra == "nodriver"
+Requires-Dist: numpy (>=2.0.0,<3.0.0)
 Requires-Dist: openpyxl (>=3.0.0,<4.0.0) ; extra == "openpyxl"
+Requires-Dist: pandarallel (>=1.0.0,<2.0.0) ; extra == "pandarallel"
 Requires-Dist: pandas (>=2.0.0,<3.0.0) ; extra == "pandas"
 Requires-Dist: pandera (>=0.22.0,<0.23.0) ; extra == "pandera"
-Requires-Dist: pendulum (>=3.0.0,<4.0.0) ; extra == "pendulum"
+Requires-Dist: pendulum (>=3.0.0,<4.0.0)
 Requires-Dist: pillow (>=11.0.0,<12.0.0) ; extra == "pillow"
-Requires-Dist: playwright (==1.47.0) ; extra == "playwright"
+Requires-Dist: playwright (==1.47.0) ; extra == "playwright" or extra == "camoufox"
+Requires-Dist: plotly (>=6.0.0,<7.0.0) ; extra == "plotly"
 Requires-Dist: pre-commit (>=4.0.0,<5.0.0)
+Requires-Dist: presidio-analyzer[phonenumbers] (>=2.0.0,<3.0.0) ; extra == "pii"
+Requires-Dist: presidio-anonymizer (>=2.0.0,<3.0.0) ; extra == "pii"
 Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0)
 Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "pyarrow"
+Requires-Dist: pycountry (>=24.0.0,<25.0.0)
 Requires-Dist: pydrive2 (>=1.0.0,<2.0.0) ; extra == "pydrive2" or extra == "drive"
 Requires-Dist: pymupdf (>=1.0.0,<2.0.0) ; extra == "pymupdf"
+Requires-Dist: pyproj (>=3.0.0,<4.0.0) ; extra == "pyproj"
+Requires-Dist: pyrate-limiter (>=3.0.0,<4.0.0) ; extra == "pyrate-limiter"
 Requires-Dist: pysocks (>=1.0.0,<2.0.0) ; extra == "pysocks"
 Requires-Dist: pyspark (>=3.0.0,<4.0.0) ; extra == "pyspark"
 Requires-Dist: pytest (>=8.0.0,<9.0.0) ; extra == "pytest"
-Requires-Dist: rapidfuzz (>=3.0.0,<4.0.0) ; extra == "rapidfuzz"
+Requires-Dist: python-string-utils (>=1.0.0,<2.0.0)
+Requires-Dist: rapidfuzz (>=3.0.0,<4.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
 Requires-Dist: retry (>=0.9.0,<0.10.0) ; extra == "retry"
+Requires-Dist: rnet (>=2.0.0,<3.0.0) ; extra == "rnet"
 Requires-Dist: shapely (>=2.0.0,<3.0.0) ; extra == "shapely"
-Requires-Dist: soda-core-mysql (>=3.0.0,<4.0.0) ; extra == "soda-core-mysql"
+Requires-Dist: soda-core-mysql-utf8-hotfix (>=3.0.0,<4.0.0) ; extra == "soda-core-mysql"
 Requires-Dist: soda-core-postgres (>=3.0.0,<4.0.0) ; extra == "soda-core-postgres"
-Requires-Dist: stem (>=1.0.0,<2.0.0) ; extra == "stem" or extra == "proxy"
+Requires-Dist: spacy (>=3.0.0,<4.0.0) ; extra == "pii"
+Requires-Dist: spacy-langdetect (>=0.1.0,<0.2.0) ; extra == "pii"
+Requires-Dist: sqlparse (>=0.5.0,<0.6.0) ; extra == "sqlparse"
+Requires-Dist: stem (>=1.0.0,<2.0.0)
 Requires-Dist: tenacity (>=9.0.0,<10.0.0)
 Requires-Dist: tqdm (>=4.0.0,<5.0.0) ; extra == "tqdm"
 Requires-Dist: typer (>=0.15.0,<0.16.0)
-Requires-Dist: unidecode (>=1.0.0,<2.0.0) ; extra == "unidecode"
+Requires-Dist: unidecode (>=1.0.0,<2.0.0)
 Requires-Dist: xmltodict (>=0.14.0,<0.15.0) ; extra == "xmltodict"
 Project-URL: Documentation, https://github.com/Data-Market/datamarket
 Project-URL: Homepage, https://datamarket.es

{datamarket-0.7.37 → datamarket-0.7.112}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.7.37"
+version = "0.7.112"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"
@@ -27,9 +27,18 @@ pendulum = "^3.0.0"
 croniter = "^3.0.0"
 dynaconf = "^3.0.0"
 jinja2 = "^3.0.0"
+inflection = "~0.5.0"
+python-string-utils = "^1.0.0"
+unidecode = "^1.0.0"
+numpy = "^2.0.0"
+pycountry = "^24.0.0"
+geopy = "^2.0.0"
+jellyfish = "^1.0.0"
+stem = "^1.0.0"
+babel = "^2.0.0"
+rapidfuzz = "^3.0.0"
 
 boto3 = { version = "~1.35.0", optional = true }
-unidecode = { version = "^1.0.0", optional = true }
 lxml = { extras = ["html-clean"], version = "^5.0.0", optional = true }
 tqdm = { version = "^4.0.0", optional = true }
 pandas = { version = "^2.0.0", optional = true }
@@ -38,22 +47,19 @@ pytest = { version = "^8.0.0", optional = true }
 playwright = { version = "1.47.0", optional = true }
 tf-playwright-stealth = { version = "^1.0.0", optional = true }
 soda-core-postgres = { version = "^3.0.0", optional = true }
-soda-core-mysql = { version = "^3.0.0", optional = true }
+soda-core-mysql-utf8-hotfix = { version = "^3.0.0", optional = true }
 fake-useragent = { version = "^2.0.0", optional = true }
 pydrive2 = { version = "^1.0.0", optional = true }
 clickhouse-driver = { version = "~0.2.0", optional = true }
-stem = { version = "^1.0.0", optional = true }
 click = { version = "^8.0.0", optional = true }
-rapidfuzz = { version = "^3.0.0", optional = true }
 demjson3 = { version = "^3.0.0", optional = true }
-geopy = { version = "^2.0.0", optional = true }
-nodriver = { version = "0.38.post1", optional = true }
+nodriver = { version = "~0.44", optional = true }
 retry = { version = "~0.9.0", optional = true }
 shapely = { version = "^2.0.0", optional = true }
 geopandas = { version = "^1.0.0", optional = true }
 chompjs = { version = "^1.0.0", optional = true }
 pillow = { version = "^11.0.0", optional = true }
-duckduckgo-search = { version = "^7.0.0", optional = true }
+ddgs = { version = "^9.0.0", optional = true }
 pysocks = { version = "^1.0.0", optional = true }
 xmltodict = { version = "~0.14.0", optional = true }
 pymupdf = { version = "^1.0.0", optional = true }
@@ -70,29 +76,38 @@ google-auth-oauthlib = { version = "^1.0.0", optional = true }
 dnspython = { version = "^2.0.0", optional = true }
 openpyxl = { version = "^3.0.0", optional = true }
 httpx = { extras = ["http2"], version = "~0.28.0", optional = true }
+camoufox = { extras = ["geoip"], version = "~0.4.11", optional = true }
+browserforge = { version = "^1.2.0", optional = true }
+presidio-analyzer = { version = "^2.0.0", optional = true, extras = [
+    "phonenumbers",
+] }
+presidio-anonymizer = { version = "^2.0.0", optional = true }
+spacy = { version = "^3.0.0", optional = true }
+spacy-langdetect = { version = "~0.1.0", optional = true }
+pandarallel = { version = "^1.0.0", optional = true }
+pyrate-limiter = { version = "^3.0.0", optional = true }
+pyproj = { version = "^3.0.0", optional = true }
+sqlparse = { version = "~0.5.0", optional = true }
+rnet = { version = "^2.0.0", optional = true }
+matplotlib = { version = "^3.0.0", optional = true }
+plotly = { version = "^6.0.0", optional = true }
 
 [tool.poetry.extras]
 boto3 = ["boto3"]
-unidecode = ["unidecode"]
 lxml = ["lxml"]
 tqdm = ["tqdm"]
-pendulum = ["pendulum"]
 pandas = ["pandas"]
 pyarrow = ["pyarrow"]
 pytest = ["pytest"]
 playwright = ["playwright"]
 playwright-stealth = ["playwright-stealth"]
 soda-core-postgres = ["soda-core-postgres"]
-soda-core-mysql = ["soda-core-mysql"]
+soda-core-mysql = ["soda-core-mysql-utf8-hotfix"]
 fake-useragent = ["fake-useragent"]
-croniter = ["croniter"]
 pydrive2 = ["pydrive2"]
 clickhouse-driver = ["clickhouse-driver"]
-stem = ["stem"]
 click = ["click"]
-rapidfuzz = ["rapidfuzz"]
 demjson3 = ["demjson3"]
-geopy = ["geopy"]
 nodriver = ["nodriver"]
 undetected-chromedriver = ["undetected-chromedriver"]
 retry = ["retry"]
@@ -100,7 +115,7 @@ shapely = ["shapely"]
 geopandas = ["geopandas"]
 chompjs = ["chompjs"]
 pillow = ["pillow"]
-duckduckgo-search = ["duckduckgo-search"]
+ddgs = ["ddgs"]
 pysocks = ["pysocks"]
 xmltodict = ["xmltodict"]
 pymupdf = ["pymupdf"]
@@ -117,12 +132,22 @@ google-auth-oauthlib = ["google-auth-oauthlib"]
 dnspython = ["dnspython"]
 openpyxl = ["openpyxl"]
 httpx = ["httpx"]
+camoufox = ["camoufox", "browserforge", "playwright"]
+pandarallel = ["pandarallel"]
+pyrate-limiter = ["pyrate-limiter"]
+pyproj = ["pyproj"]
+sqlparse = ["sqlparse"]
+rnet = ["rnet"]
+matplotlib = ["matplotlib"]
+plotly = ["plotly"]
 
 # Interface groups
 aws = ["boto3"]
 drive = ["pydrive2"]
 peerdb = ["boto3", "clickhouse-driver"]
-proxy = ["stem"]
+
+# Other groups
+pii = ["presidio-analyzer", "presidio-anonymizer", "spacy", "spacy-langdetect"]
 
 
 [build-system]

datamarket-0.7.112/src/datamarket/exceptions/__init__.py

@@ -0,0 +1 @@
+from .main import *  # noqa: F403

datamarket-0.7.112/src/datamarket/exceptions/main.py

@@ -0,0 +1,53 @@
+########################################################################################################################
+# CLASSES
+
+
+import requests
+
+
+class RedirectionDetectedError(Exception):
+    def __init__(self, message="Redirection detected!"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class NotFoundError(Exception):
+    def __init__(self, message="Not found!"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class BadRequestError(Exception):
+    def __init__(self, message="Bad request!"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class EmptyResponseError(Exception):
+    def __init__(self, message="Empty response!"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class ManagedHTTPError(Exception):
+    """Signal that this HTTP status was handled and should not be retried."""
+
+    def __init__(self, response: requests.Response, *, url: str | None = None, message: str | None = None):
+        self.response = response
+        self.request = getattr(response, "request", None)
+        self.status_code = getattr(response, "status_code", None)
+        self.url = url or (self.request.url if self.request is not None else None)
+        self.message = message
+        super().__init__(message or f"HTTP {self.status_code} for {self.url}")
+
+
+class NoWorkingProxiesError(Exception):
+    def __init__(self, message="No working proxies available"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class EnsureNewIPTimeoutError(Exception):
+    def __init__(self, message="Timed out waiting for new IP"):
+        self.message = message
+        super().__init__(self.message)