datamarket 0.7.24.1__tar.gz → 0.7.112__tar.gz

datamarket-0.7.112/PKG-INFO

@@ -0,0 +1,168 @@
+Metadata-Version: 2.4
+Name: datamarket
+Version: 0.7.112
+Summary: Utilities that integrate advanced scraping knowledge into just one library.
+License: GPL-3.0-or-later
+License-File: LICENSE
+Author: DataMarket
+Author-email: techsupport@datamarket.es
+Requires-Python: >=3.12,<4.0
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Provides-Extra: aws
+Provides-Extra: azure-storage-blob
+Provides-Extra: boto3
+Provides-Extra: camoufox
+Provides-Extra: chompjs
+Provides-Extra: click
+Provides-Extra: clickhouse-driver
+Provides-Extra: datetime
+Provides-Extra: ddgs
+Provides-Extra: demjson3
+Provides-Extra: dnspython
+Provides-Extra: drive
+Provides-Extra: fake-useragent
+Provides-Extra: geoalchemy2
+Provides-Extra: geopandas
+Provides-Extra: google-api-python-client
+Provides-Extra: google-auth-httplib2
+Provides-Extra: google-auth-oauthlib
+Provides-Extra: html2text
+Provides-Extra: httpx
+Provides-Extra: json5
+Provides-Extra: lxml
+Provides-Extra: matplotlib
+Provides-Extra: nodriver
+Provides-Extra: openpyxl
+Provides-Extra: pandarallel
+Provides-Extra: pandas
+Provides-Extra: pandera
+Provides-Extra: peerdb
+Provides-Extra: pii
+Provides-Extra: pillow
+Provides-Extra: playwright
+Provides-Extra: playwright-stealth
+Provides-Extra: plotly
+Provides-Extra: pyarrow
+Provides-Extra: pydrive2
+Provides-Extra: pymupdf
+Provides-Extra: pyproj
+Provides-Extra: pyrate-limiter
+Provides-Extra: pysocks
+Provides-Extra: pyspark
+Provides-Extra: pytest
+Provides-Extra: retry
+Provides-Extra: rnet
+Provides-Extra: shapely
+Provides-Extra: soda-core-mysql
+Provides-Extra: soda-core-postgres
+Provides-Extra: sqlparse
+Provides-Extra: tqdm
+Provides-Extra: undetected-chromedriver
+Provides-Extra: xmltodict
+Requires-Dist: SQLAlchemy (>=2.0.0,<3.0.0)
+Requires-Dist: azure-storage-blob (>=12.0.0,<13.0.0) ; extra == "azure-storage-blob"
+Requires-Dist: babel (>=2.0.0,<3.0.0)
+Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
+Requires-Dist: boto3 (>=1.35.0,<1.36.0) ; extra == "boto3" or extra == "aws" or extra == "peerdb"
+Requires-Dist: browserforge (>=1.2.0,<2.0.0) ; extra == "camoufox"
+Requires-Dist: camoufox[geoip] (>=0.4.11,<0.5.0) ; extra == "camoufox"
+Requires-Dist: chompjs (>=1.0.0,<2.0.0) ; extra == "chompjs"
+Requires-Dist: click (>=8.0.0,<9.0.0) ; extra == "click"
+Requires-Dist: clickhouse-driver (>=0.2.0,<0.3.0) ; extra == "clickhouse-driver" or extra == "peerdb"
+Requires-Dist: croniter (>=3.0.0,<4.0.0)
+Requires-Dist: datetime (>=5.0,<6.0) ; extra == "datetime"
+Requires-Dist: ddgs (>=9.0.0,<10.0.0) ; extra == "ddgs"
+Requires-Dist: demjson3 (>=3.0.0,<4.0.0) ; extra == "demjson3"
+Requires-Dist: dnspython (>=2.0.0,<3.0.0) ; extra == "dnspython"
+Requires-Dist: dynaconf (>=3.0.0,<4.0.0)
+Requires-Dist: fake-useragent (>=2.0.0,<3.0.0) ; extra == "fake-useragent"
+Requires-Dist: geoalchemy2 (>=0.17.0,<0.18.0) ; extra == "geoalchemy2"
+Requires-Dist: geopandas (>=1.0.0,<2.0.0) ; extra == "geopandas"
+Requires-Dist: geopy (>=2.0.0,<3.0.0)
+Requires-Dist: google-api-python-client (>=2.0.0,<3.0.0) ; extra == "google-api-python-client"
+Requires-Dist: google-auth-httplib2 (>=0.2.0,<0.3.0) ; extra == "google-auth-httplib2"
+Requires-Dist: google-auth-oauthlib (>=1.0.0,<2.0.0) ; extra == "google-auth-oauthlib"
+Requires-Dist: html2text (>=2024.0.0,<2025.0.0) ; extra == "html2text"
+Requires-Dist: httpx[http2] (>=0.28.0,<0.29.0) ; extra == "httpx"
+Requires-Dist: inflection (>=0.5.0,<0.6.0)
+Requires-Dist: jellyfish (>=1.0.0,<2.0.0)
+Requires-Dist: jinja2 (>=3.0.0,<4.0.0)
+Requires-Dist: json5 (>=0.10.0,<0.11.0) ; extra == "json5"
+Requires-Dist: lxml[html-clean] (>=5.0.0,<6.0.0) ; extra == "lxml"
+Requires-Dist: matplotlib (>=3.0.0,<4.0.0) ; extra == "matplotlib"
+Requires-Dist: nodriver (>=0.44,<0.45) ; extra == "nodriver"
+Requires-Dist: numpy (>=2.0.0,<3.0.0)
+Requires-Dist: openpyxl (>=3.0.0,<4.0.0) ; extra == "openpyxl"
+Requires-Dist: pandarallel (>=1.0.0,<2.0.0) ; extra == "pandarallel"
+Requires-Dist: pandas (>=2.0.0,<3.0.0) ; extra == "pandas"
+Requires-Dist: pandera (>=0.22.0,<0.23.0) ; extra == "pandera"
+Requires-Dist: pendulum (>=3.0.0,<4.0.0)
+Requires-Dist: pillow (>=11.0.0,<12.0.0) ; extra == "pillow"
+Requires-Dist: playwright (==1.47.0) ; extra == "playwright" or extra == "camoufox"
+Requires-Dist: plotly (>=6.0.0,<7.0.0) ; extra == "plotly"
+Requires-Dist: pre-commit (>=4.0.0,<5.0.0)
+Requires-Dist: presidio-analyzer[phonenumbers] (>=2.0.0,<3.0.0) ; extra == "pii"
+Requires-Dist: presidio-anonymizer (>=2.0.0,<3.0.0) ; extra == "pii"
+Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0)
+Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "pyarrow"
+Requires-Dist: pycountry (>=24.0.0,<25.0.0)
+Requires-Dist: pydrive2 (>=1.0.0,<2.0.0) ; extra == "pydrive2" or extra == "drive"
+Requires-Dist: pymupdf (>=1.0.0,<2.0.0) ; extra == "pymupdf"
+Requires-Dist: pyproj (>=3.0.0,<4.0.0) ; extra == "pyproj"
+Requires-Dist: pyrate-limiter (>=3.0.0,<4.0.0) ; extra == "pyrate-limiter"
+Requires-Dist: pysocks (>=1.0.0,<2.0.0) ; extra == "pysocks"
+Requires-Dist: pyspark (>=3.0.0,<4.0.0) ; extra == "pyspark"
+Requires-Dist: pytest (>=8.0.0,<9.0.0) ; extra == "pytest"
+Requires-Dist: python-string-utils (>=1.0.0,<2.0.0)
+Requires-Dist: rapidfuzz (>=3.0.0,<4.0.0)
+Requires-Dist: requests (>=2.0.0,<3.0.0)
+Requires-Dist: retry (>=0.9.0,<0.10.0) ; extra == "retry"
+Requires-Dist: rnet (>=2.0.0,<3.0.0) ; extra == "rnet"
+Requires-Dist: shapely (>=2.0.0,<3.0.0) ; extra == "shapely"
+Requires-Dist: soda-core-mysql-utf8-hotfix (>=3.0.0,<4.0.0) ; extra == "soda-core-mysql"
+Requires-Dist: soda-core-postgres (>=3.0.0,<4.0.0) ; extra == "soda-core-postgres"
+Requires-Dist: spacy (>=3.0.0,<4.0.0) ; extra == "pii"
+Requires-Dist: spacy-langdetect (>=0.1.0,<0.2.0) ; extra == "pii"
+Requires-Dist: sqlparse (>=0.5.0,<0.6.0) ; extra == "sqlparse"
+Requires-Dist: stem (>=1.0.0,<2.0.0)
+Requires-Dist: tenacity (>=9.0.0,<10.0.0)
+Requires-Dist: tqdm (>=4.0.0,<5.0.0) ; extra == "tqdm"
+Requires-Dist: typer (>=0.15.0,<0.16.0)
+Requires-Dist: unidecode (>=1.0.0,<2.0.0)
+Requires-Dist: xmltodict (>=0.14.0,<0.15.0) ; extra == "xmltodict"
+Project-URL: Documentation, https://github.com/Data-Market/datamarket
+Project-URL: Homepage, https://datamarket.es
+Project-URL: Repository, https://github.com/Data-Market/datamarket
+Description-Content-Type: text/markdown
+
+# DataMarket scraping core
+
+------------------------------------------------------
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+
+
+Utilities that integrate advance scraping knowledge into just one library.
+
+## Installation
+
+To install this library in your Python environment:
+
+`pip install datamarket`
+
+## Documentation
+
+This library has built functionalities for the following topics:
+
+- **Databases**: through sqlalchemy it allows to insert records and perform queries in any database.
+- **Proxies**: wide range of functions to perform HTTP requests through custom proxies or the Tor network.
+- **Tinybird**: a Python client for this popular API.
+- **Drive**: functions to upload, delete or authenticate to Google Drive.
+- **FTP**: functions to upload, delete or authenticate to an FTP, SFTP or FTPS server.
+- **Selenium**: wrapper for the main Selenium functions.
+

datamarket-0.7.112/pyproject.toml

@@ -0,0 +1,155 @@
+[tool.poetry]
+name = "datamarket"
+version = "0.7.112"
+description = "Utilities that integrate advanced scraping knowledge into just one library."
+authors = ["DataMarket <techsupport@datamarket.es>"]
+license = "GPL-3.0-or-later"
+readme = "README.md"
+homepage = "https://datamarket.es"
+repository = "https://github.com/Data-Market/datamarket"
+documentation = "https://github.com/Data-Market/datamarket"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Operating System :: OS Independent",
+]
+
+[tool.poetry.dependencies]
+python = "^3.12"
+typer = "~0.15.0"
+SQLAlchemy = "^2.0.0"
+psycopg2-binary = "^2.0.0"
+requests = "^2.0.0"
+tenacity = "^9.0.0"
+beautifulsoup4 = "^4.0.0"
+pre-commit = "^4.0.0"
+pendulum = "^3.0.0"
+croniter = "^3.0.0"
+dynaconf = "^3.0.0"
+jinja2 = "^3.0.0"
+inflection = "~0.5.0"
+python-string-utils = "^1.0.0"
+unidecode = "^1.0.0"
+numpy = "^2.0.0"
+pycountry = "^24.0.0"
+geopy = "^2.0.0"
+jellyfish = "^1.0.0"
+stem = "^1.0.0"
+babel = "^2.0.0"
+rapidfuzz = "^3.0.0"
+
+boto3 = { version = "~1.35.0", optional = true }
+lxml = { extras = ["html-clean"], version = "^5.0.0", optional = true }
+tqdm = { version = "^4.0.0", optional = true }
+pandas = { version = "^2.0.0", optional = true }
+pyarrow = { version = "^19.0.0", optional = true }
+pytest = { version = "^8.0.0", optional = true }
+playwright = { version = "1.47.0", optional = true }
+tf-playwright-stealth = { version = "^1.0.0", optional = true }
+soda-core-postgres = { version = "^3.0.0", optional = true }
+soda-core-mysql-utf8-hotfix = { version = "^3.0.0", optional = true }
+fake-useragent = { version = "^2.0.0", optional = true }
+pydrive2 = { version = "^1.0.0", optional = true }
+clickhouse-driver = { version = "~0.2.0", optional = true }
+click = { version = "^8.0.0", optional = true }
+demjson3 = { version = "^3.0.0", optional = true }
+nodriver = { version = "~0.44", optional = true }
+retry = { version = "~0.9.0", optional = true }
+shapely = { version = "^2.0.0", optional = true }
+geopandas = { version = "^1.0.0", optional = true }
+chompjs = { version = "^1.0.0", optional = true }
+pillow = { version = "^11.0.0", optional = true }
+ddgs = { version = "^9.0.0", optional = true }
+pysocks = { version = "^1.0.0", optional = true }
+xmltodict = { version = "~0.14.0", optional = true }
+pymupdf = { version = "^1.0.0", optional = true }
+html2text = { version = "^2024.0.0", optional = true }
+pyspark = { version = "^3.0.0", optional = true }
+pandera = { version = "~0.22.0", optional = true }
+json5 = { version = "~0.10.0", optional = true }
+geoalchemy2 = { version = "~0.17.0", optional = true }
+datetime = { version = "^5.0", optional = true }
+azure-storage-blob = { version = "^12.0.0", optional = true }
+google-api-python-client = { version = "^2.0.0", optional = true }
+google-auth-httplib2 = { version = "~0.2.0", optional = true }
+google-auth-oauthlib = { version = "^1.0.0", optional = true }
+dnspython = { version = "^2.0.0", optional = true }
+openpyxl = { version = "^3.0.0", optional = true }
+httpx = { extras = ["http2"], version = "~0.28.0", optional = true }
+camoufox = { extras = ["geoip"], version = "~0.4.11", optional = true }
+browserforge = { version = "^1.2.0", optional = true }
+presidio-analyzer = { version = "^2.0.0", optional = true, extras = [
+    "phonenumbers",
+] }
+presidio-anonymizer = { version = "^2.0.0", optional = true }
+spacy = { version = "^3.0.0", optional = true }
+spacy-langdetect = { version = "~0.1.0", optional = true }
+pandarallel = { version = "^1.0.0", optional = true }
+pyrate-limiter = { version = "^3.0.0", optional = true }
+pyproj = { version = "^3.0.0", optional = true }
+sqlparse = { version = "~0.5.0", optional = true }
+rnet = { version = "^2.0.0", optional = true }
+matplotlib = { version = "^3.0.0", optional = true }
+plotly = { version = "^6.0.0", optional = true }
+
+[tool.poetry.extras]
+boto3 = ["boto3"]
+lxml = ["lxml"]
+tqdm = ["tqdm"]
+pandas = ["pandas"]
+pyarrow = ["pyarrow"]
+pytest = ["pytest"]
+playwright = ["playwright"]
+playwright-stealth = ["playwright-stealth"]
+soda-core-postgres = ["soda-core-postgres"]
+soda-core-mysql = ["soda-core-mysql-utf8-hotfix"]
+fake-useragent = ["fake-useragent"]
+pydrive2 = ["pydrive2"]
+clickhouse-driver = ["clickhouse-driver"]
+click = ["click"]
+demjson3 = ["demjson3"]
+nodriver = ["nodriver"]
+undetected-chromedriver = ["undetected-chromedriver"]
+retry = ["retry"]
+shapely = ["shapely"]
+geopandas = ["geopandas"]
+chompjs = ["chompjs"]
+pillow = ["pillow"]
+ddgs = ["ddgs"]
+pysocks = ["pysocks"]
+xmltodict = ["xmltodict"]
+pymupdf = ["pymupdf"]
+html2text = ["html2text"]
+pyspark = ["pyspark"]
+pandera = ["pandera"]
+json5 = ["json5"]
+geoalchemy2 = ["geoalchemy2"]
+datetime = ["datetime"]
+azure-storage-blob = ["azure-storage-blob"]
+google-api-python-client = ["google-api-python-client"]
+google-auth-httplib2 = ["google-auth-httplib2"]
+google-auth-oauthlib = ["google-auth-oauthlib"]
+dnspython = ["dnspython"]
+openpyxl = ["openpyxl"]
+httpx = ["httpx"]
+camoufox = ["camoufox", "browserforge", "playwright"]
+pandarallel = ["pandarallel"]
+pyrate-limiter = ["pyrate-limiter"]
+pyproj = ["pyproj"]
+sqlparse = ["sqlparse"]
+rnet = ["rnet"]
+matplotlib = ["matplotlib"]
+plotly = ["plotly"]
+
+# Interface groups
+aws = ["boto3"]
+drive = ["pydrive2"]
+peerdb = ["boto3", "clickhouse-driver"]
+
+# Other groups
+pii = ["presidio-analyzer", "presidio-anonymizer", "spacy", "spacy-langdetect"]
+
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

datamarket-0.7.112/src/datamarket/exceptions/__init__.py

@@ -0,0 +1 @@
+from .main import *  # noqa: F403

datamarket-0.7.112/src/datamarket/exceptions/main.py

@@ -0,0 +1,53 @@
+########################################################################################################################
+# CLASSES
+
+
+import requests
+
+
+class RedirectionDetectedError(Exception):
+    def __init__(self, message="Redirection detected!"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class NotFoundError(Exception):
+    def __init__(self, message="Not found!"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class BadRequestError(Exception):
+    def __init__(self, message="Bad request!"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class EmptyResponseError(Exception):
+    def __init__(self, message="Empty response!"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class ManagedHTTPError(Exception):
+    """Signal that this HTTP status was handled and should not be retried."""
+
+    def __init__(self, response: requests.Response, *, url: str | None = None, message: str | None = None):
+        self.response = response
+        self.request = getattr(response, "request", None)
+        self.status_code = getattr(response, "status_code", None)
+        self.url = url or (self.request.url if self.request is not None else None)
+        self.message = message
+        super().__init__(message or f"HTTP {self.status_code} for {self.url}")
+
+
+class NoWorkingProxiesError(Exception):
+    def __init__(self, message="No working proxies available"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class EnsureNewIPTimeoutError(Exception):
+    def __init__(self, message="Timed out waiting for new IP"):
+        self.message = message
+        super().__init__(self.message)