datamarket 0.9.2__tar.gz → 0.9.4__tar.gz

datamarket-0.9.4/PKG-INFO

@@ -0,0 +1,144 @@
+Metadata-Version: 2.3
+Name: datamarket
+Version: 0.9.4
+Summary: Utilities that integrate advanced scraping knowledge into just one library.
+License: GPL-3.0-or-later
+Author: DataMarket
+Author-email: techsupport@datamarket.es
+Requires-Python: >=3.12,<4.0
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: alchemy
+Provides-Extra: aws
+Provides-Extra: azure-storage-blob
+Provides-Extra: boto3
+Provides-Extra: chompjs
+Provides-Extra: click
+Provides-Extra: clickhouse-driver
+Provides-Extra: datetime
+Provides-Extra: demjson3
+Provides-Extra: dnspython
+Provides-Extra: drive
+Provides-Extra: duckduckgo-search
+Provides-Extra: fake-useragent
+Provides-Extra: geoalchemy2
+Provides-Extra: geopandas
+Provides-Extra: geopy
+Provides-Extra: google-api-python-client
+Provides-Extra: google-auth-httplib2
+Provides-Extra: google-auth-oauthlib
+Provides-Extra: html2text
+Provides-Extra: httpx
+Provides-Extra: json5
+Provides-Extra: lxml
+Provides-Extra: nodriver
+Provides-Extra: openpyxl
+Provides-Extra: pandas
+Provides-Extra: pandera
+Provides-Extra: peerdb
+Provides-Extra: pillow
+Provides-Extra: playwright
+Provides-Extra: playwright-stealth
+Provides-Extra: proxy
+Provides-Extra: pyarrow
+Provides-Extra: pydrive2
+Provides-Extra: pymupdf
+Provides-Extra: pysocks
+Provides-Extra: pyspark
+Provides-Extra: pytest
+Provides-Extra: rapidfuzz
+Provides-Extra: retry
+Provides-Extra: shapely
+Provides-Extra: soda-core-mysql
+Provides-Extra: soda-core-postgres
+Provides-Extra: stem
+Provides-Extra: tqdm
+Provides-Extra: undetected-chromedriver
+Provides-Extra: unidecode
+Provides-Extra: xmltodict
+Requires-Dist: SQLAlchemy (>=2.0.0,<3.0.0) ; extra == "alchemy"
+Requires-Dist: azure-storage-blob (>=12.0.0,<13.0.0) ; extra == "azure-storage-blob"
+Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
+Requires-Dist: boto3 (>=1.0.0,<2.0.0) ; extra == "boto3" or extra == "aws" or extra == "peerdb"
+Requires-Dist: chompjs (>=1.0.0,<2.0.0) ; extra == "chompjs"
+Requires-Dist: click (>=8.0.0,<9.0.0) ; extra == "click"
+Requires-Dist: clickhouse-driver (>=0.2.0,<0.3.0) ; extra == "clickhouse-driver" or extra == "peerdb"
+Requires-Dist: croniter (>=3.0.0,<4.0.0)
+Requires-Dist: datetime (>=5.0,<6.0) ; extra == "datetime"
+Requires-Dist: demjson3 (>=3.0.0,<4.0.0) ; extra == "demjson3"
+Requires-Dist: dnspython (>=2.0.0,<3.0.0) ; extra == "dnspython"
+Requires-Dist: duckduckgo-search (>=7.0.0,<8.0.0) ; extra == "duckduckgo-search"
+Requires-Dist: dynaconf (>=3.0.0,<4.0.0)
+Requires-Dist: fake-useragent (>=2.0.0,<3.0.0) ; extra == "fake-useragent"
+Requires-Dist: geoalchemy2 (>=0.17.0,<0.18.0) ; extra == "geoalchemy2"
+Requires-Dist: geopandas (>=1.0.0,<2.0.0) ; extra == "geopandas"
+Requires-Dist: geopy (>=2.0.0,<3.0.0) ; extra == "geopy"
+Requires-Dist: google-api-python-client (>=2.0.0,<3.0.0) ; extra == "google-api-python-client"
+Requires-Dist: google-auth-httplib2 (>=0.2.0,<0.3.0) ; extra == "google-auth-httplib2"
+Requires-Dist: google-auth-oauthlib (>=1.0.0,<2.0.0) ; extra == "google-auth-oauthlib"
+Requires-Dist: html2text (>=2024.0.0,<2025.0.0) ; extra == "html2text"
+Requires-Dist: httpx[http2] (>=0.28.0,<0.29.0) ; extra == "httpx"
+Requires-Dist: jinja2 (>=3.0.0,<4.0.0)
+Requires-Dist: json5 (>=0.10.0,<0.11.0) ; extra == "json5"
+Requires-Dist: lxml[html-clean] (>=5.0.0,<6.0.0) ; extra == "lxml"
+Requires-Dist: nodriver (==0.38.post1) ; extra == "nodriver"
+Requires-Dist: openpyxl (>=3.0.0,<4.0.0) ; extra == "openpyxl"
+Requires-Dist: pandas (>=2.0.0,<3.0.0) ; extra == "pandas"
+Requires-Dist: pandera (>=0.22.0,<0.23.0) ; extra == "pandera"
+Requires-Dist: pendulum (>=3.0.0,<4.0.0)
+Requires-Dist: pillow (>=11.0.0,<12.0.0) ; extra == "pillow"
+Requires-Dist: playwright (==1.47.0) ; extra == "playwright"
+Requires-Dist: pre-commit (>=4.0.0,<5.0.0)
+Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0)
+Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "pyarrow"
+Requires-Dist: pydrive2 (>=1.0.0,<2.0.0) ; extra == "pydrive2" or extra == "drive"
+Requires-Dist: pymupdf (>=1.0.0,<2.0.0) ; extra == "pymupdf"
+Requires-Dist: pysocks (>=1.0.0,<2.0.0) ; extra == "pysocks"
+Requires-Dist: pyspark (>=3.0.0,<4.0.0) ; extra == "pyspark"
+Requires-Dist: pytest (>=8.0.0,<9.0.0) ; extra == "pytest"
+Requires-Dist: rapidfuzz (>=3.0.0,<4.0.0) ; extra == "rapidfuzz"
+Requires-Dist: requests (>=2.0.0,<3.0.0)
+Requires-Dist: retry (>=0.9.0,<0.10.0) ; extra == "retry"
+Requires-Dist: shapely (>=2.0.0,<3.0.0) ; extra == "shapely"
+Requires-Dist: soda-core-mysql (>=3.0.0,<4.0.0) ; extra == "soda-core-mysql"
+Requires-Dist: soda-core-postgres (>=3.0.0,<4.0.0) ; extra == "soda-core-postgres"
+Requires-Dist: stem (>=1.0.0,<2.0.0) ; extra == "stem" or extra == "proxy"
+Requires-Dist: tenacity (>=9.0.0,<10.0.0)
+Requires-Dist: tqdm (>=4.0.0,<5.0.0) ; extra == "tqdm"
+Requires-Dist: typer (>=0.15.0,<0.16.0)
+Requires-Dist: unidecode (>=1.0.0,<2.0.0) ; extra == "unidecode"
+Requires-Dist: xmltodict (>=0.14.0,<0.15.0) ; extra == "xmltodict"
+Project-URL: Documentation, https://github.com/Data-Market/datamarket
+Project-URL: Homepage, https://datamarket.es
+Project-URL: Repository, https://github.com/Data-Market/datamarket
+Description-Content-Type: text/markdown
+
+# DataMarket scraping core
+
+------------------------------------------------------
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+
+
+Utilities that integrate advance scraping knowledge into just one library.
+
+## Installation
+
+To install this library in your Python environment:
+
+`pip install datamarket`
+
+## Documentation
+
+This library has built functionalities for the following topics:
+
+- **Databases**: through sqlalchemy it allows to insert records and perform queries in any database.
+- **Proxies**: wide range of functions to perform HTTP requests through custom proxies or the Tor network.
+- **Tinybird**: a Python client for this popular API.
+- **Drive**: functions to upload, delete or authenticate to Google Drive.
+- **FTP**: functions to upload, delete or authenticate to an FTP, SFTP or FTPS server.
+- **Selenium**: wrapper for the main Selenium functions.
+

datamarket-0.9.4/pyproject.toml

@@ -0,0 +1,129 @@
+[tool.poetry]
+name = "datamarket"
+version = "0.9.4"
+description = "Utilities that integrate advanced scraping knowledge into just one library."
+authors = ["DataMarket <techsupport@datamarket.es>"]
+license = "GPL-3.0-or-later"
+readme = "README.md"
+homepage = "https://datamarket.es"
+repository = "https://github.com/Data-Market/datamarket"
+documentation = "https://github.com/Data-Market/datamarket"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Operating System :: OS Independent",
+]
+
+[tool.poetry.dependencies]
+python = "^3.12"
+typer = "~0.15.0"
+psycopg2-binary = "^2.0.0"
+requests = "^2.0.0"
+tenacity = "^9.0.0"
+beautifulsoup4 = "^4.0.0"
+pre-commit = "^4.0.0"
+pendulum = "^3.0.0"
+croniter = "^3.0.0"
+dynaconf = "^3.0.0"
+jinja2 = "^3.0.0"
+
+boto3 = { version = "^1.0.0", optional = true }
+unidecode = { version = "^1.0.0", optional = true }
+lxml = { extras = ["html-clean"], version = "^5.0.0", optional = true }
+tqdm = { version = "^4.0.0", optional = true }
+pandas = { version = "^2.0.0", optional = true }
+pyarrow = { version = "^19.0.0", optional = true }
+pytest = { version = "^8.0.0", optional = true }
+playwright = { version = "1.47.0", optional = true }
+tf-playwright-stealth = { version = "^1.0.0", optional = true }
+soda-core-postgres = { version = "^3.0.0", optional = true }
+soda-core-mysql = { version = "^3.0.0", optional = true }
+fake-useragent = { version = "^2.0.0", optional = true }
+pydrive2 = { version = "^1.0.0", optional = true }
+clickhouse-driver = { version = "~0.2.0", optional = true }
+stem = { version = "^1.0.0", optional = true }
+click = { version = "^8.0.0", optional = true }
+rapidfuzz = { version = "^3.0.0", optional = true }
+demjson3 = { version = "^3.0.0", optional = true }
+geopy = { version = "^2.0.0", optional = true }
+nodriver = { version = "0.38.post1", optional = true }
+retry = { version = "~0.9.0", optional = true }
+shapely = { version = "^2.0.0", optional = true }
+geopandas = { version = "^1.0.0", optional = true }
+chompjs = { version = "^1.0.0", optional = true }
+pillow = { version = "^11.0.0", optional = true }
+duckduckgo-search = { version = "^7.0.0", optional = true }
+pysocks = { version = "^1.0.0", optional = true }
+xmltodict = { version = "~0.14.0", optional = true }
+pymupdf = { version = "^1.0.0", optional = true }
+html2text = { version = "^2024.0.0", optional = true }
+pyspark = { version = "^3.0.0", optional = true }
+pandera = { version = "~0.22.0", optional = true }
+json5 = { version = "~0.10.0", optional = true }
+geoalchemy2 = { version = "~0.17.0", optional = true }
+datetime = { version = "^5.0", optional = true }
+azure-storage-blob = { version = "^12.0.0", optional = true }
+google-api-python-client = { version = "^2.0.0", optional = true }
+google-auth-httplib2 = { version = "~0.2.0", optional = true }
+google-auth-oauthlib = { version = "^1.0.0", optional = true }
+dnspython = { version = "^2.0.0", optional = true }
+openpyxl = { version = "^3.0.0", optional = true }
+httpx = { extras = ["http2"], version = "~0.28.0", optional = true }
+SQLAlchemy = { version = "^2.0.0", optional = true }
+
+[tool.poetry.extras]
+boto3 = ["boto3"]
+unidecode = ["unidecode"]
+lxml = ["lxml"]
+tqdm = ["tqdm"]
+pandas = ["pandas"]
+pyarrow = ["pyarrow"]
+pytest = ["pytest"]
+playwright = ["playwright"]
+playwright-stealth = ["playwright-stealth"]
+soda-core-postgres = ["soda-core-postgres"]
+soda-core-mysql = ["soda-core-mysql"]
+fake-useragent = ["fake-useragent"]
+pydrive2 = ["pydrive2"]
+clickhouse-driver = ["clickhouse-driver"]
+stem = ["stem"]
+click = ["click"]
+rapidfuzz = ["rapidfuzz"]
+demjson3 = ["demjson3"]
+geopy = ["geopy"]
+nodriver = ["nodriver"]
+undetected-chromedriver = ["undetected-chromedriver"]
+retry = ["retry"]
+shapely = ["shapely"]
+geopandas = ["geopandas"]
+chompjs = ["chompjs"]
+pillow = ["pillow"]
+duckduckgo-search = ["duckduckgo-search"]
+pysocks = ["pysocks"]
+xmltodict = ["xmltodict"]
+pymupdf = ["pymupdf"]
+html2text = ["html2text"]
+pyspark = ["pyspark"]
+pandera = ["pandera"]
+json5 = ["json5"]
+geoalchemy2 = ["geoalchemy2"]
+datetime = ["datetime"]
+azure-storage-blob = ["azure-storage-blob"]
+google-api-python-client = ["google-api-python-client"]
+google-auth-httplib2 = ["google-auth-httplib2"]
+google-auth-oauthlib = ["google-auth-oauthlib"]
+dnspython = ["dnspython"]
+openpyxl = ["openpyxl"]
+httpx = ["httpx"]
+
+# Interface groups
+aws = ["boto3"]
+drive = ["pydrive2"]
+peerdb = ["boto3", "clickhouse-driver"]
+proxy = ["stem"]
+alchemy = ["SQLAlchemy"]
+
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

{datamarket-0.9.2 → datamarket-0.9.4}/src/datamarket/interfaces/aws.py

@@ -5,6 +5,8 @@ import io
 import logging
 import boto3
 
+from ..utils.main import Config
+
 ########################################################################################################################
 # CLASSES
 
@@ -12,7 +14,7 @@ logger = logging.getLogger(__name__)
 
 
 class AWSInterface:
-    def __init__(self, config):
+    def __init__(self, config: Config) -> None:
         self.profiles = []
         self.config = config
 
@@ -31,13 +33,13 @@ class AWSInterface:
         self.current_profile = self.profiles[0] if self.profiles else None
         self._update_resources()
 
-    def _update_resources(self):
+    def _update_resources(self) -> None:
         if self.current_profile:
             self.s3 = self.current_profile["session"].resource("s3")
             self.s3_client = self.s3.meta.client
             self.bucket = self.current_profile["buckets"][0]
 
-    def switch_profile(self, profile_name: str):
+    def switch_profile(self, profile_name: str) -> None:
         for profile in self.profiles:
             if profile["profile"] == profile_name:
                 self.current_profile = profile
@@ -45,7 +47,7 @@ class AWSInterface:
                 return
         logger.warning(f"Profile {profile_name} not found")
 
-    def switch_bucket(self, bucket: str):
+    def switch_bucket(self, bucket: str) -> None:
         if bucket not in self.current_profile["buckets"]:
             logger.warning(
                 f"Bucket {bucket} not found in profile {self.current_profile['profile']}"
@@ -54,14 +56,14 @@ class AWSInterface:
 
         self.bucket = bucket
 
-    def get_file(self, s3_path: str):
+    def get_file(self, s3_path: str) -> None:
         try:
             return self.s3.Object(self.bucket, s3_path).get()
         except self.s3_client.exceptions.NoSuchKey:
             logger.info(f"{s3_path} does not exist")
 
-    def read_file_as_bytes(self, s3_path: str):
+    def read_file_as_bytes(self, s3_path: str) -> io.BytesIO:
         return io.BytesIO(self.get_file(s3_path)["Body"].read())
 
-    def upload_file(self, local_path: str, s3_path: str):
+    def upload_file(self, local_path: str, s3_path: str) -> None:
         self.s3.Bucket(self.bucket).upload_file(local_path, s3_path)

{datamarket-0.9.2 → datamarket-0.9.4}/src/datamarket/interfaces/ftp.py

@@ -5,6 +5,8 @@ import logging
 from ftplib import FTP, FTP_TLS
 from pathlib import Path
 
+from ..utils.main import Config
+
 ########################################################################################################################
 # CLASSES
 
@@ -12,7 +14,7 @@ logger = logging.getLogger(__name__)
 
 
 class FTPInterface:
-    def __init__(self, config):
+    def __init__(self, config: Config):
         if "ftp" in config:
             self.config = config["ftp"]
 

{datamarket-0.9.2 → datamarket-0.9.4}/src/datamarket/interfaces/peerdb.py

@@ -216,52 +216,26 @@ class TransientS3:
             self.config = section
             self.bucket_name = self.config["bucket"]
             self.session = boto3.Session(profile_name=self.config["profile"])
-            self.s3_client = self.session.client("s3")
+            self.s3_resource = self.session.resource("s3")
             self.credentials = self.session.get_credentials()
             self.access_key = self.credentials.access_key
             self.secret_key = self.credentials.secret_key
             self.region_name = self.session.region_name
-            self.endpoint_url = self.s3_client.meta.endpoint_url
+            self.endpoint_url = self.s3_resource.meta.endpoint_url
         else:
             logger.warning("no peerdb.s3 section in config")
 
     def delete_paths_with_schema(self, schema_name):
         logger.info(f"Deleting paths containing '{schema_name}' from S3")
 
-        paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages = paginator.paginate(Bucket=self.bucket_name, Delimiter="/")
+        bucket = self.s3_resource.Bucket(self.bucket_name)
 
-        for page in pages:
-            if "CommonPrefixes" in page:
-                for prefix in page["CommonPrefixes"]:
-                    folder = prefix["Prefix"]
-                    if schema_name in folder:
-                        self._delete_folder_contents(folder)
+        for prefix in [schema_name, f"clone_{schema_name}"]:
+            objects_to_delete = bucket.objects.filter(Prefix=prefix)
+            objects_to_delete.delete()
 
         logger.info(f"Deleted paths containing '{schema_name}' from S3")
 
-    def _delete_folder_contents(self, folder):
-        logger.info(f"Deleting contents of folder: {folder}")
-
-        paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages = paginator.paginate(Bucket=self.bucket_name, Prefix=folder)
-
-        delete_us = dict(Objects=[])
-        for page in pages:
-            if "Contents" in page:
-                for obj in page["Contents"]:
-                    delete_us["Objects"].append(dict(Key=obj["Key"]))
-
-                    # AWS limits to deleting 1000 objects at a time
-                    if len(delete_us["Objects"]) >= 1000:
-                        self.s3_client.delete_objects(Bucket=self.bucket_name, Delete=delete_us)
-                        delete_us = dict(Objects=[])
-
-        if len(delete_us["Objects"]):
-            self.s3_client.delete_objects(Bucket=self.bucket_name, Delete=delete_us)
-
-        logger.info(f"Deleted contents of folder: {folder}")
-
 
 class PeerDBInterface:
     def __init__(self, config):
@@ -308,11 +282,11 @@ class PeerDBInterface:
         if not self.docker_host_mapping or not host:
             return host
 
-        if host in ['localhost', '127.0.0.1']:
+        if host in ["localhost", "127.0.0.1"]:
             logger.debug(f"Mapping host {host} to {self.docker_host_mapping} for Docker environment")
             return self.docker_host_mapping
 
-        url_pattern = r'(localhost|127\.0\.0\.1)'
+        url_pattern = r"(localhost|127\.0\.0\.1)"
         match = re.search(url_pattern, host)
         if match:
             original_host = match.group(1)

{datamarket-0.9.2 → datamarket-0.9.4}/src/datamarket/utils/main.py

@@ -2,7 +2,6 @@
 # IMPORTS
 
 import asyncio
-import configparser
 import inspect
 import logging
 import random
@@ -16,6 +15,7 @@ from typing import Literal, Union
 
 import pendulum
 from croniter import croniter
+from configparser import RawConfigParser
 from dynaconf import Dynaconf, add_converter
 
 ########################################################################################################################
@@ -23,6 +23,8 @@ from dynaconf import Dynaconf, add_converter
 
 logger = logging.getLogger(__name__)
 
+Config = Union[RawConfigParser, Dynaconf]
+
 
 def get_granular_date(
     granularity: Union[Literal["monthly", "biweekly", "weekly", "daily"], str], tz: str = "Europe/Madrid"
@@ -64,16 +66,20 @@ def read_converter(path_str: str):
         return f.read()
 
 
-def get_config(config_file: Path, tz: str = "Europe/Madrid"):
+def get_config(
+    config_file: Path, tz: str = "Europe/Madrid"
+) -> Union[RawConfigParser, Dynaconf]:
     if Path(config_file).suffix == ".ini":
         logger.warning("Using legacy INI config reader. Please migrate to TOML")
-        cfg = configparser.RawConfigParser()
+        cfg = RawConfigParser()
         cfg.read(config_file)
         return cfg
 
     add_converter("read", read_converter)
 
     dt_now = get_granular_date("now", tz)
+    dt_weekly = get_granular_date("weekly", tz)
+    dt_biweekly = get_granular_date("biweekly", tz)
 
     config = Dynaconf(
         environments=True,
@@ -84,14 +90,17 @@ def get_config(config_file: Path, tz: str = "Europe/Madrid"):
     config.load_file(path=Path.home() / config_file.name)
 
     config.vars = {
-        "now": dt_now.strftime("%Y-%m-%d %H:%M:%S"),
-        "today": dt_now.strftime("%Y-%m-%d"),
         "year": dt_now.strftime("%Y"),
         "month": dt_now.strftime("%m"),
         "day": dt_now.strftime("%d"),
-        "biweekly_date": get_granular_date("biweekly", tz).strftime("%Y-%m-%d"),
-        "today_stripped": dt_now.strftime("%Y%m%d"),
+        "now": dt_now.strftime("%Y-%m-%d %H:%M:%S"),
         "now_stripped": dt_now.strftime("%Y%m%d%H%M%S"),
+        "today": dt_now.strftime("%Y-%m-%d"),
+        "today_stripped": dt_now.strftime("%Y%m%d"),
+        "weekly_date": dt_weekly.strftime("%Y-%m-%d"),
+        "weekly_date_stripped": dt_weekly.strftime("%Y%m%d"),
+        "biweekly_date": dt_biweekly.strftime("%Y-%m-%d"),
+        "biweekly_date_stripped": dt_biweekly.strftime("%Y%m%d"),
         "dynaconf_merge": True,
     }
 

datamarket-0.9.2/PKG-INFO

@@ -1,149 +0,0 @@
-Metadata-Version: 2.3
-Name: datamarket
-Version: 0.9.2
-Summary: Utilities that integrate advanced scraping knowledge into just one library.
-License: GPL-3.0-or-later
-Author: DataMarket
-Author-email: techsupport@datamarket.es
-Requires-Python: >=3.9,<4.0
-Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
-Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Provides-Extra: alchemy
-Provides-Extra: aws
-Provides-Extra: azure-storage-blob
-Provides-Extra: boto3
-Provides-Extra: chompjs
-Provides-Extra: click
-Provides-Extra: clickhouse-driver
-Provides-Extra: datetime
-Provides-Extra: demjson3
-Provides-Extra: dnspython
-Provides-Extra: drive
-Provides-Extra: duckduckgo-search
-Provides-Extra: fake-useragent
-Provides-Extra: geoalchemy2
-Provides-Extra: geopandas
-Provides-Extra: geopy
-Provides-Extra: google-api-python-client
-Provides-Extra: google-auth-httplib2
-Provides-Extra: google-auth-oauthlib
-Provides-Extra: html2text
-Provides-Extra: httpx
-Provides-Extra: json5
-Provides-Extra: lxml
-Provides-Extra: nodriver
-Provides-Extra: openpyxl
-Provides-Extra: pandas
-Provides-Extra: pandera
-Provides-Extra: peerdb
-Provides-Extra: pillow
-Provides-Extra: playwright
-Provides-Extra: playwright-stealth
-Provides-Extra: proxy
-Provides-Extra: pyarrow
-Provides-Extra: pydrive2
-Provides-Extra: pymupdf
-Provides-Extra: pysocks
-Provides-Extra: pyspark
-Provides-Extra: pytest
-Provides-Extra: rapidfuzz
-Provides-Extra: retry
-Provides-Extra: shapely
-Provides-Extra: soda-core-mysql
-Provides-Extra: soda-core-postgres
-Provides-Extra: stem
-Provides-Extra: tqdm
-Provides-Extra: undetected-chromedriver
-Provides-Extra: unidecode
-Provides-Extra: xmltodict
-Requires-Dist: SQLAlchemy (==2.0.36) ; extra == "alchemy"
-Requires-Dist: azure-storage-blob (==12.23.1) ; extra == "azure-storage-blob"
-Requires-Dist: beautifulsoup4 (==4.12.3)
-Requires-Dist: boto3 (==1.35.53) ; extra == "boto3" or extra == "aws" or extra == "peerdb"
-Requires-Dist: chompjs (==1.3.0) ; extra == "chompjs"
-Requires-Dist: click (==8.1.7) ; extra == "click"
-Requires-Dist: clickhouse-driver (==0.2.9) ; extra == "clickhouse-driver" or extra == "peerdb"
-Requires-Dist: croniter (==3.0.4)
-Requires-Dist: datetime (==5.5) ; extra == "datetime"
-Requires-Dist: demjson3 (==3.0.6) ; extra == "demjson3"
-Requires-Dist: dnspython (==2.7.0) ; extra == "dnspython"
-Requires-Dist: duckduckgo-search (==6.2.11b1) ; extra == "duckduckgo-search"
-Requires-Dist: dynaconf (==3.2.6)
-Requires-Dist: fake-useragent (==1.5.1) ; extra == "fake-useragent"
-Requires-Dist: geoalchemy2 (==0.15.2) ; extra == "geoalchemy2"
-Requires-Dist: geopandas (==1.0.1) ; extra == "geopandas"
-Requires-Dist: geopy (==2.4.1) ; extra == "geopy"
-Requires-Dist: google-api-python-client (==2.151.0) ; extra == "google-api-python-client"
-Requires-Dist: google-auth-httplib2 (==0.2.0) ; extra == "google-auth-httplib2"
-Requires-Dist: google-auth-oauthlib (==1.2.1) ; extra == "google-auth-oauthlib"
-Requires-Dist: html2text (==2024.2.26) ; extra == "html2text"
-Requires-Dist: httpx[http2] (==0.28.1) ; extra == "httpx"
-Requires-Dist: jinja2 (==3.1.5)
-Requires-Dist: json5 (==0.9.25) ; extra == "json5"
-Requires-Dist: lxml[html-clean] (==5.3.0) ; extra == "lxml"
-Requires-Dist: nodriver (==0.38.post1) ; extra == "nodriver"
-Requires-Dist: openpyxl (==3.1.5) ; extra == "openpyxl"
-Requires-Dist: pandas (==2.2.3) ; extra == "pandas"
-Requires-Dist: pandera (==0.20.4) ; extra == "pandera"
-Requires-Dist: pendulum (==3.0.0)
-Requires-Dist: pillow (==11.0.0) ; extra == "pillow"
-Requires-Dist: playwright (==1.47.0) ; extra == "playwright"
-Requires-Dist: playwright-stealth (==1.0.6) ; extra == "playwright-stealth"
-Requires-Dist: pre-commit (==4.0.1)
-Requires-Dist: psycopg2-binary (==2.9.10)
-Requires-Dist: pyarrow (==17.0.0) ; extra == "pyarrow"
-Requires-Dist: pydrive2 (==1.20.0) ; extra == "pydrive2" or extra == "drive"
-Requires-Dist: pymupdf (==1.24.13) ; extra == "pymupdf"
-Requires-Dist: pysocks (==1.7.1) ; extra == "pysocks"
-Requires-Dist: pyspark (==3.5.3) ; extra == "pyspark"
-Requires-Dist: pytest (==8.3.3) ; extra == "pytest"
-Requires-Dist: rapidfuzz (==3.10.1) ; extra == "rapidfuzz"
-Requires-Dist: requests (==2.32.3)
-Requires-Dist: retry (==0.9.2) ; extra == "retry"
-Requires-Dist: shapely (==2.0.6) ; extra == "shapely"
-Requires-Dist: soda-core-mysql (==3.4.4) ; extra == "soda-core-mysql"
-Requires-Dist: soda-core-postgres (==3.4.1) ; extra == "soda-core-postgres"
-Requires-Dist: stem (==1.8.2) ; extra == "stem" or extra == "proxy"
-Requires-Dist: tenacity (==9.0.0)
-Requires-Dist: tqdm (==4.66.6) ; extra == "tqdm"
-Requires-Dist: typer (==0.12.5)
-Requires-Dist: undetected-chromedriver (==3.5.5) ; extra == "undetected-chromedriver"
-Requires-Dist: unidecode (==1.3.8) ; extra == "unidecode"
-Requires-Dist: xmltodict (==0.14.2) ; extra == "xmltodict"
-Project-URL: Documentation, https://github.com/Data-Market/datamarket
-Project-URL: Homepage, https://datamarket.es
-Project-URL: Repository, https://github.com/Data-Market/datamarket
-Description-Content-Type: text/markdown
-
-# DataMarket scraping core
-
-------------------------------------------------------
-[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
-
-
-Utilities that integrate advance scraping knowledge into just one library.
-
-## Installation
-
-To install this library in your Python environment:
-
-`pip install datamarket`
-
-## Documentation
-
-This library has built functionalities for the following topics:
-
-- **Databases**: through sqlalchemy it allows to insert records and perform queries in any database.
-- **Proxies**: wide range of functions to perform HTTP requests through custom proxies or the Tor network.
-- **Tinybird**: a Python client for this popular API.
-- **Drive**: functions to upload, delete or authenticate to Google Drive.
-- **FTP**: functions to upload, delete or authenticate to an FTP, SFTP or FTPS server.
-- **Selenium**: wrapper for the main Selenium functions.
-

datamarket-0.9.2/pyproject.toml

@@ -1,130 +0,0 @@
-[tool.poetry]
-name = "datamarket"
-version = "0.9.2"
-description = "Utilities that integrate advanced scraping knowledge into just one library."
-authors = ["DataMarket <techsupport@datamarket.es>"]
-license = "GPL-3.0-or-later"
-readme = "README.md"
-homepage = "https://datamarket.es"
-repository = "https://github.com/Data-Market/datamarket"
-documentation = "https://github.com/Data-Market/datamarket"
-classifiers = [
-    "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
-    "Operating System :: OS Independent",
-]
-
-[tool.poetry.dependencies]
-python = "^3.9"
-typer = "0.12.5"
-psycopg2-binary = "2.9.10"
-requests = "2.32.3"
-tenacity = "9.0.0"
-beautifulsoup4 = "4.12.3"
-pre-commit = "4.0.1"
-pendulum = "3.0.0"
-croniter = "3.0.4"
-dynaconf = "3.2.6"
-jinja2 = "3.1.5"
-
-boto3 = { version = "1.35.53", optional = true }
-unidecode = { version = "1.3.8", optional = true }
-lxml = { extras = ["html-clean"], version = "5.3.0", optional = true }
-tqdm = { version = "4.66.6", optional = true }
-pandas = { version = "2.2.3", optional = true }
-pyarrow = { version = "17.0.0", optional = true }
-pytest = { version = "8.3.3", optional = true }
-playwright = { version = "1.47.0", optional = true }
-playwright-stealth = { version = "1.0.6", optional = true }
-soda-core-postgres = { version = "3.4.1", optional = true }
-soda-core-mysql = { version = "3.4.4", optional = true }
-fake-useragent = { version = "1.5.1", optional = true }
-pydrive2 = { version = "1.20.0", optional = true }
-clickhouse-driver = { version = "0.2.9", optional = true }
-stem = { version = "1.8.2", optional = true }
-click = { version = "8.1.7", optional = true }
-rapidfuzz = { version = "3.10.1", optional = true }
-demjson3 = { version = "3.0.6", optional = true }
-geopy = { version = "2.4.1", optional = true }
-nodriver = { version = "0.38.post1", optional = true }
-undetected-chromedriver = { version = "3.5.5", optional = true }
-retry = { version = "0.9.2", optional = true }
-shapely = { version = "2.0.6", optional = true }
-geopandas = { version = "1.0.1", optional = true }
-chompjs = { version = "1.3.0", optional = true }
-pillow = { version = "11.0.0", optional = true }
-duckduckgo-search = { version = "6.2.11b1", optional = true }
-pysocks = { version = "1.7.1", optional = true }
-xmltodict = { version = "0.14.2", optional = true }
-pymupdf = { version = "1.24.13", optional = true }
-html2text = { version = "2024.2.26", optional = true }
-pyspark = { version = "3.5.3", optional = true }
-pandera = { version = "0.20.4", optional = true }
-json5 = { version = "0.9.25", optional = true }
-geoalchemy2 = { version = "0.15.2", optional = true }
-datetime = { version = "5.5", optional = true }
-azure-storage-blob = { version = "12.23.1", optional = true }
-google-api-python-client = { version = "2.151.0", optional = true }
-google-auth-httplib2 = { version = "0.2.0", optional = true }
-google-auth-oauthlib = { version = "1.2.1", optional = true }
-dnspython = { version = "2.7.0", optional = true }
-openpyxl = { version = "3.1.5", optional = true }
-httpx = { extras = ["http2"], version = "0.28.1", optional = true }
-SQLAlchemy = { version = "2.0.36", optional = true }
-
-[tool.poetry.extras]
-boto3 = ["boto3"]
-unidecode = ["unidecode"]
-lxml = ["lxml"]
-tqdm = ["tqdm"]
-pandas = ["pandas"]
-pyarrow = ["pyarrow"]
-pytest = ["pytest"]
-playwright = ["playwright"]
-playwright-stealth = ["playwright-stealth"]
-soda-core-postgres = ["soda-core-postgres"]
-soda-core-mysql = ["soda-core-mysql"]
-fake-useragent = ["fake-useragent"]
-pydrive2 = ["pydrive2"]
-clickhouse-driver = ["clickhouse-driver"]
-stem = ["stem"]
-click = ["click"]
-rapidfuzz = ["rapidfuzz"]
-demjson3 = ["demjson3"]
-geopy = ["geopy"]
-nodriver = ["nodriver"]
-undetected-chromedriver = ["undetected-chromedriver"]
-retry = ["retry"]
-shapely = ["shapely"]
-geopandas = ["geopandas"]
-chompjs = ["chompjs"]
-pillow = ["pillow"]
-duckduckgo-search = ["duckduckgo-search"]
-pysocks = ["pysocks"]
-xmltodict = ["xmltodict"]
-pymupdf = ["pymupdf"]
-html2text = ["html2text"]
-pyspark = ["pyspark"]
-pandera = ["pandera"]
-json5 = ["json5"]
-geoalchemy2 = ["geoalchemy2"]
-datetime = ["datetime"]
-azure-storage-blob = ["azure-storage-blob"]
-google-api-python-client = ["google-api-python-client"]
-google-auth-httplib2 = ["google-auth-httplib2"]
-google-auth-oauthlib = ["google-auth-oauthlib"]
-dnspython = ["dnspython"]
-openpyxl = ["openpyxl"]
-httpx = ["httpx"]
-
-# Interface groups
-aws = ["boto3"]
-drive = ["pydrive2"]
-peerdb = ["boto3", "clickhouse-driver"]
-proxy = ["stem"]
-alchemy = ["SQLAlchemy"]
-
-
-[build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"