datamarket 0.9.3__py3-none-any.whl → 0.9.4__py3-none-any.whl

datamarket/interfaces/peerdb.py

@@ -216,52 +216,26 @@ class TransientS3:
             self.config = section
             self.bucket_name = self.config["bucket"]
             self.session = boto3.Session(profile_name=self.config["profile"])
-            self.s3_client = self.session.client("s3")
+            self.s3_resource = self.session.resource("s3")
             self.credentials = self.session.get_credentials()
             self.access_key = self.credentials.access_key
             self.secret_key = self.credentials.secret_key
             self.region_name = self.session.region_name
-            self.endpoint_url = self.s3_client.meta.endpoint_url
+            self.endpoint_url = self.s3_resource.meta.endpoint_url
         else:
             logger.warning("no peerdb.s3 section in config")
 
     def delete_paths_with_schema(self, schema_name):
         logger.info(f"Deleting paths containing '{schema_name}' from S3")
 
-        paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages = paginator.paginate(Bucket=self.bucket_name, Delimiter="/")
+        bucket = self.s3_resource.Bucket(self.bucket_name)
 
-        for page in pages:
-            if "CommonPrefixes" in page:
-                for prefix in page["CommonPrefixes"]:
-                    folder = prefix["Prefix"]
-                    if schema_name in folder:
-                        self._delete_folder_contents(folder)
+        for prefix in [schema_name, f"clone_{schema_name}"]:
+            objects_to_delete = bucket.objects.filter(Prefix=prefix)
+            objects_to_delete.delete()
 
         logger.info(f"Deleted paths containing '{schema_name}' from S3")
 
-    def _delete_folder_contents(self, folder):
-        logger.info(f"Deleting contents of folder: {folder}")
-
-        paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages = paginator.paginate(Bucket=self.bucket_name, Prefix=folder)
-
-        delete_us = dict(Objects=[])
-        for page in pages:
-            if "Contents" in page:
-                for obj in page["Contents"]:
-                    delete_us["Objects"].append(dict(Key=obj["Key"]))
-
-                    # AWS limits to deleting 1000 objects at a time
-                    if len(delete_us["Objects"]) >= 1000:
-                        self.s3_client.delete_objects(Bucket=self.bucket_name, Delete=delete_us)
-                        delete_us = dict(Objects=[])
-
-        if len(delete_us["Objects"]):
-            self.s3_client.delete_objects(Bucket=self.bucket_name, Delete=delete_us)
-
-        logger.info(f"Deleted contents of folder: {folder}")
-
 
 class PeerDBInterface:
     def __init__(self, config):
@@ -308,11 +282,11 @@ class PeerDBInterface:
         if not self.docker_host_mapping or not host:
             return host
 
-        if host in ['localhost', '127.0.0.1']:
+        if host in ["localhost", "127.0.0.1"]:
             logger.debug(f"Mapping host {host} to {self.docker_host_mapping} for Docker environment")
             return self.docker_host_mapping
 
-        url_pattern = r'(localhost|127\.0\.0\.1)'
+        url_pattern = r"(localhost|127\.0\.0\.1)"
         match = re.search(url_pattern, host)
         if match:
             original_host = match.group(1)

datamarket-0.9.4.dist-info/METADATA

@@ -0,0 +1,144 @@
+Metadata-Version: 2.3
+Name: datamarket
+Version: 0.9.4
+Summary: Utilities that integrate advanced scraping knowledge into just one library.
+License: GPL-3.0-or-later
+Author: DataMarket
+Author-email: techsupport@datamarket.es
+Requires-Python: >=3.12,<4.0
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: alchemy
+Provides-Extra: aws
+Provides-Extra: azure-storage-blob
+Provides-Extra: boto3
+Provides-Extra: chompjs
+Provides-Extra: click
+Provides-Extra: clickhouse-driver
+Provides-Extra: datetime
+Provides-Extra: demjson3
+Provides-Extra: dnspython
+Provides-Extra: drive
+Provides-Extra: duckduckgo-search
+Provides-Extra: fake-useragent
+Provides-Extra: geoalchemy2
+Provides-Extra: geopandas
+Provides-Extra: geopy
+Provides-Extra: google-api-python-client
+Provides-Extra: google-auth-httplib2
+Provides-Extra: google-auth-oauthlib
+Provides-Extra: html2text
+Provides-Extra: httpx
+Provides-Extra: json5
+Provides-Extra: lxml
+Provides-Extra: nodriver
+Provides-Extra: openpyxl
+Provides-Extra: pandas
+Provides-Extra: pandera
+Provides-Extra: peerdb
+Provides-Extra: pillow
+Provides-Extra: playwright
+Provides-Extra: playwright-stealth
+Provides-Extra: proxy
+Provides-Extra: pyarrow
+Provides-Extra: pydrive2
+Provides-Extra: pymupdf
+Provides-Extra: pysocks
+Provides-Extra: pyspark
+Provides-Extra: pytest
+Provides-Extra: rapidfuzz
+Provides-Extra: retry
+Provides-Extra: shapely
+Provides-Extra: soda-core-mysql
+Provides-Extra: soda-core-postgres
+Provides-Extra: stem
+Provides-Extra: tqdm
+Provides-Extra: undetected-chromedriver
+Provides-Extra: unidecode
+Provides-Extra: xmltodict
+Requires-Dist: SQLAlchemy (>=2.0.0,<3.0.0) ; extra == "alchemy"
+Requires-Dist: azure-storage-blob (>=12.0.0,<13.0.0) ; extra == "azure-storage-blob"
+Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
+Requires-Dist: boto3 (>=1.0.0,<2.0.0) ; extra == "boto3" or extra == "aws" or extra == "peerdb"
+Requires-Dist: chompjs (>=1.0.0,<2.0.0) ; extra == "chompjs"
+Requires-Dist: click (>=8.0.0,<9.0.0) ; extra == "click"
+Requires-Dist: clickhouse-driver (>=0.2.0,<0.3.0) ; extra == "clickhouse-driver" or extra == "peerdb"
+Requires-Dist: croniter (>=3.0.0,<4.0.0)
+Requires-Dist: datetime (>=5.0,<6.0) ; extra == "datetime"
+Requires-Dist: demjson3 (>=3.0.0,<4.0.0) ; extra == "demjson3"
+Requires-Dist: dnspython (>=2.0.0,<3.0.0) ; extra == "dnspython"
+Requires-Dist: duckduckgo-search (>=7.0.0,<8.0.0) ; extra == "duckduckgo-search"
+Requires-Dist: dynaconf (>=3.0.0,<4.0.0)
+Requires-Dist: fake-useragent (>=2.0.0,<3.0.0) ; extra == "fake-useragent"
+Requires-Dist: geoalchemy2 (>=0.17.0,<0.18.0) ; extra == "geoalchemy2"
+Requires-Dist: geopandas (>=1.0.0,<2.0.0) ; extra == "geopandas"
+Requires-Dist: geopy (>=2.0.0,<3.0.0) ; extra == "geopy"
+Requires-Dist: google-api-python-client (>=2.0.0,<3.0.0) ; extra == "google-api-python-client"
+Requires-Dist: google-auth-httplib2 (>=0.2.0,<0.3.0) ; extra == "google-auth-httplib2"
+Requires-Dist: google-auth-oauthlib (>=1.0.0,<2.0.0) ; extra == "google-auth-oauthlib"
+Requires-Dist: html2text (>=2024.0.0,<2025.0.0) ; extra == "html2text"
+Requires-Dist: httpx[http2] (>=0.28.0,<0.29.0) ; extra == "httpx"
+Requires-Dist: jinja2 (>=3.0.0,<4.0.0)
+Requires-Dist: json5 (>=0.10.0,<0.11.0) ; extra == "json5"
+Requires-Dist: lxml[html-clean] (>=5.0.0,<6.0.0) ; extra == "lxml"
+Requires-Dist: nodriver (==0.38.post1) ; extra == "nodriver"
+Requires-Dist: openpyxl (>=3.0.0,<4.0.0) ; extra == "openpyxl"
+Requires-Dist: pandas (>=2.0.0,<3.0.0) ; extra == "pandas"
+Requires-Dist: pandera (>=0.22.0,<0.23.0) ; extra == "pandera"
+Requires-Dist: pendulum (>=3.0.0,<4.0.0)
+Requires-Dist: pillow (>=11.0.0,<12.0.0) ; extra == "pillow"
+Requires-Dist: playwright (==1.47.0) ; extra == "playwright"
+Requires-Dist: pre-commit (>=4.0.0,<5.0.0)
+Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0)
+Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "pyarrow"
+Requires-Dist: pydrive2 (>=1.0.0,<2.0.0) ; extra == "pydrive2" or extra == "drive"
+Requires-Dist: pymupdf (>=1.0.0,<2.0.0) ; extra == "pymupdf"
+Requires-Dist: pysocks (>=1.0.0,<2.0.0) ; extra == "pysocks"
+Requires-Dist: pyspark (>=3.0.0,<4.0.0) ; extra == "pyspark"
+Requires-Dist: pytest (>=8.0.0,<9.0.0) ; extra == "pytest"
+Requires-Dist: rapidfuzz (>=3.0.0,<4.0.0) ; extra == "rapidfuzz"
+Requires-Dist: requests (>=2.0.0,<3.0.0)
+Requires-Dist: retry (>=0.9.0,<0.10.0) ; extra == "retry"
+Requires-Dist: shapely (>=2.0.0,<3.0.0) ; extra == "shapely"
+Requires-Dist: soda-core-mysql (>=3.0.0,<4.0.0) ; extra == "soda-core-mysql"
+Requires-Dist: soda-core-postgres (>=3.0.0,<4.0.0) ; extra == "soda-core-postgres"
+Requires-Dist: stem (>=1.0.0,<2.0.0) ; extra == "stem" or extra == "proxy"
+Requires-Dist: tenacity (>=9.0.0,<10.0.0)
+Requires-Dist: tqdm (>=4.0.0,<5.0.0) ; extra == "tqdm"
+Requires-Dist: typer (>=0.15.0,<0.16.0)
+Requires-Dist: unidecode (>=1.0.0,<2.0.0) ; extra == "unidecode"
+Requires-Dist: xmltodict (>=0.14.0,<0.15.0) ; extra == "xmltodict"
+Project-URL: Documentation, https://github.com/Data-Market/datamarket
+Project-URL: Homepage, https://datamarket.es
+Project-URL: Repository, https://github.com/Data-Market/datamarket
+Description-Content-Type: text/markdown
+
+# DataMarket scraping core
+
+------------------------------------------------------
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+
+
+Utilities that integrate advance scraping knowledge into just one library.
+
+## Installation
+
+To install this library in your Python environment:
+
+`pip install datamarket`
+
+## Documentation
+
+This library has built functionalities for the following topics:
+
+- **Databases**: through sqlalchemy it allows to insert records and perform queries in any database.
+- **Proxies**: wide range of functions to perform HTTP requests through custom proxies or the Tor network.
+- **Tinybird**: a Python client for this popular API.
+- **Drive**: functions to upload, delete or authenticate to Google Drive.
+- **FTP**: functions to upload, delete or authenticate to an FTP, SFTP or FTPS server.
+- **Selenium**: wrapper for the main Selenium functions.
+

{datamarket-0.9.3.dist-info → datamarket-0.9.4.dist-info}/RECORD

@@ -5,7 +5,7 @@ datamarket/interfaces/aws.py,sha256=R6lYdSCD6a4g9l6aFMtNDt_EX3kroe2untDhgy7XG1k,
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=Owk3D7tiF47_ZFT3Dc9h4_BaPsWtcJUbhagjpQB19q8,1900
 datamarket/interfaces/nominatim.py,sha256=_gFJ04D-ju5xn3wuaGT5Pj5jhf4F5eINpxOpuQL_dIQ,3664
-datamarket/interfaces/peerdb.py,sha256=rNQ1-THcVvrej8BEPJs9zM4VfH5dlByafOIHYN9sB2A,21833
+datamarket/interfaces/peerdb.py,sha256=FhBLJfR2EMT9Rsnj_OJXvC14E5OlXGsMrPUQ1AQlwPY,20717
 datamarket/interfaces/proxy.py,sha256=updoOStKd8-nQBbxWbnD9eOt6HksnYi-5dQ0rEySf5M,3152
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -17,7 +17,7 @@ datamarket/utils/main.py,sha256=0Abt3ww1VSPnX4AVKDcYzqDLAOEV_54iUHMLJfre2bg,6129
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.9.3.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.3.dist-info/METADATA,sha256=SY-94WLCqqxLVnPXVPzRQZ6SwP0I-HbqdCBp9Me9ySw,6329
-datamarket-0.9.3.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-datamarket-0.9.3.dist-info/RECORD,,
+datamarket-0.9.4.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.4.dist-info/METADATA,sha256=YYuxN--M3y9MX62_hG5Y1piS2TBBo_fl6MfJeLyyOZA,6360
+datamarket-0.9.4.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+datamarket-0.9.4.dist-info/RECORD,,

datamarket-0.9.3.dist-info/METADATA

@@ -1,149 +0,0 @@
-Metadata-Version: 2.3
-Name: datamarket
-Version: 0.9.3
-Summary: Utilities that integrate advanced scraping knowledge into just one library.
-License: GPL-3.0-or-later
-Author: DataMarket
-Author-email: techsupport@datamarket.es
-Requires-Python: >=3.9,<4.0
-Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
-Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Provides-Extra: alchemy
-Provides-Extra: aws
-Provides-Extra: azure-storage-blob
-Provides-Extra: boto3
-Provides-Extra: chompjs
-Provides-Extra: click
-Provides-Extra: clickhouse-driver
-Provides-Extra: datetime
-Provides-Extra: demjson3
-Provides-Extra: dnspython
-Provides-Extra: drive
-Provides-Extra: duckduckgo-search
-Provides-Extra: fake-useragent
-Provides-Extra: geoalchemy2
-Provides-Extra: geopandas
-Provides-Extra: geopy
-Provides-Extra: google-api-python-client
-Provides-Extra: google-auth-httplib2
-Provides-Extra: google-auth-oauthlib
-Provides-Extra: html2text
-Provides-Extra: httpx
-Provides-Extra: json5
-Provides-Extra: lxml
-Provides-Extra: nodriver
-Provides-Extra: openpyxl
-Provides-Extra: pandas
-Provides-Extra: pandera
-Provides-Extra: peerdb
-Provides-Extra: pillow
-Provides-Extra: playwright
-Provides-Extra: playwright-stealth
-Provides-Extra: proxy
-Provides-Extra: pyarrow
-Provides-Extra: pydrive2
-Provides-Extra: pymupdf
-Provides-Extra: pysocks
-Provides-Extra: pyspark
-Provides-Extra: pytest
-Provides-Extra: rapidfuzz
-Provides-Extra: retry
-Provides-Extra: shapely
-Provides-Extra: soda-core-mysql
-Provides-Extra: soda-core-postgres
-Provides-Extra: stem
-Provides-Extra: tqdm
-Provides-Extra: undetected-chromedriver
-Provides-Extra: unidecode
-Provides-Extra: xmltodict
-Requires-Dist: SQLAlchemy (==2.0.36) ; extra == "alchemy"
-Requires-Dist: azure-storage-blob (==12.23.1) ; extra == "azure-storage-blob"
-Requires-Dist: beautifulsoup4 (==4.12.3)
-Requires-Dist: boto3 (==1.35.53) ; extra == "boto3" or extra == "aws" or extra == "peerdb"
-Requires-Dist: chompjs (==1.3.0) ; extra == "chompjs"
-Requires-Dist: click (==8.1.7) ; extra == "click"
-Requires-Dist: clickhouse-driver (==0.2.9) ; extra == "clickhouse-driver" or extra == "peerdb"
-Requires-Dist: croniter (==3.0.4)
-Requires-Dist: datetime (==5.5) ; extra == "datetime"
-Requires-Dist: demjson3 (==3.0.6) ; extra == "demjson3"
-Requires-Dist: dnspython (==2.7.0) ; extra == "dnspython"
-Requires-Dist: duckduckgo-search (==6.2.11b1) ; extra == "duckduckgo-search"
-Requires-Dist: dynaconf (==3.2.6)
-Requires-Dist: fake-useragent (==1.5.1) ; extra == "fake-useragent"
-Requires-Dist: geoalchemy2 (==0.15.2) ; extra == "geoalchemy2"
-Requires-Dist: geopandas (==1.0.1) ; extra == "geopandas"
-Requires-Dist: geopy (==2.4.1) ; extra == "geopy"
-Requires-Dist: google-api-python-client (==2.151.0) ; extra == "google-api-python-client"
-Requires-Dist: google-auth-httplib2 (==0.2.0) ; extra == "google-auth-httplib2"
-Requires-Dist: google-auth-oauthlib (==1.2.1) ; extra == "google-auth-oauthlib"
-Requires-Dist: html2text (==2024.2.26) ; extra == "html2text"
-Requires-Dist: httpx[http2] (==0.28.1) ; extra == "httpx"
-Requires-Dist: jinja2 (==3.1.5)
-Requires-Dist: json5 (==0.9.25) ; extra == "json5"
-Requires-Dist: lxml[html-clean] (==5.3.0) ; extra == "lxml"
-Requires-Dist: nodriver (==0.38.post1) ; extra == "nodriver"
-Requires-Dist: openpyxl (==3.1.5) ; extra == "openpyxl"
-Requires-Dist: pandas (==2.2.3) ; extra == "pandas"
-Requires-Dist: pandera (==0.20.4) ; extra == "pandera"
-Requires-Dist: pendulum (==3.0.0)
-Requires-Dist: pillow (==11.0.0) ; extra == "pillow"
-Requires-Dist: playwright (==1.47.0) ; extra == "playwright"
-Requires-Dist: playwright-stealth (==1.0.6) ; extra == "playwright-stealth"
-Requires-Dist: pre-commit (==4.0.1)
-Requires-Dist: psycopg2-binary (==2.9.10)
-Requires-Dist: pyarrow (==17.0.0) ; extra == "pyarrow"
-Requires-Dist: pydrive2 (==1.20.0) ; extra == "pydrive2" or extra == "drive"
-Requires-Dist: pymupdf (==1.24.13) ; extra == "pymupdf"
-Requires-Dist: pysocks (==1.7.1) ; extra == "pysocks"
-Requires-Dist: pyspark (==3.5.3) ; extra == "pyspark"
-Requires-Dist: pytest (==8.3.3) ; extra == "pytest"
-Requires-Dist: rapidfuzz (==3.10.1) ; extra == "rapidfuzz"
-Requires-Dist: requests (==2.32.3)
-Requires-Dist: retry (==0.9.2) ; extra == "retry"
-Requires-Dist: shapely (==2.0.6) ; extra == "shapely"
-Requires-Dist: soda-core-mysql (==3.4.4) ; extra == "soda-core-mysql"
-Requires-Dist: soda-core-postgres (==3.4.1) ; extra == "soda-core-postgres"
-Requires-Dist: stem (==1.8.2) ; extra == "stem" or extra == "proxy"
-Requires-Dist: tenacity (==9.0.0)
-Requires-Dist: tqdm (==4.66.6) ; extra == "tqdm"
-Requires-Dist: typer (==0.12.5)
-Requires-Dist: undetected-chromedriver (==3.5.5) ; extra == "undetected-chromedriver"
-Requires-Dist: unidecode (==1.3.8) ; extra == "unidecode"
-Requires-Dist: xmltodict (==0.14.2) ; extra == "xmltodict"
-Project-URL: Documentation, https://github.com/Data-Market/datamarket
-Project-URL: Homepage, https://datamarket.es
-Project-URL: Repository, https://github.com/Data-Market/datamarket
-Description-Content-Type: text/markdown
-
-# DataMarket scraping core
-
-------------------------------------------------------
-[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
-
-
-Utilities that integrate advance scraping knowledge into just one library.
-
-## Installation
-
-To install this library in your Python environment:
-
-`pip install datamarket`
-
-## Documentation
-
-This library has built functionalities for the following topics:
-
-- **Databases**: through sqlalchemy it allows to insert records and perform queries in any database.
-- **Proxies**: wide range of functions to perform HTTP requests through custom proxies or the Tor network.
-- **Tinybird**: a Python client for this popular API.
-- **Drive**: functions to upload, delete or authenticate to Google Drive.
-- **FTP**: functions to upload, delete or authenticate to an FTP, SFTP or FTPS server.
-- **Selenium**: wrapper for the main Selenium functions.
-