datamarket 0.9.0__py3-none-any.whl → 0.9.2__py3-none-any.whl

datamarket/interfaces/aws.py

@@ -20,7 +20,7 @@ class AWSInterface:
             self.profiles.append(
                 {
                     "profile": profile_name,
-                    "bucket": values["bucket"],
+                    "buckets": values["buckets"],
                     "session": boto3.Session(profile_name=profile_name),
                 }
             )
@@ -35,9 +35,9 @@ class AWSInterface:
         if self.current_profile:
             self.s3 = self.current_profile["session"].resource("s3")
             self.s3_client = self.s3.meta.client
-            self.bucket = self.current_profile["bucket"]
+            self.bucket = self.current_profile["buckets"][0]
 
-    def switch_profile(self, profile_name):
+    def switch_profile(self, profile_name: str):
         for profile in self.profiles:
             if profile["profile"] == profile_name:
                 self.current_profile = profile
@@ -45,14 +45,23 @@ class AWSInterface:
                 return
         logger.warning(f"Profile {profile_name} not found")
 
-    def get_file(self, s3_path):
+    def switch_bucket(self, bucket: str):
+        if bucket not in self.current_profile["buckets"]:
+            logger.warning(
+                f"Bucket {bucket} not found in profile {self.current_profile['profile']}"
+            )
+            return
+
+        self.bucket = bucket
+
+    def get_file(self, s3_path: str):
         try:
             return self.s3.Object(self.bucket, s3_path).get()
         except self.s3_client.exceptions.NoSuchKey:
             logger.info(f"{s3_path} does not exist")
 
-    def read_file_as_bytes(self, s3_path):
+    def read_file_as_bytes(self, s3_path: str):
         return io.BytesIO(self.get_file(s3_path)["Body"].read())
 
-    def upload_file(self, local_path, s3_path):
+    def upload_file(self, local_path: str, s3_path: str):
         self.s3.Bucket(self.bucket).upload_file(local_path, s3_path)

datamarket/interfaces/proxy.py

@@ -58,15 +58,13 @@ class ProxyInterface:
         }
 
     def get_current_host_port(self):
-        hosts = self.config["hosts"].split(",") if isinstance(self.config["hosts"], str) else self.config["hosts"]
-        host_port_pairs = [hp.split(":") for hp in hosts]
+        host_port_pairs = [hp.split(":") for hp in self.config["hosts"]]
         current_host, current_port = host_port_pairs[self.current_index]
         self.current_index = (self.current_index + 1) % len(host_port_pairs)
         return current_host, current_port
 
     def get_random_host_port(self):
-        hosts = self.config["hosts"].split(",") if isinstance(self.config["hosts"], str) else self.config["hosts"]
-        host_port_pairs = [hp.split(":") for hp in hosts]
+        host_port_pairs = [hp.split(":") for hp in self.config["hosts"]]
         self.current_index = random.randint(0, len(host_port_pairs) - 1)  # noqa: S311
         return host_port_pairs[self.current_index]
 

{datamarket-0.9.0.dist-info → datamarket-0.9.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.0
+Version: 0.9.2
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: alchemy
 Provides-Extra: aws
 Provides-Extra: azure-storage-blob
 Provides-Extra: boto3
@@ -62,7 +63,7 @@ Provides-Extra: tqdm
 Provides-Extra: undetected-chromedriver
 Provides-Extra: unidecode
 Provides-Extra: xmltodict
-Requires-Dist: SQLAlchemy (==2.0.36)
+Requires-Dist: SQLAlchemy (==2.0.36) ; extra == "alchemy"
 Requires-Dist: azure-storage-blob (==12.23.1) ; extra == "azure-storage-blob"
 Requires-Dist: beautifulsoup4 (==4.12.3)
 Requires-Dist: boto3 (==1.35.53) ; extra == "boto3" or extra == "aws" or extra == "peerdb"

{datamarket-0.9.0.dist-info → datamarket-0.9.2.dist-info}/RECORD

@@ -1,12 +1,12 @@
 datamarket/__init__.py,sha256=FHS77P9qNewKMoN-p0FLEUEC60oWIYup1QkbJZP4ays,12
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/interfaces/alchemy.py,sha256=V8E1GtokxUNmrUftKTFkIpNoXaqJME7ACES2BY0znQM,4214
-datamarket/interfaces/aws.py,sha256=UztVuBn561DnU1AcjyJ16UAIS1BUD5HUxiQ4gc9EhtM,1968
+datamarket/interfaces/aws.py,sha256=UV7FPZNC1S66mqmUZ8jgQUe8y1G0B7YVJzGw_dq-FUQ,2281
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=9GQgiNBBK7njkv8ytHQaP9YLB9kI5vnUFA5gtz9J7As,1859
 datamarket/interfaces/nominatim.py,sha256=_gFJ04D-ju5xn3wuaGT5Pj5jhf4F5eINpxOpuQL_dIQ,3664
 datamarket/interfaces/peerdb.py,sha256=rNQ1-THcVvrej8BEPJs9zM4VfH5dlByafOIHYN9sB2A,21833
-datamarket/interfaces/proxy.py,sha256=8EJaW8zAMzUMIRLkdAcMkTO9qZXPIubE6vyB5ZXcRtU,3352
+datamarket/interfaces/proxy.py,sha256=updoOStKd8-nQBbxWbnD9eOt6HksnYi-5dQ0rEySf5M,3152
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/params/nominatim.py,sha256=pBYRfoBkkLBg2INbFymefmYSzaAVujQSpEro5c1hD_I,1143
@@ -17,7 +17,7 @@ datamarket/utils/main.py,sha256=agWVJ5ZFZjVrBNuMpnxN2F_edA3mMJop6dVHPBBkOqU,5775
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.9.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.0.dist-info/METADATA,sha256=-MsG-FOmTXHlmzyVzNXbvNwWET4xYr401XgjC77uNP8,6284
-datamarket-0.9.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-datamarket-0.9.0.dist-info/RECORD,,
+datamarket-0.9.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.2.dist-info/METADATA,sha256=GK7Pc3NbQ0Adn4_McbjFQ7cbwjpBLTpJjUCfOx3aUkw,6329
+datamarket-0.9.2.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+datamarket-0.9.2.dist-info/RECORD,,

{datamarket-0.9.0.dist-info → datamarket-0.9.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.0.1
+Generator: poetry-core 2.1.1
 Root-Is-Purelib: true
 Tag: py3-none-any