datamarket 0.9.2__py3-none-any.whl → 0.9.3__py3-none-any.whl

datamarket/interfaces/aws.py

@@ -5,6 +5,8 @@ import io
 import logging
 import boto3
 
+from ..utils.main import Config
+
 ########################################################################################################################
 # CLASSES
 
@@ -12,7 +14,7 @@ logger = logging.getLogger(__name__)
 
 
 class AWSInterface:
-    def __init__(self, config):
+    def __init__(self, config: Config) -> None:
         self.profiles = []
         self.config = config
 
@@ -31,13 +33,13 @@ class AWSInterface:
         self.current_profile = self.profiles[0] if self.profiles else None
         self._update_resources()
 
-    def _update_resources(self):
+    def _update_resources(self) -> None:
         if self.current_profile:
             self.s3 = self.current_profile["session"].resource("s3")
             self.s3_client = self.s3.meta.client
             self.bucket = self.current_profile["buckets"][0]
 
-    def switch_profile(self, profile_name: str):
+    def switch_profile(self, profile_name: str) -> None:
         for profile in self.profiles:
             if profile["profile"] == profile_name:
                 self.current_profile = profile
@@ -45,7 +47,7 @@ class AWSInterface:
                 return
         logger.warning(f"Profile {profile_name} not found")
 
-    def switch_bucket(self, bucket: str):
+    def switch_bucket(self, bucket: str) -> None:
         if bucket not in self.current_profile["buckets"]:
             logger.warning(
                 f"Bucket {bucket} not found in profile {self.current_profile['profile']}"
@@ -54,14 +56,14 @@ class AWSInterface:
 
         self.bucket = bucket
 
-    def get_file(self, s3_path: str):
+    def get_file(self, s3_path: str) -> None:
         try:
             return self.s3.Object(self.bucket, s3_path).get()
         except self.s3_client.exceptions.NoSuchKey:
             logger.info(f"{s3_path} does not exist")
 
-    def read_file_as_bytes(self, s3_path: str):
+    def read_file_as_bytes(self, s3_path: str) -> io.BytesIO:
         return io.BytesIO(self.get_file(s3_path)["Body"].read())
 
-    def upload_file(self, local_path: str, s3_path: str):
+    def upload_file(self, local_path: str, s3_path: str) -> None:
         self.s3.Bucket(self.bucket).upload_file(local_path, s3_path)

datamarket/interfaces/ftp.py

@@ -5,6 +5,8 @@ import logging
 from ftplib import FTP, FTP_TLS
 from pathlib import Path
 
+from ..utils.main import Config
+
 ########################################################################################################################
 # CLASSES
 
@@ -12,7 +14,7 @@ logger = logging.getLogger(__name__)
 
 
 class FTPInterface:
-    def __init__(self, config):
+    def __init__(self, config: Config):
         if "ftp" in config:
             self.config = config["ftp"]
 

datamarket/utils/main.py

@@ -2,7 +2,6 @@
 # IMPORTS
 
 import asyncio
-import configparser
 import inspect
 import logging
 import random
@@ -16,6 +15,7 @@ from typing import Literal, Union
 
 import pendulum
 from croniter import croniter
+from configparser import RawConfigParser
 from dynaconf import Dynaconf, add_converter
 
 ########################################################################################################################
@@ -23,6 +23,8 @@ from dynaconf import Dynaconf, add_converter
 
 logger = logging.getLogger(__name__)
 
+Config = Union[RawConfigParser, Dynaconf]
+
 
 def get_granular_date(
     granularity: Union[Literal["monthly", "biweekly", "weekly", "daily"], str], tz: str = "Europe/Madrid"
@@ -64,16 +66,20 @@ def read_converter(path_str: str):
         return f.read()
 
 
-def get_config(config_file: Path, tz: str = "Europe/Madrid"):
+def get_config(
+    config_file: Path, tz: str = "Europe/Madrid"
+) -> Union[RawConfigParser, Dynaconf]:
     if Path(config_file).suffix == ".ini":
         logger.warning("Using legacy INI config reader. Please migrate to TOML")
-        cfg = configparser.RawConfigParser()
+        cfg = RawConfigParser()
         cfg.read(config_file)
         return cfg
 
     add_converter("read", read_converter)
 
     dt_now = get_granular_date("now", tz)
+    dt_weekly = get_granular_date("weekly", tz)
+    dt_biweekly = get_granular_date("biweekly", tz)
 
     config = Dynaconf(
         environments=True,
@@ -84,14 +90,17 @@ def get_config(config_file: Path, tz: str = "Europe/Madrid"):
     config.load_file(path=Path.home() / config_file.name)
 
     config.vars = {
-        "now": dt_now.strftime("%Y-%m-%d %H:%M:%S"),
-        "today": dt_now.strftime("%Y-%m-%d"),
         "year": dt_now.strftime("%Y"),
         "month": dt_now.strftime("%m"),
         "day": dt_now.strftime("%d"),
-        "biweekly_date": get_granular_date("biweekly", tz).strftime("%Y-%m-%d"),
-        "today_stripped": dt_now.strftime("%Y%m%d"),
+        "now": dt_now.strftime("%Y-%m-%d %H:%M:%S"),
         "now_stripped": dt_now.strftime("%Y%m%d%H%M%S"),
+        "today": dt_now.strftime("%Y-%m-%d"),
+        "today_stripped": dt_now.strftime("%Y%m%d"),
+        "weekly_date": dt_weekly.strftime("%Y-%m-%d"),
+        "weekly_date_stripped": dt_weekly.strftime("%Y%m%d"),
+        "biweekly_date": dt_biweekly.strftime("%Y-%m-%d"),
+        "biweekly_date_stripped": dt_biweekly.strftime("%Y%m%d"),
         "dynaconf_merge": True,
     }
 

{datamarket-0.9.2.dist-info → datamarket-0.9.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.2
+Version: 0.9.3
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.2.dist-info → datamarket-0.9.3.dist-info}/RECORD

@@ -1,9 +1,9 @@
 datamarket/__init__.py,sha256=FHS77P9qNewKMoN-p0FLEUEC60oWIYup1QkbJZP4ays,12
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/interfaces/alchemy.py,sha256=V8E1GtokxUNmrUftKTFkIpNoXaqJME7ACES2BY0znQM,4214
-datamarket/interfaces/aws.py,sha256=UV7FPZNC1S66mqmUZ8jgQUe8y1G0B7YVJzGw_dq-FUQ,2281
+datamarket/interfaces/aws.py,sha256=R6lYdSCD6a4g9l6aFMtNDt_EX3kroe2untDhgy7XG1k,2384
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
-datamarket/interfaces/ftp.py,sha256=9GQgiNBBK7njkv8ytHQaP9YLB9kI5vnUFA5gtz9J7As,1859
+datamarket/interfaces/ftp.py,sha256=Owk3D7tiF47_ZFT3Dc9h4_BaPsWtcJUbhagjpQB19q8,1900
 datamarket/interfaces/nominatim.py,sha256=_gFJ04D-ju5xn3wuaGT5Pj5jhf4F5eINpxOpuQL_dIQ,3664
 datamarket/interfaces/peerdb.py,sha256=rNQ1-THcVvrej8BEPJs9zM4VfH5dlByafOIHYN9sB2A,21833
 datamarket/interfaces/proxy.py,sha256=updoOStKd8-nQBbxWbnD9eOt6HksnYi-5dQ0rEySf5M,3152
@@ -13,11 +13,11 @@ datamarket/params/nominatim.py,sha256=pBYRfoBkkLBg2INbFymefmYSzaAVujQSpEro5c1hD_
 datamarket/utils/__init__.py,sha256=8D5a8oKgqd6WA1RUkiKCn4l_PVemtyuckxQut0vDHXM,20
 datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,795
 datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
-datamarket/utils/main.py,sha256=agWVJ5ZFZjVrBNuMpnxN2F_edA3mMJop6dVHPBBkOqU,5775
+datamarket/utils/main.py,sha256=0Abt3ww1VSPnX4AVKDcYzqDLAOEV_54iUHMLJfre2bg,6129
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.9.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.2.dist-info/METADATA,sha256=GK7Pc3NbQ0Adn4_McbjFQ7cbwjpBLTpJjUCfOx3aUkw,6329
-datamarket-0.9.2.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-datamarket-0.9.2.dist-info/RECORD,,
+datamarket-0.9.3.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.3.dist-info/METADATA,sha256=SY-94WLCqqxLVnPXVPzRQZ6SwP0I-HbqdCBp9Me9ySw,6329
+datamarket-0.9.3.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+datamarket-0.9.3.dist-info/RECORD,,