datamarket 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

datamarket/interfaces/proxy.py

@@ -2,8 +2,8 @@
 # IMPORTS
 
 import logging
-import time
 import random
+import time
 
 import requests
 from stem import Signal
@@ -45,7 +45,7 @@ class ProxyInterface:
             proxy_url = self.get_proxy_url("127.0.0.1", 9050)
         else:
             current_host, current_port = self.get_random_host_port() if randomize else self.get_current_host_port()
-            
+
             user = self.config.get("user")
             password = self.config.get("password")
             use_socks = self.config.get("socks", "false").lower() == "true"
@@ -67,14 +67,12 @@ class ProxyInterface:
     def get_random_host_port(self):
         hosts = self.config["hosts"].split(",") if isinstance(self.config["hosts"], str) else self.config["hosts"]
         host_port_pairs = [hp.split(":") for hp in hosts]
-        self.current_index = random.randint(0, len(host_port_pairs) - 1)
+        self.current_index = random.randint(0, len(host_port_pairs) - 1)  # noqa: S311
         return host_port_pairs[self.current_index]
 
     def check_current_ip(self):
         try:
-            return requests.get(self.CHECK_IP_URL, proxies=self.proxies).json()[
-                "YourFuckingIPAddress"
-            ]
+            return requests.get(self.CHECK_IP_URL, proxies=self.proxies, timeout=30).json()["YourFuckingIPAddress",]
         except Exception as ex:
             logger.error(ex)
 

datamarket/utils/main.py

@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 def get_granular_date(
     granularity: Union[Literal["monthly", "biweekly", "weekly", "daily"], str], tz: str = "Europe/Madrid"
-) -> str:
+) -> pendulum.DateTime:
     """
     Returns the most recent date based on the given granularity or a custom cron expression.
 
@@ -52,7 +52,7 @@ def get_granular_date(
 
     try:
         cron = croniter(cron_pattern, now)
-        return cron.get_prev(pendulum.DateTime).strftime("%Y-%m-%d")
+        return cron.get_prev(pendulum.DateTime)
     except Exception as e:
         raise ValueError("Invalid cron expression or granularity specified.") from e
 
@@ -63,19 +63,28 @@ def read_converter(path_str: str):
 
 
 def get_config(config_file: Path, tz: str = "Europe/Madrid"):
-    if Path(config_file).suffix == "ini":
+    if Path(config_file).suffix == ".ini":
         logger.warning("Using legacy INI config reader. Please migrate to TOML")
         cfg = configparser.RawConfigParser()
         return cfg.read(config_file)
 
     add_converter("read", read_converter)
 
+    today = get_granular_date("daily", tz)
+
     config = Dynaconf(
         environments=True,
         env_switcher="SYSTYPE",
         vars={
-            "today": get_granular_date("daily", tz),
-            "biweekly_date": get_granular_date("biweekly", tz),
+            "now": today.strftime("%Y-%m-%d %H:%M:%S"),
+            "today": today.strftime("%Y-%m-%d"),
+            "year": today.strftime("%Y"),
+            "month": today.strftime("%m"),
+            "day": today.strftime("%d"),
+            "biweekly_date": get_granular_date("biweekly", tz).strftime("%Y-%m-%d"),
+            "today_stripped": today.strftime("%Y%m%d"),
+            "now_stripped": today.strftime("%Y%m%d%H%M%S"),
+            "dynaconf_merge": True,
         },
     )
 

{datamarket-0.8.0.dist-info → datamarket-0.8.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.8.0
+Version: 0.8.2
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.8.0.dist-info → datamarket-0.8.2.dist-info}/RECORD

@@ -6,18 +6,18 @@ datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIg
 datamarket/interfaces/ftp.py,sha256=9GQgiNBBK7njkv8ytHQaP9YLB9kI5vnUFA5gtz9J7As,1859
 datamarket/interfaces/nominatim.py,sha256=WkPXaug-oH5zJkuE6aXMu4-MEkGYIY7S6TekfZ2FnHY,3658
 datamarket/interfaces/peerdb.py,sha256=kJ3hdzGEACTfA_HM1b_I2rExY0x6ijCq5eOP1WObpLc,21821
-datamarket/interfaces/proxy.py,sha256=vZ42V4zXBWkW7dTGpDvBMNL45MCHe1ZrYrF8xIY25GU,3367
+datamarket/interfaces/proxy.py,sha256=8EJaW8zAMzUMIRLkdAcMkTO9qZXPIubE6vyB5ZXcRtU,3352
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/params/nominatim.py,sha256=pBYRfoBkkLBg2INbFymefmYSzaAVujQSpEro5c1hD_I,1143
 datamarket/utils/__init__.py,sha256=8D5a8oKgqd6WA1RUkiKCn4l_PVemtyuckxQut0vDHXM,20
 datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,795
 datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
-datamarket/utils/main.py,sha256=z6gbwR5RhFXYFLkkfCRk14DQsjvSnmJ_GDRd0G5PKgg,5144
+datamarket/utils/main.py,sha256=X5m9OMu8XnYYMEnQxvj8PWrT5MxJCftNv48BwDCfk1w,5533
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.8.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.8.0.dist-info/METADATA,sha256=1dr2cvGcPu3WVR-lAkWkoHRtQ31eS9uImDwvLtWTi0Q,6176
-datamarket-0.8.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-datamarket-0.8.0.dist-info/RECORD,,
+datamarket-0.8.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.8.2.dist-info/METADATA,sha256=AlcJCKZNiKtQ3S2us0-EkYuKa0Nokyz64O2ZtOu3j3w,6176
+datamarket-0.8.2.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+datamarket-0.8.2.dist-info/RECORD,,