datamarket 0.7.21__py3-none-any.whl → 0.8.0__py3-none-any.whl

datamarket/interfaces/aws.py

@@ -16,16 +16,14 @@ class AWSInterface:
         self.profiles = []
         self.config = config
 
-        for section in self.config.sections():
-            if section.startswith("aws:"):
-                profile_name = section.split(":", 1)[1]
-                self.profiles.append(
-                    {
-                        "profile": profile_name,
-                        "bucket": self.config[section]["bucket"],
-                        "session": boto3.Session(profile_name=profile_name),
-                    }
-                )
+        for profile_name, values in self.config.get("aws", {}).items():
+            self.profiles.append(
+                {
+                    "profile": profile_name,
+                    "bucket": values["bucket"],
+                    "session": boto3.Session(profile_name=profile_name),
+                }
+            )
 
         if not self.profiles:
             logger.warning("No AWS profiles found in config file")

datamarket/utils/main.py

@@ -2,14 +2,20 @@
 # IMPORTS
 
 import configparser
+import inspect
 import logging
 import random
 import re
 import shlex
+import shutil
 import subprocess
 import time
+from pathlib import Path
+from typing import Literal, Union
 
 import pendulum
+from croniter import croniter
+from dynaconf import Dynaconf, add_converter
 
 ########################################################################################################################
 # FUNCTIONS
@@ -17,10 +23,77 @@ import pendulum
 logger = logging.getLogger(__name__)
 
 
-def get_config(config_path):
-    cfg = configparser.RawConfigParser()
-    cfg.read(config_path)
-    return cfg
+def get_granular_date(
+    granularity: Union[Literal["monthly", "biweekly", "weekly", "daily"], str], tz: str = "Europe/Madrid"
+) -> str:
+    """
+    Returns the most recent date based on the given granularity or a custom cron expression.
+
+    Args:
+        granularity: Either a predefined value ("monthly", "biweekly", "weekly") or a custom cron expression.
+        tz: Timezone to use for date calculations (default: "Europe/Madrid").
+
+    Returns:
+        A string representing the most recent date in the format "YYYY-MM-DD".
+
+    Raises:
+        ValueError: If the provided granularity or cron expression is invalid.
+    """
+    now = pendulum.now(tz)
+
+    predefined_patterns = {
+        "monthly": "0 0 1 * *",
+        "biweekly": "0 0 1,15 * *",
+        "weekly": "0 0 * * MON",
+        "daily": "0 0 * * *",
+    }
+
+    cron_pattern = predefined_patterns.get(granularity, granularity)
+
+    try:
+        cron = croniter(cron_pattern, now)
+        return cron.get_prev(pendulum.DateTime).strftime("%Y-%m-%d")
+    except Exception as e:
+        raise ValueError("Invalid cron expression or granularity specified.") from e
+
+
+def read_converter(path_str: str):
+    with open(path_str) as f:
+        return f.read()
+
+
+def get_config(config_file: Path, tz: str = "Europe/Madrid"):
+    if Path(config_file).suffix == "ini":
+        logger.warning("Using legacy INI config reader. Please migrate to TOML")
+        cfg = configparser.RawConfigParser()
+        return cfg.read(config_file)
+
+    add_converter("read", read_converter)
+
+    config = Dynaconf(
+        environments=True,
+        env_switcher="SYSTYPE",
+        vars={
+            "today": get_granular_date("daily", tz),
+            "biweekly_date": get_granular_date("biweekly", tz),
+        },
+    )
+
+    config.load_file(path=config_file)
+    config.load_file(path=Path.home() / config_file.name)
+    return config
+
+
+def get_project_metadata():
+    caller_frame = inspect.stack()[1]
+    current_file_parts = Path(caller_frame.filename).resolve().parts
+    src_index = current_file_parts.index("src")
+    cmd_prefix = "dix vnc run --" if shutil.which("dix") else ""
+    pkg_name = current_file_parts[src_index + 1]
+    env_name = f"{pkg_name}_env"
+    project_path = Path(*current_file_parts[:src_index])
+
+    return {"cmd_prefix": cmd_prefix, "pkg_name": pkg_name, "env_name": env_name, "project_path": project_path}
 
 
 def set_logger(level):
@@ -34,7 +107,7 @@ def set_logger(level):
 
 
 def ban_sleep(max_time, min_time=0):
-    sleep_time = int(random.uniform(min_time, max_time))
+    sleep_time = int(random.uniform(min_time, max_time))  # noqa: S311
     logger.info(f"sleeping for {sleep_time} seconds...")
     time.sleep(sleep_time)
 

{datamarket-0.7.21.dist-info → datamarket-0.8.0.dist-info}/METADATA

@@ -1,8 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: datamarket
-Version: 0.7.21
+Version: 0.8.0
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
-Home-page: https://datamarket.es
 License: GPL-3.0-or-later
 Author: DataMarket
 Author-email: techsupport@datamarket.es
@@ -22,7 +21,6 @@ Provides-Extra: boto3
 Provides-Extra: chompjs
 Provides-Extra: click
 Provides-Extra: clickhouse-driver
-Provides-Extra: croniter
 Provides-Extra: datetime
 Provides-Extra: demjson3
 Provides-Extra: dnspython
@@ -44,7 +42,6 @@ Provides-Extra: openpyxl
 Provides-Extra: pandas
 Provides-Extra: pandera
 Provides-Extra: peerdb
-Provides-Extra: pendulum
 Provides-Extra: pillow
 Provides-Extra: playwright
 Provides-Extra: playwright-stealth
@@ -71,11 +68,12 @@ Requires-Dist: boto3 (==1.35.53) ; extra == "boto3" or extra == "aws" or extra =
 Requires-Dist: chompjs (==1.3.0) ; extra == "chompjs"
 Requires-Dist: click (==8.1.7) ; extra == "click"
 Requires-Dist: clickhouse-driver (==0.2.9) ; extra == "clickhouse-driver" or extra == "peerdb"
-Requires-Dist: croniter (==3.0.4) ; extra == "croniter"
+Requires-Dist: croniter (==3.0.4)
 Requires-Dist: datetime (==5.5) ; extra == "datetime"
 Requires-Dist: demjson3 (==3.0.6) ; extra == "demjson3"
 Requires-Dist: dnspython (==2.7.0) ; extra == "dnspython"
 Requires-Dist: duckduckgo-search (==6.2.11b1) ; extra == "duckduckgo-search"
+Requires-Dist: dynaconf (==3.2.6)
 Requires-Dist: fake-useragent (==1.5.1) ; extra == "fake-useragent"
 Requires-Dist: geoalchemy2 (==0.15.2) ; extra == "geoalchemy2"
 Requires-Dist: geopandas (==1.0.1) ; extra == "geopandas"
@@ -85,13 +83,14 @@ Requires-Dist: google-auth-httplib2 (==0.2.0) ; extra == "google-auth-httplib2"
 Requires-Dist: google-auth-oauthlib (==1.2.1) ; extra == "google-auth-oauthlib"
 Requires-Dist: html2text (==2024.2.26) ; extra == "html2text"
 Requires-Dist: httpx[http2] (==0.28.1) ; extra == "httpx"
+Requires-Dist: jinja2 (==3.1.5)
 Requires-Dist: json5 (==0.9.25) ; extra == "json5"
 Requires-Dist: lxml[html-clean] (==5.3.0) ; extra == "lxml"
 Requires-Dist: nodriver (==0.37) ; extra == "nodriver"
 Requires-Dist: openpyxl (==3.1.5) ; extra == "openpyxl"
 Requires-Dist: pandas (==2.2.3) ; extra == "pandas"
 Requires-Dist: pandera (==0.20.4) ; extra == "pandera"
-Requires-Dist: pendulum (==3.0.0) ; extra == "pendulum"
+Requires-Dist: pendulum (==3.0.0)
 Requires-Dist: pillow (==11.0.0) ; extra == "pillow"
 Requires-Dist: playwright (==1.47.0) ; extra == "playwright"
 Requires-Dist: playwright-stealth (==1.0.6) ; extra == "playwright-stealth"
@@ -116,6 +115,7 @@ Requires-Dist: undetected-chromedriver (==3.5.5) ; extra == "undetected-chromedr
 Requires-Dist: unidecode (==1.3.8) ; extra == "unidecode"
 Requires-Dist: xmltodict (==0.14.2) ; extra == "xmltodict"
 Project-URL: Documentation, https://github.com/Data-Market/datamarket
+Project-URL: Homepage, https://datamarket.es
 Project-URL: Repository, https://github.com/Data-Market/datamarket
 Description-Content-Type: text/markdown
 

{datamarket-0.7.21.dist-info → datamarket-0.8.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
 datamarket/__init__.py,sha256=FHS77P9qNewKMoN-p0FLEUEC60oWIYup1QkbJZP4ays,12
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/interfaces/alchemy.py,sha256=V8E1GtokxUNmrUftKTFkIpNoXaqJME7ACES2BY0znQM,4214
-datamarket/interfaces/aws.py,sha256=Mk9h-UcdbyNPUaYaG9wlKdd0R95xzTwtX1-_PgsBkjo,2084
+datamarket/interfaces/aws.py,sha256=UztVuBn561DnU1AcjyJ16UAIS1BUD5HUxiQ4gc9EhtM,1968
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=9GQgiNBBK7njkv8ytHQaP9YLB9kI5vnUFA5gtz9J7As,1859
 datamarket/interfaces/nominatim.py,sha256=WkPXaug-oH5zJkuE6aXMu4-MEkGYIY7S6TekfZ2FnHY,3658
@@ -13,11 +13,11 @@ datamarket/params/nominatim.py,sha256=pBYRfoBkkLBg2INbFymefmYSzaAVujQSpEro5c1hD_
 datamarket/utils/__init__.py,sha256=8D5a8oKgqd6WA1RUkiKCn4l_PVemtyuckxQut0vDHXM,20
 datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,795
 datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
-datamarket/utils/main.py,sha256=jumvHgbPiUNUTGZ2ebGcGhV6wPWV1BHig6Wranxmn4w,2696
+datamarket/utils/main.py,sha256=z6gbwR5RhFXYFLkkfCRk14DQsjvSnmJ_GDRd0G5PKgg,5144
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.7.21.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.7.21.dist-info/METADATA,sha256=ecszHimSeE45_gqA5PiBAuzMmDa55cZCr2R7NJfP4bU,6193
-datamarket-0.7.21.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-datamarket-0.7.21.dist-info/RECORD,,
+datamarket-0.8.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.8.0.dist-info/METADATA,sha256=1dr2cvGcPu3WVR-lAkWkoHRtQ31eS9uImDwvLtWTi0Q,6176
+datamarket-0.8.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+datamarket-0.8.0.dist-info/RECORD,,

{datamarket-0.7.21.dist-info → datamarket-0.8.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.1
+Generator: poetry-core 2.0.1
 Root-Is-Purelib: true
 Tag: py3-none-any