sibi-dst 0.3.52__tar.gz → 0.3.53__tar.gz

{sibi_dst-0.3.52 → sibi_dst-0.3.53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.52
+Version: 0.3.53
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -11,8 +11,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: complete
 Provides-Extra: df-helper
-Provides-Extra: geopy-helper
-Provides-Extra: osmnx-helper
+Provides-Extra: geospatial
 Requires-Dist: apache-airflow-client (>=2.10.0,<3.0.0)
 Requires-Dist: chardet (>=5.2.0,<6.0.0)
 Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
@@ -21,9 +20,9 @@ Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
 Requires-Dist: dask[complete] (>=2025.3.0,<2026.0.0)
 Requires-Dist: django (>=5.1.4,<6.0.0) ; extra == "df-helper" or extra == "complete"
 Requires-Dist: djangorestframework (>=3.15.2,<4.0.0) ; extra == "df-helper" or extra == "complete"
-Requires-Dist: folium (>=0.19.4,<0.20.0) ; extra == "osmnx-helper" or extra == "complete"
-Requires-Dist: geopandas (>=1.0.1,<2.0.0) ; extra == "osmnx-helper" or extra == "complete"
-Requires-Dist: geopy (>=2.4.1,<3.0.0) ; extra == "geopy-helper" or extra == "complete"
+Requires-Dist: folium (>=0.19.4,<0.20.0) ; extra == "geospatial" or extra == "complete"
+Requires-Dist: geopandas (>=1.0.1,<2.0.0) ; extra == "geospatial" or extra == "complete"
+Requires-Dist: geopy (>=2.4.1,<3.0.0) ; extra == "geospatial" or extra == "complete"
 Requires-Dist: gunicorn (>=23.0.0,<24.0.0)
 Requires-Dist: httpx (>=0.27.2,<0.28.0)
 Requires-Dist: ipython (>=8.29.0,<9.0.0)
@@ -31,7 +30,7 @@ Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: mysqlclient (>=2.2.6,<3.0.0) ; extra == "df-helper" or extra == "complete"
 Requires-Dist: nltk (>=3.9.1,<4.0.0)
 Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
-Requires-Dist: osmnx (>=2.0.1,<3.0.0) ; extra == "osmnx-helper" or extra == "complete"
+Requires-Dist: osmnx (>=2.0.1,<3.0.0) ; extra == "geospatial" or extra == "complete"
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: paramiko (>=3.5.0,<4.0.0)
 Requires-Dist: psutil (>=6.1.0,<7.0.0)

{sibi_dst-0.3.52 → sibi_dst-0.3.53}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.52"
+version = "0.3.53"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"
@@ -48,8 +48,7 @@ webdav4 = { extras = ["fsspec"], version = "^0.10.0" }
 
 [tool.poetry.extras]
 df_helper = ["django", "sqlalchemy", "sqlmodel", "djangorestframework", "pymysql", "psycopg2", "mysqlclient"]
-geopy_helper = ["geopy"]
-osmnx_helper = ["folium", "geopandas", "osmnx"]
+geospatial = ["geopy","folium", "geopandas", "osmnx"]
 complete = ["django", "sqlalchemy", "sqlmodel", "djangorestframework", "pymysql", "psycopg2", "mysqlclient", "geopy", "folium", "geopandas", "osmnx"]
 [build-system]
 requires = ["poetry-core"]

{sibi_dst-0.3.52 → sibi_dst-0.3.53}/sibi_dst/df_helper/_df_helper.py

@@ -74,7 +74,7 @@ class DfHelper:
     logger: Logger
     default_config: Dict = None
 
-    def __init__(self, backend='django_db', **kwargs):
+    def __init__(self, backend='sqlalchemy', **kwargs):
         # Ensure default_config is not shared across instances
         self.default_config = self.default_config or {}
         kwargs = {**self.default_config.copy(), **kwargs}
@@ -209,6 +209,7 @@ class DfHelper:
             else:
                 self.logger.debug("Regular asyncio run...")
                 return asyncio.run(self.__load_from_http(**options))
+        return dd.from_pandas(pd.DataFrame(), npartitions=1)
 
     def __load_from_sqlalchemy(self, **options):
         """

{sibi_dst-0.3.52 → sibi_dst-0.3.53}/sibi_dst/df_helper/_parquet_artifact.py

@@ -151,9 +151,21 @@ class ParquetArtifact(DfHelper):
                 'parquet_end_date': datetime.date.today().strftime('%Y-%m-%d')
             }
 
+        def custom_config():
+            try:
+                start_date = kwargs.pop('start_on')
+                end_date = kwargs.pop('end_on')
+            except KeyError:
+                raise ValueError("For period 'custom', you must provide 'start_on' in kwargs.")
+            return {
+                'parquet_start_date': start_date,
+                'parquet_end_date': end_date
+            }
+
         config_map = {
             'itd': itd_config,
-            'ytd': ytd_config
+            'ytd': ytd_config,
+            'custom': custom_config,
         }
 
         if period in config_map:

{sibi_dst-0.3.52 → sibi_dst-0.3.53}/sibi_dst/utils/data_wrapper.py

@@ -47,7 +47,8 @@ class DataWrapper:
                  show_progress: bool = False,
                  timeout: float = 60,
                  reference_date: datetime.date = None,
-                 custom_priority_map: Dict[str, int] = None):
+                 custom_priority_map: Dict[str, int] = None,
+                 max_threads: int = 3):
         self.dataclass = dataclass
         self.date_field = date_field
         self.data_path = self._ensure_forward_slash(data_path)
@@ -70,6 +71,7 @@ class DataWrapper:
         self.timeout = timeout
         self.reference_date = reference_date or datetime.date.today()
         self.priority_map = custom_priority_map or self.DEFAULT_PRIORITY_MAP
+        self.max_threads = max_threads
 
         self.start_date = self._convert_to_date(start_date)
         self.end_date = self._convert_to_date(end_date)
@@ -138,8 +140,9 @@ class DataWrapper:
 
         desc = f"Processing {self.dataclass.__name__}, task: {self._priority_label(priority)}"
         self.logger.info(f"Starting {desc.lower()}")
-
-        with ThreadPoolExecutor() as executor:
+        max_threads = min(len(dates), self.max_threads)
+        self.logger.info(f"DataWrapper Max threads set at: {max_threads}")
+        with ThreadPoolExecutor(max_workers=max_threads) as executor:
             futures = {
                 executor.submit(self._process_date_with_retry, date, max_retries): date
                 for date in dates

{sibi_dst-0.3.52 → sibi_dst-0.3.53}/sibi_dst/utils/parquet_saver.py

@@ -1,15 +1,10 @@
 import base64
+import hashlib
 import logging
-from pathlib import Path
+import warnings
 from typing import Optional
 
 import pyarrow as pa
-import fsspec
-import warnings
-import hashlib
-from s3fs import S3FileSystem
-
-from fsspec import filesystem
 
 # Suppress the specific UserWarning message
 warnings.filterwarnings("ignore")