sibi-dst 0.3.52__py3-none-any.whl → 0.3.53__py3-none-any.whl

sibi_dst/df_helper/_df_helper.py

@@ -74,7 +74,7 @@ class DfHelper:
     logger: Logger
     default_config: Dict = None
 
-    def __init__(self, backend='django_db', **kwargs):
+    def __init__(self, backend='sqlalchemy', **kwargs):
         # Ensure default_config is not shared across instances
         self.default_config = self.default_config or {}
         kwargs = {**self.default_config.copy(), **kwargs}
@@ -209,6 +209,7 @@ class DfHelper:
             else:
                 self.logger.debug("Regular asyncio run...")
                 return asyncio.run(self.__load_from_http(**options))
+        return dd.from_pandas(pd.DataFrame(), npartitions=1)
 
     def __load_from_sqlalchemy(self, **options):
         """

sibi_dst/df_helper/_parquet_artifact.py

@@ -151,9 +151,21 @@ class ParquetArtifact(DfHelper):
                 'parquet_end_date': datetime.date.today().strftime('%Y-%m-%d')
             }
 
+        def custom_config():
+            try:
+                start_date = kwargs.pop('start_on')
+                end_date = kwargs.pop('end_on')
+            except KeyError:
+                raise ValueError("For period 'custom', you must provide 'start_on' in kwargs.")
+            return {
+                'parquet_start_date': start_date,
+                'parquet_end_date': end_date
+            }
+
         config_map = {
             'itd': itd_config,
-            'ytd': ytd_config
+            'ytd': ytd_config,
+            'custom': custom_config,
         }
 
         if period in config_map:

sibi_dst/utils/data_wrapper.py

@@ -47,7 +47,8 @@ class DataWrapper:
                  show_progress: bool = False,
                  timeout: float = 60,
                  reference_date: datetime.date = None,
-                 custom_priority_map: Dict[str, int] = None):
+                 custom_priority_map: Dict[str, int] = None,
+                 max_threads: int = 3):
         self.dataclass = dataclass
         self.date_field = date_field
         self.data_path = self._ensure_forward_slash(data_path)
@@ -70,6 +71,7 @@ class DataWrapper:
         self.timeout = timeout
         self.reference_date = reference_date or datetime.date.today()
         self.priority_map = custom_priority_map or self.DEFAULT_PRIORITY_MAP
+        self.max_threads = max_threads
 
         self.start_date = self._convert_to_date(start_date)
         self.end_date = self._convert_to_date(end_date)
@@ -138,8 +140,9 @@ class DataWrapper:
 
         desc = f"Processing {self.dataclass.__name__}, task: {self._priority_label(priority)}"
         self.logger.info(f"Starting {desc.lower()}")
-
-        with ThreadPoolExecutor() as executor:
+        max_threads = min(len(dates), self.max_threads)
+        self.logger.info(f"DataWrapper Max threads set at: {max_threads}")
+        with ThreadPoolExecutor(max_workers=max_threads) as executor:
             futures = {
                 executor.submit(self._process_date_with_retry, date, max_retries): date
                 for date in dates

sibi_dst/utils/parquet_saver.py

@@ -1,15 +1,10 @@
 import base64
+import hashlib
 import logging
-from pathlib import Path
+import warnings
 from typing import Optional
 
 import pyarrow as pa
-import fsspec
-import warnings
-import hashlib
-from s3fs import S3FileSystem
-
-from fsspec import filesystem
 
 # Suppress the specific UserWarning message
 warnings.filterwarnings("ignore")

{sibi_dst-0.3.52.dist-info → sibi_dst-0.3.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.52
+Version: 0.3.53
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -11,8 +11,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: complete
 Provides-Extra: df-helper
-Provides-Extra: geopy-helper
-Provides-Extra: osmnx-helper
+Provides-Extra: geospatial
 Requires-Dist: apache-airflow-client (>=2.10.0,<3.0.0)
 Requires-Dist: chardet (>=5.2.0,<6.0.0)
 Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
@@ -21,9 +20,9 @@ Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
 Requires-Dist: dask[complete] (>=2025.3.0,<2026.0.0)
 Requires-Dist: django (>=5.1.4,<6.0.0) ; extra == "df-helper" or extra == "complete"
 Requires-Dist: djangorestframework (>=3.15.2,<4.0.0) ; extra == "df-helper" or extra == "complete"
-Requires-Dist: folium (>=0.19.4,<0.20.0) ; extra == "osmnx-helper" or extra == "complete"
-Requires-Dist: geopandas (>=1.0.1,<2.0.0) ; extra == "osmnx-helper" or extra == "complete"
-Requires-Dist: geopy (>=2.4.1,<3.0.0) ; extra == "geopy-helper" or extra == "complete"
+Requires-Dist: folium (>=0.19.4,<0.20.0) ; extra == "geospatial" or extra == "complete"
+Requires-Dist: geopandas (>=1.0.1,<2.0.0) ; extra == "geospatial" or extra == "complete"
+Requires-Dist: geopy (>=2.4.1,<3.0.0) ; extra == "geospatial" or extra == "complete"
 Requires-Dist: gunicorn (>=23.0.0,<24.0.0)
 Requires-Dist: httpx (>=0.27.2,<0.28.0)
 Requires-Dist: ipython (>=8.29.0,<9.0.0)
@@ -31,7 +30,7 @@ Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: mysqlclient (>=2.2.6,<3.0.0) ; extra == "df-helper" or extra == "complete"
 Requires-Dist: nltk (>=3.9.1,<4.0.0)
 Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
-Requires-Dist: osmnx (>=2.0.1,<3.0.0) ; extra == "osmnx-helper" or extra == "complete"
+Requires-Dist: osmnx (>=2.0.1,<3.0.0) ; extra == "geospatial" or extra == "complete"
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: paramiko (>=3.5.0,<4.0.0)
 Requires-Dist: psutil (>=6.1.0,<7.0.0)

{sibi_dst-0.3.52.dist-info → sibi_dst-0.3.53.dist-info}/RECORD

@@ -1,8 +1,8 @@
 sibi_dst/__init__.py,sha256=3pbriM7Ym5f9gew7n9cO4G_p9n-0bnxdmQ0hwBdJjr4,253
 sibi_dst/df_helper/__init__.py,sha256=McYrw2N0MsMgtawLrONXTGdyHfQWVOBUvIDbklfjb54,342
 sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=toH2QvNF-CQNJ4Bc8xreytuWr37G0EWz4ciWVdFMVqU,11646
-sibi_dst/df_helper/_df_helper.py,sha256=IS1m9r9U-eJ7EVMBqmITmre6S0JfIl6nJtPIwNI3xKY,29771
-sibi_dst/df_helper/_parquet_artifact.py,sha256=qt3WZXrE2EZs3KI0biGzm3znIZazzTh8fgiipCVr_Ic,10196
+sibi_dst/df_helper/_df_helper.py,sha256=D85n4oUdu92IN2QaPc6k9uJJ_Vm197me1aoHojuWEYs,29833
+sibi_dst/df_helper/_parquet_artifact.py,sha256=dDoR5Tq0EJViW52tD9XW_vno7hkOfA8WeAilR1mAb_g,10636
 sibi_dst/df_helper/_parquet_reader.py,sha256=L6mr2FeKtTeIn37G9EGpvOx8PwMqXb6qnEECqBaiwxo,3954
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/df_helper/backends/django/__init__.py,sha256=uWHi-DtQX5re7b2HcqoXUH3_FZWOw1VTmDf552FAkNs,256
@@ -44,13 +44,13 @@ sibi_dst/utils/clickhouse_writer.py,sha256=iAUe4_Kn2WR1xZjpLW2FOWCWfOTw6fCGMTUcW
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
-sibi_dst/utils/data_wrapper.py,sha256=pIIQxeHknUeQd0YbISkAhL-xYBK4OdijoATBY-oBznw,12114
+sibi_dst/utils/data_wrapper.py,sha256=OMmdjnUCjxDsC1T0K1JbzxKXLTiDOtxPp1ujV66bMAY,12345
 sibi_dst/utils/date_utils.py,sha256=OCJqkWl5e8fE7z11Ufz4206DUeuLMd_Gf_JGZu914Pg,18539
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
 sibi_dst/utils/filepath_generator.py,sha256=-HHO0U-PR8fysDDFwnWdHRlgqksh_RkmgBZLWv9hM7s,6669
 sibi_dst/utils/log_utils.py,sha256=eSAbi_jmMpJ8RpycakzT4S4zNkqVZDj3FY8WwnxpdXc,4623
-sibi_dst/utils/parquet_saver.py,sha256=Tucxv9jRX66VuLQZn0dPQBN7JOttBou6SF8FxqufeGE,8169
+sibi_dst/utils/parquet_saver.py,sha256=EBtd9blzk7Wb65aDBVVU0ZMHFtqjfWi_fCUt1LvyAC4,8069
 sibi_dst/utils/phone_formatter.py,sha256=tsVTDamuthFYgy4-5UwmQkPQ-FGTGH7MjZyH8utAkIY,4945
 sibi_dst/utils/storage_config.py,sha256=Cg8EOGLZ_5v9sunaQHZLYHdp5FDkgPrCVVNHF-ys5sQ,2181
 sibi_dst/utils/storage_manager.py,sha256=btecX7ggNb7rfu5EK9Xuu2q_FZA7r_rB_tfhQ8V96qc,6567
@@ -75,6 +75,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-0.3.52.dist-info/METADATA,sha256=VIiTzkdO-FsYBkie1cfuwFKEwFFSE5TL5Ory-VzPa0g,7221
-sibi_dst-0.3.52.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-0.3.52.dist-info/RECORD,,
+sibi_dst-0.3.53.dist-info/METADATA,sha256=rNy87i3acBUu27BdqZ-J73e9oy6pd8YEKCX-HrE48ig,7182
+sibi_dst-0.3.53.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.53.dist-info/RECORD,,