sibi-dst 2025.1.9__py3-none-any.whl → 2025.1.10__py3-none-any.whl

sibi_dst/df_helper/_artifact_updater_multi_wrapper.py

@@ -49,6 +49,7 @@ class ArtifactUpdaterMultiWrapperThreaded(ManagedResource):
         self.completion_times: Dict[str, float] = {}
         self.failed: List[str] = []
         self.original_classes: List[Type] = []
+        self.logger.info("ArtifactUpdaterMultiWrapperThreaded initialized")
 
     def get_artifact_classes(self, data_type: str) -> List[Type]:
         """Retrieve artifact classes by data type."""
@@ -270,6 +271,7 @@ class ArtifactUpdaterMultiWrapperAsync(ManagedResource):
         self.completion_times: Dict[str, float] = {}
         self.failed: List[str] = []
         self.original_classes: List[Type] = []
+        self.logger.info("ArtifactUpdaterMultiWrapperAsync initialized")
 
     def get_artifact_classes(self, data_type: str) -> List[Type]:
         """

sibi_dst/df_helper/_df_helper.py

@@ -28,6 +28,7 @@ class BaseBackend:
         self.logger = helper.logger
         self.debug = helper.debug
         self.total_records = helper.total_records  # no records loaded yet
+        self._entered = helper._entered  # Track if the helper is used in a context manager
 
     def load(self, **options) -> tuple[Any, Any] | Union[dd.DataFrame | pd.DataFrame]:
         """Synchronous data loading method. Must be implemented by sync backends."""
@@ -67,7 +68,7 @@ class ParquetBackend(BaseBackend):
             df = self.helper.backend_parquet.load_files()
             if options and df is not None:
                 df = FilterHandler('dask', logger=self.logger, debug=False).apply_filters(df, filters=options)
-            self.total_records = len(df)
+            self.total_records = len(df) or -1  # If df is empty, set total_records to -1
             return self.total_records, df
         except Exception as e:
             self.total_records = -1  # Reset total_records on failure
@@ -105,6 +106,12 @@ class DfHelper(ManagedResource):
         'http': HttpBackend,
     }
 
+    _BACKEND_ATTR_MAP = {
+        'sqlalchemy': 'backend_db_connection',
+        'parquet': 'backend_parquet',
+        'http': 'backend_http',
+    }
+
     default_config: Dict = None
 
     def __init__(self, backend='sqlalchemy', **kwargs):
@@ -140,9 +147,15 @@ class DfHelper(ManagedResource):
         super().__exit__(exc_type, exc_value, traceback)
 
     def _cleanup(self):
-        active_config = getattr(self, f"backend_{self.backend}", None)
+        attr_name = self._BACKEND_ATTR_MAP.get(self.backend)
+        if not attr_name:
+            self.logger.warning(f"No attribute mapping found for backend '{self.backend}'. Cleanup skipped.")
+            return
+        # Get the actual config object (e.g., self.backend_db_connection)
+        active_config = getattr(self, attr_name, None)
+
         if active_config and hasattr(active_config, "close"):
-            self.logger.debug(f"Closing resources for '{self.backend}' backend.")
+            self.logger.debug(f"Closing resources for '{self.backend}' backend using attribute '{attr_name}'.")
             active_config.close()
 
     def _get_config(self, model: T, kwargs: Dict[str, Any]) -> T:
@@ -156,6 +169,10 @@ class DfHelper(ManagedResource):
         self.total_records, df = self.backend_strategy.load(**options)
         df = self._process_loaded_data(df)
         df = self._post_process_df(df)
+        if not self._entered:
+            self.logger.warning(
+                "DfHelper instance was not used in a context manager; cleanup is being called manually.")
+            self._cleanup()
         return df.compute() if as_pandas else df
 
     async def aload(self, as_pandas=False, **options) -> Union[pd.DataFrame, dd.DataFrame]:
@@ -200,7 +217,11 @@ class DfHelper(ManagedResource):
             self.logger.warning("Cannot save to parquet; DataFrame is empty.")
             return
         fs = kwargs.pop('fs', self.fs)
-        path = kwargs.pop('parquet_storage_path', self.backend_parquet.parquet_storage_path)
+        if not fs:
+            raise ValueError("Filesystem (fs) must be provided to save to parquet.")
+        path = kwargs.pop('parquet_storage_path', None)
+        if not path:
+            raise ValueError("parquet_storage_path must be provided to save to parquet.")
         writer_config = {
             'df_result': df,
             'parquet_storage_path': path,

sibi_dst/df_helper/backends/parquet/_parquet_options.py

@@ -4,8 +4,8 @@ from typing import Optional, List
 
 import dask.dataframe as dd
 import fsspec
-from pydantic import BaseModel, model_validator, DirectoryPath, FilePath, ConfigDict
-
+import pandas as pd
+from pydantic import BaseModel, model_validator, ConfigDict
 from sibi_dst.utils import FilePathGenerator
 from sibi_dst.utils import Logger
 
@@ -93,7 +93,7 @@ class ParquetConfig(BaseModel):
         self.parquet_storage_path = self.parquet_storage_path.rstrip('/')
         if not self.fs.exists(self.parquet_storage_path):
             self.fs.mkdirs(self.parquet_storage_path, exist_ok=True)
-            #raise ValueError('Parquet storage path does not exist')
+            # raise ValueError('Parquet storage path does not exist')
         self.load_parquet = False
         if self.parquet_filename is not None:
             self.parquet_full_path = self.ensure_file_extension(
@@ -184,11 +184,36 @@ class ParquetConfig(BaseModel):
         :return: A Dask DataFrame containing loaded parquet file data.
         :rtype: dask.dataframe.DataFrame
         """
-        if self.load_parquet:
-            if self.parquet_folder_list:
-                return dd.read_parquet(self.parquet_folder_list, engine="pyarrow", filesystem=self.fs)
-            else:
-                return dd.read_parquet(self.parquet_full_path, engine="pyarrow", filesystem=self.fs)
+        if not self.load_parquet:
+            self.logger.warning("Parquet loading is disabled. Returning empty DataFrame.")
+            return dd.from_pandas(pd.DataFrame(), npartitions=1)
+
+        paths_to_load = []
+        if self.parquet_folder_list:
+            # Filter out any None values from the list
+            paths_to_load = [p for p in self.parquet_folder_list if p is not None]
+        elif self.parquet_full_path:
+            # Treat the single path as a list with one item
+            paths_to_load = [self.parquet_full_path]
+
+        if not paths_to_load:
+            self.logger.warning("No valid parquet file paths were provided. Returning empty DataFrame.")
+            return dd.from_pandas(pd.DataFrame(), npartitions=1)
+
+        try:
+            self.logger.debug(f"Attempting to load Parquet data from: {paths_to_load}")
+            return dd.read_parquet(
+                paths_to_load,
+                engine="pyarrow",
+                filesystem=self.fs,
+                exclude=["_*", ".*"]
+            )
+        except Exception as e:
+            # This robust error handling is excellent.
+            self.logger.error(f"Parquet loading failed for paths {paths_to_load}: {e}", exc_info=True)
+            self.logger.warning("Returning empty DataFrame due to loading error.")
+            return dd.from_pandas(pd.DataFrame(), npartitions=1)
+
 
     @staticmethod
     def ensure_file_extension(filepath: str, extension: str) -> str:

sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py

@@ -172,7 +172,7 @@ class SqlAlchemyConnectionConfig(BaseModel):
                 return
 
             engine_wrapper['ref_count'] -= 1
-            self.logger.debug(f"Closing config. Ref count is now {engine_wrapper['ref_count']}.")
+            self.logger.debug(f"Closing connection within engine wrapper. Ref count is now {engine_wrapper['ref_count']}.")
 
             if engine_wrapper['ref_count'] <= 0:
                 self.logger.debug(f"Disposing engine as reference count is zero. Key: {key}")

sibi_dst/utils/data_wrapper.py

@@ -153,44 +153,44 @@ class DataWrapper(ManagedResource):
             # Create a copy to avoid mutating the shared instance dictionary
             local_load_params = self.load_params.copy()
             local_load_params.update(date_filter)
-            local_class_instance = self.dataclass(**self.class_params)
-            df = local_class_instance.load(**local_load_params)
-            load_time = time.perf_counter() - load_start
-
-            if hasattr(local_class_instance, "total_records"):
-                self.logger.debug(
-                    f"Total records loaded by {local_class_instance.__class__.__name__}: {local_class_instance.total_records}")
-                if int(local_class_instance.total_records) == 0:  # If no records were loaded but not due to an error
-                    if self.mmanifest:
-                        self.mmanifest.record(
+            with self.dataclass(**self.class_params) as local_class_instance:
+                df = local_class_instance.load(**local_load_params)
+                load_time = time.perf_counter() - load_start
+
+                if hasattr(local_class_instance, "total_records"):
+                    self.logger.debug(
+                        f"Total records loaded by {local_class_instance.__class__.__name__}: {local_class_instance.total_records}")
+                    if int(local_class_instance.total_records) == 0:  # If no records were loaded but not due to an error
+                        if self.mmanifest:
+                            self.mmanifest.record(
                             full_path=path
                         )
-                    self.logger.info(f"No data found for {full_path}. Logged to missing manifest.")
-                elif int(local_class_instance.total_records) < 0:
-                    self.logger.warning(
-                        f"Negative record count ({local_class_instance.total_records}) for {full_path}. "
-                        "This may indicate an error in the data loading process."
-                    )
-                else:
-                    save_start = time.perf_counter()
-                    parquet_params ={
-                        "df_result": df,
-                        "parquet_storage_path": path,
-                        "fs": self.fs,
-                        "logger": self.logger,
-                        "debug": self.debug,
-                    }
-                    with ParquetSaver(**parquet_params) as ps:
-                        ps.save_to_parquet(self.parquet_filename, overwrite=True)
-                    save_time = time.perf_counter() - save_start
-
-                    total_time = time.perf_counter() - overall_start
-                    self.benchmarks[date] = {
-                        "load_duration": load_time,
-                        "save_duration": save_time,
-                        "total_duration": total_time
-                    }
-                    self._log_success(date, total_time, full_path)
+                        self.logger.info(f"No data found for {full_path}. Logged to missing manifest.")
+                    elif int(local_class_instance.total_records) < 0:
+                        self.logger.warning(
+                            f"Negative record count ({local_class_instance.total_records}) for {full_path}. "
+                            "This may indicate an error in the data loading process."
+                        )
+                    else:
+                        save_start = time.perf_counter()
+                        parquet_params ={
+                            "df_result": df,
+                            "parquet_storage_path": path,
+                            "fs": self.fs,
+                            "logger": self.logger,
+                            "debug": self.debug,
+                        }
+                        with ParquetSaver(**parquet_params) as ps:
+                            ps.save_to_parquet(self.parquet_filename, overwrite=True)
+                        save_time = time.perf_counter() - save_start
+
+                        total_time = time.perf_counter() - overall_start
+                        self.benchmarks[date] = {
+                            "load_duration": load_time,
+                            "save_duration": save_time,
+                            "total_duration": total_time
+                        }
+                        self._log_success(date, total_time, full_path)
         except Exception as e:
             self._log_failure(date, e)
             raise

{sibi_dst-2025.1.9.dist-info → sibi_dst-2025.1.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.1.9
+Version: 2025.1.10
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-2025.1.9.dist-info → sibi_dst-2025.1.10.dist-info}/RECORD

@@ -1,7 +1,7 @@
 sibi_dst/__init__.py,sha256=j8lZpGCJlxlLgEgeIMxZnWdqJ0g3MCs7-gsnbvPn_KY,285
 sibi_dst/df_helper/__init__.py,sha256=Jur_MO8RGPkVw0CS3XH5YIWv-d922DC_FwRDTvHHV6Y,432
-sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=10EkCYEfoWwTQbS-ahYWo6TvbtNXM8p0UqqDu0gTuyI,17426
-sibi_dst/df_helper/_df_helper.py,sha256=iBoWz2iVgLzQ3hA1EwllL62dkraKamRx2sXseu30FVI,11914
+sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=pSSw3N_ZNZCZHAiChbsF_ECyCmz0L2xCgvt9srHtPOM,17575
+sibi_dst/df_helper/_df_helper.py,sha256=BbpP0BOLDGCOE8oAxqP5ODN_HqYohQcGsh-8Dx2-sks,12885
 sibi_dst/df_helper/_parquet_artifact.py,sha256=dCvUA2bytv0wY0pFI8lxbcLwXlgGpHndS36iKfEmjLw,14310
 sibi_dst/df_helper/_parquet_reader.py,sha256=m98C0TZRroOXvVc2LpEuElrJnquGlR81E1gjI7v1hi4,3102
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -9,9 +9,9 @@ sibi_dst/df_helper/backends/http/__init__.py,sha256=d1pfgYxbiYg7E0Iw8RbJ7xfqIfJS
 sibi_dst/df_helper/backends/http/_http_config.py,sha256=eGPFdqZ5M3Tscqx2P93B6XoBEEzlmdt7yNg7PXUQnNQ,4726
 sibi_dst/df_helper/backends/parquet/__init__.py,sha256=esWJ9aSuYC26d-T01z9dPrJ1uqJzvdaPNTYRb5qXTlQ,182
 sibi_dst/df_helper/backends/parquet/_filter_handler.py,sha256=TvDf0RXta7mwJv11GNQttYJsXgFf2XDj4oLIjt4xTzA,5219
-sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=lrDn2-BbgxDor5g71LAu5LDg2g3ApGAPiQfbFTB2xNA,10702
+sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=FWExRRTlhGrOhGPyzL1tucxgoHa3nJenLLs87I2gs-I,11776
 sibi_dst/df_helper/backends/sqlalchemy/__init__.py,sha256=LjWm9B7CweTvlvFOgB90XjSe0lVLILAIYMWKPkFXFm8,265
-sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=Rsvh1nfVtqzfMhv968vNTYYIqVxYsEs4PB-O5CTSYdk,10935
+sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=AOYvWw1vxd1CwXpIakQNFln7PHzFLfp7oaOsGaG0UN8,10961
 sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py,sha256=NqBSHqeYv_1vHt6J0tez0GdMwKrP_sIRcXYXu869ZkY,13313
 sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=ibxeVqpIEsSVusP2bgcd1MNV_wJIoNgXwacltUbwTas,3194
 sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py,sha256=d_-ip-dQnWOlM8btCjoywAXpaiSuN6AaavkTGJsVQfY,3576
@@ -38,7 +38,7 @@ sibi_dst/utils/clickhouse_writer.py,sha256=mNUJoYOreIdRrEFv2mQ6pdtLi1Iz_2rALDyO6
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
-sibi_dst/utils/data_wrapper.py,sha256=deUz2760T_v42Ni1twLUcGS4ucIQM63vJnC6p8sWsb4,9470
+sibi_dst/utils/data_wrapper.py,sha256=9aYXorbrqDX53NVJ5oUnNQy6FbXYhs5osxzeMcdZpC4,9609
 sibi_dst/utils/date_utils.py,sha256=8fwPpOYqSdM3nHeNykh7Ftk-uPdFa44cEAy5S8iUNw4,18667
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
@@ -71,6 +71,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-2025.1.9.dist-info/METADATA,sha256=aGk1rY4nTE2KjIYLgIobb0ER3DhtncHp_GTqlXxxizg,2610
-sibi_dst-2025.1.9.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-2025.1.9.dist-info/RECORD,,
+sibi_dst-2025.1.10.dist-info/METADATA,sha256=8vs8tux9EiNETH_j1d-2JMDfWfhN7DysoBAa9HtJk1w,2611
+sibi_dst-2025.1.10.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-2025.1.10.dist-info/RECORD,,