sibi-dst 2025.1.10__py3-none-any.whl → 2025.1.11__py3-none-any.whl

sibi_dst/df_helper/_df_helper.py

@@ -68,6 +68,9 @@ class ParquetBackend(BaseBackend):
             df = self.helper.backend_parquet.load_files()
             if options and df is not None:
                 df = FilterHandler('dask', logger=self.logger, debug=False).apply_filters(df, filters=options)
+
+            df = df.persist()
+
             self.total_records = len(df) or -1  # If df is empty, set total_records to -1
             return self.total_records, df
         except Exception as e:

sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py

@@ -15,7 +15,7 @@ from sqlalchemy.engine import url as sqlalchemy_url
 from sqlalchemy.engine import Engine
 from sqlalchemy.exc import OperationalError, SQLAlchemyError
 from sqlalchemy.orm import sessionmaker, Session
-from sqlalchemy.pool import QueuePool, NullPool, StaticPool
+from sqlalchemy.pool import QueuePool, NullPool, StaticPool, Pool
 
 # Assuming these are your project's internal modules
 from sibi_dst.utils import Logger
@@ -54,7 +54,7 @@ class SqlAlchemyConnectionConfig(BaseModel):
     pool_timeout: int = int(os.environ.get("DB_POOL_TIMEOUT", 30))
     pool_recycle: int = int(os.environ.get("DB_POOL_RECYCLE", 1800))
     pool_pre_ping: bool = True
-    poolclass: Type[QueuePool] = QueuePool
+    poolclass: Type[Pool] = QueuePool
 
     # --- Internal & Runtime State ---
     model: Optional[Type[Any]] = None
@@ -195,7 +195,6 @@ class SqlAlchemyConnectionConfig(BaseModel):
             wrapper = self._engine_registry.get(self._engine_key_instance)
             if wrapper:
                 wrapper['active_connections'] += 1
-        # self.logger.debug(f"Connection checked out. Active: {self.active_connections}")
 
     def _on_checkin(self, *args) -> None:
         """Event listener for when a connection is returned to the pool."""
@@ -203,7 +202,6 @@ class SqlAlchemyConnectionConfig(BaseModel):
             wrapper = self._engine_registry.get(self._engine_key_instance)
             if wrapper:
                 wrapper['active_connections'] = max(0, wrapper['active_connections'] - 1)
-        # self.logger.debug(f"Connection checked in. Active: {self.active_connections}")
 
     @property
     def active_connections(self) -> int:

sibi_dst/utils/date_utils.py

@@ -4,7 +4,7 @@ from typing import Union, Tuple, Callable, Dict, Optional
 import fsspec
 import numpy as np
 import pandas as pd
-
+import dask.dataframe as dd
 from .log_utils import Logger
 
 
@@ -305,154 +305,153 @@ class FileAgeChecker:
             raise ValueError(f"Unsupported modification time format for {file_path}") from e
 
 
+# --- Vectorized Helper Functions ---
+# These replace the slow, row-by-row .apply() logic. They operate
+# on entire DataFrame partitions for maximum efficiency.
+
+def _vectorized_busday_count(
+        partition: pd.DataFrame,
+        begin_col: str,
+        end_col: str,
+        holidays: list
+) -> pd.Series:
+    """Vectorized function to count business days on a DataFrame partition."""
+    if partition.empty:
+        return pd.Series([], dtype=float)
+
+    # Convert entire columns to datetime at once, coercing errors to NaT
+    start_dates = pd.to_datetime(partition[begin_col], errors='coerce').dt.date
+    end_dates = pd.to_datetime(partition[end_col], errors='coerce').dt.date
+
+    # Create a result series filled with NaN to handle rows with invalid dates
+    result = pd.Series(np.nan, index=partition.index, dtype=float)
+
+    # Create a boolean mask for valid, non-NaT date pairs
+    valid_mask = pd.notna(start_dates) & pd.notna(end_dates)
+
+    # Perform the vectorized calculation only on the valid subset of dates
+    result.loc[valid_mask] = np.busday_count(
+        start_dates[valid_mask],
+        end_dates[valid_mask],
+        holidays=holidays
+    )
+    return result
+
+
+def _vectorized_sla_end_date(
+        partition: pd.DataFrame,
+        start_col: str,
+        n_days_col: str,
+        holidays: list
+) -> pd.Series:
+    """Vectorized function to calculate the SLA end date on a DataFrame partition."""
+    if partition.empty:
+        return pd.Series([], dtype='datetime64[ns]')
+
+    start_dates = pd.to_datetime(partition[start_col], errors='coerce').dt.date
+    sla_days = partition[n_days_col]
+
+    # Create a result series filled with NaT for rows with invalid start dates
+    result = pd.Series(pd.NaT, index=partition.index, dtype='datetime64[ns]')
+
+    # Create a boolean mask for valid start dates and sla_days
+    valid_mask = pd.notna(start_dates) & pd.notna(sla_days)
+
+    # Perform the vectorized calculation only on the valid subset
+    result.loc[valid_mask] = np.busday_offset(
+        start_dates[valid_mask],
+        sla_days[valid_mask].astype(int),  # Ensure days are integers
+        roll='forward',
+        holidays=holidays
+    )
+    return result
+
+
+# --- Refactored BusinessDays Class ---
+
 class BusinessDays:
     """
-    Provides functionality for handling business days calculations with a custom
-    holiday list. The class includes methods for calculating the number of
-    business days, modifying dates by adding business days, and applying these
-    operations to Dask DataFrames.
-
-    :ivar logger: Logger instance for logging error, warning, and debug messages.
-    :type logger: logging.Logger
-    :ivar HOLIDAY_LIST: Dictionary mapping years to lists of holiday dates.
-    :type HOLIDAY_LIST: dict
-    :ivar bd_cal: Numpy busdaycalendar object containing holidays and week mask.
-    :type bd_cal: numpy.busdaycalendar
-    :ivar holidays: Array of holiday dates used by the business day calendar.
-    :type holidays: numpy.ndarray
-    :ivar week_mask: Boolean array indicating working days within a week.
-    :type week_mask: numpy.ndarray
+    Business days calculations with a custom holiday list.
+    Supports scalar and efficient, vectorized Dask DataFrame operations.
     """
 
-    def __init__(self, holiday_list, logger):
-        """
-        Initialize a BusinessDays object with a given holiday list.
-        """
+    def __init__(self, holiday_list: dict[str, list[str]], logger) -> None:
         self.logger = logger
         self.HOLIDAY_LIST = holiday_list
-        bd_holidays = [day for year in self.HOLIDAY_LIST for day in self.HOLIDAY_LIST[year]]
-        self.bd_cal = np.busdaycalendar(holidays=bd_holidays, weekmask="1111100")
-        self.holidays = self.bd_cal.holidays
-        self.week_mask = self.bd_cal.weekmask
 
-    def get_business_days_count(self, begin_date, end_date):
-        """
-        Calculate the number of business days between two dates.
-        """
-        try:
-            begin_date = pd.to_datetime(begin_date)
-            end_date = pd.to_datetime(end_date)
-        except Exception as e:
-            raise ValueError(f"Invalid date format: {e}")
-
-        years = [str(year) for year in range(begin_date.year, end_date.year + 1)]
-        if not all(year in self.HOLIDAY_LIST for year in years):
-            raise ValueError("Not all years in date range are in the holiday list")
-
-        return np.busday_count(
-            begin_date.strftime("%Y-%m-%d"),
-            end_date.strftime("%Y-%m-%d"),
-            busdaycal=self.bd_cal,
-        )
+        # Flatten and store as tuple for determinism
+        bd_holidays = [day for year in self.HOLIDAY_LIST for day in self.HOLIDAY_LIST[year]]
+        self.holidays = tuple(bd_holidays)
 
-    def calc_business_days_from_df(self, df, begin_date_col, end_date_col, result_col="business_days"):
-        """
-        Add a column to a Dask DataFrame with the number of business days between two date columns.
-        """
-        if not all(col in df.columns for col in [begin_date_col, end_date_col]):
-            self.logger.error("Column names not found in DataFrame")
-            raise ValueError("Required columns are missing")
-
-        # Extract holidays and weekmask to recreate the busdaycalendar
-        holidays = self.bd_cal.holidays
-        weekmask = self.bd_cal.weekmask
-
-        # Define a function to calculate business days
-        def calculate_business_days(row, holidays, weekmask):
-            begin_date = pd.to_datetime(row[begin_date_col])
-            end_date = pd.to_datetime(row[end_date_col])
-            if pd.isna(begin_date) or pd.isna(end_date):
-                return np.nan
-            busdaycal = np.busdaycalendar(holidays=holidays, weekmask=weekmask)
-            return np.busday_count(
-                begin_date.strftime("%Y-%m-%d"),
-                end_date.strftime("%Y-%m-%d"),
-                busdaycal=busdaycal,
-            )
-
-        # Define a wrapper function for partition-wise operations
-        def apply_business_days(partition, holidays, weekmask):
-            return partition.apply(
-                calculate_business_days, axis=1, holidays=holidays, weekmask=weekmask
-            )
-
-        # Apply the function using map_partitions
-        df[result_col] = df.map_partitions(
-            apply_business_days,
-            holidays,
-            weekmask,
-            meta=(result_col, "int64"),
+    def get_business_days_count(
+            self,
+            begin_date: str | datetime.date | pd.Timestamp,
+            end_date: str | datetime.date | pd.Timestamp,
+    ) -> int:
+        """Scalar method to count business days between two dates."""
+        begin = pd.to_datetime(begin_date)
+        end = pd.to_datetime(end_date)
+        return int(np.busday_count(begin.date(), end.date(), holidays=list(self.holidays)))
+
+    def calc_business_days_from_df(
+            self,
+            df: dd.DataFrame,
+            begin_date_col: str,
+            end_date_col: str,
+            result_col: str = "business_days",
+    ) -> dd.DataFrame:
+        """Calculates business days between two columns in a Dask DataFrame."""
+        missing = {begin_date_col, end_date_col} - set(df.columns)
+        if missing:
+            self.logger.error(f"Missing columns: {missing}")
+            raise ValueError("Required columns are missing from DataFrame")
+
+        return df.assign(
+            **{result_col: df.map_partitions(
+                _vectorized_busday_count,
+                begin_col=begin_date_col,
+                end_col=end_date_col,
+                holidays=list(self.holidays),
+                meta=(result_col, 'f8')  # f8 is float64
+            )}
         )
 
-        return df
-
-    def add_business_days(self, start_date, n_days):
-        """
-        Add n_days business days to start_date.
-        """
-        try:
-            start_date = datetime.datetime.strptime(start_date, "%Y-%m-%d")
-        except ValueError:
-            raise ValueError("Date should be a string in the format YYYY-MM-DD")
-
-        if str(start_date.year) not in self.HOLIDAY_LIST:
-            self.logger.warning(f"Year {start_date.year} is not in the holiday list")
-
+    def add_business_days(
+            self,
+            start_date: str | datetime.date | pd.Timestamp,
+            n_days: int,
+    ) -> np.datetime64:
+        """Scalar method to add N business days to a start date."""
+        start = pd.to_datetime(start_date)
         return np.busday_offset(
-            start_date.strftime("%Y-%m-%d"),
+            start.date(),
             n_days,
-            roll="forward",
-            busdaycal=self.bd_cal,
+            roll='forward',
+            holidays=list(self.holidays),
         )
 
-    def calc_sla_end_date(self, df, start_date_col, n_days_col, result_col="sla_end_date"):
-        """
-        Add a column to a Dask DataFrame with SLA end dates based on start date and SLA days.
-        """
-        if not all(col in df.columns for col in [start_date_col, n_days_col]):
-            raise ValueError("Column names not found in DataFrame")
-
-        # Extract holidays and weekmask to recreate the busdaycalendar
-        holidays = self.bd_cal.holidays
-        weekmask = self.bd_cal.weekmask
-
-        # Define a function to calculate SLA end dates
-        def calculate_sla_end_date(row, holidays, weekmask):
-            start_date = pd.to_datetime(row[start_date_col])
-            n_days = row[n_days_col]
-            busdaycal = np.busdaycalendar(holidays=holidays, weekmask=weekmask)
-            return np.busday_offset(
-                start_date.strftime("%Y-%m-%d"),
-                n_days,
-                roll="forward",
-                busdaycal=busdaycal,
-            )
-
-        # Define a wrapper for partition-wise operation
-        def apply_sla_end_date(partition, holidays, weekmask):
-            return partition.apply(
-                calculate_sla_end_date, axis=1, holidays=holidays, weekmask=weekmask
-            )
-
-        # Apply the function using map_partitions
-        df[result_col] = df.map_partitions(
-            apply_sla_end_date,
-            holidays,
-            weekmask,
-            meta=(result_col, "object"),
+    def calc_sla_end_date(
+            self,
+            df: dd.DataFrame,
+            start_date_col: str,
+            n_days_col: str,
+            result_col: str = "sla_end_date",
+    ) -> dd.DataFrame:
+        """Calculates an SLA end date column for a Dask DataFrame."""
+        missing = {start_date_col, n_days_col} - set(df.columns)
+        if missing:
+            self.logger.error(f"Missing columns: {missing}")
+            raise ValueError("Required columns are missing from DataFrame")
+
+        return df.assign(
+            **{result_col: df.map_partitions(
+                _vectorized_sla_end_date,
+                start_col=start_date_col,
+                n_days_col=n_days_col,
+                holidays=list(self.holidays),
+                meta=(result_col, 'datetime64[ns]')
+            )}
         )
-
-        return df
 # Class enhancements
 # DateUtils.register_period('next_week', lambda: (datetime.date.today() + datetime.timedelta(days=7),
 #                                                 datetime.date.today() + datetime.timedelta(days=13)))

sibi_dst/v2/df_helper/backends/sqlmodel/_io_dask.py

@@ -1,7 +1,8 @@
 import itertools
 import dask.dataframe as dd
 import pandas as pd
-from sqlmodel import create_engine, Session, select
+
+#from sqlmodel import create_engine, Session, select
 from sibi_dst.v2.df_helper.core import FilterHandler
 from sibi_dst.v2.utils import Logger
 
@@ -116,7 +117,7 @@ class SQLModelDask:
                 return dask_df
 
         except Exception as e:
-            self.logger.error(f"Error executing query: {str(e)}")
-            self.logger.error(self.query)
+            self.logger.error(f"_io_dask:Error executing query: {str(e)}")
+            self.logger.error(f"_io_dask:{self.query})
             # In case of error, return an empty Dask DataFrame with the expected columns.
             return dd.from_pandas(pd.DataFrame(columns=ordered_columns), npartitions=1)

{sibi_dst-2025.1.10.dist-info → sibi_dst-2025.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.1.10
+Version: 2025.1.11
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-2025.1.10.dist-info → sibi_dst-2025.1.11.dist-info}/RECORD

@@ -1,7 +1,7 @@
 sibi_dst/__init__.py,sha256=j8lZpGCJlxlLgEgeIMxZnWdqJ0g3MCs7-gsnbvPn_KY,285
 sibi_dst/df_helper/__init__.py,sha256=Jur_MO8RGPkVw0CS3XH5YIWv-d922DC_FwRDTvHHV6Y,432
 sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=pSSw3N_ZNZCZHAiChbsF_ECyCmz0L2xCgvt9srHtPOM,17575
-sibi_dst/df_helper/_df_helper.py,sha256=BbpP0BOLDGCOE8oAxqP5ODN_HqYohQcGsh-8Dx2-sks,12885
+sibi_dst/df_helper/_df_helper.py,sha256=PNoN0nlzRwo_4JiaVyzmOM--LRrsJ0jB9pZqDi_kkRA,12917
 sibi_dst/df_helper/_parquet_artifact.py,sha256=dCvUA2bytv0wY0pFI8lxbcLwXlgGpHndS36iKfEmjLw,14310
 sibi_dst/df_helper/_parquet_reader.py,sha256=m98C0TZRroOXvVc2LpEuElrJnquGlR81E1gjI7v1hi4,3102
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -11,7 +11,7 @@ sibi_dst/df_helper/backends/parquet/__init__.py,sha256=esWJ9aSuYC26d-T01z9dPrJ1u
 sibi_dst/df_helper/backends/parquet/_filter_handler.py,sha256=TvDf0RXta7mwJv11GNQttYJsXgFf2XDj4oLIjt4xTzA,5219
 sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=FWExRRTlhGrOhGPyzL1tucxgoHa3nJenLLs87I2gs-I,11776
 sibi_dst/df_helper/backends/sqlalchemy/__init__.py,sha256=LjWm9B7CweTvlvFOgB90XjSe0lVLILAIYMWKPkFXFm8,265
-sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=AOYvWw1vxd1CwXpIakQNFln7PHzFLfp7oaOsGaG0UN8,10961
+sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=ycjnkhD1lWMKnLFy1bycle__jbfaWH6oI7m9ymX59c4,10783
 sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py,sha256=NqBSHqeYv_1vHt6J0tez0GdMwKrP_sIRcXYXu869ZkY,13313
 sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=ibxeVqpIEsSVusP2bgcd1MNV_wJIoNgXwacltUbwTas,3194
 sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py,sha256=d_-ip-dQnWOlM8btCjoywAXpaiSuN6AaavkTGJsVQfY,3576
@@ -39,7 +39,7 @@ sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
 sibi_dst/utils/data_wrapper.py,sha256=9aYXorbrqDX53NVJ5oUnNQy6FbXYhs5osxzeMcdZpC4,9609
-sibi_dst/utils/date_utils.py,sha256=8fwPpOYqSdM3nHeNykh7Ftk-uPdFa44cEAy5S8iUNw4,18667
+sibi_dst/utils/date_utils.py,sha256=T0uXNIG2IQfgs0AyQNsF9S6-cTujtA4GDC1IalvZVSU,18040
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
 sibi_dst/utils/filepath_generator.py,sha256=-HHO0U-PR8fysDDFwnWdHRlgqksh_RkmgBZLWv9hM7s,6669
@@ -62,7 +62,7 @@ sibi_dst/v2/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=jhgN0OO5Sk1zQF
 sibi_dst/v2/df_helper/backends/sqlalchemy/_model_builder.py,sha256=jX_mQAzl_6xdh7CTYw4uvUIX2wMp3NzXMlfbC5alOzs,13632
 sibi_dst/v2/df_helper/backends/sqlmodel/__init__.py,sha256=LcwJjVVxxrnVZalWqnz5m7r77i9tmJR0-U2k8eSQ-m8,249
 sibi_dst/v2/df_helper/backends/sqlmodel/_db_connection.py,sha256=n3CDbda0OY3X7eTeu_PR2KcZ5hYyEJL7Hroo8yQkjG8,15435
-sibi_dst/v2/df_helper/backends/sqlmodel/_io_dask.py,sha256=wVgNPo5V75aLtlZr_SIQ-yteyXq-Rg93eMfR8JCfkSo,5422
+sibi_dst/v2/df_helper/backends/sqlmodel/_io_dask.py,sha256=VyhSGZGSN0gpsGhHHpY07NkmeAvPmMyQi3ewAaE79VM,5446
 sibi_dst/v2/df_helper/backends/sqlmodel/_load_from_db.py,sha256=FIs6UrNxdJ7eDHDvTv-cJuybIue2-oCRedhW-MNe7CU,6285
 sibi_dst/v2/df_helper/backends/sqlmodel/_model_builder.py,sha256=k0dnMLkLMMvkDYDYWkGFgibW5UD8pJgB3YrEg_R7pj8,13556
 sibi_dst/v2/df_helper/core/__init__.py,sha256=rZhBh32Rgcxj4MBii-KsYVJQmrT9egiWKXk68gWKblo,197
@@ -71,6 +71,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-2025.1.10.dist-info/METADATA,sha256=8vs8tux9EiNETH_j1d-2JMDfWfhN7DysoBAa9HtJk1w,2611
-sibi_dst-2025.1.10.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-2025.1.10.dist-info/RECORD,,
+sibi_dst-2025.1.11.dist-info/METADATA,sha256=7iwn7RFfaDF_9dfpWvnNl2Al_8NHWu7l8vGhzO9BAac,2611
+sibi_dst-2025.1.11.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-2025.1.11.dist-info/RECORD,,