sibi-dst 2025.9.10__py3-none-any.whl → 2025.9.11__py3-none-any.whl

sibi_dst/df_helper/backends/parquet/_parquet_options.py

@@ -231,7 +231,7 @@ class ParquetConfig(BaseModel):
         Builds a list of path patterns for dask.read_parquet.
         Respects partition_on + start/end date if given.
         """
-        print(f"_resolve_paths_for_read: {self.partition_on}")
+        self.logger.debug(f"_resolve_paths_for_read: {self.partition_on}")
         # Partitioned dataset by column
         if self.partition_on and self.parquet_start_date and self.parquet_end_date:
             if not isinstance(self.partition_on, (list, tuple)):
@@ -244,12 +244,10 @@ class ParquetConfig(BaseModel):
             days = pd.date_range(start=start, end=end, freq="D").date
 
             base = self.parquet_storage_path.rstrip("/")
-            print("base:",base)
             result= [
                 f"{base}/{parts[0]}={d.isoformat()}/*.parquet"
                 for d in days
             ]
-            print("result:",result)
             return result
 
         # Date-ranged folders (non-partitioned, using FilePathGenerator)

sibi_dst/utils/business_days.py

@@ -1,19 +1,22 @@
+from __future__ import annotations
+
 import datetime as dt
 from typing import Any, Dict, Iterable, Optional
-from sibi_dst.utils import Logger
+
+import dask.dataframe as dd
 import numpy as np
 import pandas as pd
-import dask.dataframe as dd
+
+from sibi_dst.utils import Logger
 
 
 # ---------------- Vectorized helpers (used by Dask map_partitions) ----------------
 
 def _to_np_days(series: pd.Series) -> np.ndarray:
     """Coerce to numpy datetime64[D] with NaT-safe conversion."""
-    # Use pandas for robust parsing, then cast to date-days
     s = pd.to_datetime(series, errors="coerce")
-    # Convert to numpy datetime64[D] (day precision)
-    return s.values.astype("datetime64[D]")
+    # Return day precision array directly
+    return s.dt.floor("D").to_numpy(dtype="datetime64[D]")
 
 
 def _vectorized_busday_count(
@@ -24,8 +27,8 @@ def _vectorized_busday_count(
         weekmask: Optional[str],
         inclusive: bool,
 ) -> pd.Series:
-    start = _to_np_days(part[begin_col])  # numpy datetime64[D]
-    end = _to_np_days(part[end_col])  # numpy datetime64[D]
+    start = _to_np_days(part[begin_col])
+    end = _to_np_days(part[end_col])
 
     kwargs: Dict[str, Any] = {}
     if holidays:
@@ -38,7 +41,7 @@ def _vectorized_busday_count(
         with np.errstate(invalid="ignore"):
             end_adj = end + np.timedelta64(1, "D")
 
-    valid = (~pd.isna(start)) & (~pd.isna(end))  # numpy bool mask
+    valid = (~pd.isna(start)) & (~pd.isna(end))
     result = np.full(part.shape[0], np.nan, dtype="float64")
     if valid.any():
         counts = np.busday_count(
@@ -59,8 +62,8 @@ def _vectorized_busday_offset(
         weekmask: Optional[str],
         roll: str,
 ) -> pd.Series:
-    start = _to_np_days(part[start_col])  # numpy datetime64[D]
-    n_days = pd.to_numeric(part[n_days_col], errors="coerce").to_numpy()  # numpy float -> cast later
+    start = _to_np_days(part[start_col])
+    n_days = pd.to_numeric(part[n_days_col], errors="coerce").to_numpy()
 
     kwargs: Dict[str, Any] = {"roll": roll}
     if holidays:
@@ -68,7 +71,7 @@ def _vectorized_busday_offset(
     if weekmask:
         kwargs["weekmask"] = weekmask
 
-    valid = (~pd.isna(start)) & (~pd.isna(n_days))  # numpy bool mask
+    valid = (~pd.isna(start)) & (~pd.isna(n_days))
     out = np.full(part.shape[0], np.datetime64("NaT", "ns"), dtype="datetime64[ns]")
     if valid.any():
         offs = np.busday_offset(
@@ -86,26 +89,6 @@ def _vectorized_busday_offset(
 class BusinessDays:
     """
     Business day calculations with custom holidays and optional weekmask.
-
-    Features
-    - Scalar helpers:
-        - get_business_days_count(begin, end, inclusive=False) -> int
-        - add_business_days(start_date, n_days, roll='forward') -> np.datetime64
-    - Dask DataFrame helpers (vectorized via map_partitions):
-        - calc_business_days_from_df(df, begin_col, end_col, result_col='business_days', inclusive=False)
-        - calc_sla_end_date(df, start_date_col, n_days_col, result_col='sla_end_date', roll='forward')
-
-    Parameters
-    ----------
-    holiday_list : dict[str, list[str]] | Iterable[str]
-        Either a mapping of year -> [YYYY-MM-DD, ...] or a flat iterable of YYYY-MM-DD strings.
-    logger : Any
-        Logger with .debug/.info/.warning/.error.
-    weekmask : str | None
-        A numpy business day weekmask like '1111100' (Mon–Fri). None means default Mon–Fri.
-        Examples:
-            '1111100' -> Mon-Fri
-            '1111110' -> Mon-Sat
     """
 
     def __init__(
@@ -119,12 +102,11 @@ class BusinessDays:
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
         self.weekmask = weekmask
 
-        # Normalize holidays to a flat, sorted tuple of 'YYYY-MM-DD'
         if isinstance(holiday_list, dict):
             flat = [d for _, days in sorted(holiday_list.items()) for d in days]
         else:
             flat = list(holiday_list)
-        # Deduplicate while preserving order
+
         seen = set()
         flat_unique = []
         for d in flat:
@@ -142,7 +124,6 @@ class BusinessDays:
             *,
             inclusive: bool = False,
     ) -> int:
-        """Business days between two dates. If inclusive=True, include the end date."""
         b = pd.to_datetime(begin_date).date()
         e = pd.to_datetime(end_date).date()
 
@@ -153,11 +134,11 @@ class BusinessDays:
             kwargs["weekmask"] = self.weekmask
 
         if inclusive:
-            e_np = np.datetime64(e) + np.timedelta64(1, "D")
+            e_np = np.datetime64(e, "D") + np.timedelta64(1, "D")
         else:
-            e_np = np.datetime64(e)
+            e_np = np.datetime64(e, "D")
 
-        val = int(np.busday_count(np.datetime64(b), e_np, **kwargs))
+        val = int(np.busday_count(np.datetime64(b, "D"), e_np, **kwargs))
         return val
 
     def add_business_days(
@@ -167,11 +148,6 @@ class BusinessDays:
             *,
             roll: str = "forward",
     ) -> np.datetime64:
-        """
-        Add (or subtract) business days to a date. Returns numpy datetime64[D].
-        roll: {'forward','backward','following','preceding','modifiedfollowing',
-               'modifiedpreceding','nat'}
-        """
         s = pd.to_datetime(start_date).date()
         kwargs: Dict[str, Any] = {"roll": roll}
         if self.holidays:
@@ -179,7 +155,7 @@ class BusinessDays:
         if self.weekmask:
             kwargs["weekmask"] = self.weekmask
 
-        return np.busday_offset(np.datetime64(s), int(n_days), **kwargs)
+        return np.busday_offset(np.datetime64(s, "D"), int(n_days), **kwargs)
 
     # -------- Dask API --------
 
@@ -192,10 +168,6 @@ class BusinessDays:
             *,
             inclusive: bool = False,
     ) -> dd.DataFrame:
-        """
-        Vectorized business-day difference between two date columns.
-        Produces float64 (NaN where either side is missing).
-        """
         missing = {begin_date_col, end_date_col} - set(df.columns)
         if missing:
             self.logger.error(f"Missing columns: {missing}")
@@ -224,10 +196,6 @@ class BusinessDays:
             *,
             roll: str = "forward",
     ) -> dd.DataFrame:
-        """
-        Vectorized business-day offset for SLA end date.
-        Produces datetime64[ns] with NaT where invalid.
-        """
         missing = {start_date_col, n_days_col} - set(df.columns)
         if missing:
             self.logger.error(f"Missing columns: {missing}")

sibi_dst/utils/filepath_generator.py

@@ -160,160 +160,7 @@ class FilePathGenerator:
         # For local file, return absolute-like path without scheme or keep 'file://'? Keep scheme for consistency.
         return f"{self._protocol}://{path}"
 
-# import datetime
-# import re
-#
-# import fsspec
-#
-# from .log_utils import Logger
-#
-#
-# class FilePathGenerator:
-#     """
-#     Dynamically generates file paths by scanning directories starting from the base path
-#     and determining the innermost directory structure.
-#
-#     Now supports generating appropriate paths for both pandas and Dask.
-#     """
-#
-#     def __init__(self, base_path='', fs=None, logger=None, **kwargs):
-#         """
-#         Initialize the FilePathGenerator.
-#
-#         Parameters:
-#             base_path (str): Base directory path where data files are stored.
-#             fs (fsspec.AbstractFileSystem, optional): Filesystem object to use for file operations.
-#             logger (Logger, optional): Logger instance for logging information.
-#             **kwargs: Additional keyword arguments.
-#                 - debug (bool): If True, enables debug logging.
-#                 - storage_options (dict): Options for the filesystem (e.g., credentials, tokens).
-#                 - exclude_patterns (list): List of regex patterns to exclude from file paths.
-#                 - file_extension (str): File extension to look for (default: 'parquet').
-#         """
-#         self.base_path = base_path.rstrip('/')
-#         self.fs = fs  # Filesystem object
-#         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
-#         self.debug = kwargs.get('debug', False)
-#         self.storage_options = kwargs.get('storage_options', {})
-#         self.exclude_patterns = kwargs.get('exclude_patterns', [])
-#         self.file_extension = kwargs.get('file_extension', 'parquet').lstrip('.')
-#
-#         # If fs is not provided, initialize it based on base_path and storage_options
-#         if self.fs is None:
-#             self.fs, _ = fsspec.core.url_to_fs(self.base_path, **self.storage_options)
-#
-#     def generate_file_paths(self, start_date, end_date, engine='dask'):
-#         """
-#         Generate paths dynamically for files within the date range by scanning directories.
-#         Returns a list of file paths compatible with the specified engine.
-#
-#         Parameters:
-#             start_date (str or datetime): Start date in 'YYYY-MM-DD' format or datetime object.
-#             end_date (str or datetime): End date in 'YYYY-MM-DD' format or datetime object.
-#             engine (str): 'pandas' or 'dask' to specify which library the paths are intended for.
-#
-#         Returns:
-#             list: List of file paths.
-#         """
-#         start_date = self._convert_to_datetime(start_date)
-#         end_date = self._convert_to_datetime(end_date)
-#
-#         paths = []
-#         curr_date = start_date
-#
-#         while curr_date <= end_date:
-#             year, month, day = curr_date.year, curr_date.month, curr_date.day
-#             day_paths = self._collect_paths(year, month, day, engine)
-#             if day_paths:
-#                 paths.extend(day_paths)
-#             curr_date += datetime.timedelta(days=1)
-#
-#         return paths
-#
-#     def _collect_paths(self, year, month, day, engine):
-#         """
-#         Collect appropriate paths for a given date, depending on the engine.
-#
-#         Parameters:
-#             year (int): Year component of the date.
-#             month (int): Month component of the date.
-#             day (int): Day component of the date.
-#             engine (str): 'pandas' or 'dask'.
-#
-#         Returns:
-#             list: List of file or directory paths.
-#         """
-#         base_dir = f"{self.base_path}/{year}/{str(month).zfill(2)}/{str(day).zfill(2)}"
-#
-#         if not self.fs.exists(base_dir):
-#             if self.debug:
-#                 self.logger.debug(f"Directory does not exist: {base_dir}")
-#             return []
-#
-#         if engine == 'dask':
-#             # Collect individual file paths
-#             file_pattern = f"{base_dir}/**/*.{self.file_extension}"
-#             all_paths = self.fs.glob(file_pattern)
-#
-#             if not all_paths and self.debug:
-#                 self.logger.debug(f"No files found with pattern: {file_pattern}")
-#
-#             # Exclude unwanted files and directories
-#             filtered_paths = self._exclude_unwanted_paths(all_paths)
-#
-#             # Filter out directories
-#             file_paths = [path for path in filtered_paths if not self.fs.isdir(path)]
-#
-#         elif engine == 'pandas':
-#             # Collect dataset directories
-#             # Assume that the base_dir is a Parquet dataset
-#             if self.fs.isdir(base_dir):
-#                 file_paths = [base_dir]
-#             else:
-#                 file_paths = []
-#
-#         else:
-#             raise ValueError("Engine must be 'pandas' or 'dask'.")
-#
-#         protocol = self.fs.protocol if isinstance(self.fs.protocol, str) else self.fs.protocol[0]
-#
-#         # Ensure the protocol is included in the paths
-#         file_paths = [
-#             f"{protocol}://{path}" if not path.startswith(f"{protocol}://") else path
-#             for path in file_paths
-#         ]
-#
-#         if self.debug:
-#             self.logger.debug(f"Collected {len(file_paths)} paths from {base_dir} for engine '{engine}'")
-#
-#         return file_paths
-#
-#     def _exclude_unwanted_paths(self, paths):
-#         """
-#         Exclude paths that match any of the exclusion patterns.
-#         """
-#         # Combine default patterns with user-provided patterns
-#         exclude_patterns = self.exclude_patterns
-#
-#         # Compile regex patterns for efficiency
-#         compiled_patterns = [re.compile(pattern) for pattern in exclude_patterns]
-#
-#         # Filter out paths matching any of the exclude patterns
-#         filtered_paths = [
-#             path for path in paths
-#             if not any(pattern.match(path) for pattern in compiled_patterns)
-#         ]
-#
-#         return filtered_paths
-#
-#     @staticmethod
-#     def _convert_to_datetime(date):
-#         """Convert a date string or datetime object into a datetime object."""
-#         if isinstance(date, str):
-#             return datetime.datetime.strptime(date, '%Y-%m-%d')
-#         return date
-#
-#
+
 # """
 # Usage:
 # # Initialize the generator

{sibi_dst-2025.9.10.dist-info → sibi_dst-2025.9.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.9.10
+Version: 2025.9.11
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-2025.9.10.dist-info → sibi_dst-2025.9.11.dist-info}/RECORD

@@ -9,7 +9,7 @@ sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 sibi_dst/df_helper/backends/http/__init__.py,sha256=d1pfgYxbiYg7E0Iw8RbJ7xfqIfJShqqTBQQGU_S6OOo,105
 sibi_dst/df_helper/backends/http/_http_config.py,sha256=eGPFdqZ5M3Tscqx2P93B6XoBEEzlmdt7yNg7PXUQnNQ,4726
 sibi_dst/df_helper/backends/parquet/__init__.py,sha256=0A6BGHZLwiLBmuBBaUvEHfeWTcInvy2NbymlrI_nuXE,104
-sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=KlzmTMVHME5kmhEBRzWdWPcih-jYaslkyhHOZdQ1gLA,11802
+sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=OeeWWFhrolayg8MGZxbbAVNIC9zZB9cCXxTGGLPX0og,11746
 sibi_dst/df_helper/backends/sqlalchemy/__init__.py,sha256=LjWm9B7CweTvlvFOgB90XjSe0lVLILAIYMWKPkFXFm8,265
 sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=6705rABdh0RY0JisxD7sE62m6890hMCAv_cpyHOMSvM,8729
 sibi_dst/df_helper/backends/sqlalchemy/_db_gatekeeper.py,sha256=GQwDy2JwPUx37vpwxPM5hg4ZydilPIP824y5C_clsl0,383
@@ -48,7 +48,7 @@ sibi_dst/utils/boilerplate/base_parquet_reader.py,sha256=3kN9_bbxyX-WuJLMBWejeAp
 sibi_dst/utils/boilerplate/base_pipeline.py,sha256=LQZBACksqHO3tQ8OhWShfqjiGyda7UhrmllRq3eWQfU,5690
 sibi_dst/utils/boilerplate/base_pipeline_template.py,sha256=D5HFA4odsR2wlTY6iLg1tm57Tsh91QkoYjjX8eUgrjU,1574
 sibi_dst/utils/boilerplate/hybrid_data_loader.py,sha256=Tazn7QL3FmWKVMXxzkvxPrG_2ucsPHvSotIW9dBLoNc,6018
-sibi_dst/utils/business_days.py,sha256=dP0Xj4FhTBIvZZrZYLOHZl5zOpDAgWkD4p_1a7BOT7I,8461
+sibi_dst/utils/business_days.py,sha256=DPZExTXTt7n3IbAaEuVacm-vZgbR_Ug2bJTPBUaoP3g,6694
 sibi_dst/utils/clickhouse_writer.py,sha256=8W_dTEOKQp4pXANznVSxRqFA2H5oD8UJifiBAONpXWY,17001
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/dask_utils.py,sha256=QhFcmpH4fXAy6b3DugIX5JvH4h-P3M3hXKnBYTLRkq0,1991
@@ -59,7 +59,7 @@ sibi_dst/utils/date_utils.py,sha256=hBVWu9_cqiZ-XsLR7QY9Iek09DQKLwrY1ZlYxWlXj7g,
 sibi_dst/utils/df_utils.py,sha256=bQGromLOEdRTvbVVcuHq0vQ0fIgqhwOoD_eIp5v7VEY,10899
 sibi_dst/utils/file_age_checker.py,sha256=44B3lwH_PLwzMfiKkgvJKjKx-qSgITIXxKfNbdf_VeA,11552
 sibi_dst/utils/file_utils.py,sha256=cm__02IKCfEOzAKAZwdNIjnRL8H4XtPa6hKcj510pto,1310
-sibi_dst/utils/filepath_generator.py,sha256=Ke_OwBjLJkNMeOP0QjbLIZpSMkzhAIxKyf4hZ5P5re0,12916
+sibi_dst/utils/filepath_generator.py,sha256=Nt6JGEM01_bPCMuWs8_TnTm0CqV5Hv9o1zu9QvN4WFk,6774
 sibi_dst/utils/iceberg_saver.py,sha256=l1UWJWrLqe2OxCdP1mRyXlG9It1-F3MN_ZvHPmxqRJ4,5253
 sibi_dst/utils/log_utils.py,sha256=1xXTDfwMwWIdj37hjyXSpHx3ft2GMiXsAfxq9AArMTY,11588
 sibi_dst/utils/manifest_manager.py,sha256=9y4cV-Ig8O-ekhApp_UObTY-cTsl-bGnvKIThItEzg4,7394
@@ -95,6 +95,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-2025.9.10.dist-info/METADATA,sha256=Sy1u7f-x3I_Ap-n2NklwyhAPWk1cluc0vX5Qn_BgS7c,2761
-sibi_dst-2025.9.10.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-2025.9.10.dist-info/RECORD,,
+sibi_dst-2025.9.11.dist-info/METADATA,sha256=LPlDJ-9iwPB3hKALqJuAX3vvZX2EHv83BPD4hwVs6Hw,2761
+sibi_dst-2025.9.11.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-2025.9.11.dist-info/RECORD,,