sibi-dst 0.3.40__py3-none-any.whl → 0.3.43__py3-none-any.whl

sibi_dst/utils/date_utils.py

@@ -140,21 +140,32 @@ class DateUtils:
             'last_month': lambda: cls.get_month_range(n=-1),
             'current_year': lambda: cls.get_year_timerange(today().year),
             'current_quarter': lambda: (
-            cls.get_first_day_of_the_quarter(today()), cls.get_last_day_of_the_quarter(today())),
+                cls.get_first_day_of_the_quarter(today()), cls.get_last_day_of_the_quarter(today())),
             'ytd': lambda: (datetime.date(today().year, 1, 1), today()),
         }
 
-    def is_file_older_than(self, file_path: str, max_age_minutes: int, fs: Optional[fsspec.AbstractFileSystem] = None,
-                           ignore_missing: bool = False, verbose: bool = False) -> bool:
+
+class FileAgeChecker:
+    def __init__(self, logger=None):
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+
+    def is_file_older_than(
+            self,
+            file_path: str,
+            max_age_minutes: int,
+            fs: Optional[fsspec.AbstractFileSystem] = None,
+            ignore_missing: bool = False,
+            verbose: bool = False,
+    ) -> bool:
         """
-        Check if a file or a partitioned Parquet dataset is older than the specified max_age_minutes.
+        Check if a file or directory is older than the specified max_age_minutes.
 
-        :param file_path: Path to the file or dataset.
+        :param file_path: Path to the file or directory.
         :param max_age_minutes: Maximum allowed age in minutes.
-        :param fs: Filesystem object (e.g., S3, local). If not provided, defaults to the local filesystem.
-        :param ignore_missing: If True, treat missing files as not old. Defaults to False.
-        :param verbose: If True, log detailed information. Defaults to False.
-        :return: True if the file or dataset is older than max_age_minutes, False otherwise.
+        :param fs: Filesystem object. Defaults to local filesystem.
+        :param ignore_missing: Treat missing paths as not old if True.
+        :param verbose: Enable detailed logging.
+        :return: True if older than max_age_minutes, False otherwise.
         """
         fs = fs or fsspec.filesystem("file")
         self.logger.info(f"Checking age for {file_path}...")
@@ -165,136 +176,129 @@ class DateUtils:
                 return not ignore_missing
 
             if fs.isdir(file_path):
-                self.logger.info(f"Found that {file_path} is a directory...")
-                return self._is_directory_older_than(file_path, max_age_minutes, fs, verbose)
-
+                self.logger.info(f"Found directory: {file_path}")
+                age = self._get_directory_age_minutes(file_path, fs, verbose)
             elif fs.isfile(file_path):
-                return self._is_file_older_than(file_path, max_age_minutes, fs, verbose)
-
+                age = self._get_file_age_minutes(file_path, fs, verbose)
             else:
-                self.logger.warning(f"Path {file_path} is neither a file nor a directory.")
+                self.logger.warning(f"Path {file_path} is neither file nor directory.")
                 return True
 
+            return age > max_age_minutes
+
         except Exception as e:
-            self.logger.warning(f"Error checking age for {file_path}: {str(e)}")
+            self.logger.warning(f"Error checking {file_path}: {str(e)}")
             return True
 
-    def _is_directory_older_than(self, dir_path: str, max_age_minutes: int, fs: fsspec.AbstractFileSystem,
-                                 verbose: bool) -> bool:
+    def get_file_or_dir_age_minutes(
+            self,
+            file_path: str,
+            fs: Optional[fsspec.AbstractFileSystem] = None,
+    ) -> float:
         """
-        Check if the oldest file in a directory is older than the specified max_age_minutes.
+        Get age of file/directory in minutes. Returns infinity for errors/missing paths.
 
-        :param dir_path: Path to the directory.
-        :param max_age_minutes: Maximum allowed age in minutes.
-        :param fs: Filesystem object.
-        :param verbose: If True, log detailed information.
-        :return: True if the oldest file is older than max_age_minutes, False otherwise.
+        :param file_path: Path to check.
+        :param fs: Filesystem object. Defaults to local filesystem.
+        :return: Age in minutes or infinity if unavailable.
         """
-        all_files = fs.ls(dir_path)
-        if not all_files:
-            self.logger.info(f"No files found in dataset: {dir_path}.")
-            return True
-
-        modification_times = [
-            self._get_modification_time(fs.info(file), file)
-            for file in all_files
-            if self._is_valid_file(file, fs)
-        ]
+        fs = fs or fsspec.filesystem("file")
+        try:
+            if not fs.exists(file_path):
+                self.logger.info(f"Path not found: {file_path}")
+                return float("inf")
 
-        if not modification_times:
-            self.logger.warning(f"No valid modification times found for dataset: {dir_path}. Assuming dataset is old.")
-            return True
+            if fs.isdir(file_path):
+                return self._get_directory_age_minutes(file_path, fs, verbose=False)
+            if fs.isfile(file_path):
+                return self._get_file_age_minutes(file_path, fs, verbose=False)
 
-        oldest_modification_time = min(modification_times)
-        dataset_age_minutes = (datetime.datetime.now(
-            datetime.timezone.utc) - oldest_modification_time).total_seconds() / 60
+            self.logger.warning(f"Invalid path type: {file_path}")
+            return float("inf")
 
-        if verbose:
-            self.logger.info(
-                f"Oldest file in dataset {dir_path} is {round(dataset_age_minutes, 2)} minutes old "
-                f"(threshold: {max_age_minutes} minutes)"
-            )
+        except Exception as e:
+            self.logger.warning(f"Error getting age for {file_path}: {str(e)}")
+            return float("inf")
+
+    def _get_directory_age_minutes(
+            self,
+            dir_path: str,
+            fs: fsspec.AbstractFileSystem,
+            verbose: bool,
+    ) -> float:
+        """Calculate age of oldest file in directory."""
+        try:
+            all_files = fs.ls(dir_path)
+        except Exception as e:
+            self.logger.warning(f"Error listing {dir_path}: {str(e)}")
+            return float("inf")
 
-        return dataset_age_minutes > max_age_minutes
+        if not all_files:
+            self.logger.info(f"Empty directory: {dir_path}")
+            return float("inf")
 
-    def _is_file_older_than(self, file_path: str, max_age_minutes: int, fs: fsspec.AbstractFileSystem,
-                            verbose: bool) -> bool:
-        """
-        Check if a single file is older than the specified max_age_minutes.
+        modification_times = []
+        for file in all_files:
+            try:
+                info = fs.info(file)
+                mod_time = self._get_modification_time(info, file)
+                modification_times.append(mod_time)
+            except Exception as e:
+                self.logger.warning(f"Skipping {file}: {str(e)}")
 
-        :param file_path: Path to the file.
-        :param max_age_minutes: Maximum allowed age in minutes.
-        :param fs: Filesystem object.
-        :param verbose: If True, log detailed information.
-        :return: True if the file is older than max_age_minutes, False otherwise.
-        """
-        info = fs.info(file_path)
-        if verbose:
-            self.logger.debug(f"File info for {file_path}: {info}")
-
-        file_modification_datetime = self._get_modification_time(info, file_path)
-        file_age_minutes = (datetime.datetime.now(
-            datetime.timezone.utc) - file_modification_datetime).total_seconds() / 60
-
-        if verbose:
-            self.logger.debug(
-                f"File {file_path} is {round(file_age_minutes, 2)} minutes old "
-                f"(threshold: {max_age_minutes} minutes)"
-            )
+        if not modification_times:
+            self.logger.warning(f"No valid files in {dir_path}")
+            return float("inf")
+
+        oldest = min(modification_times)
+        age = (datetime.datetime.now(datetime.timezone.utc) - oldest).total_seconds() / 60
+        self.logger.info(f"Oldest in {dir_path}: {age:.2f} minutes")
+
+        return age
+
+    def _get_file_age_minutes(
+            self,
+            file_path: str,
+            fs: fsspec.AbstractFileSystem,
+            verbose: bool,
+    ) -> float:
+        """Calculate file age in minutes."""
+        try:
+            info = fs.info(file_path)
+            mod_time = self._get_modification_time(info, file_path)
+            age = (datetime.datetime.now(datetime.timezone.utc) - mod_time).total_seconds() / 60
 
-        return file_age_minutes > max_age_minutes
+            if verbose:
+                self.logger.debug(f"{file_path} info: {info}")
+                self.logger.debug(f"File age: {age:.2f} minutes")
 
-    def _is_valid_file(self, file_path: str, fs: fsspec.AbstractFileSystem) -> bool:
-        """
-        Check if a file is valid (exists and has a valid modification time).
+            return age
 
-        :param file_path: Path to the file.
-        :param fs: Filesystem object.
-        :return: True if the file is valid, False otherwise.
-        """
-        try:
-            fs.info(file_path)
-            return True
         except Exception as e:
-            self.logger.warning(f"Error checking file age for {file_path}: {str(e)}")
-            return False
+            self.logger.warning(f"Error processing {file_path}: {str(e)}")
+            return float("inf")
 
     def _get_modification_time(self, info: Dict, file_path: str) -> datetime.datetime:
-        """
-        Extract the modification time from file info.
+        """Extract modification time from filesystem info with timezone awareness."""
+        try:
+            if "LastModified" in info:  # S3-like
+                lm = info["LastModified"]
+                return lm if isinstance(lm, datetime.datetime) else datetime.datetime.fromisoformat(
+                    lm[:-1]).astimezone()
 
-        :param info: File info dictionary.
-        :param file_path: Path to the file (for logging purposes).
-        :return: Modification time as a timezone-aware datetime object.
-        """
-        if "LastModified" in info:  # S3-compatible filesystem
-            last_modified = info["LastModified"]
-            if isinstance(last_modified, datetime.datetime):
-                return last_modified
-            else:
-                return datetime.datetime.strptime(last_modified, "%Y-%m-%dT%H:%M:%S.%fZ").replace(
-                    tzinfo=datetime.timezone.utc)
+            if "mtime" in info:  # Local filesystem
+                return datetime.datetime.fromtimestamp(info["mtime"], tz=datetime.timezone.utc)
 
-        elif "mtime" in info:  # Local filesystem
-            return datetime.datetime.fromtimestamp(info["mtime"], tz=datetime.timezone.utc)
+            if "modified" in info:  # FTP/SSH
+                return datetime.datetime.strptime(
+                    info["modified"], "%Y-%m-%d %H:%M:%S"
+                ).replace(tzinfo=datetime.timezone.utc)
 
-        elif "modified" in info:  # FTP or SSH filesystem
-            modified_str = info["modified"]
-            try:
-                return datetime.datetime.strptime(modified_str, "%Y-%m-%d %H:%M:%S").replace(
-                    tzinfo=datetime.timezone.utc)
-            except ValueError:
-                try:
-                    return datetime.datetime.strptime(modified_str, "%b %d %H:%M").replace(
-                        year=datetime.datetime.now().year, tzinfo=datetime.timezone.utc
-                    )
-                except ValueError:
-                    self.logger.warning(f"Unsupported modification time format for {file_path}: {modified_str}")
-                    raise ValueError("Unsupported modification time format")
+            raise KeyError("No valid modification time key found")
 
-        else:  # Fallback for unsupported filesystems
-            self.logger.warning(f"Modification time not available for {file_path}.")
-            raise ValueError("Modification time not available")
+        except (KeyError, ValueError) as e:
+            self.logger.warning(f"Invalid mod time for {file_path}: {str(e)}")
+            raise ValueError(f"Unsupported modification time format for {file_path}") from e
 
 
 class BusinessDays:
@@ -315,6 +319,7 @@ class BusinessDays:
     :ivar week_mask: Boolean array indicating working days within a week.
     :type week_mask: numpy.ndarray
     """
+
     def __init__(self, holiday_list, logger):
         """
         Initialize a BusinessDays object with a given holiday list.

sibi_dst/utils/df_utils.py

@@ -14,6 +14,13 @@ class DfUtils:
         """
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
 
+    @classmethod
+    def compute_to_list(cls, series):
+        return series.compute().tolist() if hasattr(series, "compute") else series.tolist()
+
+    def extract_unique_values(self, df, *columns):
+        return {col: self.compute_to_list(df[col].dropna().unique()) for col in columns}
+
     def align_and_merge_by_type(self, df_left, df_right, type_mapping, how='left'):
         """
         Align column data types in two DataFrames based on a type mapping dictionary and perform the merge.

sibi_dst/utils/log_utils.py

@@ -1,6 +1,8 @@
 import logging
 import os
 import sys
+import time
+from typing import Optional
 
 
 class Logger:
@@ -23,15 +25,26 @@ class Logger:
     :ivar logger: The initialized logger instance used for logging messages.
     :type logger: logging.Logger
     """
-    def __init__(self, log_dir, logger_name, log_file):
+
+    def __init__(self, log_dir: str, logger_name: str, log_file: str, log_level: int = logging.DEBUG):
+        """
+        Initialize the Logger instance.
+
+        :param log_dir: Directory where logs are stored.
+        :param logger_name: Name of the logger instance.
+        :param log_file: Base name of the log file.
+        :param log_level: Logging level (defaults to DEBUG).
+        """
         self.log_dir = log_dir
         self.logger_name = logger_name
         self.log_file = log_file
+        self.log_level = log_level
         self.logger = None
 
         self._setup()
 
     def _setup(self):
+        """Set up the logger with file and console handlers."""
         # Ensure the log directory exists
         os.makedirs(self.log_dir, exist_ok=True)
 
@@ -47,45 +60,71 @@ class Logger:
 
         # Create a logger
         self.logger = logging.getLogger(self.logger_name)
-        self.logger.setLevel(logging.DEBUG)  # Log all levels DEBUG and above
+        self.logger.setLevel(self.log_level)
 
-        # Create a file handler
-        handler = logging.FileHandler(log_file_path)
+        # Create a formatter
+        formatter = logging.Formatter(
+            '[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S'
+        )
 
-        # Create a formatter and add it to the handler
-        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-        handler.setFormatter(formatter)
+        formatter.converter = time.localtime  # << Set local time explicitly
 
-        # Add the handler to the logger
-        self.logger.addHandler(handler)
+        # Create a file handler
+        file_handler = logging.FileHandler(log_file_path)
+        file_handler.setFormatter(formatter)
+        self.logger.addHandler(file_handler)
+
+        # Create a console handler (optional)
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(formatter)
+        self.logger.addHandler(console_handler)
 
     @classmethod
-    def default_logger(cls, log_dir='./logs/', logger_name=None, log_file=None):
+    def default_logger(
+            cls,
+            log_dir: str = './logs/',
+            logger_name: Optional[str] = None,
+            log_file: Optional[str] = None,
+            log_level: int = logging.INFO
+    ) -> 'Logger':
         """
         Class-level method to create a default logger with generic parameters.
-        :param log_dir: Directory where logs are stored.
+
+        :param log_dir: Directory where logs are stored (defaults to './logs/').
         :param logger_name: Name of the logger (defaults to __name__).
         :param log_file: Name of the log file (defaults to logger_name).
+        :param log_level: Logging level (defaults to INFO).
         :return: Instance of Logger.
         """
         logger_name = logger_name or __name__
         log_file = log_file or logger_name
-        return cls(log_dir=log_dir, logger_name=logger_name, log_file=log_file)
+        return cls(log_dir=log_dir, logger_name=logger_name, log_file=log_file, log_level=log_level)
 
-    def set_level(self, level):
+    def set_level(self, level: int):
+        """
+        Set the logging level for the logger.
+
+        :param level: Logging level (e.g., logging.DEBUG, logging.INFO).
+        """
         self.logger.setLevel(level)
 
-    def debug(self, msg):
+    def debug(self, msg: str):
+        """Log a debug message."""
         self.logger.debug(msg)
 
-    def info(self, msg):
+    def info(self, msg: str):
+        """Log an info message."""
         self.logger.info(msg)
 
-    def warning(self, msg):
+    def warning(self, msg: str):
+        """Log a warning message."""
         self.logger.warning(msg)
 
-    def error(self, msg):
+    def error(self, msg: str):
+        """Log an error message."""
         self.logger.error(msg)
 
-    def critical(self, msg):
+    def critical(self, msg: str):
+        """Log a critical message."""
         self.logger.critical(msg)

sibi_dst/utils/parquet_saver.py

@@ -1,4 +1,5 @@
 import base64
+import logging
 from pathlib import Path
 from typing import Optional
 
@@ -16,7 +17,7 @@ from sibi_dst.utils import Logger
 
 
 class ParquetSaver:
-    def __init__(self, df_result, parquet_storage_path, logger=None, fs=None):
+    def __init__(self, df_result, parquet_storage_path, logger=None, fs=None, debug=False):
         """
         Initialize ParquetSaver.
         :param df_result: Dask DataFrame to save.
@@ -26,8 +27,9 @@ class ParquetSaver:
         """
         self.df_result = df_result
         self.parquet_storage_path = parquet_storage_path.rstrip("/")
+        self.debug = debug
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
-
+        self.logger.set_level(logging.DEBUG if self.debug else logging.INFO)
         self.fs = fs
         self.protocol = self.parquet_storage_path.split(":")[0]
 

sibi_dst/utils/phone_formatter.py

@@ -0,0 +1,127 @@
+import re
+from enum import Enum
+from typing import Optional, Union, Callable
+
+class CountryCode(Enum):
+    """Enum for supported country codes, including phone number length and formatting rules."""
+
+    USA = ("1", 10, lambda number: f"({number[:3]}) {number[3:6]}-{number[6:]}")
+    UK = ("44", 10, lambda number: f"{number[:2]} {number[2:6]} {number[6:]}")
+    FRANCE = ("33", 9, lambda number: f"{number[:1]} {number[1:3]} {number[3:5]} {number[5:]}")
+    SPAIN = ("34", 9, lambda number: f"{number[:2]} {number[2:5]} {number[5:]}")
+    DEFAULT = ("506", 8, lambda number: f"{number[:4]}-{number[4:]}")
+
+    def __init__(self, code: str, length: int, formatter: Callable[[str], str]):
+        """
+        Initialize a CountryCode enum member.
+
+        :param code: The country code.
+        :type code: str
+        :param length: The expected length of the phone number (excluding the country code).
+        :type length: int
+        :param formatter: A function to format the phone number.
+        :type formatter: Callable[[str], str]
+        """
+        self.code = code
+        self.length = length
+        self.formatter = formatter
+
+    @property
+    def value(self) -> str:
+        """
+        Get the country code value.
+
+        :return: The country code.
+        :rtype: str
+        """
+        return self.code
+
+    def validate_length(self, number: str) -> bool:
+        """
+        Validate the length of the phone number for this country.
+
+        :param number: The phone number part to validate.
+        :type number: str
+        :return: True if the number length is valid, False otherwise.
+        :rtype: bool
+        """
+        return len(number) == self.length
+
+    def format_number(self, number: str) -> str:
+        """
+        Format the phone number according to this country's rules.
+
+        :param number: The phone number part to format.
+        :type number: str
+        :return: The formatted number.
+        :rtype: str
+        """
+        return self.formatter(number)
+
+class PhoneNumberFormatter:
+    """
+    A utility class for validating and formatting phone numbers based on country-specific rules.
+
+    The class supports phone numbers for the UK, USA, France, and Spain. It detects the country code
+    from the input or uses a default country code if missing. Phone numbers are formatted according
+    to country-specific rules.
+    """
+
+    def __init__(self, default_country_code: CountryCode = CountryCode.DEFAULT):
+        """
+        Initialize the PhoneNumberFormatter with a default country code.
+
+        :param default_country_code: The default country code to use if missing.
+        :type default_country_code: CountryCode
+        """
+        self.default_country_code = default_country_code
+
+    def format_phone_number(self, phone_number: Union[str, int, float]) -> Optional[str]:
+        """
+        Validate and format a phone number according to country-specific rules.
+
+        If the input is numeric (e.g., an integer or float), it will be converted to a string.
+        If the country code is missing, the default country code will be used. The phone number
+        will be formatted according to the detected country's rules.
+
+        :param phone_number: The phone number to validate and format. Can be a string, integer, or float.
+        :type phone_number: Union[str, int, float]
+        :return: The formatted phone number, or None if the input is invalid.
+        :rtype: Optional[str]
+        """
+        # Convert numeric input to string
+        if isinstance(phone_number, (int, float)):
+            phone_number = str(int(phone_number))  # Convert to integer first to remove decimal points
+
+        # Remove all non-digit characters
+        digits = re.sub(r"\D", "", phone_number)
+
+        # Validate the length of the phone number
+        if not digits or len(digits) < 7:  # Minimum length for a valid phone number
+            return None
+
+        # Detect the country code
+        country_code, number = self._detect_country_code(digits)
+
+        # Validate the number length for the detected country
+        if not country_code.validate_length(number):
+            return None
+
+        # Format the phone number based on the country code
+        formatted_number = country_code.format_number(number)
+
+        return f"+{country_code.value} {formatted_number}"
+
+    def _detect_country_code(self, digits: str) -> tuple[CountryCode, str]:
+        """
+        Detect the country code from the input digits.
+
+        :param digits: The phone number digits (without non-digit characters).
+        :type digits: str
+        :return: A tuple containing the detected country code and the remaining number.
+        :rtype: tuple[CountryCode, str]
+        """
+        for country_code in CountryCode:
+            if digits.startswith(country_code.value):
+                return country_code, digits[len(country_code.value):]
+        return self.default_country_code, digits

sibi_dst/utils/storage_manager.py

@@ -4,13 +4,14 @@ import fsspec
 
 
 class StorageManager:
-    def __init__(self, storage_path, fs_type="file", fs_options=None):
+    def __init__(self, storage_path, fs_type="file", fs_options=None, debug=False):
         """
         Initializes the StorageManager with the base storage path and file system settings.
         :param storage_path: Base path for the storage (e.g., "s3://my-bucket").
         :param fs_type: File system type (e.g., "file", "s3").
         :param fs_options: Dictionary of options for fsspec file system (e.g., credentials).
         """
+        self.debug = debug
         # Ensure the storage_path ends with a slash for consistency
         self.storage_path = storage_path.rstrip("/")
         self.fs_type = fs_type
@@ -35,9 +36,11 @@ class StorageManager:
         :param dirs_to_create: List of subdirectories to create.
         :param clear_existing: Whether to clear existing directories.
         """
-        print(f"Setting up directories under: {base_path}")
+        if self.debug:
+            print(f"Setting up directories under: {base_path}")
         if clear_existing:
-            print(f"Warning: All existing contents in {base_path} will be removed.")
+            if self.debug:
+                print(f"Warning: All existing contents in {base_path} will be removed.")
             if self.fs.exists(base_path):
                 self.fs.rm(base_path, recursive=True)
 
@@ -47,7 +50,8 @@ class StorageManager:
         # Create subdirectories
         for sub_directory in dirs_to_create:
             sub_path = self.join_paths(base_path, sub_directory)
-            print(f"Creating directory: {sub_path}")
+            if self.debug:
+                print(f"Creating directory: {sub_path}")
             if clear_existing and self.fs.exists(sub_path):
                 self.fs.rm(sub_path, recursive=True)
             self.fs.mkdirs(sub_path, exist_ok=True)
@@ -63,7 +67,8 @@ class StorageManager:
         # Ensure directories exist (optionally clear existing ones)
         for depot, sub_directories in depots.items():
             depot_path = self.join_paths(self.storage_path, depot)
-            print(f"Rebuilding depot at: {depot_path}")
+            if self.debug:
+                print(f"Rebuilding depot at: {depot_path}")
             self.setup_directories(depot_path, sub_directories, clear_existing=clear_existing)
 
         # Generate depot_paths dictionary
@@ -89,9 +94,11 @@ class StorageManager:
         :param depots: Dictionary where keys are depot names and values are subdirectory lists.
         :param clear_existing: Whether to clear existing directories.
         """
-        print("Rebuilding depot structure...")
+        if self.debug:
+            print("Rebuilding depot structure...")
         self.rebuild_depot_paths(depots, clear_existing=clear_existing)
-        print("Rebuild complete.")
+        if self.debug:
+            print("Rebuild complete.")
 
     def get_fs_instance(self):
         return fsspec.filesystem(self.fs_type, **self.fs_options)