sibi-dst 0.3.11__tar.gz → 0.3.12__tar.gz

{sibi_dst-0.3.11 → sibi_dst-0.3.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.11
+Version: 0.3.12
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -9,6 +9,8 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: apache-airflow-client (>=2.10.0,<3.0.0)
+Requires-Dist: chardet (>=5.2.0,<6.0.0)
+Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
 Requires-Dist: clickhouse-connect (>=0.8.7,<0.9.0)
 Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
 Requires-Dist: dask[complete] (>=2024.11.1,<2025.0.0)
@@ -29,6 +31,7 @@ Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: sqlmodel (>=0.0.22,<0.0.23)
 Requires-Dist: tornado (>=6.4.1,<7.0.0)
 Requires-Dist: tqdm (>=4.67.0,<5.0.0)
+Requires-Dist: uvicorn (>=0.32.1,<0.33.0)
 Description-Content-Type: text/markdown
 
 # sibi-dst

{sibi_dst-0.3.11 → sibi_dst-0.3.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.11"
+version = "0.3.12"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"
@@ -29,6 +29,9 @@ pytest = "^8.3.3"
 clickhouse-connect = "^0.8.7"
 clickhouse-driver = "^0.2.9"
 paramiko = "^3.5.0"
+chardet = "^5.2.0"
+charset-normalizer = "^3.4.0"
+uvicorn = "^0.32.1"
 
 
 [build-system]

{sibi_dst-0.3.11 → sibi_dst-0.3.12}/sibi_dst/df_helper/_df_helper.py

@@ -42,6 +42,7 @@ class DfHelper:
         self.dt_field=kwargs.setdefault("dt_field", None)
         self.as_pandas = kwargs.setdefault("as_pandas", False)
         kwargs.setdefault("live", True)
+        kwargs.setdefault("logger", self.logger)
         self.post_init(**kwargs)
 
 
@@ -211,6 +212,7 @@ class DfHelper:
     def save_to_parquet(self, parquet_filename: Optional[str] = None):
         ps = ParquetSaver(self.df, self.parquet_storage_path, self.logger)
         ps.save_to_parquet(parquet_filename)
+        self.logger.info(f"Parquet saved to {parquet_filename} in parquet storage: {self.parquet_storage_path}.")
 
     def save_to_clickhouse(self, database, table, order_by=None, **credentials):
         click_config ={
@@ -219,13 +221,14 @@ class DfHelper:
             'order_by': order_by or 'id',
         }
         credentials = {**credentials, **click_config}
-        cs=ClickHouseWriter(**credentials)
+        cs=ClickHouseWriter(logger=self.logger, **credentials)
         cs.save_to_clickhouse(self.df)
+        self.logger.info("Save to ClickHouse completed.")
 
     def _load_from_parquet(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
         self.df = self.plugin_parquet.load_files()
         if options:
-            self.df = ParquetFilterHandler().apply_filters_dask(self.df, options)
+            self.df = ParquetFilterHandler(logger=self.logger).apply_filters_dask(self.df, options)
         return self.df
 
     def load_period(self, **kwargs):

{sibi_dst-0.3.11 → sibi_dst-0.3.12}/sibi_dst/df_helper/plugins/http/_http_config.py

@@ -13,11 +13,10 @@ class HttpConfig(BaseModel):
     api_key: Optional[SecretStr] = None
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
-    def __init__(self, **data):
+    def __init__(self, logger=None, **data):
         super().__init__(**data)
         # Initialize the logger if not provided
-        if not self.logger:
-            self.logger = Logger(log_dir='./logs/', logger_name="HttpDataSource", log_file='http_data_source.log')
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
 
     async def fetch_data(self, **options) -> dd.DataFrame:
         """Asynchronously fetch JSON data from HTTP endpoint, substituting options into the URL path."""

{sibi_dst-0.3.11 → sibi_dst-0.3.12}/sibi_dst/df_helper/plugins/parquet/_parquet_filter_handler.py

@@ -1,7 +1,11 @@
 import pandas as pd
 import dask.dataframe as dd
+from sibi_dst.utils import Logger
 
 class ParquetFilterHandler(object):
+    def __init__(self, logger=None):
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+
     @staticmethod
     def apply_filters_dask(df, filters):
         dt_operators = ['date', 'time']

{sibi_dst-0.3.11 → sibi_dst-0.3.12}/sibi_dst/df_helper/plugins/parquet/_parquet_options.py

@@ -25,6 +25,8 @@ class ParquetConfig(BaseModel):
     @model_validator(mode='after')
     def check_parquet_params(self):
         # Configure paths based on fsspec
+        if self.logger is None:
+            self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
         self.fs = fsspec.filesystem("file") if "://" not in str(self.parquet_storage_path) else fsspec.filesystem(str(self.parquet_storage_path).split("://")[0])
 
         # Validation for parquet path

{sibi_dst-0.3.11 → sibi_dst-0.3.12}/sibi_dst/utils/__init__.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 from ._credentials import ConfigManager, ConfigLoader
 from ._log_utils import Logger
-from ._date_utils import DateUtils
+from ._date_utils import DateUtils, BusinessDays
 from ._data_utils import DataUtils
 from ._file_utils import FileUtils
 from ._filepath_generator import FilePathGenerator
@@ -17,6 +17,7 @@ __all__=[
     "ConfigLoader",
     "Logger",
     "DateUtils",
+    "BusinessDays",
     "FileUtils",
     "DataWrapper",
     "DataUtils",

{sibi_dst-0.3.11 → sibi_dst-0.3.12}/sibi_dst/utils/_data_utils.py

@@ -7,6 +7,27 @@ class DataUtils:
     def __init__(self, logger=None):
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
 
+    def transform_numeric_cols(self, df, columns, fill_value=0, dtype=int):
+        if not columns:
+            self.logger.warning('No columns specified')
+
+        columns = [column for column in columns if column in df.columns]
+        for col in columns:
+            if isinstance(df, dd.DataFrame):
+                # Replace NaN with 0, then convert to boolean
+                df[col] = df[col].map_partitions(
+                    lambda s: pd.to_numeric(s, errors='coerce')  # Convert to numeric, invalid to NaN
+                    .fillna(fill_value)  # Replace NaN with 0
+                    .astype(dtype),
+                    meta=(col, dtype)
+                )
+            else:
+                # For Pandas DataFrame, handle mixed types and invalid values
+                df[col] = pd.to_numeric(df[col], errors='coerce')  # Convert to numeric, invalid to NaN
+                df[col] = df[col].fillna(fill_value).astype(dtype)
+
+        return df
+
     @staticmethod
     def transform_numeric_columns(df, columns=None, fill_value=0, transform_func=None):
         """
@@ -14,6 +35,7 @@ class DataUtils:
 
         Parameters:
         - df (pandas.DataFrame or dask.dataframe.DataFrame): The DataFrame.
+        - columns (list of str, optional): Specific columns to transform. If None, all numeric columns are transformed.
         - fill_value (int or float): The value to replace NA values with.
         - transform_func (callable, optional): The transformation function to apply.
           If None, no additional transformation is applied.
@@ -28,31 +50,28 @@ class DataUtils:
         if not columns:
             return df
 
+        columns = [column for column in columns if column in df.columns]
         # Default transformation function (identity) if none is provided
         if transform_func is None:
             transform_func = lambda x: x
 
-        # Apply transformations
-        for col in columns:
-            dtype = df[col].dtype
-            if pd.api.types.is_integer_dtype(dtype):
-                meta_type = 'int64'
-            elif pd.api.types.is_float_dtype(dtype):
-                meta_type = 'float64'
-            else:
-                continue  # Skip non-numeric columns
+        # Batch processing for Dask
+        if isinstance(df, dd.DataFrame):
+            def transform_partition(partition):
+                # Apply transformations for all numeric columns in a single pass
+                partition[columns] = partition[columns].fillna(fill_value).map(transform_func)
+                return partition
+
+            # Apply the transformation function to all specified columns
+            df = df.map_partitions(transform_partition, meta=df)
+        else:
+            # Pandas: Vectorized operations for all specified columns
+            df[columns] = df[columns].fillna(fill_value).map(transform_func)
 
-            df[col] = df[col].fillna(fill_value).astype(meta_type)
-            if isinstance(df, dd.DataFrame):
-                df[col] = df[col].map_partitions(
-                    lambda s: s.apply(transform_func), meta=(col, meta_type)
-                )
-            else:
-                df[col] = df[col].apply(transform_func)
         return df
 
     @staticmethod
-    def transform_boolean_columns(df, columns=None, sample_size=100):
+    def transform_boolean_columns(df, columns=None):
         """
         Detect if the provided columns in a DataFrame (Pandas or Dask) contain only 0 and 1
         and convert them to boolean. Detection is performed using a sample.
@@ -96,73 +115,67 @@ class DataUtils:
         Returns:
         - pandas.DataFrame or dask.dataframe.DataFrame: Updated DataFrame with merged lookup data.
         """
-        # Check if the DataFrame is empty
+        # Return early if the DataFrame is empty
         if self.is_dataframe_empty(df):
             return df
 
-        # Extract required parameters with default values
-        source_col = kwargs.pop('source_col', None)
-        lookup_col = kwargs.pop('lookup_col', None)
-        lookup_description_col = kwargs.pop('lookup_description_col', None)
-        source_description_alias = kwargs.pop('source_description_alias', None)
-        fillna_source_description_alias = kwargs.pop('fillna_source_description_alias', False)
-        fieldnames = kwargs.get('fieldnames', None)
-        column_names = kwargs.get('column_names', None)
+        # Extract and validate required parameters
+        required_params = ['source_col', 'lookup_col', 'lookup_description_col', 'source_description_alias']
+        missing_params = [param for param in required_params if param not in kwargs]
+        if missing_params:
+            raise ValueError(f"Missing required parameters: {', '.join(missing_params)}")
 
-        # Validate required parameters
-        if not all([source_col, lookup_col, lookup_description_col, source_description_alias]):
-            raise ValueError(
-                'source_col, lookup_col, lookup_description_col, and source_description_alias must be specified'
-            )
+        source_col = kwargs.pop('source_col')
+        lookup_col = kwargs.pop('lookup_col')
+        lookup_description_col = kwargs.pop('lookup_description_col')
+        source_description_alias = kwargs.pop('source_description_alias')
+
+        # Optional parameters with default values
+        fillna_source_description_alias = kwargs.pop('fillna_source_description_alias', False)
+        fieldnames = kwargs.pop('fieldnames', (lookup_col, lookup_description_col))
+        column_names = kwargs.pop('column_names', ['temp_join_col', source_description_alias])
 
         if source_col not in df.columns:
-            self.logger.info(f'{source_col} not in DataFrame columns')
+            self.logger.info(f"{source_col} not in DataFrame columns")
             return df
 
         # Get unique IDs from source column
         ids = df[source_col].dropna().unique()
         if isinstance(ids, dd.Series):
             ids = ids.compute()
-        ids = ids.tolist()
-
-        if not ids:
-            self.logger.info(f'No IDs found in the source column: {source_col}')
+        if not len(ids):
+            self.logger.info(f"No IDs found in the source column: {source_col}")
             return df
-
-        # Set default fieldnames and column_names if not provided
-        if fieldnames is None:
-            kwargs['fieldnames'] = (lookup_col, lookup_description_col)
-        if column_names is None:
-            kwargs['column_names'] = ['temp_join_col', source_description_alias]
-
+        ids = sorted(ids.tolist())
         # Prepare kwargs for loading lookup data
         load_kwargs = kwargs.copy()
-        load_kwargs[f'{lookup_col}__in'] = ids
-
+        load_kwargs.update({
+            'fieldnames': fieldnames,
+            'column_names': column_names,
+            f'{lookup_col}__in': ids
+        })
         # Load lookup data
-        lookup_instance = classname()
+        lookup_instance = classname(debug=True, verbose_debug=True)
         result = lookup_instance.load(**load_kwargs)
-
+        if len(result.index) == 0:
+            self.logger.info(f"No IDs found in the source column: {source_col}")
+            return df
         # Determine the join column on the result DataFrame
-        if 'temp_join_col' in kwargs.get("column_names", []):
-            temp_join_col = 'temp_join_col'
-        else:
-            temp_join_col = lookup_col
+        temp_join_col = 'temp_join_col' if 'temp_join_col' in column_names else lookup_col
 
         # Merge DataFrames
         df = df.merge(result, how='left', left_on=source_col, right_on=temp_join_col)
 
         if fillna_source_description_alias and source_description_alias in df.columns:
-            df[source_description_alias] = df[source_description_alias].fillna('')
+            df[source_description_alias]=df[source_description_alias].fillna('')
 
         # Drop temp_join_col if present
-        if 'temp_join_col' in df.columns:
-            df = df.drop(columns='temp_join_col')
+        df = df.drop(columns='temp_join_col', errors='ignore')
 
         return df
 
-    @staticmethod
-    def is_dataframe_empty(df):
+
+    def is_dataframe_empty(self, df):
         """
         Check if a DataFrame (Pandas or Dask) is empty.
 
@@ -173,14 +186,30 @@ class DataUtils:
         - bool: True if the DataFrame is empty, False otherwise.
         """
         if isinstance(df, dd.DataFrame):
-            df_size = df.map_partitions(len).sum().compute()
-            return df_size == 0
-        else:
+            try:
+                return len(df.index) == 0
+            except Exception as e:
+                self.logger.error(f"Error while processing Dask DataFrame: {e}")
+                return False
+        elif isinstance(df, pd.DataFrame):
             return df.empty
+        else:
+            self.logger.error("Input must be a pandas or dask DataFrame.")
+            return False
 
     @staticmethod
-    def convert_to_datetime(df, date_fields):
+    def convert_to_datetime_dask(df, date_fields):
+        """
+        Convert specified columns in a Dask DataFrame to datetime, handling errors gracefully.
+
+        Parameters:
+        - df (dask.dataframe.DataFrame): The Dask DataFrame containing the columns.
+        - date_fields (list of str): List of column names to convert to datetime.
+
+        Returns:
+        - dask.dataframe.DataFrame: Updated DataFrame with specified columns converted to datetime.
+        """
         for col in date_fields:
             if col in df.columns:
-                df[col] = pd.to_datetime(df[col], errors='coerce')
-        return df
+                df[col] = df[col].map_partitions(pd.to_datetime, errors="coerce", meta=(col, "datetime64[ns]"))
+        return df

sibi_dst-0.3.12/sibi_dst/utils/_data_wrapper.py

@@ -0,0 +1,238 @@
+import datetime
+from typing import Type, Any, Dict, Optional
+import fsspec
+import pandas as pd
+from IPython.display import display
+from sibi_dst.utils import Logger
+from tqdm import tqdm
+from sibi_dst.utils import ParquetSaver
+
+class DataWrapper:
+    DEFAULT_MAX_AGE_MINUTES = 1440
+    DEFAULT_HISTORY_DAYS_THRESHOLD = 30
+
+    def __init__(self,
+                 dataclass: Type,
+                 date_field: str,
+                 data_path: str,
+                 parquet_filename: str,
+                 start_date: Any,
+                 end_date: Any,
+                 filesystem_type: str = "file",
+                 filesystem_options: Optional[Dict] = None,
+                 verbose: bool = False,
+                 class_params: Optional[Dict] = None,
+                 load_params: Optional[Dict] = None,
+                 reverse_order: bool = False,
+                 overwrite: bool = False,
+                 ignore_missing: bool = False,
+                 logger: Optional[Logger] = None,
+                 max_age_minutes: int = DEFAULT_MAX_AGE_MINUTES,
+                 history_days_threshold: int = DEFAULT_HISTORY_DAYS_THRESHOLD,
+                 show_progress: bool = False):
+        self.dataclass = dataclass
+        self.date_field = date_field
+        self.data_path = self.ensure_forward_slash(data_path)
+        self.parquet_filename = parquet_filename
+        self.filesystem_type = filesystem_type
+        self.filesystem_options = filesystem_options or {}
+        self.fs = fsspec.filesystem(filesystem_type, **self.filesystem_options)
+        self.verbose = verbose
+        self.class_params = class_params or {}
+        self.load_params = load_params or {}
+        self.reverse_order = reverse_order
+        self.overwrite = overwrite
+        self.ignore_missing = ignore_missing
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+        self.max_age_minutes = max_age_minutes
+        self.history_days_threshold = history_days_threshold
+        self.show_progress = show_progress
+
+        self.start_date = self.convert_to_date(start_date)
+        self.end_date = self.convert_to_date(end_date)
+
+    @staticmethod
+    def convert_to_date(date: Any) -> datetime.date:
+        if isinstance(date, datetime.date):
+            return date
+        try:
+            return pd.to_datetime(date).date()
+        except ValueError as e:
+            raise ValueError(f"Error converting {date} to datetime: {e}")
+
+    @staticmethod
+    def ensure_forward_slash(path: str) -> str:
+        return path if path.endswith('/') else path + '/'
+
+    def generate_date_range(self):
+        """Generate a range of dates between start_date and end_date."""
+        date_range = pd.date_range(self.start_date, self.end_date, freq='D')
+        if self.reverse_order:
+            date_range = date_range[::-1]
+        for date in date_range:
+            yield date.date()
+
+    def process(self):
+        """Execute the update plan following the specified hierarchy."""
+        update_plan_table = self.generate_update_plan_with_conditions()
+
+        # Display the update plan table to the user if show_progress is True
+        if self.show_progress:
+            display(update_plan_table)
+
+        # Process files according to the hierarchy, considering only `update_required` dates
+        for category, description in [
+            ("overwrite", "Processing files due to overwrite=True"),
+            ("history_days", "Processing files within history_days_threshold"),
+            ("missing_files", "Processing missing files")
+        ]:
+            # Filter dates in the category where `update_required` is True
+            dates_to_process = update_plan_table[
+                (update_plan_table["update_category"] == category) & (update_plan_table["update_required"])
+            ]["date"].tolist()
+
+            date_iterator = dates_to_process
+            if self.show_progress:
+                date_iterator = tqdm(date_iterator, desc=description, unit="date")
+
+            for current_date in date_iterator:
+                self.process_date(current_date)
+
+    def is_file_older_than(self, file_path: str) -> bool:
+        """
+        Check if a file is older than the specified max_age_minutes.
+        """
+        try:
+            info = self.fs.info(file_path)
+            file_modification_time = info['mtime']
+            file_modification_datetime = datetime.datetime.fromtimestamp(
+                file_modification_time, tz=datetime.timezone.utc
+            )
+            current_time = datetime.datetime.now(datetime.timezone.utc)
+            file_age_minutes = (current_time - file_modification_datetime).total_seconds() / 60
+
+            if self.verbose:
+                self.logger.info(
+                    f"File {file_path} is {round(file_age_minutes, 2)} minutes old "
+                    f"(threshold: {self.max_age_minutes} minutes)"
+                )
+
+            return file_age_minutes > self.max_age_minutes
+        except FileNotFoundError:
+            return True  # Treat missing files as old
+
+    def process_date(self, date: datetime.date):
+        """Process a specific date by regenerating data as necessary."""
+        folder = f'{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/'
+        full_parquet_filename = f"{folder}{self.parquet_filename}"
+
+        start_time = datetime.datetime.now()
+
+        if self.verbose:
+            self.logger.info(f"Processing {full_parquet_filename}...")
+
+        data_object = self.dataclass(**self.class_params)
+        df = data_object.load_period(dt_field=self.date_field, start=date, end=date)
+
+        if len(df.index)==0:
+            if self.verbose:
+                self.logger.info("No data found for the specified date.")
+            return
+
+        parquet_saver = ParquetSaver(df, folder, self.logger)
+        parquet_saver.save_to_parquet(self.parquet_filename, clear_existing=True)
+
+        end_time = datetime.datetime.now()
+        duration_seconds = (end_time - start_time).total_seconds()
+
+        if self.verbose:
+            self.logger.info(
+                f"Data saved to {full_parquet_filename}. Processing time: {duration_seconds:.2f} seconds"
+            )
+
+    def generate_update_plan_with_conditions(self):
+        """
+        Generate an update plan that evaluates files based on the specified hierarchy:
+        1. Overwrite (all files regenerated).
+        2. History threshold: Files within `history_days_threshold` are evaluated for `max_age_minutes`.
+        3. Missing files: Detect missing files, ignoring future dates.
+        """
+        rows = []
+
+        today = datetime.date.today()
+        history_start_date = today - datetime.timedelta(days=self.history_days_threshold)
+
+        date_range = self.generate_date_range()
+        if self.show_progress:
+            date_range = tqdm(date_range, desc=f"Evaluating update plan {self.__class__.__name__}", unit="date")
+
+        for current_date in date_range:
+            folder = f'{self.data_path}{current_date.year}/{current_date.month:02d}/{current_date.day:02d}/'
+            full_parquet_filename = f"{folder}{self.parquet_filename}"
+
+            file_exists = self.fs.exists(full_parquet_filename)
+            within_history = history_start_date <= current_date <= today
+            missing_file = not file_exists and not self.ignore_missing
+            category = None
+
+            # Hierarchy 1: Overwrite
+            if self.overwrite:
+                category = "overwrite"
+                update_required = True
+            # Hierarchy 2: History threshold evaluation
+            elif within_history:
+                if self.is_file_older_than(full_parquet_filename):
+                    category = "history_days"
+                    update_required = True
+                else:
+                    update_required = False
+            # Hierarchy 3: Missing files
+            elif missing_file and current_date <= today:
+                category = "missing_files"
+                update_required = True
+            else:
+                update_required = False
+
+            # Collect condition descriptions for the update plan table
+            rows.append({
+                "date": current_date,
+                "file_exists": file_exists,
+                "within_history": within_history,
+                "missing_file": missing_file,
+                "update_required": update_required,
+                "update_category": category
+            })
+
+        update_plan_table = pd.DataFrame(rows)
+        return update_plan_table
+
+
+
+# # Usage:
+# # wrapper = DataWrapper(
+# #     dataclass=YourDataClass,
+# #     date_field="created_at",
+# #     data_path="/path/to/data",
+# #     parquet_filename="data.parquet",
+# #     start_date="2022-01-01",
+# #     end_date="2022-12-31",
+# #     filesystem_type="file",
+# #     verbose=True
+# # )
+# # wrapper.process()
+# # wrapper = DataWrapper(
+# #    dataclass=YourDataClass,
+# #    date_field="created_at",
+# #    data_path="s3://your-bucket-name/path/to/data",
+# #    parquet_filename="data.parquet",
+# #    start_date="2022-01-01",
+# #    end_date="2022-12-31",
+# #    filesystem_type="s3",
+# #    filesystem_options={
+# #        "key": "your_aws_access_key",
+# #        "secret": "your_aws_secret_key",
+# #        "client_kwargs": {"endpoint_url": "https://s3.amazonaws.com"}
+# #    },
+# #    verbose=True
+# #)
+# #wrapper.process()