sibi-dst 0.3.15__py3-none-any.whl → 0.3.17__py3-none-any.whl

sibi_dst/df_helper/_df_helper.py

@@ -1,13 +1,16 @@
 import asyncio
 import datetime
+import logging
+import warnings
 from typing import Any, Dict, TypeVar
 from typing import Union, Optional
 
 import dask.dataframe as dd
+import dask_expr
 import pandas as pd
 from pydantic import BaseModel
 
-from sibi_dst.df_helper.core import QueryConfig, ParamsConfig
+from sibi_dst.df_helper.core import QueryConfig, ParamsConfig, FilterHandler
 from sibi_dst.utils import Logger
 from sibi_dst.utils import ParquetSaver, ClickHouseWriter
 from .plugins.django import *
@@ -18,6 +21,12 @@ from .plugins.sql_alchemy import *
 # Define a generic type variable for BaseModel subclasses
 T = TypeVar("T", bound=BaseModel)
 
+# It is considered acceptable in Django to access protected class members
+warnings.filterwarnings(
+    "ignore",
+    message="Access to a protected member _meta",
+    category=UserWarning,
+)
 
 class DfHelper:
     df: Union[dd.DataFrame, pd.DataFrame] = None
@@ -36,9 +45,12 @@ class DfHelper:
         self.default_config = self.default_config or {}
         kwargs = {**self.default_config.copy(), **kwargs}
         self.source = source
-        self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
         self.debug = kwargs.setdefault("debug", False)
-        self.verbose_debug = kwargs.setdefault("verbose_debug", False)
+        self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
+        # Configure logger level
+        self.logger.setLevel(logging.DEBUG if self.debug else logging.INFO)
+        # Configure logger level
+        self.logger.debug("Logger initialized in DEBUG mode.")
         self.parquet_storage_path = kwargs.setdefault("parquet_storage_path", None)
         self.dt_field = kwargs.setdefault("dt_field", None)
         self.as_pandas = kwargs.setdefault("as_pandas", False)
@@ -47,7 +59,7 @@ class DfHelper:
         self.post_init(**kwargs)
 
     def post_init(self, **kwargs):
-        self.logger.info(f"Source used: {self.source}")
+        self.logger.debug(f"Source used: {self.source}")
         self.plugin_query = self.__get_config(QueryConfig, kwargs)
         self.plugin_params = self.__get_config(ParamsConfig, kwargs)
         if self.source == 'django_db':
@@ -93,16 +105,15 @@ class DfHelper:
             return self._load_from_parquet(**options)
         elif self.source == 'http':
             if asyncio.get_event_loop().is_running():
-                self.logger.info("Running as a task from an event loop")
+                self.logger.debug("Running as a task from an event loop")
                 return asyncio.create_task(self._load_from_http(**options))
             else:
-                self.logger.info("Regular asyncio run...")
+                self.logger.debug("Regular asyncio run...")
                 return asyncio.run(self._load_from_http(**options))
 
     def _load_from_sqlalchemy(self, **options):
         try:
             options.setdefault("debug", self.debug)
-            options.setdefault("verbose_debug", self.verbose_debug)
             db_loader = SqlAlchemyLoadFromDb(
                 self.plugin_sqlalchemy,
                 self.plugin_query,
@@ -113,9 +124,9 @@ class DfHelper:
             self.df = db_loader.build_and_load()
             self._process_loaded_data()
             self._post_process_df()
-            self.logger.info("Data successfully loaded from sqlalchemy database.")
+            self.logger.debug("Data successfully loaded from sqlalchemy database.")
         except Exception as e:
-            self.logger.error(f"Failed to load data from sqlalchemy database: {e}: options: {options}")
+            self.logger.debug(f"Failed to load data from sqlalchemy database: {e}: options: {options}")
             self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
 
         return self.df
@@ -123,7 +134,6 @@ class DfHelper:
     def _load_from_db(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
         try:
             options.setdefault("debug", self.debug)
-            options.setdefault("verbose_debug", self.verbose_debug)
             db_loader = DjangoLoadFromDb(
                 self.plugin_django_connection,
                 self.plugin_query,
@@ -134,9 +144,9 @@ class DfHelper:
             self.df = db_loader.build_and_load()
             self._process_loaded_data()
             self._post_process_df()
-            self.logger.info("Data successfully loaded from django database.")
+            self.logger.debug("Data successfully loaded from django database.")
         except Exception as e:
-            self.logger.error(f"Failed to load data from django database: {e}")
+            self.logger.debug(f"Failed to load data from django database: {e}")
             self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
 
         return self.df
@@ -144,12 +154,12 @@ class DfHelper:
     async def _load_from_http(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
         """Delegate asynchronous HTTP data loading to HttpDataSource plugin."""
         if not self.plugin_http:
-            self.logger.error("HTTP plugin not configured properly.")
+            self.logger.debug("HTTP plugin not configured properly.")
             return dd.from_pandas(pd.DataFrame(), npartitions=1)
         try:
             self.df = await self.plugin_http.fetch_data(**options)
         except Exception as e:
-            self.logger.error(f"Failed to load data from http plugin: {e}")
+            self.logger.debug(f"Failed to load data from http plugin: {e}")
             self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
         return self.df
 
@@ -190,10 +200,10 @@ class DfHelper:
         if datetime_index and self.df.index.dtype != 'datetime64[ns]':
             self.df = self.df.map_partitions(lambda df: df.set_index(pd.to_datetime(df.index, errors='coerce')))
 
-        self.logger.info("Post-processing of DataFrame completed.")
+        self.logger.debug("Post-processing of DataFrame completed.")
 
     def _process_loaded_data(self):
-        self.logger.info(f"Type of self.df: {type(self.df)}")
+        self.logger.debug(f"Type of self.df: {type(self.df)}")
         if self.df.map_partitions(len).compute().sum() > 0:
             field_map = self.plugin_params.field_map or {}
             if isinstance(field_map, dict):
@@ -211,25 +221,30 @@ class DfHelper:
                     # Apply renaming
                     self.df = self.df.map_partitions(rename_columns, mapping=rename_mapping)
 
-            self.logger.info("Processing of loaded data completed.")
+            self.logger.debug("Processing of loaded data completed.")
 
     def save_to_parquet(self, parquet_filename: Optional[str] = None):
         ps = ParquetSaver(self.df, self.parquet_storage_path, self.logger)
         ps.save_to_parquet(parquet_filename)
-        self.logger.info(f"Parquet saved to {parquet_filename} in parquet storage: {self.parquet_storage_path}.")
+        self.logger.debug(f"Parquet saved to {parquet_filename} in parquet storage: {self.parquet_storage_path}.")
 
     def save_to_clickhouse(self, **credentials):
         if self.df.map_partitions(len).compute().sum() == 0:
-            self.logger.info("Cannot write to clickhouse since Dataframe is empty")
+            self.logger.debug("Cannot write to clickhouse since Dataframe is empty")
             return
         cs = ClickHouseWriter(logger=self.logger, **credentials)
         cs.save_to_clickhouse(self.df)
-        self.logger.info("Save to ClickHouse completed.")
+        self.logger.debug("Save to ClickHouse completed.")
 
     def _load_from_parquet(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
         self.df = self.plugin_parquet.load_files()
         if options:
+            """
+            deprecated specific filter handling to a generic one
             self.df = ParquetFilterHandler(logger=self.logger).apply_filters_dask(self.df, options)
+            
+            """
+            self.df = FilterHandler(backend='dask', logger=self.logger).apply_filters(self.df, filters=options)
         return self.df
 
     def load_period(self, **kwargs):
@@ -294,6 +309,7 @@ class DfHelper:
             elif is_datetime_field:
                 kwargs[f"{mapped_field}__date__gte"] = start
                 kwargs[f"{mapped_field}__date__lte"] = end
+        self.logger.debug(f"load_period kwargs: {kwargs}")
         return self.load(**kwargs)
 
     @staticmethod

sibi_dst/df_helper/core/__init__.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
-from ._params_config import ParamsConfig
-from ._query_config import QueryConfig
 from ._defaults import (
     django_field_conversion_map_pandas,
     django_field_conversion_map_dask,
     sqlalchemy_field_conversion_map_dask,
     normalize_sqlalchemy_type)
+from ._filter_handler import FilterHandler
+from ._params_config import ParamsConfig
+from ._query_config import QueryConfig
 
 __all__ = [
     "ParamsConfig",
@@ -14,5 +15,6 @@ __all__ = [
     "django_field_conversion_map_pandas",
     "django_field_conversion_map_dask",
     "sqlalchemy_field_conversion_map_dask",
-    "normalize_sqlalchemy_type"
-]
+    "normalize_sqlalchemy_type",
+    "FilterHandler",
+]

sibi_dst/df_helper/core/_filter_handler.py

@@ -0,0 +1,216 @@
+import datetime
+import dask.dataframe as dd
+import pandas as pd
+from sqlalchemy import func, cast
+from sqlalchemy.sql.sqltypes import Date, Time
+from sibi_dst.utils import Logger
+
+class FilterHandler:
+    def __init__(self, backend, logger=None):
+        """
+        Initialize the FilterHandler.
+
+        Args:
+            backend: The backend to use ('sqlalchemy' or 'dask').
+            logger: Optional logger for debugging purposes.
+        """
+        self.backend = backend
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)  # No-op logger if none provided
+        self.backend_methods = self._get_backend_methods(backend)
+
+    def apply_filters(self, query_or_df, model=None, filters=None):
+        """
+        Apply filters to the data source based on the backend.
+
+        Args:
+            query_or_df: SQLAlchemy query or Dask DataFrame.
+            model: SQLAlchemy model (required for SQLAlchemy backend).
+            filters: Dictionary of filters.
+
+        Returns:
+            Filtered query or DataFrame.
+        """
+        filters = filters or {}
+        for key, value in filters.items():
+            field_name, casting, operation = self._parse_filter_key(key)
+            parsed_value = self._parse_filter_value(casting, value)
+            #print(field_name, casting, operation, parsed_value)
+            # Get the column and apply backend-specific transformations
+            if self.backend == "sqlalchemy":
+                column = self.backend_methods["get_column"](field_name, model, casting)
+                condition = self.backend_methods["apply_operation"](column, operation, parsed_value)
+                query_or_df = self.backend_methods["apply_condition"](query_or_df, condition)
+
+            elif self.backend == "dask":
+                column = self.backend_methods["get_column"](query_or_df, field_name, casting)
+                condition = self.backend_methods["apply_operation"](column, operation, parsed_value)
+                query_or_df = self.backend_methods["apply_condition"](query_or_df, condition)
+            else:
+                raise ValueError(f"Unsupported backend: {self.backend}")
+
+        return query_or_df
+
+    @staticmethod
+    def _parse_filter_key(key):
+        parts = key.split("__")
+        field_name = parts[0]
+        casting = None
+        operation = "exact"
+
+        if len(parts) == 3:
+            _, casting, operation = parts
+        elif len(parts) == 2:
+            if parts[1] in FilterHandler._comparison_operators():
+                operation = parts[1]
+            elif parts[1] in FilterHandler._dt_operators() + FilterHandler._date_operators():
+                casting = parts[1]
+
+        return field_name, casting, operation
+
+
+    def _parse_filter_value(self, casting, value):
+        """
+        Convert filter value to appropriate type based on the casting (e.g., date).
+        """
+        if casting == "date":
+            if isinstance(value, str):
+                parsed = pd.Timestamp(value)  # Convert to datetime64[ns]
+                return parsed
+            if isinstance(value, list):
+                parsed = [pd.Timestamp(v) for v in value]
+                return parsed
+        elif casting == "time" and isinstance(value, str):
+            parsed = datetime.time.fromisoformat(value)
+            self.logger.debug(f"Parsed value (time): {parsed}")
+            return parsed
+        return value
+
+    @staticmethod
+    def _get_backend_methods(backend):
+        if backend == "sqlalchemy":
+            return {
+                "get_column": FilterHandler._get_sqlalchemy_column,
+                "apply_operation": FilterHandler._apply_operation_sqlalchemy,
+                "apply_condition": lambda query, condition: query.filter(condition),
+            }
+        elif backend == "dask":
+            return {
+                "get_column": FilterHandler._get_dask_column,
+                "apply_operation": FilterHandler._apply_operation_dask,
+                "apply_condition": lambda df, condition: df[condition],
+            }
+        else:
+            raise ValueError(f"Unsupported backend: {backend}")
+
+    @staticmethod
+    def _get_sqlalchemy_column(field_name, model, casting):
+        """
+        Retrieve and cast a column for SQLAlchemy based on the field name and casting.
+
+        Args:
+            field_name: The name of the field/column in the model.
+            model: The SQLAlchemy model.
+            casting: The casting type ('date', 'time', etc.).
+
+        Returns:
+            The SQLAlchemy column object, optionally cast or transformed.
+        """
+        column = getattr(model, field_name, None)
+        if not column:
+            raise AttributeError(f"Field '{field_name}' not found in model '{model.__name__}'")
+
+        if casting == "date":
+            # Cast the column to Date for whole-date comparisons
+            column = cast(column, Date)
+        elif casting == "time":
+            # Cast the column to Time for time-specific comparisons
+            column = cast(column, Time)
+        elif casting in FilterHandler._date_operators():
+            # Extract date part (e.g., year, month) using SQLAlchemy functions
+            column = func.extract(casting, column)
+
+        return column
+
+    @staticmethod
+    def _get_dask_column(df, field_name, casting):
+        """
+        Retrieve and optionally cast a column for Dask based on the field name and casting.
+
+        Args:
+            df: The Dask DataFrame.
+            field_name: The name of the field/column in the DataFrame.
+            casting: The casting type ('date', 'time', etc.).
+
+        Returns:
+            The Dask Series object, optionally cast or transformed.
+        """
+        column = dd.to_datetime(df[field_name], errors="coerce") if casting in FilterHandler._dt_operators() else df[
+            field_name]
+
+        if casting == "date":
+            column = column.dt.floor("D")  # Ensure truncation to the date level
+        elif casting in FilterHandler._date_operators():
+            column = getattr(column.dt, casting)
+
+        return column
+
+    @staticmethod
+    def _apply_operation_sqlalchemy(column, operation, value):
+        operation_map = FilterHandler._operation_map_sqlalchemy()
+        if operation not in operation_map:
+            raise ValueError(f"Unsupported operation: {operation}")
+        return operation_map[operation](column, value)
+
+    @staticmethod
+    def _apply_operation_dask(column, operation, value):
+        operation_map = FilterHandler._operation_map_dask()
+        if operation not in operation_map:
+            raise ValueError(f"Unsupported operation: {operation}")
+        return operation_map[operation](column, value)
+
+    @staticmethod
+    def _operation_map_sqlalchemy():
+        return {
+            "exact": lambda col, val: col == val,
+            "gt": lambda col, val: col > val,
+            "gte": lambda col, val: col >= val,
+            "lt": lambda col, val: col < val,
+            "lte": lambda col, val: col <= val,
+            "in": lambda col, val: col.in_(val),
+            "range": lambda col, val: col.between(val[0], val[1]),
+            "contains": lambda col, val: col.like(f"%{val}%"),
+            "startswith": lambda col, val: col.like(f"{val}%"),
+            "endswith": lambda col, val: col.like(f"%{val}"),
+            "isnull": lambda col, val: col.is_(None) if val else col.isnot(None),
+        }
+
+    @staticmethod
+    def _operation_map_dask():
+        return {
+            "exact": lambda col, val: col == val,
+            "gt": lambda col, val: col > val,
+            "gte": lambda col, val: col >= val,
+            "lt": lambda col, val: col < val,
+            "lte": lambda col, val: col <= val,
+            "in": lambda col, val: col.isin(val),
+            "range": lambda col, val: (col >= val[0]) & (col <= val[1]),
+            "contains": lambda col, val: col.str.contains(val, regex=True),
+            "startswith": lambda col, val: col.str.startswith(val),
+            "endswith": lambda col, val: col.str.endswith(val),
+            "isnull": lambda col, val: col.isnull() if val else col.notnull(),
+        }
+
+    @staticmethod
+    def _dt_operators():
+        return ["date", "time"]
+
+    @staticmethod
+    def _date_operators():
+        return ["year", "month", "day", "hour", "minute", "second", "week_day"]
+
+    @staticmethod
+    def _comparison_operators():
+        return [
+            "gte", "lte", "gt", "lt", "exact", "in", "range",
+            "contains", "startswith", "endswith", "isnull",
+        ]

sibi_dst/df_helper/plugins/django/_django_load_from_db.py

@@ -1,5 +1,8 @@
+import warnings
+
 import dask.dataframe as dd
 import pandas as pd
+from IPython.core.hooks import deprecated
 from django.db.models import Q
 
 from sibi_dst.df_helper.plugins.django import ReadFrameDask
@@ -12,13 +15,11 @@ class DjangoLoadFromDb:
     def __init__(self, db_connection, db_query, db_params, logger, **kwargs):
         self.connection_config = db_connection
         self.debug = kwargs.pop('debug', False)
-        self.verbose_debug = kwargs.pop('verbose_debug', False)
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
         if self.connection_config.model is None:
             if self.debug:
-                self.logger.critical('Model must be specified')
-                if self.verbose_debug:
-                    print('Model must be specified')
+                self.logger.debug('Model must be specified')
+
             raise ValueError('Model must be specified')
 
         self.query_config = db_query
@@ -45,7 +46,7 @@ class DjangoLoadFromDb:
             try:
                 self.df = ReadFrameDask(queryset, **self.params_config.df_params).read_frame()
             except Exception as e:
-                self.logger.critical(f'Error loading query: {str(queryset.query)}, error message: {e}')
+                self.logger.debug(f'Error loading query: {str(queryset.query)}, error message: {e}')
                 self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
         else:
             self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
@@ -69,16 +70,28 @@ class DjangoLoadFromDb:
         :param df: Dask DataFrame whose columns' data types are to be converted.
         :return: Dask DataFrame with converted column data types.
         """
-
-        def log_debug(message: str, is_verbose: bool = False):
-            """Helper to handle debug and verbose debug logging."""
-            if self.debug:
-                self.logger.debug(message)
-                if is_verbose and self.verbose_debug:
-                    print(message)
-
+        """
+            [DEPRECATED] Convert the data types of columns in a Dask DataFrame based on the field type in the Django model.
+
+            :param df: Dask DataFrame whose columns' data types are to be converted.
+            :return: Dask DataFrame with converted column data types.
+            """
+        # Emit deprecation warning
+        warnings.warn(
+            "_convert_columns is deprecated and will be removed in a future release. "
+            "Consider using <new_method_name> instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+        # Log deprecation message if debug mode is enabled
         if self.debug:
-            self.logger.info(f'Converting columns: {list(df.columns)}')
+            self.logger.warning(
+                "[DEPRECATION NOTICE] The `_convert_columns` method is deprecated and will be removed in a future release. "
+                "Consider using <new_method_name> instead."
+            )
+
+        self.logger.debug(f'Converting columns: {list(df.columns)}')
 
         # Get field information from the Django model
         model_fields = self.connection_config.model._meta.get_fields()
@@ -87,13 +100,13 @@ class DjangoLoadFromDb:
         for field_name, field_type in field_type_map.items():
             if field_name not in df.columns:
 
-                log_debug(f"Column '{field_name}' not found in DataFrame columns.")
+                self.logger.debug(f"Column '{field_name}' not found in DataFrame columns.")
                 continue
 
             conversion_func = django_field_conversion_map_dask.get(field_type)
             if not conversion_func:
                 message=f"Field type '{field_type}' not found in conversion_map."
-                log_debug(message, is_verbose=True)
+                self.logger.debug(message)
                 continue
 
             def apply_conversion(partition):
@@ -104,7 +117,7 @@ class DjangoLoadFromDb:
                     if field_name in partition.columns:
                         partition[field_name] = conversion_func(partition[field_name])
                 except Exception as e:
-                    self.logger.error(f"Error converting column '{field_name}' in partition: {str(e)}")
+                    self.logger.debug(f"Error converting column '{field_name}' in partition: {str(e)}")
                 return partition
 
             try:
@@ -113,9 +126,8 @@ class DjangoLoadFromDb:
                     apply_conversion,
                     meta=df,
                 )
-                log_debug(f"Successfully queued conversion for column '{field_name}' to type '{field_type}'.",
-                          is_verbose=True)
+                self.logger.debug(f"Successfully queued conversion for column '{field_name}' to type '{field_type}'.")
             except Exception as e:
-                log_debug(f"Failed to queue conversion for column '{field_name}': {str(e)}", is_verbose=True)
+                self.logger.debug(f"Failed to queue conversion for column '{field_name}': {str(e)}")
 
         return df

sibi_dst/df_helper/plugins/django/_io_dask.py

@@ -240,7 +240,4 @@ class ReadFrameDask:
         if verbose:
             self.update_with_verbose(dask_df, fieldnames, fields)
 
-        if isinstance(dask_df, dask_expr._collection.DataFrame):
-            dask_df = dask_df.to_legacy_dataframe()
-
         return dask_df

sibi_dst/df_helper/plugins/http/_http_config.py

@@ -31,17 +31,17 @@ class HttpConfig(BaseModel):
                 # Set up headers with API key if provided
             headers = {"Authorization": f"Bearer {self.api_key.get_secret_value()}"} if self.api_key else {}
 
-            self.logger.info(f"Fetching data from {formatted_url} with params {self.params}")
+            self.logger.debug(f"Fetching data from {formatted_url} with params {self.params}")
             async with httpx.AsyncClient() as client:
                 response = await client.get(formatted_url, params=self.params, headers=headers, timeout=self.timeout)
                 response.raise_for_status()
                 data = response.json()
                 df = dd.from_pandas(pd.json_normalize(data), npartitions=1)
-                self.logger.info("Data successfully loaded from HTTP JSON source.")
+                self.logger.debug("Data successfully loaded from HTTP JSON source.")
                 return df
         except httpx.RequestError as e:
-            self.logger.error(f"HTTP request error: {e}")
+            self.logger.debug(f"HTTP request error: {e}")
             raise
         except ValueError as e:
-            self.logger.error(f"Error parsing JSON data: {e}")
+            self.logger.debug(f"Error parsing JSON data: {e}")
             raise

sibi_dst/df_helper/plugins/sql_alchemy/_io_sqlalchemy_dask.py

@@ -8,7 +8,7 @@ from sqlalchemy.orm import sessionmaker
 
 from sibi_dst.df_helper.plugins.sql_alchemy._sqlachemy_filter_handler import SqlAlchemyFilterHandler
 from sibi_dst.utils import Logger
-
+from sibi_dst.df_helper.core import FilterHandler
 
 class SQLAlchemyDask:
     def __init__(self, model, filters, engine_url, chunk_size=1000, logger=None, debug=False):
@@ -38,7 +38,6 @@ class SQLAlchemyDask:
         """
         mapper = inspect(model)
         sqlalchemy_to_dask_dtype = {
-            #'INTEGER': pd.to_numeric(x, errors="coerce"),
             'INTEGER': 'Int64',
             'SMALLINT': 'Int64',
             'BIGINT': 'Int64',
@@ -72,11 +71,15 @@ class SQLAlchemyDask:
                 # Build query
                 self.query = select(self.model)
                 if self.filters:
-                    self.query = SqlAlchemyFilterHandler.apply_filters_sqlalchemy(self.query, self.model, self.filters)
+                    """
+                    deprecated specific filter handling to a generic one
+                    #self.query = SqlAlchemyFilterHandler.apply_filters_sqlalchemy(self.query, self.model, self.filters)
+                    """
+                    self.query = FilterHandler(backend="sqlalchemy", logger=self.logger).apply_filters(self.query, model=self.model, filters=self.filters)
                 else:
                     n_records = 100
                     self.query = self.query.limit(n_records)
-
+                self.logger.debug(f"query:{self.query}")
                 # Infer dtypes
                 dtypes = self.infer_dtypes_from_model(self.model)
                 # Get the column order from the SQLAlchemy model
@@ -124,14 +127,11 @@ class SQLAlchemyDask:
                 else:
                     dask_df = dd.from_pandas(pd.DataFrame(columns=ordered_columns), npartitions=1)
 
-                if self.debug:
-                    self.logger.info(f"Loaded {len(dask_df)} rows into Dask DataFrame.")
-
-                if isinstance(dask_df, dask_expr._collection.DataFrame):
-                    dask_df = dask_df.to_legacy_dataframe()
+                self.logger.debug(f"Loaded {len(dask_df)} rows into Dask DataFrame.")
 
                 return dask_df
 
             except Exception as e:
                 self.logger.error(f"Error executing query: {str(e)}")
+                self.logger.error(self.query)
                 return dd.from_pandas(pd.DataFrame(columns=ordered_columns), npartitions=1)

sibi_dst/df_helper/plugins/sql_alchemy/_sqlachemy_filter_handler.py

@@ -1,7 +1,9 @@
-from sqlalchemy import and_, or_, not_, func, cast
-from sqlalchemy.sql.sqltypes import Date, Time
 import datetime
 
+from sqlalchemy import func, cast
+from sqlalchemy.sql.sqltypes import Date, Time
+
+
 class SqlAlchemyFilterHandler:
     @staticmethod
     def apply_filters_sqlalchemy(query, model, filters):

sibi_dst/df_helper/plugins/sql_alchemy/_sqlalchemy_load_from_db.py

@@ -1,4 +1,5 @@
 import dask.dataframe as dd
+import dask_expr
 import pandas as pd
 
 from sibi_dst.df_helper.core import ParamsConfig, QueryConfig
@@ -28,7 +29,6 @@ class SqlAlchemyLoadFromDb:
         self.query_config = plugin_query
         self.params_config = plugin_params
         self.debug = kwargs.pop("debug", False)
-        self.verbose_debug = kwargs.pop("verbose_debug", False)
 
     def build_and_load(self) -> dd.DataFrame:
         """
@@ -40,7 +40,6 @@ class SqlAlchemyLoadFromDb:
     def _build_and_load(self) -> dd.DataFrame:
 
         try:
-            # reader = SQLAlchemyDask(model=self.model, filters=self.params_config.filters,engine_url=self.engine.url, logger=self.logger, chunk_size=1000, debug=self.debug)
             self.df = SQLAlchemyDask(
                 model=self.model,
                 filters=self.params_config.filters,
@@ -49,10 +48,13 @@ class SqlAlchemyLoadFromDb:
                 chunk_size=1000,
                 debug=self.debug).read_frame()
             if self.df is None or len(self.df.head().index) == 0:
-                self.logger.warning("Query returned no results.")
-                return dd.from_pandas(pd.DataFrame(), npartitions=1)
+                self.logger.debug("Query returned no results.")
+                dask_df=dd.from_pandas(pd.DataFrame(), npartitions=1)
 
+                return dask_df
             return self.df
         except Exception as e:
-            self.logger.error(f"Failed to load data into Dask DataFrame.{e}")
-            return dd.from_pandas(pd.DataFrame(), npartitions=1)
+            self.logger.debug(f"Failed to load data into Dask DataFrame.{e}")
+            dask_df = dd.from_pandas(pd.DataFrame(), npartitions=1)
+
+            return dask_df

sibi_dst/df_helper/plugins/sql_alchemy/_sqlalchemy_model_builder.py

@@ -59,7 +59,7 @@ class SqlAlchemyModelBuilder:
         attrs = {
             "__tablename__": self.table_name,
             "__table__": self.table,
-            #"__module__": f"{apps_label}.models",
+            "__module__": f"{apps_label}.models",
             "__mapper_args__": {"eager_defaults": True},
         }
 

sibi_dst/df_helper/plugins/sql_model/_sqlmodel_load_from_db.py

@@ -26,7 +26,6 @@ class SQLModelLoadFromDb:
         self.query_config = db_query or {}
         self.params_config = db_params or {}
         self.debug = kwargs.pop("debug", False)
-        self.verbose_debug = kwargs.pop("verbose_debug", False)
 
     def _default_logger(self):
         """Create a default logger."""
@@ -69,7 +68,7 @@ class SQLModelLoadFromDb:
                     query = query.limit(n_records)
 
                 # Debug: Log the SQL query
-                self.logger.info(f"Executing query: {str(query)}")
+                self.logger.debug(f"Executing query: {str(query)}")
 
                 # Execute the query
                 results = session.exec(query).fetchall()
@@ -79,7 +78,7 @@ class SQLModelLoadFromDb:
                 if results:
                     df = dd.from_pandas(pd.DataFrame([r.dict() for r in results]), npartitions=1)
                 else:
-                    self.logger.warning("Query returned no results.")
+                    self.logger.debug("Query returned no results.")
                     df = dd.from_pandas(pd.DataFrame(), npartitions=1)
 
             except Exception as e:

sibi_dst/utils/__init__.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
-from ._credentials import ConfigManager, ConfigLoader
+from ._credentials import *
 from ._log_utils import Logger
-from ._date_utils import DateUtils, BusinessDays
+from ._date_utils import *
 from ._data_utils import DataUtils
 from ._file_utils import FileUtils
 from ._filepath_generator import FilePathGenerator

sibi_dst/utils/_clickhouse_writer.py

@@ -34,7 +34,7 @@ class ClickHouseWriter:
         self.df = df.copy()
         self.order_by = kwargs.setdefault('order_by',self.order_by)
         if len(self.df.head().index) == 0:
-            self.logger.info("Dataframe is empty")
+            self.logger.debug("Dataframe is empty")
             return
         self._handle_missing_values()
         self._connect()
@@ -51,7 +51,7 @@ class ClickHouseWriter:
                 user=self.clickhouse_user,
                 password=self.clickhouse_password
             )
-            self.logger.info("Connected to ClickHouse")
+            self.logger.debug("Connected to ClickHouse")
         except Exception as e:
             self.logger.error(e)
             raise
@@ -80,7 +80,7 @@ class ClickHouseWriter:
     def _drop_table(self):
         if self.client:
             self.client.command('DROP TABLE IF EXISTS {}'.format(self.clickhouse_table))
-            self.logger.info(f"Dropped table {self.clickhouse_table}")
+            self.logger.debug(f"Dropped table {self.clickhouse_table}")
 
     def _create_table_from_dask(self, engine=None):
         if engine is None:
@@ -88,18 +88,18 @@ class ClickHouseWriter:
         dtypes = self.df.dtypes
         clickhouse_schema = self._generate_clickhouse_schema(dtypes,self.dtype_to_clickhouse)
         create_table_sql= f"CREATE TABLE IF NOT EXISTS {self.clickhouse_table} ({clickhouse_schema}) {engine};"
-        self.logger.info(f"Creating table SQL:{create_table_sql}")
+        self.logger.debug(f"Creating table SQL:{create_table_sql}")
         if self.client:
             self.client.command(create_table_sql)
-            self.logger.info("Created table '{}'".format(self.clickhouse_table))
+            self.logger.debug("Created table '{}'".format(self.clickhouse_table))
 
     def _handle_missing_values(self):
         """
         Handle missing values in the Dask DataFrame before writing to ClickHouse.
         """
-        self.logger.info("Checking for missing values...")
+        self.logger.debug("Checking for missing values...")
         missing_counts = self.df.isnull().sum().compute()
-        self.logger.info(f"Missing values per column:\n{missing_counts}")
+        self.logger.debug(f"Missing values per column:\n{missing_counts}")
 
         # Replace missing values based on column types
         def replace_missing_values(df):
@@ -116,14 +116,14 @@ class ClickHouseWriter:
 
         # Apply replacement
         self.df = replace_missing_values(self.df)
-        self.logger.info("Missing values replaced.")
+        self.logger.debug("Missing values replaced.")
 
     def _write_data(self):
         """
         Writes the Dask DataFrame to a ClickHouse table partition by partition.
         """
         if len(self.df.head().index) == 0:
-            self.logger.info("No data found. Nothing written.")
+            self.logger.debug("No data found. Nothing written.")
             return
 
         for i, partition in enumerate(self.df.to_delayed()):
@@ -132,10 +132,10 @@ class ClickHouseWriter:
                 df = partition.compute()
 
                 if df.empty:
-                    self.logger.info(f"Partition {i} is empty. Skipping...")
+                    self.logger.debug(f"Partition {i} is empty. Skipping...")
                     continue
 
-                self.logger.info(f"Writing partition {i} with {len(df)} rows to ClickHouse.")
+                self.logger.debug(f"Writing partition {i} with {len(df)} rows to ClickHouse.")
 
                 # Write the partition to the ClickHouse table
                 self.client.insert_df(self.clickhouse_table, df)
@@ -148,7 +148,7 @@ class ClickHouseWriter:
         Ensures a separate client instance is used per thread to avoid session conflicts.
         """
         if len(self.df.index) == 0:
-            self.logger.info("No data found. Nothing written.")
+            self.logger.debug("No data found. Nothing written.")
             return
 
         def create_client():
@@ -170,13 +170,13 @@ class ClickHouseWriter:
             Write a single partition to ClickHouse using a separate client instance.
             """
             try:
-                self.logger.info(f"Starting to process partition {index}")
+                self.logger.debug(f"Starting to process partition {index}")
                 client = create_client()  # Create a new client for the thread
 
                 # Compute the Dask partition into a Pandas DataFrame
                 df = partition.compute()
                 if df.empty:
-                    self.logger.info(f"Partition {index} is empty. Skipping...")
+                    self.logger.debug(f"Partition {index} is empty. Skipping...")
                     return
 
                 # Convert DataFrame to list of tuples
@@ -184,7 +184,7 @@ class ClickHouseWriter:
                 columns = df.columns.tolist()
 
                 # Perform the insert
-                self.logger.info(f"Writing partition {index} with {len(df)} rows to ClickHouse.")
+                self.logger.debug(f"Writing partition {index} with {len(df)} rows to ClickHouse.")
                 client.execute(f"INSERT INTO {self.clickhouse_table} ({', '.join(columns)}) VALUES", data)
 
             except Exception as e:
@@ -192,7 +192,7 @@ class ClickHouseWriter:
             finally:
                 if 'client' in locals() and hasattr(client, 'close'):
                     client.close()
-                    self.logger.info(f"Closed client for partition {index}")
+                    self.logger.debug(f"Closed client for partition {index}")
 
         try:
             # Get delayed partitions and enumerate them

sibi_dst/utils/_data_utils.py

@@ -68,6 +68,7 @@ class DataUtils:
         - pandas.DataFrame or dask.dataframe.DataFrame: Updated DataFrame with merged lookup data.
         """
         # Return early if the DataFrame is empty
+        debug = kwargs.setdefault("debug", False)
         if self.is_dataframe_empty(df):
             return df
 
@@ -88,7 +89,7 @@ class DataUtils:
         column_names = kwargs.pop('column_names', ['temp_join_col', source_description_alias])
 
         if source_col not in df.columns:
-            self.logger.info(f"{source_col} not in DataFrame columns")
+            self.logger.debug(f"{source_col} not in DataFrame columns")
             return df
 
         # Get unique IDs from source column
@@ -99,7 +100,7 @@ class DataUtils:
 
         # Check if any IDs are found
         if not len(ids):
-            self.logger.info(f"No IDs found in the source column: {source_col}")
+            self.logger.debug(f"No IDs found in the source column: {source_col}")
             return df
 
         # Convert to a list only if necessary and sort
@@ -114,10 +115,10 @@ class DataUtils:
             f'{lookup_col}__in': ids
         })
         # Load lookup data
-        lookup_instance = classname(debug=True, verbose_debug=True)
+        lookup_instance = classname(debug=debug)
         result = lookup_instance.load(**load_kwargs)
         if len(result.index) == 0:
-            self.logger.info(f"No IDs found in the source column: {source_col}")
+            self.logger.debug(f"No IDs found in the source column: {source_col}")
             return df
         # Determine the join column on the result DataFrame
         temp_join_col = 'temp_join_col' if 'temp_join_col' in column_names else lookup_col

sibi_dst/utils/_data_wrapper.py

@@ -1,12 +1,15 @@
 import datetime
 from typing import Type, Any, Dict, Optional
+
 import fsspec
 import pandas as pd
 from IPython.display import display
-from sibi_dst.utils import Logger
 from tqdm import tqdm
+
+from sibi_dst.utils import Logger
 from sibi_dst.utils import ParquetSaver
 
+
 class DataWrapper:
     DEFAULT_MAX_AGE_MINUTES = 1440
     DEFAULT_HISTORY_DAYS_THRESHOLD = 30
@@ -112,7 +115,7 @@ class DataWrapper:
             file_age_minutes = (current_time - file_modification_datetime).total_seconds() / 60
 
             if self.verbose:
-                self.logger.info(
+                self.logger.debug(
                     f"File {file_path} is {round(file_age_minutes, 2)} minutes old "
                     f"(threshold: {self.max_age_minutes} minutes)"
                 )
@@ -129,14 +132,14 @@ class DataWrapper:
         start_time = datetime.datetime.now()
 
         if self.verbose:
-            self.logger.info(f"Processing {full_parquet_filename}...")
+            self.logger.debug(f"Processing {full_parquet_filename}...")
 
         data_object = self.dataclass(**self.class_params)
         df = data_object.load_period(dt_field=self.date_field, start=date, end=date)
 
         if len(df.index)==0:
             if self.verbose:
-                self.logger.info("No data found for the specified date.")
+                self.logger.debug("No data found for the specified date.")
             return
 
         parquet_saver = ParquetSaver(df, folder, self.logger)
@@ -146,7 +149,7 @@ class DataWrapper:
         duration_seconds = (end_time - start_time).total_seconds()
 
         if self.verbose:
-            self.logger.info(
+            self.logger.debug(
                 f"Data saved to {full_parquet_filename}. Processing time: {duration_seconds:.2f} seconds"
             )
 

sibi_dst/utils/_df_utils.py

@@ -85,7 +85,7 @@ class DfUtils:
         # Ensure all specified columns exist in the DataFrame
         missing_columns = [col for col, _, _ in conditions if col not in df.columns]
         if missing_columns:
-            self.logger.info(f"The following columns are missing in the DataFrame: {', '.join(missing_columns)}")
+            self.logger.debug(f"The following columns are missing in the DataFrame: {', '.join(missing_columns)}")
             return df
 
         # Build the combined filtering condition
@@ -117,7 +117,7 @@ class DfUtils:
             DataFrame: Grouped DataFrame with counts.
         """
         if debug:
-            self.logger.info(f"Grouping by: {group_by_expr}")
+            self.logger.debug(f"Grouping by: {group_by_expr}")
 
         df_grouped = df.groupby(by=group_by_expr).size().reset_index(name=group_expr)
         return df_grouped
@@ -141,7 +141,7 @@ class DfUtils:
 
         if debug:
             df_duplicates = df[df.duplicated(subset=duplicate_expr)]
-            self.logger.info(f"Duplicate Rows based on columns {duplicate_expr} are:\n{df_duplicates}")
+            self.logger.debug(f"Duplicate Rows based on columns {duplicate_expr} are:\n{df_duplicates}")
 
         if sort_field:
             if isinstance(df, dd.DataFrame):
@@ -224,9 +224,9 @@ class DfUtils:
         Returns:
             DataFrame: Resampled pivot table.
         """
-        if isinstance(df, dd.DataFrame):
+        if isinstance(df, dd.core.DataFrame):
             # Implement Dask-compatible pivot and resample
-            self.logger.info("Performing summarization with Dask DataFrame.")
+            self.logger.debug("Performing summarization with Dask DataFrame.")
             # Ensure the index is a datetime for resampling
             if not isinstance(df.index, (pd.DatetimeIndex, dd.core.DatetimeIndex)):
                 self.logger.warning("Index is not a DatetimeIndex. Converting index to datetime.")

sibi_dst/utils/_log_utils.py

@@ -55,6 +55,9 @@ class Logger:
         log_file = log_file or logger_name
         return cls(log_dir=log_dir, logger_name=logger_name, log_file=log_file)
 
+    def setLevel(self, level):
+        self.logger.setLevel(level)
+
     def debug(self, msg):
         self.logger.debug(msg)
 

sibi_dst/utils/_parquet_saver.py

@@ -1,18 +1,16 @@
-import datetime
 from pathlib import Path
 from typing import Optional
 
-import dask.dataframe as dd
+import dask_expr
 import fsspec
-import pandas as pd
 import pyarrow as pa
+
 from sibi_dst.utils import Logger
 
+
 class ParquetSaver:
     def __init__(self, df_result, parquet_storage_path, logger=None):
         # Ensure df_result is a Dask DataFrame
-        if not isinstance(df_result, dd.DataFrame):
-            df_result = dd.from_pandas(df_result, npartitions=1)
         self.df_result = df_result
         self.parquet_storage_path = parquet_storage_path
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
@@ -106,106 +104,3 @@ class ParquetSaver:
             str(full_path), engine="pyarrow", schema=schema, write_index=False
         )
 
-# import datetime
-# from pathlib import Path
-# from typing import Optional
-#
-# import dask.dataframe as dd
-# import fsspec
-# import pandas as pd
-# import pyarrow as pa
-# from sibi_dst.utils import Logger
-#
-# class ParquetSaver:
-#     def __init__(self, df_result, parquet_storage_path, logger):
-#         self.df_result = df_result
-#         self.parquet_storage_path = parquet_storage_path
-#         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
-#
-#
-#     def save_to_parquet(self, parquet_filename: Optional[str] = None, clear_existing=True):
-#         full_path = self._construct_full_path(parquet_filename)
-#
-#         if len(self.df_result) == 0:
-#             self.logger.warning('No data to save')
-#             return  # Exit early if there's no data to save
-#
-#         # Ensure directory exists and clear if necessary
-#         self._ensure_directory_exists(full_path, clear_existing=True)
-#
-#         # Define schema and save DataFrame to parquet
-#         schema = self._define_schema()
-#         self._convert_dtypes(schema)
-#         self._save_dataframe_to_parquet(full_path, schema)
-#
-#     def _define_schema(self) -> pa.Schema:
-#         """Define a PyArrow schema dynamically based on df_result column types."""
-#         pandas_dtype_to_pa = {
-#             'object': pa.string(),
-#             'string': pa.string(),
-#             'Int64': pa.int64(),
-#             'int64': pa.int64(),
-#             'float64': pa.float64(),
-#             'bool': pa.bool_(),
-#             'boolean': pa.bool_(),  # pandas nullable boolean
-#             'datetime64[ns]': pa.timestamp('ns'),
-#             'timedelta[ns]': pa.duration('ns')
-#         }
-#
-#         fields = [
-#             pa.field(col, pandas_dtype_to_pa.get(str(dtype), pa.string()))
-#             for col, dtype in self.df_result.dtypes.items()
-#         ]
-#         return pa.schema(fields)
-#
-#     def _convert_dtypes(self, schema: pa.Schema):
-#         """Convert DataFrame columns to match the specified schema."""
-#         dtype_mapping = {}
-#         for field in schema:
-#             col_name = field.name
-#             if col_name in self.df_result.columns:
-#                 if pa.types.is_string(field.type):
-#                     dtype_mapping[col_name] = 'string'
-#                 elif pa.types.is_int64(field.type):
-#                     dtype_mapping[col_name] = 'Int64'  # pandas nullable integer
-#                 elif pa.types.is_float64(field.type):
-#                     dtype_mapping[col_name] = 'float64'
-#                 elif pa.types.is_boolean(field.type):
-#                     dtype_mapping[col_name] = 'boolean'  # pandas nullable boolean
-#                 elif pa.types.is_timestamp(field.type):
-#                     dtype_mapping[col_name] = 'datetime64[ns]'
-#                 else:
-#                     dtype_mapping[col_name] = 'object'  # Fallback to object
-#         self.df_result = self.df_result.astype(dtype_mapping)
-#
-#     def _construct_full_path(self, parquet_filename: Optional[str]) -> Path:
-#         """Construct and return the full path for the parquet file."""
-#         fs, base_path = fsspec.core.url_to_fs(self.parquet_storage_path)
-#         parquet_filename = parquet_filename or "default.parquet"
-#         return Path(base_path) / parquet_filename
-#
-#     @staticmethod
-#     def _ensure_directory_exists(full_path: Path, clear_existing=False):
-#         """Ensure that the directory for the path exists, clearing it if specified."""
-#         fs, _ = fsspec.core.url_to_fs(str(full_path))
-#         directory = str(full_path.parent)
-#
-#         if fs.exists(directory):
-#             if clear_existing:
-#                 fs.rm(directory, recursive=True)
-#         else:
-#             fs.mkdirs(directory, exist_ok=True)
-#
-#     def _save_dataframe_to_parquet(self, full_path: Path, schema: pa.Schema):
-#         """Save the DataFrame to parquet with fsspec using specified schema."""
-#         fs, _ = fsspec.core.url_to_fs(str(full_path))
-#         if fs.exists(full_path):
-#             fs.rm(full_path, recursive=True)
-#         if isinstance(self.df_result, dd.DataFrame):
-#             self.df_result.to_parquet(
-#                 str(full_path), engine="pyarrow", schema=schema, write_index=False
-#             )
-#         elif isinstance(self.df_result, pd.DataFrame):
-#             dd.from_pandas(self.df_result, npartitions=1).to_parquet(
-#                 str(full_path), engine="pyarrow", schema=schema, write_index=False
-#             )

{sibi_dst-0.3.15.dist-info → sibi_dst-0.3.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.15
+Version: 0.3.17
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -13,6 +13,7 @@ Requires-Dist: chardet (>=5.2.0,<6.0.0)
 Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
 Requires-Dist: clickhouse-connect (>=0.8.7,<0.9.0)
 Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
+Requires-Dist: dask-expr (>=1.1.20,<2.0.0)
 Requires-Dist: dask[complete] (>=2024.11.1,<2025.0.0)
 Requires-Dist: django (>=5.1.4,<6.0.0)
 Requires-Dist: djangorestframework (>=3.15.2,<4.0.0)

{sibi_dst-0.3.15.dist-info → sibi_dst-0.3.17.dist-info}/RECORD

@@ -1,45 +1,46 @@
 sibi_dst/__init__.py,sha256=1KaC0LYTHxjpENq-NXI325WcEYZ8GCBrHGkLoFxEcu0,251
 sibi_dst/df_helper/__init__.py,sha256=JXJBY47G6wOYhzNI646OBl3pSGWIy4282-3qPGYHU7w,167
-sibi_dst/df_helper/_df_helper.py,sha256=ZWhPj9K5q_amJ7eBOrvwAvncxRnI-baveKWWQWfyND8,13354
+sibi_dst/df_helper/_df_helper.py,sha256=yOapAc3MLQnylGKs0TG4Nmf8gaLdM7Nvzt4H1bEp8ik,13898
 sibi_dst/df_helper/_parquet_artifact.py,sha256=f5oHwXtsNW6-ONSFsRB0AniVefA0THzP92J-nugp9vo,4973
-sibi_dst/df_helper/core/__init__.py,sha256=NSYY_evzq6XEkO06Nz6xLH5KznzRGI44cLbrnN3zHXQ,503
+sibi_dst/df_helper/core/__init__.py,sha256=o4zDwgVmaijde3oix0ezb6KLxI5QFy-SGUhFTDVFLT4,569
 sibi_dst/df_helper/core/_defaults.py,sha256=pJU-lX7w4nrt0Anx35j08mVr_0oMGn1bTA_iCl_p1qI,6700
+sibi_dst/df_helper/core/_filter_handler.py,sha256=SYZqpX4Vt6GAGR0L0LohlDOdjLLWQXJDiWWqFG-lSu0,8563
 sibi_dst/df_helper/core/_params_config.py,sha256=hO-PddoaGjFebqJFgtn76WwVHcCjzPW3z5i3NyK6mDw,3475
 sibi_dst/df_helper/core/_query_config.py,sha256=HEiyR_fBJjIMum-PSQroY3KaefQ2SpW1w1SQS8oT-NU,489
 sibi_dst/df_helper/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/df_helper/plugins/django/__init__.py,sha256=VkvYql-PUyCKOfoKx5aGdUAki8X-kULfhnCvgSQzHDQ,311
 sibi_dst/df_helper/plugins/django/_django_db_connection.py,sha256=9fGvXXgqPk_TC7BvaSljxTgNDWoXi_LZxKabEgExznM,1654
-sibi_dst/df_helper/plugins/django/_django_load_from_db.py,sha256=NrkBb5LM1A_vo3wAotqj2sVVYIuTfFGrQqIXk3xOoDs,5162
+sibi_dst/df_helper/plugins/django/_django_load_from_db.py,sha256=NSPNCNzvEAR-4AuTSPQWJsidBvT9zRHAN6L3JC1xRV0,5591
 sibi_dst/df_helper/plugins/django/_django_sql_model_builder.py,sha256=GprCh2c6PFlRBUCir4hh8pmf4Eqb-4OIK6Vz_xXQjMw,14864
-sibi_dst/df_helper/plugins/django/_io_dask.py,sha256=aGaHziEMWK4zk9kkMNq2QtVevqVOCWqoAlXT1lVgRok,9198
+sibi_dst/df_helper/plugins/django/_io_dask.py,sha256=P3WmkuFzmWRzFchjsVD2OElIR3stuevwDH9G6Mu8IWE,9080
 sibi_dst/df_helper/plugins/django/_io_dask_alt.py,sha256=zDjLyYxBeL0ffn3yfE_7vqMLMpeEEk2o-zMr66sKkDw,6827
 sibi_dst/df_helper/plugins/http/__init__.py,sha256=AG9JSDRyVna2r1yxCQ9HcY32EaGnzWsfKgNLgPpSXjY,102
-sibi_dst/df_helper/plugins/http/_http_config.py,sha256=WH0d4vsxfZRhWrWI4iTVAnhsdY3421SBr9kXYZVfeYQ,2126
+sibi_dst/df_helper/plugins/http/_http_config.py,sha256=NN3bol7NgBTDv70yOX7hJkazt1-dAAdFWVkYyHdIXsI,2128
 sibi_dst/df_helper/plugins/parquet/__init__.py,sha256=ClkyIsIh_ovEwqm0dTrkXImbPjLDTVHW2NQqqfQwWAw,187
 sibi_dst/df_helper/plugins/parquet/_parquet_filter_handler.py,sha256=6iFvblnVq0qj89QvieQuYxe_2RPX5ArKfq5zBcEIj90,3660
 sibi_dst/df_helper/plugins/parquet/_parquet_options.py,sha256=suJC7LfNEWAo-7_R62YTMSRku3k8orysft83VxRUems,4394
 sibi_dst/df_helper/plugins/sql_alchemy/__init__.py,sha256=FHorj40SbHc0OBzQ_ieG6MG-HLbf0tw6I_5eoIjJkOI,369
-sibi_dst/df_helper/plugins/sql_alchemy/_io_sqlalchemy_dask.py,sha256=1WQ390XBFWOSXo0ea6-hz1LM6Ppi-j6ToZYr7sQBldE,5330
-sibi_dst/df_helper/plugins/sql_alchemy/_sqlachemy_filter_handler.py,sha256=H8ypUjLKzYYl9BerfJjX_Uv9qBVkBR-wZiQlh3uRQXg,4669
+sibi_dst/df_helper/plugins/sql_alchemy/_io_sqlalchemy_dask.py,sha256=fna8xZL8Ij6uCM_tZINO8vPdpJZaXs41gGzR4xn5zd8,5531
+sibi_dst/df_helper/plugins/sql_alchemy/_sqlachemy_filter_handler.py,sha256=KShsLJYGVxN0ps9Wot7fF0nR0wW9WzcPIcWZ9f5vdBo,4654
 sibi_dst/df_helper/plugins/sql_alchemy/_sqlalchemy_db_connection.py,sha256=HtMsfH5com4dLVJxh3wdMUpQI3mz0cKDJz0CmFS2S8U,1648
-sibi_dst/df_helper/plugins/sql_alchemy/_sqlalchemy_load_from_db.py,sha256=balWGKun0NKIfhLZW-_DCOhKuTzTo_C2NwZoKFwuSJo,2329
-sibi_dst/df_helper/plugins/sql_alchemy/_sqlalchemy_model_builder.py,sha256=rzzZdcRB5TS9uJ3ZIGQiNf04e3u2akqJEsoGCuyPE3c,4467
+sibi_dst/df_helper/plugins/sql_alchemy/_sqlalchemy_load_from_db.py,sha256=DBIM4kk86GxWkyiEZ4dSl_DdKa9SMvANCbympfzOqgQ,2169
+sibi_dst/df_helper/plugins/sql_alchemy/_sqlalchemy_model_builder.py,sha256=IQK2jOXMNJRQOSD0VQ0p11BeDGlvxD8NfFRilw9Go80,4466
 sibi_dst/df_helper/plugins/sql_model/__init__.py,sha256=MXd4OOdTqR4cENSV733SGodPO6eQMCexANs-3w0qL5U,226
 sibi_dst/df_helper/plugins/sql_model/_sqlmodel_db_connection.py,sha256=6jmMjKIv5Btysj3kZMaXQ98IqKQkhnOC-JWtb1B8rus,4265
-sibi_dst/df_helper/plugins/sql_model/_sqlmodel_load_from_db.py,sha256=bLD4tEcGDKkJCfSO4b13_89tzVJcpz55I6uw9D4ERnE,3751
-sibi_dst/utils/__init__.py,sha256=nkX7tASNn57kw998YdqQQGY8qXv2J4LC4-g0GoQSiic,831
+sibi_dst/df_helper/plugins/sql_model/_sqlmodel_load_from_db.py,sha256=jYwkIz7_E9Z6Mqw1a9TCWKWD146Tbx7mcQFxIpmKgKU,3686
+sibi_dst/utils/__init__.py,sha256=TV229dPIIEzU5qCLI1G6fnCZW-VirUwSuffp7z7OTFg,783
 sibi_dst/utils/_airflow_manager.py,sha256=rlt3eolR5QvtxWhAtBTCpHXvxftnKM-ibPMv3fVwNZk,7524
-sibi_dst/utils/_clickhouse_writer.py,sha256=kNBQeDn3D4Javrz5L8uU_5itf8Mrvm9l29uxcmcKlbg,8555
+sibi_dst/utils/_clickhouse_writer.py,sha256=JcnWN2635ATCOaFiB6NYglNXDwqKw0jC7Urs9WOZE20,8571
 sibi_dst/utils/_credentials.py,sha256=8i6z7y3y5S-6mSk4xrT2AwhzCA32mTn1n1iYX9IVyHk,1724
-sibi_dst/utils/_data_utils.py,sha256=uw0SW9G4GrvTX4IdUd8fmsMTMEG5aXOFcWOv4Au3H5g,7016
-sibi_dst/utils/_data_wrapper.py,sha256=SmNv1UoZLq7ovRVy4wipsWLMidKJXcRTp4HtxmaCQdk,9399
+sibi_dst/utils/_data_utils.py,sha256=ch4j5FEs8ZnniUzpbeLO-b4Yco_6nwCu71xHaVqMGi4,7050
+sibi_dst/utils/_data_wrapper.py,sha256=_hLZhKqSxcfXe8IyWM2paBxtW2JlOCq2jYhNGcInPi4,9406
 sibi_dst/utils/_date_utils.py,sha256=KYB07puKDrSG8tOm_i1HGX0TjLNUtSWjwfsCYBmW9co,10619
-sibi_dst/utils/_df_utils.py,sha256=9_dNYoZ9_ofU0t_sxMdsXALWCuh02gvqUrei-6Lhr6w,10910
+sibi_dst/utils/_df_utils.py,sha256=pjEfkof9hggXQgYerG0p4DXrwBeIRynJFg4IX3Yrb4c,10919
 sibi_dst/utils/_file_utils.py,sha256=5EN90c8N1n9d-_xwz2RzaYcXRMQY_rws2Q3EA3pNAog,1254
 sibi_dst/utils/_filepath_generator.py,sha256=ytPSZ9GYOnnSP25zwA-0NjFHupPRZyXwixWnn_68_n0,6686
-sibi_dst/utils/_log_utils.py,sha256=AAenyubYUjk77WqiaNkjgkxws3dnAMIdaGl2Ryz_cA4,2245
-sibi_dst/utils/_parquet_saver.py,sha256=-A0o_vucyYe7wlwiby_0_yS-ZfT2GHwImyQHrCIBNwk,9051
+sibi_dst/utils/_log_utils.py,sha256=rPp8z1UglwvqzBOOAvMOct0syQZ-54gGYafnJDRYZN4,2313
+sibi_dst/utils/_parquet_saver.py,sha256=3BK0XXgMOOAdIw4OzbwMxmDrzDw3_MKi8RTpulIVUe0,4367
 sibi_dst/utils/_storage_manager.py,sha256=KP2HBXnLUMMquqcO30ecfuoU7g1z8RtaV3Dv0TvEXoY,3856
-sibi_dst-0.3.15.dist-info/METADATA,sha256=0XU32Bgt1RYV7Y12lmDxq_YmHaXya5d2qMYfYP8Yic0,2090
-sibi_dst-0.3.15.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-sibi_dst-0.3.15.dist-info/RECORD,,
+sibi_dst-0.3.17.dist-info/METADATA,sha256=Aw__Wr7myZwJfWGRNFy0Ye5FLXUnGf6b14GW5KBDGtE,2133
+sibi_dst-0.3.17.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+sibi_dst-0.3.17.dist-info/RECORD,,