sibi-dst 0.3.15__tar.gz → 0.3.16__tar.gz

{sibi_dst-0.3.15 → sibi_dst-0.3.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.15
+Version: 0.3.16
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -13,6 +13,7 @@ Requires-Dist: chardet (>=5.2.0,<6.0.0)
 Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
 Requires-Dist: clickhouse-connect (>=0.8.7,<0.9.0)
 Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
+Requires-Dist: dask-expr (>=1.1.20,<2.0.0)
 Requires-Dist: dask[complete] (>=2024.11.1,<2025.0.0)
 Requires-Dist: django (>=5.1.4,<6.0.0)
 Requires-Dist: djangorestframework (>=3.15.2,<4.0.0)

{sibi_dst-0.3.15 → sibi_dst-0.3.16}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.15"
+version = "0.3.16"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"
@@ -33,6 +33,7 @@ charset-normalizer = "^3.4.0"
 uvicorn = "^0.32.1"
 sqlalchemy = "^2.0.36"
 djangorestframework = "^3.15.2"
+dask-expr = "^1.1.20"
 
 
 [build-system]

{sibi_dst-0.3.15 → sibi_dst-0.3.16}/sibi_dst/df_helper/_df_helper.py

@@ -1,13 +1,16 @@
 import asyncio
 import datetime
+import logging
+import warnings
 from typing import Any, Dict, TypeVar
 from typing import Union, Optional
 
 import dask.dataframe as dd
+import dask_expr
 import pandas as pd
 from pydantic import BaseModel
 
-from sibi_dst.df_helper.core import QueryConfig, ParamsConfig
+from sibi_dst.df_helper.core import QueryConfig, ParamsConfig, FilterHandler
 from sibi_dst.utils import Logger
 from sibi_dst.utils import ParquetSaver, ClickHouseWriter
 from .plugins.django import *
@@ -18,6 +21,12 @@ from .plugins.sql_alchemy import *
 # Define a generic type variable for BaseModel subclasses
 T = TypeVar("T", bound=BaseModel)
 
+# It is considered acceptable in Django to access protected class members
+warnings.filterwarnings(
+    "ignore",
+    message="Access to a protected member _meta",
+    category=UserWarning,
+)
 
 class DfHelper:
     df: Union[dd.DataFrame, pd.DataFrame] = None
@@ -36,9 +45,12 @@ class DfHelper:
         self.default_config = self.default_config or {}
         kwargs = {**self.default_config.copy(), **kwargs}
         self.source = source
-        self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
         self.debug = kwargs.setdefault("debug", False)
-        self.verbose_debug = kwargs.setdefault("verbose_debug", False)
+        self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
+        # Configure logger level
+        self.logger.setLevel(logging.DEBUG if self.debug else logging.INFO)
+        # Configure logger level
+        self.logger.debug("Logger initialized in DEBUG mode.")
         self.parquet_storage_path = kwargs.setdefault("parquet_storage_path", None)
         self.dt_field = kwargs.setdefault("dt_field", None)
         self.as_pandas = kwargs.setdefault("as_pandas", False)
@@ -47,7 +59,7 @@ class DfHelper:
         self.post_init(**kwargs)
 
     def post_init(self, **kwargs):
-        self.logger.info(f"Source used: {self.source}")
+        self.logger.debug(f"Source used: {self.source}")
         self.plugin_query = self.__get_config(QueryConfig, kwargs)
         self.plugin_params = self.__get_config(ParamsConfig, kwargs)
         if self.source == 'django_db':
@@ -93,16 +105,15 @@ class DfHelper:
             return self._load_from_parquet(**options)
         elif self.source == 'http':
             if asyncio.get_event_loop().is_running():
-                self.logger.info("Running as a task from an event loop")
+                self.logger.debug("Running as a task from an event loop")
                 return asyncio.create_task(self._load_from_http(**options))
             else:
-                self.logger.info("Regular asyncio run...")
+                self.logger.debug("Regular asyncio run...")
                 return asyncio.run(self._load_from_http(**options))
 
     def _load_from_sqlalchemy(self, **options):
         try:
             options.setdefault("debug", self.debug)
-            options.setdefault("verbose_debug", self.verbose_debug)
             db_loader = SqlAlchemyLoadFromDb(
                 self.plugin_sqlalchemy,
                 self.plugin_query,
@@ -113,9 +124,9 @@ class DfHelper:
             self.df = db_loader.build_and_load()
             self._process_loaded_data()
             self._post_process_df()
-            self.logger.info("Data successfully loaded from sqlalchemy database.")
+            self.logger.debug("Data successfully loaded from sqlalchemy database.")
         except Exception as e:
-            self.logger.error(f"Failed to load data from sqlalchemy database: {e}: options: {options}")
+            self.logger.debug(f"Failed to load data from sqlalchemy database: {e}: options: {options}")
             self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
 
         return self.df
@@ -123,7 +134,6 @@ class DfHelper:
     def _load_from_db(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
         try:
             options.setdefault("debug", self.debug)
-            options.setdefault("verbose_debug", self.verbose_debug)
             db_loader = DjangoLoadFromDb(
                 self.plugin_django_connection,
                 self.plugin_query,
@@ -134,9 +144,9 @@ class DfHelper:
             self.df = db_loader.build_and_load()
             self._process_loaded_data()
             self._post_process_df()
-            self.logger.info("Data successfully loaded from django database.")
+            self.logger.debug("Data successfully loaded from django database.")
         except Exception as e:
-            self.logger.error(f"Failed to load data from django database: {e}")
+            self.logger.debug(f"Failed to load data from django database: {e}")
             self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
 
         return self.df
@@ -144,12 +154,12 @@ class DfHelper:
     async def _load_from_http(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
         """Delegate asynchronous HTTP data loading to HttpDataSource plugin."""
         if not self.plugin_http:
-            self.logger.error("HTTP plugin not configured properly.")
+            self.logger.debug("HTTP plugin not configured properly.")
             return dd.from_pandas(pd.DataFrame(), npartitions=1)
         try:
             self.df = await self.plugin_http.fetch_data(**options)
         except Exception as e:
-            self.logger.error(f"Failed to load data from http plugin: {e}")
+            self.logger.debug(f"Failed to load data from http plugin: {e}")
             self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
         return self.df
 
@@ -190,10 +200,10 @@ class DfHelper:
         if datetime_index and self.df.index.dtype != 'datetime64[ns]':
             self.df = self.df.map_partitions(lambda df: df.set_index(pd.to_datetime(df.index, errors='coerce')))
 
-        self.logger.info("Post-processing of DataFrame completed.")
+        self.logger.debug("Post-processing of DataFrame completed.")
 
     def _process_loaded_data(self):
-        self.logger.info(f"Type of self.df: {type(self.df)}")
+        self.logger.debug(f"Type of self.df: {type(self.df)}")
         if self.df.map_partitions(len).compute().sum() > 0:
             field_map = self.plugin_params.field_map or {}
             if isinstance(field_map, dict):
@@ -211,25 +221,30 @@ class DfHelper:
                     # Apply renaming
                     self.df = self.df.map_partitions(rename_columns, mapping=rename_mapping)
 
-            self.logger.info("Processing of loaded data completed.")
+            self.logger.debug("Processing of loaded data completed.")
 
     def save_to_parquet(self, parquet_filename: Optional[str] = None):
         ps = ParquetSaver(self.df, self.parquet_storage_path, self.logger)
         ps.save_to_parquet(parquet_filename)
-        self.logger.info(f"Parquet saved to {parquet_filename} in parquet storage: {self.parquet_storage_path}.")
+        self.logger.debug(f"Parquet saved to {parquet_filename} in parquet storage: {self.parquet_storage_path}.")
 
     def save_to_clickhouse(self, **credentials):
         if self.df.map_partitions(len).compute().sum() == 0:
-            self.logger.info("Cannot write to clickhouse since Dataframe is empty")
+            self.logger.debug("Cannot write to clickhouse since Dataframe is empty")
             return
         cs = ClickHouseWriter(logger=self.logger, **credentials)
         cs.save_to_clickhouse(self.df)
-        self.logger.info("Save to ClickHouse completed.")
+        self.logger.debug("Save to ClickHouse completed.")
 
     def _load_from_parquet(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
         self.df = self.plugin_parquet.load_files()
         if options:
+            """
+            deprecated specific filter handling to a generic one
             self.df = ParquetFilterHandler(logger=self.logger).apply_filters_dask(self.df, options)
+            
+            """
+            self.df = FilterHandler(backend='dask', logger=self.logger).apply_filters(self.df, filters=options)
         return self.df
 
     def load_period(self, **kwargs):
@@ -294,6 +309,7 @@ class DfHelper:
             elif is_datetime_field:
                 kwargs[f"{mapped_field}__date__gte"] = start
                 kwargs[f"{mapped_field}__date__lte"] = end
+        self.logger.debug(f"load_period kwargs: {kwargs}")
         return self.load(**kwargs)
 
     @staticmethod

{sibi_dst-0.3.15 → sibi_dst-0.3.16}/sibi_dst/df_helper/core/__init__.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
-from ._params_config import ParamsConfig
-from ._query_config import QueryConfig
 from ._defaults import (
     django_field_conversion_map_pandas,
     django_field_conversion_map_dask,
     sqlalchemy_field_conversion_map_dask,
     normalize_sqlalchemy_type)
+from ._filter_handler import FilterHandler
+from ._params_config import ParamsConfig
+from ._query_config import QueryConfig
 
 __all__ = [
     "ParamsConfig",
@@ -14,5 +15,6 @@ __all__ = [
     "django_field_conversion_map_pandas",
     "django_field_conversion_map_dask",
     "sqlalchemy_field_conversion_map_dask",
-    "normalize_sqlalchemy_type"
-]
+    "normalize_sqlalchemy_type",
+    "FilterHandler",
+]

sibi_dst-0.3.16/sibi_dst/df_helper/core/_filter_handler.py

@@ -0,0 +1,216 @@
+import datetime
+import dask.dataframe as dd
+import pandas as pd
+from sqlalchemy import func, cast
+from sqlalchemy.sql.sqltypes import Date, Time
+from sibi_dst.utils import Logger
+
+class FilterHandler:
+    def __init__(self, backend, logger=None):
+        """
+        Initialize the FilterHandler.
+
+        Args:
+            backend: The backend to use ('sqlalchemy' or 'dask').
+            logger: Optional logger for debugging purposes.
+        """
+        self.backend = backend
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)  # No-op logger if none provided
+        self.backend_methods = self._get_backend_methods(backend)
+
+    def apply_filters(self, query_or_df, model=None, filters=None):
+        """
+        Apply filters to the data source based on the backend.
+
+        Args:
+            query_or_df: SQLAlchemy query or Dask DataFrame.
+            model: SQLAlchemy model (required for SQLAlchemy backend).
+            filters: Dictionary of filters.
+
+        Returns:
+            Filtered query or DataFrame.
+        """
+        filters = filters or {}
+        for key, value in filters.items():
+            field_name, casting, operation = self._parse_filter_key(key)
+            parsed_value = self._parse_filter_value(casting, value)
+            #print(field_name, casting, operation, parsed_value)
+            # Get the column and apply backend-specific transformations
+            if self.backend == "sqlalchemy":
+                column = self.backend_methods["get_column"](field_name, model, casting)
+                condition = self.backend_methods["apply_operation"](column, operation, parsed_value)
+                query_or_df = self.backend_methods["apply_condition"](query_or_df, condition)
+
+            elif self.backend == "dask":
+                column = self.backend_methods["get_column"](query_or_df, field_name, casting)
+                condition = self.backend_methods["apply_operation"](column, operation, parsed_value)
+                query_or_df = self.backend_methods["apply_condition"](query_or_df, condition)
+            else:
+                raise ValueError(f"Unsupported backend: {self.backend}")
+
+        return query_or_df
+
+    @staticmethod
+    def _parse_filter_key(key):
+        parts = key.split("__")
+        field_name = parts[0]
+        casting = None
+        operation = "exact"
+
+        if len(parts) == 3:
+            _, casting, operation = parts
+        elif len(parts) == 2:
+            if parts[1] in FilterHandler._comparison_operators():
+                operation = parts[1]
+            elif parts[1] in FilterHandler._dt_operators() + FilterHandler._date_operators():
+                casting = parts[1]
+
+        return field_name, casting, operation
+
+
+    def _parse_filter_value(self, casting, value):
+        """
+        Convert filter value to appropriate type based on the casting (e.g., date).
+        """
+        if casting == "date":
+            if isinstance(value, str):
+                parsed = pd.Timestamp(value)  # Convert to datetime64[ns]
+                return parsed
+            if isinstance(value, list):
+                parsed = [pd.Timestamp(v) for v in value]
+                return parsed
+        elif casting == "time" and isinstance(value, str):
+            parsed = datetime.time.fromisoformat(value)
+            self.logger.debug(f"Parsed value (time): {parsed}")
+            return parsed
+        return value
+
+    @staticmethod
+    def _get_backend_methods(backend):
+        if backend == "sqlalchemy":
+            return {
+                "get_column": FilterHandler._get_sqlalchemy_column,
+                "apply_operation": FilterHandler._apply_operation_sqlalchemy,
+                "apply_condition": lambda query, condition: query.filter(condition),
+            }
+        elif backend == "dask":
+            return {
+                "get_column": FilterHandler._get_dask_column,
+                "apply_operation": FilterHandler._apply_operation_dask,
+                "apply_condition": lambda df, condition: df[condition],
+            }
+        else:
+            raise ValueError(f"Unsupported backend: {backend}")
+
+    @staticmethod
+    def _get_sqlalchemy_column(field_name, model, casting):
+        """
+        Retrieve and cast a column for SQLAlchemy based on the field name and casting.
+
+        Args:
+            field_name: The name of the field/column in the model.
+            model: The SQLAlchemy model.
+            casting: The casting type ('date', 'time', etc.).
+
+        Returns:
+            The SQLAlchemy column object, optionally cast or transformed.
+        """
+        column = getattr(model, field_name, None)
+        if not column:
+            raise AttributeError(f"Field '{field_name}' not found in model '{model.__name__}'")
+
+        if casting == "date":
+            # Cast the column to Date for whole-date comparisons
+            column = cast(column, Date)
+        elif casting == "time":
+            # Cast the column to Time for time-specific comparisons
+            column = cast(column, Time)
+        elif casting in FilterHandler._date_operators():
+            # Extract date part (e.g., year, month) using SQLAlchemy functions
+            column = func.extract(casting, column)
+
+        return column
+
+    @staticmethod
+    def _get_dask_column(df, field_name, casting):
+        """
+        Retrieve and optionally cast a column for Dask based on the field name and casting.
+
+        Args:
+            df: The Dask DataFrame.
+            field_name: The name of the field/column in the DataFrame.
+            casting: The casting type ('date', 'time', etc.).
+
+        Returns:
+            The Dask Series object, optionally cast or transformed.
+        """
+        column = dd.to_datetime(df[field_name], errors="coerce") if casting in FilterHandler._dt_operators() else df[
+            field_name]
+
+        if casting == "date":
+            column = column.dt.floor("D")  # Ensure truncation to the date level
+        elif casting in FilterHandler._date_operators():
+            column = getattr(column.dt, casting)
+
+        return column
+
+    @staticmethod
+    def _apply_operation_sqlalchemy(column, operation, value):
+        operation_map = FilterHandler._operation_map_sqlalchemy()
+        if operation not in operation_map:
+            raise ValueError(f"Unsupported operation: {operation}")
+        return operation_map[operation](column, value)
+
+    @staticmethod
+    def _apply_operation_dask(column, operation, value):
+        operation_map = FilterHandler._operation_map_dask()
+        if operation not in operation_map:
+            raise ValueError(f"Unsupported operation: {operation}")
+        return operation_map[operation](column, value)
+
+    @staticmethod
+    def _operation_map_sqlalchemy():
+        return {
+            "exact": lambda col, val: col == val,
+            "gt": lambda col, val: col > val,
+            "gte": lambda col, val: col >= val,
+            "lt": lambda col, val: col < val,
+            "lte": lambda col, val: col <= val,
+            "in": lambda col, val: col.in_(val),
+            "range": lambda col, val: col.between(val[0], val[1]),
+            "contains": lambda col, val: col.like(f"%{val}%"),
+            "startswith": lambda col, val: col.like(f"{val}%"),
+            "endswith": lambda col, val: col.like(f"%{val}"),
+            "isnull": lambda col, val: col.is_(None) if val else col.isnot(None),
+        }
+
+    @staticmethod
+    def _operation_map_dask():
+        return {
+            "exact": lambda col, val: col == val,
+            "gt": lambda col, val: col > val,
+            "gte": lambda col, val: col >= val,
+            "lt": lambda col, val: col < val,
+            "lte": lambda col, val: col <= val,
+            "in": lambda col, val: col.isin(val),
+            "range": lambda col, val: (col >= val[0]) & (col <= val[1]),
+            "contains": lambda col, val: col.str.contains(val, regex=True),
+            "startswith": lambda col, val: col.str.startswith(val),
+            "endswith": lambda col, val: col.str.endswith(val),
+            "isnull": lambda col, val: col.isnull() if val else col.notnull(),
+        }
+
+    @staticmethod
+    def _dt_operators():
+        return ["date", "time"]
+
+    @staticmethod
+    def _date_operators():
+        return ["year", "month", "day", "hour", "minute", "second", "week_day"]
+
+    @staticmethod
+    def _comparison_operators():
+        return [
+            "gte", "lte", "gt", "lt", "exact", "in", "range",
+            "contains", "startswith", "endswith", "isnull",
+        ]

{sibi_dst-0.3.15 → sibi_dst-0.3.16}/sibi_dst/df_helper/plugins/django/_django_load_from_db.py

@@ -1,5 +1,8 @@
+import warnings
+
 import dask.dataframe as dd
 import pandas as pd
+from IPython.core.hooks import deprecated
 from django.db.models import Q
 
 from sibi_dst.df_helper.plugins.django import ReadFrameDask
@@ -12,13 +15,11 @@ class DjangoLoadFromDb:
     def __init__(self, db_connection, db_query, db_params, logger, **kwargs):
         self.connection_config = db_connection
         self.debug = kwargs.pop('debug', False)
-        self.verbose_debug = kwargs.pop('verbose_debug', False)
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
         if self.connection_config.model is None:
             if self.debug:
-                self.logger.critical('Model must be specified')
-                if self.verbose_debug:
-                    print('Model must be specified')
+                self.logger.debug('Model must be specified')
+
             raise ValueError('Model must be specified')
 
         self.query_config = db_query
@@ -45,7 +46,7 @@ class DjangoLoadFromDb:
             try:
                 self.df = ReadFrameDask(queryset, **self.params_config.df_params).read_frame()
             except Exception as e:
-                self.logger.critical(f'Error loading query: {str(queryset.query)}, error message: {e}')
+                self.logger.debug(f'Error loading query: {str(queryset.query)}, error message: {e}')
                 self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
         else:
             self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
@@ -69,16 +70,28 @@ class DjangoLoadFromDb:
         :param df: Dask DataFrame whose columns' data types are to be converted.
         :return: Dask DataFrame with converted column data types.
         """
-
-        def log_debug(message: str, is_verbose: bool = False):
-            """Helper to handle debug and verbose debug logging."""
-            if self.debug:
-                self.logger.debug(message)
-                if is_verbose and self.verbose_debug:
-                    print(message)
-
+        """
+            [DEPRECATED] Convert the data types of columns in a Dask DataFrame based on the field type in the Django model.
+
+            :param df: Dask DataFrame whose columns' data types are to be converted.
+            :return: Dask DataFrame with converted column data types.
+            """
+        # Emit deprecation warning
+        warnings.warn(
+            "_convert_columns is deprecated and will be removed in a future release. "
+            "Consider using <new_method_name> instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+        # Log deprecation message if debug mode is enabled
         if self.debug:
-            self.logger.info(f'Converting columns: {list(df.columns)}')
+            self.logger.warning(
+                "[DEPRECATION NOTICE] The `_convert_columns` method is deprecated and will be removed in a future release. "
+                "Consider using <new_method_name> instead."
+            )
+
+        self.logger.debug(f'Converting columns: {list(df.columns)}')
 
         # Get field information from the Django model
         model_fields = self.connection_config.model._meta.get_fields()
@@ -87,13 +100,13 @@ class DjangoLoadFromDb:
         for field_name, field_type in field_type_map.items():
             if field_name not in df.columns:
 
-                log_debug(f"Column '{field_name}' not found in DataFrame columns.")
+                self.logger.debug(f"Column '{field_name}' not found in DataFrame columns.")
                 continue
 
             conversion_func = django_field_conversion_map_dask.get(field_type)
             if not conversion_func:
                 message=f"Field type '{field_type}' not found in conversion_map."
-                log_debug(message, is_verbose=True)
+                self.logger.debug(message)
                 continue
 
             def apply_conversion(partition):
@@ -104,7 +117,7 @@ class DjangoLoadFromDb:
                     if field_name in partition.columns:
                         partition[field_name] = conversion_func(partition[field_name])
                 except Exception as e:
-                    self.logger.error(f"Error converting column '{field_name}' in partition: {str(e)}")
+                    self.logger.debug(f"Error converting column '{field_name}' in partition: {str(e)}")
                 return partition
 
             try:
@@ -113,9 +126,8 @@ class DjangoLoadFromDb:
                     apply_conversion,
                     meta=df,
                 )
-                log_debug(f"Successfully queued conversion for column '{field_name}' to type '{field_type}'.",
-                          is_verbose=True)
+                self.logger.debug(f"Successfully queued conversion for column '{field_name}' to type '{field_type}'.")
             except Exception as e:
-                log_debug(f"Failed to queue conversion for column '{field_name}': {str(e)}", is_verbose=True)
+                self.logger.debug(f"Failed to queue conversion for column '{field_name}': {str(e)}")
 
         return df

{sibi_dst-0.3.15 → sibi_dst-0.3.16}/sibi_dst/df_helper/plugins/django/_io_dask.py

@@ -240,7 +240,4 @@ class ReadFrameDask:
         if verbose:
             self.update_with_verbose(dask_df, fieldnames, fields)
 
-        if isinstance(dask_df, dask_expr._collection.DataFrame):
-            dask_df = dask_df.to_legacy_dataframe()
-
         return dask_df

{sibi_dst-0.3.15 → sibi_dst-0.3.16}/sibi_dst/df_helper/plugins/http/_http_config.py

@@ -31,17 +31,17 @@ class HttpConfig(BaseModel):
                 # Set up headers with API key if provided
             headers = {"Authorization": f"Bearer {self.api_key.get_secret_value()}"} if self.api_key else {}
 
-            self.logger.info(f"Fetching data from {formatted_url} with params {self.params}")
+            self.logger.debug(f"Fetching data from {formatted_url} with params {self.params}")
             async with httpx.AsyncClient() as client:
                 response = await client.get(formatted_url, params=self.params, headers=headers, timeout=self.timeout)
                 response.raise_for_status()
                 data = response.json()
                 df = dd.from_pandas(pd.json_normalize(data), npartitions=1)
-                self.logger.info("Data successfully loaded from HTTP JSON source.")
+                self.logger.debug("Data successfully loaded from HTTP JSON source.")
                 return df
         except httpx.RequestError as e:
-            self.logger.error(f"HTTP request error: {e}")
+            self.logger.debug(f"HTTP request error: {e}")
             raise
         except ValueError as e:
-            self.logger.error(f"Error parsing JSON data: {e}")
+            self.logger.debug(f"Error parsing JSON data: {e}")
             raise

{sibi_dst-0.3.15 → sibi_dst-0.3.16}/sibi_dst/df_helper/plugins/sql_alchemy/_io_sqlalchemy_dask.py

@@ -8,7 +8,7 @@ from sqlalchemy.orm import sessionmaker
 
 from sibi_dst.df_helper.plugins.sql_alchemy._sqlachemy_filter_handler import SqlAlchemyFilterHandler
 from sibi_dst.utils import Logger
-
+from sibi_dst.df_helper.core import FilterHandler
 
 class SQLAlchemyDask:
     def __init__(self, model, filters, engine_url, chunk_size=1000, logger=None, debug=False):
@@ -38,7 +38,6 @@ class SQLAlchemyDask:
         """
         mapper = inspect(model)
         sqlalchemy_to_dask_dtype = {
-            #'INTEGER': pd.to_numeric(x, errors="coerce"),
             'INTEGER': 'Int64',
             'SMALLINT': 'Int64',
             'BIGINT': 'Int64',
@@ -72,11 +71,15 @@ class SQLAlchemyDask:
                 # Build query
                 self.query = select(self.model)
                 if self.filters:
-                    self.query = SqlAlchemyFilterHandler.apply_filters_sqlalchemy(self.query, self.model, self.filters)
+                    """
+                    deprecated specific filter handling to a generic one
+                    #self.query = SqlAlchemyFilterHandler.apply_filters_sqlalchemy(self.query, self.model, self.filters)
+                    """
+                    self.query = FilterHandler(backend="sqlalchemy", logger=self.logger).apply_filters(self.query, model=self.model, filters=self.filters)
                 else:
                     n_records = 100
                     self.query = self.query.limit(n_records)
-
+                self.logger.debug(f"query:{self.query}")
                 # Infer dtypes
                 dtypes = self.infer_dtypes_from_model(self.model)
                 # Get the column order from the SQLAlchemy model
@@ -124,14 +127,11 @@ class SQLAlchemyDask:
                 else:
                     dask_df = dd.from_pandas(pd.DataFrame(columns=ordered_columns), npartitions=1)
 
-                if self.debug:
-                    self.logger.info(f"Loaded {len(dask_df)} rows into Dask DataFrame.")
-
-                if isinstance(dask_df, dask_expr._collection.DataFrame):
-                    dask_df = dask_df.to_legacy_dataframe()
+                self.logger.debug(f"Loaded {len(dask_df)} rows into Dask DataFrame.")
 
                 return dask_df
 
             except Exception as e:
                 self.logger.error(f"Error executing query: {str(e)}")
+                self.logger.error(self.query)
                 return dd.from_pandas(pd.DataFrame(columns=ordered_columns), npartitions=1)

{sibi_dst-0.3.15 → sibi_dst-0.3.16}/sibi_dst/df_helper/plugins/sql_alchemy/_sqlachemy_filter_handler.py

@@ -1,7 +1,9 @@
-from sqlalchemy import and_, or_, not_, func, cast
-from sqlalchemy.sql.sqltypes import Date, Time
 import datetime
 
+from sqlalchemy import func, cast
+from sqlalchemy.sql.sqltypes import Date, Time
+
+
 class SqlAlchemyFilterHandler:
     @staticmethod
     def apply_filters_sqlalchemy(query, model, filters):