sibi-dst 2025.8.3__py3-none-any.whl → 2025.8.5__py3-none-any.whl

sibi_dst/df_helper/_df_helper.py

@@ -7,7 +7,7 @@ import pandas as pd
 from fsspec import AbstractFileSystem
 from pydantic import BaseModel
 
-from sibi_dst.df_helper.core import QueryConfig, ParamsConfig, FilterHandler
+from sibi_dst.df_helper.core import QueryConfig, ParamsConfig
 from sibi_dst.utils import ManagedResource, ParquetSaver, ClickHouseWriter
 from .backends.http import HttpConfig
 from .backends.parquet import ParquetConfig
@@ -54,16 +54,13 @@ class SqlAlchemyBackend(BaseBackend):
 class ParquetBackend(BaseBackend):
     def load(self, **options):
         try:
-            df = self.helper.backend_parquet.load_files()
+            df = self.helper.backend_parquet.load_files(**options)
             if self._is_empty(df):
                 return -1, self._empty_like(df)
-
-            if options and df is not None:
-                df = FilterHandler("dask", logger=self.logger, debug=False).apply_filters(df, filters=options)
-                nrows = self._row_count(df)
-                if nrows == 0:
-                    self.logger.debug("No records after filters; returning empty DataFrame.")
-                    return 0, self._empty_like(df)
+            nrows = self._row_count(df)
+            if nrows == 0:
+                self.logger.debug("No records after filters; returning empty DataFrame.")
+                return 0, self._empty_like(df)
 
             df = df.persist()
             self.total_records = self._row_count(df) or -1
@@ -185,18 +182,20 @@ class DfHelper(ManagedResource):
         return model(**model_kwargs)
 
     # ---------- load/aload ----------
-    def load(self, as_pandas=False, **options) -> Union[pd.DataFrame, dd.DataFrame]:
+    def load(self, *, persist: bool = False, as_pandas: bool = False, **options) -> Union[pd.DataFrame, dd.DataFrame]:
         self.logger.debug(f"Loading data from {self.backend} backend with options: {options}")
         self.total_records, df = self.backend_strategy.load(**options)
         df = self._process_loaded_data(df)
         df = self._post_process_df(df)
-        self.logger.debug(f"Finished loading data from {self.backend} backend with options: {options}")
+        #self.logger.debug(f"Finished loading data from {self.backend} backend with options: {options}")
+        df = df.persist() if persist else df
         return df.compute() if as_pandas else df
 
-    async def aload(self, as_pandas=False, **options) -> Union[pd.DataFrame, dd.DataFrame]:
+    async def aload(self, *, persist: bool = False, as_pandas: bool = False, **options) -> Union[pd.DataFrame, dd.DataFrame]:
         self.total_records, df = await self.backend_strategy.aload(**options)
         df = self._process_loaded_data(df)
         df = self._post_process_df(df)
+        df = df.persist() if persist else df
         return df.compute() if as_pandas else df
 
     # ---------- dataframe post-processing ----------
@@ -298,295 +297,4 @@ class DfHelper(ManagedResource):
         self.logger.debug(f"Period load generated filters: {kwargs}")
         return kwargs
 
-# from __future__ import annotations
-#
-# from typing import Any, Dict, Optional, Union, TypeVar
-#
-# import dask.dataframe as dd
-# import pandas as pd
-# from fsspec import AbstractFileSystem
-# from pydantic import BaseModel
-#
-# from sibi_dst.df_helper.core import QueryConfig, ParamsConfig, FilterHandler
-# from sibi_dst.utils import ManagedResource
-# from sibi_dst.utils import ParquetSaver, ClickHouseWriter
-# from .backends.http import HttpConfig
-# from .backends.parquet import ParquetConfig
-# from .backends.sqlalchemy import SqlAlchemyConnectionConfig, SqlAlchemyLoadFromDb
-#
-# T = TypeVar("T", bound=BaseModel)
-#
-#
-# # --- Backend Strategy Pattern Implementation ---
-#
-# class BaseBackend:
-#     """Abstract base class defining clear sync and async loading interfaces."""
-#
-#     def __init__(self, helper: DfHelper):
-#         self.helper = helper
-#         self.logger = helper.logger
-#         self.debug = helper.debug
-#         self.total_records = helper.total_records  # no records loaded yet
-#
-#     def load(self, **options) -> tuple[Any, Any] | Union[dd.DataFrame | pd.DataFrame]:
-#         """Synchronous data loading method. Must be implemented by sync backends."""
-#         raise NotImplementedError(f"Backend '{self.__class__.__name__}' does not support synchronous loading.")
-#
-#     async def aload(self, **options) -> tuple[Any, Any] | Union[dd.DataFrame | pd.DataFrame]:
-#         """Asynchronous data loading method. By default, it calls the sync version."""
-#         return self.load(**options)
-#
-#
-# class SqlAlchemyBackend(BaseBackend):
-#     def load(self, **options) -> tuple[Any, Any] | Union[dd.DataFrame | pd.DataFrame]:
-#         try:
-#             # Process incoming filter options into the ParamsConfig object
-#             if options and hasattr(self.helper._backend_params, 'parse_params'):
-#                 self.helper._backend_params.parse_params(options)
-#
-#             with SqlAlchemyLoadFromDb(
-#                 plugin_sqlalchemy=self.helper.backend_db_connection,
-#                 plugin_query=self.helper._backend_query,
-#                 plugin_params=self.helper._backend_params,
-#                 logger=self.logger,
-#                 debug=self.debug
-#             ) as db_loader:
-#                 self.total_records, result = db_loader.build_and_load()
-#                 return self.total_records, result
-#         except Exception as e:
-#             self.logger.error(f"Failed to load data from sqlalchemy: {e}", exc_info=self.debug)
-#             return -1, dd.from_pandas(pd.DataFrame(), npartitions=1)
-#
-#
-# class ParquetBackend(BaseBackend):
-#     """This backend is also purely synchronous."""
-#
-#     def load(self, **options) -> tuple[Any, Any] | Union[dd.DataFrame | pd.DataFrame]:
-#         try:
-#             df = self.helper.backend_parquet.load_files()
-#             if len(df.head(1)) == 0:
-#                 return -1, dd.from_pandas(pd.DataFrame(), npartitions=1)
-#             if options and df is not None:
-#                 df = FilterHandler('dask', logger=self.logger, debug=False).apply_filters(df, filters=options)
-#                 if len(df.head(1)) == 0:
-#                     self.logger.debug("No records found after applying filters; returning empty DataFrame.")
-#                     return -1, dd.from_pandas(pd.DataFrame(), npartitions=1)
-#             df = df.persist()
-#
-#             self.total_records = len(df) or -1  # If df is empty, set total_records to -1
-#             return self.total_records, df
-#         except Exception as e:
-#             self.total_records = -1  # Reset total_records on failure
-#             self.logger.error(f"Failed to load data from parquet: {e}")
-#             return -1, dd.from_pandas(pd.DataFrame(), npartitions=1)
-#
-#
-# class HttpBackend(BaseBackend):
-#     """This backend is purely asynchronous."""
-#
-#     def load(self, **options) -> tuple[Any, Any] | Union[dd.DataFrame | pd.DataFrame]:
-#         # This will correctly fail by raising NotImplementedError from the base class.
-#         return self.helper.backend_http.fetch_data(**options)
-#
-#     async def aload(self, **options) -> tuple[Any, Any] | Union[pd.DataFrame, dd.DataFrame]:
-#         if not self.helper.backend_http:
-#             self.logger.warning("HTTP plugin not configured properly.")
-#             self.total_records = -1
-#             return self.total_records, dd.from_pandas(pd.DataFrame(), npartitions=1)
-#         result = await self.helper.backend_http.fetch_data(**options)
-#         self.total_records = len(result)
-#         return self.total_records, result
-#
-#
-# # --- Main DfHelper Facade Class ---
-#
-# class DfHelper(ManagedResource):
-#     """
-#     A reusable utility for loading data. It provides both sync (`load`) and
-#     async (`aload`) methods to accommodate different backends.
-#     """
-#     _BACKEND_STRATEGIES = {
-#         'sqlalchemy': SqlAlchemyBackend,
-#         'parquet': ParquetBackend,
-#         'http': HttpBackend,
-#     }
-#
-#     _BACKEND_ATTR_MAP = {
-#         'sqlalchemy': 'backend_db_connection',
-#         'parquet': 'backend_parquet',
-#         'http': 'backend_http',
-#     }
-#
-#     default_config: Dict = None
-#
-#     def __init__(self, backend='sqlalchemy', **kwargs):
-#         self.default_config = self.default_config or {}
-#         kwargs = {**self.default_config.copy(), **kwargs}
-#         super().__init__(**kwargs)
-#         self.backend = backend
-#
-#         # Need to set default values for backend-specific configurations
-#         kwargs.setdefault("debug", self.debug)
-#         kwargs.setdefault("fs", self.fs)
-#         kwargs.setdefault("logger", self.logger)
-#         self.total_records = -1  # Initialize total_records to -1 to indicate no records loaded yet
-#         self._backend_query = self._get_config(QueryConfig, kwargs)
-#         self._backend_params = self._get_config(ParamsConfig, kwargs)
-#         self.backend_db_connection: Optional[SqlAlchemyConnectionConfig] = None
-#         self.backend_parquet: Optional[ParquetConfig] = None
-#         self.backend_http: Optional[HttpConfig] = None
-#
-#         if self.backend == 'sqlalchemy':
-#             self.backend_db_connection = self._get_config(SqlAlchemyConnectionConfig, kwargs)
-#         elif self.backend == 'parquet':
-#             self.backend_parquet = self._get_config(ParquetConfig, kwargs)
-#         elif self.backend == 'http':
-#             self.backend_http = self._get_config(HttpConfig, kwargs)
-#
-#         strategy_class = self._BACKEND_STRATEGIES.get(self.backend)
-#         if not strategy_class: raise ValueError(f"Unsupported backend: {self.backend}")
-#         self.backend_strategy = strategy_class(self)
-#
-#     def _cleanup(self):
-#         attr_name = self._BACKEND_ATTR_MAP.get(self.backend)
-#         if not attr_name:
-#             self.logger.warning(f"No attribute mapping found for backend '{self.backend}'. Cleanup skipped.")
-#             return
-#         # Get the actual config object (e.g., self.backend_db_connection)
-#         active_config = getattr(self, attr_name, None)
-#
-#         if active_config and hasattr(active_config, "close"):
-#             self.logger.debug(f"Closing resources for '{self.backend}' backend using attribute '{attr_name}'.")
-#             active_config.close()
-#
-#     async def _acleanup(self):
-#         self.logger.warning("DfHelper instance was not used in an async context manager; cleanup is being called manually.")
-#         attr_name = self._BACKEND_ATTR_MAP.get(self.backend)
-#         if not attr_name:
-#             self.logger.warning(f"No attribute mapping found for backend '{self.backend}'. Cleanup skipped.")
-#             return
-#         # Get the actual config object (e.g., self.backend_db_connection)
-#         active_config = getattr(self, attr_name, None)
-#         if active_config and hasattr(active_config, "aclose"):
-#             self.logger.debug(f"Closing resources for '{self.backend}' backend using attribute '{attr_name}'.")
-#             await active_config.aclose()
-#
-#     def _get_config(self, model: T, kwargs: Dict[str, Any]) -> T:
-#         recognized_keys = set(model.model_fields.keys())
-#         model_kwargs = {k: kwargs[k] for k in recognized_keys if k in kwargs}
-#         return model(**model_kwargs)
-#
-#     def load(self, as_pandas=False, **options) -> Union[pd.DataFrame, dd.DataFrame]:
-#         """Loads data synchronously. Fails if backend is async-only."""
-#         self.logger.debug(f"Loading data from {self.backend} backend with options: {options}")
-#         self.total_records, df = self.backend_strategy.load(**options)
-#         df = self._process_loaded_data(df)
-#         df = self._post_process_df(df)
-#         self.logger.debug(f"Finished loading data from {self.backend} backend with options: {options}")
-#         return df.compute() if as_pandas else df
-#
-#     async def aload(self, as_pandas=False, **options) -> Union[pd.DataFrame, dd.DataFrame]:
-#         """Loads data asynchronously from any backend."""
-#         self.total_records, df = await self.backend_strategy.aload(**options)
-#         df = self._process_loaded_data(df)
-#         df = self._post_process_df(df)
-#         return df.compute() if as_pandas else df
-#
-#     def _post_process_df(self, df: dd.DataFrame) -> dd.DataFrame:
-#         self.logger.debug("Post-processing DataFrame.")
-#         df_params = self._backend_params.df_params
-#         if not df_params: return df
-#         fieldnames, column_names, index_col = (df_params.get("fieldnames"), df_params.get("column_names"),
-#                                                df_params.get("index_col"))
-#         if not any([fieldnames, column_names, index_col]): return df
-#
-#         if fieldnames:
-#             valid_fieldnames = [f for f in fieldnames if f in df.columns]
-#             if len(valid_fieldnames) < len(fieldnames): self.logger.warning(
-#                 f"Missing columns for filtering: {set(fieldnames) - set(valid_fieldnames)}")
-#             df = df[valid_fieldnames]
-#         if column_names:
-#             if len(df.columns) != len(column_names): raise ValueError(
-#                 f"Length mismatch: DataFrame has {len(df.columns)} columns, but {len(column_names)} names were provided.")
-#             df = df.rename(columns=dict(zip(df.columns, column_names)))
-#         if index_col:
-#             if index_col not in df.columns: raise ValueError(f"Index column '{index_col}' not found in DataFrame.")
-#             df = df.set_index(index_col)
-#         self.logger.debug("Post-processing complete.")
-#         return df
-#
-#     def _process_loaded_data(self, df: dd.DataFrame) -> dd.DataFrame:
-#         field_map = self._backend_params.field_map or {}
-#         if not isinstance(field_map, dict) or not field_map: return df
-#         if hasattr(df, 'npartitions') and df.npartitions == 1 and not len(df.head(1)): return df
-#         self.logger.debug("Processing loaded data...applying rename mapping if necessary.")
-#         rename_mapping = {k: v for k, v in field_map.items() if k in df.columns}
-#         if rename_mapping: df = df.rename(columns=rename_mapping)
-#         self.logger.debug("Rename mapping complete...")
-#         return df
-#
-#     def save_to_parquet(self, df: dd.DataFrame, parquet_filename: str, **kwargs):
-#         """Saves a Dask DataFrame to a Parquet file with validation."""
-#
-#         # Use .get() for cleaner access to optional arguments.
-#         fs: AbstractFileSystem = kwargs.get('fs', self.fs)
-#         path: str = kwargs.get('parquet_storage_path')
-#
-#         # Guard clauses to fail fast with clear errors.
-#         if not fs:
-#             raise ValueError("A filesystem (fs) must be provided to save the parquet file.")
-#         if not path:
-#             raise ValueError("A 'parquet_storage_path' keyword argument must be provided.")
-#
-#         # An efficient, idiomatic way to check if a Dask DataFrame is empty.
-#         if len(df.head(1)) == 0:
-#             self.logger.warning("Skipping save: The provided DataFrame is empty.")
-#             return
-#
-#         with ParquetSaver(
-#                 df_result=df,
-#                 parquet_storage_path=path,
-#                 fs=fs,
-#                 debug=self.debug,
-#                 logger=self.logger,
-#                 verbose=self.verbose,
-#                 **kwargs
-#         ) as saver:
-#             saver.save_to_parquet(parquet_filename)
-#
-#         self.logger.debug(f"Successfully saved '{parquet_filename}' to '{path}'.")
-#
-#     def save_to_clickhouse(self, df: dd.DataFrame, **credentials):
-#         if hasattr(df, 'npartitions') and df.npartitions == 1 and not len(df.head(1)):
-#             self.logger.warning("Cannot write to ClickHouse; DataFrame is empty.")
-#             return
-#
-#         with ClickHouseWriter(debug=self.debug, logger=self.logger, verbose=self.verbose, **credentials) as writer:
-#             writer.save_to_clickhouse(df)
-#             self.logger.debug("Save to ClickHouse completed.")
-#
-#     def load_period(self, dt_field: str, start: str, end: str, **kwargs) -> Union[pd.DataFrame, dd.DataFrame]:
-#         """Synchronous convenience method for loading a date range."""
-#         final_kwargs = self._prepare_period_filters(dt_field, start, end, **kwargs)
-#         return self.load(**final_kwargs)
-#
-#     async def aload_period(self, dt_field: str, start: str, end: str, **kwargs) -> Union[pd.DataFrame, dd.DataFrame]:
-#         """Asynchronous convenience method for loading a date range."""
-#         final_kwargs = self._prepare_period_filters(dt_field, start, end, **kwargs)
-#         return await self.aload(**final_kwargs)
-#
-#     def _prepare_period_filters(self, dt_field: str, start: str, end: str, **kwargs) -> dict:
-#         start_date, end_date = pd.to_datetime(start).date(), pd.to_datetime(end).date()
-#         if start_date > end_date:
-#             raise ValueError("'start' date cannot be later than 'end' date.")
-#         field_map = self._backend_params.field_map or {}
-#         reverse_map = {v: k for k, v in field_map.items()} if field_map else {}
-#         if len(reverse_map) != len(field_map):
-#             self.logger.warning("field_map values are not unique; reverse mapping may be unreliable.")
-#         mapped_field = reverse_map.get(dt_field, dt_field)
-#         if start_date == end_date:
-#             kwargs[f"{mapped_field}__date"] = start_date
-#         else:
-#             kwargs[f"{mapped_field}__date__range"] = [start_date, end_date]
-#         self.logger.debug(f"Period load generated filters: {kwargs}")
-#         return kwargs
+

sibi_dst/df_helper/backends/parquet/__init__.py

@@ -1,9 +1,7 @@
 from __future__ import annotations
 
-from ._filter_handler import ParquetFilterHandler
 from ._parquet_options import *
 
 __all__ = [
     "ParquetConfig",
-    "ParquetFilterHandler",
 ]

sibi_dst/df_helper/backends/parquet/_parquet_options.py

@@ -6,6 +6,8 @@ import dask.dataframe as dd
 import fsspec
 import pandas as pd
 from pydantic import BaseModel, model_validator, ConfigDict
+
+from sibi_dst.df_helper.core import FilterHandler
 from sibi_dst.utils import FilePathGenerator
 from sibi_dst.utils import Logger
 
@@ -175,40 +177,79 @@ class ParquetConfig(BaseModel):
                 total_size += self.fs.size(path)
         return total_size
 
-    def load_files(self):
+    def load_files(self, **filters):
         """
-        Loads parquet files into a Dask DataFrame based on the specified conditions. This
-        method checks if parquet file loading is enabled and loads either from a list of
-        parquet folder paths or a single specified parquet path.
-
-        :return: A Dask DataFrame containing loaded parquet file data.
-        :rtype: dask.dataframe.DataFrame
+        Loads parquet files into a Dask DataFrame based on the specified conditions.
+        Supports Parquet predicate pushdown (pyarrow) + residual Dask mask.
         """
         if not self.load_parquet:
             self.logger.warning("Parquet loading is disabled. Returning empty DataFrame.")
             return dd.from_pandas(pd.DataFrame(), npartitions=1)
 
+        # Resolve paths
         paths_to_load = []
         if self.parquet_folder_list:
-            # Filter out any None values from the list
-            paths_to_load = [p for p in self.parquet_folder_list if p is not None]
+            paths_to_load = [p for p in self.parquet_folder_list if p]
         elif self.parquet_full_path:
-            # Treat the single path as a list with one item
             paths_to_load = [self.parquet_full_path]
 
         if not paths_to_load:
             self.logger.warning("No valid parquet file paths were provided. Returning empty DataFrame.")
             return dd.from_pandas(pd.DataFrame(), npartitions=1)
 
+        # Prepare filters
+        fh = None
+        expr = None
+        pq_filters = None
+        residual_filters = None
+        if filters:
+            fh = FilterHandler(backend="dask", debug=self.debug, logger=self.logger)
+
+            # Use the compiler + pushdown split so we don't double-apply
+            try:
+                # If you added split_pushdown_and_residual earlier:
+                pq_filters, residual_filters = fh.split_pushdown_and_residual(filters)
+                expr = fh.compile_filters(residual_filters) if residual_filters else None
+            except AttributeError:
+                # Fallback if you didn't add split_*: push everything down and also mask (redundant but correct)
+                expr = fh.compile_filters(filters)
+                pq_filters = expr.to_parquet_filters()
+
         try:
             self.logger.debug(f"Attempting to load Parquet data from: {paths_to_load}")
-            dd_result=dd.read_parquet(
-                paths_to_load,
-                engine="pyarrow",
-                filesystem=self.fs,
-                exclude=["_*", ".*"]
-            )
+
+            # Optional: prune columns. Keep it simple unless you want to compute from filters.
+            columns = None  # or a concrete list if you know it
+
+            if fh and pq_filters:
+                self.logger.debug(f"Applying Parquet filters: {pq_filters}")
+                dd_result = dd.read_parquet(
+                    paths_to_load,
+                    engine="pyarrow",
+                    filesystem=self.fs,  # your fsspec filesystem (e.g., s3fs)
+                    filters=pq_filters,
+                    columns=columns,
+                    gather_statistics=False,   # uncomment if you have *many* files and don't need global stats
+                )
+                # Apply only residual mask (if any)
+                if expr is not None:
+                    dd_result = dd_result[expr.mask(dd_result)]
+            else:
+                dd_result = dd.read_parquet(
+                    paths_to_load,
+                    engine="pyarrow",
+                    filesystem=self.fs,
+                    columns=columns,
+                    gather_statistics=False,
+                )
+                # If we didn't push down, but have filters, apply them here
+                if expr is None and fh and filters:
+                    expr = fh.compile_filters(filters)
+                if expr is not None:
+                    dd_result = dd_result[expr.mask(dd_result)]
+
             return dd_result
+
         except FileNotFoundError as e:
             self.logger.debug(f"Parquet files not found at paths {paths_to_load}: {e}")
             self.logger.debug("Returning empty DataFrame due to missing parquet files.")

sibi_dst/df_helper/core/_filter_handler.py

@@ -1,4 +1,6 @@
 import datetime
+from dataclasses import dataclass
+from typing import Optional, Dict, Any, List, Union, Tuple, Callable
 
 import dask.dataframe as dd
 import pandas as pd
@@ -8,69 +10,231 @@ from sqlalchemy.sql.sqltypes import Date, Time
 from sibi_dst.utils import Logger
 
 
+# -------------------- Deferred filter expression AST --------------------
+class Expr:
+    def mask(self, df: dd.DataFrame) -> dd.Series:
+        raise NotImplementedError
+
+    def to_parquet_filters(self) -> List[Union[Tuple[str, str, Any], List[Tuple[str, str, Any]]]]:
+        # By default, nothing to push down
+        return []
+
+    def __and__(self, other: "Expr") -> "Expr": return And(self, other)
+    def __or__(self, other: "Expr") -> "Expr":  return Or(self, other)
+    def __invert__(self) -> "Expr":            return Not(self)
+
+
+@dataclass(frozen=True)
+class TrueExpr(Expr):
+    """Matches all rows; useful as a neutral starting point."""
+    def mask(self, df: dd.DataFrame) -> dd.Series:
+        return df.map_partitions(lambda p: pd.Series(True, index=p.index),
+                                 meta=pd.Series(dtype=bool))
+
+
+@dataclass(frozen=True)
+class ColOp(Expr):
+    field: str
+    casting: Optional[str]
+    op: str
+    value: Any
+    handler: "FilterHandler"   # reuse your parsing + Dask ops
+
+    def mask(self, df: dd.DataFrame) -> dd.Series:
+        col = self.handler._get_dask_column(df, self.field, self.casting)
+        val = self.handler._parse_filter_value(self.casting, self.value)
+        return self.handler._apply_operation_dask(col, self.op, val)
+
+    def to_parquet_filters(self):
+        # Only basic comparisons can be pushed down
+        if self.op not in {"exact", "gt", "gte", "lt", "lte", "in", "range"}:
+            return []
+        val = self.handler._parse_filter_value(self.casting, self.value)
+        if self.casting == "date":
+            if self.op == "range" and isinstance(val, (list, tuple)) and len(val) == 2:
+                lo, hi = pd.Timestamp(val[0]), pd.Timestamp(val[1])
+                return [(self.field, ">=", lo), (self.field, "<=", hi)]
+            if isinstance(val, list):
+                val = [pd.Timestamp(v) for v in val]
+            else:
+                val = pd.Timestamp(val)
+        if self.op == "exact": return [(self.field, "=", val)]
+        if self.op in {"gt","gte","lt","lte"}:
+            sym = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[self.op]
+            return [(self.field, sym, val)]
+        if self.op == "in":   return [(self.field, "in", list(val) if not isinstance(val, list) else val)]
+        if self.op == "range":
+            lo, hi = val
+            return [(self.field, ">=", lo), (self.field, "<=", hi)]
+        return []
+
+
+@dataclass(frozen=True)
+class And(Expr):
+    left: Expr; right: Expr
+    def mask(self, df: dd.DataFrame) -> dd.Series: return self.left.mask(df) & self.right.mask(df)
+    def to_parquet_filters(self):
+        # AND = concatenate both sides' AND-terms
+        return [*self.left.to_parquet_filters(), *self.right.to_parquet_filters()]
+
+
+@dataclass(frozen=True)
+class Or(Expr):
+    left: Expr; right: Expr
+    def mask(self, df: dd.DataFrame) -> dd.Series: return self.left.mask(df) | self.right.mask(df)
+    def to_parquet_filters(self):
+        # OR must be returned as list-of-lists; if either side has non-pushdown, defer to mask
+        lf, rf = self.left.to_parquet_filters(), self.right.to_parquet_filters()
+        if not lf or not rf:
+            return []
+        return [lf, rf]
+
+
+@dataclass(frozen=True)
+class Not(Expr):
+    inner: Expr
+    def mask(self, df: dd.DataFrame) -> dd.Series: return ~self.inner.mask(df)
+    def to_parquet_filters(self): return []
+
+
+# -------------------- Filter handler --------------------
 class FilterHandler:
     """
-    Handles the application of filters to data sources with support for SQLAlchemy and Dask backends.
-
-    The FilterHandler class abstracts the process of applying filters to various backends, specifically
-    SQLAlchemy queries and Dask DataFrames. It supports multiple filtering operations, including
-    exact matches, comparisons, and string-related operations such as contains and regex. The handler
-    automatically determines and applies backend-specific processing, enabling seamless integration with
-    different data models or backends.
-
-    :ivar backend: The backend in use ('sqlalchemy' or 'dask').
-    :type backend: str
-    :ivar logger: An optional logger instance for debugging and logging purposes.
-    :type logger: Logger
-    :ivar backend_methods: A dictionary mapping backend-specific methods for column retrieval and operation application.
-    :type backend_methods: dict
+    Handles the application of filters to SQLAlchemy and Dask backends.
+    Also compiles dicts into deferred expressions (Expr) and can split
+    pushdown-friendly predicates from residual ones.
     """
     def __init__(self, backend, logger=None, debug=False):
-        """
-        Initialize the FilterHandler.
-
-        Args:
-            backend: The backend to use ('sqlalchemy' or 'dask').
-            logger: Optional logger for debugging purposes.
-        """
         self.backend = backend
-        self.logger = logger or Logger.default_logger(
-            logger_name=self.__class__.__name__)  # No-op logger if none provided
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
         self.logger.set_level(Logger.DEBUG if debug else Logger.INFO)
         self.backend_methods = self._get_backend_methods(backend)
 
-    def apply_filters(self, query_or_df, model=None, filters=None):
-        """
-        Apply filters to the data source based on the backend.
+    # --------- NEW: pushdown helpers ---------
+    def _pushdown_ops(self) -> set[str]:
+        """Ops that can be translated to PyArrow parquet filters."""
+        return {"exact", "gt", "gte", "lt", "lte", "in", "range"}
 
-        Args:
-            query_or_df: SQLAlchemy query or Dask DataFrame.
-            model: SQLAlchemy model (required for SQLAlchemy backend).
-            filters: Dictionary of filters.
+    def to_parquet_filters(self, filters: Optional[Dict[str, Any]] = None
+                           ) -> List[Tuple[str, str, Any]]:
+        """
+        Convert a subset of filters into PyArrow parquet filters (AND semantics).
+        Unsupported ops are skipped here and should be applied later as a Dask mask.
+        """
+        filters = filters or {}
+        out: List[Tuple[str, str, Any]] = []
 
-        Returns:
-            Filtered query or DataFrame.
+        for key, value in filters.items():
+            field, casting, op = self._parse_filter_key(key)
+            if op not in self._pushdown_ops():
+                continue
+
+            val = self._parse_filter_value(casting, value)
+
+            # Normalize dates to Timestamp for Arrow
+            if casting == "date":
+                if op == "range" and isinstance(val, (list, tuple)) and len(val) == 2:
+                    lo, hi = pd.Timestamp(val[0]), pd.Timestamp(val[1])
+                    out.extend([(field, ">=", lo), (field, "<=", hi)])
+                    continue
+                if isinstance(val, list):
+                    val = [pd.Timestamp(v) for v in val]
+                else:
+                    val = pd.Timestamp(val)
+
+            if op == "exact":
+                out.append((field, "=", val))
+            elif op in {"gt", "gte", "lt", "lte"}:
+                sym = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[op]
+                out.append((field, sym, val))
+            elif op == "in":
+                out.append((field, "in", list(val) if not isinstance(val, list) else val))
+            elif op == "range":
+                lo, hi = val
+                out.extend([(field, ">=", lo), (field, "<=", hi)])
+
+        return out
+
+    def split_pushdown_and_residual(self, filters: Dict[str, Any]
+                                    ) -> Tuple[List[Tuple[str, str, Any]], Dict[str, Any]]:
+        """
+        Split input filter dict into:
+          - parquet_filters: list of (col, op, val) tuples for dd.read_parquet(..., filters=...)
+          - residual_filters: dict to be applied later via a Dask boolean mask
+        """
+        push_keys = set()
+        for key in filters.keys():
+            _, casting, op = self._parse_filter_key(key)
+            if op in self._pushdown_ops():
+                push_keys.add(key)
+
+        pushdown_subset = {k: filters[k] for k in push_keys}
+        parquet_filters = self.to_parquet_filters(pushdown_subset)
+        residual_filters = {k: v for k, v in filters.items() if k not in push_keys}
+        return parquet_filters, residual_filters
+
+    # --------- Expression compiler / mask builder ---------
+    def compile_filters(self, filters: Optional[Dict[str, Any]] = None) -> Expr:
+        """
+        Compile a dict into a deferred expression tree (no df required).
+        Supports boolean forms: {"$and": [...]}, {"$or": [...]}, {"$not": {...}}.
+        Default combination for plain dicts: AND of all terms.
         """
+        filters = filters or {}
+        if not filters:
+            return TrueExpr()
+
+        # boolean forms
+        if "$and" in filters:
+            expr = TrueExpr()
+            for sub in filters["$and"]:
+                expr = expr & self.compile_filters(sub)
+            return expr
+
+        if "$or" in filters:
+            subs = [self.compile_filters(sub) for sub in filters["$or"]]
+            if not subs: return TrueExpr()
+            expr = subs[0]
+            for s in subs[1:]:
+                expr = expr | s
+            return expr
+
+        if "$not" in filters:
+            return ~self.compile_filters(filters["$not"])
+
+        # plain dict => AND across keys
+        expr: Expr = TrueExpr()
+        for key, value in filters.items():
+            field, casting, op = self._parse_filter_key(key)
+            expr = expr & ColOp(field=field, casting=casting, op=op, value=value, handler=self)
+        return expr
+
+    def build_mask_fn(self, filters: Optional[Dict[str, Any]] = None) -> Callable[[dd.DataFrame], dd.Series]:
+        """Return a callable (df -> boolean mask) without touching df now."""
+        expr = self.compile_filters(filters)
+        def _fn(df: dd.DataFrame) -> dd.Series:
+            return expr.mask(df)
+        return _fn
+
+    # --------- Existing “apply now” API (kept as-is) ---------
+    def apply_filters(self, query_or_df, model=None, filters=None):
         filters = filters or {}
         for key, value in filters.items():
             field_name, casting, operation = self._parse_filter_key(key)
             parsed_value = self._parse_filter_value(casting, value)
-            # print(field_name, casting, operation, parsed_value)
-            # Get the column and apply backend-specific transformations
             if self.backend == "sqlalchemy":
                 column = self.backend_methods["get_column"](field_name, model, casting)
                 condition = self.backend_methods["apply_operation"](column, operation, parsed_value)
                 query_or_df = self.backend_methods["apply_condition"](query_or_df, condition)
-
             elif self.backend == "dask":
                 column = self.backend_methods["get_column"](query_or_df, field_name, casting)
                 condition = self.backend_methods["apply_operation"](column, operation, parsed_value)
                 query_or_df = self.backend_methods["apply_condition"](query_or_df, condition)
             else:
                 raise ValueError(f"Unsupported backend: {self.backend}")
-
         return query_or_df
 
+    # --------- Parsing & backend plumbing (unchanged) ---------
     @staticmethod
     def _parse_filter_key(key):
         parts = key.split("__")
@@ -120,29 +284,15 @@ class FilterHandler:
 
     @staticmethod
     def _get_sqlalchemy_column(field_name, model, casting):
-        """
-        Retrieve and cast a column for SQLAlchemy based on the field name and casting.
-
-        Args:
-            field_name: The name of the field/column in the model.
-            model: The SQLAlchemy model.
-            casting: The casting type ('date', 'time', etc.).
-
-        Returns:
-            The SQLAlchemy column object, optionally cast or transformed.
-        """
         column = getattr(model, field_name, None)
         if not column:
             raise AttributeError(f"Field '{field_name}' not found in model '{model.__name__}'")
 
         if casting == "date":
-            # Cast the column to Date for whole-date comparisons
             column = cast(column, Date)
         elif casting == "time":
-            # Cast the column to Time for time-specific comparisons
             column = cast(column, Time)
         elif casting in FilterHandler._date_operators():
-            # Extract date part (e.g., year, month) using SQLAlchemy functions
             column = func.extract(casting, column)
 
         return column
@@ -196,13 +346,13 @@ class FilterHandler:
             "isnull": lambda col, val: col.is_(None) if val else col.isnot(None),
             "not_exact": lambda col, val: col != val,
             "not_contains": lambda col, val: ~col.like(f"%{val}%"),
-            "not_in": lambda col, val: ~col.in_(val),  # Custom operation
-            "regex": lambda col, val: col.op("~")(val),  # Custom operation
-            "icontains": lambda col, val: col.ilike(f"%{val}%"),  # Custom operation
-            "istartswith": lambda col, val: col.ilike(f"{val}%"),  # Custom operation
-            "iendswith": lambda col, val: col.ilike(f"%{val}"),  # Custom operation
-            "iexact": lambda col, val: col.ilike(val),  # Added iexact
-            "iregex": lambda col, val: col.op("~*")(val),  # Added iregex
+            "not_in": lambda col, val: ~col.in_(val),
+            "regex": lambda col, val: col.op("~")(val),
+            "icontains": lambda col, val: col.ilike(f"%{val}%"),
+            "istartswith": lambda col, val: col.ilike(f"{val}%"),
+            "iendswith": lambda col, val: col.ilike(f"%{val}"),
+            "iexact": lambda col, val: col.ilike(val),
+            "iregex": lambda col, val: col.op("~*")(val),
         }
 
     @staticmethod
@@ -214,7 +364,7 @@ class FilterHandler:
             "lt": lambda col, val: col < val,
             "lte": lambda col, val: col <= val,
 
-            # <-- type-safe "in" and "not_in"
+            # type-safe "in" and "not_in"
             "in": lambda col, val: FilterHandler._align_in_types(col, val)[0].isin(
                 FilterHandler._align_in_types(col, val)[1]),
             "not_in": lambda col, val: ~FilterHandler._align_in_types(col, val)[0].isin(
@@ -228,12 +378,9 @@ class FilterHandler:
             "endswith": lambda col, val: FilterHandler._as_str(col).str.endswith(val, na=False),
             "not_contains": lambda col, val: ~FilterHandler._as_str(col).str.contains(val, regex=True, na=False),
             "regex": lambda col, val: FilterHandler._as_str(col).str.contains(val, regex=True, na=False),
-            "icontains": lambda col, val: FilterHandler._as_str(col).str.contains(val, case=False, regex=True,
-                                                                                  na=False),
-            "istartswith": lambda col, val: FilterHandler._as_str(col).str.lower().str.startswith(str(val).lower(),
-                                                                                                  na=False),
-            "iendswith": lambda col, val: FilterHandler._as_str(col).str.lower().str.endswith(str(val).lower(),
-                                                                                              na=False),
+            "icontains": lambda col, val: FilterHandler._as_str(col).str.contains(val, case=False, regex=True, na=False),
+            "istartswith": lambda col, val: FilterHandler._as_str(col).str.lower().str.startswith(str(val).lower(), na=False),
+            "iendswith": lambda col, val: FilterHandler._as_str(col).str.lower().str.endswith(str(val).lower(), na=False),
             "iexact": lambda col, val: FilterHandler._as_str(col).str.lower() == str(val).lower(),
             "iregex": lambda col, val: FilterHandler._as_str(col).str.contains(val, case=False, regex=True, na=False),
 
@@ -243,12 +390,10 @@ class FilterHandler:
 
     @staticmethod
     def _as_str(col):
-        # Force a reliable string view (works with object, categorical, etc.)
         return col.astype("string").fillna("")
 
     @staticmethod
     def _strip_tz(col):
-        # Make tz-aware datetimes naive so they compare to tz-naive filter values
         import pandas as pd
         def _part(s: pd.Series) -> pd.Series:
             try:
@@ -258,12 +403,10 @@ class FilterHandler:
                     return s.dt.tz_localize(None)
                 except Exception:
                     return s
-
         return col.map_partitions(_part, meta=col._meta)
 
     @staticmethod
     def _time_to_seconds(t):
-        # t can be datetime.time or a "HH:MM[:SS]" str
         if isinstance(t, str):
             t = datetime.time.fromisoformat(t)
         return t.hour * 3600 + t.minute * 60 + t.second
@@ -288,10 +431,6 @@ class FilterHandler:
 
     @staticmethod
     def _align_in_types(col, val):
-        """
-        Return (coerced_col, coerced_values) with compatible dtypes
-        so that .isin(...) behaves as expected across partitions.
-        """
         # normalize val to a list
         if isinstance(val, (set, tuple)):
             vals = list(val)
@@ -300,7 +439,6 @@ class FilterHandler:
         else:
             vals = [val]
 
-        # try numeric alignment first if column is numeric-like
         kind = getattr(getattr(col, "dtype", None), "kind", None)
         if kind in ("i", "u"):  # integer
             def to_ints(xs):
@@ -309,13 +447,10 @@ class FilterHandler:
                     try:
                         out.append(int(x))
                     except Exception:
-                        # if any value can't be int, fall back to strings below
                         return None
                 return out
-
             ints = to_ints(vals)
             if ints is not None:
-                # nullable Int64 handles missing values
                 return col.astype("Int64"), ints
 
         if kind in ("f",):  # float
@@ -327,10 +462,8 @@ class FilterHandler:
                     except Exception:
                         return None
                 return out
-
             flts = to_floats(vals)
             if flts is not None:
                 return col.astype("float64"), flts
 
-        # fallback: compare as strings (robust across object/categorical/mixed)
-        return FilterHandler._as_str(col), [str(x) for x in vals]
+        return FilterHandler._as_str(col), [str(x) for x in vals]

{sibi_dst-2025.8.3.dist-info → sibi_dst-2025.8.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.8.3
+Version: 2025.8.5
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-2025.8.3.dist-info → sibi_dst-2025.8.5.dist-info}/RECORD

@@ -2,15 +2,14 @@ sibi_dst/__init__.py,sha256=D01Z2Ds4zES8uz5Zp7qOWD0EcfCllWgew7AWt2X1SQg,445
 sibi_dst/df_helper/__init__.py,sha256=CyDXtFhRnMrycktxNO8jGGkP0938QiScl56kMZS1Sf8,578
 sibi_dst/df_helper/_artifact_updater_async.py,sha256=0lUwel-IkmKewRnmMv9GtuT-P6SivkIKtgOHvKchHlc,8462
 sibi_dst/df_helper/_artifact_updater_threaded.py,sha256=M5GNZismOqMmBrcyfolP1DPv87VILQf_P18is_epn50,7238
-sibi_dst/df_helper/_df_helper.py,sha256=uXG7Ku8ttHuP2kVlMVilek6tkTzpKCJGhw-O0K1JS18,27550
+sibi_dst/df_helper/_df_helper.py,sha256=g1ftfSMO40l60EJWRLE0DDZvbIowrqvG1GMf2zXqYGw,12957
 sibi_dst/df_helper/_parquet_artifact.py,sha256=tqYOjwxHV1MsADmn-RNFuVI_RrEvvmCJHZieRcsVXuc,12334
 sibi_dst/df_helper/_parquet_reader.py,sha256=tFq0OQVczozbKZou93vscokp2R6O2DIJ1zHbZqVjagc,3069
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/df_helper/backends/http/__init__.py,sha256=d1pfgYxbiYg7E0Iw8RbJ7xfqIfJShqqTBQQGU_S6OOo,105
 sibi_dst/df_helper/backends/http/_http_config.py,sha256=eGPFdqZ5M3Tscqx2P93B6XoBEEzlmdt7yNg7PXUQnNQ,4726
-sibi_dst/df_helper/backends/parquet/__init__.py,sha256=esWJ9aSuYC26d-T01z9dPrJ1uqJzvdaPNTYRb5qXTlQ,182
-sibi_dst/df_helper/backends/parquet/_filter_handler.py,sha256=TvDf0RXta7mwJv11GNQttYJsXgFf2XDj4oLIjt4xTzA,5219
-sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=FusVcLVysitLoc8Ui_zU4JMhdHW1MMn4i0vnMbl2K84,12017
+sibi_dst/df_helper/backends/parquet/__init__.py,sha256=0A6BGHZLwiLBmuBBaUvEHfeWTcInvy2NbymlrI_nuXE,104
+sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=V6y1Vco3_uY4UBF79_JPd1CFK5DpNsnGYHCc5PDPGZo,13798
 sibi_dst/df_helper/backends/sqlalchemy/__init__.py,sha256=LjWm9B7CweTvlvFOgB90XjSe0lVLILAIYMWKPkFXFm8,265
 sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=R3_WY_lsQrfQwD6yAzH66MqvsgZdMd0HKcVChDQcbpM,8401
 sibi_dst/df_helper/backends/sqlalchemy/_db_gatekeeper.py,sha256=GQwDy2JwPUx37vpwxPM5hg4ZydilPIP824y5C_clsl0,383
@@ -20,7 +19,7 @@ sibi_dst/df_helper/backends/sqlalchemy/_model_registry.py,sha256=MHk64f5WDOKHQ_L
 sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py,sha256=RiCaVPME5wzgZ9xUGY0JOs_c2C0KcDIbTeMGpPupIa0,5242
 sibi_dst/df_helper/core/__init__.py,sha256=LfmTqFh6GUZup-g95bcXgAxX7J5Hkve7ftLE_CJg_AE,409
 sibi_dst/df_helper/core/_defaults.py,sha256=9UMEMu2wXznO5UzEhnQ82f_ZazZ20JRyRXIi3HP3gDw,4043
-sibi_dst/df_helper/core/_filter_handler.py,sha256=CYyeSmCyy7qVw_duRfBeGzEKaSQyyM-ZN9U8KsjwxXM,14295
+sibi_dst/df_helper/core/_filter_handler.py,sha256=9C30zrT8wSGy1X8ryiTWc0XfnbpeoHndHgoOcHKOPOo,19309
 sibi_dst/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxRrQKE5FQRxcEWsac,6736
 sibi_dst/df_helper/core/_query_config.py,sha256=1ApqmuSGXTC3CdF-xMsSbCa3V2Z5hOP3Wq5huhzZwqY,439
 sibi_dst/df_helper/data_cleaner.py,sha256=lkxQoXLvGzXCicFUimnA5nen5qkrO1oxgl_p2Be2o8w,5183
@@ -79,6 +78,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-2025.8.3.dist-info/METADATA,sha256=zdQXSnLpJ6bVQPpI-N4fnwB2ajCzyyRrGFzmEfpzjvk,2610
-sibi_dst-2025.8.3.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-2025.8.3.dist-info/RECORD,,
+sibi_dst-2025.8.5.dist-info/METADATA,sha256=ADWrf_9UI4NiTWslrJ0LgfmHTTdxSSCIc0AaP-mqSQg,2610
+sibi_dst-2025.8.5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-2025.8.5.dist-info/RECORD,,

sibi_dst/df_helper/backends/parquet/_filter_handler.py

@@ -1,126 +0,0 @@
-import dask.dataframe as dd
-import pandas as pd
-
-from sibi_dst.utils import Logger
-
-
-class ParquetFilterHandler(object):
-    """
-    Handles parquet filtering operations using dask dataframes.
-
-    This class is designed to apply complex filtering logic on dask dataframes
-    based on specified filter criteria. It includes support for operations such
-    as exact matches, ranges, string pattern matches, and null checks. Additionally,
-    it handles datetime-related field filtering including precise truncations and
-    specific date/time attributes.
-
-    :ivar logger: Logger object to handle logging within the class. Defaults to the class-level logger.
-    :type logger: Logger
-    """
-    def __init__(self, logger=None, debug=False):
-        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
-        self.logger.set_level(Logger.DEBUG if debug else Logger.INFO)
-
-    @staticmethod
-    def apply_filters_dask(df, filters):
-        """
-        Applies a set of filters to a Dask DataFrame, enabling complex filtering operations
-        such as comparisons, ranges, string match operations, and more. Handles special
-        cases for datetime operations, including casting and extracting specific datetime
-        components for filtering.
-
-        :param df: Dask DataFrame to which the filters will be applied.
-        :type df: dask.dataframe.DataFrame
-        :param filters: Dictionary defining the filtering logic, where the keys specify
-            the column name and filter operation, and the values specify the corresponding
-            filter values to apply.
-        :type filters: dict
-        :return: A filtered Dask DataFrame based on the defined logic in the filters.
-        :rtype: dask.dataframe.DataFrame
-        :raises ValueError: If an unsupported operation is encountered in the filters.
-        """
-        dt_operators = ['date', 'time']
-        date_operators = ['year', 'month', 'day', 'hour', 'minute', 'second', 'week_day']
-        comparison_operators = [
-            'gte',
-            'lte',
-            'gt',
-            'lt',
-            'exact',
-            'in',
-            'range',
-            'contains',
-            'icontains',
-            'startswith',
-            'endswith',
-            'isnull'
-        ]
-
-        operation_map = {
-            'exact': lambda col, val: col == val,
-            'gt': lambda col, val: col > val,
-            'gte': lambda col, val: col >= val,
-            'lt': lambda col, val: col < val,
-            'lte': lambda col, val: col <= val,
-            'in': lambda col, val: col.isin(val),
-            'range': lambda col, val: (col >= val[0]) & (col <= val[1]),
-            'contains': lambda col, val: col.str.contains(val, regex=True),
-            'icontains': lambda col, val: col.str.contains(val, case=False),
-            'startswith': lambda col, val: col.str.startswith(val),
-            'endswith': lambda col, val: col.str.endswith(val),
-            'isnull': lambda col, val: col.isnull() if val else col.notnull(),
-        }
-
-        def parse_filter_value(casting, value):
-            """
-            Convert filter value to appropriate type based on the casting (e.g., date).
-            """
-            if casting == 'date':
-                if isinstance(value, str):
-                    return pd.Timestamp(value)  # Convert to datetime64[ns]
-                if isinstance(value, list):
-                    return [pd.Timestamp(v) for v in value]  # Convert list elements
-            return value
-
-        def get_temp_col(dask_df, field_name, casting):
-            """
-            Handle datetime conversion and field retrieval.
-            """
-            temp_col = dd.to_datetime(dask_df[field_name], errors='coerce') if casting in dt_operators else dask_df[
-                field_name]
-            if casting == 'date':
-                temp_col = temp_col.dt.floor('D')  # Keep it as datetime64[ns] truncated to the day level
-            elif casting in date_operators:
-                temp_col = getattr(temp_col.dt, casting)
-            return temp_col
-
-        for key, value in filters.items():
-            parts = key.split('__')
-            field_name = parts[0]
-            casting = None
-            operation = 'exact'
-
-            if len(parts) == 3:
-                # Adjust logic based on the parts
-                _, casting, operation = parts
-            elif len(parts) == 2:
-                # Could be either a casting or an operation
-                if parts[1] in comparison_operators:
-                    operation = parts[1]
-                elif parts[1] in dt_operators + date_operators:
-                    casting = parts[1]
-
-            # Convert the filter value to the correct type
-            parsed_value = parse_filter_value(casting, value)
-
-            # Get the column to filter
-            temp_col = get_temp_col(df, field_name, casting)
-
-            if operation in operation_map:
-                # Apply the filter operation
-                condition = operation_map[operation](temp_col, parsed_value)
-                df = df[condition]
-            else:
-                raise ValueError(f"Unsupported operation: {operation}")
-
-        return df