sibi-dst 2025.8.2__tar.gz → 2025.8.3__tar.gz

{sibi_dst-2025.8.2 → sibi_dst-2025.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.8.2
+Version: 2025.8.3
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-2025.8.2 → sibi_dst-2025.8.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "2025.8.2"
+version = "2025.8.3"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

{sibi_dst-2025.8.2 → sibi_dst-2025.8.3}/sibi_dst/df_helper/_df_helper.py

@@ -55,23 +55,42 @@ class ParquetBackend(BaseBackend):
     def load(self, **options):
         try:
             df = self.helper.backend_parquet.load_files()
-            if len(df.head(1)) == 0:
-                return -1, dd.from_pandas(pd.DataFrame(), npartitions=1)
+            if self._is_empty(df):
+                return -1, self._empty_like(df)
 
             if options and df is not None:
                 df = FilterHandler("dask", logger=self.logger, debug=False).apply_filters(df, filters=options)
-                if len(df.head(1)) == 0:
+                nrows = self._row_count(df)
+                if nrows == 0:
                     self.logger.debug("No records after filters; returning empty DataFrame.")
-                    return -1, dd.from_pandas(pd.DataFrame(), npartitions=1)
+                    return 0, self._empty_like(df)
 
             df = df.persist()
-            self.total_records = len(df) or -1
+            self.total_records = self._row_count(df) or -1
             return self.total_records, df
+
         except Exception as e:
-            self.total_records = -1
+            self.total_records = -1  # Reset total_records on failure
             self.logger.error(f"Failed to load data from parquet: {e}", exc_info=self.debug)
             return -1, dd.from_pandas(pd.DataFrame(), npartitions=1)
 
+    def _is_empty(self, ddf) -> bool:
+        """True if no rows across all partitions."""
+        try:
+            # head with npartitions=-1 walks partitions until it gets n rows
+            return ddf.head(1, npartitions=-1).shape[0] == 0
+        except Exception:
+            return True
+
+    def _row_count(self, ddf) -> int:
+        """Reliable row count for Dask DataFrame."""
+        return int(ddf.map_partitions(len).sum().compute())
+
+    def _empty_like(self, ddf):
+        """Return an empty Dask DF with the SAME columns/dtypes."""
+        empty_pdf = ddf._meta.iloc[0:0]
+        return dd.from_pandas(empty_pdf, npartitions=1)
+
 
 class HttpBackend(BaseBackend):
     def load(self, **options):

{sibi_dst-2025.8.2 → sibi_dst-2025.8.3}/sibi_dst/df_helper/core/_filter_handler.py

@@ -89,20 +89,16 @@ class FilterHandler:
         return field_name, casting, operation
 
     def _parse_filter_value(self, casting, value):
-        """
-        Convert filter value to appropriate type based on the casting (e.g., date).
-        """
         if casting == "date":
             if isinstance(value, str):
-                parsed = pd.Timestamp(value)  # Convert to datetime64[ns]
-                return parsed
+                return pd.Timestamp(value)
+            if isinstance(value, list):
+                return [pd.Timestamp(v) for v in value]
+        elif casting == "time":
+            # convert to seconds since midnight
             if isinstance(value, list):
-                parsed = [pd.Timestamp(v) for v in value]
-                return parsed
-        elif casting == "time" and isinstance(value, str):
-            parsed = datetime.time.fromisoformat(value)
-            self.logger.debug(f"Parsed value (time): {parsed}")
-            return parsed
+                return [self._time_to_seconds(v) for v in value]
+            return self._time_to_seconds(value)
         return value
 
     @staticmethod
@@ -153,24 +149,20 @@ class FilterHandler:
 
     @staticmethod
     def _get_dask_column(df, field_name, casting):
-        """
-        Retrieve and optionally cast a column for Dask based on the field name and casting.
+        needs_dt = casting in (FilterHandler._dt_operators() + FilterHandler._date_operators())
+        column = dd.to_datetime(df[field_name], errors="coerce") if needs_dt else df[field_name]
 
-        Args:
-            df: The Dask DataFrame.
-            field_name: The name of the field/column in the DataFrame.
-            casting: The casting type ('date', 'time', etc.).
-
-        Returns:
-            The Dask Series object, optionally cast or transformed.
-        """
-        column = dd.to_datetime(df[field_name], errors="coerce") if casting in FilterHandler._dt_operators() else df[
-            field_name]
+        if needs_dt:
+            column = FilterHandler._strip_tz(column)
 
         if casting == "date":
-            column = column.dt.floor("D")  # Ensure truncation to the date level
+            column = column.dt.floor("D")
+        elif casting == "time":
+            # compare as "seconds since midnight"
+            column = (column.dt.hour * 3600 + column.dt.minute * 60 + column.dt.second)
         elif casting in FilterHandler._date_operators():
-            column = getattr(column.dt, casting)
+            attr = "weekday" if casting == "week_day" else casting
+            column = getattr(column.dt, attr)
 
         return column
 
@@ -221,23 +213,61 @@ class FilterHandler:
             "gte": lambda col, val: col >= val,
             "lt": lambda col, val: col < val,
             "lte": lambda col, val: col <= val,
-            "in": lambda col, val: col.isin(val),
+
+            # <-- type-safe "in" and "not_in"
+            "in": lambda col, val: FilterHandler._align_in_types(col, val)[0].isin(
+                FilterHandler._align_in_types(col, val)[1]),
+            "not_in": lambda col, val: ~FilterHandler._align_in_types(col, val)[0].isin(
+                FilterHandler._align_in_types(col, val)[1]),
+
             "range": lambda col, val: (col >= val[0]) & (col <= val[1]),
-            "contains": lambda col, val: col.str.contains(val, regex=True),
-            "startswith": lambda col, val: col.str.startswith(val),
-            "endswith": lambda col, val: col.str.endswith(val),
+
+            # robust string ops (dtype-agnostic)
+            "contains": lambda col, val: FilterHandler._as_str(col).str.contains(val, regex=True, na=False),
+            "startswith": lambda col, val: FilterHandler._as_str(col).str.startswith(val, na=False),
+            "endswith": lambda col, val: FilterHandler._as_str(col).str.endswith(val, na=False),
+            "not_contains": lambda col, val: ~FilterHandler._as_str(col).str.contains(val, regex=True, na=False),
+            "regex": lambda col, val: FilterHandler._as_str(col).str.contains(val, regex=True, na=False),
+            "icontains": lambda col, val: FilterHandler._as_str(col).str.contains(val, case=False, regex=True,
+                                                                                  na=False),
+            "istartswith": lambda col, val: FilterHandler._as_str(col).str.lower().str.startswith(str(val).lower(),
+                                                                                                  na=False),
+            "iendswith": lambda col, val: FilterHandler._as_str(col).str.lower().str.endswith(str(val).lower(),
+                                                                                              na=False),
+            "iexact": lambda col, val: FilterHandler._as_str(col).str.lower() == str(val).lower(),
+            "iregex": lambda col, val: FilterHandler._as_str(col).str.contains(val, case=False, regex=True, na=False),
+
             "isnull": lambda col, val: col.isnull() if val else col.notnull(),
             "not_exact": lambda col, val: col != val,
-            "not_contains": lambda col, val: ~col.str.contains(val, regex=True),
-            "not_in": lambda col, val: ~col.isin(val),  # Custom operation
-            "regex": lambda col, val: col.str.contains(val, regex=True),  # Custom operation
-            "icontains": lambda col, val: col.str.contains(val, case=False, regex=True),  # Custom operation
-            "istartswith": lambda col, val: col.str.startswith(val, case=False),  # Custom operation
-            "iendswith": lambda col, val: col.str.endswith(val, case=False),  # Custom operation
-            "iexact": lambda col, val: col.str.contains(f"^{val}$", case=False, regex=True),  # Added iexact
-            "iregex": lambda col, val: col.str.contains(val, case=False, regex=True),  # Added iregex
         }
 
+    @staticmethod
+    def _as_str(col):
+        # Force a reliable string view (works with object, categorical, etc.)
+        return col.astype("string").fillna("")
+
+    @staticmethod
+    def _strip_tz(col):
+        # Make tz-aware datetimes naive so they compare to tz-naive filter values
+        import pandas as pd
+        def _part(s: pd.Series) -> pd.Series:
+            try:
+                return s.dt.tz_convert("UTC").dt.tz_localize(None)
+            except Exception:
+                try:
+                    return s.dt.tz_localize(None)
+                except Exception:
+                    return s
+
+        return col.map_partitions(_part, meta=col._meta)
+
+    @staticmethod
+    def _time_to_seconds(t):
+        # t can be datetime.time or a "HH:MM[:SS]" str
+        if isinstance(t, str):
+            t = datetime.time.fromisoformat(t)
+        return t.hour * 3600 + t.minute * 60 + t.second
+
     @staticmethod
     def _dt_operators():
         return ["date", "time"]
@@ -255,3 +285,52 @@ class FilterHandler:
             "regex", "icontains", "istartswith", "iendswith",
             "iexact", "iregex"
         ]
+
+    @staticmethod
+    def _align_in_types(col, val):
+        """
+        Return (coerced_col, coerced_values) with compatible dtypes
+        so that .isin(...) behaves as expected across partitions.
+        """
+        # normalize val to a list
+        if isinstance(val, (set, tuple)):
+            vals = list(val)
+        elif isinstance(val, list):
+            vals = val
+        else:
+            vals = [val]
+
+        # try numeric alignment first if column is numeric-like
+        kind = getattr(getattr(col, "dtype", None), "kind", None)
+        if kind in ("i", "u"):  # integer
+            def to_ints(xs):
+                out = []
+                for x in xs:
+                    try:
+                        out.append(int(x))
+                    except Exception:
+                        # if any value can't be int, fall back to strings below
+                        return None
+                return out
+
+            ints = to_ints(vals)
+            if ints is not None:
+                # nullable Int64 handles missing values
+                return col.astype("Int64"), ints
+
+        if kind in ("f",):  # float
+            def to_floats(xs):
+                out = []
+                for x in xs:
+                    try:
+                        out.append(float(x))
+                    except Exception:
+                        return None
+                return out
+
+            flts = to_floats(vals)
+            if flts is not None:
+                return col.astype("float64"), flts
+
+        # fallback: compare as strings (robust across object/categorical/mixed)
+        return FilterHandler._as_str(col), [str(x) for x in vals]