upgini 1.2.155.dev1__tar.gz → 1.2.156.dev1__tar.gz

{upgini-1.2.155.dev1 → upgini-1.2.156.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.155.dev1
+Version: 1.2.156.dev1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.156.dev1/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.156.dev1"

{upgini-1.2.155.dev1 → upgini-1.2.156.dev1}/src/upgini/autofe/date.py

@@ -8,7 +8,7 @@ from pandas.core.arrays.timedeltas import TimedeltaArray
 from pydantic import BaseModel, __version__ as pydantic_version
 
 from upgini.autofe.operator import PandasOperator, ParametrizedOperator
-from upgini.autofe.utils import pydantic_validator
+from upgini.autofe.utils import bin_index, bin_index_many, bin_index_vectorized, pydantic_validator
 
 
 def get_pydantic_version():
@@ -254,26 +254,33 @@ class DatePercentileBase(PandasOperator, abc.ABC):
         left = pd.to_datetime(left, unit=self.date_unit)
 
         bounds = self._get_bounds(left)
+        values = pd.to_numeric(right, errors="coerce").to_numpy(dtype=np.float64, copy=False)
+        bounds_list = bounds.tolist()
+        result = np.full(len(values), np.nan)
+
+        if not bounds_list:
+            return pd.Series(result, index=right.index).astype(pd.Int64Dtype()).astype("category")
+
+        bounds_lengths = {len(b) for b in bounds_list if isinstance(b, (list, np.ndarray))}
+        if len(bounds_lengths) == 1 and all(isinstance(b, (list, np.ndarray)) for b in bounds_list):
+            bounds_2d = np.asarray(bounds_list, dtype=np.float64)
+            if bounds_2d.ndim == 1:
+                result = bin_index_vectorized(values, bounds_2d)
+            else:
+                result = bin_index_many(values, bounds_2d)
+        else:
+            for i, row_bounds in enumerate(bounds_list):
+                if isinstance(row_bounds, (list, np.ndarray)) and len(row_bounds) > 0:
+                    result[i] = bin_index(values[i], row_bounds)
 
-        return (
-            right.index.to_series()
-            .apply(lambda i: self._perc(right[i], bounds[i]))
-            .astype(pd.Int64Dtype())
-            .astype("category")
-        )
+        return pd.Series(result, index=right.index).astype(pd.Int64Dtype()).astype("category")
 
     @abc.abstractmethod
     def _get_bounds(self, date_col: pd.Series) -> pd.Series:
         pass
 
     def _perc(self, f, bounds):
-        if f is None or np.isnan(f):
-            return np.nan
-        hit = np.where(f >= np.array(bounds))[0]
-        if hit.size > 0:
-            return np.max(hit) + 1
-        else:
-            return np.nan
+        return bin_index(f, bounds)
 
     def get_params(self) -> Dict[str, Optional[str]]:
         res = super().get_params()
@@ -313,13 +320,15 @@ class DatePercentile(DatePercentileBase):
         return value
 
     def _get_bounds(self, date_col: pd.Series) -> pd.Series:
-        months = date_col.dt.month
-        years = date_col.dt.year
-
-        month_diffs = 12 * (years - (self.zero_year or 0)) + (months - (self.zero_month or 0))
-        return month_diffs.apply(
-            lambda d: np.array(self.zero_bounds if self.zero_bounds is not None else []) + d * self.step
-        )
+        zero_bounds = self.zero_bounds if self.zero_bounds is not None else []
+        if not zero_bounds:
+            return pd.Series([[] for _ in range(len(date_col))], index=date_col.index)
+
+        month_diffs = (
+            12 * (date_col.dt.year - (self.zero_year or 0)) + (date_col.dt.month - (self.zero_month or 0))
+        ).to_numpy()
+        bounds_2d = np.asarray(zero_bounds, dtype=np.float64) + month_diffs[:, None] * self.step
+        return pd.Series(list(bounds_2d), index=date_col.index)
 
 
 class DatePercentileMethod2(DatePercentileBase):

{upgini-1.2.155.dev1 → upgini-1.2.156.dev1}/src/upgini/autofe/timeseries/delta.py

@@ -23,6 +23,8 @@ class DeltaBase(TimeSeriesBase):
     def _calculate_delta(self, x: Union[pd.DataFrame, pd.Series]) -> Union[pd.DataFrame, pd.Series]:
         return_series = isinstance(x, pd.Series)
         x = pd.DataFrame(x)
+        value_col = x.columns[-1]
+        x[value_col] = pd.to_numeric(x[value_col], errors="coerce").astype("float64")
         lag = Lag(lag_size=self.delta_size, lag_unit=self.delta_unit)
         x.iloc[:, -1] = x.iloc[:, -1] - lag._aggregate(x.iloc[:, -1])
         return x.iloc[:, -1] if return_series else x

{upgini-1.2.155.dev1 → upgini-1.2.156.dev1}/src/upgini/autofe/timeseries/trend.py

@@ -54,12 +54,14 @@ class TrendCoefficient(TimeSeriesBase, ParametrizedOperator):
         return_series = isinstance(x, pd.Series)
         x = pd.DataFrame(x)
         resampled = (
-            x.iloc[:, -1].resample(f"{self.step_size}{self.step_unit}").fillna(method="ffill").fillna(method="bfill")
+            x.iloc[:, -1].resample(f"{self.step_size}{self.step_unit}").ffill().bfill()
         )
         idx = np.arange(len(resampled))
+        value_col = x.columns[-1]
+        x[value_col] = pd.to_numeric(x[value_col], errors="coerce").astype("float64")
         try:
             coeffs = np.polyfit(idx, resampled, 1)
-            x.iloc[:, -1] = coeffs[0]
+            x.iloc[:, -1] = float(coeffs[0])
         except np.linalg.LinAlgError:
-            x.iloc[:, -1] = 0
+            x.iloc[:, -1] = 0.0
         return x.iloc[:, -1] if return_series else x

{upgini-1.2.155.dev1 → upgini-1.2.156.dev1}/src/upgini/autofe/timeseries/volatility.py

@@ -65,6 +65,8 @@ class EWMAVolatility(VolatilityBase, ParametrizedOperator):
     def _ewma_vol(self, x):
         return_series = isinstance(x, pd.Series)
         x = pd.DataFrame(x)
+        value_col = x.columns[-1]
+        x[value_col] = pd.to_numeric(x[value_col], errors="coerce").astype("float64")
         returns = self._get_returns(x.iloc[:, -1], f"{self.step_size}{self.step_unit}")
         x.iloc[:, -1] = returns.ewm(span=self.window_size).std()
         return x.iloc[:, -1] if return_series else x
@@ -93,6 +95,8 @@ class RollingVolBase(VolatilityBase):
     ) -> Union[pd.DataFrame, pd.Series]:
         return_series = isinstance(x, pd.Series)
         x = pd.DataFrame(x)
+        value_col = x.columns[-1]
+        x[value_col] = pd.to_numeric(x[value_col], errors="coerce").astype("float64")
         returns = self._get_returns(x.iloc[:, -1], f"{self.step_size}{self.step_unit}")
         if abs_returns:
             returns = returns.abs()

{upgini-1.2.155.dev1 → upgini-1.2.156.dev1}/src/upgini/autofe/unary.py

@@ -4,7 +4,7 @@ import numpy as np
 import pandas as pd
 
 from upgini.autofe.operator import PandasOperator, ParametrizedOperator, VectorizableMixin
-from upgini.autofe.utils import pydantic_validator
+from upgini.autofe.utils import bin_index, bin_index_vectorized, pydantic_validator
 
 
 class Abs(PandasOperator, VectorizableMixin):
@@ -163,16 +163,13 @@ class Bin(PandasOperator):
     is_categorical: bool = True
 
     def calculate_unary(self, data: pd.Series) -> pd.Series:
-        return data.apply(self._bin, bounds=self.bin_bounds).fillna(-1).astype(int).astype("category")
+        bounds_arr = np.asarray(self.bin_bounds, dtype=np.float64)
+        values = pd.to_numeric(data, errors="coerce").to_numpy(dtype=np.float64, copy=False)
+        result = bin_index_vectorized(values, bounds_arr)
+        return pd.Series(result, index=data.index).fillna(-1).astype(int).astype("category")
 
     def _bin(self, f, bounds):
-        if f is None or np.isnan(f):
-            return np.nan
-        hit = np.where(f >= np.array(bounds))[0]
-        if hit.size > 0:
-            return np.max(hit) + 1
-        else:
-            return np.nan
+        return bin_index(f, bounds)
 
     def get_params(self) -> Dict[str, Optional[str]]:
         res = super().get_params()

{upgini-1.2.155.dev1 → upgini-1.2.156.dev1}/src/upgini/autofe/utils.py

@@ -3,8 +3,9 @@ Utility functions for autofe module.
 """
 
 import functools
-from typing import Callable
+from typing import Callable, Union
 
+import numpy as np
 from pydantic import BaseModel
 
 
@@ -111,3 +112,44 @@ def pydantic_copy_method(obj):
         return obj.model_copy
     else:
         return obj.copy
+
+
+def bin_index(value: Union[float, int, None], bounds) -> float:
+    if value is None or (isinstance(value, float) and np.isnan(value)):
+        return np.nan
+    bounds_arr = np.asarray(bounds, dtype=np.float64)
+    if bounds_arr.size == 0 or value < bounds_arr[0]:
+        return np.nan
+    return np.searchsorted(bounds_arr, value, side="right")
+
+
+def bin_index_vectorized(values: np.ndarray, bounds: np.ndarray) -> np.ndarray:
+    n = len(values)
+    result = np.full(n, np.nan)
+    bounds_arr = np.asarray(bounds, dtype=np.float64)
+    if bounds_arr.size == 0:
+        return result
+    valid = ~np.isnan(values)
+    if not valid.any():
+        return result
+    valid_values = values[valid]
+    idx = np.searchsorted(bounds_arr, valid_values, side="right").astype(np.float64)
+    below = valid_values < bounds_arr[0]
+    if below.any():
+        idx[below] = np.nan
+    result[valid] = idx
+    return result
+
+
+def bin_index_many(values: np.ndarray, bounds_2d: np.ndarray) -> np.ndarray:
+    n = len(values)
+    result = np.full(n, np.nan)
+    for i in range(n):
+        v = values[i]
+        if np.isnan(v):
+            continue
+        bounds_row = bounds_2d[i]
+        if bounds_row.size == 0 or v < bounds_row[0]:
+            continue
+        result[i] = np.searchsorted(bounds_row, v, side="right")
+    return result

{upgini-1.2.155.dev1 → upgini-1.2.156.dev1}/src/upgini/features_enricher.py

@@ -1537,10 +1537,15 @@ class FeaturesEnricher(TransformerMixin):
 
         checking_eval_set_df[date_column] = date_converter.to_date_ms(eval_set_dates[selected_eval_set_idx].to_frame())
 
-        cat_features = [c for c in cat_features if c in checking_eval_set_df.columns]
+        baseline_score_column = self._get_renamed_baseline_score_column()
+        psi_df = checking_eval_set_df
+        if baseline_score_column and baseline_score_column in psi_df.columns:
+            psi_df = psi_df.drop(columns=[baseline_score_column])
+
+        cat_features = [c for c in cat_features if c in psi_df.columns]
 
         psi_values_sparse = calculate_sparsity_psi(
-            checking_eval_set_df, cat_features, date_column, self.logger, model_task_type
+            psi_df, cat_features, date_column, self.logger, model_task_type
         )
 
         self.logger.info(f"PSI values by sparsity: {psi_values_sparse}")
@@ -1550,7 +1555,7 @@ class FeaturesEnricher(TransformerMixin):
             self.logger.info(f"Unstable by sparsity features ({stability_threshold}): {sorted(unstable_by_sparsity)}")
 
         psi_values = calculate_features_psi(
-            checking_eval_set_df, cat_features, date_column, self.logger, model_task_type, stability_agg_func
+            psi_df, cat_features, date_column, self.logger, model_task_type, stability_agg_func
         )
 
         self.logger.info(f"PSI values by value: {psi_values}")
@@ -1564,6 +1569,8 @@ class FeaturesEnricher(TransformerMixin):
         }
 
         total_unstable_features = sorted(set(unstable_by_sparsity + unstable_by_value))
+        if baseline_score_column:
+            total_unstable_features = [f for f in total_unstable_features if f != baseline_score_column]
 
         return total_unstable_features
 
@@ -1753,6 +1760,15 @@ class FeaturesEnricher(TransformerMixin):
                             raise ValidationError(self.bundle.get("cat_feature_search_key").format(cat_feature))
         return cat_features, search_keys_for_metrics
 
+    def _get_renamed_baseline_score_column(self, columns_renaming: dict[str, str] | None = None) -> str | None:
+        if self.baseline_score_column is None:
+            return None
+        if columns_renaming:
+            return columns_renaming.get(self.baseline_score_column, self.baseline_score_column)
+        if self.fit_columns_renaming:
+            return self.fit_columns_renaming.get(self.baseline_score_column, self.baseline_score_column)
+        return self.baseline_score_column
+
     def _get_cat_features_for_psi(
         self,
         client_cat_features: list[str] | None,

{upgini-1.2.155.dev1 → upgini-1.2.156.dev1}/src/upgini/utils/feature_info.py

@@ -104,7 +104,8 @@ def _get_feature_sample(feature_meta: FeaturesMetadataV2, data: Optional[pd.Data
     if data is not None and len(data) > 0 and feature_meta.name in data.columns:
         if len(data) > 3:
             rand = np.random.RandomState(42)
-            feature_sample = rand.choice(data[feature_meta.name].dropna().unique(), 3).tolist()
+            unique_values = sorted(data[feature_meta.name].dropna().unique(), key=str)
+            feature_sample = rand.choice(unique_values, 3, replace=False).tolist()
         else:
             feature_sample = data[feature_meta.name].dropna().unique().tolist()
         if len(feature_sample) > 0 and isinstance(feature_sample[0], float):

upgini-1.2.155.dev1/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.155.dev1"