upgini 1.2.87.dev2__py3-none-any.whl → 1.2.87.dev4__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.87.dev2"
+__version__ = "1.2.87.dev4"

upgini/features_enricher.py

@@ -2299,11 +2299,16 @@ if response.status_code == 200:
 
             self.__log_debug_information(validated_X, validated_y, exclude_features_sources=exclude_features_sources)
 
-            self.__validate_search_keys(self.search_keys, self.search_id)
+            filtered_columns = self.__filtered_enriched_features(
+                importance_threshold, max_features, trace_id, validated_X
+            )
+            # If there are no important features, return original dataframe
+            if not filtered_columns:
+                msg = self.bundle.get("no_important_features_for_transform")
+                self.__log_warning(msg, show_support_link=True)
+                return X, {c: c for c in X.columns}, [], dict()
 
-            if len(self.feature_names_) == 0:
-                self.logger.warning(self.bundle.get("no_important_features_for_transform"))
-                return X, {c: c for c in X.columns}, [], {}
+            self.__validate_search_keys(self.search_keys, self.search_id)
 
             if self._has_paid_features(exclude_features_sources):
                 msg = self.bundle.get("transform_with_paid_features")
@@ -2444,6 +2449,8 @@ if response.status_code == 200:
             # Explode multiple search keys
             df, unnest_search_keys = self._explode_multiple_search_keys(df, search_keys, columns_renaming)
 
+            # Convert search keys and generate features on them
+
             email_column = self._get_email_column(search_keys)
             hem_column = self._get_hem_column(search_keys)
             if email_column:
@@ -2484,6 +2491,26 @@ if response.status_code == 200:
                 converter = PostalCodeSearchKeyConverter(postal_code)
                 df = converter.convert(df)
 
+            # TODO return X + generated features
+            # external_features = [fm for fm in features_meta if fm.shap_value > 0 and fm.source != "etalon"]
+            # if not external_features:
+            #     # Unexplode dataframe back to original shape
+            #     if len(unnest_search_keys) > 0:
+            #         df = df.groupby(ENTITY_SYSTEM_RECORD_ID).first().reset_index()
+                
+            #     # Get important features from etalon source
+            #     etalon_features = [fm.name for fm in features_meta if fm.shap_value > 0 and fm.source == "etalon"]
+                
+            #     # Select only etalon features that exist in dataframe
+            #     available_etalon_features = [f for f in etalon_features if f in df.columns]
+                
+            #     # Return original dataframe with only important etalon features
+            #     result = df[available_etalon_features].copy()
+            #     result.index = validated_Xy.index
+                
+            #     return result, columns_renaming, generated_features, search_keys
+            #     ...
+
             meaning_types = {}
             meaning_types.update({col: FileColumnMeaningType.FEATURE for col in features_for_transform})
             meaning_types.update({col: key.value for col, key in search_keys.items()})
@@ -2637,9 +2664,6 @@ if response.status_code == 200:
                 for c in itertools.chain(validated_Xy.columns.tolist(), generated_features)
                 if c not in self.dropped_client_feature_names_
             ]
-            filtered_columns = self.__filtered_enriched_features(
-                importance_threshold, max_features, trace_id, validated_X
-            )
             selecting_columns.extend(
                 c for c in filtered_columns if c in result.columns and c not in validated_X.columns
             )
@@ -2942,7 +2966,10 @@ if response.status_code == 200:
                 self.__log_warning(fintech_warning)
         df, full_duplicates_warning = clean_full_duplicates(df, self.logger, bundle=self.bundle)
         if full_duplicates_warning:
-            self.__log_warning(full_duplicates_warning)
+            if len(df) == 0:
+                raise ValidationError(full_duplicates_warning)
+            else:
+                self.__log_warning(full_duplicates_warning)
 
         # Explode multiple search keys
         df = self.__add_fit_system_record_id(

upgini/metrics.py

@@ -6,13 +6,23 @@ import re
 from collections import defaultdict
 from copy import deepcopy
 from dataclasses import dataclass
-from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Protocol,
+    Tuple,
+    Union,
+    runtime_checkable,
+)
 
 import lightgbm as lgb
 import numpy as np
 import pandas as pd
 from catboost import CatBoostClassifier, CatBoostRegressor
-from category_encoders.cat_boost import CatBoostEncoder
 from lightgbm import LGBMClassifier, LGBMRegressor
 from numpy import log1p
 from pandas.api.types import is_float_dtype, is_integer_dtype, is_numeric_dtype
@@ -32,10 +42,7 @@ except ImportError:
     available_scorers = SCORERS
 from sklearn.metrics import mean_squared_error
 from sklearn.metrics._regression import _check_reg_targets, check_consistent_length
-from sklearn.model_selection import (  # , TimeSeriesSplit
-    BaseCrossValidator,
-    TimeSeriesSplit,
-)
+from sklearn.model_selection import BaseCrossValidator, TimeSeriesSplit
 
 from upgini.errors import ValidationError
 from upgini.metadata import ModelTaskType
@@ -57,6 +64,16 @@ CATBOOST_REGRESSION_PARAMS = {
     "allow_writing_files": False,
 }
 
+CATBOOST_TS_PARAMS = {
+    "learning_rate": 0.05,
+    "early_stopping_rounds": 20,
+    "use_best_model": True,
+    "one_hot_max_size": 100,
+    "verbose": False,
+    "random_state": 42,
+    "allow_writing_files": False,
+}
+
 CATBOOST_BINARY_PARAMS = {
     "iterations": 250,
     "learning_rate": 0.05,
@@ -311,6 +328,7 @@ class EstimatorWrapper:
         self.target_type = target_type
         self.add_params = add_params
         self.cv_estimators = None
+        self.cv_cat_encoders: Optional[List[Optional[HasTransform]]] = None
         self.groups = groups
         self.text_features = text_features
         self.logger = logger or logging.getLogger()
@@ -437,7 +455,9 @@ class EstimatorWrapper:
 
         return x, y, {}
 
-    def calculate_shap(self, x: pd.DataFrame, y: pd.Series, estimator) -> Optional[Dict[str, float]]:
+    def calculate_shap(
+        self, x: pd.DataFrame, y: pd.Series, estimator, cat_encoder: Optional[HasTransform]
+    ) -> Optional[Dict[str, float]]:
         return None
 
     def cross_val_predict(
@@ -468,9 +488,11 @@ class EstimatorWrapper:
                 fit_params=fit_params,
                 return_estimator=True,
                 error_score="raise",
+                random_state=DEFAULT_RANDOM_STATE,
             )
             metrics_by_fold = cv_results["test_score"]
             self.cv_estimators = cv_results["estimator"]
+            self.cv_cat_encoders = cv_results["cat_encoder"]
 
             self.check_fold_metrics(metrics_by_fold)
 
@@ -478,14 +500,14 @@ class EstimatorWrapper:
 
             splits = self.cv.split(x, y, groups)
 
-            for estimator, split in zip(self.cv_estimators, splits):
+            for estimator, cat_encoder, split in zip(self.cv_estimators, self.cv_cat_encoders, splits):
                 _, validation_idx = split
                 cv_x = x.iloc[validation_idx]
                 if isinstance(y, pd.Series):
                     cv_y = y.iloc[validation_idx]
                 else:
                     cv_y = y[validation_idx]
-                shaps = self.calculate_shap(cv_x, cv_y, estimator)
+                shaps = self.calculate_shap(cv_x, cv_y, estimator, cat_encoder)
                 if shaps is not None:
                     for feature, shap_value in shaps.items():
                         shap_values_all_folds[feature].append(shap_value)
@@ -525,8 +547,19 @@ class EstimatorWrapper:
             metric, metric_std = roc_auc_score(y, x[baseline_score_column]), None
         else:
             metrics = []
-            for est in self.cv_estimators:
-                metrics.append(self.scorer(est, x, y))
+            for est, cat_encoder in zip(self.cv_estimators, self.cv_cat_encoders):
+                x_copy = x.copy()
+                if cat_encoder is not None:
+                    if hasattr(cat_encoder, "feature_names_in_"):
+                        encoded = cat_encoder.transform(x_copy[cat_encoder.feature_names_in_])
+                    else:
+                        encoded = cat_encoder.transform(x[self.cat_features])
+                    if isinstance(self.cv, TimeSeriesSplit) or isinstance(self.cv, BlockedTimeSeriesSplit):
+                        encoded = encoded.astype(int)
+                    else:
+                        encoded = encoded.astype("category")
+                    x_copy[self.cat_features] = encoded
+                metrics.append(self.scorer(est, x_copy, y))
 
             metric, metric_std = self._calculate_metric_from_folds(metrics)
         return _CrossValResults(metric=metric, metric_std=metric_std, shap_values=None)
@@ -549,7 +582,7 @@ class EstimatorWrapper:
         text_features: Optional[List[str]] = None,
         add_params: Optional[Dict[str, Any]] = None,
         groups: Optional[List[str]] = None,
-        has_time: Optional[bool] = None,
+        has_time: bool = False,
     ) -> EstimatorWrapper:
         scorer, metric_name, multiplier = define_scorer(target_type, scoring)
         kwargs = {
@@ -576,7 +609,10 @@ class EstimatorWrapper:
                     params = _get_add_params(params, add_params)
                     estimator = CatBoostWrapper(CatBoostClassifier(**params), **kwargs)
                 elif target_type == ModelTaskType.REGRESSION:
-                    params = _get_add_params(params, CATBOOST_REGRESSION_PARAMS)
+                    if not isinstance(cv, TimeSeriesSplit) and not isinstance(cv, BlockedTimeSeriesSplit):
+                        params = _get_add_params(params, CATBOOST_TS_PARAMS)
+                    else:
+                        params = _get_add_params(params, CATBOOST_REGRESSION_PARAMS)
                     params = _get_add_params(params, add_params)
                     estimator = CatBoostWrapper(CatBoostRegressor(**params), **kwargs)
                 else:
@@ -767,15 +803,24 @@ class CatBoostWrapper(EstimatorWrapper):
             else:
                 raise e
 
-    def calculate_shap(self, x: pd.DataFrame, y: pd.Series, estimator) -> Optional[Dict[str, float]]:
+    def calculate_shap(self, x: pd.DataFrame, y: pd.Series, estimator, cat_encoder) -> Optional[Dict[str, float]]:
         try:
             from catboost import Pool
 
+            cat_features = None
+            if cat_encoder is not None:
+                if isinstance(self.cv, TimeSeriesSplit) or isinstance(self.cv, BlockedTimeSeriesSplit):
+                    encoded = cat_encoder.transform(x[self.cat_features]).astype(int)
+                else:
+                    encoded = cat_encoder.transform(x[self.cat_features])
+                    cat_features = encoded.columns.to_list()
+                x[self.cat_features] = encoded
+
             # Create Pool for fold data, if need (for example, when categorical features are present)
             fold_pool = Pool(
                 x,
                 y,
-                cat_features=self.cat_features,
+                cat_features=cat_features,
                 text_features=self.text_features,
                 embedding_features=self.grouped_embedding_features,
             )
@@ -832,7 +877,6 @@ class LightGBMWrapper(EstimatorWrapper):
             text_features=text_features,
             logger=logger,
         )
-        self.cat_encoder = None
         self.n_classes = None
 
     def _prepare_to_fit(self, x: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, pd.Series, np.ndarray, dict]:
@@ -844,10 +888,10 @@ class LightGBMWrapper(EstimatorWrapper):
                 params["eval_metric"] = "auc"
             params["callbacks"] = [lgb.early_stopping(stopping_rounds=LIGHTGBM_EARLY_STOPPING_ROUNDS, verbose=False)]
         if self.cat_features:
-            encoder = CatBoostEncoder(random_state=DEFAULT_RANDOM_STATE, cols=self.cat_features, return_df=True)
-            encoded = encoder.fit_transform(x[self.cat_features].astype("object"), y_numpy).astype("category")
-            x[self.cat_features] = encoded
-            self.cat_encoder = encoder
+            for c in self.cat_features:
+                if x[c].dtype != "category":
+                    x[c] = x[c].astype("category")
+
         for c in x.columns:
             if x[c].dtype not in ["category", "int64", "float64", "bool"]:
                 self.logger.warning(f"Feature {c} is not numeric and will be dropped")
@@ -857,15 +901,26 @@ class LightGBMWrapper(EstimatorWrapper):
 
     def _prepare_to_calculate(self, x: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, np.ndarray, dict]:
         x, y_numpy, params = super()._prepare_to_calculate(x, y)
-        if self.cat_features is not None and self.cat_encoder is not None:
-            encoded = self.cat_encoder.transform(x[self.cat_features].astype("object"), y_numpy).astype("category")
-            x[self.cat_features] = encoded
+        if self.cat_features:
+            for c in self.cat_features:
+                if x[c].dtype != "category":
+                    x[c] = x[c].astype("category")
         return x, y_numpy, params
 
-    def calculate_shap(self, x: pd.DataFrame, y: pd.Series, estimator) -> Optional[Dict[str, float]]:
+    def calculate_shap(
+        self, x: pd.DataFrame, y: pd.Series, estimator, cat_encoder: Optional[HasTransform]
+    ) -> Optional[Dict[str, float]]:
         try:
+            x_copy = x.copy()
+            if cat_encoder is not None:
+                if isinstance(self.cv, TimeSeriesSplit) or isinstance(self.cv, BlockedTimeSeriesSplit):
+                    encoded = cat_encoder.transform(x_copy[self.cat_features]).astype(int)
+                else:
+                    encoded = cat_encoder.transform(x_copy[self.cat_features]).astype("category")
+                x_copy[self.cat_features] = encoded
+
             shap_matrix = estimator.predict(
-                x,
+                x_copy,
                 predict_disable_shape_check=True,
                 raw_score=True,
                 pred_leaf=False,
@@ -924,10 +979,10 @@ class OtherEstimatorWrapper(EstimatorWrapper):
         num_features = [col for col in x.columns if col not in self.cat_features]
         x[num_features] = x[num_features].fillna(-999)
         if self.cat_features:
-            encoder = CatBoostEncoder(random_state=DEFAULT_RANDOM_STATE, return_df=True)
-            encoded = encoder.fit_transform(x[self.cat_features].astype("object"), y_numpy).astype("category")
-            x[self.cat_features] = encoded
-            self.cat_encoder = encoder
+            for c in self.cat_features:
+                if x[c].dtype != "category":
+                    x[c] = x[c].astype("category")
+            params["cat_features"] = self.cat_features
         for c in x.columns:
             if x[c].dtype not in ["category", "int64", "float64", "bool"]:
                 self.logger.warning(f"Feature {c} is not numeric and will be dropped")
@@ -938,15 +993,22 @@ class OtherEstimatorWrapper(EstimatorWrapper):
     def _prepare_to_calculate(self, x: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, np.ndarray, dict]:
         x, y_numpy, params = super()._prepare_to_calculate(x, y)
         if self.cat_features is not None:
+            for c in self.cat_features:
+                if x[c].dtype != "category":
+                    x[c] = x[c].astype("category")
             num_features = [col for col in x.columns if col not in self.cat_features]
-            x[num_features] = x[num_features].fillna(-999)
-            if self.cat_features and self.cat_encoder is not None:
-                x[self.cat_features] = self.cat_encoder.transform(
-                    x[self.cat_features].astype("object"), y_numpy
-                ).astype("category")
+        else:
+            num_features = x.columns
+        x[num_features] = x[num_features].fillna(-999)
+
         return x, y_numpy, params
 
 
+@runtime_checkable
+class HasTransform(Protocol):
+    def transform(self, X: pd.DataFrame, y: Optional[Union[pd.Series, np.ndarray]] = None) -> pd.DataFrame: ...
+
+
 def validate_scoring_argument(scoring: Union[Callable, str, None]):
     if scoring is None:
         return

upgini/resource_bundle/strings.properties

@@ -156,7 +156,7 @@ dataset_too_few_rows=X size should be at least {} rows after validation
 dataset_too_many_rows_registered=X rows limit for transform is {}. Please sample X
 dataset_empty_column_names=Some column names are empty. Add names please
 dataset_full_duplicates={:.5f}% of the rows are fully duplicated
-dataset_diff_target_duplicates={:.4f}% of rows ({}) in X and eval_set are duplicates with different y values. These rows will be deleted as incorrect\nIncorrect row indexes: {}
+dataset_diff_target_duplicates={:.4f}% of rows ({}) in X and eval_set are duplicates with different y values. These rows will be deleted as incorrect\nSample of incorrect row indexes: {}
 dataset_train_diff_target_duplicates_fintech={:.4f}% of rows ({}) in X are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
 dataset_eval_diff_target_duplicates_fintech={:.4f}% of rows ({}) in eval{} X are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
 dataset_drop_old_dates=We don't have data before '2000-01-01' and removed all earlier records from the search dataset

upgini/utils/datetime_utils.py

@@ -251,99 +251,107 @@ def is_time_series(df: pd.DataFrame, date_col: str) -> bool:
 
 
 def is_blocked_time_series(df: pd.DataFrame, date_col: str, search_keys: List[str]) -> bool:
-    df = df.copy()
-    seconds = "datetime_seconds"
-    if isinstance(df[date_col].dtype, pd.PeriodDtype):
-        df[date_col] = df[date_col].dt.to_timestamp()
-    else:
-        df[date_col] = pd.to_datetime(df[date_col])
-    df[date_col] = df[date_col].dt.tz_localize(None)
-    df[seconds] = (df[date_col] - df[date_col].dt.floor("D")).dt.seconds
-
-    seconds_without_na = df[seconds].dropna()
-    columns_to_drop = [c for c in search_keys if c != date_col] + [seconds]
-    df.drop(columns=columns_to_drop, inplace=True)
-    # Date, not datetime
-    if (seconds_without_na != 0).any() and seconds_without_na.nunique() > 1:
-        return False
+    try:
+        df = df.copy()
+        seconds = "datetime_seconds"
+        if isinstance(df[date_col].dtype, pd.PeriodDtype):
+            df[date_col] = df[date_col].dt.to_timestamp()
+        elif is_numeric_dtype(df[date_col]):
+            df[date_col] = pd.to_datetime(df[date_col], unit="ms")
+        else:
+            df[date_col] = pd.to_datetime(df[date_col])
+        df[date_col] = df[date_col].dt.tz_localize(None)
+        df[seconds] = (df[date_col] - df[date_col].dt.floor("D")).dt.seconds
+
+        seconds_without_na = df[seconds].dropna()
+        columns_to_drop = [c for c in search_keys if c != date_col] + [seconds]
+        df.drop(columns=columns_to_drop, inplace=True)
+        # Date, not datetime
+        if (seconds_without_na != 0).any() and seconds_without_na.nunique() > 1:
+            return False
 
-    nunique_dates = df[date_col].nunique()
-    # Unique dates count more than 270
-    if nunique_dates < 270:
-        return False
+        nunique_dates = df[date_col].nunique()
+        # Unique dates count more than 270
+        if nunique_dates < 270:
+            return False
 
-    min_date = df[date_col].min()
-    max_date = df[date_col].max()
-    days_delta = (max_date - min_date).days + 1
-    # Missing dates less than 30% (unique dates count and days delta between earliest and latest dates)
-    if nunique_dates / days_delta < 0.3:
-        return False
+        min_date = df[date_col].min()
+        max_date = df[date_col].max()
+        days_delta = (max_date - min_date).days + 1
+        # Missing dates less than 30% (unique dates count and days delta between earliest and latest dates)
+        if nunique_dates / days_delta < 0.3:
+            return False
 
-    accumulated_changing_columns = set()
+        accumulated_changing_columns = set()
 
-    def check_differences(group: pd.DataFrame):
-        changing_columns = group.columns[group.nunique(dropna=False) > 1].to_list()
-        accumulated_changing_columns.update(changing_columns)
+        def check_differences(group: pd.DataFrame):
+            changing_columns = group.columns[group.nunique(dropna=False) > 1].to_list()
+            accumulated_changing_columns.update(changing_columns)
 
-    def is_multiple_rows(group: pd.DataFrame) -> bool:
-        return group.shape[0] > 1
+        def is_multiple_rows(group: pd.DataFrame) -> bool:
+            return group.shape[0] > 1
 
-    grouped = df.groupby(date_col)[[c for c in df.columns if c != date_col]]
-    dates_with_multiple_rows = grouped.apply(is_multiple_rows).sum()
+        grouped = df.groupby(date_col)[[c for c in df.columns if c != date_col]]
+        dates_with_multiple_rows = grouped.apply(is_multiple_rows).sum()
 
-    # share of dates with more than one record is more than 99%
-    if dates_with_multiple_rows / nunique_dates < 0.99:
-        return False
+        # share of dates with more than one record is more than 99%
+        if dates_with_multiple_rows / nunique_dates < 0.99:
+            return False
 
-    if df.shape[1] <= 3:
-        return True
+        if df.shape[1] <= 3:
+            return True
 
-    grouped.apply(check_differences)
-    return len(accumulated_changing_columns) <= 2
+        grouped.apply(check_differences)
+        return len(accumulated_changing_columns) <= 2
+    except Exception:
+        return False
 
 
 def is_dates_distribution_valid(
     df: pd.DataFrame,
     search_keys: Dict[str, SearchKey],
 ) -> bool:
-    maybe_date_col = SearchKey.find_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
+    try:
+        maybe_date_col = SearchKey.find_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
 
-    if EVAL_SET_INDEX in df.columns:
-        X = df.query(f"{EVAL_SET_INDEX} == 0")
-    else:
-        X = df
+        if EVAL_SET_INDEX in df.columns:
+            X = df.query(f"{EVAL_SET_INDEX} == 0")
+        else:
+            X = df
 
-    if maybe_date_col is None:
-        for col in X.columns:
-            if col in search_keys:
-                continue
-            try:
-                if isinstance(X[col].dtype, pd.PeriodDtype):
+        if maybe_date_col is None:
+            for col in X.columns:
+                if col in search_keys:
+                    continue
+                try:
+                    if isinstance(X[col].dtype, pd.PeriodDtype):
+                        pass
+                    elif pd.__version__ >= "2.0.0":
+                        # Format mixed to avoid massive warnings
+                        pd.to_datetime(X[col], format="mixed")
+                    else:
+                        pd.to_datetime(X[col])
+                    maybe_date_col = col
+                    break
+                except Exception:
                     pass
-                elif pd.__version__ >= "2.0.0":
-                    # Format mixed to avoid massive warnings
-                    pd.to_datetime(X[col], format="mixed")
-                else:
-                    pd.to_datetime(X[col])
-                maybe_date_col = col
-                break
-            except Exception:
-                pass
-
-    if maybe_date_col is None:
-        return
-
-    if isinstance(X[maybe_date_col].dtype, pd.PeriodDtype):
-        dates = X[maybe_date_col].dt.to_timestamp().dt.date
-    elif pd.__version__ >= "2.0.0":
-        dates = pd.to_datetime(X[maybe_date_col], format="mixed").dt.date
-    else:
-        dates = pd.to_datetime(X[maybe_date_col]).dt.date
-
-    date_counts = dates.value_counts().sort_index()
-
-    date_counts_1 = date_counts[: round(len(date_counts) / 2)]
-    date_counts_2 = date_counts[round(len(date_counts) / 2) :]
-    ratio = date_counts_2.mean() / date_counts_1.mean()
-
-    return ratio >= 0.8 and ratio <= 1.2
+
+        if maybe_date_col is None:
+            return
+
+        if isinstance(X[maybe_date_col].dtype, pd.PeriodDtype):
+            dates = X[maybe_date_col].dt.to_timestamp().dt.date
+        elif pd.__version__ >= "2.0.0":
+            dates = pd.to_datetime(X[maybe_date_col], format="mixed").dt.date
+        else:
+            dates = pd.to_datetime(X[maybe_date_col]).dt.date
+
+        date_counts = dates.value_counts().sort_index()
+
+        date_counts_1 = date_counts[: round(len(date_counts) / 2)]
+        date_counts_2 = date_counts[round(len(date_counts) / 2) :]
+        ratio = date_counts_2.mean() / date_counts_1.mean()
+
+        return ratio >= 0.8 and ratio <= 1.2
+    except Exception:
+        return False

upgini/utils/deduplicate_utils.py

@@ -192,7 +192,7 @@ def clean_full_duplicates(
         unique_columns.remove(TARGET)
         marked_duplicates = df.duplicated(subset=unique_columns, keep=False)
         if marked_duplicates.sum() > 0:
-            dups_indices = df[marked_duplicates].index.to_list()
+            dups_indices = df[marked_duplicates].index.to_list()[:100]
             nrows_after_tgt_dedup = len(df.drop_duplicates(subset=unique_columns, keep=False))
             num_dup_rows = nrows_after_full_dedup - nrows_after_tgt_dedup
             share_tgt_dedup = 100 * num_dup_rows / nrows_after_full_dedup

upgini/utils/sklearn_ext.py

@@ -1,4 +1,5 @@
 import functools
+import inspect
 import numbers
 import time
 import warnings
@@ -9,6 +10,7 @@ from traceback import format_exc
 
 import numpy as np
 import scipy.sparse as sp
+from category_encoders import CatBoostEncoder
 from joblib import Parallel, logger
 from scipy.sparse import issparse
 from sklearn import config_context, get_config
@@ -16,10 +18,13 @@ from sklearn.base import clone, is_classifier
 from sklearn.exceptions import FitFailedWarning, NotFittedError
 from sklearn.metrics import check_scoring
 from sklearn.metrics._scorer import _MultimetricScorer
-from sklearn.model_selection import StratifiedKFold, check_cv
+from sklearn.model_selection import StratifiedKFold, TimeSeriesSplit, check_cv
+from sklearn.preprocessing import OrdinalEncoder
 from sklearn.utils.fixes import np_version, parse_version
 from sklearn.utils.validation import indexable
 
+from upgini.utils.blocked_time_series import BlockedTimeSeriesSplit
+
 # from sklearn.model_selection import cross_validate as original_cross_validate
 
 _DEFAULT_TAGS = {
@@ -59,6 +64,7 @@ def cross_validate(
     return_train_score=False,
     return_estimator=False,
     error_score=np.nan,
+    random_state=None,
 ):
     """Evaluate metric(s) by cross-validation and also record fit/score times.
 
@@ -279,6 +285,8 @@ def cross_validate(
                 return_times=True,
                 return_estimator=return_estimator,
                 error_score=error_score,
+                is_timeseries=isinstance(cv, TimeSeriesSplit) or isinstance(cv, BlockedTimeSeriesSplit),
+                random_state=random_state,
             )
             for train, test in cv.split(x, y, groups)
         )
@@ -296,6 +304,7 @@ def cross_validate(
         ret = {}
         ret["fit_time"] = results["fit_time"]
         ret["score_time"] = results["score_time"]
+        ret["cat_encoder"] = results["cat_encoder"]
 
         if return_estimator:
             ret["estimator"] = results["estimator"]
@@ -320,16 +329,16 @@ def cross_validate(
             else:
                 shuffle = False
             if hasattr(cv, "random_state") and shuffle:
-                random_state = cv.random_state
+                cv_random_state = cv.random_state
             else:
-                random_state = None
+                cv_random_state = None
             return cross_validate(
                 estimator,
                 x,
                 y,
                 groups=groups,
                 scoring=scoring,
-                cv=StratifiedKFold(n_splits=cv.get_n_splits(), shuffle=shuffle, random_state=random_state),
+                cv=StratifiedKFold(n_splits=cv.get_n_splits(), shuffle=shuffle, random_state=cv_random_state),
                 n_jobs=n_jobs,
                 verbose=verbose,
                 fit_params=fit_params,
@@ -337,21 +346,46 @@ def cross_validate(
                 return_train_score=return_train_score,
                 return_estimator=return_estimator,
                 error_score=error_score,
+                random_state=random_state,
             )
         raise e
 
 
-def is_catboost_estimator(estimator):
+def _is_catboost_estimator(estimator):
     try:
         from catboost import CatBoostClassifier, CatBoostRegressor
+
         return isinstance(estimator, (CatBoostClassifier, CatBoostRegressor))
     except ImportError:
         return False
 
 
-def is_lightgbm_estimator(estimator):
+def _supports_cat_features(estimator) -> bool:
+    """Check if estimator's fit method accepts cat_features parameter.
+
+    Parameters
+    ----------
+    estimator : estimator object
+        The estimator to check.
+
+    Returns
+    -------
+    bool
+        True if estimator's fit method accepts cat_features parameter, False otherwise.
+    """
+    try:
+        # Get the signature of the fit method
+        fit_params = inspect.signature(estimator.fit).parameters
+        # Check if cat_features is in the parameters
+        return "cat_features" in fit_params
+    except (AttributeError, ValueError):
+        return False
+
+
+def _is_lightgbm_estimator(estimator):
     try:
         from lightgbm import LGBMClassifier, LGBMRegressor
+
         return isinstance(estimator, (LGBMClassifier, LGBMRegressor))
     except ImportError:
         return False
@@ -375,6 +409,8 @@ def _fit_and_score(
     split_progress=None,
     candidate_progress=None,
     error_score=np.nan,
+    is_timeseries=False,
+    random_state=None,
 ):
     """Fit estimator and compute scores for a given dataset split.
 
@@ -509,13 +545,24 @@ def _fit_and_score(
 
     result = {}
     try:
+        if "cat_features" in fit_params and fit_params["cat_features"]:
+            X_train, y_train, X_test, y_test, cat_features, cat_encoder = _encode_cat_features(
+                X_train, y_train, X_test, y_test, fit_params["cat_features"], estimator, is_timeseries, random_state
+            )
+            if cat_features and _supports_cat_features(estimator):
+                fit_params["cat_features"] = cat_features
+            else:
+                del fit_params["cat_features"]
+        else:
+            cat_encoder = None
+        result["cat_encoder"] = cat_encoder
         if y_train is None:
             estimator.fit(X_train, **fit_params)
         else:
-            if is_catboost_estimator(estimator):
+            if _is_catboost_estimator(estimator):
                 fit_params = fit_params.copy()
                 fit_params["eval_set"] = [(X_test, y_test)]
-            elif is_lightgbm_estimator(estimator):
+            elif _is_lightgbm_estimator(estimator):
                 fit_params = fit_params.copy()
                 fit_params["eval_set"] = [(X_test, y_test)]
             estimator.fit(X_train, y_train, **fit_params)
@@ -1245,3 +1292,60 @@ def _num_samples(x):
         return len(x)
     except TypeError as type_error:
         raise TypeError(message) from type_error
+
+
+def _encode_cat_features(X_train, y_train, X_test, y_test, cat_features, estimator, is_timeseries, random_state):
+    if _is_catboost_estimator(estimator):
+        if is_timeseries:
+            # Fit encoder on training fold
+            encoder = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1)
+            encoder.fit(X_train[cat_features], y_train)
+
+            X_train[cat_features] = encoder.transform(X_train[cat_features]).astype(int)
+            X_test[cat_features] = encoder.transform(X_test[cat_features]).astype(int)
+
+            # Don't use as categorical features, so CatBoost will not encode them
+            return X_train, y_train, X_test, y_test, [], encoder
+        else:
+            return X_train, y_train, X_test, y_test, cat_features, None
+    else:
+        if is_timeseries:
+            # Fit encoder on training fold
+            encoder = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1)
+            encoder.fit(X_train[cat_features], y_train)
+
+            # Progressive encoding on train (using y)
+            X_train[cat_features] = encoder.transform(X_train[cat_features], y_train).astype(int)
+
+            # Static encoding on validation (no y)
+            X_test[cat_features] = encoder.transform(X_test[cat_features]).astype(int)
+
+            return X_train, y_train, X_test, y_test, [], encoder
+        else:
+            # Shuffle train data
+            X_train_shuffled, y_train_shuffled = _shuffle_pair(
+                X_train[cat_features].astype("object"), y_train, random_state
+            )
+
+            # Fit encoder on training fold
+            encoder = CatBoostEncoder(random_state=random_state, cols=cat_features)
+            encoder.fit(X_train_shuffled, y_train_shuffled)
+
+            # Progressive encoding on train (using y)
+            X_train[cat_features] = encoder.transform(X_train[cat_features], y_train).astype("category")
+
+            # Static encoding on validation (no y)
+            X_test[cat_features] = encoder.transform(X_test[cat_features]).astype("category")
+
+            return X_train, y_train, X_test, y_test, cat_features, encoder
+
+
+def _shuffle_pair(X, y, random_state):
+    # If X doesn't have reseted index there could be a problem
+    # shuffled_idx = np.random.RandomState(random_state).permutation(len(X))
+    # return X.iloc[shuffled_idx], pd.Series(y).iloc[shuffled_idx]
+
+    Xy = X.copy()
+    Xy["target"] = y
+    Xy_shuffled = Xy.sample(frac=1, random_state=random_state)
+    return Xy_shuffled.drop(columns="target"), Xy_shuffled["target"]

{upgini-1.2.87.dev2.dist-info → upgini-1.2.87.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.87.dev2
+Version: 1.2.87.dev4
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.87.dev2.dist-info → upgini-1.2.87.dev4.dist-info}/RECORD

@@ -1,12 +1,12 @@
-upgini/__about__.py,sha256=lMGQTkNeO4UaNk9zHrLiiQXvmrSDuq_oo4cWNvv6Lho,28
+upgini/__about__.py,sha256=snYX5GSOXf809cKcpmiRzx30DuIAydReavaEB237z1A,28
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=fRtqSkXNONLnPe6cCL967GMt349FTIpXzy_u8LUKncw,35354
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=n8KBoBgJApLiRv4wXeSgfS-PfbB1D5aDOJfFnL0q6v8,214487
+upgini/features_enricher.py,sha256=8KJiPXTFSiJUl5hJPEhMwhpXqPnGm3LrX31pKwlYe3k,215900
 upgini/http.py,sha256=6Qcepv0tDC72mBBJxYHnA2xqw6QwFaKrXN8o4vju8Es,44372
 upgini/metadata.py,sha256=zt_9k0iQbWXuiRZcel4ORNPdQKt6Ou69ucZD_E1Q46o,12341
-upgini/metrics.py,sha256=7j8wcU-5xh_jSzqGpY7SiWwo1FEAhcPelFVOVu0qwzs,43118
+upgini/metrics.py,sha256=64M7RGbr9dItbXPYqWmeKhpBGHO4B69eV9Rj6P18_qg,45228
 upgini/search_task.py,sha256=Q5HjBpLIB3OCxAD1zNv5yQ3ZNJx696WCK_-H35_y7Rs,17912
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
 upgini/version_validator.py,sha256=DvbaAvuYFoJqYt0fitpsk6Xcv-H1BYDJYHUMxaKSH_Y,1509
@@ -38,7 +38,7 @@ upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 upgini/normalizer/normalize_utils.py,sha256=g2TcDXZeJp9kAFO2sTqZ4CAsN4J1qHNgoJHZ8gtzUWo,7376
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=U_ewTI-qPww4X3WcFG3qDf_jv2vo6RrlCehVDjqtzEI,27991
+upgini/resource_bundle/strings.properties,sha256=Q__3SNuespbG9bRJ9Gq4E_w665NPe8EZ7Pcng8B1V8Y,28001
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=7GpjYqjOp58vYcJLiX__1R5wjUlyQbxvHJ2klFnup_M,6389
@@ -51,8 +51,8 @@ upgini/utils/blocked_time_series.py,sha256=Uqr3vp4YqNclj2-PzEYqVy763GSXHn86sbpIl
 upgini/utils/country_utils.py,sha256=lY-eXWwFVegdVENFttbvLcgGDjFO17Sex8hd2PyJaRk,6937
 upgini/utils/custom_loss_utils.py,sha256=kieNZYBYZm5ZGBltF1F_jOSF4ea6C29rYuCyiDcqVNY,3857
 upgini/utils/cv_utils.py,sha256=w6FQb9nO8BWDx88EF83NpjPLarK4eR4ia0Wg0kLBJC4,3525
-upgini/utils/datetime_utils.py,sha256=UpM2Ljri8rCqHBrSGhmjViGDheLiOYWARFcGaFvC7JE,13858
-upgini/utils/deduplicate_utils.py,sha256=jm9ARZ0fbJFF3aJqj-xm_T6lNh-WErM0H0h6B_L1xQc,8948
+upgini/utils/datetime_utils.py,sha256=UL1ernnawW0LV9mPDpCIc6sFy0HUhFscWVNwfH4V7rI,14366
+upgini/utils/deduplicate_utils.py,sha256=EpBVCov42-FJIAPfa4jY_ZRct3N2MFaC7i-oJNZ_MGI,8954
 upgini/utils/display_utils.py,sha256=hAeWEcJtPDg8fAVcMNrNB-azFD2WJp1nvbPAhR7SeP4,12071
 upgini/utils/email_utils.py,sha256=pZ2vCfNxLIPUhxr0-OlABNXm12jjU44isBk8kGmqQzA,5277
 upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0--cOFTwJ0,1074
@@ -64,13 +64,13 @@ upgini/utils/mstats.py,sha256=u3gQVUtDRbyrOQK6V1UJ2Rx1QbkSNYGjXa6m3Z_dPVs,6286
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
-upgini/utils/sklearn_ext.py,sha256=HpaNQaKJisgNE7IZ71n7uswxTj7kbPglU2G3s1sORAc,45042
+upgini/utils/sklearn_ext.py,sha256=Mdxz0tc-9zT4QyNccA3B86fY4l0MnLDr94POVdYeCT4,49332
 upgini/utils/sort.py,sha256=8uuHs2nfSMVnz8GgvbOmgMB1PgEIZP1uhmeRFxcwnYw,7039
 upgini/utils/target_utils.py,sha256=LRN840dzx78-wg7ftdxAkp2c1eu8-JDvkACiRThm4HE,16832
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.87.dev2.dist-info/METADATA,sha256=7Mt_3MedPG9E1Ie9vtf8T67JlPCGd8PnISUSrN91ctU,49167
-upgini-1.2.87.dev2.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.87.dev2.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.87.dev2.dist-info/RECORD,,
+upgini-1.2.87.dev4.dist-info/METADATA,sha256=PpZ-d4CiDjy-RnXvTGmyEXh-Q_Mjkdf1UaGyVFniqCw,49167
+upgini-1.2.87.dev4.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.87.dev4.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.87.dev4.dist-info/RECORD,,