upgini 1.2.14a3616.dev3__tar.gz → 1.2.16__tar.gz

{upgini-1.2.14a3616.dev3 → upgini-1.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.14a3616.dev3
+Version: 1.2.16
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -145,7 +145,7 @@ Description-Content-Type: text/markdown
 
 ## 💼 Tutorials
 
-###  [Search of relevant external features & Automated feature generation for Salary predicton task (use as a template)](https://github.com/upgini/upgini/blob/main/notebooks/Upgini_Features_search%26generation.ipynb)
+###  [Search of relevant external features & Automated feature generation for Salary prediction task (use as a template)](https://github.com/upgini/upgini/blob/main/notebooks/Upgini_Features_search%26generation.ipynb)
 
 * The goal is to predict salary for data science job postning based on information about employer and job description.
 * Following this guide, you'll learn how to **search & auto generate new relevant features with Upgini library**

{upgini-1.2.14a3616.dev3 → upgini-1.2.16}/README.md

@@ -103,7 +103,7 @@
 
 ## 💼 Tutorials
 
-###  [Search of relevant external features & Automated feature generation for Salary predicton task (use as a template)](https://github.com/upgini/upgini/blob/main/notebooks/Upgini_Features_search%26generation.ipynb)
+###  [Search of relevant external features & Automated feature generation for Salary prediction task (use as a template)](https://github.com/upgini/upgini/blob/main/notebooks/Upgini_Features_search%26generation.ipynb)
 
 * The goal is to predict salary for data science job postning based on information about employer and job description.
 * Following this guide, you'll learn how to **search & auto generate new relevant features with Upgini library**

upgini-1.2.16/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.16"

{upgini-1.2.14a3616.dev3 → upgini-1.2.16}/src/upgini/dataset.py

@@ -53,7 +53,8 @@ class Dataset:  # (pd.DataFrame):
     FIT_SAMPLE_THRESHOLD = 200_000
     FIT_SAMPLE_WITH_EVAL_SET_ROWS = 200_000
     FIT_SAMPLE_WITH_EVAL_SET_THRESHOLD = 200_000
-    MIN_SAMPLE_THRESHOLD = 5_000
+    BINARY_MIN_SAMPLE_THRESHOLD = 5_000
+    MULTICLASS_MIN_SAMPLE_THRESHOLD = 25_000
     IMBALANCE_THESHOLD = 0.6
     BINARY_BOOTSTRAP_LOOPS = 5
     MULTICLASS_BOOTSTRAP_LOOPS = 2
@@ -225,7 +226,7 @@ class Dataset:  # (pd.DataFrame):
             train_segment = self.data
 
         if self.task_type == ModelTaskType.MULTICLASS or (
-            self.task_type == ModelTaskType.BINARY and len(train_segment) > self.MIN_SAMPLE_THRESHOLD
+            self.task_type == ModelTaskType.BINARY and len(train_segment) > self.BINARY_MIN_SAMPLE_THRESHOLD
         ):
             count = len(train_segment)
             target_column = self.etalon_def_checked.get(FileColumnMeaningType.TARGET.value, TARGET)
@@ -253,6 +254,7 @@ class Dataset:  # (pd.DataFrame):
             min_class_percent = self.IMBALANCE_THESHOLD / target_classes_count
             min_class_threshold = min_class_percent * count
 
+            # If min class count less than 30% for binary or (60 / classes_count)% for multiclass
             if min_class_count < min_class_threshold:
                 self.imbalanced = True
                 self.data = balance_undersample(
@@ -260,7 +262,8 @@ class Dataset:  # (pd.DataFrame):
                     target_column=target_column,
                     task_type=self.task_type,
                     random_state=self.random_state,
-                    imbalance_threshold=self.IMBALANCE_THESHOLD,
+                    binary_min_sample_threshold=self.BINARY_MIN_SAMPLE_THRESHOLD,
+                    multiclass_min_sample_threshold=self.MULTICLASS_MIN_SAMPLE_THRESHOLD,
                     binary_bootstrap_loops=self.BINARY_BOOTSTRAP_LOOPS,
                     multiclass_bootstrap_loops=self.MULTICLASS_BOOTSTRAP_LOOPS,
                     logger=self.logger,

{upgini-1.2.14a3616.dev3 → upgini-1.2.16}/src/upgini/features_enricher.py

@@ -336,6 +336,7 @@ class FeaturesEnricher(TransformerMixin):
         self.exclude_columns = exclude_columns
         self.baseline_score_column = baseline_score_column
         self.add_date_if_missing = add_date_if_missing
+        self.features_info_display_handle = None
 
     def _get_api_key(self):
         return self._api_key
@@ -871,6 +872,13 @@ class FeaturesEnricher(TransformerMixin):
                 else None
             )
 
+            if self.X is None:
+                self.X = X
+            if self.y is None:
+                self.y = y
+            if self.eval_set is None:
+                self.eval_set = effective_eval_set
+
             try:
                 self.__log_debug_information(
                     validated_X,
@@ -938,14 +946,14 @@ class FeaturesEnricher(TransformerMixin):
 
                 gc.collect()
 
+                if fitting_X.shape[1] == 0 and fitting_enriched_X.shape[1] == 0:
+                    print(self.bundle.get("metrics_no_important_free_features"))
+                    self.logger.warning("No client or free relevant ADS features found to calculate metrics")
+                    self.warning_counter.increment()
+                    return None
+
                 print(self.bundle.get("metrics_start"))
                 with Spinner():
-                    if fitting_X.shape[1] == 0 and fitting_enriched_X.shape[1] == 0:
-                        print(self.bundle.get("metrics_no_important_free_features"))
-                        self.logger.warning("No client or free relevant ADS features found to calculate metrics")
-                        self.warning_counter.increment()
-                        return None
-
                     self._check_train_and_eval_target_distribution(y_sorted, fitting_eval_set_dict)
 
                     has_date = SearchKey.find_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME]) is not None
@@ -989,7 +997,7 @@ class FeaturesEnricher(TransformerMixin):
                             text_features=self.generate_features,
                             has_date=has_date,
                         )
-                        etalon_metric = baseline_estimator.cross_val_predict(
+                        etalon_metric, _ = baseline_estimator.cross_val_predict(
                             fitting_X, y_sorted, self.baseline_score_column
                         )
                         if etalon_metric is None:
@@ -1023,7 +1031,13 @@ class FeaturesEnricher(TransformerMixin):
                             text_features=self.generate_features,
                             has_date=has_date,
                         )
-                        enriched_metric = enriched_estimator.cross_val_predict(fitting_enriched_X, enriched_y_sorted)
+                        enriched_metric, enriched_shaps = enriched_estimator.cross_val_predict(
+                            fitting_enriched_X, enriched_y_sorted
+                        )
+
+                        if enriched_shaps is not None:
+                            self._update_shap_values(enriched_shaps)
+
                         if enriched_metric is None:
                             self.logger.warning(
                                 f"Enriched {metric} on train combined features is None (maybe all features was removed)"
@@ -1156,13 +1170,6 @@ class FeaturesEnricher(TransformerMixin):
                     elif uplift_col in metrics_df.columns and (metrics_df[uplift_col] < 0).any():
                         self.logger.warning("Uplift is negative")
 
-                    if self.X is None:
-                        self.X = X
-                    if self.y is None:
-                        self.y = y
-                    if self.eval_set is None:
-                        self.eval_set = effective_eval_set
-
                     return metrics_df
             except Exception as e:
                 error_message = "Failed to calculate metrics" + (
@@ -1187,6 +1194,48 @@ class FeaturesEnricher(TransformerMixin):
             finally:
                 self.logger.info(f"Calculating metrics elapsed time: {time.time() - start_time}")
 
+    def _update_shap_values(self, new_shaps: Dict[str, float]):
+        new_shaps = {
+            feature: self._round_shap_value(shap)
+            for feature, shap in new_shaps.items()
+            if feature in self.feature_names_
+        }
+        features_importances = list(new_shaps.items())
+        features_importances.sort(key=lambda m: (-m[1], m[0]))
+        self.feature_names_, self.feature_importances_ = zip(*features_importances)
+        self.feature_names_ = list(self.feature_names_)
+        self.feature_importances_ = list(self.feature_importances_)
+
+        feature_name_header = self.bundle.get("features_info_name")
+        shap_value_header = self.bundle.get("features_info_shap")
+
+        def update_shap(row):
+            return new_shaps.get(row[feature_name_header], row[shap_value_header])
+
+        self.features_info[shap_value_header] = self.features_info.apply(update_shap, axis=1)
+        self._internal_features_info[shap_value_header] = self._internal_features_info.apply(update_shap, axis=1)
+        self._features_info_without_links[shap_value_header] = self._features_info_without_links.apply(
+            update_shap, axis=1
+        )
+        self.logger.info(f"Recalculated SHAP values:\n{self._features_info_without_links}")
+
+        self.features_info.sort_values(by=shap_value_header, ascending=False, inplace=True)
+        self._internal_features_info.sort_values(by=shap_value_header, ascending=False, inplace=True)
+        self._features_info_without_links.sort_values(by=shap_value_header, ascending=False, inplace=True)
+
+        if self.features_info_display_handle:
+            try:
+                _ = get_ipython()  # type: ignore
+
+                display_html_dataframe(
+                    self.features_info,
+                    self._features_info_without_links,
+                    self.bundle.get("relevant_features_header"),
+                    display_handle=self.features_info_display_handle,
+                )
+            except (ImportError, NameError):
+                print(self._internal_features_info)
+
     def _check_train_and_eval_target_distribution(self, y, eval_set_dict):
         uneven_distribution = False
         for eval_set in eval_set_dict.values():
@@ -1515,11 +1564,19 @@ class FeaturesEnricher(TransformerMixin):
             self.logger.info("No external features selected. So use only input datasets for metrics calculation")
             return self.__sample_only_input(validated_X, validated_y, eval_set, is_demo_dataset)
         # TODO save and check if dataset was deduplicated - use imbalance branch for such case
-        elif not self.imbalanced and not exclude_features_sources and is_input_same_as_fit:
+        elif (
+            not self.imbalanced
+            and not exclude_features_sources
+            and is_input_same_as_fit
+            and self.df_with_original_index is not None
+        ):
             self.logger.info("Dataset is not imbalanced, so use enriched_X from fit")
             return self.__sample_balanced(eval_set, trace_id, remove_outliers_calc_metrics)
         else:
-            self.logger.info("Dataset is imbalanced or exclude_features_sources or X was passed. Run transform")
+            self.logger.info(
+                "Dataset is imbalanced or exclude_features_sources or X was passed or this is saved search."
+                " Run transform"
+            )
             print(self.bundle.get("prepare_data_for_metrics"))
             return self.__sample_imbalanced(
                 validated_X,
@@ -1577,8 +1634,8 @@ class FeaturesEnricher(TransformerMixin):
             df = generator.generate(df)
             generated_features.extend(generator.generated_features)
 
-        normalizer = Normalizer(search_keys, generated_features, self.bundle, self.logger, self.warning_counter)
-        df = normalizer.normalize(df)
+        normalizer = Normalizer(self.bundle, self.logger, self.warning_counter)
+        df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
         columns_renaming = normalizer.columns_renaming
 
         df = clean_full_duplicates(df, logger=self.logger, silent=True, bundle=self.bundle)
@@ -2017,10 +2074,8 @@ class FeaturesEnricher(TransformerMixin):
                 df = generator.generate(df)
                 generated_features.extend(generator.generated_features)
 
-            normalizer = Normalizer(
-                search_keys, generated_features, self.bundle, self.logger, self.warning_counter, silent_mode
-            )
-            df = normalizer.normalize(df)
+            normalizer = Normalizer(self.bundle, self.logger, self.warning_counter, silent_mode)
+            df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
             columns_renaming = normalizer.columns_renaming
 
             # Don't pass all features in backend on transform
@@ -2449,16 +2504,13 @@ class FeaturesEnricher(TransformerMixin):
         if is_numeric_dtype(df[self.TARGET_NAME]) and has_date:
             self._validate_PSI(df.sort_values(by=maybe_date_column))
 
-        normalizer = Normalizer(
-            self.fit_search_keys, self.fit_generated_features, self.bundle, self.logger, self.warning_counter
+        normalizer = Normalizer(self.bundle, self.logger, self.warning_counter)
+        df, self.fit_search_keys, self.fit_generated_features = normalizer.normalize(
+            df, self.fit_search_keys, self.fit_generated_features
         )
-        df = normalizer.normalize(df)
-        columns_renaming = normalizer.columns_renaming
-        self.fit_columns_renaming = columns_renaming
+        self.fit_columns_renaming = normalizer.columns_renaming
 
-        self.__adjust_cv(
-            df, normalizer.search_keys, self.model_task_type
-        )
+        self.__adjust_cv(df)
 
         df = remove_fintech_duplicates(
             df, self.fit_search_keys, date_format=self.date_format, logger=self.logger, bundle=self.bundle
@@ -2472,7 +2524,7 @@ class FeaturesEnricher(TransformerMixin):
         self.df_with_original_index = df.copy()
         # TODO check maybe need to drop _time column from df_with_original_index
 
-        df, unnest_search_keys = self._explode_multiple_search_keys(df, self.fit_search_keys, columns_renaming)
+        df, unnest_search_keys = self._explode_multiple_search_keys(df, self.fit_search_keys, self.fit_columns_renaming)
 
         # Convert EMAIL to HEM after unnesting to do it only with one column
         email_column = self._get_email_column(self.fit_search_keys)
@@ -2482,7 +2534,7 @@ class FeaturesEnricher(TransformerMixin):
                 email_column,
                 hem_column,
                 self.fit_search_keys,
-                columns_renaming,
+                self.fit_columns_renaming,
                 list(unnest_search_keys.keys()),
                 self.logger,
             )
@@ -2493,7 +2545,7 @@ class FeaturesEnricher(TransformerMixin):
             converter = IpSearchKeyConverter(
                 ip_column,
                 self.fit_search_keys,
-                columns_renaming,
+                self.fit_columns_renaming,
                 list(unnest_search_keys.keys()),
                 self.bundle,
                 self.logger,
@@ -2524,7 +2576,7 @@ class FeaturesEnricher(TransformerMixin):
         features_columns = [c for c in df.columns if c not in non_feature_columns]
 
         features_to_drop = FeaturesValidator(self.logger).validate(
-            df, features_columns, self.generate_features, self.warning_counter, columns_renaming
+            df, features_columns, self.generate_features, self.warning_counter, self.fit_columns_renaming
         )
         self.fit_dropped_features.update(features_to_drop)
         df = df.drop(columns=features_to_drop)
@@ -2565,7 +2617,7 @@ class FeaturesEnricher(TransformerMixin):
             rest_client=self.rest_client,
             logger=self.logger,
         )
-        dataset.columns_renaming = columns_renaming
+        dataset.columns_renaming = self.fit_columns_renaming
 
         self.passed_features = [
             column for column, meaning_type in meaning_types.items() if meaning_type == FileColumnMeaningType.FEATURE
@@ -2712,22 +2764,22 @@ class FeaturesEnricher(TransformerMixin):
         if not self.warning_counter.has_warnings():
             self.__display_support_link(self.bundle.get("all_ok_community_invite"))
 
-    def __adjust_cv(self, df: pd.DataFrame, search_keys: Dict[str, SearchKey], model_task_type: ModelTaskType):
-        date_column = SearchKey.find_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
+    def __adjust_cv(self, df: pd.DataFrame):
+        date_column = SearchKey.find_key(self.fit_search_keys, [SearchKey.DATE, SearchKey.DATETIME])
         # Check Multivariate time series
         if (
             self.cv is None
             and date_column
-            and model_task_type == ModelTaskType.REGRESSION
-            and len({SearchKey.PHONE, SearchKey.EMAIL, SearchKey.HEM}.intersection(search_keys.keys())) == 0
-            and is_blocked_time_series(df, date_column, list(search_keys.keys()) + [TARGET])
+            and self.model_task_type == ModelTaskType.REGRESSION
+            and len({SearchKey.PHONE, SearchKey.EMAIL, SearchKey.HEM}.intersection(self.fit_search_keys.keys())) == 0
+            and is_blocked_time_series(df, date_column, list(self.fit_search_keys.keys()) + [TARGET])
         ):
             msg = self.bundle.get("multivariate_timeseries_detected")
             self.__override_cv(CVType.blocked_time_series, msg, print_warning=False)
-        elif self.cv is None and model_task_type != ModelTaskType.REGRESSION:
+        elif self.cv is None and self.model_task_type != ModelTaskType.REGRESSION:
             msg = self.bundle.get("group_k_fold_in_classification")
             self.__override_cv(CVType.group_k_fold, msg, print_warning=self.cv is not None)
-            group_columns = self._get_group_columns(df, search_keys)
+            group_columns = self._get_group_columns(df, self.fit_search_keys)
             self.runtime_parameters.properties["cv_params.group_columns"] = ",".join(group_columns)
             self.runtime_parameters.properties["cv_params.shuffle_kfold"] = "True"
 
@@ -3379,6 +3431,13 @@ class FeaturesEnricher(TransformerMixin):
 
         return result_train, result_eval_sets
 
+    @staticmethod
+    def _round_shap_value(shap: float) -> float:
+        if shap > 0.0 and shap < 0.0001:
+            return 0.0001
+        else:
+            return round(shap, 4)
+
     def __prepare_feature_importances(self, trace_id: str, x_columns: List[str], silent=False):
         llm_source = "LLM with external data augmentation"
         if self._search_task is None:
@@ -3396,12 +3455,6 @@ class FeaturesEnricher(TransformerMixin):
         features_info_without_links = []
         internal_features_info = []
 
-        def round_shap_value(shap: float) -> float:
-            if shap > 0.0 and shap < 0.0001:
-                return 0.0001
-            else:
-                return round(shap, 4)
-
         def list_or_single(lst: List[str], single: str):
             return lst or ([single] if single else [])
 
@@ -3434,7 +3487,7 @@ class FeaturesEnricher(TransformerMixin):
 
             feature_sample = []
             self.feature_names_.append(feature_meta.name)
-            self.feature_importances_.append(round_shap_value(feature_meta.shap_value))
+            self.feature_importances_.append(self._round_shap_value(feature_meta.shap_value))
             if feature_meta.name in features_df.columns:
                 feature_sample = np.random.choice(features_df[feature_meta.name].dropna().unique(), 3).tolist()
                 if len(feature_sample) > 0 and isinstance(feature_sample[0], float):
@@ -3473,7 +3526,7 @@ class FeaturesEnricher(TransformerMixin):
             features_info.append(
                 {
                     self.bundle.get("features_info_name"): feature_name,
-                    self.bundle.get("features_info_shap"): round_shap_value(feature_meta.shap_value),
+                    self.bundle.get("features_info_shap"): self._round_shap_value(feature_meta.shap_value),
                     self.bundle.get("features_info_hitrate"): feature_meta.hit_rate,
                     self.bundle.get("features_info_value_preview"): feature_sample,
                     self.bundle.get("features_info_provider"): provider,
@@ -3484,7 +3537,7 @@ class FeaturesEnricher(TransformerMixin):
             features_info_without_links.append(
                 {
                     self.bundle.get("features_info_name"): internal_feature_name,
-                    self.bundle.get("features_info_shap"): round_shap_value(feature_meta.shap_value),
+                    self.bundle.get("features_info_shap"): self._round_shap_value(feature_meta.shap_value),
                     self.bundle.get("features_info_hitrate"): feature_meta.hit_rate,
                     self.bundle.get("features_info_value_preview"): feature_sample,
                     self.bundle.get("features_info_provider"): internal_provider,
@@ -3496,7 +3549,7 @@ class FeaturesEnricher(TransformerMixin):
                 {
                     self.bundle.get("features_info_name"): internal_feature_name,
                     "feature_link": feature_meta.doc_link,
-                    self.bundle.get("features_info_shap"): round_shap_value(feature_meta.shap_value),
+                    self.bundle.get("features_info_shap"): self._round_shap_value(feature_meta.shap_value),
                     self.bundle.get("features_info_hitrate"): feature_meta.hit_rate,
                     self.bundle.get("features_info_value_preview"): feature_sample,
                     self.bundle.get("features_info_provider"): internal_provider,
@@ -3776,8 +3829,11 @@ class FeaturesEnricher(TransformerMixin):
             print(Format.GREEN + Format.BOLD + msg + Format.END)
             self.logger.info(msg)
             if len(self.feature_names_) > 0:
-                display_html_dataframe(
-                    self.features_info, self._features_info_without_links, self.bundle.get("relevant_features_header")
+                self.features_info_display_handle = display_html_dataframe(
+                    self.features_info,
+                    self._features_info_without_links,
+                    self.bundle.get("relevant_features_header"),
+                    display_id="features_info",
                 )
 
                 display_html_dataframe(

{upgini-1.2.14a3616.dev3 → upgini-1.2.16}/src/upgini/metrics.py

@@ -3,13 +3,14 @@ from __future__ import annotations
 import inspect
 import logging
 import re
+from collections import defaultdict
 from copy import deepcopy
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import catboost
 import numpy as np
 import pandas as pd
-from catboost import CatBoostClassifier, CatBoostRegressor
+from catboost import CatBoost, CatBoostClassifier, CatBoostRegressor, Pool
 from numpy import log1p
 from pandas.api.types import is_numeric_dtype
 from sklearn.metrics import check_scoring, get_scorer, make_scorer, roc_auc_score
@@ -288,9 +289,12 @@ class EstimatorWrapper:
         x, y, _ = self._prepare_data(x, y)
         return x, y, {}
 
+    def calculate_shap(self, x: pd.DataFrame, y: pd.Series, estimator) -> Optional[Dict[str, float]]:
+        return None
+
     def cross_val_predict(
         self, x: pd.DataFrame, y: np.ndarray, baseline_score_column: Optional[Any] = None
-    ) -> Optional[float]:
+    ) -> Tuple[Optional[float], Optional[Dict[str, float]]]:
         x, y, groups, fit_params = self._prepare_to_fit(x, y)
 
         if x.shape[1] == 0:
@@ -298,6 +302,7 @@ class EstimatorWrapper:
 
         scorer = check_scoring(self.estimator, scoring=self.scorer)
 
+        shap_values_all_folds = defaultdict(list)
         if baseline_score_column is not None and self.metric_name == "GINI":
             self.logger.info("Calculate baseline GINI on passed baseline_score_column and target")
             metric = roc_auc_score(y, x[baseline_score_column])
@@ -319,7 +324,29 @@ class EstimatorWrapper:
             self.check_fold_metrics(metrics_by_fold)
 
             metric = np.mean(metrics_by_fold) * self.multiplier
-        return self.post_process_metric(metric)
+
+            splits = self.cv.split(x, y, groups)
+
+            for estimator, split in zip(self.cv_estimators, splits):
+                _, validation_idx = split
+                cv_x = x.iloc[validation_idx]
+                cv_y = y[validation_idx]
+                shaps = self.calculate_shap(cv_x, cv_y, estimator)
+                if shaps is not None:
+                    for feature, shap_value in shaps.items():
+                        # shap_values_all_folds[feature] = shap_values_all_folds.get(feature, []) + shap_value.tolist()
+                        shap_values_all_folds[feature].extend(shap_value.tolist())
+
+        if shap_values_all_folds:
+            average_shap_values = {
+                feature: np.mean(np.array(shaps)) for feature, shaps in shap_values_all_folds.items() if len(shaps) > 0
+            }
+            if len(average_shap_values) == 0:
+                average_shap_values = None
+        else:
+            average_shap_values = None
+
+        return self.post_process_metric(metric), average_shap_values
 
     def check_fold_metrics(self, metrics_by_fold: List[float]):
         first_metric_sign = 1 if metrics_by_fold[0] >= 0 else -1
@@ -453,6 +480,7 @@ class CatBoostWrapper(EstimatorWrapper):
         )
         self.cat_features = None
         self.emb_features = None
+        self.grouped_embedding_features = None
         self.exclude_features = []
 
     def _prepare_to_fit(self, x: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, np.ndarray, np.ndarray, dict]:
@@ -462,17 +490,16 @@ class CatBoostWrapper(EstimatorWrapper):
         if hasattr(CatBoostClassifier, "get_embedding_feature_indices"):
             emb_pattern = r"(.+)_emb\d+"
             self.emb_features = [c for c in x.columns if re.match(emb_pattern, c) and is_numeric_dtype(x[c])]
-            embedding_features = []
             if len(self.emb_features) > 3:  # There is no reason to reduce embeddings dimension with less than 4
                 self.logger.info(
                     "Embedding features count more than 3, so group them into one vector for CatBoost: "
                     f"{self.emb_features}"
                 )
-                x, embedding_features = self.group_embeddings(x)
-                params["embedding_features"] = embedding_features
+                x, self.grouped_embedding_features = self.group_embeddings(x)
+                params["embedding_features"] = self.grouped_embedding_features
             else:
                 self.logger.info(f"Embedding features count less than 3, so use them separately: {self.emb_features}")
-                self.emb_features = []
+                self.grouped_embedding_features = None
         else:
             self.logger.warning(f"Embedding features are not supported by Catboost version {catboost.__version__}")
 
@@ -488,7 +515,7 @@ class CatBoostWrapper(EstimatorWrapper):
             self.logger.warning(f"Text features are not supported by this Catboost version {catboost.__version__}")
 
         # Find rest categorical features
-        self.cat_features = _get_cat_features(x, self.text_features, embedding_features)
+        self.cat_features = _get_cat_features(x, self.text_features, self.grouped_embedding_features)
         # x = fill_na_cat_features(x, self.cat_features)
         unique_cat_features = []
         for name in self.cat_features:
@@ -548,7 +575,7 @@ class CatBoostWrapper(EstimatorWrapper):
 
     def cross_val_predict(
         self, x: pd.DataFrame, y: np.ndarray, baseline_score_column: Optional[Any] = None
-    ) -> Optional[float]:
+    ) -> Tuple[Optional[float], Optional[Dict[str, float]]]:
         try:
             return super().cross_val_predict(x, y, baseline_score_column)
         except Exception as e:
@@ -573,6 +600,36 @@ class CatBoostWrapper(EstimatorWrapper):
             else:
                 raise e
 
+    def calculate_shap(self, x: pd.DataFrame, y: pd.Series, estimator: CatBoost) -> Optional[Dict[str, float]]:
+        try:
+            # Create Pool for fold data, if need (for example, when categorical features are present)
+            fold_pool = Pool(
+                x,
+                y,
+                cat_features=self.cat_features,
+                text_features=self.text_features,
+                embedding_features=self.grouped_embedding_features,
+            )
+
+            # Get SHAP values of current estimator
+            shap_values_fold = estimator.get_feature_importance(data=fold_pool, type="ShapValues")
+
+            # Remove last columns (base value) and flatten
+            if self.target_type == ModelTaskType.MULTICLASS:
+                all_shaps = shap_values_fold[:, :, :-1]
+                all_shaps = [all_shaps[:, :, k].flatten() for k in range(all_shaps.shape[2])]
+            else:
+                all_shaps = shap_values_fold[:, :-1]
+                all_shaps = [all_shaps[:, k].flatten() for k in range(all_shaps.shape[1])]
+
+            all_shaps = np.abs(all_shaps)
+
+            return dict(zip(estimator.feature_names_, all_shaps))
+
+        except Exception:
+            self.logger.exception("Failed to recalculate new SHAP values")
+            return None
+
 
 class LightGBMWrapper(EstimatorWrapper):
     def __init__(

{upgini-1.2.14a3616.dev3 → upgini-1.2.16}/src/upgini/normalizer/normalize_utils.py

@@ -1,6 +1,6 @@
 import hashlib
 from logging import Logger, getLogger
-from typing import Dict, List
+from typing import Dict, List, Tuple
 
 import numpy as np
 import pandas as pd
@@ -35,22 +35,25 @@ class Normalizer:
 
     def __init__(
         self,
-        search_keys: Dict[str, SearchKey],
-        generated_features: List[str],
         bundle: ResourceBundle = None,
         logger: Logger = None,
         warnings_counter: WarningCounter = None,
         silent_mode=False,
     ):
-        self.search_keys = search_keys
-        self.generated_features = generated_features
         self.bundle = bundle or get_custom_bundle()
         self.logger = logger or getLogger()
         self.warnings_counter = warnings_counter or WarningCounter()
         self.silent_mode = silent_mode
         self.columns_renaming = {}
+        self.search_keys = {}
+        self.generated_features = []
+
+    def normalize(
+        self, df: pd.DataFrame, search_keys: Dict[str, SearchKey], generated_features: List[str]
+    ) -> Tuple[pd.DataFrame, Dict[str, SearchKey], List[str]]:
+        self.search_keys = search_keys.copy()
+        self.generated_features = generated_features.copy()
 
-    def normalize(self, df: pd.DataFrame) -> pd.DataFrame:
         df = df.copy()
         df = self._rename_columns(df)
 
@@ -68,21 +71,25 @@ class Normalizer:
 
         df = self.__convert_features_types(df)
 
-        return df
+        return df, self.search_keys, self.generated_features
 
     def _rename_columns(self, df: pd.DataFrame):
         # logger.info("Replace restricted symbols in column names")
         new_columns = []
         dup_counter = 0
         for column in df.columns:
-            if column in [
-                TARGET,
-                EVAL_SET_INDEX,
-                SYSTEM_RECORD_ID,
-                ENTITY_SYSTEM_RECORD_ID,
-                SEARCH_KEY_UNNEST,
-                DateTimeSearchKeyConverter.DATETIME_COL,
-            ] + self.generated_features:
+            if (
+                column
+                in [
+                    TARGET,
+                    EVAL_SET_INDEX,
+                    SYSTEM_RECORD_ID,
+                    ENTITY_SYSTEM_RECORD_ID,
+                    SEARCH_KEY_UNNEST,
+                    DateTimeSearchKeyConverter.DATETIME_COL,
+                ]
+                + self.generated_features
+            ):
                 self.columns_renaming[column] = column
                 new_columns.append(column)
                 continue

{upgini-1.2.14a3616.dev3 → upgini-1.2.16}/src/upgini/resource_bundle/strings.properties

@@ -203,11 +203,18 @@ email_detected=Emails detected in column `{}`. It will be used as a search key\n
 email_detected_not_registered=Emails detected in column `{}`. It can be used only with api_key from profile.upgini.com\nSee docs to turn off the automatic detection: https://github.com/upgini/upgini/blob/main/README.md#turn-off-autodetection-for-search-key-columns
 phone_detected=Phone numbers detected in column `{}`. It can be used only with api_key from profile.upgini.com\nSee docs to turn off the automatic detection: https://github.com/upgini/upgini/blob/main/README.md#turn-off-autodetection-for-search-key-columns
 phone_detected_not_registered=\nWARNING: Phone numbers detected in column `{}`. It can be used only with api_key from profile.upgini.com\nSee docs to turn off the automatic detection: https://github.com/upgini/upgini/blob/main/README.md#turn-off-autodetection-for-search-key-columns
-target_type_detected=\nDetected task type: {}\n
+target_type_detected=\nDetected task type: {}. Reason: {}\nYou can set task type manually with argument `model_task_type` of FeaturesEnricher constructor if task type detected incorrectly\n
+binary_target_reason=only two unique label-values observed
+non_numeric_multiclass_reason=non-numeric label values observed
+few_unique_label_multiclass_reason=few unique label-values observed and can be considered as categorical
+date_search_key_regression_reason=date search key is present, treating as regression
+many_unique_label_regression_reason=many unique label-values or non-integer floating point values observed
+limited_int_multiclass_reason=integer-like values with limited unique values observed
 # all_ok_community_invite=Chat with us in Slack community:
 all_ok_community_invite=❓ Support request 
 too_small_for_metrics=Your train dataset or one of eval datasets contains less than 500 rows. For such dataset Upgini will not calculate accuracy metrics. Please increase the number of rows in the training dataset to calculate accuracy metrics
 imbalance_multiclass=Class {0} is on 25% quantile of classes distribution ({1} records in train dataset). \nDownsample classes with records more than {1}.
+imbalanced_target=\nWARNING: Target is imbalanced and will be undersampled. Frequency of the rarest class `{}` is {}
 loss_selection_info=Using loss `{}` for feature selection
 loss_calc_metrics_info=Using loss `{}` for metrics calculation with default estimator
 

{upgini-1.2.14a3616.dev3 → upgini-1.2.16}/src/upgini/utils/display_utils.py

@@ -9,6 +9,7 @@ from typing import Callable, List, Optional
 
 import pandas as pd
 from xhtml2pdf import pisa
+
 from upgini.__about__ import __version__
 
 
@@ -72,7 +73,9 @@ def make_table(df: pd.DataFrame, wrap_long_string=None) -> str:
     )
 
 
-def display_html_dataframe(df: pd.DataFrame, internal_df: pd.DataFrame, header: str):
+def display_html_dataframe(
+    df: pd.DataFrame, internal_df: pd.DataFrame, header: str, display_id: Optional[str] = None, display_handle=None
+):
     if not ipython_available():
         print(header)
         print(internal_df)
@@ -133,7 +136,10 @@ def display_html_dataframe(df: pd.DataFrame, internal_df: pd.DataFrame, header:
             {table_html}
         </div>
         """
-    display(HTML(result_html))
+    if display_handle:
+        return display_handle.update(HTML(result_html))
+    else:
+        return display(HTML(result_html), display_id=display_id)
 
 
 def make_html_report(

{upgini-1.2.14a3616.dev3 → upgini-1.2.16}/src/upgini/utils/target_utils.py

@@ -24,49 +24,83 @@ def define_task(
 ) -> ModelTaskType:
     if logger is None:
         logger = logging.getLogger()
+
+    # Replace inf and -inf with NaN to handle extreme values correctly
+    y = y.replace([np.inf, -np.inf], np.nan, inplace=False)
+
+    # Drop NaN values from the target
     target = y.dropna()
+
+    # Check if target is numeric and finite
     if is_numeric_dtype(target):
         target = target.loc[np.isfinite(target)]
     else:
+        # If not numeric, drop empty strings as well
         target = target.loc[target != ""]
+
+    # Raise error if there are no valid values left in the target
     if len(target) == 0:
         raise ValidationError(bundle.get("empty_target"))
+
+    # Count unique values in the target
     target_items = target.nunique()
+
+    # Raise error if all target values are the same
     if target_items == 1:
         raise ValidationError(bundle.get("dataset_constant_target"))
+
+    reason = ""  # Will store the reason for selecting the task type
+
+    # Binary classification case: exactly two unique values
     if target_items == 2:
         task = ModelTaskType.BINARY
+        reason = bundle.get("binary_target_reason")
     else:
+        # Attempt to convert target to numeric
         try:
             target = pd.to_numeric(target)
             is_numeric = True
         except Exception:
             is_numeric = False
 
-        # If any value is non numeric - multiclass
+        # If target cannot be converted to numeric, assume multiclass classification
         if not is_numeric:
             task = ModelTaskType.MULTICLASS
+            reason = bundle.get("non_numeric_multiclass_reason")
         else:
+            # Multiclass classification: few unique values and integer encoding
             if target.nunique() <= 50 and is_int_encoding(target.unique()):
                 task = ModelTaskType.MULTICLASS
+                reason = bundle.get("few_unique_label_multiclass_reason")
+            # Regression case: if there is date, assume regression
             elif has_date:
                 task = ModelTaskType.REGRESSION
+                reason = bundle.get("date_search_key_regression_reason")
             else:
+                # Remove zero values and recalculate unique ratio
                 non_zero_target = target[target != 0]
                 target_items = non_zero_target.nunique()
                 target_ratio = target_items / len(non_zero_target)
+
+                # Use unique_ratio to determine whether to classify as regression or multiclass
                 if (
-                    (target.dtype.kind == "f" and np.any(target != target.astype(int)))  # any non integer
+                    (target.dtype.kind == "f" and np.any(target != target.astype(int)))  # Non-integer float values
                     or target_items > 50
-                    or target_ratio > 0.2
+                    or target_ratio > 0.2  # If non-zero values have high ratio of uniqueness
                 ):
                     task = ModelTaskType.REGRESSION
+                    reason = bundle.get("many_unique_label_regression_reason")
                 else:
                     task = ModelTaskType.MULTICLASS
+                    reason = bundle.get("limited_int_multiclass_reason")
 
-    logger.info(f"Detected task type: {task}")
+    # Log or print the reason for the selected task type
+    logger.info(f"Detected task type: {task} (Reason: {reason})")
+
+    # Print task type and reason if silent mode is off
     if not silent:
-        print(bundle.get("target_type_detected").format(task))
+        print(bundle.get("target_type_detected").format(task, reason))
+
     return task
 
 
@@ -81,8 +115,8 @@ def balance_undersample(
     target_column: str,
     task_type: ModelTaskType,
     random_state: int,
-    imbalance_threshold: int = 0.2,
-    min_sample_threshold: int = 5000,
+    binary_min_sample_threshold: int = 5000,
+    multiclass_min_sample_threshold: int = 25000,
     binary_bootstrap_loops: int = 5,
     multiclass_bootstrap_loops: int = 2,
     logger: Optional[logging.Logger] = None,
@@ -96,52 +130,60 @@ def balance_undersample(
     if SYSTEM_RECORD_ID not in df.columns:
         raise Exception("System record id must be presented for undersampling")
 
-    count = len(df)
+    # count = len(df)
     target = df[target_column].copy()
-    target_classes_count = target.nunique()
+    # target_classes_count = target.nunique()
 
     vc = target.value_counts()
     max_class_value = vc.index[0]
     min_class_value = vc.index[len(vc) - 1]
     max_class_count = vc[max_class_value]
     min_class_count = vc[min_class_value]
+    num_classes = len(vc)
 
-    min_class_percent = imbalance_threshold / target_classes_count
-    min_class_threshold = int(min_class_percent * count)
+    # min_class_percent = imbalance_threshold / target_classes_count
+    # min_class_threshold = int(min_class_percent * count)
 
     resampled_data = df
     df = df.copy().sort_values(by=SYSTEM_RECORD_ID)
     if task_type == ModelTaskType.MULTICLASS:
-        # Sort classes by rows count and find 25% quantile class
-        classes = vc.index
-        quantile25_idx = int(0.75 * len(classes)) - 1
-        quantile25_class = classes[quantile25_idx]
-        quantile25_class_cnt = vc[quantile25_class]
-
-        if max_class_count > (quantile25_class_cnt * multiclass_bootstrap_loops):
-            msg = bundle.get("imbalance_multiclass").format(quantile25_class, quantile25_class_cnt)
+        if len(df) > multiclass_min_sample_threshold and max_class_count > (
+            min_class_count * multiclass_bootstrap_loops
+        ):
+
+            # msg = bundle.get("imbalance_multiclass").format(min_class_value, min_class_count)
+            msg = bundle.get("imbalanced_target").format(min_class_value, min_class_count)
             logger.warning(msg)
             print(msg)
             if warning_counter:
                 warning_counter.increment()
 
-            # 25% and lower classes will stay as is. Higher classes will be downsampled
             sample_strategy = dict()
-            for class_idx in range(quantile25_idx):
-                # compare class count with count_of_quantile25_class * 2
-                class_value = classes[class_idx]
+            for class_value in vc.index:
+                if class_value == min_class_value:
+                    continue
                 class_count = vc[class_value]
-                sample_strategy[class_value] = min(class_count, quantile25_class_cnt * multiclass_bootstrap_loops)
+                sample_size = min(
+                    class_count,
+                    multiclass_bootstrap_loops
+                    * (
+                        min_class_count
+                        + max((multiclass_min_sample_threshold - num_classes * min_class_count) / (num_classes - 1), 0)
+                    ),
+                )
+                sample_strategy[class_value] = int(sample_size)
+            logger.info(f"Rebalance sample strategy: {sample_strategy}. Min class count: {min_class_count}")
             sampler = RandomUnderSampler(sampling_strategy=sample_strategy, random_state=random_state)
             X = df[SYSTEM_RECORD_ID]
             X = X.to_frame(SYSTEM_RECORD_ID)
             new_x, _ = sampler.fit_resample(X, target)  # type: ignore
 
             resampled_data = df[df[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
-    elif len(df) > min_sample_threshold and min_class_count < min_sample_threshold / 2:
-        msg = bundle.get("dataset_rarest_class_less_threshold").format(
-            min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
-        )
+    elif len(df) > binary_min_sample_threshold:
+        # msg = bundle.get("dataset_rarest_class_less_threshold").format(
+        #     min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
+        # )
+        msg = bundle.get("imbalanced_target").format(min_class_value, min_class_count)
         logger.warning(msg)
         print(msg)
         if warning_counter:
@@ -150,30 +192,38 @@ def balance_undersample(
         # fill up to min_sample_threshold by majority class
         minority_class = df[df[target_column] == min_class_value]
         majority_class = df[df[target_column] != min_class_value]
-        sample_size = min(len(majority_class), min_sample_threshold - min_class_count)
+        # sample_size = min(len(majority_class), min_sample_threshold - min_class_count)
+        sample_size = min(
+            max_class_count,
+            binary_bootstrap_loops * (min_class_count + max(binary_min_sample_threshold - 2 * min_class_count, 0)),
+        )
+        logger.info(
+            f"Min class count: {min_class_count}. Max class count: {max_class_count}."
+            f" Rebalance sample size: {sample_size}"
+        )
         sampled_majority_class = majority_class.sample(n=sample_size, random_state=random_state)
         resampled_data = df[
             (df[SYSTEM_RECORD_ID].isin(minority_class[SYSTEM_RECORD_ID]))
             | (df[SYSTEM_RECORD_ID].isin(sampled_majority_class[SYSTEM_RECORD_ID]))
         ]
 
-    elif max_class_count > min_class_count * binary_bootstrap_loops:
-        msg = bundle.get("dataset_rarest_class_less_threshold").format(
-            min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
-        )
-        logger.warning(msg)
-        print(msg)
-        if warning_counter:
-            warning_counter.increment()
-
-        sampler = RandomUnderSampler(
-            sampling_strategy={max_class_value: binary_bootstrap_loops * min_class_count}, random_state=random_state
-        )
-        X = df[SYSTEM_RECORD_ID]
-        X = X.to_frame(SYSTEM_RECORD_ID)
-        new_x, _ = sampler.fit_resample(X, target)  # type: ignore
-
-        resampled_data = df[df[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
+    # elif max_class_count > min_class_count * binary_bootstrap_loops:
+    #     msg = bundle.get("dataset_rarest_class_less_threshold").format(
+    #         min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
+    #     )
+    #     logger.warning(msg)
+    #     print(msg)
+    #     if warning_counter:
+    #         warning_counter.increment()
+
+    #     sampler = RandomUnderSampler(
+    #         sampling_strategy={max_class_value: binary_bootstrap_loops * min_class_count}, random_state=random_state
+    #     )
+    #     X = df[SYSTEM_RECORD_ID]
+    #     X = X.to_frame(SYSTEM_RECORD_ID)
+    #     new_x, _ = sampler.fit_resample(X, target)  # type: ignore
+
+    #     resampled_data = df[df[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
 
     logger.info(f"Shape after rebalance resampling: {resampled_data}")
     return resampled_data

upgini-1.2.14a3616.dev3/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.14a3616.dev3"