upgini 1.2.16__py3-none-any.whl → 1.2.16a1__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.16"
+__version__ = "1.2.16a1"

upgini/features_enricher.py

@@ -336,7 +336,6 @@ class FeaturesEnricher(TransformerMixin):
         self.exclude_columns = exclude_columns
         self.baseline_score_column = baseline_score_column
         self.add_date_if_missing = add_date_if_missing
-        self.features_info_display_handle = None
 
     def _get_api_key(self):
         return self._api_key
@@ -1035,9 +1034,7 @@ class FeaturesEnricher(TransformerMixin):
                             fitting_enriched_X, enriched_y_sorted
                         )
 
-                        if enriched_shaps is not None:
-                            self._update_shap_values(enriched_shaps)
-
+                        print(f"Calculated enriched shaps: {enriched_shaps}")
                         if enriched_metric is None:
                             self.logger.warning(
                                 f"Enriched {metric} on train combined features is None (maybe all features was removed)"
@@ -1195,17 +1192,6 @@ class FeaturesEnricher(TransformerMixin):
                 self.logger.info(f"Calculating metrics elapsed time: {time.time() - start_time}")
 
     def _update_shap_values(self, new_shaps: Dict[str, float]):
-        new_shaps = {
-            feature: self._round_shap_value(shap)
-            for feature, shap in new_shaps.items()
-            if feature in self.feature_names_
-        }
-        features_importances = list(new_shaps.items())
-        features_importances.sort(key=lambda m: (-m[1], m[0]))
-        self.feature_names_, self.feature_importances_ = zip(*features_importances)
-        self.feature_names_ = list(self.feature_names_)
-        self.feature_importances_ = list(self.feature_importances_)
-
         feature_name_header = self.bundle.get("features_info_name")
         shap_value_header = self.bundle.get("features_info_shap")
 
@@ -1213,28 +1199,9 @@ class FeaturesEnricher(TransformerMixin):
             return new_shaps.get(row[feature_name_header], row[shap_value_header])
 
         self.features_info[shap_value_header] = self.features_info.apply(update_shap, axis=1)
-        self._internal_features_info[shap_value_header] = self._internal_features_info.apply(update_shap, axis=1)
-        self._features_info_without_links[shap_value_header] = self._features_info_without_links.apply(
-            update_shap, axis=1
-        )
-        self.logger.info(f"Recalculated SHAP values:\n{self._features_info_without_links}")
-
         self.features_info.sort_values(by=shap_value_header, ascending=False, inplace=True)
-        self._internal_features_info.sort_values(by=shap_value_header, ascending=False, inplace=True)
-        self._features_info_without_links.sort_values(by=shap_value_header, ascending=False, inplace=True)
 
-        if self.features_info_display_handle:
-            try:
-                _ = get_ipython()  # type: ignore
-
-                display_html_dataframe(
-                    self.features_info,
-                    self._features_info_without_links,
-                    self.bundle.get("relevant_features_header"),
-                    display_handle=self.features_info_display_handle,
-                )
-            except (ImportError, NameError):
-                print(self._internal_features_info)
+        # TODO redraw
 
     def _check_train_and_eval_target_distribution(self, y, eval_set_dict):
         uneven_distribution = False
@@ -1564,19 +1531,11 @@ class FeaturesEnricher(TransformerMixin):
             self.logger.info("No external features selected. So use only input datasets for metrics calculation")
             return self.__sample_only_input(validated_X, validated_y, eval_set, is_demo_dataset)
         # TODO save and check if dataset was deduplicated - use imbalance branch for such case
-        elif (
-            not self.imbalanced
-            and not exclude_features_sources
-            and is_input_same_as_fit
-            and self.df_with_original_index is not None
-        ):
+        elif not self.imbalanced and not exclude_features_sources and is_input_same_as_fit:
             self.logger.info("Dataset is not imbalanced, so use enriched_X from fit")
             return self.__sample_balanced(eval_set, trace_id, remove_outliers_calc_metrics)
         else:
-            self.logger.info(
-                "Dataset is imbalanced or exclude_features_sources or X was passed or this is saved search."
-                " Run transform"
-            )
+            self.logger.info("Dataset is imbalanced or exclude_features_sources or X was passed. Run transform")
             print(self.bundle.get("prepare_data_for_metrics"))
             return self.__sample_imbalanced(
                 validated_X,
@@ -1635,7 +1594,9 @@ class FeaturesEnricher(TransformerMixin):
             generated_features.extend(generator.generated_features)
 
         normalizer = Normalizer(self.bundle, self.logger, self.warning_counter)
-        df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
+        df, search_keys, generated_features = normalizer.normalize(
+            df, search_keys, generated_features
+        )
         columns_renaming = normalizer.columns_renaming
 
         df = clean_full_duplicates(df, logger=self.logger, silent=True, bundle=self.bundle)
@@ -2075,7 +2036,9 @@ class FeaturesEnricher(TransformerMixin):
                 generated_features.extend(generator.generated_features)
 
             normalizer = Normalizer(self.bundle, self.logger, self.warning_counter, silent_mode)
-            df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
+            df, search_keys, generated_features = normalizer.normalize(
+                df, search_keys, generated_features
+            )
             columns_renaming = normalizer.columns_renaming
 
             # Don't pass all features in backend on transform
@@ -3431,13 +3394,6 @@ class FeaturesEnricher(TransformerMixin):
 
         return result_train, result_eval_sets
 
-    @staticmethod
-    def _round_shap_value(shap: float) -> float:
-        if shap > 0.0 and shap < 0.0001:
-            return 0.0001
-        else:
-            return round(shap, 4)
-
     def __prepare_feature_importances(self, trace_id: str, x_columns: List[str], silent=False):
         llm_source = "LLM with external data augmentation"
         if self._search_task is None:
@@ -3455,6 +3411,12 @@ class FeaturesEnricher(TransformerMixin):
         features_info_without_links = []
         internal_features_info = []
 
+        def round_shap_value(shap: float) -> float:
+            if shap > 0.0 and shap < 0.0001:
+                return 0.0001
+            else:
+                return round(shap, 4)
+
         def list_or_single(lst: List[str], single: str):
             return lst or ([single] if single else [])
 
@@ -3487,7 +3449,7 @@ class FeaturesEnricher(TransformerMixin):
 
             feature_sample = []
             self.feature_names_.append(feature_meta.name)
-            self.feature_importances_.append(self._round_shap_value(feature_meta.shap_value))
+            self.feature_importances_.append(round_shap_value(feature_meta.shap_value))
             if feature_meta.name in features_df.columns:
                 feature_sample = np.random.choice(features_df[feature_meta.name].dropna().unique(), 3).tolist()
                 if len(feature_sample) > 0 and isinstance(feature_sample[0], float):
@@ -3526,7 +3488,7 @@ class FeaturesEnricher(TransformerMixin):
             features_info.append(
                 {
                     self.bundle.get("features_info_name"): feature_name,
-                    self.bundle.get("features_info_shap"): self._round_shap_value(feature_meta.shap_value),
+                    self.bundle.get("features_info_shap"): round_shap_value(feature_meta.shap_value),
                     self.bundle.get("features_info_hitrate"): feature_meta.hit_rate,
                     self.bundle.get("features_info_value_preview"): feature_sample,
                     self.bundle.get("features_info_provider"): provider,
@@ -3537,7 +3499,7 @@ class FeaturesEnricher(TransformerMixin):
             features_info_without_links.append(
                 {
                     self.bundle.get("features_info_name"): internal_feature_name,
-                    self.bundle.get("features_info_shap"): self._round_shap_value(feature_meta.shap_value),
+                    self.bundle.get("features_info_shap"): round_shap_value(feature_meta.shap_value),
                     self.bundle.get("features_info_hitrate"): feature_meta.hit_rate,
                     self.bundle.get("features_info_value_preview"): feature_sample,
                     self.bundle.get("features_info_provider"): internal_provider,
@@ -3549,7 +3511,7 @@ class FeaturesEnricher(TransformerMixin):
                 {
                     self.bundle.get("features_info_name"): internal_feature_name,
                     "feature_link": feature_meta.doc_link,
-                    self.bundle.get("features_info_shap"): self._round_shap_value(feature_meta.shap_value),
+                    self.bundle.get("features_info_shap"): round_shap_value(feature_meta.shap_value),
                     self.bundle.get("features_info_hitrate"): feature_meta.hit_rate,
                     self.bundle.get("features_info_value_preview"): feature_sample,
                     self.bundle.get("features_info_provider"): internal_provider,
@@ -3829,11 +3791,8 @@ class FeaturesEnricher(TransformerMixin):
             print(Format.GREEN + Format.BOLD + msg + Format.END)
             self.logger.info(msg)
             if len(self.feature_names_) > 0:
-                self.features_info_display_handle = display_html_dataframe(
-                    self.features_info,
-                    self._features_info_without_links,
-                    self.bundle.get("relevant_features_header"),
-                    display_id="features_info",
+                display_html_dataframe(
+                    self.features_info, self._features_info_without_links, self.bundle.get("relevant_features_header")
                 )
 
                 display_html_dataframe(

upgini/metrics.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
+from collections import defaultdict
 import inspect
 import logging
 import re
-from collections import defaultdict
 from copy import deepcopy
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
@@ -339,10 +339,10 @@ class EstimatorWrapper:
 
         if shap_values_all_folds:
             average_shap_values = {
-                feature: np.mean(np.array(shaps)) for feature, shaps in shap_values_all_folds.items() if len(shaps) > 0
+                feature: np.mean(shaps)
+                for feature, shaps
+                in shap_values_all_folds.items()
             }
-            if len(average_shap_values) == 0:
-                average_shap_values = None
         else:
             average_shap_values = None
 
@@ -480,7 +480,6 @@ class CatBoostWrapper(EstimatorWrapper):
         )
         self.cat_features = None
         self.emb_features = None
-        self.grouped_embedding_features = None
         self.exclude_features = []
 
     def _prepare_to_fit(self, x: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, np.ndarray, np.ndarray, dict]:
@@ -490,16 +489,17 @@ class CatBoostWrapper(EstimatorWrapper):
         if hasattr(CatBoostClassifier, "get_embedding_feature_indices"):
             emb_pattern = r"(.+)_emb\d+"
             self.emb_features = [c for c in x.columns if re.match(emb_pattern, c) and is_numeric_dtype(x[c])]
+            embedding_features = []
             if len(self.emb_features) > 3:  # There is no reason to reduce embeddings dimension with less than 4
                 self.logger.info(
                     "Embedding features count more than 3, so group them into one vector for CatBoost: "
                     f"{self.emb_features}"
                 )
-                x, self.grouped_embedding_features = self.group_embeddings(x)
-                params["embedding_features"] = self.grouped_embedding_features
+                x, embedding_features = self.group_embeddings(x)
+                params["embedding_features"] = embedding_features
             else:
                 self.logger.info(f"Embedding features count less than 3, so use them separately: {self.emb_features}")
-                self.grouped_embedding_features = None
+                self.emb_features = []
         else:
             self.logger.warning(f"Embedding features are not supported by Catboost version {catboost.__version__}")
 
@@ -515,7 +515,7 @@ class CatBoostWrapper(EstimatorWrapper):
             self.logger.warning(f"Text features are not supported by this Catboost version {catboost.__version__}")
 
         # Find rest categorical features
-        self.cat_features = _get_cat_features(x, self.text_features, self.grouped_embedding_features)
+        self.cat_features = _get_cat_features(x, self.text_features, embedding_features)
         # x = fill_na_cat_features(x, self.cat_features)
         unique_cat_features = []
         for name in self.cat_features:
@@ -603,13 +603,7 @@ class CatBoostWrapper(EstimatorWrapper):
     def calculate_shap(self, x: pd.DataFrame, y: pd.Series, estimator: CatBoost) -> Optional[Dict[str, float]]:
         try:
             # Create Pool for fold data, if need (for example, when categorical features are present)
-            fold_pool = Pool(
-                x,
-                y,
-                cat_features=self.cat_features,
-                text_features=self.text_features,
-                embedding_features=self.grouped_embedding_features,
-            )
+            fold_pool = Pool(x, y, cat_features=self.cat_features)
 
             # Get SHAP values of current estimator
             shap_values_fold = estimator.get_feature_importance(data=fold_pool, type="ShapValues")
@@ -627,7 +621,6 @@ class CatBoostWrapper(EstimatorWrapper):
             return dict(zip(estimator.feature_names_, all_shaps))
 
         except Exception:
-            self.logger.exception("Failed to recalculate new SHAP values")
             return None
 
 

upgini/normalizer/normalize_utils.py

@@ -49,7 +49,10 @@ class Normalizer:
         self.generated_features = []
 
     def normalize(
-        self, df: pd.DataFrame, search_keys: Dict[str, SearchKey], generated_features: List[str]
+        self,
+        df: pd.DataFrame,
+        search_keys: Dict[str, SearchKey],
+        generated_features: List[str],
     ) -> Tuple[pd.DataFrame, Dict[str, SearchKey], List[str]]:
         self.search_keys = search_keys.copy()
         self.generated_features = generated_features.copy()

upgini/utils/display_utils.py

@@ -9,7 +9,6 @@ from typing import Callable, List, Optional
 
 import pandas as pd
 from xhtml2pdf import pisa
-
 from upgini.__about__ import __version__
 
 
@@ -73,9 +72,7 @@ def make_table(df: pd.DataFrame, wrap_long_string=None) -> str:
     )
 
 
-def display_html_dataframe(
-    df: pd.DataFrame, internal_df: pd.DataFrame, header: str, display_id: Optional[str] = None, display_handle=None
-):
+def display_html_dataframe(df: pd.DataFrame, internal_df: pd.DataFrame, header: str, display_id: str):
     if not ipython_available():
         print(header)
         print(internal_df)
@@ -136,10 +133,7 @@ def display_html_dataframe(
             {table_html}
         </div>
         """
-    if display_handle:
-        return display_handle.update(HTML(result_html))
-    else:
-        return display(HTML(result_html), display_id=display_id)
+    return display(HTML(result_html))
 
 
 def make_html_report(

upgini/utils/features_validator.py

@@ -58,6 +58,10 @@ class FeaturesValidator:
 
         columns_renaming = columns_renaming or {}
 
+        if features_for_generate:
+            empty_or_constant_features = [
+                f for f in empty_or_constant_features if columns_renaming.get(f, f) not in features_for_generate
+            ]
         if empty_or_constant_features:
             msg = bundle.get("empty_or_contant_features").format(
                 [columns_renaming.get(f, f) for f in empty_or_constant_features]

{upgini-1.2.16.dist-info → upgini-1.2.16a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.16
+Version: 1.2.16a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -145,7 +145,7 @@ Description-Content-Type: text/markdown
 
 ## 💼 Tutorials
 
-###  [Search of relevant external features & Automated feature generation for Salary prediction task (use as a template)](https://github.com/upgini/upgini/blob/main/notebooks/Upgini_Features_search%26generation.ipynb)
+###  [Search of relevant external features & Automated feature generation for Salary predicton task (use as a template)](https://github.com/upgini/upgini/blob/main/notebooks/Upgini_Features_search%26generation.ipynb)
 
 * The goal is to predict salary for data science job postning based on information about employer and job description.
 * Following this guide, you'll learn how to **search & auto generate new relevant features with Upgini library**

{upgini-1.2.16.dist-info → upgini-1.2.16a1.dist-info}/RECORD

@@ -1,13 +1,13 @@
-upgini/__about__.py,sha256=7RIJ-Nh9kHJf5O5NvahUeP8DNXq6oIzbYcIt_yKv0lQ,23
+upgini/__about__.py,sha256=vMDC8s3UWLhN6avUSjtfizIVhxWIHW-WKTw04ha19HE,25
 upgini/__init__.py,sha256=M64LwQTBa-5Jz24Zm2h8rWwlKQQ1J8nP7gGgIciS0WU,589
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=iPFiMJtk4HF1ytw9wCQr8H9RfoOKj_TIo8XYZKWgcMc,31331
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=d3-QHyb8vYuWBImHumeA-BBrgT--5V-L91WgU1vduR8,190892
+upgini/features_enricher.py,sha256=oEWJjD3v4v_0fZr8ZWSzqFCs08yJrjVTDMNPEFsFL_E,188978
 upgini/http.py,sha256=21asexflvavydzCOONJDGQBtQanCElrbnqLXakJ9Cu8,42880
 upgini/lazy_import.py,sha256=74gQ8JuA48BGRLxAo7lNHNKY2D2emMxrUxKGdxVGhuY,1012
 upgini/metadata.py,sha256=osmzdNESeh7yP3BZday6N9Q3eaIHfzhhRM1d6NSgcf0,11223
-upgini/metrics.py,sha256=lhLqFv1tLWNzx3ULELo3MMSqI8eBoHL7P5jKpG8a6PE,33899
+upgini/metrics.py,sha256=zs_gnjZCdk8AUYOj-mD7V1k-8Gn2EfHcXvK7J6RWOxA,33492
 upgini/search_task.py,sha256=qxUxAD-bed-FpZYmTB_4orW7YJsW_O6a1TcgnZIRFr4,17307
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
 upgini/version_validator.py,sha256=ddSKUK_-eGJB3NgrqOMoWJU-OxQ253WsNLp8aqJkaIM,1389
@@ -27,7 +27,7 @@ upgini/data_source/data_source_publisher.py,sha256=X-8aGtVgzGmxyXkMVBoBLIGDMb4lY
 upgini/mdc/__init__.py,sha256=aM08nIWFc2gWdWUa3_IuEnNND0cQPkBGnYpRMnfFN8k,1019
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/normalizer/normalize_utils.py,sha256=Lv75lq7M46z9cAIutwkdKZtPZkWblgoRzToAJ1BwY8A,7709
+upgini/normalizer/normalize_utils.py,sha256=w7S4yQZkdlBptC7peqmrn8zqs-Z0RPq2rp78IZuoE7M,7734
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
 upgini/resource_bundle/strings.properties,sha256=eqJP6bGu12zFuQJqMY03QbMhppcdwIfL2bsJWaqmuZ4,27221
@@ -44,10 +44,10 @@ upgini/utils/custom_loss_utils.py,sha256=kieNZYBYZm5ZGBltF1F_jOSF4ea6C29rYuCyiDc
 upgini/utils/cv_utils.py,sha256=w6FQb9nO8BWDx88EF83NpjPLarK4eR4ia0Wg0kLBJC4,3525
 upgini/utils/datetime_utils.py,sha256=4tsGeehU0KS6wqNsc9gEEWZ9s6T9E0UReUIO3rSuXNU,12174
 upgini/utils/deduplicate_utils.py,sha256=NpaPtBYXwUtfKTRHWrtz2uUq6tZN6C_Nd719ydPRF2Q,8484
-upgini/utils/display_utils.py,sha256=ntmOs8VchrkPFVNp4iEhdXRRtYx823vJ51saECoY6Bk,11175
+upgini/utils/display_utils.py,sha256=kOY3lKKbJDIb424TFAF0wQiFUhcARTy2Flz0bQ2M8NY,11014
 upgini/utils/email_utils.py,sha256=j0Ug1R_0AnCg1Y92zIZ4XMwvKo3G5_pcOlBN1OH_gZs,5191
 upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0--cOFTwJ0,1074
-upgini/utils/features_validator.py,sha256=yiOdzVtpArELMufzAa9mtWq32lETB6sIF-w3Yvl3vV8,3614
+upgini/utils/features_validator.py,sha256=URNywJnfPVpRKGAK9drJIdyHarGczB298y9QGQwOVGE,3818
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
 upgini/utils/ip_utils.py,sha256=Q6vb7Sr5Khx3Sq3eENjW2qCXKej_S5jZbneH6zEOkzQ,5171
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
@@ -57,7 +57,7 @@ upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,4
 upgini/utils/target_utils.py,sha256=qHzZRmICFbLNCrmVqGkaBcjm91L2ERRZMppci36acV4,10085
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.2.16.dist-info/METADATA,sha256=hmLjva5M3m8LpawvBgEjKCrilENMBJAfax6mDnygVyw,48578
-upgini-1.2.16.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.16.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.16.dist-info/RECORD,,
+upgini-1.2.16a1.dist-info/METADATA,sha256=0w4SeT93Uz51cWP6Y0uHw0Eh2iMVqkUIOjlaaD_Jduw,48579
+upgini-1.2.16a1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.16a1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.16a1.dist-info/RECORD,,