upgini 1.2.29a7__py3-none-any.whl → 1.2.30__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.29a7"
+__version__ = "1.2.30"

upgini/features_enricher.py

@@ -54,6 +54,7 @@ from upgini.metadata import (
     SYSTEM_RECORD_ID,
     TARGET,
     CVType,
+    FeaturesMetadataV2,
     FileColumnMeaningType,
     ModelTaskType,
     RuntimeParameters,
@@ -349,6 +350,7 @@ class FeaturesEnricher(TransformerMixin):
         self.add_date_if_missing = add_date_if_missing
         self.features_info_display_handle = None
         self.data_sources_display_handle = None
+        self.autofe_features_display_handle = None
         self.report_button_handle = None
 
     def _get_api_key(self):
@@ -1048,7 +1050,7 @@ class FeaturesEnricher(TransformerMixin):
                         enriched_shaps = enriched_cv_result.shap_values
 
                         if enriched_shaps is not None:
-                            self._update_shap_values(enriched_shaps)
+                            self._update_shap_values(trace_id, validated_X.columns.to_list(), enriched_shaps)
 
                         if enriched_metric is None:
                             self.logger.warning(
@@ -1208,37 +1210,11 @@ class FeaturesEnricher(TransformerMixin):
             finally:
                 self.logger.info(f"Calculating metrics elapsed time: {time.time() - start_time}")
 
-    def _update_shap_values(self, new_shaps: Dict[str, float]):
+    def _update_shap_values(self, trace_id: str, x_columns: List[str], new_shaps: Dict[str, float]):
         new_shaps = {
             feature: _round_shap_value(shap) for feature, shap in new_shaps.items() if feature in self.feature_names_
         }
-        features_importances = list(new_shaps.items())
-        features_importances.sort(key=lambda m: (-m[1], m[0]))
-        self.feature_names_, self.feature_importances_ = zip(*features_importances)
-        self.feature_names_ = list(self.feature_names_)
-        self.feature_importances_ = list(self.feature_importances_)
-
-        feature_name_header = self.bundle.get("features_info_name")
-        shap_value_header = self.bundle.get("features_info_shap")
-
-        def update_shap(row):
-            return new_shaps.get(row[feature_name_header], row[shap_value_header])
-
-        self.features_info[shap_value_header] = self.features_info.apply(update_shap, axis=1)
-        self._internal_features_info[shap_value_header] = self._internal_features_info.apply(update_shap, axis=1)
-        self._features_info_without_links[shap_value_header] = self._features_info_without_links.apply(
-            update_shap, axis=1
-        )
-        self.logger.info(f"Recalculated SHAP values:\n{self._features_info_without_links}")
-
-        self.features_info.sort_values(by=shap_value_header, ascending=False, inplace=True)
-        self._internal_features_info.sort_values(by=shap_value_header, ascending=False, inplace=True)
-        self._features_info_without_links.sort_values(by=shap_value_header, ascending=False, inplace=True)
-
-        self.relevant_data_sources = self._group_relevant_data_sources(self.features_info, self.bundle)
-        self._relevant_data_sources_wo_links = self._group_relevant_data_sources(
-            self._features_info_without_links, self.bundle
-        )
+        self.__prepare_feature_importances(trace_id, x_columns, new_shaps, silent=True)
 
         if self.features_info_display_handle is not None:
             try:
@@ -1251,7 +1227,7 @@ class FeaturesEnricher(TransformerMixin):
                     display_handle=self.features_info_display_handle,
                 )
             except (ImportError, NameError):
-                print(self._internal_features_info)
+                pass
         if self.data_sources_display_handle is not None:
             try:
                 _ = get_ipython()  # type: ignore
@@ -1259,11 +1235,24 @@ class FeaturesEnricher(TransformerMixin):
                 display_html_dataframe(
                     self.relevant_data_sources,
                     self._relevant_data_sources_wo_links,
-                    self.bundle.get("relevant_features_header"),
+                    self.bundle.get("relevant_data_sources_header"),
                     display_handle=self.data_sources_display_handle,
                 )
             except (ImportError, NameError):
-                print(self._relevant_data_sources_wo_links)
+                pass
+        if self.autofe_features_display_handle is not None:
+            try:
+                _ = get_ipython()  # type: ignore
+                autofe_descriptions_df = self.get_autofe_features_description()
+                if autofe_descriptions_df is not None:
+                    display_html_dataframe(
+                        df=autofe_descriptions_df,
+                        internal_df=autofe_descriptions_df,
+                        header=self.bundle.get("autofe_descriptions_header"),
+                        display_handle=self.autofe_features_display_handle,
+                    )
+            except (ImportError, NameError):
+                pass
         if self.report_button_handle is not None:
             try:
                 _ = get_ipython()  # type: ignore
@@ -1999,9 +1988,19 @@ class FeaturesEnricher(TransformerMixin):
         file_metadata = self._search_task.get_file_metadata(str(uuid.uuid4()))
         search_keys = file_metadata.search_types()
         if SearchKey.IPV6_ADDRESS in search_keys:
-            search_keys.remove(SearchKey.IPV6_ADDRESS)
+            # search_keys.remove(SearchKey.IPV6_ADDRESS)
+            search_keys.pop(SearchKey.IPV6_ADDRESS, None)
 
-        keys = "{" + ", ".join([f'"{key.name}": "{key_example(key)}"' for key in search_keys]) + "}"
+        keys = (
+            "{"
+            + ", ".join(
+                [
+                    f'"{key.name}": {{"name": "{name}", "value": "{key_example(key)}"}}'
+                    for key, name in search_keys.items()
+                ]
+            )
+            + "}"
+        )
         features_for_transform = self._search_task.get_features_for_transform()
         if features_for_transform:
             original_features_for_transform = [
@@ -2804,7 +2803,12 @@ class FeaturesEnricher(TransformerMixin):
         autofe_description = self.get_autofe_features_description()
         if autofe_description is not None:
             self.logger.info(f"AutoFE descriptions: {autofe_description}")
-            display_html_dataframe(autofe_description, autofe_description, "*Description of AutoFE feature names")
+            self.autofe_features_display_handle = display_html_dataframe(
+                df=autofe_description,
+                internal_df=autofe_description,
+                header=self.bundle.get("autofe_descriptions_header"),
+                display_id="autofe_descriptions",
+            )
 
         if self._has_paid_features(exclude_features_sources):
             if calculate_metrics is not None and calculate_metrics:
@@ -3516,7 +3520,9 @@ class FeaturesEnricher(TransformerMixin):
 
         return result_train, result_eval_sets
 
-    def __prepare_feature_importances(self, trace_id: str, x_columns: List[str], silent=False):
+    def __prepare_feature_importances(
+            self, trace_id: str, x_columns: List[str], updated_shaps: Optional[Dict[str, float]] = None, silent=False
+    ):
         if self._search_task is None:
             raise NotFittedError(self.bundle.get("transform_unfitted_enricher"))
         features_meta = self._search_task.get_all_features_metadata_v2()
@@ -3533,6 +3539,10 @@ class FeaturesEnricher(TransformerMixin):
         features_info_without_links = []
         internal_features_info = []
 
+        if updated_shaps is not None:
+            for fm in features_meta:
+                fm.shap_value = updated_shaps.get(fm.name, 0.0)
+
         features_meta.sort(key=lambda m: (-m.shap_value, m.name))
         for feature_meta in features_meta:
             if feature_meta.name in original_names_dict.keys():
@@ -3585,7 +3595,22 @@ class FeaturesEnricher(TransformerMixin):
             autofe_meta = self._search_task.get_autofe_metadata()
             if autofe_meta is None:
                 return None
-            features_meta = self._search_task.get_all_features_metadata_v2()
+            if len(self._internal_features_info) != 0:
+
+                def to_feature_meta(row):
+                    fm = FeaturesMetadataV2(
+                        name=row[bundle.get("features_info_name")],
+                        type="",
+                        source="",
+                        hit_rate=row[bundle.get("features_info_hitrate")],
+                        shap_value=row[bundle.get("features_info_shap")],
+                        data_source=row[bundle.get("features_info_source")],
+                    )
+                    return fm
+
+                features_meta = self._internal_features_info.apply(to_feature_meta, axis=1).to_list()
+            else:
+                features_meta = self._search_task.get_all_features_metadata_v2()
 
             def get_feature_by_name(name: str):
                 for m in features_meta:
@@ -3614,27 +3639,32 @@ class FeaturesEnricher(TransformerMixin):
                     self.logger.warning(f"Feature meta for display index {m.display_index} not found")
                     continue
                 description["shap"] = feature_meta.shap_value
-                description["Sources"] = feature_meta.data_source.replace("AutoFE: features from ", "").replace(
-                    "AutoFE: feature from ", ""
-                )
-                description["Feature name"] = feature_meta.name
+                description[self.bundle.get("autofe_descriptions_sources")] = feature_meta.data_source.replace(
+                    "AutoFE: features from ", ""
+                ).replace("AutoFE: feature from ", "")
+                description[self.bundle.get("autofe_descriptions_feature_name")] = feature_meta.name
 
                 feature_idx = 1
                 for bc in m.base_columns:
-                    description[f"Feature {feature_idx}"] = bc.hashed_name
+                    description[self.bundle.get("autofe_descriptions_feature").format(feature_idx)] = bc.hashed_name
                     feature_idx += 1
 
-                description["Function"] = ",".join(sorted(autofe_feature.get_all_operand_names()))
+                description[self.bundle.get("autofe_descriptions_function")] = ",".join(
+                    sorted(autofe_feature.get_all_operand_names())
+                )
 
                 descriptions.append(description)
 
             if len(descriptions) == 0:
                 return None
 
-            descriptions_df = pd.DataFrame(descriptions)
-            descriptions_df.fillna("", inplace=True)
-            descriptions_df.sort_values(by="shap", ascending=False, inplace=True)
-            descriptions_df.drop(columns="shap", inplace=True)
+            descriptions_df = (
+                pd.DataFrame(descriptions)
+                .fillna("")
+                .sort_values(by="shap", ascending=False)
+                .drop(columns="shap")
+                .reset_index(drop=True)
+            )
             return descriptions_df
 
         except Exception:

upgini/metadata.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from enum import Enum
-from typing import Any, Dict, List, Optional, Set, Union
+from typing import Any, Dict, List, Optional, Union
 
 from pydantic import BaseModel
 
@@ -228,13 +228,13 @@ class FileMetadata(BaseModel):
                 return c
         return None
 
-    def search_types(self) -> Set[SearchKey]:
-        search_keys = set()
+    def search_types(self) -> Dict[SearchKey, str]:
+        search_keys = dict()
         for keys_group in self.searchKeys:
             for key in keys_group:
                 column = self.column_by_name(key)
                 if column:
-                    search_keys.add(SearchKey.from_meaning_type(column.meaningType))
+                    search_keys[SearchKey.from_meaning_type(column.meaningType)] = column.name
         return search_keys
 
 

upgini/resource_bundle/strings.properties

@@ -251,6 +251,14 @@ relevant_data_sources_header=Relevant data sources
 relevant_data_sources_all_shap=All features SHAP
 relevant_data_sources_number=Number of relevant features
 
+# Autofe descriptions
+autofe_descriptions_header=*Description of AutoFE feature names
+autofe_descriptions_sources=Sources
+autofe_descriptions_feature_name=Feature name
+autofe_descriptions_feature=Feature {}
+autofe_descriptions_function=Function
+
+
 # Quality metrics table
 quality_metrics_header=Accuracy after enrichment
 quality_metrics_train_segment=Train

{upgini-1.2.29a7.dist-info → upgini-1.2.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.29a7
+Version: 1.2.30
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.29a7.dist-info → upgini-1.2.30.dist-info}/RECORD

@@ -1,12 +1,12 @@
-upgini/__about__.py,sha256=zQC-_yiNcwPq8o3NPpgr0tGKyMXiUXgF1aIDtN0fDEk,25
+upgini/__about__.py,sha256=WGj1CvVJizDkAvd9BtLpwcsI-hzacJoXGbC8sVpoHYk,23
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=iPFiMJtk4HF1ytw9wCQr8H9RfoOKj_TIo8XYZKWgcMc,31331
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=tBdArcifhTLuDIL4D_eRB1gIBt9ayTVU7Ox2fVKE68c,191300
+upgini/features_enricher.py,sha256=lNfu5Z40NmkkGJScKAwe_0VBtL8liePifuAlKE_flfA,192053
 upgini/http.py,sha256=plZGTGoi1h2edd8Cnjt4eYB8t4NbBGnZz7DtPTByiNc,42885
 upgini/lazy_import.py,sha256=74gQ8JuA48BGRLxAo7lNHNKY2D2emMxrUxKGdxVGhuY,1012
-upgini/metadata.py,sha256=osmzdNESeh7yP3BZday6N9Q3eaIHfzhhRM1d6NSgcf0,11223
+upgini/metadata.py,sha256=lUa2xYhBhnCeTqNt6lWc9iP_YuikYGIsDSn8Vwyjv1I,11235
 upgini/metrics.py,sha256=hr7UwLphbZ_FEglLuO2lzr_pFgxOJ4c3WBeg7H-fNqY,35521
 upgini/search_task.py,sha256=qxUxAD-bed-FpZYmTB_4orW7YJsW_O6a1TcgnZIRFr4,17307
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
@@ -30,7 +30,7 @@ upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 upgini/normalizer/normalize_utils.py,sha256=Ft2MwSgVoBilXAORAOYAuwPD79GOLfwn4qQE3IUFzzg,7218
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=fOAeLTsnx8xvJK-7RPFXprATG0n56jeCdse8sQTuVX8,26674
+upgini/resource_bundle/strings.properties,sha256=bKw_rjZZTomLJhQBqiM7_P2EoRq45_Ng2gP4WE6MRBE,26921
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=7GpjYqjOp58vYcJLiX__1R5wjUlyQbxvHJ2klFnup_M,6389
@@ -59,7 +59,7 @@ upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,4
 upgini/utils/target_utils.py,sha256=PU77nIhTz7IHbC4rpTpxrVxib6cdpRL9F1dhkjIffLY,10225
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.29a7.dist-info/METADATA,sha256=sE4t490pcKTOegDZx5S7gX4eh9j_pk8zHl5xKe8Qy08,48580
-upgini-1.2.29a7.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.29a7.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.29a7.dist-info/RECORD,,
+upgini-1.2.30.dist-info/METADATA,sha256=93iMDL28nXF2DJaDP-oUS2CeCFHgnjk2zmAhge2LAHg,48578
+upgini-1.2.30.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.30.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.30.dist-info/RECORD,,