upgini 1.2.30__py3-none-any.whl → 1.2.30a8__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.30"
+__version__ = "1.2.30a8"

upgini/features_enricher.py

@@ -54,7 +54,6 @@ from upgini.metadata import (
     SYSTEM_RECORD_ID,
     TARGET,
     CVType,
-    FeaturesMetadataV2,
     FileColumnMeaningType,
     ModelTaskType,
     RuntimeParameters,
@@ -350,7 +349,6 @@ class FeaturesEnricher(TransformerMixin):
         self.add_date_if_missing = add_date_if_missing
         self.features_info_display_handle = None
         self.data_sources_display_handle = None
-        self.autofe_features_display_handle = None
         self.report_button_handle = None
 
     def _get_api_key(self):
@@ -1050,7 +1048,7 @@ class FeaturesEnricher(TransformerMixin):
                         enriched_shaps = enriched_cv_result.shap_values
 
                         if enriched_shaps is not None:
-                            self._update_shap_values(trace_id, validated_X.columns.to_list(), enriched_shaps)
+                            self._update_shap_values(enriched_shaps)
 
                         if enriched_metric is None:
                             self.logger.warning(
@@ -1210,11 +1208,37 @@ class FeaturesEnricher(TransformerMixin):
             finally:
                 self.logger.info(f"Calculating metrics elapsed time: {time.time() - start_time}")
 
-    def _update_shap_values(self, trace_id: str, x_columns: List[str], new_shaps: Dict[str, float]):
+    def _update_shap_values(self, new_shaps: Dict[str, float]):
         new_shaps = {
             feature: _round_shap_value(shap) for feature, shap in new_shaps.items() if feature in self.feature_names_
         }
-        self.__prepare_feature_importances(trace_id, x_columns, new_shaps, silent=True)
+        features_importances = list(new_shaps.items())
+        features_importances.sort(key=lambda m: (-m[1], m[0]))
+        self.feature_names_, self.feature_importances_ = zip(*features_importances)
+        self.feature_names_ = list(self.feature_names_)
+        self.feature_importances_ = list(self.feature_importances_)
+
+        feature_name_header = self.bundle.get("features_info_name")
+        shap_value_header = self.bundle.get("features_info_shap")
+
+        def update_shap(row):
+            return new_shaps.get(row[feature_name_header], row[shap_value_header])
+
+        self.features_info[shap_value_header] = self.features_info.apply(update_shap, axis=1)
+        self._internal_features_info[shap_value_header] = self._internal_features_info.apply(update_shap, axis=1)
+        self._features_info_without_links[shap_value_header] = self._features_info_without_links.apply(
+            update_shap, axis=1
+        )
+        self.logger.info(f"Recalculated SHAP values:\n{self._features_info_without_links}")
+
+        self.features_info.sort_values(by=shap_value_header, ascending=False, inplace=True)
+        self._internal_features_info.sort_values(by=shap_value_header, ascending=False, inplace=True)
+        self._features_info_without_links.sort_values(by=shap_value_header, ascending=False, inplace=True)
+
+        self.relevant_data_sources = self._group_relevant_data_sources(self.features_info, self.bundle)
+        self._relevant_data_sources_wo_links = self._group_relevant_data_sources(
+            self._features_info_without_links, self.bundle
+        )
 
         if self.features_info_display_handle is not None:
             try:
@@ -1227,7 +1251,7 @@ class FeaturesEnricher(TransformerMixin):
                     display_handle=self.features_info_display_handle,
                 )
             except (ImportError, NameError):
-                pass
+                print(self._internal_features_info)
         if self.data_sources_display_handle is not None:
             try:
                 _ = get_ipython()  # type: ignore
@@ -1235,24 +1259,11 @@ class FeaturesEnricher(TransformerMixin):
                 display_html_dataframe(
                     self.relevant_data_sources,
                     self._relevant_data_sources_wo_links,
-                    self.bundle.get("relevant_data_sources_header"),
+                    self.bundle.get("relevant_features_header"),
                     display_handle=self.data_sources_display_handle,
                 )
             except (ImportError, NameError):
-                pass
-        if self.autofe_features_display_handle is not None:
-            try:
-                _ = get_ipython()  # type: ignore
-                autofe_descriptions_df = self.get_autofe_features_description()
-                if autofe_descriptions_df is not None:
-                    display_html_dataframe(
-                        df=autofe_descriptions_df,
-                        internal_df=autofe_descriptions_df,
-                        header=self.bundle.get("autofe_descriptions_header"),
-                        display_handle=self.autofe_features_display_handle,
-                    )
-            except (ImportError, NameError):
-                pass
+                print(self._relevant_data_sources_wo_links)
         if self.report_button_handle is not None:
             try:
                 _ = get_ipython()  # type: ignore
@@ -1436,11 +1447,7 @@ class FeaturesEnricher(TransformerMixin):
         client_features = [
             c
             for c in X_sampled.columns.to_list()
-            if (
-                not self.select_features
-                or c in self.feature_names_
-                or (self.fit_columns_renaming is not None and self.fit_columns_renaming.get(c) in self.feature_names_)
-            )
+            if (not self.select_features or c in self.feature_names_)
             and c
             not in (
                 excluding_search_keys
@@ -1657,10 +1664,7 @@ class FeaturesEnricher(TransformerMixin):
         generated_features = []
         if date_column is not None:
             converter = DateTimeSearchKeyConverter(date_column, self.date_format, self.logger, self.bundle)
-            # Leave original date column values
-            df_with_date_features = converter.convert(df, keep_time=True)
-            df_with_date_features[date_column] = df[date_column]
-            df = df_with_date_features
+            df = converter.convert(df, keep_time=True)
             generated_features = converter.generated_features
 
         email_columns = SearchKey.find_all_keys(search_keys, SearchKey.EMAIL)
@@ -1669,10 +1673,9 @@ class FeaturesEnricher(TransformerMixin):
             df = generator.generate(df)
             generated_features.extend(generator.generated_features)
 
-        # normalizer = Normalizer(self.bundle, self.logger)
-        # df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
-        # columns_renaming = normalizer.columns_renaming
-        columns_renaming = {c: c for c in df.columns}
+        normalizer = Normalizer(self.bundle, self.logger)
+        df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
+        columns_renaming = normalizer.columns_renaming
 
         df, _ = clean_full_duplicates(df, logger=self.logger, bundle=self.bundle)
 
@@ -1988,19 +1991,9 @@ class FeaturesEnricher(TransformerMixin):
         file_metadata = self._search_task.get_file_metadata(str(uuid.uuid4()))
         search_keys = file_metadata.search_types()
         if SearchKey.IPV6_ADDRESS in search_keys:
-            # search_keys.remove(SearchKey.IPV6_ADDRESS)
-            search_keys.pop(SearchKey.IPV6_ADDRESS, None)
+            search_keys.remove(SearchKey.IPV6_ADDRESS)
 
-        keys = (
-            "{"
-            + ", ".join(
-                [
-                    f'"{key.name}": {{"name": "{name}", "value": "{key_example(key)}"}}'
-                    for key, name in search_keys.items()
-                ]
-            )
-            + "}"
-        )
+        keys = "{" + ", ".join([f'"{key.name}": "{key_example(key)}"' for key in search_keys]) + "}"
         features_for_transform = self._search_task.get_features_for_transform()
         if features_for_transform:
             original_features_for_transform = [
@@ -2112,7 +2105,7 @@ class FeaturesEnricher(TransformerMixin):
             date_column = SearchKey.find_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
             if date_column is not None:
                 converter = DateTimeSearchKeyConverter(date_column, self.date_format, self.logger, bundle=self.bundle)
-                df = converter.convert(df, keep_time=True)
+                df = converter.convert(df)
                 self.logger.info(f"Date column after convertion: {df[date_column]}")
                 generated_features.extend(converter.generated_features)
             else:
@@ -2207,12 +2200,11 @@ class FeaturesEnricher(TransformerMixin):
 
             if add_fit_system_record_id:
                 df = self.__add_fit_system_record_id(df, search_keys, SYSTEM_RECORD_ID)
+                if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
+                    df = df.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL)
                 df = df.rename(columns={SYSTEM_RECORD_ID: SORT_ID})
                 features_not_to_pass.append(SORT_ID)
 
-            if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
-                df = df.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL)
-
             # search keys might be changed after explode
             columns_for_system_record_id = sorted(list(search_keys.keys()) + features_for_transform)
             df[SYSTEM_RECORD_ID] = pd.util.hash_pandas_object(df[columns_for_system_record_id], index=False).astype(
@@ -2231,7 +2223,7 @@ class FeaturesEnricher(TransformerMixin):
 
             combined_search_keys = combine_search_keys(search_keys.keys())
 
-            df_without_features = df.drop(columns=features_not_to_pass, errors="ignore")
+            df_without_features = df.drop(columns=features_not_to_pass)
 
             df_without_features, full_duplicates_warning = clean_full_duplicates(
                 df_without_features, self.logger, bundle=self.bundle
@@ -2803,12 +2795,7 @@ class FeaturesEnricher(TransformerMixin):
         autofe_description = self.get_autofe_features_description()
         if autofe_description is not None:
             self.logger.info(f"AutoFE descriptions: {autofe_description}")
-            self.autofe_features_display_handle = display_html_dataframe(
-                df=autofe_description,
-                internal_df=autofe_description,
-                header=self.bundle.get("autofe_descriptions_header"),
-                display_id="autofe_descriptions",
-            )
+            display_html_dataframe(autofe_description, autofe_description, "*Description of AutoFE feature names")
 
         if self._has_paid_features(exclude_features_sources):
             if calculate_metrics is not None and calculate_metrics:
@@ -3520,9 +3507,7 @@ class FeaturesEnricher(TransformerMixin):
 
         return result_train, result_eval_sets
 
-    def __prepare_feature_importances(
-            self, trace_id: str, x_columns: List[str], updated_shaps: Optional[Dict[str, float]] = None, silent=False
-    ):
+    def __prepare_feature_importances(self, trace_id: str, x_columns: List[str], silent=False):
         if self._search_task is None:
             raise NotFittedError(self.bundle.get("transform_unfitted_enricher"))
         features_meta = self._search_task.get_all_features_metadata_v2()
@@ -3539,10 +3524,6 @@ class FeaturesEnricher(TransformerMixin):
         features_info_without_links = []
         internal_features_info = []
 
-        if updated_shaps is not None:
-            for fm in features_meta:
-                fm.shap_value = updated_shaps.get(fm.name, 0.0)
-
         features_meta.sort(key=lambda m: (-m.shap_value, m.name))
         for feature_meta in features_meta:
             if feature_meta.name in original_names_dict.keys():
@@ -3595,22 +3576,7 @@ class FeaturesEnricher(TransformerMixin):
             autofe_meta = self._search_task.get_autofe_metadata()
             if autofe_meta is None:
                 return None
-            if len(self._internal_features_info) != 0:
-
-                def to_feature_meta(row):
-                    fm = FeaturesMetadataV2(
-                        name=row[bundle.get("features_info_name")],
-                        type="",
-                        source="",
-                        hit_rate=row[bundle.get("features_info_hitrate")],
-                        shap_value=row[bundle.get("features_info_shap")],
-                        data_source=row[bundle.get("features_info_source")],
-                    )
-                    return fm
-
-                features_meta = self._internal_features_info.apply(to_feature_meta, axis=1).to_list()
-            else:
-                features_meta = self._search_task.get_all_features_metadata_v2()
+            features_meta = self._search_task.get_all_features_metadata_v2()
 
             def get_feature_by_name(name: str):
                 for m in features_meta:
@@ -3639,32 +3605,27 @@ class FeaturesEnricher(TransformerMixin):
                     self.logger.warning(f"Feature meta for display index {m.display_index} not found")
                     continue
                 description["shap"] = feature_meta.shap_value
-                description[self.bundle.get("autofe_descriptions_sources")] = feature_meta.data_source.replace(
-                    "AutoFE: features from ", ""
-                ).replace("AutoFE: feature from ", "")
-                description[self.bundle.get("autofe_descriptions_feature_name")] = feature_meta.name
+                description["Sources"] = feature_meta.data_source.replace("AutoFE: features from ", "").replace(
+                    "AutoFE: feature from ", ""
+                )
+                description["Feature name"] = feature_meta.name
 
                 feature_idx = 1
                 for bc in m.base_columns:
-                    description[self.bundle.get("autofe_descriptions_feature").format(feature_idx)] = bc.hashed_name
+                    description[f"Feature {feature_idx}"] = bc.hashed_name
                     feature_idx += 1
 
-                description[self.bundle.get("autofe_descriptions_function")] = ",".join(
-                    sorted(autofe_feature.get_all_operand_names())
-                )
+                description["Function"] = ",".join(sorted(autofe_feature.get_all_operand_names()))
 
                 descriptions.append(description)
 
             if len(descriptions) == 0:
                 return None
 
-            descriptions_df = (
-                pd.DataFrame(descriptions)
-                .fillna("")
-                .sort_values(by="shap", ascending=False)
-                .drop(columns="shap")
-                .reset_index(drop=True)
-            )
+            descriptions_df = pd.DataFrame(descriptions)
+            descriptions_df.fillna("", inplace=True)
+            descriptions_df.sort_values(by="shap", ascending=False, inplace=True)
+            descriptions_df.drop(columns="shap", inplace=True)
             return descriptions_df
 
         except Exception:

upgini/http.py

@@ -882,7 +882,7 @@ class _RestClient:
         if content_type:
             headers[_RestClient.CONTENT_TYPE_HEADER_NAME] = content_type
         if trace_id:
-            headers[_RestClient.TRACE_ID_HEADER_NAME] = str(trace_id)
+            headers[_RestClient.TRACE_ID_HEADER_NAME] = trace_id
         for header_key, header_value in additional_headers.items():
             headers[header_key] = header_value
         return headers

upgini/metadata.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from enum import Enum
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Set, Union
 
 from pydantic import BaseModel
 
@@ -228,13 +228,13 @@ class FileMetadata(BaseModel):
                 return c
         return None
 
-    def search_types(self) -> Dict[SearchKey, str]:
-        search_keys = dict()
+    def search_types(self) -> Set[SearchKey]:
+        search_keys = set()
         for keys_group in self.searchKeys:
             for key in keys_group:
                 column = self.column_by_name(key)
                 if column:
-                    search_keys[SearchKey.from_meaning_type(column.meaningType)] = column.name
+                    search_keys.add(SearchKey.from_meaning_type(column.meaningType))
         return search_keys
 
 

upgini/resource_bundle/strings.properties

@@ -251,14 +251,6 @@ relevant_data_sources_header=Relevant data sources
 relevant_data_sources_all_shap=All features SHAP
 relevant_data_sources_number=Number of relevant features
 
-# Autofe descriptions
-autofe_descriptions_header=*Description of AutoFE feature names
-autofe_descriptions_sources=Sources
-autofe_descriptions_feature_name=Feature name
-autofe_descriptions_feature=Feature {}
-autofe_descriptions_function=Function
-
-
 # Quality metrics table
 quality_metrics_header=Accuracy after enrichment
 quality_metrics_train_segment=Train

upgini/utils/email_utils.py

@@ -39,7 +39,7 @@ class EmailDomainGenerator:
         for email_col in self.email_columns:
             domain_feature = email_col + self.DOMAIN_SUFFIX
             if domain_feature not in df.columns:
-                df[domain_feature] = df[email_col].apply(self._email_to_domain).astype("string")
+                df[domain_feature] = df[email_col].apply(self._email_to_domain)
                 self.generated_features.append(domain_feature)
         return df
 

{upgini-1.2.30.dist-info → upgini-1.2.30a8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.30
+Version: 1.2.30a8
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.30.dist-info → upgini-1.2.30a8.dist-info}/RECORD

@@ -1,12 +1,12 @@
-upgini/__about__.py,sha256=WGj1CvVJizDkAvd9BtLpwcsI-hzacJoXGbC8sVpoHYk,23
+upgini/__about__.py,sha256=EAN8NQ-4wbTNJ2NY82nkDVQrc6de5dh2FXWEmQ2YlyI,25
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=iPFiMJtk4HF1ytw9wCQr8H9RfoOKj_TIo8XYZKWgcMc,31331
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=lNfu5Z40NmkkGJScKAwe_0VBtL8liePifuAlKE_flfA,192053
-upgini/http.py,sha256=plZGTGoi1h2edd8Cnjt4eYB8t4NbBGnZz7DtPTByiNc,42885
+upgini/features_enricher.py,sha256=yiMy36rV0j5VCO6mwmUEbDrQTEpQSJDFEDlBLpgO5DU,190878
+upgini/http.py,sha256=21asexflvavydzCOONJDGQBtQanCElrbnqLXakJ9Cu8,42880
 upgini/lazy_import.py,sha256=74gQ8JuA48BGRLxAo7lNHNKY2D2emMxrUxKGdxVGhuY,1012
-upgini/metadata.py,sha256=lUa2xYhBhnCeTqNt6lWc9iP_YuikYGIsDSn8Vwyjv1I,11235
+upgini/metadata.py,sha256=osmzdNESeh7yP3BZday6N9Q3eaIHfzhhRM1d6NSgcf0,11223
 upgini/metrics.py,sha256=hr7UwLphbZ_FEglLuO2lzr_pFgxOJ4c3WBeg7H-fNqY,35521
 upgini/search_task.py,sha256=qxUxAD-bed-FpZYmTB_4orW7YJsW_O6a1TcgnZIRFr4,17307
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
@@ -30,7 +30,7 @@ upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 upgini/normalizer/normalize_utils.py,sha256=Ft2MwSgVoBilXAORAOYAuwPD79GOLfwn4qQE3IUFzzg,7218
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=bKw_rjZZTomLJhQBqiM7_P2EoRq45_Ng2gP4WE6MRBE,26921
+upgini/resource_bundle/strings.properties,sha256=fOAeLTsnx8xvJK-7RPFXprATG0n56jeCdse8sQTuVX8,26674
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=7GpjYqjOp58vYcJLiX__1R5wjUlyQbxvHJ2klFnup_M,6389
@@ -46,7 +46,7 @@ upgini/utils/cv_utils.py,sha256=w6FQb9nO8BWDx88EF83NpjPLarK4eR4ia0Wg0kLBJC4,3525
 upgini/utils/datetime_utils.py,sha256=F61i2vZCB6eUy4WwodDyPi50XKPbhOHsxDrU6tGa6CM,13133
 upgini/utils/deduplicate_utils.py,sha256=SMZx9IKIhWI5HqXepfKiQb3uDJrogQZtG6jcWuMo5Z4,8855
 upgini/utils/display_utils.py,sha256=DsBjJ8jEYAh8BPgfAbzq5imoGFV6IACP20PQ78BQCX0,11964
-upgini/utils/email_utils.py,sha256=GbnhHJn1nhUBytmK6PophYqaoq4t7Lp6i0-O0Gd3RV8,5265
+upgini/utils/email_utils.py,sha256=sCe7G-04-Wu9tRnL15l4Q6RVetCamJZyLjxLkWbxS-A,5248
 upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0--cOFTwJ0,1074
 upgini/utils/feature_info.py,sha256=Tp_2g5-rCjY4NpzKhzxwNxuqH5FFL8vG94OU5kH6wzk,6702
 upgini/utils/features_validator.py,sha256=lEfmk4DoxZ4ooOE1HC0ZXtUb_lFKRFHIrnFULZ4_rL8,3746
@@ -59,7 +59,7 @@ upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,4
 upgini/utils/target_utils.py,sha256=PU77nIhTz7IHbC4rpTpxrVxib6cdpRL9F1dhkjIffLY,10225
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.30.dist-info/METADATA,sha256=93iMDL28nXF2DJaDP-oUS2CeCFHgnjk2zmAhge2LAHg,48578
-upgini-1.2.30.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.30.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.30.dist-info/RECORD,,
+upgini-1.2.30a8.dist-info/METADATA,sha256=rrYHyEwsXHfbMsfaCvXgeVZ_WgXjyf3wrt_Pq7mPJBE,48580
+upgini-1.2.30a8.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+upgini-1.2.30a8.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.30a8.dist-info/RECORD,,

{upgini-1.2.30.dist-info → upgini-1.2.30a8.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: hatchling 1.24.2
 Root-Is-Purelib: true
 Tag: py3-none-any