upgini 1.2.29a6__tar.gz → 1.2.30__tar.gz

{upgini-1.2.29a6 → upgini-1.2.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.29a6
+Version: 1.2.30
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.30/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.30"

{upgini-1.2.29a6 → upgini-1.2.30}/src/upgini/features_enricher.py

@@ -350,6 +350,7 @@ class FeaturesEnricher(TransformerMixin):
         self.add_date_if_missing = add_date_if_missing
         self.features_info_display_handle = None
         self.data_sources_display_handle = None
+        self.autofe_features_display_handle = None
         self.report_button_handle = None
 
     def _get_api_key(self):
@@ -1049,7 +1050,7 @@ class FeaturesEnricher(TransformerMixin):
                         enriched_shaps = enriched_cv_result.shap_values
 
                         if enriched_shaps is not None:
-                            self._update_shap_values(enriched_shaps)
+                            self._update_shap_values(trace_id, validated_X.columns.to_list(), enriched_shaps)
 
                         if enriched_metric is None:
                             self.logger.warning(
@@ -1209,37 +1210,11 @@ class FeaturesEnricher(TransformerMixin):
             finally:
                 self.logger.info(f"Calculating metrics elapsed time: {time.time() - start_time}")
 
-    def _update_shap_values(self, new_shaps: Dict[str, float]):
+    def _update_shap_values(self, trace_id: str, x_columns: List[str], new_shaps: Dict[str, float]):
         new_shaps = {
             feature: _round_shap_value(shap) for feature, shap in new_shaps.items() if feature in self.feature_names_
         }
-        features_importances = list(new_shaps.items())
-        features_importances.sort(key=lambda m: (-m[1], m[0]))
-        self.feature_names_, self.feature_importances_ = zip(*features_importances)
-        self.feature_names_ = list(self.feature_names_)
-        self.feature_importances_ = list(self.feature_importances_)
-
-        feature_name_header = self.bundle.get("features_info_name")
-        shap_value_header = self.bundle.get("features_info_shap")
-
-        def update_shap(row):
-            return new_shaps.get(row[feature_name_header], row[shap_value_header])
-
-        self.features_info[shap_value_header] = self.features_info.apply(update_shap, axis=1)
-        self._internal_features_info[shap_value_header] = self._internal_features_info.apply(update_shap, axis=1)
-        self._features_info_without_links[shap_value_header] = self._features_info_without_links.apply(
-            update_shap, axis=1
-        )
-        self.logger.info(f"Recalculated SHAP values:\n{self._features_info_without_links}")
-
-        self.features_info.sort_values(by=shap_value_header, ascending=False, inplace=True)
-        self._internal_features_info.sort_values(by=shap_value_header, ascending=False, inplace=True)
-        self._features_info_without_links.sort_values(by=shap_value_header, ascending=False, inplace=True)
-
-        self.relevant_data_sources = self._group_relevant_data_sources(self.features_info, self.bundle)
-        self._relevant_data_sources_wo_links = self._group_relevant_data_sources(
-            self._features_info_without_links, self.bundle
-        )
+        self.__prepare_feature_importances(trace_id, x_columns, new_shaps, silent=True)
 
         if self.features_info_display_handle is not None:
             try:
@@ -1252,7 +1227,7 @@ class FeaturesEnricher(TransformerMixin):
                     display_handle=self.features_info_display_handle,
                 )
             except (ImportError, NameError):
-                print(self._internal_features_info)
+                pass
         if self.data_sources_display_handle is not None:
             try:
                 _ = get_ipython()  # type: ignore
@@ -1260,11 +1235,24 @@ class FeaturesEnricher(TransformerMixin):
                 display_html_dataframe(
                     self.relevant_data_sources,
                     self._relevant_data_sources_wo_links,
-                    self.bundle.get("relevant_features_header"),
+                    self.bundle.get("relevant_data_sources_header"),
                     display_handle=self.data_sources_display_handle,
                 )
             except (ImportError, NameError):
-                print(self._relevant_data_sources_wo_links)
+                pass
+        if self.autofe_features_display_handle is not None:
+            try:
+                _ = get_ipython()  # type: ignore
+                autofe_descriptions_df = self.get_autofe_features_description()
+                if autofe_descriptions_df is not None:
+                    display_html_dataframe(
+                        df=autofe_descriptions_df,
+                        internal_df=autofe_descriptions_df,
+                        header=self.bundle.get("autofe_descriptions_header"),
+                        display_handle=self.autofe_features_display_handle,
+                    )
+            except (ImportError, NameError):
+                pass
         if self.report_button_handle is not None:
             try:
                 _ = get_ipython()  # type: ignore
@@ -1448,7 +1436,11 @@ class FeaturesEnricher(TransformerMixin):
         client_features = [
             c
             for c in X_sampled.columns.to_list()
-            if (not self.select_features or c in self.feature_names_)
+            if (
+                not self.select_features
+                or c in self.feature_names_
+                or (self.fit_columns_renaming is not None and self.fit_columns_renaming.get(c) in self.feature_names_)
+            )
             and c
             not in (
                 excluding_search_keys
@@ -1665,7 +1657,10 @@ class FeaturesEnricher(TransformerMixin):
         generated_features = []
         if date_column is not None:
             converter = DateTimeSearchKeyConverter(date_column, self.date_format, self.logger, self.bundle)
-            df = converter.convert(df, keep_time=True)
+            # Leave original date column values
+            df_with_date_features = converter.convert(df, keep_time=True)
+            df_with_date_features[date_column] = df[date_column]
+            df = df_with_date_features
             generated_features = converter.generated_features
 
         email_columns = SearchKey.find_all_keys(search_keys, SearchKey.EMAIL)
@@ -1674,9 +1669,10 @@ class FeaturesEnricher(TransformerMixin):
             df = generator.generate(df)
             generated_features.extend(generator.generated_features)
 
-        normalizer = Normalizer(self.bundle, self.logger)
-        df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
-        columns_renaming = normalizer.columns_renaming
+        # normalizer = Normalizer(self.bundle, self.logger)
+        # df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
+        # columns_renaming = normalizer.columns_renaming
+        columns_renaming = {c: c for c in df.columns}
 
         df, _ = clean_full_duplicates(df, logger=self.logger, bundle=self.bundle)
 
@@ -1992,9 +1988,19 @@ class FeaturesEnricher(TransformerMixin):
         file_metadata = self._search_task.get_file_metadata(str(uuid.uuid4()))
         search_keys = file_metadata.search_types()
         if SearchKey.IPV6_ADDRESS in search_keys:
-            search_keys.remove(SearchKey.IPV6_ADDRESS)
+            # search_keys.remove(SearchKey.IPV6_ADDRESS)
+            search_keys.pop(SearchKey.IPV6_ADDRESS, None)
 
-        keys = "{" + ", ".join([f'"{key.name}": "{key_example(key)}"' for key in search_keys]) + "}"
+        keys = (
+            "{"
+            + ", ".join(
+                [
+                    f'"{key.name}": {{"name": "{name}", "value": "{key_example(key)}"}}'
+                    for key, name in search_keys.items()
+                ]
+            )
+            + "}"
+        )
         features_for_transform = self._search_task.get_features_for_transform()
         if features_for_transform:
             original_features_for_transform = [
@@ -2106,7 +2112,7 @@ class FeaturesEnricher(TransformerMixin):
             date_column = SearchKey.find_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
             if date_column is not None:
                 converter = DateTimeSearchKeyConverter(date_column, self.date_format, self.logger, bundle=self.bundle)
-                df = converter.convert(df)
+                df = converter.convert(df, keep_time=True)
                 self.logger.info(f"Date column after convertion: {df[date_column]}")
                 generated_features.extend(converter.generated_features)
             else:
@@ -2201,11 +2207,12 @@ class FeaturesEnricher(TransformerMixin):
 
             if add_fit_system_record_id:
                 df = self.__add_fit_system_record_id(df, search_keys, SYSTEM_RECORD_ID)
-                if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
-                    df = df.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL)
                 df = df.rename(columns={SYSTEM_RECORD_ID: SORT_ID})
                 features_not_to_pass.append(SORT_ID)
 
+            if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
+                df = df.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL)
+
             # search keys might be changed after explode
             columns_for_system_record_id = sorted(list(search_keys.keys()) + features_for_transform)
             df[SYSTEM_RECORD_ID] = pd.util.hash_pandas_object(df[columns_for_system_record_id], index=False).astype(
@@ -2224,7 +2231,7 @@ class FeaturesEnricher(TransformerMixin):
 
             combined_search_keys = combine_search_keys(search_keys.keys())
 
-            df_without_features = df.drop(columns=features_not_to_pass)
+            df_without_features = df.drop(columns=features_not_to_pass, errors="ignore")
 
             df_without_features, full_duplicates_warning = clean_full_duplicates(
                 df_without_features, self.logger, bundle=self.bundle
@@ -2339,7 +2346,9 @@ class FeaturesEnricher(TransformerMixin):
                 if c not in self.dropped_client_feature_names_
             ]
             filtered_columns = self.__filtered_enriched_features(importance_threshold, max_features)
-            selecting_columns.extend(c for c in filtered_columns if c in result.columns and c not in validated_X.columns)
+            selecting_columns.extend(
+                c for c in filtered_columns if c in result.columns and c not in validated_X.columns
+            )
             if add_fit_system_record_id:
                 selecting_columns.append(SORT_ID)
 
@@ -2794,7 +2803,12 @@ class FeaturesEnricher(TransformerMixin):
         autofe_description = self.get_autofe_features_description()
         if autofe_description is not None:
             self.logger.info(f"AutoFE descriptions: {autofe_description}")
-            display_html_dataframe(autofe_description, autofe_description, "*Description of AutoFE feature names")
+            self.autofe_features_display_handle = display_html_dataframe(
+                df=autofe_description,
+                internal_df=autofe_description,
+                header=self.bundle.get("autofe_descriptions_header"),
+                display_id="autofe_descriptions",
+            )
 
         if self._has_paid_features(exclude_features_sources):
             if calculate_metrics is not None and calculate_metrics:
@@ -3506,7 +3520,9 @@ class FeaturesEnricher(TransformerMixin):
 
         return result_train, result_eval_sets
 
-    def __prepare_feature_importances(self, trace_id: str, x_columns: List[str], silent=False):
+    def __prepare_feature_importances(
+            self, trace_id: str, x_columns: List[str], updated_shaps: Optional[Dict[str, float]] = None, silent=False
+    ):
         if self._search_task is None:
             raise NotFittedError(self.bundle.get("transform_unfitted_enricher"))
         features_meta = self._search_task.get_all_features_metadata_v2()
@@ -3523,6 +3539,10 @@ class FeaturesEnricher(TransformerMixin):
         features_info_without_links = []
         internal_features_info = []
 
+        if updated_shaps is not None:
+            for fm in features_meta:
+                fm.shap_value = updated_shaps.get(fm.name, 0.0)
+
         features_meta.sort(key=lambda m: (-m.shap_value, m.name))
         for feature_meta in features_meta:
             if feature_meta.name in original_names_dict.keys():
@@ -3544,7 +3564,6 @@ class FeaturesEnricher(TransformerMixin):
             ):
                 continue
 
-
             self.feature_names_.append(feature_meta.name)
             self.feature_importances_.append(_round_shap_value(feature_meta.shap_value))
 
@@ -3576,7 +3595,22 @@ class FeaturesEnricher(TransformerMixin):
             autofe_meta = self._search_task.get_autofe_metadata()
             if autofe_meta is None:
                 return None
-            features_meta = self._search_task.get_all_features_metadata_v2()
+            if len(self._internal_features_info) != 0:
+
+                def to_feature_meta(row):
+                    fm = FeaturesMetadataV2(
+                        name=row[bundle.get("features_info_name")],
+                        type="",
+                        source="",
+                        hit_rate=row[bundle.get("features_info_hitrate")],
+                        shap_value=row[bundle.get("features_info_shap")],
+                        data_source=row[bundle.get("features_info_source")],
+                    )
+                    return fm
+
+                features_meta = self._internal_features_info.apply(to_feature_meta, axis=1).to_list()
+            else:
+                features_meta = self._search_task.get_all_features_metadata_v2()
 
             def get_feature_by_name(name: str):
                 for m in features_meta:
@@ -3605,27 +3639,32 @@ class FeaturesEnricher(TransformerMixin):
                     self.logger.warning(f"Feature meta for display index {m.display_index} not found")
                     continue
                 description["shap"] = feature_meta.shap_value
-                description["Sources"] = feature_meta.data_source.replace("AutoFE: features from ", "").replace(
-                    "AutoFE: feature from ", ""
-                )
-                description["Feature name"] = feature_meta.name
+                description[self.bundle.get("autofe_descriptions_sources")] = feature_meta.data_source.replace(
+                    "AutoFE: features from ", ""
+                ).replace("AutoFE: feature from ", "")
+                description[self.bundle.get("autofe_descriptions_feature_name")] = feature_meta.name
 
                 feature_idx = 1
                 for bc in m.base_columns:
-                    description[f"Feature {feature_idx}"] = bc.hashed_name
+                    description[self.bundle.get("autofe_descriptions_feature").format(feature_idx)] = bc.hashed_name
                     feature_idx += 1
 
-                description["Function"] = ",".join(sorted(autofe_feature.get_all_operand_names()))
+                description[self.bundle.get("autofe_descriptions_function")] = ",".join(
+                    sorted(autofe_feature.get_all_operand_names())
+                )
 
                 descriptions.append(description)
 
             if len(descriptions) == 0:
                 return None
 
-            descriptions_df = pd.DataFrame(descriptions)
-            descriptions_df.fillna("", inplace=True)
-            descriptions_df.sort_values(by="shap", ascending=False, inplace=True)
-            descriptions_df.drop(columns="shap", inplace=True)
+            descriptions_df = (
+                pd.DataFrame(descriptions)
+                .fillna("")
+                .sort_values(by="shap", ascending=False)
+                .drop(columns="shap")
+                .reset_index(drop=True)
+            )
             return descriptions_df
 
         except Exception:

{upgini-1.2.29a6 → upgini-1.2.30}/src/upgini/http.py

@@ -882,7 +882,7 @@ class _RestClient:
         if content_type:
             headers[_RestClient.CONTENT_TYPE_HEADER_NAME] = content_type
         if trace_id:
-            headers[_RestClient.TRACE_ID_HEADER_NAME] = trace_id
+            headers[_RestClient.TRACE_ID_HEADER_NAME] = str(trace_id)
         for header_key, header_value in additional_headers.items():
             headers[header_key] = header_value
         return headers

{upgini-1.2.29a6 → upgini-1.2.30}/src/upgini/metadata.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from enum import Enum
-from typing import Any, Dict, List, Optional, Set, Union
+from typing import Any, Dict, List, Optional, Union
 
 from pydantic import BaseModel
 
@@ -228,13 +228,13 @@ class FileMetadata(BaseModel):
                 return c
         return None
 
-    def search_types(self) -> Set[SearchKey]:
-        search_keys = set()
+    def search_types(self) -> Dict[SearchKey, str]:
+        search_keys = dict()
         for keys_group in self.searchKeys:
             for key in keys_group:
                 column = self.column_by_name(key)
                 if column:
-                    search_keys.add(SearchKey.from_meaning_type(column.meaningType))
+                    search_keys[SearchKey.from_meaning_type(column.meaningType)] = column.name
         return search_keys
 
 

{upgini-1.2.29a6 → upgini-1.2.30}/src/upgini/resource_bundle/strings.properties

@@ -251,6 +251,14 @@ relevant_data_sources_header=Relevant data sources
 relevant_data_sources_all_shap=All features SHAP
 relevant_data_sources_number=Number of relevant features
 
+# Autofe descriptions
+autofe_descriptions_header=*Description of AutoFE feature names
+autofe_descriptions_sources=Sources
+autofe_descriptions_feature_name=Feature name
+autofe_descriptions_feature=Feature {}
+autofe_descriptions_function=Function
+
+
 # Quality metrics table
 quality_metrics_header=Accuracy after enrichment
 quality_metrics_train_segment=Train

{upgini-1.2.29a6 → upgini-1.2.30}/src/upgini/utils/datetime_utils.py

@@ -114,10 +114,12 @@ class DateTimeSearchKeyConverter:
             period_suffix = f"_{period}" if column != "day_in_quarter" else ""
             sin_feature = f"datetime_{column}_sin{period_suffix}"
             cos_feature = f"datetime_{column}_cos{period_suffix}"
-            df[sin_feature] = np.sin(2 * np.pi * df[column] / period)
-            df[cos_feature] = np.cos(2 * np.pi * df[column] / period)
-            self.generated_features.append(sin_feature)
-            self.generated_features.append(cos_feature)
+            if sin_feature not in df.columns:
+                df[sin_feature] = np.sin(2 * np.pi * df[column] / period)
+                self.generated_features.append(sin_feature)
+            if cos_feature not in df.columns:
+                df[cos_feature] = np.cos(2 * np.pi * df[column] / period)
+                self.generated_features.append(cos_feature)
 
         df["quarter"] = df[self.date_column].dt.quarter
 

{upgini-1.2.29a6 → upgini-1.2.30}/src/upgini/utils/email_utils.py

@@ -38,8 +38,9 @@ class EmailDomainGenerator:
     def generate(self, df: pd.DataFrame) -> pd.DataFrame:
         for email_col in self.email_columns:
             domain_feature = email_col + self.DOMAIN_SUFFIX
-            df[domain_feature] = df[email_col].apply(self._email_to_domain)
-            self.generated_features.append(domain_feature)
+            if domain_feature not in df.columns:
+                df[domain_feature] = df[email_col].apply(self._email_to_domain).astype("string")
+                self.generated_features.append(domain_feature)
         return df
 
     @staticmethod

{upgini-1.2.29a6 → upgini-1.2.30}/src/upgini/utils/features_validator.py

@@ -2,6 +2,7 @@ import logging
 from logging import Logger
 from typing import Dict, List, Optional, Tuple
 
+import numpy as np
 import pandas as pd
 from pandas.api.types import is_integer_dtype, is_object_dtype, is_string_dtype
 
@@ -83,10 +84,21 @@ class FeaturesValidator:
         return [
             i
             for i in df
-            if (is_object_dtype(df[i]) or is_string_dtype(df[i]) or is_integer_dtype(df[i]))
+            if (is_object_dtype(df[i]) or is_string_dtype(df[i]) or FeaturesValidator.__is_integer(df[i]))
             and (df[i].nunique(dropna=False) / row_count >= 0.85)
         ]
 
+    @staticmethod
+    def __is_integer(series: pd.Series) -> bool:
+        return (
+            is_integer_dtype(series)
+            or series.dropna()
+            .apply(
+                lambda f: (float.is_integer(f) and abs(f) < np.iinfo(np.int64).max) if isinstance(f, float) else False
+            )
+            .all()
+        )
+
     @staticmethod
     def find_constant_features(df: pd.DataFrame) -> List[str]:
         return [i for i in df if df[i].nunique() <= 1]

upgini-1.2.29a6/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.29a6"