upgini 1.2.66a3818.dev1__py3-none-any.whl → 1.2.68__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.66a3818.dev1"
+__version__ = "1.2.68"

upgini/autofe/feature.py

@@ -112,11 +112,7 @@ class Feature:
 
     def get_hash(self) -> str:
         return hashlib.sha256(
-            "_".join(
-                [self.op.get_hash_component()]
-                + [ch.op.get_hash_component() for ch in self.children if isinstance(ch, Feature)]
-                + [ch.get_display_name() for ch in self.children]
-            ).encode("utf-8")
+            "_".join([self.op.get_hash_component()] + [ch.get_display_name() for ch in self.children]).encode("utf-8")
         ).hexdigest()[:8]
 
     def set_alias(self, alias: str) -> "Feature":

upgini/features_enricher.py

@@ -308,7 +308,8 @@ class FeaturesEnricher(TransformerMixin):
                     self._search_task = search_task.poll_result(trace_id, quiet=True, check_fit=True)
                     file_metadata = self._search_task.get_file_metadata(trace_id)
                     x_columns = [c.originalName or c.name for c in file_metadata.columns]
-                    self.__prepare_feature_importances(trace_id, x_columns, silent=True)
+                    df = pd.DataFrame(columns=x_columns)
+                    self.__prepare_feature_importances(trace_id, df, silent=True)
                     # TODO validate search_keys with search_keys from file_metadata
                     print(self.bundle.get("search_by_task_id_finish"))
                     self.logger.debug(f"Successfully initialized with search_id: {search_id}")
@@ -1087,7 +1088,7 @@ class FeaturesEnricher(TransformerMixin):
                         enriched_shaps = enriched_cv_result.shap_values
 
                         if enriched_shaps is not None:
-                            self._update_shap_values(trace_id, validated_X.columns.to_list(), enriched_shaps)
+                            self._update_shap_values(trace_id, fitting_X, enriched_shaps)
 
                         if enriched_metric is None:
                             self.logger.warning(
@@ -1255,14 +1256,14 @@ class FeaturesEnricher(TransformerMixin):
             finally:
                 self.logger.info(f"Calculating metrics elapsed time: {time.time() - start_time}")
 
-    def _update_shap_values(self, trace_id: str, x_columns: List[str], new_shaps: Dict[str, float]):
+    def _update_shap_values(self, trace_id: str, df: pd.DataFrame, new_shaps: Dict[str, float]):
         renaming = self.fit_columns_renaming or {}
         new_shaps = {
             renaming.get(feature, feature): _round_shap_value(shap)
             for feature, shap in new_shaps.items()
             if feature in self.feature_names_ or renaming.get(feature, feature) in self.feature_names_
         }
-        self.__prepare_feature_importances(trace_id, x_columns, new_shaps)
+        self.__prepare_feature_importances(trace_id, df, new_shaps)
 
         if self.features_info_display_handle is not None:
             try:
@@ -3021,7 +3022,7 @@ if response.status_code == 200:
             msg = self.bundle.get("features_not_generated").format(unused_features_for_generation)
             self.__log_warning(msg)
 
-        self.__prepare_feature_importances(trace_id, validated_X.columns.to_list() + self.fit_generated_features)
+        self.__prepare_feature_importances(trace_id, df)
 
         self.__show_selected_features(self.fit_search_keys)
 
@@ -3796,7 +3797,7 @@ if response.status_code == 200:
         return result_train, result_eval_sets
 
     def __prepare_feature_importances(
-        self, trace_id: str, x_columns: List[str], updated_shaps: Optional[Dict[str, float]] = None, silent=False
+        self, trace_id: str, df: pd.DataFrame, updated_shaps: Optional[Dict[str, float]] = None, silent=False
     ):
         if self._search_task is None:
             raise NotFittedError(self.bundle.get("transform_unfitted_enricher"))
@@ -3807,6 +3808,8 @@ if response.status_code == 200:
         original_names_dict = {c.name: c.originalName for c in self._search_task.get_file_metadata(trace_id).columns}
         features_df = self._search_task.get_all_initial_raw_features(trace_id, metrics_calculation=True)
 
+        df = df.rename(columns=original_names_dict)
+
         self.feature_names_ = []
         self.dropped_client_feature_names_ = []
         self.feature_importances_ = []
@@ -3825,7 +3828,7 @@ if response.status_code == 200:
             if feature_meta.name in original_names_dict.keys():
                 feature_meta.name = original_names_dict[feature_meta.name]
 
-            is_client_feature = feature_meta.name in x_columns
+            is_client_feature = feature_meta.name in df.columns
 
             # TODO make a decision about selected features based on special flag from mlb
             if original_shaps.get(feature_meta.name, 0.0) == 0.0:
@@ -3845,7 +3848,7 @@ if response.status_code == 200:
             self.feature_names_.append(feature_meta.name)
             self.feature_importances_.append(_round_shap_value(feature_meta.shap_value))
 
-            df_for_sample = features_df if feature_meta.name in features_df.columns else self.X
+            df_for_sample = features_df if feature_meta.name in features_df.columns else df
             feature_info = FeatureInfo.from_metadata(feature_meta, df_for_sample, is_client_feature)
             features_info.append(feature_info.to_row(self.bundle))
             features_info_without_links.append(feature_info.to_row_without_links(self.bundle))

upgini/utils/feature_info.py

@@ -88,8 +88,11 @@ class FeatureInfo:
 
 
 def _get_feature_sample(feature_meta: FeaturesMetadataV2, data: Optional[pd.DataFrame]) -> str:
-    if data is not None and feature_meta.name in data.columns:
-        feature_sample = np.random.choice(data[feature_meta.name].dropna().unique(), 3).tolist()
+    if data is not None and len(data) > 0 and feature_meta.name in data.columns:
+        if len(data) > 3:
+            feature_sample = np.random.choice(data[feature_meta.name].dropna().unique(), 3).tolist()
+        else:
+            feature_sample = data[feature_meta.name].dropna().unique().tolist()
         if len(feature_sample) > 0 and isinstance(feature_sample[0], float):
             feature_sample = [round(f, 4) for f in feature_sample]
         feature_sample = [str(f) for f in feature_sample]
@@ -123,7 +126,11 @@ def _get_provider(feature_meta: FeaturesMetadataV2, is_client_feature: bool) ->
 
 
 def _get_internal_provider(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
-    return "" if is_client_feature else (feature_meta.data_provider or "Upgini")
+    providers = _list_or_single(feature_meta.data_providers, feature_meta.data_provider)
+    if providers:
+        return ", ".join(providers)
+    else:
+        return "" if is_client_feature else (feature_meta.data_provider or "Upgini")
 
 
 def _get_source(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
@@ -137,13 +144,17 @@ def _get_source(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> st
 
 
 def _get_internal_source(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
-    return feature_meta.data_source or (
-        LLM_SOURCE
-        if not feature_meta.name.endswith("_country")
-        and not feature_meta.name.endswith("_postal_code")
-        and not is_client_feature
-        else ""
-    )
+    sources = _list_or_single(feature_meta.data_sources, feature_meta.data_source)
+    if sources:
+        return ", ".join(sources)
+    else:
+        return feature_meta.data_source or (
+            LLM_SOURCE
+            if not feature_meta.name.endswith("_country")
+            and not feature_meta.name.endswith("_postal_code")
+            and not is_client_feature
+            else ""
+            )
 
 
 def _list_or_single(lst: List[str], single: str):
@@ -161,7 +172,7 @@ def _to_anchor(link: str, value: str) -> str:
         return f"<a href='{link}' target='_blank' rel='noopener noreferrer'>{value}</a>"
 
 
-def _make_links(names: List[str], links: List[str]):
+def _make_links(names: List[str], links: List[str]) -> str:
     all_links = [_to_anchor(link, name) for name, link in itertools.zip_longest(names, links)]
     return ",".join(all_links)
 

upgini/utils/sort.py

@@ -39,6 +39,11 @@ def sort_columns(
     sorted_keys = sorted(search_keys.keys(), key=lambda x: str(search_keys.get(x)))
     sorted_keys = [k for k in sorted_keys if k in df.columns and k not in exclude_columns]
 
+    duplicate_names = df.columns[df.columns.duplicated()].unique()
+    if len(duplicate_names) > 0:
+        logger.warning(f"WARNING: Found columns with duplicate names: {list(duplicate_names)}")
+        df = df[list(set(df.columns))]
+
     other_columns = sorted(
         [
             c
@@ -49,7 +54,7 @@ def sort_columns(
     target = target_column if isinstance(target_column, pd.Series) else df[target_column]
     target = prepare_target(target, model_task_type)
     sort_dict = get_sort_columns_dict(
-        df[sorted_keys + other_columns], target, sorted_keys, omit_nan=True, sort_all_columns=sort_all_columns
+        df[sorted_keys + other_columns], target, sorted_keys, sort_all_columns=sort_all_columns
     )
     other_columns = [c for c in other_columns if c in sort_dict]
     columns_for_sort = sorted_keys + sorted(other_columns, key=lambda e: sort_dict[e], reverse=True)
@@ -60,7 +65,6 @@ def get_sort_columns_dict(
     df: pd.DataFrame,
     target: pd.Series,
     sorted_keys: List[str],
-    omit_nan: bool,
     n_jobs: Optional[int] = None,
     sort_all_columns: bool = False,
 ) -> Dict[str, Any]:
@@ -78,6 +82,13 @@ def get_sort_columns_dict(
         return {}
 
     df = df[columns_for_sort]
+    df_with_target = pd.concat([df, target], axis=1)
+    # Drop rows where target is NaN
+    df_with_target = df_with_target.loc[~target.isna()]
+    df = df_with_target.iloc[:, :-1]
+    target = df_with_target.iloc[:, -1]
+    df = df.fillna(df.mean())
+    omit_nan = False
     hashes = [hash_series(df[col]) for col in columns_for_sort]
     df = np.asarray(df, dtype=np.float32)
     correlations = get_sort_columns_correlations(df, target, omit_nan, n_jobs)

{upgini-1.2.66a3818.dev1.dist-info → upgini-1.2.68.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.66a3818.dev1
+Version: 1.2.68
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.66a3818.dev1.dist-info → upgini-1.2.68.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=NbAl7_TAPRLWAiByFYGbEOi4eRvu1Erxk-b19Z5nTRs,33
+upgini/__about__.py,sha256=36MiqZwik3NwGoXvug_voP86Q4bwFpauhq0m3rJ2Avc,23
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=OGjpeFHbj3lWiZTOHTpWEoMMDmFY1FlNC44FKktoZvU,34956
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=nXGBMC42VPAmqQKXbEqZJFIHiGj6F_G2AwhurA8LuQs,205351
+upgini/features_enricher.py,sha256=YXG5uUBN1Qo-3X5EUV4Y--Pyqbvg4Gta3WIoWQMTYkU,205359
 upgini/http.py,sha256=ud0Cp7h0jNeHuuZGpU_1dAAEiabGoJjGxc1X5oeBQr4,43496
 upgini/lazy_import.py,sha256=74gQ8JuA48BGRLxAo7lNHNKY2D2emMxrUxKGdxVGhuY,1012
 upgini/metadata.py,sha256=Jh6YTaS00m_nbaOY_owvlSyn9zgkErkqu8iTr9ZjKI8,12279
@@ -17,7 +17,7 @@ upgini/autofe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/autofe/all_operators.py,sha256=rdjF5eaE4bC6Q4eu_el5Z7ekYt8DjOFermz2bePPbUc,333
 upgini/autofe/binary.py,sha256=MnQuFiERpocjCPQUjOljlsq5FE-04GPfwtNjzvfNMyU,7671
 upgini/autofe/date.py,sha256=I07psJerrxOcHao91PdSCk9X6KWu61IBVyFRLjGNgK8,10730
-upgini/autofe/feature.py,sha256=y1x3wijhTVBmloayQAHiscqKU9Ll8kLcGm1PdvS357I,14910
+upgini/autofe/feature.py,sha256=xgu6bVIlUJ5PCUgoXQRNcGkcMOhj-_BdDRmkB_qRFS4,14766
 upgini/autofe/groupby.py,sha256=IYmQV9uoCdRcpkeWZj_kI3ObzoNCNx3ff3h8sTL01tk,3603
 upgini/autofe/operator.py,sha256=EOffJw6vKXpEh5yymqb1RFNJPxGxmnHdFRo9dB5SCFo,4969
 upgini/autofe/unary.py,sha256=yVgPvtfnPSOhrii0YgezddmgWPwyOBCR0JutaIkdTTc,4658
@@ -56,7 +56,7 @@ upgini/utils/deduplicate_utils.py,sha256=SMZx9IKIhWI5HqXepfKiQb3uDJrogQZtG6jcWuM
 upgini/utils/display_utils.py,sha256=DsBjJ8jEYAh8BPgfAbzq5imoGFV6IACP20PQ78BQCX0,11964
 upgini/utils/email_utils.py,sha256=pZ2vCfNxLIPUhxr0-OlABNXm12jjU44isBk8kGmqQzA,5277
 upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0--cOFTwJ0,1074
-upgini/utils/feature_info.py,sha256=0rOXSyCj-sw-8migWP0ge8qrOzGU50dQvH0JUJUrDfQ,6766
+upgini/utils/feature_info.py,sha256=m1tQcT3hTChPAiXzpk0WQcEqElj8KgeCifEJFa7-gss,7247
 upgini/utils/features_validator.py,sha256=lEfmk4DoxZ4ooOE1HC0ZXtUb_lFKRFHIrnFULZ4_rL8,3746
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
 upgini/utils/ip_utils.py,sha256=TSQ_qDsLlVnm09X1HacpabEf_HNqSWpxBF4Sdc2xs08,6580
@@ -65,12 +65,12 @@ upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,1
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
 upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,44511
-upgini/utils/sort.py,sha256=GfWfCIbfK7e7BvSPZZNJD-PEtiN19DnTCEQkeefHHxI,6491
+upgini/utils/sort.py,sha256=H79A17NMoHtLbqLCPFx_MBUloLZcDKjOba_H4gCE3t8,6965
 upgini/utils/target_utils.py,sha256=b1GzO8_gMcwXSZ2v98CY50MJJBzKbWHId_BJGybXfkM,16579
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.66a3818.dev1.dist-info/METADATA,sha256=RlvM_n0dDfEJ6-4PCEiyh7bXHCDZjjdTOOP7uGjQd-M,49123
-upgini-1.2.66a3818.dev1.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-upgini-1.2.66a3818.dev1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.66a3818.dev1.dist-info/RECORD,,
+upgini-1.2.68.dist-info/METADATA,sha256=XJ67N1OUXdu0fMTjlA9hFcg3HPnN4KXiiqXYvNWD3Dk,49113
+upgini-1.2.68.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.68.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.68.dist-info/RECORD,,

{upgini-1.2.66a3818.dev1.dist-info → upgini-1.2.68.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.24.2
+Generator: hatchling 1.25.0
 Root-Is-Purelib: true
 Tag: py3-none-any