upgini 1.2.117a1__tar.gz → 1.2.119__tar.gz

{upgini-1.2.117a1 → upgini-1.2.119}/.gitignore

@@ -111,6 +111,7 @@ env10/
 .env10/
 .env310/
 env11/
+env12/
 venv/
 ENV/
 env.bak/

{upgini-1.2.117a1 → upgini-1.2.119}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.117a1
+Version: 1.2.119
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.119/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.119"

{upgini-1.2.117a1 → upgini-1.2.119}/src/upgini/features_enricher.py

@@ -854,7 +854,7 @@ class FeaturesEnricher(TransformerMixin):
                     raise e
             finally:
                 self.logger.info(f"Transform elapsed time: {time.time() - start_time}")
-            
+
             return result
 
     def calculate_metrics(
@@ -1423,8 +1423,15 @@ class FeaturesEnricher(TransformerMixin):
         # Find latest eval set or earliest if all eval sets are before train set
         date_column = self._get_date_column(search_keys)
 
-        # Get minimum date from main dataset X
-        main_min_date = X[date_column].dropna().min()
+        x_date = X[date_column].dropna()
+        if not is_numeric_dtype(x_date):
+            x_date = pd.to_datetime(x_date).dt.floor("D").astype(np.int64) / 10**6
+        main_min_date = x_date.min()
+
+        for eval_x, _ in eval_set:
+            eval_x_date = eval_x[date_column].dropna()
+            if not is_numeric_dtype(eval_x_date):
+                eval_x[date_column] = pd.to_datetime(eval_x_date).dt.floor("D").astype(np.int64) / 10**6
 
         # Find minimum date for each eval_set and compare with main dataset
         eval_dates = []
@@ -1433,8 +1440,11 @@ class FeaturesEnricher(TransformerMixin):
                 if len(eval_x) < 1000:
                     self.logger.warning(f"Eval_set {i} has less than 1000 rows. It will be ignored for stability check")
                     continue
-                eval_min_date = eval_x[date_column].dropna().min()
-                eval_max_date = eval_x[date_column].dropna().max()
+                eval_x_date = eval_x[date_column].dropna()
+                if not is_numeric_dtype(eval_x_date):
+                    eval_x_date = pd.to_datetime(eval_x_date).dt.floor("D").astype(np.int64) / 10**6
+                eval_min_date = eval_x_date.min()
+                eval_max_date = eval_x_date.max()
                 eval_dates.append((i, eval_min_date, eval_max_date))
 
         if not eval_dates:
@@ -1460,6 +1470,10 @@ class FeaturesEnricher(TransformerMixin):
         checking_eval_set_df = checking_eval_set_df.copy()
 
         checking_eval_set_df[date_column] = eval_set_dates[selected_eval_set_idx]
+        if not is_numeric_dtype(checking_eval_set_df[date_column]):
+            checking_eval_set_df[date_column] = (
+                pd.to_datetime(checking_eval_set_df[date_column]).dt.floor("D").astype(np.int64) / 10**6
+            )
 
         psi_values_sparse = calculate_sparsity_psi(
             checking_eval_set_df, cat_features, date_column, self.logger, model_task_type
@@ -1727,7 +1741,8 @@ class FeaturesEnricher(TransformerMixin):
             c
             for c in (validated_X.columns.to_list() + generated_features)
             if (not self.fit_select_features or c in set(self.feature_names_).union(self.id_columns or []))
-            and c not in (
+            and c
+            not in (
                 excluding_search_keys
                 + list(self.fit_dropped_features)
                 + [DateTimeSearchKeyConverter.DATETIME_COL, SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID]
@@ -2201,7 +2216,8 @@ class FeaturesEnricher(TransformerMixin):
             progress_callback=progress_callback,
             add_fit_system_record_id=True,
         )
-        if enriched_df is None:
+        if enriched_df is None or len(enriched_df) == 0 or len(enriched_df.columns) == 0:
+            self.logger.warning(f"Empty enriched dataframe returned: {enriched_df}, returning None")
             return None
 
         x_columns = [
@@ -2505,7 +2521,7 @@ if response.status_code == 200:
             if len(self.feature_names_) == 0:
                 msg = self.bundle.get("no_important_features_for_transform")
                 self.__log_warning(msg, show_support_link=True)
-                return X, {c: c for c in X.columns}, [], dict()
+                return None, {}, [], self.search_keys
 
             self.__validate_search_keys(self.search_keys, self.search_id)
 
@@ -2513,7 +2529,7 @@ if response.status_code == 200:
                 msg = self.bundle.get("transform_with_paid_features")
                 self.logger.warning(msg)
                 self.__display_support_link(msg)
-                return None, {c: c for c in X.columns}, [], {}
+                return None, {}, [], self.search_keys
 
             features_meta = self._search_task.get_all_features_metadata_v2()
             online_api_features = [fm.name for fm in features_meta if fm.from_online_api and fm.shap_value > 0]
@@ -2536,7 +2552,7 @@ if response.status_code == 200:
                         self.logger.warning(msg)
                         print(msg)
                         show_request_quote_button()
-                        return None, {c: c for c in X.columns}, [], {}
+                        return None, {}, [], {}
                     else:
                         msg = self.bundle.get("transform_usage_info").format(
                             transform_usage.limit, transform_usage.transformed_rows
@@ -2606,14 +2622,33 @@ if response.status_code == 200:
 
             # If there are no external features, we don't call backend on transform
             external_features = [fm for fm in features_meta if fm.shap_value > 0 and fm.source != "etalon"]
-            if not external_features:
+            if len(external_features) == 0:
                 self.logger.warning(
                     "No external features found, returning original dataframe"
                     f" with generated important features: {self.feature_names_}"
                 )
-                filtered_columns = [c for c in self.feature_names_ if c in df.columns]
-                self.logger.warning(f"Filtered columns by existance in dataframe: {filtered_columns}")
-                return df[filtered_columns], columns_renaming, generated_features, search_keys
+                df = df.rename(columns=columns_renaming)
+                generated_features = [columns_renaming.get(c, c) for c in generated_features]
+                search_keys = {columns_renaming.get(c, c): t for c, t in search_keys.items()}
+                selecting_columns = self._selecting_input_and_generated_columns(
+                    validated_Xy, generated_features, keep_input, trace_id
+                )
+                self.logger.warning(f"Filtered columns by existance in dataframe: {selecting_columns}")
+                if add_fit_system_record_id:
+                    df = self._add_fit_system_record_id(
+                        df,
+                        search_keys,
+                        SYSTEM_RECORD_ID,
+                        TARGET,
+                        columns_renaming,
+                        self.id_columns,
+                        self.cv,
+                        self.model_task_type,
+                        self.logger,
+                        self.bundle,
+                    )
+                    selecting_columns.append(SYSTEM_RECORD_ID)
+                return df[selecting_columns], columns_renaming, generated_features, search_keys
 
             # Don't pass all features in backend on transform
             runtime_parameters = self._get_copy_of_runtime_parameters()
@@ -2831,29 +2866,12 @@ if response.status_code == 200:
                 how="left",
             )
 
-            fit_input_columns = [c.originalName for c in self._search_task.get_file_metadata(trace_id).columns]
-            new_columns_on_transform = [c for c in validated_Xy.columns if c not in fit_input_columns]
-
-            selected_generated_features = [
-                c for c in generated_features if not self.fit_select_features or c in self.feature_names_
-            ]
-            if keep_input is True:
-                selected_input_columns = [
-                    c
-                    for c in validated_Xy.columns
-                    if not self.fit_select_features
-                    or c in self.feature_names_
-                    or c in new_columns_on_transform
-                    or c in self.search_keys
-                    or c in (self.id_columns or [])
-                    or c in [EVAL_SET_INDEX, TARGET]  # transform for metrics calculation
-                ]
-            else:
-                selected_input_columns = []
-
-            selecting_columns = selected_input_columns + selected_generated_features
+            selecting_columns = self._selecting_input_and_generated_columns(
+                validated_Xy, generated_features, keep_input, trace_id
+            )
             selecting_columns.extend(
-                c for c in result.columns
+                c
+                for c in result.columns
                 if c in self.feature_names_ and c not in selecting_columns and c not in validated_Xy.columns
             )
             if add_fit_system_record_id:
@@ -2881,6 +2899,35 @@ if response.status_code == 200:
 
             return result, columns_renaming, generated_features, search_keys
 
+    def _selecting_input_and_generated_columns(
+        self,
+        validated_Xy: pd.DataFrame,
+        generated_features: list[str],
+        keep_input: bool,
+        trace_id: str,
+    ):
+        fit_input_columns = [c.originalName for c in self._search_task.get_file_metadata(trace_id).columns]
+        new_columns_on_transform = [c for c in validated_Xy.columns if c not in fit_input_columns]
+
+        selected_generated_features = [
+            c for c in generated_features if not self.fit_select_features or c in self.feature_names_
+        ]
+        if keep_input is True:
+            selected_input_columns = [
+                c
+                for c in validated_Xy.columns
+                if not self.fit_select_features
+                or c in self.feature_names_
+                or c in new_columns_on_transform
+                or c in self.search_keys
+                or c in (self.id_columns or [])
+                or c in [EVAL_SET_INDEX, TARGET]  # transform for metrics calculation
+            ]
+        else:
+            selected_input_columns = []
+
+        return selected_input_columns + selected_generated_features
+
     def __validate_search_keys(self, search_keys: dict[str, SearchKey], search_id: str | None = None):
         if (search_keys is None or len(search_keys) == 0) and self.country_code is None:
             if search_id:
@@ -3708,6 +3755,23 @@ if response.status_code == 200:
             else:
                 raise ValidationError(self.bundle.get("eval_x_and_x_diff_shape"))
 
+        if any(validated_eval_X.dtypes != X.dtypes):
+            x_types = X.dtypes
+            eval_types = validated_eval_X.dtypes
+            # Find columns with different types
+            diff_cols = [
+                (col, x_types[col], eval_types[col]) for col in x_types.index if x_types[col] != eval_types[col]
+            ]
+            diff_col_names = [col for col, _, _ in diff_cols]
+            # print columns with different types
+            print("Columns with different types:")
+            for col, x_type, eval_type in diff_cols:
+                print("-" * 50)
+                print(f"Column: {col}")
+                print(f"X type:        {x_type}")
+                print(f"Eval_set type: {eval_type}")
+            raise ValidationError(self.bundle.get("eval_x_and_x_diff_dtypes").format(diff_col_names))
+
         if _num_samples(validated_eval_X) != _num_samples(eval_y):
             raise ValidationError(
                 self.bundle.get("x_and_y_diff_size_eval_set").format(
@@ -3782,9 +3846,7 @@ if response.status_code == 200:
         return Xy[X.columns].copy(), Xy[TARGET].copy()
 
     @staticmethod
-    def _sort_by_system_record_id(
-        X: pd.DataFrame, y: pd.Series, cv: CVType | None
-    ) -> tuple[pd.DataFrame, pd.Series]:
+    def _sort_by_system_record_id(X: pd.DataFrame, y: pd.Series, cv: CVType | None) -> tuple[pd.DataFrame, pd.Series]:
         if cv not in [CVType.time_series, CVType.blocked_time_series]:
             record_id_column = ENTITY_SYSTEM_RECORD_ID if ENTITY_SYSTEM_RECORD_ID in X else SYSTEM_RECORD_ID
             Xy = X.copy()
@@ -4420,7 +4482,8 @@ if response.status_code == 200:
 
         if len(features_info) > 0:
             self.features_info = pd.DataFrame(features_info)
-            if self.features_info[self.bundle.get("features_info_psi")].isna().all():
+            # If all psi values are 0 or null, drop psi column
+            if self.features_info[self.bundle.get("features_info_psi")].fillna(0.0).eq(0.0).all():
                 self.features_info.drop(columns=[self.bundle.get("features_info_psi")], inplace=True)
             self._features_info_without_links = pd.DataFrame(features_info_without_links)
             self._internal_features_info = pd.DataFrame(internal_features_info)

{upgini-1.2.117a1 → upgini-1.2.119}/src/upgini/metrics.py

@@ -847,7 +847,7 @@ class CatBoostWrapper(EstimatorWrapper):
 
             feature_importance = {}
             for i, col in enumerate(x.columns):
-                feature_importance[col] = np.mean(np.abs(shap_values[:, i]))
+                feature_importance[col] = float(np.mean(np.abs(shap_values[:, i])))
 
             return feature_importance
 
@@ -922,6 +922,7 @@ class LightGBMWrapper(EstimatorWrapper):
                     encoded = cat_encoder.transform(x_copy[self.cat_features]).astype(int)
                 else:
                     encoded = cat_encoder.transform(x_copy[self.cat_features]).astype("category")
+                x_copy = x_copy.drop(columns=self.cat_features, errors="ignore")
                 x_copy[self.cat_features] = encoded
 
             shap_matrix = estimator.predict(
@@ -943,7 +944,7 @@ class LightGBMWrapper(EstimatorWrapper):
 
             feature_importance = {}
             for i, col in enumerate(x.columns):
-                feature_importance[col] = np.mean(np.abs(shap_matrix[:, i]))
+                feature_importance[col] = float(np.mean(np.abs(shap_matrix[:, i])))
 
             return feature_importance
 

{upgini-1.2.117a1 → upgini-1.2.119}/src/upgini/resource_bundle/strings.properties

@@ -123,6 +123,7 @@ unsupported_type_eval_set=Unsupported type of eval_set: {}. It should be list of
 eval_set_invalid_tuple_size=eval_set contains a tuple of size {}. It should contain only pairs of X and y or X only
 unsupported_x_type_eval_set=Unsupported type of X in eval_set: {}. Use pandas.DataFrame, pandas.Series or numpy.ndarray or list.
 eval_x_and_x_diff_shape=The column set in eval_set are differ from the column set in X
+eval_x_and_x_diff_dtypes=The column types in eval_set are different from the column types in X: {}
 unsupported_y_type_eval_set=Unsupported type of y in eval_set: {}. Use pandas.Series, numpy.ndarray or list
 y_is_constant_eval_set=y in eval_set is a constant. Relevant feature search requires a non-constant y
 x_and_y_diff_size_eval_set=X and y in eval_set contain different number of rows: {}, {}

{upgini-1.2.117a1 → upgini-1.2.119}/src/upgini/utils/display_utils.py

@@ -269,19 +269,22 @@ def make_html_report(
                 if search_keys is not None
                 else ""
             }
-            {"<h3>All relevant features. Accuracy after enrichment</h3>" + make_table(metrics_df)
-             if metrics_df is not None
-             else ""
+            {
+                "<h3>All relevant features. Accuracy after enrichment</h3>" + make_table(metrics_df)
+                if metrics_df is not None
+                else ""
             }
-            {"<h3>Relevant data sources</h3>" + make_table(relevant_datasources_df)
-             if len(relevant_datasources_df) > 0
-             else ""
+            {
+                "<h3>Relevant data sources</h3>" + make_table(relevant_datasources_df)
+                if len(relevant_datasources_df) > 0
+                else ""
             }
             <h3>All relevant features. Listing ({len(relevant_features_df)} items)</h3>
             {make_table(relevant_features_df, wrap_long_string=25)}
-            {"<h3>Description of AutoFE feature names</h3>" + make_table(autofe_descriptions_df, wrap_long_string=25)
-             if autofe_descriptions_df is not None
-             else ""
+            {
+                "<h3>Description of AutoFE feature names</h3>" + make_table(autofe_descriptions_df, wrap_long_string=25)
+                if autofe_descriptions_df is not None and len(autofe_descriptions_df) > 0
+                else ""
             }
             <p>To buy found data sources, please contact: <a href='mailto:sales@upgini.com'>sales@upgini.com</a></p>
             <p>Best regards, </br><b>Upgini Team</b></p>

{upgini-1.2.117a1 → upgini-1.2.119}/src/upgini/utils/psi.py

@@ -82,9 +82,6 @@ def calculate_features_psi(
 ) -> dict[str, float]:
     empty_res = {col: 0.0 for col in df.columns if col not in [TARGET, date_column]}
 
-    if not is_numeric_dtype(df[date_column]):
-        df[date_column] = pd.to_datetime(df[date_column]).dt.floor("D").astype(np.int64) / 10**6
-
     # Filter out rows with missing dates
     df = df[df[date_column].notna()].copy()
 

upgini-1.2.117a1/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.117a1"