upgini 1.2.114a1__tar.gz → 1.2.114a3__tar.gz

{upgini-1.2.114a1 → upgini-1.2.114a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.114a1
+Version: 1.2.114a3
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.114a3/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.114a3"

{upgini-1.2.114a1 → upgini-1.2.114a3}/src/upgini/dataset.py

@@ -50,7 +50,7 @@ except Exception:
 
 class Dataset:
     MIN_ROWS_COUNT = 100
-    MAX_ROWS = 100_000
+    MAX_ROWS = 200_000
     IMBALANCE_THESHOLD = 0.6
     MIN_TARGET_CLASS_ROWS = 100
     MAX_MULTICLASS_CLASS_COUNT = 100
@@ -184,7 +184,19 @@ class Dataset:
     def __validate_target(self):
         # self.logger.info("Validating target")
         target_column = self.etalon_def_checked.get(FileColumnMeaningType.TARGET.value, "")
-        target = self.data[target_column]
+
+        oot_indices = []
+        if EVAL_SET_INDEX in self.data.columns:
+            for eval_set_index in self.data[EVAL_SET_INDEX].unique():
+                eval_set = self.data[self.data[EVAL_SET_INDEX] == eval_set_index]
+                if eval_set[target_column].isna().all():
+                    oot_indices.append(eval_set_index)
+
+        df_to_check = self.data.copy()
+        if oot_indices:
+            df_to_check = df_to_check[~df_to_check[EVAL_SET_INDEX].isin(oot_indices)]
+
+        target = df_to_check[target_column]
 
         if self.task_type == ModelTaskType.BINARY:
             if not is_integer_dtype(target):
@@ -201,7 +213,7 @@ class Dataset:
         elif self.task_type == ModelTaskType.MULTICLASS:
             if not is_integer_dtype(target):
                 try:
-                    target = self.data[target_column].astype("category").cat.codes
+                    target = target.astype("category").cat.codes
                 except Exception:
                     self.logger.exception("Failed to cast target to category codes for multiclass task type")
                     raise ValidationError(self.bundle.get("dataset_invalid_multiclass_target").format(target.dtype))
@@ -335,10 +347,30 @@ class Dataset:
         all_valid_message = self.bundle.get("validation_all_valid_message")
         invalid_message = self.bundle.get("validation_invalid_message")
 
+        oot_indices = []
+        if EVAL_SET_INDEX in self.data.columns:
+            for eval_set_index in self.data[EVAL_SET_INDEX].unique():
+                eval_set = self.data[self.data[EVAL_SET_INDEX] == eval_set_index]
+                if eval_set[target].isna().all():
+                    oot_indices.append(eval_set_index)
+
         for col in columns_to_validate:
-            self.data[f"{col}_is_valid"] = ~self.data[col].isnull()
             if validate_target and target is not None and col == target:
-                self.data.loc[self.data[target] == np.inf, f"{col}_is_valid"] = False
+                if oot_indices:
+                    mask_not_oot = ~self.data[EVAL_SET_INDEX].isin(oot_indices)
+                    invalid_target_mask = (
+                        self.data[col].isnull() | (self.data[col] == np.inf) | (self.data[col] == -np.inf)
+                    )
+                    # Initialize as valid and mark invalid only for non-OOT rows with NaN or +/-inf
+                    self.data[f"{col}_is_valid"] = True
+                    self.data.loc[mask_not_oot & invalid_target_mask, f"{col}_is_valid"] = False
+                else:
+                    # No OOT: mark invalid where target is NaN or +/-inf
+                    self.data[f"{col}_is_valid"] = ~(
+                        self.data[col].isnull() | (self.data[col] == np.inf) | (self.data[col] == -np.inf)
+                    )
+            else:
+                self.data[f"{col}_is_valid"] = ~self.data[col].isnull()
 
             if col in mandatory_columns:
                 self.data["valid_mandatory"] = self.data["valid_mandatory"] & self.data[f"{col}_is_valid"]

{upgini-1.2.114a1 → upgini-1.2.114a3}/src/upgini/features_enricher.py

@@ -398,16 +398,15 @@ class FeaturesEnricher(TransformerMixin):
 
     api_key = property(_get_api_key, _set_api_key)
 
-    @staticmethod
-    def _check_eval_set(eval_set, X, bundle: ResourceBundle):
+    def _check_eval_set(self, eval_set, X):
         checked_eval_set = []
         if eval_set is None:
             return checked_eval_set
         if isinstance(eval_set, tuple):
             eval_set = [eval_set]
         if not isinstance(eval_set, list):
-            raise ValidationError(bundle.get("unsupported_type_eval_set").format(type(eval_set)))
-        for eval_pair in eval_set or []:
+            raise ValidationError(self.bundle.get("unsupported_type_eval_set").format(type(eval_set)))
+        for i, eval_pair in enumerate(eval_set or [], 1):
             # Handle OOT
             if isinstance(eval_pair, pd.DataFrame):
                 empty_target = pd.Series([np.nan] * len(eval_pair), index=eval_pair.index)
@@ -417,12 +416,17 @@ class FeaturesEnricher(TransformerMixin):
                 eval_pair = (eval_pair[0], empty_target)
 
             if not isinstance(eval_pair, tuple) or len(eval_pair) != 2:
-                raise ValidationError(bundle.get("eval_set_invalid_tuple_size").format(len(eval_pair)))
+                raise ValidationError(self.bundle.get("eval_set_invalid_tuple_size").format(len(eval_pair)))
             if eval_pair[1] is None:
                 empty_target = pd.Series([np.nan] * len(eval_pair[0]), index=eval_pair[0].index)
                 eval_pair = (eval_pair[0], empty_target)
-            if not is_frames_equal(X, eval_pair[0], bundle):
+
+            if not is_frames_equal(X, eval_pair[0], self.bundle):
                 checked_eval_set.append(eval_pair)
+            else:
+                msg = f"Eval set {i} is equal to train set and will be ignored"
+                self.logger.warning(msg)
+                print(msg)
         return checked_eval_set
 
     def fit(
@@ -517,7 +521,7 @@ class FeaturesEnricher(TransformerMixin):
             try:
                 self.X = X
                 self.y = y
-                self.eval_set = self._check_eval_set(eval_set, X, self.bundle)
+                self.eval_set = self._check_eval_set(eval_set, X)
                 self.dump_input(trace_id, X, y, self.eval_set)
                 self.__set_select_features(select_features)
                 self.__inner_fit(
@@ -678,7 +682,7 @@ class FeaturesEnricher(TransformerMixin):
             try:
                 self.X = X
                 self.y = y
-                self.eval_set = self._check_eval_set(eval_set, X, self.bundle)
+                self.eval_set = self._check_eval_set(eval_set, X)
                 self.__set_select_features(select_features)
                 self.dump_input(trace_id, X, y, self.eval_set)
 
@@ -953,7 +957,7 @@ class FeaturesEnricher(TransformerMixin):
             effective_X = X if X is not None else self.X
             effective_y = y if y is not None else self.y
             effective_eval_set = eval_set if eval_set is not None else self.eval_set
-            effective_eval_set = self._check_eval_set(effective_eval_set, effective_X, self.bundle)
+            effective_eval_set = self._check_eval_set(effective_eval_set, effective_X)
 
             if (
                 self._search_task is None
@@ -1471,14 +1475,17 @@ class FeaturesEnricher(TransformerMixin):
         date_column = self._get_date_column(search_keys)
 
         # Get minimum date from main dataset X
-        main_min_date = X[date_column].min()
+        main_min_date = X[date_column].dropna().min()
 
         # Find minimum date for each eval_set and compare with main dataset
         eval_dates = []
         for i, (eval_x, _) in enumerate(eval_set):
             if date_column in eval_x.columns:
-                eval_min_date = eval_x[date_column].min()
-                eval_max_date = eval_x[date_column].max()
+                if len(eval_x) < 1000:
+                    self.logger.warning(f"Eval_set {i} has less than 1000 rows. It will be ignored for stability check")
+                    continue
+                eval_min_date = eval_x[date_column].dropna().min()
+                eval_max_date = eval_x[date_column].dropna().max()
                 eval_dates.append((i, eval_min_date, eval_max_date))
 
         if not eval_dates:
@@ -1509,6 +1516,8 @@ class FeaturesEnricher(TransformerMixin):
             checking_eval_set_df, cat_features, date_column, self.logger, model_task_type
         )
 
+        self.logger.info(f"PSI values by sparsity: {psi_values_sparse}")
+
         unstable_by_sparsity = [feature for feature, psi in psi_values_sparse.items() if psi > stability_threshold]
         if unstable_by_sparsity:
             self.logger.info(f"Unstable by sparsity features: {sorted(unstable_by_sparsity)}")
@@ -1517,6 +1526,8 @@ class FeaturesEnricher(TransformerMixin):
             checking_eval_set_df, cat_features, date_column, self.logger, model_task_type
         )
 
+        self.logger.info(f"PSI values by value: {psi_values}")
+
         unstable_by_value = [feature for feature, psi in psi_values.items() if psi > stability_threshold]
         if unstable_by_value:
             self.logger.info(f"Unstable by value features: {sorted(unstable_by_value)}")
@@ -1679,7 +1690,7 @@ class FeaturesEnricher(TransformerMixin):
         if X is None:
             return True, self.X, self.y, self.eval_set
 
-        checked_eval_set = self._check_eval_set(eval_set, X, self.bundle)
+        checked_eval_set = self._check_eval_set(eval_set, X)
 
         if (
             X is self.X
@@ -1783,7 +1794,7 @@ class FeaturesEnricher(TransformerMixin):
     ):
         is_input_same_as_fit, X, y, eval_set = self._is_input_same_as_fit(X, y, eval_set)
         is_demo_dataset = hash_input(X, y, eval_set) in DEMO_DATASET_HASHES
-        checked_eval_set = self._check_eval_set(eval_set, X, self.bundle)
+        checked_eval_set = self._check_eval_set(eval_set, X)
         validated_X, validated_y, validated_eval_set = self._validate_train_eval(X, y, checked_eval_set, silent=True)
 
         sampled_data = self._get_enriched_for_metrics(
@@ -3246,6 +3257,15 @@ if response.status_code == 200:
             else:
                 self.__log_warning(full_duplicates_warning)
 
+        # Check if OOT eval set still more than 1000 rows
+        if EVAL_SET_INDEX in df.columns:
+            for eval_set_index in df[EVAL_SET_INDEX].unique():
+                if eval_set_index == 0:
+                    continue
+                eval_set_df = df[df[EVAL_SET_INDEX] == eval_set_index]
+                if np.all(pd.isna(eval_set_df[TARGET])) and len(eval_set_df) < 1000:
+                    self.__log_warning(self.bundle.get("oot_eval_set_too_small_after_dedup").format(eval_set_index + 1))
+
         # Explode multiple search keys
         df = self.__add_fit_system_record_id(
             df, self.fit_search_keys, ENTITY_SYSTEM_RECORD_ID, TARGET, self.fit_columns_renaming
@@ -3823,7 +3843,7 @@ if response.status_code == 200:
                 if isinstance(eval_set, tuple):
                     eval_set = [eval_set]
                 for eval in eval_set:
-                    is_oot = eval[1].isna().all()
+                    is_oot = np.all(pd.isna(eval[1]))
                     if not is_oot:
                         if self.baseline_score_column not in eval[0].columns:
                             raise ValidationError(self.bundle.get("baseline_score_column_not_exists"))

{upgini-1.2.114a1 → upgini-1.2.114a3}/src/upgini/resource_bundle/strings.properties

@@ -165,6 +165,7 @@ dataset_too_many_rows_registered=X rows limit for transform is {}. Please sample
 dataset_empty_column_names=Some column names are empty. Add names please
 dataset_full_duplicates={:.5f}% of the rows are fully duplicated
 dataset_diff_target_duplicates={:.4f}% of rows ({}) in X and eval_set are duplicates with different y values. These rows will be deleted as incorrect\nSample of incorrect row indexes: {}
+dataset_diff_target_duplicates_oot={:.4f}% of rows ({}) in OOT eval_set are duplicates with train or another eval_set. These rows will be deleted from OOT\nSample of incorrect row indexes: {}
 dataset_train_diff_target_duplicates_fintech={:.4f}% of rows ({}) in X are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
 dataset_eval_diff_target_duplicates_fintech={:.4f}% of rows ({}) in eval{} X are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
 dataset_drop_old_dates=We don't have data before '2000-01-01' and removed all earlier records from the search dataset
@@ -185,6 +186,7 @@ dataset_invalid_column_type=Unsupported data type of column {}: {}
 dataset_invalid_filter=Unknown field in filter_features. Should be {'min_importance', 'max_psi', 'max_count', 'selected_features'}.
 dataset_too_big_file=Too big size of dataframe X for processing. Please reduce number of rows or columns
 dataset_transform_diff_fit=You try to enrich dataset that column names are different from the train dataset column names that you used on the fit stage. Please make the column names the same as in the train dataset and restart.
+oot_eval_set_too_small_after_dedup=OOT eval set {} has less than 1000 rows after deduplication. It will be ignored for stability check
 binary_small_dataset=The least populated class in Target contains less than 1000 rows.\nSmall numbers of observations may negatively affect the number of selected features and quality of your ML model.\nUpgini recommends you increase the number of observations in the least populated class.\n
 all_search_keys_invalid=All search keys are invalid
 all_emails_invalid=All values in column {} are invalid emails    # Metrics validation

{upgini-1.2.114a1 → upgini-1.2.114a3}/src/upgini/utils/deduplicate_utils.py

@@ -134,10 +134,12 @@ def remove_fintech_duplicates(
     logger.info(f"Train dataset shape after clean fintech duplicates: {train_df.shape}")
 
     # Process each eval_set part separately
+    oot_eval_dfs = []
     new_eval_dfs = []
     for i, eval_df in enumerate(eval_dfs, 1):
         # Skip OOT
         if eval_df[TARGET].isna().all():
+            oot_eval_dfs.append(eval_df)
             continue
         logger.info(f"Eval {i} dataset shape before clean fintech duplicates: {eval_df.shape}")
         cleaned_eval_df, eval_warning = process_df(eval_df, i)
@@ -148,8 +150,8 @@ def remove_fintech_duplicates(
 
     # Combine the processed train and eval parts back into one dataset
     logger.info(f"Dataset shape before clean fintech duplicates: {df.shape}")
-    if new_eval_dfs:
-        df = pd.concat([train_df] + new_eval_dfs)
+    if new_eval_dfs or oot_eval_dfs:
+        df = pd.concat([train_df] + new_eval_dfs + oot_eval_dfs, ignore_index=False)
     else:
         df = train_df
     logger.info(f"Dataset shape after clean fintech duplicates: {df.shape}")
@@ -196,30 +198,30 @@ def clean_full_duplicates(
 
         # Separate rows to exclude from deduplication:
         # for each eval_set_index != 0 check separately, all TARGET values are NaN
-        excluded_from_dedup = pd.DataFrame()
         df_for_dedup = df
+        oot_df = None
 
         if EVAL_SET_INDEX in df.columns:
-            excluded_parts = []
-            # Get all unique eval_set_index values, except 0
-            unique_eval_indices = df[df[EVAL_SET_INDEX] != 0][EVAL_SET_INDEX].unique()
-
-            for eval_idx in unique_eval_indices:
+            oot_eval_dfs = []
+            other_dfs = []
+            for eval_idx in df[EVAL_SET_INDEX].unique():
                 eval_subset = df[df[EVAL_SET_INDEX] == eval_idx]
                 # Check that all TARGET values for this specific eval_set_index are NaN
-                if len(eval_subset) > 0 and eval_subset[TARGET].isna().all():
-                    excluded_parts.append(eval_subset)
+                if eval_idx != 0 and eval_subset[TARGET].isna().all():
+                    oot_eval_dfs.append(eval_subset)
                     logger.info(
                         f"Excluded {len(eval_subset)} rows from deduplication "
                         f"(eval_set_index={eval_idx} and all TARGET values are NaN)"
                     )
+                else:
+                    other_dfs.append(eval_subset)
+
+            if oot_eval_dfs:
+                oot_df = pd.concat(oot_eval_dfs, ignore_index=False)
+                df_for_dedup = pd.concat(other_dfs, ignore_index=False)
+            else:
+                df_for_dedup = df
 
-            # Combine all excluded parts
-            if excluded_parts:
-                excluded_from_dedup = pd.concat(excluded_parts, ignore_index=False)
-                # Remove excluded rows from dataframe for deduplication
-                excluded_indices = excluded_from_dedup.index
-                df_for_dedup = df[~df.index.isin(excluded_indices)]
         marked_duplicates = df_for_dedup.duplicated(subset=unique_columns, keep=False)
         if marked_duplicates.sum() > 0:
             dups_indices = df_for_dedup[marked_duplicates].index.to_list()[:100]
@@ -231,8 +233,18 @@ def clean_full_duplicates(
             df_for_dedup = df_for_dedup.drop_duplicates(subset=unique_columns, keep=False)
             logger.info(f"Dataset shape after clean invalid target duplicates: {df_for_dedup.shape}")
         # Combine back excluded rows
-        if len(excluded_from_dedup) > 0:
-            df = pd.concat([df_for_dedup, excluded_from_dedup], ignore_index=False)
+        if oot_df is not None:
+            df = pd.concat([df_for_dedup, oot_df], ignore_index=False)
+            marked_duplicates = df.duplicated(subset=unique_columns, keep=False)
+            if marked_duplicates.sum() > 0:
+                dups_indices = df[marked_duplicates].index.to_list()[:100]
+                nrows_after_tgt_dedup = len(df.drop_duplicates(subset=unique_columns, keep=False))
+                num_dup_rows = len(df) - nrows_after_tgt_dedup
+                share_tgt_dedup = 100 * num_dup_rows / len(df)
+                msg = bundle.get("dataset_diff_target_duplicates_oot").format(
+                    share_tgt_dedup, num_dup_rows, dups_indices
+                )
+                df = df.drop_duplicates(subset=unique_columns, keep="first")
             logger.info(f"Final dataset shape after adding back excluded rows: {df.shape}")
         else:
             df = df_for_dedup

{upgini-1.2.114a1 → upgini-1.2.114a3}/src/upgini/utils/psi.py

@@ -77,7 +77,7 @@ def calculate_features_psi(
     psi_features_params: StabilityParams = DEFAULT_FEATURES_PARAMS,
     psi_target_params: StabilityParams = DEFAULT_TARGET_PARAMS,
 ) -> Dict[str, float]:
-    empty_res = pd.Series(index=df.columns, data=0)
+    empty_res = {col: 0.0 for col in df.columns if col not in [TARGET, date_column]}
 
     if not is_numeric_dtype(df[date_column]):
         df[date_column] = pd.to_datetime(df[date_column]).dt.floor("D").astype(np.int64) / 10**6
@@ -113,9 +113,9 @@ def calculate_features_psi(
             cat_top_pct=psi_target_params.cat_top_pct,
             agg_func=target_agg_func,
         )
-        if target_psi is None:
+        if target_psi is None or np.isnan(target_psi):
             logger.info("Cannot determine target PSI. Skip feature PSI check")
-            return pd.Series(index=df.columns, data=0)
+            return empty_res
 
         if target_psi > psi_target_params.threshold:
             logger.info(

{upgini-1.2.114a1 → upgini-1.2.114a3}/src/upgini/utils/sample_utils.py

@@ -5,7 +5,7 @@ from typing import Callable, List, Optional
 import numpy as np
 import pandas as pd
 
-from upgini.metadata import SYSTEM_RECORD_ID, CVType, ModelTaskType
+from upgini.metadata import EVAL_SET_INDEX, SYSTEM_RECORD_ID, TARGET, CVType, ModelTaskType
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.utils.target_utils import balance_undersample
 from upgini.utils.ts_utils import get_most_frequent_time_unit, trunc_datetime
@@ -117,6 +117,22 @@ def sample(
             **kwargs,
         )
 
+    # separate OOT
+    oot_dfs = []
+    other_dfs = []
+    if EVAL_SET_INDEX in df.columns:
+        for eval_set_index in df[EVAL_SET_INDEX].unique():
+            eval_df = df[df[EVAL_SET_INDEX] == eval_set_index]
+            if TARGET in eval_df.columns and eval_df[TARGET].isna().all():
+                oot_dfs.append(eval_df)
+            else:
+                other_dfs.append(eval_df)
+    if len(oot_dfs) > 0:
+        oot_df = pd.concat(oot_dfs, ignore_index=False)
+        df = pd.concat(other_dfs, ignore_index=False)
+    else:
+        oot_df = None
+
     num_samples = _num_samples(df)
     if num_samples > fit_sample_threshold:
         logger.info(
@@ -126,6 +142,18 @@ def sample(
         df = df.sample(n=fit_sample_rows, random_state=random_state)
         logger.info(f"Shape after threshold resampling: {df.shape}")
 
+    if oot_df is not None:
+        num_samples_oot = _num_samples(oot_df)
+        if num_samples_oot > fit_sample_threshold:
+            logger.info(
+                f"OOT has size {num_samples_oot} more than threshold {fit_sample_threshold} "
+                f"and will be downsampled to {fit_sample_rows}"
+            )
+            oot_df = oot_df.sample(n=fit_sample_rows, random_state=random_state)
+        df = pd.concat([df, oot_df], ignore_index=False)
+
+    logger.info(f"Dataset size after downsampling: {len(df)}")
+
     return df
 
 
@@ -175,7 +203,7 @@ def sample_time_series_train_eval(
             )
         if logger is not None:
             logger.info(f"Eval set size: {len(eval_df)}")
-        df = pd.concat([train_df, eval_df])
+        df = pd.concat([train_df, eval_df], ignore_index=False)
 
     elif len(train_df) > max_rows:
         df = sample_time_series_trunc(

{upgini-1.2.114a1 → upgini-1.2.114a3}/src/upgini/utils/target_utils.py

@@ -6,7 +6,7 @@ import pandas as pd
 from pandas.api.types import is_bool_dtype, is_datetime64_any_dtype, is_numeric_dtype
 
 from upgini.errors import ValidationError
-from upgini.metadata import SYSTEM_RECORD_ID, ModelTaskType
+from upgini.metadata import EVAL_SET_INDEX, SYSTEM_RECORD_ID, ModelTaskType
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle, bundle
 from upgini.sampler.random_under_sampler import RandomUnderSampler
 
@@ -132,6 +132,11 @@ def balance_undersample(
     if SYSTEM_RECORD_ID not in df.columns:
         raise Exception("System record id must be presented for undersampling")
 
+    # Rebalance and send to server only train data
+    # because eval set data will be sent separately in transform for metrics
+    if EVAL_SET_INDEX in df.columns:
+        df = df[df[EVAL_SET_INDEX] == 0]
+
     target = df[target_column].copy()
 
     vc = target.value_counts()

upgini-1.2.114a1/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.114a1"