upgini 1.2.146a2__tar.gz → 1.2.146a9__tar.gz

{upgini-1.2.146a2 → upgini-1.2.146a9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: upgini
-Version: 1.2.146a2
+Version: 1.2.146a9
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.146a9/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.146a9"

{upgini-1.2.146a2 → upgini-1.2.146a9}/src/upgini/dataset.py

@@ -58,7 +58,7 @@ class Dataset:
     MAX_ROWS = 3_000_000
     MIN_SUPPORTED_DATE_TS = 946684800000  # 2000-01-01
     MAX_FEATURES_COUNT = 3500
-    MAX_UPLOADING_FILE_SIZE = 268435456  # 256 Mb
+    MAX_UPLOADING_FILE_SIZE = 536_870_912  # 512 Mb
     MAX_STRING_FEATURE_LENGTH = 24573
     FORCE_SAMPLE_SIZE = 7_000
 
@@ -304,10 +304,11 @@ class Dataset:
         ):
             keys_to_validate.remove(ipv4_column)
 
-        mandatory_columns = [target]
+        mandatory_columns = {target} if target is not None else set()
         columns_to_validate = mandatory_columns.copy()
-        columns_to_validate.extend(keys_to_validate)
-        columns_to_validate = set([i for i in columns_to_validate if i is not None])
+        columns_to_validate.update(keys_to_validate)
+        if len(columns_to_validate) == 0:
+            return
 
         nrows = len(self.data)
         validation_stats = {}

{upgini-1.2.146a2 → upgini-1.2.146a9}/src/upgini/features_enricher.py

@@ -11,6 +11,7 @@ import uuid
 from collections import Counter
 from copy import deepcopy
 from dataclasses import dataclass
+from pathlib import Path
 from threading import Thread
 from typing import Any, Callable
 
@@ -277,6 +278,8 @@ class FeaturesEnricher(TransformerMixin):
         self.autodetected_search_keys: dict[str, SearchKey] | None = None
         self.imbalanced = False
         self.fit_select_features = True
+        self.true_one_hot_groups: dict[str, list[str]] | None = None
+        self.pseudo_one_hot_groups: dict[str, list[str]] | None = None
         self.__cached_sampled_datasets: dict[str, tuple[pd.DataFrame, pd.DataFrame, pd.Series, dict, dict, dict]] = (
             dict()
         )
@@ -679,9 +682,6 @@ class FeaturesEnricher(TransformerMixin):
                 self.__set_select_features(select_features)
                 self.dump_input(X, y, self.eval_set)
 
-                if _num_samples(drop_duplicates(X)) > Dataset.MAX_ROWS:
-                    raise ValidationError(self.bundle.get("dataset_too_many_rows_registered").format(Dataset.MAX_ROWS))
-
                 self.__inner_fit(
                     X,
                     y,
@@ -2049,6 +2049,9 @@ class FeaturesEnricher(TransformerMixin):
             generated_features.extend(generator.generated_features)
 
         normalizer = Normalizer(self.bundle, self.logger)
+        # TODO restore these properties from the server
+        normalizer.true_one_hot_groups = self.true_one_hot_groups
+        normalizer.pseudo_one_hot_groups = self.pseudo_one_hot_groups
         df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
         columns_renaming = normalizer.columns_renaming
 
@@ -2664,6 +2667,9 @@ if response.status_code == 200:
             generated_features.extend(generator.generated_features)
 
         normalizer = Normalizer(self.bundle, self.logger)
+        # TODO restore these properties from the server
+        normalizer.true_one_hot_groups = self.true_one_hot_groups
+        normalizer.pseudo_one_hot_groups = self.pseudo_one_hot_groups
         df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
         columns_renaming = normalizer.columns_renaming
 
@@ -2831,85 +2837,103 @@ if response.status_code == 200:
         del df
         gc.collect()
 
-        dataset = Dataset(
-            "sample_" + str(uuid.uuid4()),
-            df=df_without_features,
-            meaning_types=meaning_types,
-            search_keys=combined_search_keys,
-            unnest_search_keys=unnest_search_keys,
-            id_columns=self.__get_renamed_id_columns(columns_renaming),
-            date_column=self._get_date_column(search_keys),
-            date_format=self.date_format,
-            sample_config=self.sample_config,
-            rest_client=self.rest_client,
-            logger=self.logger,
-            bundle=self.bundle,
-            warning_callback=self.__log_warning,
-        )
-        dataset.columns_renaming = columns_renaming
+        def invoke_validation(df: pd.DataFrame):
+
+            dataset = Dataset(
+                "sample_" + str(uuid.uuid4()),
+                df=df,
+                meaning_types=meaning_types,
+                search_keys=combined_search_keys,
+                unnest_search_keys=unnest_search_keys,
+                id_columns=self.__get_renamed_id_columns(columns_renaming),
+                date_column=self._get_date_column(search_keys),
+                date_format=self.date_format,
+                sample_config=self.sample_config,
+                rest_client=self.rest_client,
+                logger=self.logger,
+                bundle=self.bundle,
+                warning_callback=self.__log_warning,
+            )
+            dataset.columns_renaming = columns_renaming
+
+            validation_task = self._search_task.validation(
+                self._get_trace_id(),
+                dataset,
+                start_time=start_time,
+                extract_features=True,
+                runtime_parameters=runtime_parameters,
+                exclude_features_sources=exclude_features_sources,
+                metrics_calculation=metrics_calculation,
+                silent_mode=silent_mode,
+                progress_bar=progress_bar,
+                progress_callback=progress_callback,
+            )
 
-        validation_task = self._search_task.validation(
-            self._get_trace_id(),
-            dataset,
-            start_time=start_time,
-            extract_features=True,
-            runtime_parameters=runtime_parameters,
-            exclude_features_sources=exclude_features_sources,
-            metrics_calculation=metrics_calculation,
-            silent_mode=silent_mode,
-            progress_bar=progress_bar,
-            progress_callback=progress_callback,
-        )
+            del df, dataset
+            gc.collect()
 
-        del df_without_features, dataset
-        gc.collect()
+            if not silent_mode:
+                print(self.bundle.get("polling_transform_task").format(validation_task.search_task_id))
+                if not self.__is_registered:
+                    print(self.bundle.get("polling_unregister_information"))
 
-        if not silent_mode:
-            print(self.bundle.get("polling_transform_task").format(validation_task.search_task_id))
-            if not self.__is_registered:
-                print(self.bundle.get("polling_unregister_information"))
+            progress = self.get_progress(validation_task)
+            progress.recalculate_eta(time.time() - start_time)
+            if progress_bar is not None:
+                progress_bar.progress = progress.to_progress_bar()
+            if progress_callback is not None:
+                progress_callback(progress)
+            prev_progress: SearchProgress | None = None
+            polling_period_seconds = 1
+            try:
+                while progress.stage != ProgressStage.DOWNLOADING.value:
+                    if prev_progress is None or prev_progress.percent != progress.percent:
+                        progress.recalculate_eta(time.time() - start_time)
+                    else:
+                        progress.update_eta(prev_progress.eta - polling_period_seconds)
+                    prev_progress = progress
+                    if progress_bar is not None:
+                        progress_bar.progress = progress.to_progress_bar()
+                    if progress_callback is not None:
+                        progress_callback(progress)
+                    if progress.stage == ProgressStage.FAILED.value:
+                        raise Exception(progress.error_message)
+                    time.sleep(polling_period_seconds)
+                    progress = self.get_progress(validation_task)
+            except KeyboardInterrupt as e:
+                print(self.bundle.get("search_stopping"))
+                self.rest_client.stop_search_task_v2(self._get_trace_id(), validation_task.search_task_id)
+                self.logger.warning(f"Search {validation_task.search_task_id} stopped by user")
+                print(self.bundle.get("search_stopped"))
+                raise e
 
-        progress = self.get_progress(validation_task)
-        progress.recalculate_eta(time.time() - start_time)
-        if progress_bar is not None:
-            progress_bar.progress = progress.to_progress_bar()
-        if progress_callback is not None:
-            progress_callback(progress)
-        prev_progress: SearchProgress | None = None
-        polling_period_seconds = 1
-        try:
-            while progress.stage != ProgressStage.DOWNLOADING.value:
-                if prev_progress is None or prev_progress.percent != progress.percent:
-                    progress.recalculate_eta(time.time() - start_time)
-                else:
-                    progress.update_eta(prev_progress.eta - polling_period_seconds)
-                prev_progress = progress
-                if progress_bar is not None:
-                    progress_bar.progress = progress.to_progress_bar()
-                if progress_callback is not None:
-                    progress_callback(progress)
-                if progress.stage == ProgressStage.FAILED.value:
-                    raise Exception(progress.error_message)
-                time.sleep(polling_period_seconds)
-                progress = self.get_progress(validation_task)
-        except KeyboardInterrupt as e:
-            print(self.bundle.get("search_stopping"))
-            self.rest_client.stop_search_task_v2(self._get_trace_id(), validation_task.search_task_id)
-            self.logger.warning(f"Search {validation_task.search_task_id} stopped by user")
-            print(self.bundle.get("search_stopped"))
-            raise e
-
-        validation_task.poll_result(self._get_trace_id(), quiet=True)
-
-        seconds_left = time.time() - start_time
-        progress = SearchProgress(97.0, ProgressStage.DOWNLOADING, seconds_left)
-        if progress_bar is not None:
-            progress_bar.progress = progress.to_progress_bar()
-        if progress_callback is not None:
-            progress_callback(progress)
+            validation_task.poll_result(self._get_trace_id(), quiet=True)
 
-        if not silent_mode:
-            print(self.bundle.get("transform_start"))
+            seconds_left = time.time() - start_time
+            progress = SearchProgress(97.0, ProgressStage.DOWNLOADING, seconds_left)
+            if progress_bar is not None:
+                progress_bar.progress = progress.to_progress_bar()
+            if progress_callback is not None:
+                progress_callback(progress)
+
+            if not silent_mode:
+                print(self.bundle.get("transform_start"))
+
+            return validation_task.get_all_validation_raw_features(self._get_trace_id(), metrics_calculation)
+
+        if len(df_without_features) <= Dataset.MAX_ROWS:
+            result_features = invoke_validation(df_without_features)
+        else:
+            self.logger.warning(
+                f"Dataset has more than {Dataset.MAX_ROWS} rows: {len(df_without_features)}, "
+                f"splitting into chunks of {Dataset.MAX_ROWS} rows"
+            )
+            result_features_list = []
+
+            for i in range(0, len(df_without_features), Dataset.MAX_ROWS):
+                chunk = df_without_features.iloc[i : i + Dataset.MAX_ROWS]
+                result_features_list.append(invoke_validation(chunk))
+            result_features = pd.concat(result_features_list)
 
         # Prepare input DataFrame for __enrich by concatenating generated ids and client features
         df_before_explode = df_before_explode.rename(columns=columns_renaming)
@@ -2922,8 +2946,6 @@ if response.status_code == 200:
             axis=1,
         ).set_index(validated_Xy.index)
 
-        result_features = validation_task.get_all_validation_raw_features(self._get_trace_id(), metrics_calculation)
-
         result = self.__enrich(
             combined_df,
             result_features,
@@ -2974,12 +2996,38 @@ if response.status_code == 200:
         fit_dropped_features = self.fit_dropped_features or file_meta.droppedColumns or []
         fit_input_columns = [c.originalName for c in file_meta.columns]
         original_dropped_features = [self.fit_columns_renaming.get(c, c) for c in fit_dropped_features]
+        true_one_hot_features = (
+            [f for group in self.true_one_hot_groups.values() for f in group] if self.true_one_hot_groups else []
+        )
         new_columns_on_transform = [
-            c for c in validated_Xy.columns if c not in fit_input_columns and c not in original_dropped_features
+            c
+            for c in validated_Xy.columns
+            if c not in fit_input_columns and c not in original_dropped_features and c not in true_one_hot_features
         ]
         fit_original_search_keys = self._get_fit_search_keys_with_original_names()
 
         selected_generated_features = [c for c in generated_features if c in self.feature_names_]
+        selected_true_one_hot_features = (
+            [
+                c
+                for cat_feature, group in self.true_one_hot_groups.items()
+                for c in group
+                if cat_feature in self.feature_names_
+            ]
+            if self.true_one_hot_groups
+            else []
+        )
+        selected_pseudo_one_hot_features = (
+            [
+                feature
+                for group in self.pseudo_one_hot_groups.values()
+                if any(f in self.feature_names_ for f in group)
+                for feature in group
+            ]
+            if self.pseudo_one_hot_groups
+            else []
+        )
+
         if keep_input is True:
             selected_input_columns = [
                 c
@@ -2998,11 +3046,14 @@ if response.status_code == 200:
         if DEFAULT_INDEX in selected_input_columns:
             selected_input_columns.remove(DEFAULT_INDEX)
 
-        return selected_input_columns + selected_generated_features
+        return (
+            selected_input_columns
+            + selected_generated_features
+            + selected_true_one_hot_features
+            + selected_pseudo_one_hot_features
+        )
 
-    def _validate_empty_search_keys(
-        self, search_keys: dict[str, SearchKey], is_transform: bool = False
-    ):
+    def _validate_empty_search_keys(self, search_keys: dict[str, SearchKey], is_transform: bool = False):
         if (search_keys is None or len(search_keys) == 0) and self.country_code is None:
             if is_transform:
                 self.logger.debug("Transform started without search_keys")
@@ -3169,7 +3220,7 @@ if response.status_code == 200:
         else:
             only_train_df = df
 
-        self.imbalanced = is_imbalanced(only_train_df, self.model_task_type, self.sample_config, self.bundle)
+        self.imbalanced = is_imbalanced(only_train_df, self.model_task_type, self.sample_config, self.bundle, self.__log_warning)
         if self.imbalanced:
             # Exclude eval sets from fit because they will be transformed before metrics calculation
             df = only_train_df
@@ -3242,6 +3293,8 @@ if response.status_code == 200:
             df, self.fit_search_keys, self.fit_generated_features
         )
         self.fit_columns_renaming = normalizer.columns_renaming
+        self.true_one_hot_groups = normalizer.true_one_hot_groups
+        self.pseudo_one_hot_groups = normalizer.pseudo_one_hot_groups
         if normalizer.removed_datetime_features:
             self.fit_dropped_features.update(normalizer.removed_datetime_features)
             original_removed_datetime_features = [
@@ -3259,7 +3312,11 @@ if response.status_code == 200:
         features_columns = [c for c in df.columns if c not in non_feature_columns]
 
         features_to_drop, feature_validator_warnings = FeaturesValidator(self.logger).validate(
-            df, features_columns, self.generate_features, self.fit_columns_renaming
+            df,
+            features_columns,
+            self.generate_features,
+            self.fit_columns_renaming,
+            [f for group in self.pseudo_one_hot_groups.values() for f in group] if self.pseudo_one_hot_groups else [],
         )
         if feature_validator_warnings:
             for warning in feature_validator_warnings:
@@ -3822,8 +3879,7 @@ if response.status_code == 200:
                 elif self.columns_for_online_api:
                     msg = self.bundle.get("oot_with_online_sources_not_supported").format(eval_set_index)
                 if msg:
-                    print(msg)
-                    self.logger.warning(msg)
+                    self.__log_warning(msg)
                     df = df[df[EVAL_SET_INDEX] != eval_set_index]
         return df
 
@@ -4768,7 +4824,7 @@ if response.status_code == 200:
         elif self.autodetect_search_keys:
             valid_search_keys = self.__detect_missing_search_keys(x, valid_search_keys, is_demo_dataset)
 
-        if all(k == SearchKey.CUSTOM_KEY for k in valid_search_keys.values()):
+        if len(valid_search_keys) > 0 and all(k == SearchKey.CUSTOM_KEY for k in valid_search_keys.values()):
             if self.__is_registered:
                 msg = self.bundle.get("only_custom_keys")
             else:
@@ -5027,37 +5083,55 @@ if response.status_code == 200:
                         X_ = X_.to_frame()
 
                     with tempfile.TemporaryDirectory() as tmp_dir:
-                        X_.to_parquet(f"{tmp_dir}/x.parquet", compression="zstd")
-                        x_digest_sha256 = file_hash(f"{tmp_dir}/x.parquet")
+                        x_file_name = f"{tmp_dir}/x.parquet"
+                        X_.to_parquet(x_file_name, compression="zstd")
+                        uploading_file_size = Path(x_file_name).stat().st_size
+                        if uploading_file_size > Dataset.MAX_UPLOADING_FILE_SIZE:
+                            self.logger.warning(
+                                f"Uploading file x.parquet is too large: {uploading_file_size} bytes. Skip it"
+                            )
+                            return
+                        x_digest_sha256 = file_hash(x_file_name)
                         if self.rest_client.is_file_uploaded(trace_id_, x_digest_sha256):
                             self.logger.info(
                                 f"File x.parquet was already uploaded with digest {x_digest_sha256}, skipping"
                             )
                         else:
-                            self.rest_client.dump_input_file(
-                                trace_id_, f"{tmp_dir}/x.parquet", "x.parquet", x_digest_sha256
-                            )
+                            self.rest_client.dump_input_file(trace_id_, x_file_name, "x.parquet", x_digest_sha256)
 
                         if y_ is not None:
                             if isinstance(y_, pd.Series):
                                 y_ = y_.to_frame()
-                            y_.to_parquet(f"{tmp_dir}/y.parquet", compression="zstd")
-                            y_digest_sha256 = file_hash(f"{tmp_dir}/y.parquet")
+                            y_file_name = f"{tmp_dir}/y.parquet"
+                            y_.to_parquet(y_file_name, compression="zstd")
+                            uploading_file_size = Path(y_file_name).stat().st_size
+                            if uploading_file_size > Dataset.MAX_UPLOADING_FILE_SIZE:
+                                self.logger.warning(
+                                    f"Uploading file y.parquet is too large: {uploading_file_size} bytes. Skip it"
+                                )
+                                return
+                            y_digest_sha256 = file_hash(y_file_name)
                             if self.rest_client.is_file_uploaded(trace_id_, y_digest_sha256):
                                 self.logger.info(
                                     f"File y.parquet was already uploaded with digest {y_digest_sha256}, skipping"
                                 )
                             else:
-                                self.rest_client.dump_input_file(
-                                    trace_id_, f"{tmp_dir}/y.parquet", "y.parquet", y_digest_sha256
-                                )
+                                self.rest_client.dump_input_file(trace_id_, y_file_name, "y.parquet", y_digest_sha256)
 
                             if eval_set_ is not None and len(eval_set_) > 0:
                                 for idx, (eval_x_, eval_y_) in enumerate(eval_set_):
                                     if isinstance(eval_x_, pd.Series):
                                         eval_x_ = eval_x_.to_frame()
-                                    eval_x_.to_parquet(f"{tmp_dir}/eval_x_{idx}.parquet", compression="zstd")
-                                    eval_x_digest_sha256 = file_hash(f"{tmp_dir}/eval_x_{idx}.parquet")
+                                    eval_x_file_name = f"{tmp_dir}/eval_x_{idx}.parquet"
+                                    eval_x_.to_parquet(eval_x_file_name, compression="zstd")
+                                    uploading_file_size = Path(eval_x_file_name).stat().st_size
+                                    if uploading_file_size > Dataset.MAX_UPLOADING_FILE_SIZE:
+                                        self.logger.warning(
+                                            f"Uploading file eval_x_{idx}.parquet is too large: "
+                                            f"{uploading_file_size} bytes. Skip it"
+                                        )
+                                        return
+                                    eval_x_digest_sha256 = file_hash(eval_x_file_name)
                                     if self.rest_client.is_file_uploaded(trace_id_, eval_x_digest_sha256):
                                         self.logger.info(
                                             f"File eval_x_{idx}.parquet was already uploaded with"
@@ -5066,15 +5140,23 @@ if response.status_code == 200:
                                     else:
                                         self.rest_client.dump_input_file(
                                             trace_id_,
-                                            f"{tmp_dir}/eval_x_{idx}.parquet",
+                                            eval_x_file_name,
                                             f"eval_x_{idx}.parquet",
                                             eval_x_digest_sha256,
                                         )
 
                                     if isinstance(eval_y_, pd.Series):
                                         eval_y_ = eval_y_.to_frame()
-                                    eval_y_.to_parquet(f"{tmp_dir}/eval_y_{idx}.parquet", compression="zstd")
-                                    eval_y_digest_sha256 = file_hash(f"{tmp_dir}/eval_y_{idx}.parquet")
+                                    eval_y_file_name = f"{tmp_dir}/eval_y_{idx}.parquet"
+                                    eval_y_.to_parquet(eval_y_file_name, compression="zstd")
+                                    uploading_file_size = Path(eval_y_file_name).stat().st_size
+                                    if uploading_file_size > Dataset.MAX_UPLOADING_FILE_SIZE:
+                                        self.logger.warning(
+                                            f"Uploading file eval_y_{idx}.parquet is too large: "
+                                            f"{uploading_file_size} bytes. Skip it"
+                                        )
+                                        return
+                                    eval_y_digest_sha256 = file_hash(eval_y_file_name)
                                     if self.rest_client.is_file_uploaded(trace_id_, eval_y_digest_sha256):
                                         self.logger.info(
                                             f"File eval_y_{idx}.parquet was already uploaded"
@@ -5083,7 +5165,7 @@ if response.status_code == 200:
                                     else:
                                         self.rest_client.dump_input_file(
                                             trace_id_,
-                                            f"{tmp_dir}/eval_y_{idx}.parquet",
+                                            eval_y_file_name,
                                             f"eval_y_{idx}.parquet",
                                             eval_y_digest_sha256,
                                         )

{upgini-1.2.146a2 → upgini-1.2.146a9}/src/upgini/normalizer/normalize_utils.py

@@ -26,6 +26,7 @@ from upgini.utils import find_numbers_with_decimal_comma
 from upgini.utils.country_utils import CountrySearchKeyConverter
 from upgini.utils.datetime_utils import DateTimeConverter
 from upgini.utils.ip_utils import IpSearchKeyConverter
+from upgini.utils.one_hot_encoder import OneHotDecoder
 from upgini.utils.phone_utils import PhoneSearchKeyConverter
 from upgini.utils.postal_code_utils import PostalCodeSearchKeyConverter
 
@@ -45,6 +46,8 @@ class Normalizer:
         self.search_keys = {}
         self.generated_features = []
         self.removed_datetime_features = []
+        self.true_one_hot_groups: dict[str, list[str]] | None = None
+        self.pseudo_one_hot_groups: dict[str, list[str]] | None = None
 
     def normalize(
         self, df: pd.DataFrame, search_keys: Dict[str, SearchKey], generated_features: List[str]
@@ -53,6 +56,9 @@ class Normalizer:
         self.generated_features = generated_features.copy()
 
         df = df.copy()
+
+        df = self._convert_one_hot_encoded_columns(df)
+
         df = self._rename_columns(df)
 
         df = self._remove_dates_from_features(df)
@@ -77,6 +83,15 @@ class Normalizer:
 
         return df, self.search_keys, self.generated_features
 
+    def _convert_one_hot_encoded_columns(self, df: pd.DataFrame):
+        if self.true_one_hot_groups is not None or self.pseudo_one_hot_groups is not None:
+            df = OneHotDecoder.decode_with_cached_groups(
+                df, self.true_one_hot_groups, self.pseudo_one_hot_groups
+            )
+        else:
+            df, self.true_one_hot_groups, self.pseudo_one_hot_groups = OneHotDecoder.decode(df)
+        return df
+
     def _rename_columns(self, df: pd.DataFrame):
         # logger.info("Replace restricted symbols in column names")
         new_columns = []

{upgini-1.2.146a2 → upgini-1.2.146a9}/src/upgini/resource_bundle/strings.properties

@@ -176,7 +176,8 @@ dataset_invalid_multiclass_target=Unexpected dtype of target for multiclass task
 dataset_invalid_regression_target=Unexpected dtype of target for regression task type: {}. Expected float
 dataset_invalid_timeseries_target=Unexpected dtype of target for timeseries task type: {}. Expected float
 dataset_to_many_multiclass_targets=The number of target classes {} exceeds the allowed threshold: {}. Please, correct your data and try again
-dataset_rarest_class_less_min=Count of rows with the rarest class `{}` is {}, minimum count must be > {} for each class\nPlease, remove rows with rarest class from your dataframe
+dataset_rarest_class_less_min=Count of rows with the rarest class `{}` is {}, minimum count must be > {} for each class
+#\nPlease, remove rows with rarest class from your dataframe
 dataset_rarest_class_less_threshold=Target is imbalanced and will be undersampled to the rarest class. Frequency of the rarest class `{}` is {}\nMinimum number of observations for each class to avoid undersampling {} ({}%)
 dataset_date_features=Columns {} is a datetime or period type but not used as a search key, removed from X
 dataset_too_many_features=Too many features. Maximum number of features is {}
@@ -231,7 +232,8 @@ limited_int_multiclass_reason=integer-like values with limited unique values obs
 all_ok_community_invite=❓ Support request 
 too_small_for_metrics=Your train dataset or one of eval datasets contains less than 500 rows. For such dataset Upgini will not calculate accuracy metrics. Please increase the number of rows in the training dataset to calculate accuracy metrics
 imbalance_multiclass=Class {0} is on 25% quantile of classes distribution ({1} records in train dataset). \nDownsample classes with records more than {1}.
-imbalanced_target=\nTarget is imbalanced and will be undersampled. Frequency of the rarest class `{}` is {}
+rare_target_classes_drop=Drop rare target classes with <0.01% freq: {}
+imbalanced_target=Target is imbalanced and will be undersampled. Frequency of the rarest class `{}` is {}
 loss_selection_info=Using loss `{}` for feature selection
 loss_calc_metrics_info=Using loss `{}` for metrics calculation with default estimator
 forced_balance_undersample=For quick data retrieval, your dataset has been sampled. To use data search without data sampling please contact support (sales@upgini.com)

{upgini-1.2.146a2 → upgini-1.2.146a9}/src/upgini/utils/datetime_utils.py

@@ -8,7 +8,7 @@ from dateutil.relativedelta import relativedelta
 from pandas.api.types import is_numeric_dtype
 
 from upgini.errors import ValidationError
-from upgini.metadata import EVAL_SET_INDEX, SearchKey
+from upgini.metadata import CURRENT_DATE_COL, EVAL_SET_INDEX, SearchKey
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.utils.base_search_key_detector import BaseSearchKeyDetector
 
@@ -418,12 +418,12 @@ def is_dates_distribution_valid(
                 except Exception:
                     pass
 
-        if maybe_date_col is None:
-            return
+        if maybe_date_col is None or maybe_date_col == CURRENT_DATE_COL:
+            return True
 
         # Don't check if date column is constant
         if X[maybe_date_col].nunique() <= 1:
-            return
+            return True
 
         if isinstance(X[maybe_date_col].dtype, pd.PeriodDtype):
             dates = X[maybe_date_col].dt.to_timestamp().dt.date

{upgini-1.2.146a2 → upgini-1.2.146a9}/src/upgini/utils/features_validator.py

@@ -23,12 +23,18 @@ class FeaturesValidator:
         features: List[str],
         features_for_generate: Optional[List[str]] = None,
         columns_renaming: Optional[Dict[str, str]] = None,
+        pseudo_one_hot_encoded_features: Optional[List[str]] = None,
     ) -> Tuple[List[str], List[str]]:
-        one_hot_encoded_features = []
         empty_or_constant_features = []
         high_cardinality_features = []
         warnings = []
 
+        pseudo_one_hot_encoded_features = [
+            renamed
+            for renamed, original in columns_renaming.items()
+            if original in pseudo_one_hot_encoded_features or []
+        ]
+
         for f in features:
             column = df[f]
             if is_object_dtype(column):
@@ -38,20 +44,11 @@ class FeaturesValidator:
 
             if len(value_counts) == 1:
                 empty_or_constant_features.append(f)
-            elif most_frequent_percent >= 0.99:
-                if self.is_one_hot_encoded(column):
-                    one_hot_encoded_features.append(f)
-                else:
-                    empty_or_constant_features.append(f)
+            elif most_frequent_percent >= 0.99 and f not in pseudo_one_hot_encoded_features:
+                empty_or_constant_features.append(f)
 
         columns_renaming = columns_renaming or {}
 
-        if one_hot_encoded_features and len(one_hot_encoded_features) > 1:
-            msg = bundle.get("one_hot_encoded_features").format(
-                [columns_renaming.get(f, f) for f in one_hot_encoded_features]
-            )
-            warnings.append(msg)
-
         if empty_or_constant_features:
             msg = bundle.get("empty_or_contant_features").format(
                 [columns_renaming.get(f, f) for f in empty_or_constant_features]
@@ -98,41 +95,3 @@ class FeaturesValidator:
     @staticmethod
     def find_constant_features(df: pd.DataFrame) -> List[str]:
         return [i for i in df if df[i].nunique() <= 1]
-
-    @staticmethod
-    def is_one_hot_encoded(series: pd.Series) -> bool:
-        try:
-            # All rows should be the same type
-            if series.apply(lambda x: type(x)).nunique() != 1:
-                return False
-
-            # First, handle string representations of True/False
-            series_copy = series.copy()
-            if series_copy.dtype == "object" or series_copy.dtype == "string":
-                # Convert string representations of boolean values to numeric
-                series_copy = series_copy.astype(str).str.strip().str.lower()
-                series_copy = series_copy.replace({"true": "1", "false": "0"})
-
-            # Column contains only 0 and 1 (as strings or numbers or booleans)
-            series_copy = series_copy.astype(float)
-            if set(series_copy.unique()) != {0.0, 1.0}:
-                return False
-
-            series_copy = series_copy.astype(int)
-
-            # Column doesn't contain any NaN, np.NaN, space, null, etc.
-            if not (series_copy.isin([0, 1])).all():
-                return False
-
-            vc = series_copy.value_counts()
-            # Column should contain both 0 and 1
-            if len(vc) != 2:
-                return False
-
-            # Minority class is 1
-            if vc[1] >= vc[0]:
-                return False
-
-            return True
-        except ValueError:
-            return False

upgini-1.2.146a9/src/upgini/utils/one_hot_encoder.py

@@ -0,0 +1,215 @@
+import numpy as np
+import pandas as pd
+
+
+class OneHotDecoder:
+
+    def encode(df: pd.DataFrame, category_columns: list[str]) -> pd.DataFrame:
+        """
+        Encode categorical columns into one-hot encoded columns.
+        """
+        return pd.get_dummies(df, columns=category_columns, prefix_sep="")
+
+    def decode(df: pd.DataFrame) -> (pd.DataFrame, dict[str, list[str]], dict[str, list[str]]):
+        """
+        Detect one-hot encoded column groups and collapse each group into a single
+        categorical column. For each row, all active bits in the group are
+        encoded into a unique category using a bitmask over the group's columns
+        (ordered by numeric suffix). Rows with zero active bits are set to NA.
+
+        Returns a new DataFrame with transformed columns.
+        """
+        one_hot_candidate_groups = OneHotDecoder._group_one_hot_fast(df.columns)
+        true_one_hot_groups: dict[str, list[str]] = {}
+
+        # 1) Detect valid one-hot groups (filter candidates by column-level checks)
+        for group_name, column_candidates in one_hot_candidate_groups.items():
+            group_columns: list[str] = []
+            for column in column_candidates:
+                value_counts = df[column].value_counts(dropna=False, normalize=True)
+                most_frequent_percent = value_counts.iloc[0]
+                if most_frequent_percent >= 0.6 and OneHotDecoder._is_one_hot_encoded(df[column]):
+                    group_columns.append(column)
+            if len(group_columns) > 1:
+                true_one_hot_groups[group_name] = group_columns
+
+        # 2) Transform: replace each detected group with one categorical column
+        if not true_one_hot_groups:
+            return df, {}, {}
+
+        result_df = df.copy()
+        pseudo_one_hot_groups: dict[str, list[str]] = {}
+        for group_name, group_columns in true_one_hot_groups.items():
+            sub = result_df[group_columns].copy()
+            for c in group_columns:
+                s = sub[c]
+                if s.dtype == "object" or s.dtype == "string":
+                    s = s.astype(str).str.strip().str.lower()
+                    s = s.replace({"true": "1", "false": "0"})
+                s = pd.to_numeric(s, errors="coerce")
+                sub[c] = s
+
+            # 3) Find pseudo one-hot encoded columns when there are multiple ones in one row
+            if any(sub.sum(axis=1) > 1):
+                pseudo_one_hot_groups[group_name] = group_columns
+                result_df[group_columns] = result_df[group_columns].astype("string")
+                continue
+
+            # Coerce values to numeric 0/1 handling common textual forms
+            sub = sub.fillna(0.0)
+            # Binarize strictly to 0/1
+            bin_values = (sub.to_numpy() > 0.5).astype(np.int64)
+            # Map single active bit to exact numeric suffix from column name
+            row_sums = bin_values.sum(axis=1)
+            argmax_idx = bin_values.argmax(axis=1)
+            suffix_arr = np.array(
+                [int(OneHotDecoder._split_prefix_numeric_suffix(col)[1]) for col in group_columns], dtype=np.int64
+            )
+            codes = suffix_arr[argmax_idx]
+            categorical_series = pd.Series(codes, index=sub.index)
+            # Keep only rows with exactly one active bit; else set NA
+            categorical_series = categorical_series.where(row_sums == 1, other=pd.NA)
+            # Use pandas nullable integer dtype to keep NA with integer codes
+            result_df[group_name] = categorical_series.astype("Int64").astype("string")
+
+            # Drop original one-hot columns of the group
+            result_df = result_df.drop(columns=group_columns)
+
+        for group_name in pseudo_one_hot_groups:
+            del true_one_hot_groups[group_name]
+
+        return result_df, true_one_hot_groups, pseudo_one_hot_groups
+
+    def decode_with_cached_groups(
+        df: pd.DataFrame, true_one_hot_groups: dict[str, list[str]], pseudo_one_hot_groups: dict[str, list[str]]
+    ) -> pd.DataFrame:
+        """
+        Decode one-hot encoded columns with cached groups.
+        """
+        result_df = df.copy()
+        # 1. Transform regular one-hot groups back to categorical
+        if true_one_hot_groups:
+            for group_name, group_columns in true_one_hot_groups.items():
+                sub = result_df[group_columns].copy()
+                for c in group_columns:
+                    s = sub[c]
+                    if s.dtype == "object" or s.dtype == "string":
+                        s = s.astype(str).str.strip().str.lower()
+                        s = s.replace({"true": "1", "false": "0"})
+                    s = pd.to_numeric(s, errors="coerce")
+                    sub[c] = s
+                sub = sub.fillna(0.0)
+                bin_values = (sub.to_numpy() > 0.5).astype(np.int64)
+                row_sums = bin_values.sum(axis=1)
+                argmax_idx = bin_values.argmax(axis=1)
+                suffix_arr = np.array(
+                    [int(OneHotDecoder._split_prefix_numeric_suffix(col)[1]) for col in group_columns], dtype=np.int64
+                )
+                codes = suffix_arr[argmax_idx]
+                categorical_series = pd.Series(codes, index=sub.index)
+                categorical_series = categorical_series.where(row_sums == 1, other=pd.NA)
+                result_df[group_name] = categorical_series.astype("Int64").astype("string")
+                result_df = result_df.drop(columns=group_columns)
+        # 2. Convert pseudo-one-hot features to string
+        if pseudo_one_hot_groups:
+            for _, group_columns in pseudo_one_hot_groups.items():
+                result_df[group_columns] = result_df[group_columns].astype("string")
+        return result_df
+
+    @staticmethod
+    def _is_ascii_digit(c: str) -> bool:
+        return "0" <= c <= "9"
+
+    @staticmethod
+    def _split_prefix_numeric_suffix(name: str) -> tuple[str, str] | None:
+        """
+        Return (prefix, numeric_suffix) if name ends with ASCII digits and isn't all digits.
+        Otherwise None.
+        """
+        if not name or not OneHotDecoder._is_ascii_digit(name[-1]):
+            return None
+        i = len(name) - 1
+        while i >= 0 and OneHotDecoder._is_ascii_digit(name[i]):
+            i -= 1
+        if i < 0:
+            # Entire string is digits -> reject
+            return None
+        return name[: i + 1], name[i + 1 :]  # prefix, suffix
+
+    @staticmethod
+    def _group_one_hot_fast(
+        candidates: list[str], min_group_size: int = 2, require_consecutive: bool = True
+    ) -> dict[str, list[str]]:
+        """
+        Group OHE-like columns by (prefix, numeric_suffix).
+        - Only keeps groups with size >= min_group_size (default: 2).
+        - Each group's columns are sorted by numeric suffix (int).
+        Returns: {prefix: [col_names_sorted]}.
+        """
+        if min_group_size < 2:
+            raise ValueError("min_group_size must be >= 2.")
+
+        # 1) Collect by prefix with parsed numeric suffix
+        groups: dict[str, list[(int, str)]] = {}
+        for s in candidates:
+            sp = OneHotDecoder._split_prefix_numeric_suffix(s)
+            if sp is None:
+                continue
+            prefix, sfx = sp
+            groups.setdefault(prefix, []).append((int(sfx), s))
+
+        # 2) Filter and finalize
+        out: dict[str, list[str]] = {}
+        for prefix, pairs in groups.items():
+            if len(pairs) < min_group_size:
+                continue
+            pairs.sort(key=lambda t: t[0])  # sort by numeric suffix
+            if require_consecutive:
+                suffixes = [num for num, _ in pairs]
+                # no duplicates
+                if len(suffixes) != len(set(suffixes)):
+                    continue
+                # strictly consecutive run with step=1
+                start = suffixes[0]
+                if any(suffixes[i] != start + i for i in range(len(suffixes))):
+                    continue
+            out[prefix] = [name for _, name in pairs]
+
+        return out
+
+    def _is_one_hot_encoded(series: pd.Series) -> bool:
+        try:
+            # All rows should be the same type
+            if series.apply(lambda x: type(x)).nunique() != 1:
+                return False
+
+            # First, handle string representations of True/False
+            series_copy = series.copy()
+            if series_copy.dtype == "object" or series_copy.dtype == "string":
+                # Convert string representations of boolean values to numeric
+                series_copy = series_copy.astype(str).str.strip().str.lower()
+                series_copy = series_copy.replace({"true": "1", "false": "0"})
+
+            # Column contains only 0 and 1 (as strings or numbers or booleans)
+            series_copy = series_copy.astype(float)
+            if set(series_copy.unique()) != {0.0, 1.0}:
+                return False
+
+            series_copy = series_copy.astype(int)
+
+            # Column doesn't contain any NaN, np.NaN, space, null, etc.
+            if not (series_copy.isin([0, 1])).all():
+                return False
+
+            vc = series_copy.value_counts()
+            # Column should contain both 0 and 1
+            if len(vc) != 2:
+                return False
+
+            # Minority class is 1
+            if vc[1] >= vc[0]:
+                return False
+
+            return True
+        except ValueError:
+            return False

{upgini-1.2.146a2 → upgini-1.2.146a9}/src/upgini/utils/target_utils.py

@@ -117,6 +117,7 @@ def is_imbalanced(
     task_type: ModelTaskType,
     sample_config: SampleConfig,
     bundle: ResourceBundle,
+    warning_callback: Optional[Callable] = None,
 ) -> bool:
     if task_type is None or not task_type.is_classification():
         return False
@@ -144,7 +145,8 @@ def is_imbalanced(
         msg = bundle.get("dataset_rarest_class_less_min").format(
             min_class_value, min_class_count, MIN_TARGET_CLASS_ROWS
         )
-        raise ValidationError(msg)
+        if warning_callback is not None:
+            warning_callback(msg)
 
     min_class_percent = IMBALANCE_THESHOLD / target_classes_count
     min_class_threshold = min_class_percent * count
@@ -196,14 +198,34 @@ def balance_undersample(
     resampled_data = df
     df = df.copy().sort_values(by=SYSTEM_RECORD_ID)
     if task_type == ModelTaskType.MULTICLASS:
+        # Remove rare classes which have <0.01% of samples
+        total_count = len(df)
+        # Always preserve two most frequent classes, even if they are rare
+        top_two_classes = list(vc.index[:2])
+        rare_classes_all = [cls for cls, cnt in vc.items() if cnt / total_count < 0.0001]
+        rare_classes = [cls for cls in rare_classes_all if cls not in top_two_classes]
+        if rare_classes:
+            msg = bundle.get("rare_target_classes_drop").format(rare_classes)
+            logger.warning(msg)
+            warning_callback(msg)
+            df = df[~df[target_column].isin(rare_classes)]
+            target = df[target_column].copy()
+            vc = target.value_counts()
+            max_class_value = vc.index[0]
+            min_class_value = vc.index[len(vc) - 1]
+            max_class_count = vc[max_class_value]
+            min_class_count = vc[min_class_value]
+            num_classes = len(vc)
+
         if len(df) > multiclass_min_sample_threshold and max_class_count > (
             min_class_count * multiclass_bootstrap_loops
         ):
 
             msg = bundle.get("imbalanced_target").format(min_class_value, min_class_count)
-            logger.warning(msg)
             if warning_callback is not None:
                 warning_callback(msg)
+            else:
+                logger.warning(msg)
 
             sample_strategy = dict()
             for class_value in vc.index:
@@ -228,9 +250,10 @@ def balance_undersample(
             resampled_data = df[df[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
     elif len(df) > binary_min_sample_threshold:
         msg = bundle.get("imbalanced_target").format(min_class_value, min_class_count)
-        logger.warning(msg)
         if warning_callback is not None:
             warning_callback(msg)
+        else:
+            logger.warning(msg)
 
         # fill up to min_sample_threshold by majority class
         minority_class = df[df[target_column] == min_class_value]

upgini-1.2.146a2/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.146a2"