upgini 1.1.252a5__py3-none-any.whl → 1.1.253__py3-none-any.whl

upgini/data_source/data_source_publisher.py

@@ -40,7 +40,7 @@ class DataSourcePublisher:
         if logs_enabled:
             self.logger = LoggerFactory().get_logger(endpoint, api_key)
         else:
-            self.logger = logging.getLogger("muted_logger")
+            self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
 
     def place(

upgini/dataset.py

@@ -15,9 +15,9 @@ from pandas.api.types import (
     is_float_dtype,
     is_integer_dtype,
     is_numeric_dtype,
+    is_period_dtype,
     is_string_dtype,
 )
-from pandas.core.dtypes.common import is_period_dtype
 
 from upgini.errors import ValidationError
 from upgini.http import ProgressStage, SearchProgress, _RestClient
@@ -39,10 +39,10 @@ from upgini.metadata import (
 )
 from upgini.normalizer.phone_normalizer import PhoneNormalizer
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
+from upgini.sampler.random_under_sampler import RandomUnderSampler
 from upgini.search_task import SearchTask
-from upgini.utils import combine_search_keys
+from upgini.utils import combine_search_keys, find_numbers_with_decimal_comma
 from upgini.utils.email_utils import EmailSearchKeyConverter
-from upgini.utils.target_utils import balance_undersample
 
 try:
     from upgini.utils.progress_bar import CustomProgressBar as ProgressBar
@@ -61,8 +61,6 @@ class Dataset:  # (pd.DataFrame):
     FIT_SAMPLE_WITH_EVAL_SET_THRESHOLD = 200_000
     MIN_SAMPLE_THRESHOLD = 5_000
     IMBALANCE_THESHOLD = 0.4
-    BINARY_BOOTSTRAP_LOOPS = 5
-    MULTICLASS_BOOTSTRAP_LOOPS = 2
     MIN_TARGET_CLASS_ROWS = 100
     MAX_MULTICLASS_CLASS_COUNT = 100
     MIN_SUPPORTED_DATE_TS = 946684800000  # 2000-01-01
@@ -224,45 +222,6 @@ class Dataset:  # (pd.DataFrame):
                 if max_length > self.MAX_STRING_FEATURE_LENGTH:
                     self.data[col] = self.data[col].astype("str").str.slice(stop=self.MAX_STRING_FEATURE_LENGTH)
 
-    def __clean_duplicates(self, silent_mode: bool = False):
-        """Clean DataSet from full duplicates."""
-        # self.logger.info("Clean full duplicates")
-        nrows = len(self.data)
-        if nrows == 0:
-            return
-        # Remove absolute duplicates (exclude system_record_id)
-        unique_columns = self.data.columns.tolist()
-        unique_columns.remove(SYSTEM_RECORD_ID)
-        self.logger.info(f"Dataset shape before clean duplicates: {self.data.shape}")
-        self.data.drop_duplicates(subset=unique_columns, inplace=True)
-        self.logger.info(f"Dataset shape after clean duplicates: {self.data.shape}")
-        nrows_after_full_dedup = len(self.data)
-        share_full_dedup = 100 * (1 - nrows_after_full_dedup / nrows)
-        if share_full_dedup > 0:
-            msg = self.bundle.get("dataset_full_duplicates").format(share_full_dedup)
-            self.logger.warning(msg)
-            # if not silent_mode:
-            #     print(msg)
-            # self.warning_counter.increment()
-        target_column = self.etalon_def_checked.get(FileColumnMeaningType.TARGET.value)
-        if target_column is not None:
-            unique_columns.remove(target_column)
-            marked_duplicates = self.data.duplicated(subset=unique_columns, keep=False)
-            if marked_duplicates.sum() > 0:
-                dups_indices = self.data[marked_duplicates].index.to_list()
-                nrows_after_tgt_dedup = len(self.data.drop_duplicates(subset=unique_columns))
-                num_dup_rows = nrows_after_full_dedup - nrows_after_tgt_dedup
-                share_tgt_dedup = 100 * num_dup_rows / nrows_after_full_dedup
-
-                msg = self.bundle.get("dataset_diff_target_duplicates").format(
-                    share_tgt_dedup, num_dup_rows, dups_indices
-                )
-                self.logger.warning(msg)
-                if not silent_mode:
-                    print(msg)
-                self.data.drop_duplicates(subset=unique_columns, keep=False, inplace=True)
-                self.logger.info(f"Dataset shape after clean invalid target duplicates: {self.data.shape}")
-
     def __convert_bools(self):
         """Convert bool columns True -> 1, False -> 0"""
         # self.logger.info("Converting bool to int")
@@ -280,12 +239,10 @@ class Dataset:  # (pd.DataFrame):
     def __correct_decimal_comma(self):
         """Check DataSet for decimal commas and fix them"""
         # self.logger.info("Correct decimal commas")
-        tmp = self.data.head(10)
-        # all columns with sep="," will have dtype == 'object', i.e string
-        # sep="." will be casted to numeric automatically
-        cls_to_check = [i for i in tmp.columns if is_string_dtype(tmp[i])]
-        for col in cls_to_check:
-            if tmp[col].astype("string").str.match("^[0-9]+,[0-9]*$").any():
+        columns_to_fix = find_numbers_with_decimal_comma(self.data)
+        if len(columns_to_fix) > 0:
+            self.logger.warning(f"Convert strings with decimal comma to float: {columns_to_fix}")
+            for col in columns_to_fix:
                 self.data[col] = self.data[col].astype("string").str.replace(",", ".").astype(np.float64)
 
     @staticmethod
@@ -504,8 +461,10 @@ class Dataset:  # (pd.DataFrame):
             self.task_type == ModelTaskType.BINARY and len(train_segment) > self.MIN_SAMPLE_THRESHOLD
         ):
             count = len(train_segment)
-            target_column = self.etalon_def_checked.get(FileColumnMeaningType.TARGET.value, TARGET)
-            target = train_segment[target_column]
+            min_class_count = count
+            min_class_value = None
+            target_column = self.etalon_def_checked.get(FileColumnMeaningType.TARGET.value, "")
+            target = train_segment[target_column].copy()
             target_classes_count = target.nunique()
 
             if target_classes_count > self.MAX_MULTICLASS_CLASS_COUNT:
@@ -515,9 +474,12 @@ class Dataset:  # (pd.DataFrame):
                 self.logger.warning(msg)
                 raise ValidationError(msg)
 
-            vc = target.value_counts()
-            min_class_value = vc.index[len(vc) - 1]
-            min_class_count = vc[min_class_value]
+            unique_target = target.unique()
+            for v in list(unique_target):  # type: ignore
+                current_class_count = len(train_segment.loc[target == v])
+                if current_class_count < min_class_count:
+                    min_class_count = current_class_count
+                    min_class_value = v
 
             if min_class_count < self.MIN_TARGET_CLASS_ROWS:
                 msg = self.bundle.get("dataset_rarest_class_less_min").format(
@@ -530,19 +492,53 @@ class Dataset:  # (pd.DataFrame):
             min_class_threshold = min_class_percent * count
 
             if min_class_count < min_class_threshold:
-                self.imbalanced = True
-                self.data = balance_undersample(
-                    df=train_segment,
-                    target_column=target_column,
-                    task_type=self.task_type,
-                    random_state=self.random_state,
-                    imbalance_threshold=self.IMBALANCE_THESHOLD,
-                    binary_bootstrap_loops=self.BINARY_BOOTSTRAP_LOOPS,
-                    multiclass_bootstrap_loops=self.MULTICLASS_BOOTSTRAP_LOOPS,
-                    logger=self.logger,
-                    bundle=self.bundle,
-                    warning_counter=self.warning_counter,
+                msg = self.bundle.get("dataset_rarest_class_less_threshold").format(
+                    min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
                 )
+                self.logger.warning(msg)
+                print(msg)
+                self.warning_counter.increment()
+
+                train_segment = train_segment.copy().sort_values(by=SYSTEM_RECORD_ID)
+                if self.task_type == ModelTaskType.MULTICLASS:
+                    # Sort classes by rows count and find 25% quantile class
+                    classes = target.value_counts().index
+                    quantile25_idx = int(0.75 * len(classes))
+                    quantile25_class = classes[quantile25_idx]
+                    count_of_quantile25_class = len(target[target == quantile25_class])
+                    msg = self.bundle.get("imbalance_multiclass").format(quantile25_class, count_of_quantile25_class)
+                    self.logger.warning(msg)
+                    print(msg)
+                    # 25% and lower classes will stay as is. Higher classes will be downsampled
+                    parts = []
+                    for class_idx in range(quantile25_idx):
+                        sampled = train_segment[train_segment[target_column] == classes[class_idx]].sample(
+                            n=count_of_quantile25_class, random_state=self.random_state
+                        )
+                        parts.append(sampled)
+                    for class_idx in range(quantile25_idx, len(classes)):
+                        parts.append(train_segment[train_segment[target_column] == classes[class_idx]])
+                    resampled_data = pd.concat(parts)
+                elif self.task_type == ModelTaskType.BINARY and min_class_count < self.MIN_SAMPLE_THRESHOLD / 2:
+                    minority_class = train_segment[train_segment[target_column] == min_class_value]
+                    majority_class = train_segment[train_segment[target_column] != min_class_value]
+                    sampled_majority_class = majority_class.sample(
+                        n=self.MIN_SAMPLE_THRESHOLD - min_class_count, random_state=self.random_state
+                    )
+                    resampled_data = train_segment[
+                        (train_segment[SYSTEM_RECORD_ID].isin(minority_class[SYSTEM_RECORD_ID]))
+                        | (train_segment[SYSTEM_RECORD_ID].isin(sampled_majority_class[SYSTEM_RECORD_ID]))
+                    ]
+                else:
+                    sampler = RandomUnderSampler(random_state=self.random_state)
+                    X = train_segment[SYSTEM_RECORD_ID]
+                    X = X.to_frame(SYSTEM_RECORD_ID)
+                    new_x, _ = sampler.fit_resample(X, target)  # type: ignore
+                    resampled_data = train_segment[train_segment[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
+
+                self.data = resampled_data
+                self.logger.info(f"Shape after rebalance resampling: {self.data.shape}")
+                self.imbalanced = True
 
         # Resample over fit threshold
         if not self.imbalanced and EVAL_SET_INDEX in self.data.columns:

upgini/features_enricher.py

@@ -16,7 +16,13 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
 import numpy as np
 import pandas as pd
-from pandas.api.types import is_numeric_dtype, is_string_dtype
+from pandas.api.types import (
+    is_bool,
+    is_datetime64_any_dtype,
+    is_numeric_dtype,
+    is_period_dtype,
+    is_string_dtype,
+)
 from scipy.stats import ks_2samp
 from sklearn.base import TransformerMixin
 from sklearn.exceptions import NotFittedError
@@ -54,7 +60,7 @@ from upgini.metrics import EstimatorWrapper, validate_scoring_argument
 from upgini.resource_bundle import ResourceBundle, bundle, get_custom_bundle
 from upgini.search_task import SearchTask
 from upgini.spinner import Spinner
-from upgini.utils import combine_search_keys
+from upgini.utils import combine_search_keys, find_numbers_with_decimal_comma
 from upgini.utils.country_utils import CountrySearchKeyDetector
 from upgini.utils.custom_loss_utils import (
     get_additional_params_custom_loss,
@@ -215,7 +221,7 @@ class FeaturesEnricher(TransformerMixin):
         if logs_enabled:
             self.logger = LoggerFactory().get_logger(endpoint, self._api_key, client_ip, client_visitorid)
         else:
-            self.logger = logging.getLogger("muted_logger")
+            self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
 
         if len(kwargs) > 0:
@@ -1323,16 +1329,52 @@ class FeaturesEnricher(TransformerMixin):
         fitting_X = X_sorted[client_features].copy()
         fitting_enriched_X = enriched_X_sorted[client_features + existing_filtered_enriched_features].copy()
 
-        # Detect and drop high cardinality columns in train
-        columns_with_high_cardinality = FeaturesValidator.find_high_cardinality(fitting_X)
-        columns_with_high_cardinality = [
-            c for c in columns_with_high_cardinality if c not in (self.generate_features or [])
+        # Don't do this because one hot encoded client features will be removed
+        # # Detect and drop high cardinality columns in train
+        # columns_with_high_cardinality = FeaturesValidator.find_high_cardinality(fitting_X)
+        # columns_with_high_cardinality = [
+        #     c for c in columns_with_high_cardinality if c not in (self.generate_features or [])
+        # ]
+        # if len(columns_with_high_cardinality) > 0:
+        #     self.logger.warning(
+        #         f"High cardinality columns {columns_with_high_cardinality} will be dropped for metrics calculation"
+        #     )
+        #     fitting_X = fitting_X.drop(columns=columns_with_high_cardinality, errors="ignore")
+        #     fitting_enriched_X = fitting_enriched_X.drop(columns=columns_with_high_cardinality, errors="ignore")
+
+        # Detect and drop constant columns
+        constant_columns = FeaturesValidator.find_constant_features(fitting_X)
+        if len(constant_columns) > 0:
+            self.logger.warning(f"Constant columns {constant_columns} will be dropped for metrics calculation")
+            fitting_X = fitting_X.drop(columns=constant_columns, errors="ignore")
+            fitting_enriched_X = fitting_enriched_X.drop(columns=constant_columns, errors="ignore")
+
+        # Remove datetime features
+        datetime_features = [
+            f for f in fitting_X.columns if is_datetime64_any_dtype(fitting_X[f]) or is_period_dtype(fitting_X[f])
         ]
-        self.logger.info(
-            f"Columns {columns_with_high_cardinality} will be dropped for metrics calculation due to high cardinality"
-        )
-        fitting_X = fitting_X.drop(columns=columns_with_high_cardinality, errors="ignore")
-        fitting_enriched_X = fitting_enriched_X.drop(columns=columns_with_high_cardinality, errors="ignore")
+        if len(datetime_features) > 0:
+            self.logger.warning(self.bundle.get("dataset_date_features").format(datetime_features))
+            fitting_X = fitting_X.drop(columns=datetime_features, errors="ignore")
+            fitting_enriched_X = fitting_enriched_X.drop(columns=datetime_features, errors="ignore")
+
+        bool_columns = []
+        for col in fitting_X.columns:
+            if is_bool(fitting_X[col]):
+                bool_columns.append(col)
+                fitting_X[col] = fitting_X[col].astype(str)
+                fitting_enriched_X[col] = fitting_enriched_X[col].astype(str)
+        if len(bool_columns) > 0:
+            self.logger.warning(f"Bool columns {bool_columns} was converted to string for metrics calculation")
+
+        decimal_columns_to_fix = find_numbers_with_decimal_comma(fitting_X)
+        if len(decimal_columns_to_fix) > 0:
+            self.logger.warning(f"Convert strings with decimal comma to float: {decimal_columns_to_fix}")
+            for col in decimal_columns_to_fix:
+                fitting_X[col] = fitting_X[col].astype("string").str.replace(",", ".").astype(np.float64)
+                fitting_enriched_X[col] = (
+                    fitting_enriched_X[col].astype("string").str.replace(",", ".").astype(np.float64)
+                )
 
         fitting_eval_set_dict = dict()
         for idx, eval_tuple in eval_set_sampled_dict.items():
@@ -1346,11 +1388,31 @@ class FeaturesEnricher(TransformerMixin):
                 client_features + existing_filtered_enriched_features
             ].copy()
 
-            # Drop high cardinality columns in eval set
-            fitting_eval_X = fitting_eval_X.drop(columns=columns_with_high_cardinality, errors="ignore")
-            fitting_enriched_eval_X = fitting_enriched_eval_X.drop(
-                columns=columns_with_high_cardinality, errors="ignore"
-            )
+            # # Drop high cardinality features in eval set
+            # if len(columns_with_high_cardinality) > 0:
+            #     fitting_eval_X = fitting_eval_X.drop(columns=columns_with_high_cardinality, errors="ignore")
+            #     fitting_enriched_eval_X = fitting_enriched_eval_X.drop(
+            #         columns=columns_with_high_cardinality, errors="ignore"
+            #     )
+            # Drop constant features in eval_set
+            if len(constant_columns) > 0:
+                fitting_eval_X = fitting_eval_X.drop(columns=constant_columns, errors="ignore")
+                fitting_enriched_eval_X = fitting_enriched_eval_X.drop(columns=constant_columns, errors="ignore")
+            # Drop datetime features in eval_set
+            if len(datetime_features) > 0:
+                fitting_eval_X = fitting_eval_X.drop(columns=datetime_features, errors="ignore")
+                fitting_enriched_eval_X = fitting_enriched_eval_X.drop(columns=datetime_features, errors="ignore")
+            # Convert bool to string in eval_set
+            if len(bool_columns) > 0:
+                fitting_eval_X[col] = fitting_eval_X[col].astype(str)
+                fitting_enriched_eval_X[col] = fitting_enriched_eval_X[col].astype(str)
+            # Correct string features with decimal commas
+            if len(decimal_columns_to_fix) > 0:
+                for col in decimal_columns_to_fix:
+                    fitting_eval_X[col] = fitting_eval_X[col].astype("string").str.replace(",", ".").astype(np.float64)
+                    fitting_enriched_eval_X[col] = (
+                        fitting_enriched_eval_X[col].astype("string").str.replace(",", ".").astype(np.float64)
+                    )
 
             fitting_eval_set_dict[idx] = (
                 fitting_eval_X,
@@ -1398,6 +1460,7 @@ class FeaturesEnricher(TransformerMixin):
         elif len(self.feature_importances_) == 0:
             self.logger.info("No external features selected. So use only input datasets for metrics calculation")
             return self.__sample_only_input(validated_X, validated_y, eval_set, is_demo_dataset)
+        # TODO save and check if dataset was deduplicated - use imbalance branch for such case
         elif not self.imbalanced and not exclude_features_sources and is_input_same_as_fit:
             self.logger.info("Dataset is not imbalanced, so use enriched_X from fit")
             return self.__sample_balanced(eval_set, trace_id, remove_outliers_calc_metrics)
@@ -1438,6 +1501,8 @@ class FeaturesEnricher(TransformerMixin):
                 eval_xy[EVAL_SET_INDEX] = idx + 1
                 df = pd.concat([df, eval_xy])
 
+        df = clean_full_duplicates(df, logger=self.logger, silent=True, bundle=self.bundle)
+
         num_samples = _num_samples(df)
         sample_threshold, sample_rows = (
             (Dataset.FIT_SAMPLE_WITH_EVAL_SET_THRESHOLD, Dataset.FIT_SAMPLE_WITH_EVAL_SET_ROWS)
@@ -1561,14 +1626,7 @@ class FeaturesEnricher(TransformerMixin):
                 eval_df_with_index[EVAL_SET_INDEX] = idx + 1
                 df = pd.concat([df, eval_df_with_index])
 
-            _, df = remove_fintech_duplicates(
-                df,
-                self.search_keys,
-                date_format=self.date_format,
-                logger=self.logger,
-                silent=True,
-                bundle=self.bundle,
-            )
+            df = clean_full_duplicates(df, logger=self.logger, silent=True, bundle=self.bundle)
 
             # downsample if need to eval_set threshold
             num_samples = _num_samples(df)
@@ -1653,9 +1711,7 @@ class FeaturesEnricher(TransformerMixin):
 
         self.__cached_sampled_datasets = (X_sampled, y_sampled, enriched_X, eval_set_sampled_dict, self.search_keys)
 
-        return self.__mk_sampled_data_tuple(
-            X_sampled, y_sampled, enriched_X, eval_set_sampled_dict, self.search_keys
-        )
+        return self.__mk_sampled_data_tuple(X_sampled, y_sampled, enriched_X, eval_set_sampled_dict, self.search_keys)
 
     def __mk_sampled_data_tuple(
         self,
@@ -2179,11 +2235,10 @@ class FeaturesEnricher(TransformerMixin):
 
         df = self.__add_country_code(df, self.fit_search_keys)
 
-        need_full_defuplication, df = remove_fintech_duplicates(
+        df = remove_fintech_duplicates(
             df, self.fit_search_keys, date_format=self.date_format, logger=self.logger, bundle=self.bundle
         )
-        if need_full_defuplication:
-            df = clean_full_duplicates(df, self.logger, bundle=self.bundle)
+        df = clean_full_duplicates(df, self.logger, bundle=self.bundle)
 
         date_column = self._get_date_column(self.fit_search_keys)
         self.__adjust_cv(df, date_column, model_task_type)
@@ -2806,8 +2861,9 @@ class FeaturesEnricher(TransformerMixin):
         # save original order or rows
         original_index_name = df.index.name
         index_name = df.index.name or DEFAULT_INDEX
-        df = df.reset_index().reset_index(drop=True)
-        df = df.rename(columns={index_name: ORIGINAL_INDEX})
+        original_order_name = "original_order"
+        df = df.reset_index().rename(columns={index_name: ORIGINAL_INDEX})
+        df = df.reset_index().rename(columns={DEFAULT_INDEX: original_order_name})
 
         # order by date and idempotent order by other keys
         if self.cv not in [CVType.time_series, CVType.blocked_time_series]:
@@ -2847,7 +2903,7 @@ class FeaturesEnricher(TransformerMixin):
         # return original order
         df = df.set_index(ORIGINAL_INDEX)
         df.index.name = original_index_name
-        # df = df.sort_index()
+        df = df.sort_values(by=original_order_name).drop(columns=original_order_name)
 
         meaning_types[SYSTEM_RECORD_ID] = FileColumnMeaningType.SYSTEM_RECORD_ID
         return df
@@ -2966,6 +3022,7 @@ class FeaturesEnricher(TransformerMixin):
         return result_train, result_eval_sets
 
     def __prepare_feature_importances(self, trace_id: str, x_columns: List[str], silent=False):
+        llm_source = "LLM with external data augmentation"
         if self._search_task is None:
             raise NotFittedError(self.bundle.get("transform_unfitted_enricher"))
         features_meta = self._search_task.get_all_features_metadata_v2()
@@ -2990,6 +3047,20 @@ class FeaturesEnricher(TransformerMixin):
         def list_or_single(lst: List[str], single: str):
             return lst or ([single] if single else [])
 
+        def to_anchor(link: str, value: str) -> str:
+            if not value:
+                return ""
+            elif not link:
+                return value
+            elif value == llm_source:
+                return value
+            else:
+                return f"<a href='{link}' target='_blank' rel='noopener noreferrer'>{value}</a>"
+
+        def make_links(names: List[str], links: List[str]):
+            all_links = [to_anchor(link, name) for name, link in itertools.zip_longest(names, links)]
+            return ",".join(all_links)
+
         features_meta.sort(key=lambda m: (-m.shap_value, m.name))
         for feature_meta in features_meta:
             if feature_meta.name in original_names_dict.keys():
@@ -3015,18 +3086,6 @@ class FeaturesEnricher(TransformerMixin):
                 if len(feature_sample) > 30:
                     feature_sample = feature_sample[:30] + "..."
 
-            def to_anchor(link: str, value: str) -> str:
-                if not value:
-                    return ""
-                elif not link:
-                    return value
-                else:
-                    return f"<a href='{link}' target='_blank' rel='noopener noreferrer'>{value}</a>"
-
-            def make_links(names: List[str], links: List[str]):
-                all_links = [to_anchor(link, name) for name, link in itertools.zip_longest(names, links)]
-                return ",".join(all_links)
-
             internal_provider = feature_meta.data_provider or "Upgini"
             providers = list_or_single(feature_meta.data_providers, feature_meta.data_provider)
             provider_links = list_or_single(feature_meta.data_provider_links, feature_meta.data_provider_link)
@@ -3036,7 +3095,7 @@ class FeaturesEnricher(TransformerMixin):
                 provider = to_anchor("https://upgini.com", "Upgini")
 
             internal_source = feature_meta.data_source or (
-                "LLM with external data augmentation"
+                llm_source
                 if not feature_meta.name.endswith("_country") and not feature_meta.name.endswith("_postal_code")
                 else ""
             )

upgini/search_task.py

@@ -57,7 +57,7 @@ class SearchTask:
         if logger is not None:
             self.logger = logger
         else:
-            self.logger = logging.getLogger("muted_logger")
+            self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
         self.provider_metadata_v2: Optional[List[ProviderTaskMetadataV2]] = None
         self.unused_features_for_generation: Optional[List[str]] = None

upgini/utils/__init__.py

@@ -1,6 +1,9 @@
 import itertools
 from typing import List, Tuple
 
+import pandas as pd
+from pandas.api.types import is_string_dtype
+
 
 def combine_search_keys(search_keys: List[str]) -> List[Tuple[str]]:
     combined_search_keys = []
@@ -8,3 +11,14 @@ def combine_search_keys(search_keys: List[str]) -> List[Tuple[str]]:
         for subset in itertools.combinations(search_keys, L):
             combined_search_keys.append(subset)
     return combined_search_keys
+
+
+def find_numbers_with_decimal_comma(df: pd.DataFrame) -> pd.DataFrame:
+    tmp = df.head(10)
+    # all columns with sep="," will have dtype == 'object', i.e string
+    # sep="." will be casted to numeric automatically
+    return [
+        col
+        for col in tmp.columns
+        if is_string_dtype(tmp[col]) and tmp[col].astype("string").str.match("^[0-9]+,[0-9]*$").any()
+    ]

upgini/utils/datetime_utils.py

@@ -31,7 +31,7 @@ class DateTimeSearchKeyConverter:
         if logger is not None:
             self.logger = logger
         else:
-            self.logger = logging.getLogger("muted_logger")
+            self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
         self.generated_features: List[str] = []
         self.bundle = bundle or get_custom_bundle()

upgini/utils/deduplicate_utils.py

@@ -1,9 +1,9 @@
 from logging import Logger
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Union
 
 import pandas as pd
 
-from upgini.metadata import SYSTEM_RECORD_ID, TARGET, ModelTaskType, SearchKey
+from upgini.metadata import SORT_ID, SYSTEM_RECORD_ID, TARGET, ModelTaskType, SearchKey
 from upgini.resource_bundle import ResourceBundle
 from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
 from upgini.utils.target_utils import define_task
@@ -16,17 +16,15 @@ def remove_fintech_duplicates(
     logger: Optional[Logger] = None,
     silent=False,
     bundle: ResourceBundle = None,
-) -> Tuple[bool, pd.DataFrame]:
+) -> pd.DataFrame:
     # Base checks
-    need_full_deduplication = True
-
     date_col = _get_column_by_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
     if define_task(df[TARGET], date_col is not None, silent=True) != ModelTaskType.BINARY:
-        return need_full_deduplication, df
+        return df
 
     date_col = _get_column_by_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
     if date_col is None:
-        return need_full_deduplication, df
+        return df
 
     personal_cols = []
     phone_col = _get_column_by_key(search_keys, SearchKey.PHONE)
@@ -39,13 +37,13 @@ def remove_fintech_duplicates(
     if hem_col:
         personal_cols.append(hem_col)
     if len(personal_cols) == 0:
-        return need_full_deduplication, df
+        return df
 
     sub_df = df[personal_cols + [date_col, TARGET]]
 
     # Fast check for duplicates by personal keys
     if not sub_df[personal_cols].duplicated().any():
-        return need_full_deduplication, df
+        return df
 
     grouped_by_personal_cols = sub_df.groupby(personal_cols, group_keys=False)
 
@@ -54,21 +52,19 @@ def remove_fintech_duplicates(
     total = len(uniques)
     diff_dates = len(uniques[uniques > 1])
     if diff_dates / total >= 0.6:
-        return need_full_deduplication, df
+        return df
 
     # Additional checks
 
-    need_full_deduplication = False
-
     duplicates = sub_df.duplicated(personal_cols, keep=False)
     duplicate_rows = sub_df[duplicates]
     if len(duplicate_rows) == 0:
-        return need_full_deduplication, df
+        return df
 
     # if there is no different target values in personal keys duplicate rows
     nonunique_target_groups = grouped_by_personal_cols[TARGET].nunique() > 1
     if nonunique_target_groups.sum() == 0:
-        return need_full_deduplication, df
+        return df
 
     def has_diff_target_within_60_days(rows):
         rows = rows.sort_values(by=date_col)
@@ -96,7 +92,7 @@ def remove_fintech_duplicates(
         df = df[~df.index.isin(rows_to_remove.index)]
         logger.info(f"Dataset shape after clean fintech duplicates: {df.shape}")
 
-    return need_full_deduplication, df
+    return df
 
 
 def clean_full_duplicates(
@@ -109,8 +105,8 @@ def clean_full_duplicates(
     unique_columns = df.columns.tolist()
     if SYSTEM_RECORD_ID in unique_columns:
         unique_columns.remove(SYSTEM_RECORD_ID)
-    if "sort_id" in unique_columns:
-        unique_columns.remove("sort_id")
+    if SORT_ID in unique_columns:
+        unique_columns.remove(SORT_ID)
     logger.info(f"Dataset shape before clean duplicates: {df.shape}")
     df = df.drop_duplicates(subset=unique_columns)
     logger.info(f"Dataset shape after clean duplicates: {df.shape}")

upgini/utils/features_validator.py

@@ -55,7 +55,7 @@ class FeaturesValidator:
         return empty_or_constant_features
 
     @staticmethod
-    def find_high_cardinality(df: pd.DataFrame):
+    def find_high_cardinality(df: pd.DataFrame) -> List[str]:
         # Remove high cardinality columns
         row_count = df.shape[0]
         return [
@@ -63,3 +63,11 @@ class FeaturesValidator:
             for i in df
             if (is_string_dtype(df[i]) or is_integer_dtype(df[i])) and (df[i].nunique() / row_count >= 0.9)
         ]
+
+    @staticmethod
+    def find_constant_features(df: pd.DataFrame) -> List[str]:
+        return [
+            i
+            for i in df
+            if df[i].nunique() == 1
+        ]

upgini/utils/target_utils.py

@@ -6,10 +6,8 @@ import pandas as pd
 from pandas.api.types import is_numeric_dtype
 
 from upgini.errors import ValidationError
-from upgini.metadata import SYSTEM_RECORD_ID, ModelTaskType
-from upgini.resource_bundle import ResourceBundle, bundle, get_custom_bundle
-from upgini.sampler.random_under_sampler import RandomUnderSampler
-from upgini.utils.warning_counter import WarningCounter
+from upgini.metadata import ModelTaskType
+from upgini.resource_bundle import bundle
 
 
 def correct_string_target(y: Union[pd.Series, np.ndarray]) -> Union[pd.Series, np.ndarray]:
@@ -74,110 +72,3 @@ def is_int_encoding(unique_values):
     return set(unique_values) == set(range(len(unique_values))) or set(unique_values) == set(
         range(1, len(unique_values) + 1)
     )
-
-
-def balance_undersample(
-    df: pd.DataFrame,
-    target_column: str,
-    task_type: ModelTaskType,
-    random_state: int,
-    imbalance_threshold: int = 0.2,
-    min_sample_threshold: int = 5000,
-    binary_bootstrap_loops: int = 5,
-    multiclass_bootstrap_loops: int = 2,
-    logger: Optional[logging.Logger] = None,
-    bundle: Optional[ResourceBundle] = None,
-    warning_counter: Optional[WarningCounter] = None,
-) -> pd.DataFrame:
-    if logger is None:
-        logger = logging.getLogger("muted_logger")
-        logger.setLevel("FATAL")
-    bundle = bundle or get_custom_bundle()
-    if SYSTEM_RECORD_ID not in df.columns:
-        raise Exception("System record id must be presented for undersampling")
-
-    count = len(df)
-    target = df[target_column].copy()
-    target_classes_count = target.nunique()
-
-    vc = target.value_counts()
-    max_class_value = vc.index[0]
-    min_class_value = vc.index[len(vc) - 1]
-    max_class_count = vc[max_class_value]
-    min_class_count = vc[min_class_value]
-
-    min_class_percent = imbalance_threshold / target_classes_count
-    min_class_threshold = min_class_percent * count
-
-    resampled_data = df
-    df = df.copy().sort_values(by=SYSTEM_RECORD_ID)
-    if task_type == ModelTaskType.MULTICLASS:
-        # Sort classes by rows count and find 25% quantile class
-        classes = vc.index
-        quantile25_idx = int(0.75 * len(classes)) - 1
-        quantile25_class = classes[quantile25_idx]
-        quantile25_class_cnt = vc[quantile25_class]
-
-        if max_class_count > (quantile25_class_cnt * multiclass_bootstrap_loops):
-            msg = bundle.get("imbalance_multiclass").format(quantile25_class, quantile25_class_cnt)
-            logger.warning(msg)
-            print(msg)
-            if warning_counter:
-                warning_counter.increment()
-
-            # 25% and lower classes will stay as is. Higher classes will be downsampled
-            sample_strategy = dict()
-            for class_idx in range(quantile25_idx):
-                # compare class count with count_of_quantile25_class * 2
-                class_value = classes[class_idx]
-                class_count = vc[class_value]
-                sample_strategy[class_value] = min(class_count, quantile25_class_cnt * multiclass_bootstrap_loops)
-            sampler = RandomUnderSampler(
-                sampling_strategy=sample_strategy, random_state=random_state
-            )
-            X = df[SYSTEM_RECORD_ID]
-            X = X.to_frame(SYSTEM_RECORD_ID)
-            new_x, _ = sampler.fit_resample(X, target)  # type: ignore
-
-            resampled_data = df[df[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
-    elif len(df) > min_sample_threshold and min_class_count < min_sample_threshold / 2:
-        msg = bundle.get("dataset_rarest_class_less_threshold").format(
-            min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
-        )
-        logger.warning(msg)
-        print(msg)
-        if warning_counter:
-            warning_counter.increment()
-
-        # fill up to min_sample_threshold by majority class
-        minority_class = df[df[target_column] == min_class_value]
-        majority_class = df[df[target_column] != min_class_value]
-        sample_size = min(len(majority_class, min_sample_threshold - min_class_count))
-        sampled_majority_class = majority_class.sample(
-            n=sample_size, random_state=random_state
-        )
-        resampled_data = df[
-            (df[SYSTEM_RECORD_ID].isin(minority_class[SYSTEM_RECORD_ID]))
-            | (df[SYSTEM_RECORD_ID].isin(sampled_majority_class[SYSTEM_RECORD_ID]))
-        ]
-
-    elif max_class_count > min_class_count * binary_bootstrap_loops:
-        msg = bundle.get("dataset_rarest_class_less_threshold").format(
-            min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
-        )
-        logger.warning(msg)
-        print(msg)
-        if warning_counter:
-            warning_counter.increment()
-
-        sampler = RandomUnderSampler(
-            sampling_strategy={max_class_value: binary_bootstrap_loops * min_class_count}, random_state=random_state
-        )
-        X = df[SYSTEM_RECORD_ID]
-        X = X.to_frame(SYSTEM_RECORD_ID)
-        new_x, _ = sampler.fit_resample(X, target)  # type: ignore
-
-        resampled_data = df[df[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
-
-    logger.info(f"Shape after rebalance resampling: {resampled_data}")
-    return resampled_data

{upgini-1.1.252a5.dist-info → upgini-1.1.253.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.252a5
+Version: 1.1.253
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.252a5.dist-info → upgini-1.1.253.dist-info}/RECORD

@@ -1,13 +1,13 @@
 upgini/__init__.py,sha256=asENHgEVHQBIkV-e_0IhE_ZWqkCG6398U3ZLrNzAH6k,407
 upgini/ads.py,sha256=mre6xn44wcC_fg63iLT_kTh4mViZqR9AKRJZAtpQz8Y,2592
-upgini/dataset.py,sha256=jNYcD86UhmC-3in51bnX51uoFFgXo0gfSvuHxag1VyE,47816
+upgini/dataset.py,sha256=tLa0aEcT7XwVJz1AawXCIEj3vxsSBi-geKicuYpRIMw,48196
 upgini/errors.py,sha256=pdzQl3MKuK52yvncxMWMRWeSIOGhUFzpQoszoRFBOk0,958
-upgini/features_enricher.py,sha256=VQUrzZw4uPNw0PiunE7GjOebELOlelvaOcYh0aL9yeY,168016
+upgini/features_enricher.py,sha256=dP6Oyhi4erESEGlVFA_j67lqhNqNvbkfM4FGpE_WaTU,171760
 upgini/fingerprint.js,sha256=VygVIQlN1v4NGZfjHqtRogOw8zjTnnMNJg_f7M5iGQU,33442
 upgini/http.py,sha256=eSG4gOpmCGlXmB6KIPNzAG8tRZNUjyYpMeUeHw_2li4,42264
 upgini/metadata.py,sha256=fwVxtkR6Mn4iRoOqV6BfMJvJrx65I3YwZUMbZjhPyOI,9673
 upgini/metrics.py,sha256=LS2MgEKgmn9VEXsKzxv3pBZ-q71mTnpWu6vL8fYgpo4,26727
-upgini/search_task.py,sha256=tmJ17WUxv3J5NWrYUJB_NKdZ792Ifz8Z8UnDXeQnpss,17077
+upgini/search_task.py,sha256=5n4qGJmtu48s0-FHAtF3L5qVLMd1JVW3FJlM8dFbh-s,17063
 upgini/spinner.py,sha256=Dm1dQ5F_z_Ua2odLxZX7OypcOX9tSx_vE5MGaKtUmfw,1118
 upgini/version_validator.py,sha256=rDIncP6BEko4J2F2hUcMOtKm_vZbI4ICWcNcw8hrwM4,1400
 upgini/ads_management/__init__.py,sha256=qzyisOToVRP-tquAJD1PblZhNtMrOB8FiyF9JvfkvgE,50
@@ -21,7 +21,7 @@ upgini/autofe/operand.py,sha256=Rhy7Ky3we-I1Su1--dS4xdsO3K8neV4rqM_Q4xYE4ug,2779
 upgini/autofe/unary.py,sha256=gyMkrx9bfa3o19zS-4JaRlScHrfeZGBsYe7d_6ePT-0,2853
 upgini/autofe/vector.py,sha256=Qk7VmdwURNwVw7fIMEspWEo7HTiyUWCYIqu3hcWQQio,507
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/data_source/data_source_publisher.py,sha256=LZ8iZHGrGVP1c7xIk57zio9OqN3VmBS_S3NW9Xs2pL4,15124
+upgini/data_source/data_source_publisher.py,sha256=ZMNyh1x1S3QkXkA-PTtBQ-sbOiANtNioEQs8VoQ24Lk,15110
 upgini/mdc/__init__.py,sha256=ETDh3JKbrDdPMOECiYLAa8lvKYe68mv4IY6fZa9FimA,1126
 upgini/mdc/context.py,sha256=Sl1S_InKlzzRxYqwJ2k24lawJdCKWgGJ-RIRfvzWJrk,1468
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -34,29 +34,29 @@ upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=CC-DvPbrN7zp5--SVFuUqkVmdWM_5F7R0Do98ETV82U,6421
 upgini/sampler/random_under_sampler.py,sha256=XU4c2swPIFxVXHOPpxgM2bUao0Xm-aoMmd6fKjIuV5s,4068
 upgini/sampler/utils.py,sha256=PYOk3kKSnFlyxcpdtDNLBEEhTB4lO_iP7pQHqeUcmAc,20211
-upgini/utils/__init__.py,sha256=XDL_YTIPiCbd8BPHwAx4HMDfLXP5xH8NFPEI0V9wOr4,336
+upgini/utils/__init__.py,sha256=dQ4-s8-sZ5eOBZ-mH3gEwDHTdI0wI1bUAVgVqUKKPx4,786
 upgini/utils/base_search_key_detector.py,sha256=DGwhXLvc8i5VZWMDr0rncFfV5GEHdsCSnLGon_W9TPs,859
 upgini/utils/blocked_time_series.py,sha256=dMz5ewk3PsoeOrc3lDzInCVPS9u_2XQkV0W6PuMMjPg,3380
 upgini/utils/country_utils.py,sha256=1KXhLSNqkNYVL3on8-zK0Arc_SspUH7AMZvGZICysOU,6462
 upgini/utils/custom_loss_utils.py,sha256=DBslpjWGPt7xTeypt78baR59012SYphbPsO_YLKdilo,3972
 upgini/utils/cv_utils.py,sha256=Tn01RJvpZGZh0PUQUimlBkV-AXwe7s6yjCNFtw352Uc,3525
-upgini/utils/datetime_utils.py,sha256=b8pyNhrC8ni6apsLQivQOiKqu-37pU4EF3nNHPZqiN8,8713
-upgini/utils/deduplicate_utils.py,sha256=GRPwD8bXZNspKvf19W3SrYjqg1qQMDlZD-BDkHnKYyo,6176
+upgini/utils/datetime_utils.py,sha256=P5no4mFgYpEP6oY524ebTKvKc3TBMJzAYpWdj210_Fw,8699
+upgini/utils/deduplicate_utils.py,sha256=ckJrpU8Ruc_vcwIPTopbUjyJuNiseLHNAbQlLfhUCxo,5888
 upgini/utils/display_utils.py,sha256=tiq5sFOfMwkKCjQ7OGdyK_twe0Qdr9F3mzkW1QXSDog,10664
 upgini/utils/email_utils.py,sha256=3CvHXTSzlgLyGsQOXfRYVfFhfPy6OXG4uXOBWRaLfHg,3479
 upgini/utils/fallback_progress_bar.py,sha256=cdbd1XGcWm4Ed4eAqV2_St3z7uC_kkH22gEyrN5ub6M,1090
-upgini/utils/features_validator.py,sha256=iP8muF3PUf_aP9m7O3i3LPMuJPTNbw8rCAWqgvDt_h8,2369
+upgini/utils/features_validator.py,sha256=VexG-9p63ni66Hf9T7dgP4iUAhpXqwo3sgMwBK_eii8,2565
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
 upgini/utils/ip_utils.py,sha256=Zf3F2cnQmOCH09QLQHetpjMFu1PnD0cTmDymn0SnSy8,1672
 upgini/utils/phone_utils.py,sha256=JNSkF8G6mgsN8Czy11pamaJdsY6rBINEMpi7jbVt_RA,408
 upgini/utils/postal_code_utils.py,sha256=_8CR9tBqsPptQsmMUvnrCAmBaMIQSWH3JfJ4ly3x_zs,409
 upgini/utils/progress_bar.py,sha256=iNXyqT3vKCeHpfiG5HHwr7Lk2cTtKViM93Fl8iZnjGc,1564
 upgini/utils/sklearn_ext.py,sha256=fvuTWJ5AnT3ED9KSaQu_yIgW2JR19hFlaGDoVP3k60g,44027
-upgini/utils/target_utils.py,sha256=OzW1dlhW0tQj5FBR-iIIjdpzqIGfGFRoYePppP8yRhw,7204
+upgini/utils/target_utils.py,sha256=DH812qcZ7Pvf9WVVb33fbwQjb1W9h1hXRNCCiG7Y6tI,2563
 upgini/utils/track_info.py,sha256=EPcJ13Jqa17_T0JjM37Ac9kWDz5Zk0GVsIZKutOb8aU,5207
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.1.252a5.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.252a5.dist-info/METADATA,sha256=xynsF_WVgmRp0hcYCqjqvGfF3PcaBXUzzwqKlGX0HbE,48158
-upgini-1.1.252a5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-upgini-1.1.252a5.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
-upgini-1.1.252a5.dist-info/RECORD,,
+upgini-1.1.253.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.253.dist-info/METADATA,sha256=6FwSFP4xzkd9GTHCyToBORKRQEriGSZKJPs1O1ujbcI,48156
+upgini-1.1.253.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+upgini-1.1.253.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
+upgini-1.1.253.dist-info/RECORD,,