upgini 1.2.14__tar.gz → 1.2.14a1__tar.gz

{upgini-1.2.14 → upgini-1.2.14a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.14
+Version: 1.2.14a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.14a1/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.14a1"

{upgini-1.2.14 → upgini-1.2.14a1}/src/upgini/dataset.py

@@ -253,6 +253,7 @@ class Dataset:  # (pd.DataFrame):
             min_class_percent = self.IMBALANCE_THESHOLD / target_classes_count
             min_class_threshold = min_class_percent * count
 
+            # If min class count less than 30% for binary or (60 / classes_count)% for multiclass
             if min_class_count < min_class_threshold:
                 self.imbalanced = True
                 self.data = balance_undersample(

{upgini-1.2.14 → upgini-1.2.14a1}/src/upgini/features_enricher.py

@@ -1577,8 +1577,8 @@ class FeaturesEnricher(TransformerMixin):
             df = generator.generate(df)
             generated_features.extend(generator.generated_features)
 
-        normalizer = Normalizer(self.bundle, self.logger, self.warning_counter)
-        df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
+        normalizer = Normalizer(search_keys, generated_features, self.bundle, self.logger, self.warning_counter)
+        df = normalizer.normalize(df)
         columns_renaming = normalizer.columns_renaming
 
         df = clean_full_duplicates(df, logger=self.logger, silent=True, bundle=self.bundle)
@@ -2017,8 +2017,10 @@ class FeaturesEnricher(TransformerMixin):
                 df = generator.generate(df)
                 generated_features.extend(generator.generated_features)
 
-            normalizer = Normalizer(self.bundle, self.logger, self.warning_counter, silent_mode)
-            df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
+            normalizer = Normalizer(
+                search_keys, generated_features, self.bundle, self.logger, self.warning_counter, silent_mode
+            )
+            df = normalizer.normalize(df)
             columns_renaming = normalizer.columns_renaming
 
             # Don't pass all features in backend on transform
@@ -2447,13 +2449,14 @@ class FeaturesEnricher(TransformerMixin):
         if is_numeric_dtype(df[self.TARGET_NAME]) and has_date:
             self._validate_PSI(df.sort_values(by=maybe_date_column))
 
-        normalizer = Normalizer(self.bundle, self.logger, self.warning_counter)
-        df, self.fit_search_keys, self.fit_generated_features = normalizer.normalize(
-            df, self.fit_search_keys, self.fit_generated_features
-        )
-        self.fit_columns_renaming = normalizer.columns_renaming
+        self.__adjust_cv(df, maybe_date_column, self.model_task_type)
 
-        self.__adjust_cv(df)
+        normalizer = Normalizer(
+            self.fit_search_keys, self.fit_generated_features, self.bundle, self.logger, self.warning_counter
+        )
+        df = normalizer.normalize(df)
+        columns_renaming = normalizer.columns_renaming
+        self.fit_columns_renaming = columns_renaming
 
         df = remove_fintech_duplicates(
             df, self.fit_search_keys, date_format=self.date_format, logger=self.logger, bundle=self.bundle
@@ -2467,7 +2470,7 @@ class FeaturesEnricher(TransformerMixin):
         self.df_with_original_index = df.copy()
         # TODO check maybe need to drop _time column from df_with_original_index
 
-        df, unnest_search_keys = self._explode_multiple_search_keys(df, self.fit_search_keys, self.fit_columns_renaming)
+        df, unnest_search_keys = self._explode_multiple_search_keys(df, self.fit_search_keys, columns_renaming)
 
         # Convert EMAIL to HEM after unnesting to do it only with one column
         email_column = self._get_email_column(self.fit_search_keys)
@@ -2477,7 +2480,7 @@ class FeaturesEnricher(TransformerMixin):
                 email_column,
                 hem_column,
                 self.fit_search_keys,
-                self.fit_columns_renaming,
+                columns_renaming,
                 list(unnest_search_keys.keys()),
                 self.logger,
             )
@@ -2488,7 +2491,7 @@ class FeaturesEnricher(TransformerMixin):
             converter = IpSearchKeyConverter(
                 ip_column,
                 self.fit_search_keys,
-                self.fit_columns_renaming,
+                columns_renaming,
                 list(unnest_search_keys.keys()),
                 self.bundle,
                 self.logger,
@@ -2519,7 +2522,7 @@ class FeaturesEnricher(TransformerMixin):
         features_columns = [c for c in df.columns if c not in non_feature_columns]
 
         features_to_drop = FeaturesValidator(self.logger).validate(
-            df, features_columns, self.generate_features, self.warning_counter, self.fit_columns_renaming
+            df, features_columns, self.generate_features, self.warning_counter, columns_renaming
         )
         self.fit_dropped_features.update(features_to_drop)
         df = df.drop(columns=features_to_drop)
@@ -2560,7 +2563,7 @@ class FeaturesEnricher(TransformerMixin):
             rest_client=self.rest_client,
             logger=self.logger,
         )
-        dataset.columns_renaming = self.fit_columns_renaming
+        dataset.columns_renaming = columns_renaming
 
         self.passed_features = [
             column for column, meaning_type in meaning_types.items() if meaning_type == FileColumnMeaningType.FEATURE
@@ -2707,24 +2710,24 @@ class FeaturesEnricher(TransformerMixin):
         if not self.warning_counter.has_warnings():
             self.__display_support_link(self.bundle.get("all_ok_community_invite"))
 
-    def __adjust_cv(self, df: pd.DataFrame):
-        date_column = SearchKey.find_key(self.fit_search_keys, [SearchKey.DATE, SearchKey.DATETIME])
+    def __adjust_cv(self, df: pd.DataFrame, date_column: pd.Series, model_task_type: ModelTaskType):
         # Check Multivariate time series
         if (
             self.cv is None
             and date_column
-            and self.model_task_type == ModelTaskType.REGRESSION
+            and model_task_type == ModelTaskType.REGRESSION
             and len({SearchKey.PHONE, SearchKey.EMAIL, SearchKey.HEM}.intersection(self.fit_search_keys.keys())) == 0
             and is_blocked_time_series(df, date_column, list(self.fit_search_keys.keys()) + [TARGET])
         ):
             msg = self.bundle.get("multivariate_timeseries_detected")
             self.__override_cv(CVType.blocked_time_series, msg, print_warning=False)
-        elif self.cv is None and self.model_task_type != ModelTaskType.REGRESSION:
+        elif (
+            self.cv is None
+            and model_task_type != ModelTaskType.REGRESSION
+            and self._get_group_columns(df, self.fit_search_keys)
+        ):
             msg = self.bundle.get("group_k_fold_in_classification")
             self.__override_cv(CVType.group_k_fold, msg, print_warning=self.cv is not None)
-            group_columns = self._get_group_columns(df, self.fit_search_keys)
-            self.runtime_parameters.properties["cv_params.group_columns"] = ",".join(group_columns)
-            self.runtime_parameters.properties["cv_params.shuffle_kfold"] = "True"
 
     def __override_cv(self, cv: CVType, msg: str, print_warning: bool = True):
         if print_warning:

{upgini-1.2.14 → upgini-1.2.14a1}/src/upgini/normalizer/normalize_utils.py

@@ -1,6 +1,6 @@
 import hashlib
 from logging import Logger, getLogger
-from typing import Dict, List, Tuple
+from typing import Dict, List
 
 import numpy as np
 import pandas as pd
@@ -35,25 +35,22 @@ class Normalizer:
 
     def __init__(
         self,
+        search_keys: Dict[str, SearchKey],
+        generated_features: List[str],
         bundle: ResourceBundle = None,
         logger: Logger = None,
         warnings_counter: WarningCounter = None,
         silent_mode=False,
     ):
+        self.search_keys = search_keys
+        self.generated_features = generated_features
         self.bundle = bundle or get_custom_bundle()
         self.logger = logger or getLogger()
         self.warnings_counter = warnings_counter or WarningCounter()
         self.silent_mode = silent_mode
         self.columns_renaming = {}
-        self.search_keys = {}
-        self.generated_features = []
-
-    def normalize(
-        self, df: pd.DataFrame, search_keys: Dict[str, SearchKey], generated_features: List[str]
-    ) -> Tuple[pd.DataFrame, Dict[str, SearchKey], List[str]]:
-        self.search_keys = search_keys.copy()
-        self.generated_features = generated_features.copy()
 
+    def normalize(self, df: pd.DataFrame) -> pd.DataFrame:
         df = df.copy()
         df = self._rename_columns(df)
 
@@ -71,25 +68,21 @@ class Normalizer:
 
         df = self.__convert_features_types(df)
 
-        return df, self.search_keys, self.generated_features
+        return df
 
     def _rename_columns(self, df: pd.DataFrame):
         # logger.info("Replace restricted symbols in column names")
         new_columns = []
         dup_counter = 0
         for column in df.columns:
-            if (
-                column
-                in [
-                    TARGET,
-                    EVAL_SET_INDEX,
-                    SYSTEM_RECORD_ID,
-                    ENTITY_SYSTEM_RECORD_ID,
-                    SEARCH_KEY_UNNEST,
-                    DateTimeSearchKeyConverter.DATETIME_COL,
-                ]
-                + self.generated_features
-            ):
+            if column in [
+                TARGET,
+                EVAL_SET_INDEX,
+                SYSTEM_RECORD_ID,
+                ENTITY_SYSTEM_RECORD_ID,
+                SEARCH_KEY_UNNEST,
+                DateTimeSearchKeyConverter.DATETIME_COL,
+            ] + self.generated_features:
                 self.columns_renaming[column] = column
                 new_columns.append(column)
                 continue

{upgini-1.2.14 → upgini-1.2.14a1}/src/upgini/utils/target_utils.py

@@ -150,30 +150,34 @@ def balance_undersample(
         # fill up to min_sample_threshold by majority class
         minority_class = df[df[target_column] == min_class_value]
         majority_class = df[df[target_column] != min_class_value]
-        sample_size = min(len(majority_class), min_sample_threshold - min_class_count)
+        # sample_size = min(len(majority_class), min_sample_threshold - min_class_count)
+        sample_size = min(
+            max_class_count,
+            binary_bootstrap_loops * (min_class_count + max(min_sample_threshold - 2 * min_class_count, 0)),
+        )
         sampled_majority_class = majority_class.sample(n=sample_size, random_state=random_state)
         resampled_data = df[
             (df[SYSTEM_RECORD_ID].isin(minority_class[SYSTEM_RECORD_ID]))
             | (df[SYSTEM_RECORD_ID].isin(sampled_majority_class[SYSTEM_RECORD_ID]))
         ]
 
-    elif max_class_count > min_class_count * binary_bootstrap_loops:
-        msg = bundle.get("dataset_rarest_class_less_threshold").format(
-            min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
-        )
-        logger.warning(msg)
-        print(msg)
-        if warning_counter:
-            warning_counter.increment()
-
-        sampler = RandomUnderSampler(
-            sampling_strategy={max_class_value: binary_bootstrap_loops * min_class_count}, random_state=random_state
-        )
-        X = df[SYSTEM_RECORD_ID]
-        X = X.to_frame(SYSTEM_RECORD_ID)
-        new_x, _ = sampler.fit_resample(X, target)  # type: ignore
-
-        resampled_data = df[df[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
+    # elif max_class_count > min_class_count * binary_bootstrap_loops:
+    #     msg = bundle.get("dataset_rarest_class_less_threshold").format(
+    #         min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
+    #     )
+    #     logger.warning(msg)
+    #     print(msg)
+    #     if warning_counter:
+    #         warning_counter.increment()
+
+    #     sampler = RandomUnderSampler(
+    #         sampling_strategy={max_class_value: binary_bootstrap_loops * min_class_count}, random_state=random_state
+    #     )
+    #     X = df[SYSTEM_RECORD_ID]
+    #     X = X.to_frame(SYSTEM_RECORD_ID)
+    #     new_x, _ = sampler.fit_resample(X, target)  # type: ignore
+
+    #     resampled_data = df[df[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
 
     logger.info(f"Shape after rebalance resampling: {resampled_data}")
     return resampled_data

upgini-1.2.14/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.14"