upgini 1.2.14__py3-none-any.whl → 1.2.14a1__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.14"
+__version__ = "1.2.14a1"

upgini/dataset.py

@@ -253,6 +253,7 @@ class Dataset:  # (pd.DataFrame):
             min_class_percent = self.IMBALANCE_THESHOLD / target_classes_count
             min_class_threshold = min_class_percent * count
 
+            # If min class count less than 30% for binary or (60 / classes_count)% for multiclass
             if min_class_count < min_class_threshold:
                 self.imbalanced = True
                 self.data = balance_undersample(

upgini/features_enricher.py

@@ -1577,8 +1577,8 @@ class FeaturesEnricher(TransformerMixin):
             df = generator.generate(df)
             generated_features.extend(generator.generated_features)
 
-        normalizer = Normalizer(self.bundle, self.logger, self.warning_counter)
-        df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
+        normalizer = Normalizer(search_keys, generated_features, self.bundle, self.logger, self.warning_counter)
+        df = normalizer.normalize(df)
         columns_renaming = normalizer.columns_renaming
 
         df = clean_full_duplicates(df, logger=self.logger, silent=True, bundle=self.bundle)
@@ -2017,8 +2017,10 @@ class FeaturesEnricher(TransformerMixin):
                 df = generator.generate(df)
                 generated_features.extend(generator.generated_features)
 
-            normalizer = Normalizer(self.bundle, self.logger, self.warning_counter, silent_mode)
-            df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
+            normalizer = Normalizer(
+                search_keys, generated_features, self.bundle, self.logger, self.warning_counter, silent_mode
+            )
+            df = normalizer.normalize(df)
             columns_renaming = normalizer.columns_renaming
 
             # Don't pass all features in backend on transform
@@ -2447,13 +2449,14 @@ class FeaturesEnricher(TransformerMixin):
         if is_numeric_dtype(df[self.TARGET_NAME]) and has_date:
             self._validate_PSI(df.sort_values(by=maybe_date_column))
 
-        normalizer = Normalizer(self.bundle, self.logger, self.warning_counter)
-        df, self.fit_search_keys, self.fit_generated_features = normalizer.normalize(
-            df, self.fit_search_keys, self.fit_generated_features
-        )
-        self.fit_columns_renaming = normalizer.columns_renaming
+        self.__adjust_cv(df, maybe_date_column, self.model_task_type)
 
-        self.__adjust_cv(df)
+        normalizer = Normalizer(
+            self.fit_search_keys, self.fit_generated_features, self.bundle, self.logger, self.warning_counter
+        )
+        df = normalizer.normalize(df)
+        columns_renaming = normalizer.columns_renaming
+        self.fit_columns_renaming = columns_renaming
 
         df = remove_fintech_duplicates(
             df, self.fit_search_keys, date_format=self.date_format, logger=self.logger, bundle=self.bundle
@@ -2467,7 +2470,7 @@ class FeaturesEnricher(TransformerMixin):
         self.df_with_original_index = df.copy()
         # TODO check maybe need to drop _time column from df_with_original_index
 
-        df, unnest_search_keys = self._explode_multiple_search_keys(df, self.fit_search_keys, self.fit_columns_renaming)
+        df, unnest_search_keys = self._explode_multiple_search_keys(df, self.fit_search_keys, columns_renaming)
 
         # Convert EMAIL to HEM after unnesting to do it only with one column
         email_column = self._get_email_column(self.fit_search_keys)
@@ -2477,7 +2480,7 @@ class FeaturesEnricher(TransformerMixin):
                 email_column,
                 hem_column,
                 self.fit_search_keys,
-                self.fit_columns_renaming,
+                columns_renaming,
                 list(unnest_search_keys.keys()),
                 self.logger,
             )
@@ -2488,7 +2491,7 @@ class FeaturesEnricher(TransformerMixin):
             converter = IpSearchKeyConverter(
                 ip_column,
                 self.fit_search_keys,
-                self.fit_columns_renaming,
+                columns_renaming,
                 list(unnest_search_keys.keys()),
                 self.bundle,
                 self.logger,
@@ -2519,7 +2522,7 @@ class FeaturesEnricher(TransformerMixin):
         features_columns = [c for c in df.columns if c not in non_feature_columns]
 
         features_to_drop = FeaturesValidator(self.logger).validate(
-            df, features_columns, self.generate_features, self.warning_counter, self.fit_columns_renaming
+            df, features_columns, self.generate_features, self.warning_counter, columns_renaming
         )
         self.fit_dropped_features.update(features_to_drop)
         df = df.drop(columns=features_to_drop)
@@ -2560,7 +2563,7 @@ class FeaturesEnricher(TransformerMixin):
             rest_client=self.rest_client,
             logger=self.logger,
         )
-        dataset.columns_renaming = self.fit_columns_renaming
+        dataset.columns_renaming = columns_renaming
 
         self.passed_features = [
             column for column, meaning_type in meaning_types.items() if meaning_type == FileColumnMeaningType.FEATURE
@@ -2707,24 +2710,24 @@ class FeaturesEnricher(TransformerMixin):
         if not self.warning_counter.has_warnings():
             self.__display_support_link(self.bundle.get("all_ok_community_invite"))
 
-    def __adjust_cv(self, df: pd.DataFrame):
-        date_column = SearchKey.find_key(self.fit_search_keys, [SearchKey.DATE, SearchKey.DATETIME])
+    def __adjust_cv(self, df: pd.DataFrame, date_column: pd.Series, model_task_type: ModelTaskType):
         # Check Multivariate time series
         if (
             self.cv is None
             and date_column
-            and self.model_task_type == ModelTaskType.REGRESSION
+            and model_task_type == ModelTaskType.REGRESSION
             and len({SearchKey.PHONE, SearchKey.EMAIL, SearchKey.HEM}.intersection(self.fit_search_keys.keys())) == 0
             and is_blocked_time_series(df, date_column, list(self.fit_search_keys.keys()) + [TARGET])
         ):
             msg = self.bundle.get("multivariate_timeseries_detected")
             self.__override_cv(CVType.blocked_time_series, msg, print_warning=False)
-        elif self.cv is None and self.model_task_type != ModelTaskType.REGRESSION:
+        elif (
+            self.cv is None
+            and model_task_type != ModelTaskType.REGRESSION
+            and self._get_group_columns(df, self.fit_search_keys)
+        ):
             msg = self.bundle.get("group_k_fold_in_classification")
             self.__override_cv(CVType.group_k_fold, msg, print_warning=self.cv is not None)
-            group_columns = self._get_group_columns(df, self.fit_search_keys)
-            self.runtime_parameters.properties["cv_params.group_columns"] = ",".join(group_columns)
-            self.runtime_parameters.properties["cv_params.shuffle_kfold"] = "True"
 
     def __override_cv(self, cv: CVType, msg: str, print_warning: bool = True):
         if print_warning:

upgini/normalizer/normalize_utils.py

@@ -1,6 +1,6 @@
 import hashlib
 from logging import Logger, getLogger
-from typing import Dict, List, Tuple
+from typing import Dict, List
 
 import numpy as np
 import pandas as pd
@@ -35,25 +35,22 @@ class Normalizer:
 
     def __init__(
         self,
+        search_keys: Dict[str, SearchKey],
+        generated_features: List[str],
         bundle: ResourceBundle = None,
         logger: Logger = None,
         warnings_counter: WarningCounter = None,
         silent_mode=False,
     ):
+        self.search_keys = search_keys
+        self.generated_features = generated_features
         self.bundle = bundle or get_custom_bundle()
         self.logger = logger or getLogger()
         self.warnings_counter = warnings_counter or WarningCounter()
         self.silent_mode = silent_mode
         self.columns_renaming = {}
-        self.search_keys = {}
-        self.generated_features = []
-
-    def normalize(
-        self, df: pd.DataFrame, search_keys: Dict[str, SearchKey], generated_features: List[str]
-    ) -> Tuple[pd.DataFrame, Dict[str, SearchKey], List[str]]:
-        self.search_keys = search_keys.copy()
-        self.generated_features = generated_features.copy()
 
+    def normalize(self, df: pd.DataFrame) -> pd.DataFrame:
         df = df.copy()
         df = self._rename_columns(df)
 
@@ -71,25 +68,21 @@ class Normalizer:
 
         df = self.__convert_features_types(df)
 
-        return df, self.search_keys, self.generated_features
+        return df
 
     def _rename_columns(self, df: pd.DataFrame):
         # logger.info("Replace restricted symbols in column names")
         new_columns = []
         dup_counter = 0
         for column in df.columns:
-            if (
-                column
-                in [
-                    TARGET,
-                    EVAL_SET_INDEX,
-                    SYSTEM_RECORD_ID,
-                    ENTITY_SYSTEM_RECORD_ID,
-                    SEARCH_KEY_UNNEST,
-                    DateTimeSearchKeyConverter.DATETIME_COL,
-                ]
-                + self.generated_features
-            ):
+            if column in [
+                TARGET,
+                EVAL_SET_INDEX,
+                SYSTEM_RECORD_ID,
+                ENTITY_SYSTEM_RECORD_ID,
+                SEARCH_KEY_UNNEST,
+                DateTimeSearchKeyConverter.DATETIME_COL,
+            ] + self.generated_features:
                 self.columns_renaming[column] = column
                 new_columns.append(column)
                 continue

upgini/utils/target_utils.py

@@ -150,30 +150,34 @@ def balance_undersample(
         # fill up to min_sample_threshold by majority class
         minority_class = df[df[target_column] == min_class_value]
         majority_class = df[df[target_column] != min_class_value]
-        sample_size = min(len(majority_class), min_sample_threshold - min_class_count)
+        # sample_size = min(len(majority_class), min_sample_threshold - min_class_count)
+        sample_size = min(
+            max_class_count,
+            binary_bootstrap_loops * (min_class_count + max(min_sample_threshold - 2 * min_class_count, 0)),
+        )
         sampled_majority_class = majority_class.sample(n=sample_size, random_state=random_state)
         resampled_data = df[
             (df[SYSTEM_RECORD_ID].isin(minority_class[SYSTEM_RECORD_ID]))
             | (df[SYSTEM_RECORD_ID].isin(sampled_majority_class[SYSTEM_RECORD_ID]))
         ]
 
-    elif max_class_count > min_class_count * binary_bootstrap_loops:
-        msg = bundle.get("dataset_rarest_class_less_threshold").format(
-            min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
-        )
-        logger.warning(msg)
-        print(msg)
-        if warning_counter:
-            warning_counter.increment()
-
-        sampler = RandomUnderSampler(
-            sampling_strategy={max_class_value: binary_bootstrap_loops * min_class_count}, random_state=random_state
-        )
-        X = df[SYSTEM_RECORD_ID]
-        X = X.to_frame(SYSTEM_RECORD_ID)
-        new_x, _ = sampler.fit_resample(X, target)  # type: ignore
-
-        resampled_data = df[df[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
+    # elif max_class_count > min_class_count * binary_bootstrap_loops:
+    #     msg = bundle.get("dataset_rarest_class_less_threshold").format(
+    #         min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
+    #     )
+    #     logger.warning(msg)
+    #     print(msg)
+    #     if warning_counter:
+    #         warning_counter.increment()
+
+    #     sampler = RandomUnderSampler(
+    #         sampling_strategy={max_class_value: binary_bootstrap_loops * min_class_count}, random_state=random_state
+    #     )
+    #     X = df[SYSTEM_RECORD_ID]
+    #     X = X.to_frame(SYSTEM_RECORD_ID)
+    #     new_x, _ = sampler.fit_resample(X, target)  # type: ignore
+
+    #     resampled_data = df[df[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
 
     logger.info(f"Shape after rebalance resampling: {resampled_data}")
     return resampled_data

{upgini-1.2.14.dist-info → upgini-1.2.14a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.14
+Version: 1.2.14a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.14.dist-info → upgini-1.2.14a1.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=mxzjGyB-ihJR05pHYcBiUe_XT5X9wj6cBHLxOXBhAeM,23
+upgini/__about__.py,sha256=HLgDtt9GUzL7hBuQQkMyI0Uz-5Ms9FchFLQelf9FVWU,25
 upgini/__init__.py,sha256=M64LwQTBa-5Jz24Zm2h8rWwlKQQ1J8nP7gGgIciS0WU,589
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
-upgini/dataset.py,sha256=olZ-OHSfBNoBSCo7R5t7uCLukI2nO7afpx_A-HCiJLk,31067
+upgini/dataset.py,sha256=OEcmfHYUDbV5idrx4zMVI4yZ6bFpoiyP7EvBhPXlgeA,31165
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=vRC7g6n6XQxSrvzXk6NJjP0ZytDQhWR4sTAo4Hp7gmA,188319
+upgini/features_enricher.py,sha256=HJJZbZScVrl6ugDBQE71m7om5-ahvMyEnAqZNw-OEJ0,188058
 upgini/http.py,sha256=21asexflvavydzCOONJDGQBtQanCElrbnqLXakJ9Cu8,42880
 upgini/lazy_import.py,sha256=74gQ8JuA48BGRLxAo7lNHNKY2D2emMxrUxKGdxVGhuY,1012
 upgini/metadata.py,sha256=osmzdNESeh7yP3BZday6N9Q3eaIHfzhhRM1d6NSgcf0,11223
@@ -27,7 +27,7 @@ upgini/data_source/data_source_publisher.py,sha256=X-8aGtVgzGmxyXkMVBoBLIGDMb4lY
 upgini/mdc/__init__.py,sha256=aM08nIWFc2gWdWUa3_IuEnNND0cQPkBGnYpRMnfFN8k,1019
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/normalizer/normalize_utils.py,sha256=Lv75lq7M46z9cAIutwkdKZtPZkWblgoRzToAJ1BwY8A,7709
+upgini/normalizer/normalize_utils.py,sha256=bHRPWCNrUvt2R9qMX6dZFCJ0i8ENVCQ2Rw3dHH9IJEg,7447
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
 upgini/resource_bundle/strings.properties,sha256=9kvmcUrsSFUCrzOiN0Ozf-lQ2H8Igz5gATUPoHMOaU4,26456
@@ -54,10 +54,10 @@ upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,1
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
 upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,44511
-upgini/utils/target_utils.py,sha256=BVtDmrmFMKerSUWaNOIEdzsYHIFiODdpnWbE50QDPDc,7864
+upgini/utils/target_utils.py,sha256=SbO9CmvWIwR2pNzPZAMVDRI3cb0O-3NFggLs5cfGPxY,8071
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.2.14.dist-info/METADATA,sha256=xIbSnwYAnie1HKKRe_6MEUGG0BaZRtvWAsRlc2vzlBw,48577
-upgini-1.2.14.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-upgini-1.2.14.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.14.dist-info/RECORD,,
+upgini-1.2.14a1.dist-info/METADATA,sha256=jkOw-XYOr5RrO16cLakpeXDP-fl2Zs4xx2hAVgNYUow,48579
+upgini-1.2.14a1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.14a1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.14a1.dist-info/RECORD,,

{upgini-1.2.14.dist-info → upgini-1.2.14a1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.24.2
+Generator: hatchling 1.25.0
 Root-Is-Purelib: true
 Tag: py3-none-any