upgini 1.1.253a5__py3-none-any.whl → 1.1.253a3261__py3-none-any.whl

upgini/data_source/data_source_publisher.py

@@ -40,7 +40,7 @@ class DataSourcePublisher:
         if logs_enabled:
             self.logger = LoggerFactory().get_logger(endpoint, api_key)
         else:
-            self.logger = logging.getLogger("muted_logger")
+            self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
 
     def place(

upgini/dataset.py

@@ -39,10 +39,10 @@ from upgini.metadata import (
 )
 from upgini.normalizer.phone_normalizer import PhoneNormalizer
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
+from upgini.sampler.random_under_sampler import RandomUnderSampler
 from upgini.search_task import SearchTask
 from upgini.utils import combine_search_keys, find_numbers_with_decimal_comma
 from upgini.utils.email_utils import EmailSearchKeyConverter
-from upgini.utils.target_utils import balance_undersample
 
 try:
     from upgini.utils.progress_bar import CustomProgressBar as ProgressBar
@@ -61,8 +61,6 @@ class Dataset:  # (pd.DataFrame):
     FIT_SAMPLE_WITH_EVAL_SET_THRESHOLD = 200_000
     MIN_SAMPLE_THRESHOLD = 5_000
     IMBALANCE_THESHOLD = 0.4
-    BINARY_BOOTSTRAP_LOOPS = 5
-    MULTICLASS_BOOTSTRAP_LOOPS = 2
     MIN_TARGET_CLASS_ROWS = 100
     MAX_MULTICLASS_CLASS_COUNT = 100
     MIN_SUPPORTED_DATE_TS = 946684800000  # 2000-01-01
@@ -463,8 +461,10 @@ class Dataset:  # (pd.DataFrame):
             self.task_type == ModelTaskType.BINARY and len(train_segment) > self.MIN_SAMPLE_THRESHOLD
         ):
             count = len(train_segment)
-            target_column = self.etalon_def_checked.get(FileColumnMeaningType.TARGET.value, TARGET)
-            target = train_segment[target_column]
+            min_class_count = count
+            min_class_value = None
+            target_column = self.etalon_def_checked.get(FileColumnMeaningType.TARGET.value, "")
+            target = train_segment[target_column].copy()
             target_classes_count = target.nunique()
 
             if target_classes_count > self.MAX_MULTICLASS_CLASS_COUNT:
@@ -474,9 +474,12 @@ class Dataset:  # (pd.DataFrame):
                 self.logger.warning(msg)
                 raise ValidationError(msg)
 
-            vc = target.value_counts()
-            min_class_value = vc.index[len(vc) - 1]
-            min_class_count = vc[min_class_value]
+            unique_target = target.unique()
+            for v in list(unique_target):  # type: ignore
+                current_class_count = len(train_segment.loc[target == v])
+                if current_class_count < min_class_count:
+                    min_class_count = current_class_count
+                    min_class_value = v
 
             if min_class_count < self.MIN_TARGET_CLASS_ROWS:
                 msg = self.bundle.get("dataset_rarest_class_less_min").format(
@@ -489,19 +492,53 @@ class Dataset:  # (pd.DataFrame):
             min_class_threshold = min_class_percent * count
 
             if min_class_count < min_class_threshold:
-                self.imbalanced = True
-                self.data = balance_undersample(
-                    df=train_segment,
-                    target_column=target_column,
-                    task_type=self.task_type,
-                    random_state=self.random_state,
-                    imbalance_threshold=self.IMBALANCE_THESHOLD,
-                    binary_bootstrap_loops=self.BINARY_BOOTSTRAP_LOOPS,
-                    multiclass_bootstrap_loops=self.MULTICLASS_BOOTSTRAP_LOOPS,
-                    logger=self.logger,
-                    bundle=self.bundle,
-                    warning_counter=self.warning_counter,
+                msg = self.bundle.get("dataset_rarest_class_less_threshold").format(
+                    min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
                 )
+                self.logger.warning(msg)
+                print(msg)
+                self.warning_counter.increment()
+
+                train_segment = train_segment.copy().sort_values(by=SYSTEM_RECORD_ID)
+                if self.task_type == ModelTaskType.MULTICLASS:
+                    # Sort classes by rows count and find 25% quantile class
+                    classes = target.value_counts().index
+                    quantile25_idx = int(0.75 * len(classes))
+                    quantile25_class = classes[quantile25_idx]
+                    count_of_quantile25_class = len(target[target == quantile25_class])
+                    msg = self.bundle.get("imbalance_multiclass").format(quantile25_class, count_of_quantile25_class)
+                    self.logger.warning(msg)
+                    print(msg)
+                    # 25% and lower classes will stay as is. Higher classes will be downsampled
+                    parts = []
+                    for class_idx in range(quantile25_idx):
+                        sampled = train_segment[train_segment[target_column] == classes[class_idx]].sample(
+                            n=count_of_quantile25_class, random_state=self.random_state
+                        )
+                        parts.append(sampled)
+                    for class_idx in range(quantile25_idx, len(classes)):
+                        parts.append(train_segment[train_segment[target_column] == classes[class_idx]])
+                    resampled_data = pd.concat(parts)
+                elif self.task_type == ModelTaskType.BINARY and min_class_count < self.MIN_SAMPLE_THRESHOLD / 2:
+                    minority_class = train_segment[train_segment[target_column] == min_class_value]
+                    majority_class = train_segment[train_segment[target_column] != min_class_value]
+                    sampled_majority_class = majority_class.sample(
+                        n=self.MIN_SAMPLE_THRESHOLD - min_class_count, random_state=self.random_state
+                    )
+                    resampled_data = train_segment[
+                        (train_segment[SYSTEM_RECORD_ID].isin(minority_class[SYSTEM_RECORD_ID]))
+                        | (train_segment[SYSTEM_RECORD_ID].isin(sampled_majority_class[SYSTEM_RECORD_ID]))
+                    ]
+                else:
+                    sampler = RandomUnderSampler(random_state=self.random_state)
+                    X = train_segment[SYSTEM_RECORD_ID]
+                    X = X.to_frame(SYSTEM_RECORD_ID)
+                    new_x, _ = sampler.fit_resample(X, target)  # type: ignore
+                    resampled_data = train_segment[train_segment[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
+
+                self.data = resampled_data
+                self.logger.info(f"Shape after rebalance resampling: {self.data.shape}")
+                self.imbalanced = True
 
         # Resample over fit threshold
         if not self.imbalanced and EVAL_SET_INDEX in self.data.columns:

upgini/features_enricher.py

@@ -221,7 +221,7 @@ class FeaturesEnricher(TransformerMixin):
         if logs_enabled:
             self.logger = LoggerFactory().get_logger(endpoint, self._api_key, client_ip, client_visitorid)
         else:
-            self.logger = logging.getLogger("muted_logger")
+            self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
 
         if len(kwargs) > 0:
@@ -3047,6 +3047,20 @@ class FeaturesEnricher(TransformerMixin):
         def list_or_single(lst: List[str], single: str):
             return lst or ([single] if single else [])
 
+        def to_anchor(link: str, value: str) -> str:
+            if not value:
+                return ""
+            elif not link:
+                return value
+            elif value == llm_source:
+                return value
+            else:
+                return f"<a href='{link}' target='_blank' rel='noopener noreferrer'>{value}</a>"
+
+        def make_links(names: List[str], links: List[str]):
+            all_links = [to_anchor(link, name) for name, link in itertools.zip_longest(names, links)]
+            return ",".join(all_links)
+
         features_meta.sort(key=lambda m: (-m.shap_value, m.name))
         for feature_meta in features_meta:
             if feature_meta.name in original_names_dict.keys():
@@ -3072,20 +3086,6 @@ class FeaturesEnricher(TransformerMixin):
                 if len(feature_sample) > 30:
                     feature_sample = feature_sample[:30] + "..."
 
-            def to_anchor(link: str, value: str) -> str:
-                if not value:
-                    return ""
-                elif not link:
-                    return value
-                elif value == llm_source:
-                    return value
-                else:
-                    return f"<a href='{link}' target='_blank' rel='noopener noreferrer'>{value}</a>"
-
-            def make_links(names: List[str], links: List[str]):
-                all_links = [to_anchor(link, name) for name, link in itertools.zip_longest(names, links)]
-                return ",".join(all_links)
-
             internal_provider = feature_meta.data_provider or "Upgini"
             providers = list_or_single(feature_meta.data_providers, feature_meta.data_provider)
             provider_links = list_or_single(feature_meta.data_provider_links, feature_meta.data_provider_link)

upgini/search_task.py

@@ -57,7 +57,7 @@ class SearchTask:
         if logger is not None:
             self.logger = logger
         else:
-            self.logger = logging.getLogger("muted_logger")
+            self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
         self.provider_metadata_v2: Optional[List[ProviderTaskMetadataV2]] = None
         self.unused_features_for_generation: Optional[List[str]] = None

upgini/utils/datetime_utils.py

@@ -31,7 +31,7 @@ class DateTimeSearchKeyConverter:
         if logger is not None:
             self.logger = logger
         else:
-            self.logger = logging.getLogger("muted_logger")
+            self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
         self.generated_features: List[str] = []
         self.bundle = bundle or get_custom_bundle()

upgini/utils/target_utils.py

@@ -6,10 +6,8 @@ import pandas as pd
 from pandas.api.types import is_numeric_dtype
 
 from upgini.errors import ValidationError
-from upgini.metadata import SYSTEM_RECORD_ID, ModelTaskType
-from upgini.resource_bundle import ResourceBundle, bundle, get_custom_bundle
-from upgini.sampler.random_under_sampler import RandomUnderSampler
-from upgini.utils.warning_counter import WarningCounter
+from upgini.metadata import ModelTaskType
+from upgini.resource_bundle import bundle
 
 
 def correct_string_target(y: Union[pd.Series, np.ndarray]) -> Union[pd.Series, np.ndarray]:
@@ -74,110 +72,3 @@ def is_int_encoding(unique_values):
     return set(unique_values) == set(range(len(unique_values))) or set(unique_values) == set(
         range(1, len(unique_values) + 1)
     )
-
-
-def balance_undersample(
-    df: pd.DataFrame,
-    target_column: str,
-    task_type: ModelTaskType,
-    random_state: int,
-    imbalance_threshold: int = 0.2,
-    min_sample_threshold: int = 5000,
-    binary_bootstrap_loops: int = 5,
-    multiclass_bootstrap_loops: int = 2,
-    logger: Optional[logging.Logger] = None,
-    bundle: Optional[ResourceBundle] = None,
-    warning_counter: Optional[WarningCounter] = None,
-) -> pd.DataFrame:
-    if logger is None:
-        logger = logging.getLogger("muted_logger")
-        logger.setLevel("FATAL")
-    bundle = bundle or get_custom_bundle()
-    if SYSTEM_RECORD_ID not in df.columns:
-        raise Exception("System record id must be presented for undersampling")
-
-    count = len(df)
-    target = df[target_column].copy()
-    target_classes_count = target.nunique()
-
-    vc = target.value_counts()
-    max_class_value = vc.index[0]
-    min_class_value = vc.index[len(vc) - 1]
-    max_class_count = vc[max_class_value]
-    min_class_count = vc[min_class_value]
-
-    min_class_percent = imbalance_threshold / target_classes_count
-    min_class_threshold = min_class_percent * count
-
-    resampled_data = df
-    df = df.copy().sort_values(by=SYSTEM_RECORD_ID)
-    if task_type == ModelTaskType.MULTICLASS:
-        # Sort classes by rows count and find 25% quantile class
-        classes = vc.index
-        quantile25_idx = int(0.75 * len(classes)) - 1
-        quantile25_class = classes[quantile25_idx]
-        quantile25_class_cnt = vc[quantile25_class]
-
-        if max_class_count > (quantile25_class_cnt * multiclass_bootstrap_loops):
-            msg = bundle.get("imbalance_multiclass").format(quantile25_class, quantile25_class_cnt)
-            logger.warning(msg)
-            print(msg)
-            if warning_counter:
-                warning_counter.increment()
-
-            # 25% and lower classes will stay as is. Higher classes will be downsampled
-            sample_strategy = dict()
-            for class_idx in range(quantile25_idx):
-                # compare class count with count_of_quantile25_class * 2
-                class_value = classes[class_idx]
-                class_count = vc[class_value]
-                sample_strategy[class_value] = min(class_count, quantile25_class_cnt * multiclass_bootstrap_loops)
-            sampler = RandomUnderSampler(
-                sampling_strategy=sample_strategy, random_state=random_state
-            )
-            X = df[SYSTEM_RECORD_ID]
-            X = X.to_frame(SYSTEM_RECORD_ID)
-            new_x, _ = sampler.fit_resample(X, target)  # type: ignore
-
-            resampled_data = df[df[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
-    elif len(df) > min_sample_threshold and min_class_count < min_sample_threshold / 2:
-        msg = bundle.get("dataset_rarest_class_less_threshold").format(
-            min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
-        )
-        logger.warning(msg)
-        print(msg)
-        if warning_counter:
-            warning_counter.increment()
-
-        # fill up to min_sample_threshold by majority class
-        minority_class = df[df[target_column] == min_class_value]
-        majority_class = df[df[target_column] != min_class_value]
-        sample_size = min(len(majority_class), min_sample_threshold - min_class_count)
-        sampled_majority_class = majority_class.sample(
-            n=sample_size, random_state=random_state
-        )
-        resampled_data = df[
-            (df[SYSTEM_RECORD_ID].isin(minority_class[SYSTEM_RECORD_ID]))
-            | (df[SYSTEM_RECORD_ID].isin(sampled_majority_class[SYSTEM_RECORD_ID]))
-        ]
-
-    elif max_class_count > min_class_count * binary_bootstrap_loops:
-        msg = bundle.get("dataset_rarest_class_less_threshold").format(
-            min_class_value, min_class_count, min_class_threshold, min_class_percent * 100
-        )
-        logger.warning(msg)
-        print(msg)
-        if warning_counter:
-            warning_counter.increment()
-
-        sampler = RandomUnderSampler(
-            sampling_strategy={max_class_value: binary_bootstrap_loops * min_class_count}, random_state=random_state
-        )
-        X = df[SYSTEM_RECORD_ID]
-        X = X.to_frame(SYSTEM_RECORD_ID)
-        new_x, _ = sampler.fit_resample(X, target)  # type: ignore
-
-        resampled_data = df[df[SYSTEM_RECORD_ID].isin(new_x[SYSTEM_RECORD_ID])]
-
-    logger.info(f"Shape after rebalance resampling: {resampled_data}")
-    return resampled_data

{upgini-1.1.253a5.dist-info → upgini-1.1.253a3261.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.253a5
+Version: 1.1.253a3261
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers
@@ -18,12 +18,13 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: Telecommunications Industry
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
-Requires-Python: >=3.8,<3.11
+Requires-Python: >=3.7,<3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: python-dateutil >=2.8.0

{upgini-1.1.253a5.dist-info → upgini-1.1.253a3261.dist-info}/RECORD

@@ -1,13 +1,13 @@
 upgini/__init__.py,sha256=asENHgEVHQBIkV-e_0IhE_ZWqkCG6398U3ZLrNzAH6k,407
 upgini/ads.py,sha256=mre6xn44wcC_fg63iLT_kTh4mViZqR9AKRJZAtpQz8Y,2592
-upgini/dataset.py,sha256=sL1w8dVmXkuV5PWaBTUpz8X8joW6mIOLyPbSAlSbpR0,45576
+upgini/dataset.py,sha256=tLa0aEcT7XwVJz1AawXCIEj3vxsSBi-geKicuYpRIMw,48196
 upgini/errors.py,sha256=pdzQl3MKuK52yvncxMWMRWeSIOGhUFzpQoszoRFBOk0,958
-upgini/features_enricher.py,sha256=1f92VFY3ZyVGdTTY81BPQs6OjrSAQJc0U41C6QsWLQ0,171822
+upgini/features_enricher.py,sha256=dP6Oyhi4erESEGlVFA_j67lqhNqNvbkfM4FGpE_WaTU,171760
 upgini/fingerprint.js,sha256=VygVIQlN1v4NGZfjHqtRogOw8zjTnnMNJg_f7M5iGQU,33442
 upgini/http.py,sha256=eSG4gOpmCGlXmB6KIPNzAG8tRZNUjyYpMeUeHw_2li4,42264
 upgini/metadata.py,sha256=fwVxtkR6Mn4iRoOqV6BfMJvJrx65I3YwZUMbZjhPyOI,9673
 upgini/metrics.py,sha256=LS2MgEKgmn9VEXsKzxv3pBZ-q71mTnpWu6vL8fYgpo4,26727
-upgini/search_task.py,sha256=tmJ17WUxv3J5NWrYUJB_NKdZ792Ifz8Z8UnDXeQnpss,17077
+upgini/search_task.py,sha256=5n4qGJmtu48s0-FHAtF3L5qVLMd1JVW3FJlM8dFbh-s,17063
 upgini/spinner.py,sha256=Dm1dQ5F_z_Ua2odLxZX7OypcOX9tSx_vE5MGaKtUmfw,1118
 upgini/version_validator.py,sha256=rDIncP6BEko4J2F2hUcMOtKm_vZbI4ICWcNcw8hrwM4,1400
 upgini/ads_management/__init__.py,sha256=qzyisOToVRP-tquAJD1PblZhNtMrOB8FiyF9JvfkvgE,50
@@ -21,7 +21,7 @@ upgini/autofe/operand.py,sha256=Rhy7Ky3we-I1Su1--dS4xdsO3K8neV4rqM_Q4xYE4ug,2779
 upgini/autofe/unary.py,sha256=gyMkrx9bfa3o19zS-4JaRlScHrfeZGBsYe7d_6ePT-0,2853
 upgini/autofe/vector.py,sha256=Qk7VmdwURNwVw7fIMEspWEo7HTiyUWCYIqu3hcWQQio,507
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/data_source/data_source_publisher.py,sha256=LZ8iZHGrGVP1c7xIk57zio9OqN3VmBS_S3NW9Xs2pL4,15124
+upgini/data_source/data_source_publisher.py,sha256=ZMNyh1x1S3QkXkA-PTtBQ-sbOiANtNioEQs8VoQ24Lk,15110
 upgini/mdc/__init__.py,sha256=ETDh3JKbrDdPMOECiYLAa8lvKYe68mv4IY6fZa9FimA,1126
 upgini/mdc/context.py,sha256=Sl1S_InKlzzRxYqwJ2k24lawJdCKWgGJ-RIRfvzWJrk,1468
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -40,7 +40,7 @@ upgini/utils/blocked_time_series.py,sha256=dMz5ewk3PsoeOrc3lDzInCVPS9u_2XQkV0W6P
 upgini/utils/country_utils.py,sha256=1KXhLSNqkNYVL3on8-zK0Arc_SspUH7AMZvGZICysOU,6462
 upgini/utils/custom_loss_utils.py,sha256=DBslpjWGPt7xTeypt78baR59012SYphbPsO_YLKdilo,3972
 upgini/utils/cv_utils.py,sha256=Tn01RJvpZGZh0PUQUimlBkV-AXwe7s6yjCNFtw352Uc,3525
-upgini/utils/datetime_utils.py,sha256=b8pyNhrC8ni6apsLQivQOiKqu-37pU4EF3nNHPZqiN8,8713
+upgini/utils/datetime_utils.py,sha256=P5no4mFgYpEP6oY524ebTKvKc3TBMJzAYpWdj210_Fw,8699
 upgini/utils/deduplicate_utils.py,sha256=ckJrpU8Ruc_vcwIPTopbUjyJuNiseLHNAbQlLfhUCxo,5888
 upgini/utils/display_utils.py,sha256=tiq5sFOfMwkKCjQ7OGdyK_twe0Qdr9F3mzkW1QXSDog,10664
 upgini/utils/email_utils.py,sha256=3CvHXTSzlgLyGsQOXfRYVfFhfPy6OXG4uXOBWRaLfHg,3479
@@ -52,11 +52,11 @@ upgini/utils/phone_utils.py,sha256=JNSkF8G6mgsN8Czy11pamaJdsY6rBINEMpi7jbVt_RA,4
 upgini/utils/postal_code_utils.py,sha256=_8CR9tBqsPptQsmMUvnrCAmBaMIQSWH3JfJ4ly3x_zs,409
 upgini/utils/progress_bar.py,sha256=iNXyqT3vKCeHpfiG5HHwr7Lk2cTtKViM93Fl8iZnjGc,1564
 upgini/utils/sklearn_ext.py,sha256=fvuTWJ5AnT3ED9KSaQu_yIgW2JR19hFlaGDoVP3k60g,44027
-upgini/utils/target_utils.py,sha256=WVhhxpQVvnhsDV7ctlds51VFg7hz59S_MFUSoRZFszw,7204
+upgini/utils/target_utils.py,sha256=DH812qcZ7Pvf9WVVb33fbwQjb1W9h1hXRNCCiG7Y6tI,2563
 upgini/utils/track_info.py,sha256=EPcJ13Jqa17_T0JjM37Ac9kWDz5Zk0GVsIZKutOb8aU,5207
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.1.253a5.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.253a5.dist-info/METADATA,sha256=sYd-VQbiUxqSOb9cgjavIjagvPxM1uyO7fgAT97UZg8,48158
-upgini-1.1.253a5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-upgini-1.1.253a5.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
-upgini-1.1.253a5.dist-info/RECORD,,
+upgini-1.1.253a3261.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.253a3261.dist-info/METADATA,sha256=rriUm7SPiC9cSL8HPrx8AZ4zVyiOdlHMw5V6rGRy2Bk,48211
+upgini-1.1.253a3261.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+upgini-1.1.253a3261.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
+upgini-1.1.253a3261.dist-info/RECORD,,