upgini 1.2.91a3906.dev1__py3-none-any.whl → 1.2.92__py3-none-any.whl

upgini/utils/target_utils.py

@@ -1,17 +1,14 @@
 import logging
-from typing import Callable, List, Optional, Union
+from typing import Callable, Optional, Union
 
 import numpy as np
 import pandas as pd
 from pandas.api.types import is_bool_dtype, is_datetime64_any_dtype, is_numeric_dtype
 
 from upgini.errors import ValidationError
-from upgini.metadata import SYSTEM_RECORD_ID, CVType, ModelTaskType
-from upgini.resource_bundle import ResourceBundle, bundle, get_custom_bundle
+from upgini.metadata import SYSTEM_RECORD_ID, ModelTaskType
+from upgini.resource_bundle import ResourceBundle, get_custom_bundle, bundle
 from upgini.sampler.random_under_sampler import RandomUnderSampler
-from upgini.utils.ts_utils import get_most_frequent_time_unit, trunc_datetime
-
-TS_MIN_DIFFERENT_IDS_RATIO = 0.2
 
 
 def prepare_target(y: Union[pd.Series, np.ndarray], target_type: ModelTaskType) -> Union[pd.Series, np.ndarray]:
@@ -204,199 +201,6 @@ def balance_undersample(
     return resampled_data
 
 
-def balance_undersample_forced(
-    df: pd.DataFrame,
-    target_column: str,
-    id_columns: Optional[List[str]],
-    date_column: str,
-    task_type: ModelTaskType,
-    cv_type: Optional[CVType],
-    random_state: int,
-    sample_size: int = 7000,
-    logger: Optional[logging.Logger] = None,
-    bundle: Optional[ResourceBundle] = None,
-    warning_callback: Optional[Callable] = None,
-):
-    if len(df) <= sample_size:
-        return df
-
-    if logger is None:
-        logger = logging.getLogger("muted_logger")
-        logger.setLevel("FATAL")
-    bundle = bundle or get_custom_bundle()
-    if SYSTEM_RECORD_ID not in df.columns:
-        raise Exception("System record id must be presented for undersampling")
-
-    msg = bundle.get("forced_balance_undersample")
-    logger.info(msg)
-    if warning_callback is not None:
-        warning_callback(msg)
-
-    target = df[target_column].copy()
-
-    vc = target.value_counts()
-    max_class_value = vc.index[0]
-    min_class_value = vc.index[len(vc) - 1]
-    max_class_count = vc[max_class_value]
-    min_class_count = vc[min_class_value]
-
-    resampled_data = df
-    df = df.copy().sort_values(by=SYSTEM_RECORD_ID)
-    if cv_type is not None and cv_type.is_time_series():
-        logger.warning(f"Sampling time series dataset from {len(df)} to {sample_size}")
-        resampled_data = balance_undersample_time_series_trunc(
-            df,
-            id_columns=id_columns,
-            date_column=date_column,
-            sample_size=sample_size,
-            random_state=random_state,
-            logger=logger,
-        )
-    elif task_type in [ModelTaskType.MULTICLASS, ModelTaskType.REGRESSION]:
-        logger.warning(f"Sampling dataset from {len(df)} to {sample_size}")
-        resampled_data = df.sample(n=sample_size, random_state=random_state)
-    else:
-        msg = bundle.get("imbalanced_target").format(min_class_value, min_class_count)
-        logger.warning(msg)
-
-        # fill up to min_sample_threshold by majority class
-        minority_class = df[df[target_column] == min_class_value]
-        majority_class = df[df[target_column] != min_class_value]
-        logger.info(
-            f"Min class count: {min_class_count}. Max class count: {max_class_count}."
-            f" Rebalance sample size: {sample_size}"
-        )
-        if len(minority_class) > (sample_size / 2):
-            sampled_minority_class = minority_class.sample(n=int(sample_size / 2), random_state=random_state)
-        else:
-            sampled_minority_class = minority_class
-
-        if len(majority_class) > (sample_size) / 2:
-            sampled_majority_class = majority_class.sample(n=int(sample_size / 2), random_state=random_state)
-
-        resampled_data = df[
-            (df[SYSTEM_RECORD_ID].isin(sampled_minority_class[SYSTEM_RECORD_ID]))
-            | (df[SYSTEM_RECORD_ID].isin(sampled_majority_class[SYSTEM_RECORD_ID]))
-        ]
-
-    logger.info(f"Shape after forced rebalance resampling: {resampled_data}")
-    return resampled_data
-
-
-DEFAULT_HIGH_FREQ_TRUNC_LENGTHS = [pd.DateOffset(years=2, months=6), pd.DateOffset(years=2, days=7)]
-DEFAULT_LOW_FREQ_TRUNC_LENGTHS = [pd.DateOffset(years=7), pd.DateOffset(years=5)]
-DEFAULT_TIME_UNIT_THRESHOLD = pd.Timedelta(weeks=4)
-
-
-def balance_undersample_time_series_trunc(
-    df: pd.DataFrame,
-    id_columns: Optional[List[str]],
-    date_column: str,
-    sample_size: int,
-    random_state: int = 42,
-    logger: Optional[logging.Logger] = None,
-    highfreq_trunc_lengths: List[pd.DateOffset] = DEFAULT_HIGH_FREQ_TRUNC_LENGTHS,
-    lowfreq_trunc_lengths: List[pd.DateOffset] = DEFAULT_LOW_FREQ_TRUNC_LENGTHS,
-    time_unit_threshold: pd.Timedelta = DEFAULT_TIME_UNIT_THRESHOLD,
-    **kwargs,
-):
-    if id_columns is None:
-        id_columns = []
-    # Convert date column to datetime
-    dates_df = df[id_columns + [date_column]].copy()
-    dates_df[date_column] = pd.to_datetime(dates_df[date_column], unit="ms")
-
-    time_unit = get_most_frequent_time_unit(dates_df, id_columns, date_column)
-    if logger is not None:
-        logger.info(f"Time unit: {time_unit}")
-
-    if time_unit is None:
-        if logger is not None:
-            logger.info("Cannot detect time unit, returning original dataset")
-        return df
-
-    if time_unit < time_unit_threshold:
-        for trunc_length in highfreq_trunc_lengths:
-            sampled_df = trunc_datetime(dates_df, id_columns, date_column, trunc_length, logger=logger)
-            if len(sampled_df) <= sample_size:
-                break
-        if len(sampled_df) > sample_size:
-            sampled_df = balance_undersample_time_series(
-                sampled_df, id_columns, date_column, sample_size, random_state, logger=logger, **kwargs
-            )
-    else:
-        for trunc_length in lowfreq_trunc_lengths:
-            sampled_df = trunc_datetime(dates_df, id_columns, date_column, trunc_length, logger=logger)
-            if len(sampled_df) <= sample_size:
-                break
-        if len(sampled_df) > sample_size:
-            sampled_df = balance_undersample_time_series(
-                sampled_df, id_columns, date_column, sample_size, random_state, logger=logger, **kwargs
-            )
-
-    return df.loc[sampled_df.index]
-
-
-def balance_undersample_time_series(
-    df: pd.DataFrame,
-    id_columns: List[str],
-    date_column: str,
-    sample_size: int,
-    random_state: int = 42,
-    min_different_ids_ratio: float = TS_MIN_DIFFERENT_IDS_RATIO,
-    prefer_recent_dates: bool = True,
-    logger: Optional[logging.Logger] = None,
-):
-    def ensure_tuple(x):
-        return tuple([x]) if not isinstance(x, tuple) else x
-
-    random_state = np.random.RandomState(random_state)
-
-    if not id_columns:
-        id_columns = [date_column]
-    ids_sort = df.groupby(id_columns)[date_column].aggregate(["max", "count"]).T.to_dict()
-    ids_sort = {
-        ensure_tuple(k): (
-            (v["max"], v["count"], random_state.rand()) if prefer_recent_dates else (v["count"], random_state.rand())
-        )
-        for k, v in ids_sort.items()
-    }
-    id_counts = df[id_columns].value_counts()
-    id_counts.index = [ensure_tuple(i) for i in id_counts.index]
-    id_counts = id_counts.sort_index(key=lambda x: [ids_sort[y] for y in x], ascending=False).cumsum()
-    id_counts = id_counts[id_counts <= sample_size]
-    min_different_ids = max(int(len(df[id_columns].drop_duplicates()) * min_different_ids_ratio), 1)
-
-    def id_mask(sample_index: pd.Index) -> pd.Index:
-        if isinstance(sample_index, pd.MultiIndex):
-            return pd.MultiIndex.from_frame(df[id_columns]).isin(sample_index)
-        else:
-            return df[id_columns[0]].isin(sample_index)
-
-    if len(id_counts) < min_different_ids:
-        if logger is not None:
-            logger.info(
-                f"Different ids count {len(id_counts)} for sample size {sample_size}"
-                f" is less than min different ids {min_different_ids}, sampling time window"
-            )
-        date_counts = df.groupby(id_columns)[date_column].nunique().sort_values(ascending=False)
-        ids_to_sample = date_counts.index[:min_different_ids] if len(id_counts) > 0 else date_counts.index
-        mask = id_mask(ids_to_sample)
-        df = df[mask]
-        sample_date_counts = df[date_column].value_counts().sort_index(ascending=False).cumsum()
-        sample_date_counts = sample_date_counts[sample_date_counts <= sample_size]
-        df = df[df[date_column].isin(sample_date_counts.index)]
-    else:
-        if len(id_columns) > 1:
-            id_counts.index = pd.MultiIndex.from_tuples(id_counts.index)
-        else:
-            id_counts.index = [i[0] for i in id_counts.index]
-        mask = id_mask(id_counts.index)
-        df = df[mask]
-
-    return df
-
-
 def calculate_psi(expected: pd.Series, actual: pd.Series) -> Union[float, Exception]:
     try:
         df = pd.concat([expected, actual])

{upgini-1.2.91a3906.dev1.dist-info → upgini-1.2.92.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.91a3906.dev1
+Version: 1.2.92
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.91a3906.dev1.dist-info → upgini-1.2.92.dist-info}/RECORD

@@ -1,12 +1,12 @@
-upgini/__about__.py,sha256=Lvq_aBqVHaMSRjnG_m_EJn-gwVd5w5KGUuuS1LiJ_dY,33
+upgini/__about__.py,sha256=wXo9Q87kBdNAVEzs4oUkI_3AmrQDgiMvfXa7xRn9cOE,23
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
-upgini/dataset.py,sha256=fRtqSkXNONLnPe6cCL967GMt349FTIpXzy_u8LUKncw,35354
+upgini/dataset.py,sha256=e6JDYTZ2AwC5aF-dqclKZKkiKrHo2f6cFmMQO2ZZmjM,32724
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=rieH8wjC1c_q2LYZoju8KZyshokNzFpwVtrCtG88w3s,215940
+upgini/features_enricher.py,sha256=wFeqZ30Dkhiyath--Jg6uVVoTwCdPJ42Rbe_smr1ue4,218465
 upgini/http.py,sha256=4i7fQwrwU3WzDUOWzrgR-4C8eJwj_5dBwRAR-UjUtlc,44345
-upgini/metadata.py,sha256=zt_9k0iQbWXuiRZcel4ORNPdQKt6Ou69ucZD_E1Q46o,12341
-upgini/metrics.py,sha256=zIOaiyfQLedU9Fk4877drnlWh-KiImSkZpPeiq6Xr1E,45295
+upgini/metadata.py,sha256=vsbbHyPCP3Rs8WkeDgQg99uAA_zmsbDStAT-NwDYhO4,12455
+upgini/metrics.py,sha256=Bc1L9DUmEL8OWwNvIEjPjw5EyHSZbiu3v2hWyBmedis,45313
 upgini/search_task.py,sha256=Q5HjBpLIB3OCxAD1zNv5yQ3ZNJx696WCK_-H35_y7Rs,17912
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
 upgini/version_validator.py,sha256=DvbaAvuYFoJqYt0fitpsk6Xcv-H1BYDJYHUMxaKSH_Y,1509
@@ -19,7 +19,7 @@ upgini/autofe/date.py,sha256=MM1S-6imNSzCDOhbNnmsc_bwSqUWBcS8vWAdHF8j1kY,11134
 upgini/autofe/feature.py,sha256=cu4xXjzVVF13ZV4RxuTrysK2qCfezlRCMOzCKRo1rNs,15558
 upgini/autofe/groupby.py,sha256=IYmQV9uoCdRcpkeWZj_kI3ObzoNCNx3ff3h8sTL01tk,3603
 upgini/autofe/operator.py,sha256=EOffJw6vKXpEh5yymqb1RFNJPxGxmnHdFRo9dB5SCFo,4969
-upgini/autofe/unary.py,sha256=N76Pehn-hO8FWlSdqJ2Wm-yoU1MSR7m6yb2GWYBcumU,5933
+upgini/autofe/unary.py,sha256=Sx11IoHRh5nwyALzjgG9GQOrVNIs8NZ1JzunAJuN66A,5731
 upgini/autofe/utils.py,sha256=dYrtyAM8Vcc_R8u4dNo54IsGrHKagTHDJTKhGho0bRg,2967
 upgini/autofe/vector.py,sha256=jHs0nNTOaHspYUlxW7fjQepk4cvr_JDQ65L1OCiVsds,1360
 upgini/autofe/timeseries/__init__.py,sha256=PGwwDAMwvkXl3el12tXVEmZUgDUvlmIPlXtROm6bD18,738
@@ -38,7 +38,7 @@ upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 upgini/normalizer/normalize_utils.py,sha256=g2TcDXZeJp9kAFO2sTqZ4CAsN4J1qHNgoJHZ8gtzUWo,7376
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=SxO1uWFAc1s7BOFi01OyEI3ajklUKBhs8LkKrstImIg,28290
+upgini/resource_bundle/strings.properties,sha256=Hfpr2-I5Ws6ugIN1QSz549OHayZeLYglRsbrGDT6g9g,28491
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=7GpjYqjOp58vYcJLiX__1R5wjUlyQbxvHJ2klFnup_M,6389
@@ -56,7 +56,7 @@ upgini/utils/deduplicate_utils.py,sha256=EpBVCov42-FJIAPfa4jY_ZRct3N2MFaC7i-oJNZ
 upgini/utils/display_utils.py,sha256=hAeWEcJtPDg8fAVcMNrNB-azFD2WJp1nvbPAhR7SeP4,12071
 upgini/utils/email_utils.py,sha256=pZ2vCfNxLIPUhxr0-OlABNXm12jjU44isBk8kGmqQzA,5277
 upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0--cOFTwJ0,1074
-upgini/utils/feature_info.py,sha256=Q9HN6A-fvfVD-irFWrmOqqZG9RsUSvh5MTY_k0xu-tE,7287
+upgini/utils/feature_info.py,sha256=b3RvAeOHSEu-ZXWTrf42Dll_3ZUBL0pw7sdk7hgUKD0,7284
 upgini/utils/features_validator.py,sha256=lEfmk4DoxZ4ooOE1HC0ZXtUb_lFKRFHIrnFULZ4_rL8,3746
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
 upgini/utils/ip_utils.py,sha256=TSQ_qDsLlVnm09X1HacpabEf_HNqSWpxBF4Sdc2xs08,6580
@@ -64,13 +64,14 @@ upgini/utils/mstats.py,sha256=u3gQVUtDRbyrOQK6V1UJ2Rx1QbkSNYGjXa6m3Z_dPVs,6286
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
+upgini/utils/sample_utils.py,sha256=ETLPKQU_YngiYbdlnEoF2h7QS-3oN8et54q3Qs2ZAbA,15417
 upgini/utils/sklearn_ext.py,sha256=jLJWAKkqQinV15Z4y1ZnsN3c-fKFwXTsprs00COnyVU,49315
 upgini/utils/sort.py,sha256=8uuHs2nfSMVnz8GgvbOmgMB1PgEIZP1uhmeRFxcwnYw,7039
-upgini/utils/target_utils.py,sha256=mVZ8wrkBb-tzEnVZwZw0m-Y0Sojb5t-wIsACRH05nIw,16890
+upgini/utils/target_utils.py,sha256=i3Xt5l9ybB2_nF_ma5cfPuL3OeFTs2dY2xDI0p4Azpg,9049
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.91a3906.dev1.dist-info/METADATA,sha256=YhyG3QmIoZnjZm_YW29eCRBdyfETpzOk7yPhlNb-lcE,49546
-upgini-1.2.91a3906.dev1.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-upgini-1.2.91a3906.dev1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.91a3906.dev1.dist-info/RECORD,,
+upgini-1.2.92.dist-info/METADATA,sha256=yXqDsCwRNGqlytVFuoBL04Swo6xYo5lsk9_YHj-6PfQ,49536
+upgini-1.2.92.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+upgini-1.2.92.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.92.dist-info/RECORD,,