upgini 1.2.91a3884.dev1__py3-none-any.whl → 1.2.91a3884.dev3__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.91a3884.dev1"
+__version__ = "1.2.91a3884.dev3"

upgini/resource_bundle/strings.properties

@@ -196,6 +196,7 @@ timeseries_invalid_test_size_type=test_size={} should be a float in the (0, 1) r
 timeseries_splits_more_than_samples=Number of splits={} can't be more than number of samples={}
 timeseries_invalid_test_size=Wrong number of samples in a test fold: (test_size * n_samples / n_splits) <= 1
 date_and_id_columns_duplicates=Found {} duplicate rows by date and id_columns. Please remove them and try again
+missing_ids_in_eval_set=Following ids are present in eval set but not in sampled train set: {}. They will be removed from eval set.
     # Upload ads validation
 ads_upload_too_few_rows=At least 1000 records per sample are needed. Increase the sample size for evaluation and resubmit the data
 ads_upload_search_key_not_found=Search key {} wasn't found in dataframe columns

upgini/utils/sample_utils.py

@@ -92,12 +92,13 @@ def sample(
         fit_sample_rows = sample_config.fit_sample_rows
 
     if cv_type is not None and cv_type.is_time_series():
-        return sample_time_series_trunc(
+        return sample_time_series_train_eval(
             df,
-            sample_columns.ids,
-            sample_columns.date,
+            sample_columns,
             sample_config.fit_sample_rows_ts,
-            random_state,
+            trim_threshold=fit_sample_threshold,
+            max_rows=fit_sample_rows,
+            random_state=random_state,
             logger=logger,
             **kwargs,
         )
@@ -128,6 +129,68 @@ def sample(
     return df
 
 
+def sample_time_series_train_eval(
+    df: pd.DataFrame,
+    sample_columns: SampleColumns,
+    sample_size: int,
+    trim_threshold: int,
+    max_rows: int,
+    random_state: int = 42,
+    logger: Optional[logging.Logger] = None,
+    bundle: Optional[ResourceBundle] = None,
+    **kwargs,
+):
+    if sample_columns.eval_set_index in df.columns:
+        train_df = df[df[sample_columns.eval_set_index] == 0]
+        eval_df = df[df[sample_columns.eval_set_index] > 0]
+    else:
+        train_df = df
+        eval_df = None
+
+    train_df = sample_time_series_trunc(
+        train_df, sample_columns.ids, sample_columns.date, sample_size, random_state, logger=logger, **kwargs
+    )
+    if sample_columns.ids and eval_df is not None:
+        missing_ids = (
+            eval_df[~eval_df[sample_columns.ids].isin(np.unique(train_df[sample_columns.ids]))][sample_columns.ids]
+            .dropna()
+            .drop_duplicates()
+            .values.tolist()
+        )
+        if missing_ids:
+            bundle = bundle or get_custom_bundle()
+            print(bundle.get("missing_ids_in_eval_set").format(missing_ids))
+            eval_df = eval_df.merge(train_df[sample_columns.ids].drop_duplicates())
+
+    if eval_df is not None:
+        if len(eval_df) > trim_threshold - len(train_df):
+            eval_df = sample_time_series_trunc(
+                eval_df,
+                sample_columns.ids,
+                sample_columns.date,
+                max_rows - len(train_df),
+                random_state,
+                logger=logger,
+                **kwargs,
+            )
+        df = pd.concat([train_df, eval_df])
+
+    elif len(train_df) > max_rows:
+        df = sample_time_series_trunc(
+            train_df,
+            sample_columns.ids,
+            sample_columns.date,
+            max_rows,
+            random_state,
+            logger=logger,
+            **kwargs,
+        )
+    else:
+        df = train_df
+
+    return df
+
+
 def sample_time_series_trunc(
     df: pd.DataFrame,
     id_columns: Optional[List[str]],
@@ -189,6 +252,7 @@ def sample_time_series(
     min_different_ids_ratio: float = TS_MIN_DIFFERENT_IDS_RATIO,
     prefer_recent_dates: bool = True,
     logger: Optional[logging.Logger] = None,
+    **kwargs,
 ):
     def ensure_tuple(x):
         return tuple([x]) if not isinstance(x, tuple) else x
@@ -242,9 +306,7 @@ def sample_time_series(
 
 def balance_undersample_forced(
     df: pd.DataFrame,
-    target_column: str,
-    id_columns: Optional[List[str]],
-    date_column: str,
+    sample_columns: SampleColumns,
     task_type: ModelTaskType,
     cv_type: Optional[CVType],
     random_state: int,
@@ -268,7 +330,7 @@ def balance_undersample_forced(
     if warning_callback is not None:
         warning_callback(msg)
 
-    target = df[target_column].copy()
+    target = df[sample_columns.target].copy()
 
     vc = target.value_counts()
     max_class_value = vc.index[0]
@@ -280,11 +342,12 @@ def balance_undersample_forced(
     df = df.copy().sort_values(by=SYSTEM_RECORD_ID)
     if cv_type is not None and cv_type.is_time_series():
         logger.warning(f"Sampling time series dataset from {len(df)} to {sample_size}")
-        resampled_data = sample_time_series_trunc(
+        resampled_data = sample_time_series_train_eval(
             df,
-            id_columns=id_columns,
-            date_column=date_column,
+            sample_columns=sample_columns,
             sample_size=sample_size,
+            trim_threshold=sample_size,
+            max_rows=sample_size,
             random_state=random_state,
             logger=logger,
         )
@@ -296,8 +359,8 @@ def balance_undersample_forced(
         logger.warning(msg)
 
         # fill up to min_sample_threshold by majority class
-        minority_class = df[df[target_column] == min_class_value]
-        majority_class = df[df[target_column] != min_class_value]
+        minority_class = df[df[sample_columns.target] == min_class_value]
+        majority_class = df[df[sample_columns.target] != min_class_value]
         logger.info(
             f"Min class count: {min_class_count}. Max class count: {max_class_count}."
             f" Rebalance sample size: {sample_size}"

{upgini-1.2.91a3884.dev1.dist-info → upgini-1.2.91a3884.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.91a3884.dev1
+Version: 1.2.91a3884.dev3
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.91a3884.dev1.dist-info → upgini-1.2.91a3884.dev3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-upgini/__about__.py,sha256=71INSTd7K-9v6Q1BhlXvzqKYEhEddj5rcyg_0HuQwMU,33
+upgini/__about__.py,sha256=55Sg-JLu4aw-5ANNPanS_ciHPSsxXTa8YndbgltGREA,33
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=c0rZ-ydrnCdrTzx10WZl4WbO3LdyuF0fUCRD8Ugjitg,33093
@@ -38,7 +38,7 @@ upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 upgini/normalizer/normalize_utils.py,sha256=g2TcDXZeJp9kAFO2sTqZ4CAsN4J1qHNgoJHZ8gtzUWo,7376
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=YvW_vyK1klVdvkWFripy8cBH-wGjzsyomoe3Pd20LjY,28359
+upgini/resource_bundle/strings.properties,sha256=Hfpr2-I5Ws6ugIN1QSz549OHayZeLYglRsbrGDT6g9g,28491
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=7GpjYqjOp58vYcJLiX__1R5wjUlyQbxvHJ2klFnup_M,6389
@@ -64,14 +64,14 @@ upgini/utils/mstats.py,sha256=u3gQVUtDRbyrOQK6V1UJ2Rx1QbkSNYGjXa6m3Z_dPVs,6286
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
-upgini/utils/sample_utils.py,sha256=g4OZwxN4LnLmQs3ZCebZDRuztQL0vFkdv8m2IBiv0S0,13204
+upgini/utils/sample_utils.py,sha256=PpMXRVTPKi6TyAo0gPhF0OmXmutecHdonM7WYUsB1Wo,15249
 upgini/utils/sklearn_ext.py,sha256=jLJWAKkqQinV15Z4y1ZnsN3c-fKFwXTsprs00COnyVU,49315
 upgini/utils/sort.py,sha256=8uuHs2nfSMVnz8GgvbOmgMB1PgEIZP1uhmeRFxcwnYw,7039
 upgini/utils/target_utils.py,sha256=i3Xt5l9ybB2_nF_ma5cfPuL3OeFTs2dY2xDI0p4Azpg,9049
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.91a3884.dev1.dist-info/METADATA,sha256=7y90EE_TXo-9fNsSlsQ7cZxoxvet0C2-ccrGGa9WX-k,49546
-upgini-1.2.91a3884.dev1.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-upgini-1.2.91a3884.dev1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.91a3884.dev1.dist-info/RECORD,,
+upgini-1.2.91a3884.dev3.dist-info/METADATA,sha256=teoc8dCmv4mb2eBV6QruZag3xnwK3YAdlKCHuIKllXw,49546
+upgini-1.2.91a3884.dev3.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+upgini-1.2.91a3884.dev3.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.91a3884.dev3.dist-info/RECORD,,