upgini 1.1.264__tar.gz → 1.1.264a1__tar.gz

{upgini-1.1.264/src/upgini.egg-info → upgini-1.1.264a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.264
+Version: 1.1.264a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.264 → upgini-1.1.264a1}/setup.py

@@ -40,7 +40,7 @@ def send_log(msg: str):
 
 
 here = Path(__file__).parent.resolve()
-version = "1.1.264"
+version = "1.1.264a1"
 try:
     send_log(f"Start setup PyLib version {version}")
     setup(

{upgini-1.1.264 → upgini-1.1.264a1}/src/upgini/features_enricher.py

@@ -70,7 +70,6 @@ from upgini.utils.datetime_utils import (
     DateTimeSearchKeyConverter,
     is_blocked_time_series,
     is_time_series,
-    validate_dates_distribution,
 )
 from upgini.utils.deduplicate_utils import (
     clean_full_duplicates,
@@ -1686,6 +1685,9 @@ class FeaturesEnricher(TransformerMixin):
             df = validated_X.copy()
 
             df[TARGET] = validated_y
+
+            df = clean_full_duplicates(df, logger=self.logger, silent=True, bundle=self.bundle)
+
             num_samples = _num_samples(df)
             if num_samples > Dataset.FIT_SAMPLE_THRESHOLD:
                 self.logger.info(f"Downsampling from {num_samples} to {Dataset.FIT_SAMPLE_ROWS}")
@@ -1920,6 +1922,7 @@ class FeaturesEnricher(TransformerMixin):
 
             meaning_types = {col: key.value for col, key in search_keys.items()}
             non_keys_columns = [column for column in df.columns if column not in search_keys.keys()]
+            # Don't pass 
             if email_converted_to_hem:
                 non_keys_columns.append(email_column)
 
@@ -1941,6 +1944,7 @@ class FeaturesEnricher(TransformerMixin):
             if add_fit_system_record_id:
                 df = self.__add_fit_system_record_id(df, dict(), search_keys)
                 df = df.rename(columns={SYSTEM_RECORD_ID: SORT_ID})
+                non_keys_columns.append(SORT_ID)
 
             columns_for_system_record_id = sorted(list(search_keys.keys()) + (original_features_for_transform or []))
 
@@ -2217,10 +2221,6 @@ class FeaturesEnricher(TransformerMixin):
         self.fit_search_keys = self.search_keys.copy()
         self.fit_search_keys = self.__prepare_search_keys(validated_X, self.fit_search_keys, is_demo_dataset)
 
-        validate_dates_distribution(
-            validated_X, self.fit_search_keys, self.logger, self.bundle, self.warning_counter
-        )
-
         has_date = self._get_date_column(self.fit_search_keys) is not None
         model_task_type = self.model_task_type or define_task(validated_y, has_date, self.logger)
         self._validate_binary_observations(validated_y, model_task_type)
@@ -2883,26 +2883,35 @@ class FeaturesEnricher(TransformerMixin):
 
         # order by date and idempotent order by other keys
         if self.cv not in [CVType.time_series, CVType.blocked_time_series]:
+            sort_exclude_columns = [original_order_name, ORIGINAL_INDEX, EVAL_SET_INDEX, TARGET, "__target"]
             if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
                 date_column = DateTimeSearchKeyConverter.DATETIME_COL
+                sort_exclude_columns.append(self._get_date_column(search_keys))
             else:
                 date_column = self._get_date_column(search_keys)
             sort_columns = [date_column] if date_column is not None else []
 
-            other_search_keys = sorted(
+            other_columns = sorted(
                 [
-                    sk
-                    for sk, key_type in search_keys.items()
-                    if key_type not in [SearchKey.DATE, SearchKey.DATETIME]
-                    and sk in df.columns
-                    and df[sk].nunique() > 1  # don't use constant keys for hash
+                    c
+                    for c in df.columns
+                    if c not in sort_columns
+                    and c not in sort_exclude_columns
+                    and df[c].nunique() > 1
                 ]
+                # [
+                #     sk
+                #     for sk, key_type in search_keys.items()
+                #     if key_type not in [SearchKey.DATE, SearchKey.DATETIME]
+                #     and sk in df.columns
+                #     and df[sk].nunique() > 1  # don't use constant keys for hash
+                # ]
             )
 
             search_keys_hash = "search_keys_hash"
-            if len(other_search_keys) > 0:
+            if len(other_columns) > 0:
                 sort_columns.append(search_keys_hash)
-                df[search_keys_hash] = pd.util.hash_pandas_object(df[sorted(other_search_keys)], index=False)
+                df[search_keys_hash] = pd.util.hash_pandas_object(df[other_columns], index=False)
 
             df = df.sort_values(by=sort_columns)
 

{upgini-1.1.264 → upgini-1.1.264a1}/src/upgini/resource_bundle/strings.properties

@@ -111,7 +111,6 @@ x_is_empty=X is empty
 y_is_empty=y is empty
 x_contains_reserved_column_name=Column name {} is reserved. Please rename column and try again
 missing_generate_feature=\nWARNING: Feature {} specified in `generate_features` is not present in input columns: {}
-x_unstable_by_date=\nWARNING: Your training sample is unstable in number of rows per date. It is recommended to redesign the training sample.
     # eval set validation
 unsupported_type_eval_set=Unsupported type of eval_set: {}. It should be list of tuples with two elements: X and y
 eval_set_invalid_tuple_size=eval_set contains a tuple of size {}. It should contain only pairs of X and y
@@ -146,7 +145,8 @@ dataset_too_many_rows_registered=X rows limit for transform is {}. Please sample
 dataset_empty_column_names=Some column names are empty. Add names please
 dataset_full_duplicates=\nWARNING: {:.5f}% of the rows are fully duplicated
 dataset_diff_target_duplicates=\nWARNING: {:.4f}% of rows ({}) in X and eval_set are duplicates with different y values. These rows will be deleted as incorrect\nIncorrect row indexes: {}
-dataset_diff_target_duplicates_fintech=\nWARNING: {:.4f}% of rows ({}) in X and eval_set are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
+dataset_train_diff_target_duplicates_fintech=\nWARNING: {:.4f}% of rows ({}) in X are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
+dataset_eval_diff_target_duplicates_fintech=\nWARNING: {:.4f}% of rows ({}) in eval{} X are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
 dataset_drop_old_dates=\nWARNING: We don't have data before '2000-01-01' and removed all earlier records from the search dataset
 dataset_all_dates_old=There is empty train dataset after removing data before '2000-01-01'
 dataset_invalid_target_type=Unexpected dtype of target for binary task type: {}. Expected int or bool

{upgini-1.1.264 → upgini-1.1.264a1}/src/upgini/utils/datetime_utils.py

@@ -1,7 +1,7 @@
 import datetime
 import logging
 import re
-from typing import Dict, List, Optional
+from typing import List, Optional
 
 import numpy as np
 import pandas as pd
@@ -9,9 +9,7 @@ from dateutil.relativedelta import relativedelta
 from pandas.api.types import is_numeric_dtype, is_period_dtype, is_string_dtype
 
 from upgini.errors import ValidationError
-from upgini.metadata import SearchKey
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
-from upgini.utils.warning_counter import WarningCounter
 
 DATE_FORMATS = [
     "%Y-%m-%d",
@@ -227,49 +225,3 @@ def is_blocked_time_series(df: pd.DataFrame, date_col: str, search_keys: List[st
 
     is_diff_less_than_two_columns = grouped.apply(check_differences)
     return is_diff_less_than_two_columns.all()
-
-
-def validate_dates_distribution(
-    X: pd.DataFrame,
-    search_keys: Dict[str, SearchKey],
-    logger: Optional[logging.Logger] = None,
-    bundle: Optional[ResourceBundle] = None,
-    warning_counter: Optional[WarningCounter] = None,
-):
-    maybe_date_col = None
-    for key, key_type in search_keys.items():
-        if key_type in [SearchKey.DATE, SearchKey.DATETIME]:
-            maybe_date_col = key
-
-    if maybe_date_col is None:
-        for col in X.columns:
-            if col in search_keys:
-                continue
-            try:
-                pd.to_datetime(X[col])
-                maybe_date_col = col
-                break
-            except Exception:
-                pass
-
-    if maybe_date_col is None:
-        return
-
-    dates = pd.to_datetime(X[maybe_date_col]).dt.date
-
-    date_counts = dates.value_counts().sort_index()
-
-    date_counts_1 = date_counts[: round(len(date_counts) / 2)]
-    date_counts_2 = date_counts[round(len(date_counts) / 2) :]
-    ratio = date_counts_2.mean() / date_counts_1.mean()
-
-    if ratio > 1.2 or ratio < 0.8:
-        if warning_counter is not None:
-            warning_counter.increment()
-        if logger is None:
-            logger = logging.getLogger("muted_logger")
-            logger.setLevel("FATAL")
-        bundle = bundle or get_custom_bundle()
-        msg = bundle.get("x_unstable_by_date")
-        print(msg)
-        logger.warning(msg)

{upgini-1.1.264 → upgini-1.1.264a1}/src/upgini/utils/deduplicate_utils.py

@@ -3,7 +3,7 @@ from typing import Dict, List, Optional, Union
 
 import pandas as pd
 
-from upgini.metadata import SORT_ID, SYSTEM_RECORD_ID, TARGET, ModelTaskType, SearchKey
+from upgini.metadata import EVAL_SET_INDEX, SORT_ID, SYSTEM_RECORD_ID, TARGET, ModelTaskType, SearchKey
 from upgini.resource_bundle import ResourceBundle
 from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
 from upgini.utils.target_utils import define_task
@@ -78,20 +78,58 @@ def remove_fintech_duplicates(
     rows_with_diff_target = grouped_by_personal_cols.filter(has_diff_target_within_60_days)
     if len(rows_with_diff_target) > 0:
         unique_keys_to_delete = rows_with_diff_target[personal_cols].drop_duplicates()
-        rows_to_remove = pd.merge(df.reset_index(), unique_keys_to_delete, on=personal_cols)
-        rows_to_remove = rows_to_remove.set_index(df.index.name or "index")
-        perc = len(rows_to_remove) * 100 / len(df)
-        msg = bundle.get("dataset_diff_target_duplicates_fintech").format(
-            perc, len(rows_to_remove), rows_to_remove.index.to_list()
-        )
-        if not silent:
-            print(msg)
-        if logger:
-            logger.warning(msg)
-        logger.info(f"Dataset shape before clean fintech duplicates: {df.shape}")
-        df = df[~df.index.isin(rows_to_remove.index)]
-        logger.info(f"Dataset shape after clean fintech duplicates: {df.shape}")
-
+        if EVAL_SET_INDEX not in df.columns:
+            rows_to_remove = pd.merge(df.reset_index(), unique_keys_to_delete, on=personal_cols)
+            rows_to_remove = rows_to_remove.set_index(df.index.name or "index")
+            perc = len(rows_to_remove) * 100 / len(df)
+            msg = bundle.get("dataset_train_diff_target_duplicates_fintech").format(
+                perc, len(rows_to_remove), rows_to_remove.index.to_list()
+            )
+            if not silent:
+                print(msg)
+            if logger:
+                logger.warning(msg)
+            logger.info(f"Dataset shape before clean fintech duplicates: {df.shape}")
+            df = df[~df.index.isin(rows_to_remove.index)]
+            logger.info(f"Dataset shape after clean fintech duplicates: {df.shape}")
+        else:
+            # Indices in train and eval_set can be the same so we remove rows from them separately
+            train = df.query(f"{EVAL_SET_INDEX} == 0")
+            train_rows_to_remove = pd.merge(train.reset_index(), unique_keys_to_delete, on=personal_cols)
+            train_rows_to_remove = train_rows_to_remove.set_index(train.index.name or "index")
+            train_perc = len(train_rows_to_remove) * 100 / len(train)
+            msg = bundle.get("dataset_train_diff_target_duplicates_fintech").format(
+                train_perc, len(train_rows_to_remove), train_rows_to_remove.index.to_list()
+            )
+            if not silent:
+                print(msg)
+            if logger:
+                logger.warning(msg)
+            logger.info(f"Train dataset shape before clean fintech duplicates: {train.shape}")
+            train = train[~train.index.isin(train_rows_to_remove.index)]
+            logger.info(f"Train dataset shape after clean fintech duplicates: {train.shape}")
+
+            evals = [df.query(f"{EVAL_SET_INDEX} == {i}") for i in df[EVAL_SET_INDEX].unique() if i != 0]
+            new_evals = []
+            for i, eval in enumerate(evals):
+                eval_rows_to_remove = pd.merge(eval.reset_index(), unique_keys_to_delete, on=personal_cols)
+                eval_rows_to_remove = eval_rows_to_remove.set_index(eval.index.name or "index")
+                eval_perc = len(eval_rows_to_remove) * 100 / len(eval)
+                msg = bundle.get("dataset_eval_diff_target_duplicates_fintech").format(
+                    eval_perc, len(eval_rows_to_remove), i + 1, eval_rows_to_remove.index.to_list()
+                )
+                if not silent:
+                    print(msg)
+                if logger:
+                    logger.warning(msg)
+                logger.info(f"Eval {i + 1} dataset shape before clean fintech duplicates: {eval.shape}")
+                eval = eval[~eval.index.isin(eval_rows_to_remove.index)]
+                logger.info(f"Eval {i + 1} dataset shape after clean fintech duplicates: {eval.shape}")
+                new_evals.append(eval)
+
+            logger.info(f"Dataset shape before clean fintech duplicates: {df.shape}")
+            df = pd.concat([train] + new_evals)
+            logger.info(f"Dataset shape after clean fintech duplicates: {df.shape}")
     return df
 
 
@@ -101,14 +139,18 @@ def clean_full_duplicates(
     nrows = len(df)
     if nrows == 0:
         return df
-    # Remove absolute duplicates (exclude system_record_id)
+    # Remove full duplicates (exclude system_record_id, sort_id and eval_set_index)
     unique_columns = df.columns.tolist()
     if SYSTEM_RECORD_ID in unique_columns:
         unique_columns.remove(SYSTEM_RECORD_ID)
     if SORT_ID in unique_columns:
         unique_columns.remove(SORT_ID)
+    if EVAL_SET_INDEX in unique_columns:
+        unique_columns.remove(EVAL_SET_INDEX)
     logger.info(f"Dataset shape before clean duplicates: {df.shape}")
-    df = df.drop_duplicates(subset=unique_columns)
+    # Train segment goes first so if duplicates are found in train and eval set
+    # then we keep unique rows in train segment
+    df = df.drop_duplicates(subset=unique_columns, keep="first")
     logger.info(f"Dataset shape after clean duplicates: {df.shape}")
     nrows_after_full_dedup = len(df)
     share_full_dedup = 100 * (1 - nrows_after_full_dedup / nrows)
@@ -123,7 +165,7 @@ def clean_full_duplicates(
         marked_duplicates = df.duplicated(subset=unique_columns, keep=False)
         if marked_duplicates.sum() > 0:
             dups_indices = df[marked_duplicates].index.to_list()
-            nrows_after_tgt_dedup = len(df.drop_duplicates(subset=unique_columns))
+            nrows_after_tgt_dedup = len(df.drop_duplicates(subset=unique_columns, keep=False))
             num_dup_rows = nrows_after_full_dedup - nrows_after_tgt_dedup
             share_tgt_dedup = 100 * num_dup_rows / nrows_after_full_dedup
 
@@ -133,6 +175,7 @@ def clean_full_duplicates(
                 print(msg)
             df = df.drop_duplicates(subset=unique_columns, keep=False)
             logger.info(f"Dataset shape after clean invalid target duplicates: {df.shape}")
+
     return df
 
 

{upgini-1.1.264 → upgini-1.1.264a1}/src/upgini/utils/target_utils.py

@@ -132,7 +132,9 @@ def balance_undersample(
                 class_value = classes[class_idx]
                 class_count = vc[class_value]
                 sample_strategy[class_value] = min(class_count, quantile25_class_cnt * multiclass_bootstrap_loops)
-            sampler = RandomUnderSampler(sampling_strategy=sample_strategy, random_state=random_state)
+            sampler = RandomUnderSampler(
+                sampling_strategy=sample_strategy, random_state=random_state
+            )
             X = df[SYSTEM_RECORD_ID]
             X = X.to_frame(SYSTEM_RECORD_ID)
             new_x, _ = sampler.fit_resample(X, target)  # type: ignore
@@ -151,7 +153,9 @@ def balance_undersample(
         minority_class = df[df[target_column] == min_class_value]
         majority_class = df[df[target_column] != min_class_value]
         sample_size = min(len(majority_class), min_sample_threshold - min_class_count)
-        sampled_majority_class = majority_class.sample(n=sample_size, random_state=random_state)
+        sampled_majority_class = majority_class.sample(
+            n=sample_size, random_state=random_state
+        )
         resampled_data = df[
             (df[SYSTEM_RECORD_ID].isin(minority_class[SYSTEM_RECORD_ID]))
             | (df[SYSTEM_RECORD_ID].isin(sampled_majority_class[SYSTEM_RECORD_ID]))

{upgini-1.1.264 → upgini-1.1.264a1/src/upgini.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.264
+Version: 1.1.264a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.264 → upgini-1.1.264a1}/tests/test_datetime_utils.py

@@ -1,13 +1,7 @@
-import numpy as np
 import pandas as pd
+import numpy as np
 
-from upgini.metadata import SearchKey
-from upgini.utils.datetime_utils import (
-    is_blocked_time_series,
-    is_time_series,
-    validate_dates_distribution,
-)
-from upgini.utils.warning_counter import WarningCounter
+from upgini.utils.datetime_utils import is_blocked_time_series, is_time_series
 
 pd.set_option("mode.chained_assignment", "raise")
 
@@ -189,25 +183,3 @@ def test_multivariate_time_series():
     assert not is_blocked_time_series(df, "date", ["date"])
 
     assert is_blocked_time_series(df, "date", ["date", "feature3"])
-
-
-def test_validate_dates_distribution():
-    df = pd.DataFrame({"date": ["2020-01-01"] * 10 + ["2020-02-01"] * 20 + ["2020-03-01"] * 30 + ["2020-04-01"] * 40})
-    warning_counter = WarningCounter()
-    validate_dates_distribution(df, {}, warning_counter=warning_counter)
-    assert warning_counter.has_warnings()
-
-    df = pd.DataFrame({"date": ["2020-05-01"] * 10 + ["2020-02-01"] * 20 + ["2020-03-01"] * 30 + ["2020-04-01"] * 40})
-    warning_counter = WarningCounter()
-    validate_dates_distribution(df, {}, warning_counter=warning_counter)
-    assert not warning_counter.has_warnings()
-
-    df = pd.DataFrame(
-        {
-            "date2": ["2020-05-01"] * 10 + ["2020-02-01"] * 20 + ["2020-03-01"] * 30 + ["2020-04-01"] * 40,
-            "date1": ["2020-01-01"] * 10 + ["2020-02-01"] * 20 + ["2020-03-01"] * 30 + ["2020-04-01"] * 40,
-        }
-    )
-    warning_counter = WarningCounter()
-    validate_dates_distribution(df, {"date1": SearchKey.DATE}, warning_counter=warning_counter)
-    assert warning_counter.has_warnings()

{upgini-1.1.264 → upgini-1.1.264a1}/tests/test_features_enricher.py

@@ -2164,8 +2164,6 @@ def test_idempotent_order_with_imbalanced_dataset(requests_mock: Mocker):
 
             actual_result_df = result_wrapper.df.sort_values(by="system_record_id").reset_index(drop=True)
             # actual_result_df.to_parquet(expected_result_path)
-            actual_result_df["phone_num_a54a33"] = actual_result_df["phone_num_a54a33"].astype("Int64")
-            actual_result_df["rep_date_f5d6bb"] = actual_result_df["rep_date_f5d6bb"].astype("Int64")
             assert_frame_equal(actual_result_df, expected_result_df)
 
         for i in range(5):