upgini 1.1.264__py3-none-any.whl → 1.1.265__py3-none-any.whl

upgini/features_enricher.py

@@ -1686,6 +1686,9 @@ class FeaturesEnricher(TransformerMixin):
             df = validated_X.copy()
 
             df[TARGET] = validated_y
+
+            df = clean_full_duplicates(df, logger=self.logger, silent=True, bundle=self.bundle)
+
             num_samples = _num_samples(df)
             if num_samples > Dataset.FIT_SAMPLE_THRESHOLD:
                 self.logger.info(f"Downsampling from {num_samples} to {Dataset.FIT_SAMPLE_ROWS}")
@@ -1920,6 +1923,7 @@ class FeaturesEnricher(TransformerMixin):
 
             meaning_types = {col: key.value for col, key in search_keys.items()}
             non_keys_columns = [column for column in df.columns if column not in search_keys.keys()]
+
             if email_converted_to_hem:
                 non_keys_columns.append(email_column)
 
@@ -1941,6 +1945,7 @@ class FeaturesEnricher(TransformerMixin):
             if add_fit_system_record_id:
                 df = self.__add_fit_system_record_id(df, dict(), search_keys)
                 df = df.rename(columns={SYSTEM_RECORD_ID: SORT_ID})
+                non_keys_columns.append(SORT_ID)
 
             columns_for_system_record_id = sorted(list(search_keys.keys()) + (original_features_for_transform or []))
 
@@ -2883,26 +2888,35 @@ class FeaturesEnricher(TransformerMixin):
 
         # order by date and idempotent order by other keys
         if self.cv not in [CVType.time_series, CVType.blocked_time_series]:
+            sort_exclude_columns = [original_order_name, ORIGINAL_INDEX, EVAL_SET_INDEX, TARGET, "__target"]
             if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
                 date_column = DateTimeSearchKeyConverter.DATETIME_COL
+                sort_exclude_columns.append(self._get_date_column(search_keys))
             else:
                 date_column = self._get_date_column(search_keys)
             sort_columns = [date_column] if date_column is not None else []
 
-            other_search_keys = sorted(
+            other_columns = sorted(
                 [
-                    sk
-                    for sk, key_type in search_keys.items()
-                    if key_type not in [SearchKey.DATE, SearchKey.DATETIME]
-                    and sk in df.columns
-                    and df[sk].nunique() > 1  # don't use constant keys for hash
+                    c
+                    for c in df.columns
+                    if c not in sort_columns
+                    and c not in sort_exclude_columns
+                    and df[c].nunique() > 1
                 ]
+                # [
+                #     sk
+                #     for sk, key_type in search_keys.items()
+                #     if key_type not in [SearchKey.DATE, SearchKey.DATETIME]
+                #     and sk in df.columns
+                #     and df[sk].nunique() > 1  # don't use constant keys for hash
+                # ]
             )
 
             search_keys_hash = "search_keys_hash"
-            if len(other_search_keys) > 0:
+            if len(other_columns) > 0:
                 sort_columns.append(search_keys_hash)
-                df[search_keys_hash] = pd.util.hash_pandas_object(df[sorted(other_search_keys)], index=False)
+                df[search_keys_hash] = pd.util.hash_pandas_object(df[other_columns], index=False)
 
             df = df.sort_values(by=sort_columns)
 

upgini/resource_bundle/strings.properties

@@ -146,7 +146,8 @@ dataset_too_many_rows_registered=X rows limit for transform is {}. Please sample
 dataset_empty_column_names=Some column names are empty. Add names please
 dataset_full_duplicates=\nWARNING: {:.5f}% of the rows are fully duplicated
 dataset_diff_target_duplicates=\nWARNING: {:.4f}% of rows ({}) in X and eval_set are duplicates with different y values. These rows will be deleted as incorrect\nIncorrect row indexes: {}
-dataset_diff_target_duplicates_fintech=\nWARNING: {:.4f}% of rows ({}) in X and eval_set are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
+dataset_train_diff_target_duplicates_fintech=\nWARNING: {:.4f}% of rows ({}) in X are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
+dataset_eval_diff_target_duplicates_fintech=\nWARNING: {:.4f}% of rows ({}) in eval{} X are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
 dataset_drop_old_dates=\nWARNING: We don't have data before '2000-01-01' and removed all earlier records from the search dataset
 dataset_all_dates_old=There is empty train dataset after removing data before '2000-01-01'
 dataset_invalid_target_type=Unexpected dtype of target for binary task type: {}. Expected int or bool

upgini/utils/deduplicate_utils.py

@@ -3,7 +3,7 @@ from typing import Dict, List, Optional, Union
 
 import pandas as pd
 
-from upgini.metadata import SORT_ID, SYSTEM_RECORD_ID, TARGET, ModelTaskType, SearchKey
+from upgini.metadata import EVAL_SET_INDEX, SORT_ID, SYSTEM_RECORD_ID, TARGET, ModelTaskType, SearchKey
 from upgini.resource_bundle import ResourceBundle
 from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
 from upgini.utils.target_utils import define_task
@@ -78,20 +78,58 @@ def remove_fintech_duplicates(
     rows_with_diff_target = grouped_by_personal_cols.filter(has_diff_target_within_60_days)
     if len(rows_with_diff_target) > 0:
         unique_keys_to_delete = rows_with_diff_target[personal_cols].drop_duplicates()
-        rows_to_remove = pd.merge(df.reset_index(), unique_keys_to_delete, on=personal_cols)
-        rows_to_remove = rows_to_remove.set_index(df.index.name or "index")
-        perc = len(rows_to_remove) * 100 / len(df)
-        msg = bundle.get("dataset_diff_target_duplicates_fintech").format(
-            perc, len(rows_to_remove), rows_to_remove.index.to_list()
-        )
-        if not silent:
-            print(msg)
-        if logger:
-            logger.warning(msg)
-        logger.info(f"Dataset shape before clean fintech duplicates: {df.shape}")
-        df = df[~df.index.isin(rows_to_remove.index)]
-        logger.info(f"Dataset shape after clean fintech duplicates: {df.shape}")
-
+        if EVAL_SET_INDEX not in df.columns:
+            rows_to_remove = pd.merge(df.reset_index(), unique_keys_to_delete, on=personal_cols)
+            rows_to_remove = rows_to_remove.set_index(df.index.name or "index")
+            perc = len(rows_to_remove) * 100 / len(df)
+            msg = bundle.get("dataset_train_diff_target_duplicates_fintech").format(
+                perc, len(rows_to_remove), rows_to_remove.index.to_list()
+            )
+            if not silent:
+                print(msg)
+            if logger:
+                logger.warning(msg)
+            logger.info(f"Dataset shape before clean fintech duplicates: {df.shape}")
+            df = df[~df.index.isin(rows_to_remove.index)]
+            logger.info(f"Dataset shape after clean fintech duplicates: {df.shape}")
+        else:
+            # Indices in train and eval_set can be the same so we remove rows from them separately
+            train = df.query(f"{EVAL_SET_INDEX} == 0")
+            train_rows_to_remove = pd.merge(train.reset_index(), unique_keys_to_delete, on=personal_cols)
+            train_rows_to_remove = train_rows_to_remove.set_index(train.index.name or "index")
+            train_perc = len(train_rows_to_remove) * 100 / len(train)
+            msg = bundle.get("dataset_train_diff_target_duplicates_fintech").format(
+                train_perc, len(train_rows_to_remove), train_rows_to_remove.index.to_list()
+            )
+            if not silent:
+                print(msg)
+            if logger:
+                logger.warning(msg)
+            logger.info(f"Train dataset shape before clean fintech duplicates: {train.shape}")
+            train = train[~train.index.isin(train_rows_to_remove.index)]
+            logger.info(f"Train dataset shape after clean fintech duplicates: {train.shape}")
+
+            evals = [df.query(f"{EVAL_SET_INDEX} == {i}") for i in df[EVAL_SET_INDEX].unique() if i != 0]
+            new_evals = []
+            for i, eval in enumerate(evals):
+                eval_rows_to_remove = pd.merge(eval.reset_index(), unique_keys_to_delete, on=personal_cols)
+                eval_rows_to_remove = eval_rows_to_remove.set_index(eval.index.name or "index")
+                eval_perc = len(eval_rows_to_remove) * 100 / len(eval)
+                msg = bundle.get("dataset_eval_diff_target_duplicates_fintech").format(
+                    eval_perc, len(eval_rows_to_remove), i + 1, eval_rows_to_remove.index.to_list()
+                )
+                if not silent:
+                    print(msg)
+                if logger:
+                    logger.warning(msg)
+                logger.info(f"Eval {i + 1} dataset shape before clean fintech duplicates: {eval.shape}")
+                eval = eval[~eval.index.isin(eval_rows_to_remove.index)]
+                logger.info(f"Eval {i + 1} dataset shape after clean fintech duplicates: {eval.shape}")
+                new_evals.append(eval)
+
+            logger.info(f"Dataset shape before clean fintech duplicates: {df.shape}")
+            df = pd.concat([train] + new_evals)
+            logger.info(f"Dataset shape after clean fintech duplicates: {df.shape}")
     return df
 
 
@@ -101,14 +139,18 @@ def clean_full_duplicates(
     nrows = len(df)
     if nrows == 0:
         return df
-    # Remove absolute duplicates (exclude system_record_id)
+    # Remove full duplicates (exclude system_record_id, sort_id and eval_set_index)
     unique_columns = df.columns.tolist()
     if SYSTEM_RECORD_ID in unique_columns:
         unique_columns.remove(SYSTEM_RECORD_ID)
     if SORT_ID in unique_columns:
         unique_columns.remove(SORT_ID)
+    if EVAL_SET_INDEX in unique_columns:
+        unique_columns.remove(EVAL_SET_INDEX)
     logger.info(f"Dataset shape before clean duplicates: {df.shape}")
-    df = df.drop_duplicates(subset=unique_columns)
+    # Train segment goes first so if duplicates are found in train and eval set
+    # then we keep unique rows in train segment
+    df = df.drop_duplicates(subset=unique_columns, keep="first")
     logger.info(f"Dataset shape after clean duplicates: {df.shape}")
     nrows_after_full_dedup = len(df)
     share_full_dedup = 100 * (1 - nrows_after_full_dedup / nrows)
@@ -123,7 +165,7 @@ def clean_full_duplicates(
         marked_duplicates = df.duplicated(subset=unique_columns, keep=False)
         if marked_duplicates.sum() > 0:
             dups_indices = df[marked_duplicates].index.to_list()
-            nrows_after_tgt_dedup = len(df.drop_duplicates(subset=unique_columns))
+            nrows_after_tgt_dedup = len(df.drop_duplicates(subset=unique_columns, keep=False))
             num_dup_rows = nrows_after_full_dedup - nrows_after_tgt_dedup
             share_tgt_dedup = 100 * num_dup_rows / nrows_after_full_dedup
 
@@ -133,6 +175,7 @@ def clean_full_duplicates(
                 print(msg)
             df = df.drop_duplicates(subset=unique_columns, keep=False)
             logger.info(f"Dataset shape after clean invalid target duplicates: {df.shape}")
+
     return df
 
 

{upgini-1.1.264.dist-info → upgini-1.1.265.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.264
+Version: 1.1.265
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.264.dist-info → upgini-1.1.265.dist-info}/RECORD

@@ -2,7 +2,7 @@ upgini/__init__.py,sha256=asENHgEVHQBIkV-e_0IhE_ZWqkCG6398U3ZLrNzAH6k,407
 upgini/ads.py,sha256=mre6xn44wcC_fg63iLT_kTh4mViZqR9AKRJZAtpQz8Y,2592
 upgini/dataset.py,sha256=xb4gIANyGbdcuM8Awyq2pJPiH_3k_LEbETApJgAoRBA,45529
 upgini/errors.py,sha256=pdzQl3MKuK52yvncxMWMRWeSIOGhUFzpQoszoRFBOk0,958
-upgini/features_enricher.py,sha256=DUWMptGZT4fIC3Bfu8jqezBCTnZz3aT3xHGpJtPxhqU,172308
+upgini/features_enricher.py,sha256=5rc9vcsCBwmRDb8aAPOFGmkRbC7_zGJGPlaSvkytqCk,172880
 upgini/fingerprint.js,sha256=VygVIQlN1v4NGZfjHqtRogOw8zjTnnMNJg_f7M5iGQU,33442
 upgini/http.py,sha256=zaO86LBBLmkieGbgYifk29eVoPCxXimZQ8YkQtKcM0I,42244
 upgini/metadata.py,sha256=fwVxtkR6Mn4iRoOqV6BfMJvJrx65I3YwZUMbZjhPyOI,9673
@@ -29,7 +29,7 @@ upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 upgini/normalizer/phone_normalizer.py,sha256=lhwsPEnfyjeIsndW2EcQGZksXYsfxaQ1ghAzVYoDRKM,9927
 upgini/resource_bundle/__init__.py,sha256=hdvbqL0b0xMWbY6-kiYGsW1ro2GMiWpxxsO9uCv-h9Q,8379
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=8f9l853-JuRZklYNsrp3PS9lL4KWxSIP31m01FiKBG0,25370
+upgini/resource_bundle/strings.properties,sha256=_bEfgRl2a9sgoy2RxvIf26NemnCW5CM-1AWWpljwZQE,25664
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=CC-DvPbrN7zp5--SVFuUqkVmdWM_5F7R0Do98ETV82U,6421
@@ -42,7 +42,7 @@ upgini/utils/country_utils.py,sha256=1KXhLSNqkNYVL3on8-zK0Arc_SspUH7AMZvGZICysOU
 upgini/utils/custom_loss_utils.py,sha256=DBslpjWGPt7xTeypt78baR59012SYphbPsO_YLKdilo,3972
 upgini/utils/cv_utils.py,sha256=Tn01RJvpZGZh0PUQUimlBkV-AXwe7s6yjCNFtw352Uc,3525
 upgini/utils/datetime_utils.py,sha256=4ii5WphAHlb_NRmdJx35VZpTarJbAr-AnDw3XSzUSow,10346
-upgini/utils/deduplicate_utils.py,sha256=ckJrpU8Ruc_vcwIPTopbUjyJuNiseLHNAbQlLfhUCxo,5888
+upgini/utils/deduplicate_utils.py,sha256=6AbARehUCghJZ4PppFtrej2s3gFRruh41MEm6mzakHs,8607
 upgini/utils/display_utils.py,sha256=LKoSwjrE0xgS5_cqVhc2og2CQ1UCZ1nTI2VKboIhoQA,10858
 upgini/utils/email_utils.py,sha256=3CvHXTSzlgLyGsQOXfRYVfFhfPy6OXG4uXOBWRaLfHg,3479
 upgini/utils/fallback_progress_bar.py,sha256=cdbd1XGcWm4Ed4eAqV2_St3z7uC_kkH22gEyrN5ub6M,1090
@@ -56,8 +56,8 @@ upgini/utils/sklearn_ext.py,sha256=fvuTWJ5AnT3ED9KSaQu_yIgW2JR19hFlaGDoVP3k60g,4
 upgini/utils/target_utils.py,sha256=5BHcOsBRb4z7P8t3e9rsdXUWUUI7DBmQMmv-x6RwzHM,7152
 upgini/utils/track_info.py,sha256=EPcJ13Jqa17_T0JjM37Ac9kWDz5Zk0GVsIZKutOb8aU,5207
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.1.264.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.264.dist-info/METADATA,sha256=9v-Bdg9n5Pp1-eJqFTOFYl1r0TkmFEaa3AAfZEAtbB0,48156
-upgini-1.1.264.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-upgini-1.1.264.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
-upgini-1.1.264.dist-info/RECORD,,
+upgini-1.1.265.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.265.dist-info/METADATA,sha256=HX-CwFFNgXRRuZ00TELhLI1-3ufrny1K0uZc9p0JWdA,48156
+upgini-1.1.265.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+upgini-1.1.265.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
+upgini-1.1.265.dist-info/RECORD,,