upgini 1.1.236a2__tar.gz → 1.1.237a2__tar.gz

{upgini-1.1.236a2/src/upgini.egg-info → upgini-1.1.237a2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.236a2
+Version: 1.1.237a2
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.236a2 → upgini-1.1.237a2}/setup.py

@@ -40,7 +40,7 @@ def send_log(msg: str):
 
 
 here = Path(__file__).parent.resolve()
-version = "1.1.236a2"
+version = "1.1.237a2"
 try:
     send_log(f"Start setup PyLib version {version}")
     setup(

{upgini-1.1.236a2 → upgini-1.1.237a2}/src/upgini/dataset.py

@@ -36,12 +36,14 @@ from upgini.metadata import (
     NumericInterval,
     RuntimeParameters,
     SearchCustomization,
+    SearchKey,
 )
 from upgini.normalizer.phone_normalizer import PhoneNormalizer
 from upgini.resource_bundle import bundle
 from upgini.sampler.random_under_sampler import RandomUnderSampler
 from upgini.search_task import SearchTask
 from upgini.utils import combine_search_keys
+from upgini.utils.deduplicate_utils import remove_fintech_duplicates
 from upgini.utils.email_utils import EmailSearchKeyConverter
 
 try:
@@ -382,7 +384,7 @@ class Dataset:  # (pd.DataFrame):
 
             if is_string_dtype(self.data[postal_code]):
                 try:
-                    self.data[postal_code] = self.data[postal_code].astype("Float64").astype("Int64").astype("string")
+                    self.data[postal_code] = self.data[postal_code].astype("float64").astype("Int64").astype("string")
                 except Exception:
                     pass
             elif is_float_dtype(self.data[postal_code]):
@@ -820,6 +822,13 @@ class Dataset:  # (pd.DataFrame):
         self.__validate_dataset(validate_target, silent_mode)
 
         if validate_target:
+            search_keys = {
+                col: SearchKey.from_meaning_type(key_type)
+                for col, key_type in self.meaning_types.items()
+                if SearchKey.from_meaning_type(key_type) is not None
+                }
+            self.data = remove_fintech_duplicates(self.data, search_keys, self.logger)
+
             self.__validate_target()
 
             self.__resample()

{upgini-1.1.236a2 → upgini-1.1.237a2}/src/upgini/features_enricher.py

@@ -64,6 +64,7 @@ from upgini.utils.datetime_utils import (
     is_blocked_time_series,
     is_time_series,
 )
+from upgini.utils.deduplicate_utils import remove_fintech_duplicates
 from upgini.utils.display_utils import (
     display_html_dataframe,
     do_without_pandas_limits,
@@ -1183,8 +1184,8 @@ class FeaturesEnricher(TransformerMixin):
             converter = DateTimeSearchKeyConverter(date_column, self.date_format, self.logger)
             extended_X = converter.convert(extended_X, keep_time=True)
             generated_features.extend(converter.generated_features)
-        email_column = self.__get_email_column(search_keys)
-        hem_column = self.__get_hem_column(search_keys)
+        email_column = self._get_email_column(search_keys)
+        hem_column = self._get_hem_column(search_keys)
         if email_column:
             converter = EmailSearchKeyConverter(email_column, hem_column, search_keys, self.logger)
             extended_X = converter.convert(extended_X)
@@ -1505,6 +1506,8 @@ class FeaturesEnricher(TransformerMixin):
                 eval_df_with_index[TARGET] = eval_y
                 eval_df_with_index[EVAL_SET_INDEX] = idx + 1
                 df_with_eval_set_index = pd.concat([df_with_eval_set_index, eval_df_with_index])
+            
+            df_with_eval_set_index = remove_fintech_duplicates(df_with_eval_set_index, self.search_keys, self.logger)
 
             # downsample if need to eval_set threshold
             num_samples = _num_samples(df_with_eval_set_index)
@@ -1741,8 +1744,8 @@ class FeaturesEnricher(TransformerMixin):
                 generated_features.extend(converter.generated_features)
             else:
                 self.logger.info("Input dataset hasn't date column")
-            email_column = self.__get_email_column(search_keys)
-            hem_column = self.__get_hem_column(search_keys)
+            email_column = self._get_email_column(search_keys)
+            hem_column = self._get_hem_column(search_keys)
             email_converted_to_hem = False
             if email_column:
                 converter = EmailSearchKeyConverter(email_column, hem_column, search_keys, self.logger)
@@ -2081,8 +2084,8 @@ class FeaturesEnricher(TransformerMixin):
             self.fit_generated_features.extend(converter.generated_features)
         else:
             self.logger.info("Input dataset hasn't date column")
-        email_column = self.__get_email_column(self.fit_search_keys)
-        hem_column = self.__get_hem_column(self.fit_search_keys)
+        email_column = self._get_email_column(self.fit_search_keys)
+        hem_column = self._get_hem_column(self.fit_search_keys)
         email_converted_to_hem = False
         if email_column:
             converter = EmailSearchKeyConverter(email_column, hem_column, self.fit_search_keys, self.logger)
@@ -2615,16 +2618,22 @@ class FeaturesEnricher(TransformerMixin):
         return [col for col, t in search_keys.items() if t not in [SearchKey.DATE, SearchKey.DATETIME]]
 
     @staticmethod
-    def __get_email_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
+    def _get_email_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
         for col, t in search_keys.items():
             if t == SearchKey.EMAIL:
                 return col
 
     @staticmethod
-    def __get_hem_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
+    def _get_hem_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
         for col, t in search_keys.items():
             if t == SearchKey.HEM:
                 return col
+    
+    @staticmethod
+    def _get_phone_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
+        for col, t in search_keys.items():
+            if t == SearchKey.PHONE:
+                return col
 
     def __add_fit_system_record_id(
         self, df: pd.DataFrame, meaning_types: Dict[str, FileColumnMeaningType], search_keys: Dict[str, SearchKey]

{upgini-1.1.236a2 → upgini-1.1.237a2}/src/upgini/resource_bundle/strings.properties

@@ -142,6 +142,7 @@ dataset_empty_column_names=Some column names are empty. Add names please
 dataset_too_long_column_name=Column {} is too long: {} characters. Remove this column or trim length to 50 characters
 dataset_full_duplicates=\nWARNING: {:.5f}% of the rows are fully duplicated
 dataset_diff_target_duplicates=\nWARNING: {:.4f}% of rows ({}) in X and eval_set are duplicates with different y values. These rows will be deleted as incorrect\nIncorrect row indexes: {}
+dataset_diff_target_duplicates_fintech=\nWARNING: {:.4f}% of rows ({}) in X and eval_set are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
 dataset_drop_old_dates=\nWARNING: We don't have data before '2000-01-01' and removed all earlier records from the search dataset
 dataset_all_dates_old=There is empty train dataset after removing data before '2000-01-01'
 dataset_invalid_target_type=Unexpected dtype of target for binary task type: {}. Expected int or bool

{upgini-1.1.236a2 → upgini-1.1.237a2}/src/upgini/utils/datetime_utils.py

@@ -61,9 +61,22 @@ class DateTimeSearchKeyConverter:
         elif is_period_dtype(df[self.date_column]):
             df[self.date_column] = pd.to_datetime(df[self.date_column].astype("string"))
         elif is_numeric_dtype(df[self.date_column]):
-            msg = f"Unsupported type of date column {self.date_column}. Convert to datetime please."
-            self.logger.warning(msg)
-            raise ValidationError(msg)
+            # 315532801 - 2524608001    - seconds
+            # 315532801000 - 2524608001000 - milliseconds
+            # 315532801000000 - 2524608001000000 - microseconds
+            # 315532801000000000 - 2524608001000000000 - nanoseconds
+            if df[self.date_column].apply(lambda x: 10**16 < x).all():
+                df[self.date_column] = pd.to_datetime(df[self.date_column], unit="ns")
+            elif df[self.date_column].apply(lambda x: 10**14 < x < 10**16).all():
+                df[self.date_column] = pd.to_datetime(df[self.date_column], unit="us")
+            elif df[self.date_column].apply(lambda x: 10**11 < x < 10**14).all():
+                df[self.date_column] = pd.to_datetime(df[self.date_column], unit="ms")
+            elif df[self.date_column].apply(lambda x: 0 < x < 10*11).all():
+                df[self.date_column] = pd.to_datetime(df[self.date_column], unit="s")
+            else:
+                msg = f"Unsupported type of date column {self.date_column}. Convert to datetime please."
+                self.logger.warning(msg)
+                raise ValidationError(msg)
 
         # If column with date is datetime then extract seconds of the day and minute of the hour
         # as additional features

upgini-1.1.237a2/src/upgini/utils/deduplicate_utils.py

@@ -0,0 +1,72 @@
+from logging import Logger
+from typing import Dict, List, Optional, Union
+
+import pandas as pd
+
+from upgini.metadata import TARGET, ModelTaskType, SearchKey
+from upgini.resource_bundle import bundle
+from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
+from upgini.utils.target_utils import define_task
+
+
+def remove_fintech_duplicates(df: pd.DataFrame, 
+                              search_keys: Dict[str, SearchKey], 
+                              logger: Optional[Logger] = None) -> pd.DataFrame:
+    if define_task(df.target, silent=True) != ModelTaskType.BINARY:
+        return df
+    
+    date_col = _get_column_by_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
+    if date_col is None:
+        return df
+    
+    personal_cols = []
+    phone_col = _get_column_by_key(search_keys, SearchKey.PHONE)
+    if phone_col:
+        personal_cols.append(phone_col)
+    email_col = _get_column_by_key(search_keys, SearchKey.EMAIL)
+    if email_col:
+        personal_cols.append(email_col)
+    hem_col = _get_column_by_key(search_keys, SearchKey.HEM)
+    if hem_col:
+        personal_cols.append(hem_col)
+    if len(personal_cols) == 0:
+        return df
+    
+    duplicates = df.duplicated(personal_cols, keep=False)
+    duplicate_rows = df[duplicates]
+    if len(duplicate_rows) == 0:
+        return df
+    
+    grouped_by_personal_cols = df.groupby(personal_cols, group_keys=False)
+    
+    uniques = grouped_by_personal_cols[date_col].nunique()
+    total = len(uniques)
+    diff_dates = len(uniques[uniques > 1])
+    if diff_dates / total >= 0.6:
+        return df
+    
+    if grouped_by_personal_cols[TARGET].apply(lambda x: len(x.unique()) == 1).all():
+        return df
+
+    def has_diff_target_within_60_days(rows):
+        rows = rows.sort_values(by=date_col)
+        return len(rows[rows[TARGET].ne(rows[TARGET].shift()) & (rows[date_col].diff() < 60 * 24 * 60 * 60 * 1000)]) > 0
+    
+    df = DateTimeSearchKeyConverter(date_col).convert(df)
+    grouped_by_personal_cols = df.groupby(personal_cols, group_keys=False)
+    rows_with_diff_target = grouped_by_personal_cols.filter(has_diff_target_within_60_days)
+    if len(rows_with_diff_target) > 0:
+        perc = len(rows_with_diff_target) * 100 / len(df)
+        msg = bundle.get("dataset_diff_target_duplicates_fintech").format(perc, len(rows_with_diff_target), rows_with_diff_target.index.to_list())
+        print(msg)
+        if logger:
+            logger.warning(msg)
+        df = df[~df.index.isin(rows_with_diff_target.index)]
+    
+    return df
+
+
+def _get_column_by_key(search_keys: Dict[str, SearchKey], keys: Union[SearchKey, List[SearchKey]]) -> Optional[str]:
+    for col, key_type in search_keys.items():
+        if (isinstance(keys, list) and key_type in keys) or key_type == keys:
+            return col

{upgini-1.1.236a2 → upgini-1.1.237a2/src/upgini.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.236a2
+Version: 1.1.237a2
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.236a2 → upgini-1.1.237a2}/src/upgini.egg-info/SOURCES.txt

@@ -49,6 +49,7 @@ src/upgini/utils/country_utils.py
 src/upgini/utils/custom_loss_utils.py
 src/upgini/utils/cv_utils.py
 src/upgini/utils/datetime_utils.py
+src/upgini/utils/deduplicate_utils.py
 src/upgini/utils/display_utils.py
 src/upgini/utils/email_utils.py
 src/upgini/utils/fallback_progress_bar.py