upgini 1.1.236a1__py3-none-any.whl → 1.1.237__py3-none-any.whl

upgini/data_source/data_source_publisher.py

@@ -62,6 +62,7 @@ class DataSourcePublisher:
         trace_id = str(uuid.uuid4())
 
         with MDC(trace_id=trace_id):
+            task_id = None
             try:
                 if data_table_uri is None or not data_table_uri.startswith("bq://"):
                     raise ValidationError(
@@ -148,6 +149,12 @@ class DataSourcePublisher:
                 self.logger.info(msg)
                 print(msg)
                 return data_table_id
+            except KeyboardInterrupt:
+                if task_id is not None:
+                    msg = f"Stopping AdsManagementTask {task_id}"
+                    print(msg)
+                    self.logger.info(msg)
+                    self._rest_client.stop_ads_management_task(task_id, trace_id)
             except Exception:
                 self.logger.exception("Failed to register data table")
                 raise

upgini/dataset.py

@@ -382,7 +382,7 @@ class Dataset:  # (pd.DataFrame):
 
             if is_string_dtype(self.data[postal_code]):
                 try:
-                    self.data[postal_code] = self.data[postal_code].astype("Float64").astype("Int64").astype("string")
+                    self.data[postal_code] = self.data[postal_code].astype("float64").astype("Int64").astype("string")
                 except Exception:
                     pass
             elif is_float_dtype(self.data[postal_code]):

upgini/features_enricher.py

@@ -21,8 +21,8 @@ from scipy.stats import ks_2samp
 from sklearn.base import TransformerMixin
 from sklearn.exceptions import NotFittedError
 from sklearn.model_selection import BaseCrossValidator
-from upgini.autofe.feature import Feature
 
+from upgini.autofe.feature import Feature
 from upgini.data_source.data_source_publisher import CommercialSchema
 from upgini.dataset import Dataset
 from upgini.errors import HttpError, ValidationError
@@ -1450,15 +1450,17 @@ class FeaturesEnricher(TransformerMixin):
             rows_to_drop=rows_to_drop,
         )
 
-        original_df_sampled = self.df_with_original_index[self.df_with_original_index[SYSTEM_RECORD_ID].isin(fit_features[SYSTEM_RECORD_ID])]
+        original_df_sampled = self.df_with_original_index[
+            self.df_with_original_index[SYSTEM_RECORD_ID].isin(fit_features[SYSTEM_RECORD_ID])
+            ]
         enriched_X = drop_existing_columns(enriched_Xy, TARGET)
         if EVAL_SET_INDEX in original_df_sampled.columns:
-            X_sampled = original_df_sampled.query(f"{EVAL_SET_INDEX} == 0")
+            Xy_sampled = original_df_sampled.query(f"{EVAL_SET_INDEX} == 0")
         else:
-            X_sampled = original_df_sampled
-        X_sampled = drop_existing_columns(X_sampled, [SYSTEM_RECORD_ID, EVAL_SET_INDEX, TARGET])
+            Xy_sampled = original_df_sampled
+        X_sampled = drop_existing_columns(Xy_sampled, [SYSTEM_RECORD_ID, EVAL_SET_INDEX, TARGET])
         search_keys = self.fit_search_keys
-        y_sampled = enriched_Xy[TARGET].copy()
+        y_sampled = Xy_sampled[TARGET].copy()
 
         self.logger.info(f"Shape of enriched_X: {enriched_X.shape}")
         self.logger.info(f"Shape of X after sampling: {X_sampled.shape}")
@@ -1472,9 +1474,9 @@ class FeaturesEnricher(TransformerMixin):
 
             for idx in range(len(eval_set)):
                 enriched_eval_X = drop_existing_columns(enriched_eval_sets[idx + 1], TARGET)
-                eval_X_sampled = original_df_sampled.query(f"{EVAL_SET_INDEX} == {idx + 1}")
-                eval_X_sampled = drop_existing_columns(eval_X_sampled, [SYSTEM_RECORD_ID, EVAL_SET_INDEX, TARGET])
-                eval_y_sampled = enriched_eval_sets[idx + 1][TARGET].copy()
+                eval_Xy_sampled = original_df_sampled.query(f"{EVAL_SET_INDEX} == {idx + 1}")
+                eval_X_sampled = drop_existing_columns(eval_Xy_sampled, [SYSTEM_RECORD_ID, EVAL_SET_INDEX, TARGET])
+                eval_y_sampled = eval_Xy_sampled[TARGET].copy()
                 eval_set_sampled_dict[idx] = (eval_X_sampled, enriched_eval_X, eval_y_sampled)
 
         self.__cached_sampled_datasets = (X_sampled, y_sampled, enriched_X, eval_set_sampled_dict, search_keys)

upgini/http.py

@@ -289,7 +289,7 @@ class _RestClient:
     GET_ALL_ADS_DESCRIPTIONS_URI = "private/api/v2/ads/descriptions"
     GET_ACTIVE_ADS_DEFINITIONS_URI = "private/api/v2/ads/definitions"
     UPLOAD_ONLINE_URI = "private/api/v2/ads/upload-online"
-    UPLOAD_ONLINE_ALL_URI = "private/api/v2/ads/upload-online-all"
+    STOP_ADS_MANAGEMENT_TASK_URI_FMT = "private/api/v2/ads/management-task/{0}/stop"
 
     ACCESS_TOKEN_HEADER_NAME = "Authorization"
     CONTENT_TYPE_HEADER_NAME = "Content-Type"
@@ -751,6 +751,10 @@ class _RestClient:
         response = self._with_unauth_retry(lambda: self._send_post_req(api_path, trace_id, request, result_format=None))
         return response["adsManagementTaskId"]
 
+    def stop_ads_management_task(self, ads_management_task_id: str, trace_id: str):
+        api_path = self.STOP_ADS_MANAGEMENT_TASK_URI_FMT.format(ads_management_task_id)
+        self._with_unauth_retry(lambda: self._send_post_req(api_path, trace_id))
+
     # ---
 
     def _send_get_req(self, api_path: str, trace_id: Optional[str], additional_headers: Optional[dict] = None):

upgini/utils/deduplicate_utils.py

@@ -0,0 +1,72 @@
+from logging import Logger
+from typing import Dict, List, Optional, Union
+
+import pandas as pd
+
+from upgini.metadata import TARGET, ModelTaskType, SearchKey
+from upgini.resource_bundle import bundle
+from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
+from upgini.utils.target_utils import define_task
+
+
+def remove_fintech_duplicates(df: pd.DataFrame, 
+                              search_keys: Dict[str, SearchKey], 
+                              logger: Optional[Logger] = None) -> pd.DataFrame:
+    if define_task(df.target, silent=True) != ModelTaskType.BINARY:
+        return df
+    
+    date_col = _get_column_by_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
+    if date_col is None:
+        return df
+    
+    personal_cols = []
+    phone_col = _get_column_by_key(search_keys, SearchKey.PHONE)
+    if phone_col:
+        personal_cols.append(phone_col)
+    email_col = _get_column_by_key(search_keys, SearchKey.EMAIL)
+    if email_col:
+        personal_cols.append(email_col)
+    hem_col = _get_column_by_key(search_keys, SearchKey.HEM)
+    if hem_col:
+        personal_cols.append(hem_col)
+    if len(personal_cols) == 0:
+        return df
+    
+    duplicates = df.duplicated(personal_cols, keep=False)
+    duplicate_rows = df[duplicates]
+    if len(duplicate_rows) == 0:
+        return df
+    
+    grouped_by_personal_cols = df.groupby(personal_cols, group_keys=False)
+    
+    uniques = grouped_by_personal_cols[date_col].nunique()
+    total = len(uniques)
+    diff_dates = len(uniques[uniques > 1])
+    if diff_dates / total >= 0.6:
+        return df
+    
+    if grouped_by_personal_cols[TARGET].apply(lambda x: len(x.unique()) == 1).all():
+        return df
+
+    def has_diff_target_within_60_days(rows):
+        rows = rows.sort_values(by=date_col)
+        return len(rows[rows[TARGET].ne(rows[TARGET].shift()) & (rows[date_col].diff() < 60 * 24 * 60 * 60 * 1000)]) > 0
+    
+    df = DateTimeSearchKeyConverter(date_col).convert(df)
+    grouped_by_personal_cols = df.groupby(personal_cols, group_keys=False)
+    rows_with_diff_target = grouped_by_personal_cols.filter(has_diff_target_within_60_days)
+    if len(rows_with_diff_target) > 0:
+        perc = len(rows_with_diff_target) * 100 / len(df)
+        msg = bundle.get("dataset_diff_target_duplicates_fintech").format(perc, len(rows_with_diff_target), rows_with_diff_target.index.to_list())
+        print(msg)
+        if logger:
+            logger.warning(msg)
+        df = df[~df.index.isin(rows_with_diff_target.index)]
+    
+    return df
+
+
+def _get_column_by_key(search_keys: Dict[str, SearchKey], keys: Union[SearchKey, List[SearchKey]]) -> Optional[str]:
+    for col, key_type in search_keys.items():
+        if (isinstance(keys, list) and key_type in keys) or key_type == keys:
+            return col

{upgini-1.1.236a1.dist-info → upgini-1.1.237.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.236a1
+Version: 1.1.237
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.236a1.dist-info → upgini-1.1.237.dist-info}/RECORD

@@ -1,10 +1,10 @@
 upgini/__init__.py,sha256=asENHgEVHQBIkV-e_0IhE_ZWqkCG6398U3ZLrNzAH6k,407
 upgini/ads.py,sha256=mre6xn44wcC_fg63iLT_kTh4mViZqR9AKRJZAtpQz8Y,2592
-upgini/dataset.py,sha256=7z9zbVvd1_MiufmoZlCwEHwQ25Q2DX_0g9PFcSMlqMY,49764
+upgini/dataset.py,sha256=BCgqbol6se2ZhVeaHC0DOOEnak-3jjeCAsHvjCQ_qRE,49764
 upgini/errors.py,sha256=BqpvfhW2jJW5fa5KXj0alhXatGl-WK4xTl309-QNLp8,959
-upgini/features_enricher.py,sha256=UVvLzfHNb3v-2CH8tLtVcvoQTcDQ5yKNGe3TQg4jaEQ,160363
+upgini/features_enricher.py,sha256=D-Rm00P8Cnl4A_730jAflWcGyEo425p0r1azwoU__B0,160381
 upgini/fingerprint.js,sha256=VygVIQlN1v4NGZfjHqtRogOw8zjTnnMNJg_f7M5iGQU,33442
-upgini/http.py,sha256=HzUSZudCdISJGUqHC1gAT1v_x1n_dIFVDJW4z3Q7DCs,41204
+upgini/http.py,sha256=b1nF0YL4n-kgoz3Ju_r0pKWn_-cGs56rugr8YtA08HA,41476
 upgini/metadata.py,sha256=FZ5CQluLLWrfrBVThSIes1SW6wcs7n50aNZwzYnHiF0,9584
 upgini/metrics.py,sha256=YeYHJtEIs8OG-EzidG-nbSYB919pjZ4MMbdcZ_jfV2s,23639
 upgini/search_task.py,sha256=7YxH1zrUHMmePO0VbPBBCJjeoer7jAC0Gltc9EVAOIg,17126
@@ -21,7 +21,7 @@ upgini/autofe/operand.py,sha256=8WqEoSIA5rEWCK1xuC303E4NW5a72GZ5jUMAEj4skII,2291
 upgini/autofe/unary.py,sha256=7TBe7PCt7l_XQEqu_G5g_TC2cW3tppL7uPDcX8xsqz0,2731
 upgini/autofe/vector.py,sha256=Qk7VmdwURNwVw7fIMEspWEo7HTiyUWCYIqu3hcWQQio,507
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/data_source/data_source_publisher.py,sha256=zFu0WMKwPM11gPZHq8dpsBP7s4wmTtBqYoDEakgNxoY,13725
+upgini/data_source/data_source_publisher.py,sha256=xvHi4N4m32eqB_h_qtY1wAt1dXekM5PdNL2T9JzFQD4,14051
 upgini/mdc/__init__.py,sha256=CuKmWYCqAnmiq1S7wgMzJhSCTsXuoeiZWXSfzw0lyig,1152
 upgini/mdc/context.py,sha256=eVNEubcgkiAP139Vna2qtUBZJWoy15rWWAuB0TFv54E,1484
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -40,6 +40,7 @@ upgini/utils/country_utils.py,sha256=9BXSXoGm3nVoOZE_bRENY-KMkwMUFvAF3Au0zxUNA1o
 upgini/utils/custom_loss_utils.py,sha256=DBslpjWGPt7xTeypt78baR59012SYphbPsO_YLKdilo,3972
 upgini/utils/cv_utils.py,sha256=6pSSL_Ft_8C6n6aInJeiyeSBD7McjsMxKZpHqSBV0uY,2491
 upgini/utils/datetime_utils.py,sha256=P56e7gcgAogJYfs2Blzk1uypxb9yrFzNaeJpMCRm6Zc,7716
+upgini/utils/deduplicate_utils.py,sha256=o-XY0hbqikQTzwpX0nyl34j_oiBQTefCvRgLHkZRkTE,2795
 upgini/utils/display_utils.py,sha256=tiq5sFOfMwkKCjQ7OGdyK_twe0Qdr9F3mzkW1QXSDog,10664
 upgini/utils/email_utils.py,sha256=MhCLUAWqbp81xRyKizauNhVx6t_MFeJQRQ8pFM7EpFo,3480
 upgini/utils/fallback_progress_bar.py,sha256=f-VzVbiO6oU9WoKzEgoegYotixdiKanGlvdQCOGC-NY,1128
@@ -53,8 +54,8 @@ upgini/utils/sklearn_ext.py,sha256=IMx2La70AXAggApVpT7sMEjWqVWon5AMZt4MARDsIMQ,4
 upgini/utils/target_utils.py,sha256=cu52icjhDIPpEStHYMXrD2hIl9gzvfnxZr0Ra5osV0k,1616
 upgini/utils/track_info.py,sha256=DVNVZmXUb4f25DSPEuUNEFx49hNEBfmuY9iSW5jkMnI,5708
 upgini/utils/warning_counter.py,sha256=vnmdFo5-7GBkU2bK9h_uC0K0Y_wtfcYstxOdeRfacO0,228
-upgini-1.1.236a1.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.236a1.dist-info/METADATA,sha256=sN7MK3RFbL1ZAmkoeDYkESEGE1vvQvO_0eddGFPvz6c,48346
-upgini-1.1.236a1.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-upgini-1.1.236a1.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
-upgini-1.1.236a1.dist-info/RECORD,,
+upgini-1.1.237.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.237.dist-info/METADATA,sha256=A5a-AaJ1JMDFrGIwcV3sjlF3zMfWB9yrEvzKx2ZM6HE,48344
+upgini-1.1.237.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+upgini-1.1.237.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
+upgini-1.1.237.dist-info/RECORD,,