upgini 1.1.237a2__py3-none-any.whl → 1.1.239a1__py3-none-any.whl

upgini/autofe/operand.py

@@ -1,5 +1,5 @@
 from pydantic import BaseModel
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple, Union
 import abc
 import pandas as pd
 import numpy as np
@@ -59,6 +59,16 @@ class PandasOperand(Operand, abc.ABC):
         df_from.loc[np.nan] = np.nan
         return df_to.fillna(np.nan).apply(lambda x: df_from.loc[x])
 
+    def _round_value(self, value: Union[pd.Series, pd.DataFrame]) -> Union[pd.Series, pd.DataFrame]:
+        if isinstance(value, pd.DataFrame):
+            return value.apply(self._round_value, axis=1)
+
+        if np.issubdtype(value.dtype, np.floating):
+            precision = np.finfo(value.dtype).precision
+            return np.trunc(value * 10**precision) / (10**precision)
+        else:
+            return value
+
 
 class VectorizableMixin(Operand):
     def validate_calculation(self, input_columns: List[str], **kwargs) -> Tuple[str, List[str]]:

upgini/autofe/unary.py

@@ -22,10 +22,10 @@ class Log(PandasOperand):
     output_type = "float"
 
     def calculate_unary(self, data: pd.Series) -> pd.Series:
-        return np.log(np.abs(data.replace(0, np.nan)))
+        return self._round_value(np.log(np.abs(data.replace(0, np.nan))))
 
     def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
-        return np.log(data.replace(0, np.nan).abs())
+        return self._round_value(np.log(data.replace(0, np.nan).abs()))
 
 
 class Sqrt(PandasOperand):
@@ -35,10 +35,10 @@ class Sqrt(PandasOperand):
     output_type = "float"
 
     def calculate_unary(self, data: pd.Series) -> pd.Series:
-        return np.sqrt(np.abs(data))
+        return self._round_value(np.sqrt(np.abs(data)))
 
     def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
-        return np.sqrt(data.abs())
+        return self._round_value(np.sqrt(data.abs()))
 
 
 class Square(PandasOperand):
@@ -60,10 +60,10 @@ class Sigmoid(PandasOperand):
     output_type = "float"
 
     def calculate_unary(self, data: pd.Series) -> pd.Series:
-        return 1 / (1 + np.exp(-data))
+        return self._round_value(1 / (1 + np.exp(-data)))
 
     def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
-        return 1 / (1 + np.exp(-data))
+        return self._round_value(1 / (1 + np.exp(-data)))
 
 
 class Floor(PandasOperand):

upgini/data_source/data_source_publisher.py

@@ -62,6 +62,7 @@ class DataSourcePublisher:
         trace_id = str(uuid.uuid4())
 
         with MDC(trace_id=trace_id):
+            task_id = None
             try:
                 if data_table_uri is None or not data_table_uri.startswith("bq://"):
                     raise ValidationError(
@@ -148,6 +149,12 @@ class DataSourcePublisher:
                 self.logger.info(msg)
                 print(msg)
                 return data_table_id
+            except KeyboardInterrupt:
+                if task_id is not None:
+                    msg = f"Stopping AdsManagementTask {task_id}"
+                    print(msg)
+                    self.logger.info(msg)
+                    self._rest_client.stop_ads_management_task(task_id, trace_id)
             except Exception:
                 self.logger.exception("Failed to register data table")
                 raise

upgini/dataset.py

@@ -36,14 +36,12 @@ from upgini.metadata import (
     NumericInterval,
     RuntimeParameters,
     SearchCustomization,
-    SearchKey,
 )
 from upgini.normalizer.phone_normalizer import PhoneNormalizer
 from upgini.resource_bundle import bundle
 from upgini.sampler.random_under_sampler import RandomUnderSampler
 from upgini.search_task import SearchTask
 from upgini.utils import combine_search_keys
-from upgini.utils.deduplicate_utils import remove_fintech_duplicates
 from upgini.utils.email_utils import EmailSearchKeyConverter
 
 try:
@@ -83,7 +81,6 @@ class Dataset:  # (pd.DataFrame):
         endpoint: Optional[str] = None,
         api_key: Optional[str] = None,
         logger: Optional[logging.Logger] = None,
-        client_ip: Optional[str] = None,
         warning_counter: Optional[WarningCounter] = None,
         **kwargs,
     ):
@@ -127,7 +124,6 @@ class Dataset:  # (pd.DataFrame):
         else:
             self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
-        self.client_ip = client_ip
         self.warning_counter = warning_counter or WarningCounter()
 
     def __len__(self):
@@ -822,13 +818,6 @@ class Dataset:  # (pd.DataFrame):
         self.__validate_dataset(validate_target, silent_mode)
 
         if validate_target:
-            search_keys = {
-                col: SearchKey.from_meaning_type(key_type)
-                for col, key_type in self.meaning_types.items()
-                if SearchKey.from_meaning_type(key_type) is not None
-                }
-            self.data = remove_fintech_duplicates(self.data, search_keys, self.logger)
-
             self.__validate_target()
 
             self.__resample()
@@ -1028,7 +1017,7 @@ class Dataset:  # (pd.DataFrame):
             task_type=self.task_type,
             endpoint=self.endpoint,
             api_key=self.api_key,
-            client_ip=self.client_ip,
+            logger=self.logger,
         )
 
     def validation(
@@ -1098,7 +1087,7 @@ class Dataset:  # (pd.DataFrame):
             initial_search_task_id=initial_search_task_id,
             endpoint=self.endpoint,
             api_key=self.api_key,
-            client_ip=self.client_ip,
+            logger=self.logger,
         )
 
     def prepare_uploading_file(self, base_path: str) -> str:

upgini/features_enricher.py

@@ -21,8 +21,8 @@ from scipy.stats import ks_2samp
 from sklearn.base import TransformerMixin
 from sklearn.exceptions import NotFittedError
 from sklearn.model_selection import BaseCrossValidator
-from upgini.autofe.feature import Feature
 
+from upgini.autofe.feature import Feature
 from upgini.data_source.data_source_publisher import CommercialSchema
 from upgini.dataset import Dataset
 from upgini.errors import HttpError, ValidationError
@@ -64,7 +64,6 @@ from upgini.utils.datetime_utils import (
     is_blocked_time_series,
     is_time_series,
 )
-from upgini.utils.deduplicate_utils import remove_fintech_duplicates
 from upgini.utils.display_utils import (
     display_html_dataframe,
     do_without_pandas_limits,
@@ -181,17 +180,19 @@ class FeaturesEnricher(TransformerMixin):
         exclude_columns: Optional[List[str]] = None,
         baseline_score_column: Optional[Any] = None,
         client_ip: Optional[str] = None,
+        client_visitorid: Optional[str] = None,
         **kwargs,
     ):
         self._api_key = api_key or os.environ.get(UPGINI_API_KEY)
         if api_key is not None and not isinstance(api_key, str):
             raise ValidationError(f"api_key should be `string`, but passed: `{api_key}`")
-        self.rest_client = get_rest_client(endpoint, self._api_key)
+        self.rest_client = get_rest_client(endpoint, self._api_key, client_ip, client_visitorid)
         self.client_ip = client_ip
+        self.client_visitorid = client_visitorid
 
         self.logs_enabled = logs_enabled
         if logs_enabled:
-            self.logger = LoggerFactory().get_logger(endpoint, self._api_key, client_ip)
+            self.logger = LoggerFactory().get_logger(endpoint, self._api_key, client_ip, client_visitorid)
         else:
             self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
@@ -232,7 +233,7 @@ class FeaturesEnricher(TransformerMixin):
         self.feature_importances_ = []
         self.search_id = search_id
         if search_id:
-            search_task = SearchTask(search_id, endpoint=self.endpoint, api_key=self._api_key, client_ip=client_ip)
+            search_task = SearchTask(search_id, endpoint=self.endpoint, api_key=self._api_key, logger=self.logger)
 
             print(bundle.get("search_by_task_id_start"))
             trace_id = str(uuid.uuid4())
@@ -296,7 +297,7 @@ class FeaturesEnricher(TransformerMixin):
     def _set_api_key(self, api_key: str):
         self._api_key = api_key
         if self.logs_enabled:
-            self.logger = LoggerFactory().get_logger(self.endpoint, self._api_key, self.client_ip)
+            self.logger = LoggerFactory().get_logger(self.endpoint, self._api_key, self.client_ip, self.client_visitorid)
 
     api_key = property(_get_api_key, _set_api_key)
 
@@ -679,7 +680,7 @@ class FeaturesEnricher(TransformerMixin):
                     return None
 
                 if not metrics_calculation:
-                    transform_usage = get_rest_client(self.endpoint, self.api_key).get_current_transform_usage(trace_id)
+                    transform_usage = self.rest_client.get_current_transform_usage(trace_id)
                     self.logger.info(f"Current transform usage: {transform_usage}. Transforming {len(X)} rows")
                     if transform_usage.has_limit:
                         if len(X) > transform_usage.rest_rows:
@@ -1184,8 +1185,8 @@ class FeaturesEnricher(TransformerMixin):
             converter = DateTimeSearchKeyConverter(date_column, self.date_format, self.logger)
             extended_X = converter.convert(extended_X, keep_time=True)
             generated_features.extend(converter.generated_features)
-        email_column = self._get_email_column(search_keys)
-        hem_column = self._get_hem_column(search_keys)
+        email_column = self.__get_email_column(search_keys)
+        hem_column = self.__get_hem_column(search_keys)
         if email_column:
             converter = EmailSearchKeyConverter(email_column, hem_column, search_keys, self.logger)
             extended_X = converter.convert(extended_X)
@@ -1451,7 +1452,9 @@ class FeaturesEnricher(TransformerMixin):
             rows_to_drop=rows_to_drop,
         )
 
-        original_df_sampled = self.df_with_original_index[self.df_with_original_index[SYSTEM_RECORD_ID].isin(fit_features[SYSTEM_RECORD_ID])]
+        original_df_sampled = self.df_with_original_index[
+            self.df_with_original_index[SYSTEM_RECORD_ID].isin(fit_features[SYSTEM_RECORD_ID])
+            ]
         enriched_X = drop_existing_columns(enriched_Xy, TARGET)
         if EVAL_SET_INDEX in original_df_sampled.columns:
             Xy_sampled = original_df_sampled.query(f"{EVAL_SET_INDEX} == 0")
@@ -1506,8 +1509,6 @@ class FeaturesEnricher(TransformerMixin):
                 eval_df_with_index[TARGET] = eval_y
                 eval_df_with_index[EVAL_SET_INDEX] = idx + 1
                 df_with_eval_set_index = pd.concat([df_with_eval_set_index, eval_df_with_index])
-            
-            df_with_eval_set_index = remove_fintech_duplicates(df_with_eval_set_index, self.search_keys, self.logger)
 
             # downsample if need to eval_set threshold
             num_samples = _num_samples(df_with_eval_set_index)
@@ -1744,8 +1745,8 @@ class FeaturesEnricher(TransformerMixin):
                 generated_features.extend(converter.generated_features)
             else:
                 self.logger.info("Input dataset hasn't date column")
-            email_column = self._get_email_column(search_keys)
-            hem_column = self._get_hem_column(search_keys)
+            email_column = self.__get_email_column(search_keys)
+            hem_column = self.__get_hem_column(search_keys)
             email_converted_to_hem = False
             if email_column:
                 converter = EmailSearchKeyConverter(email_column, hem_column, search_keys, self.logger)
@@ -1806,7 +1807,6 @@ class FeaturesEnricher(TransformerMixin):
                 api_key=self.api_key,  # type: ignore
                 date_format=self.date_format,  # type: ignore
                 logger=self.logger,
-                client_ip=self.client_ip,
             )
             dataset.meaning_types = meaning_types
             dataset.search_keys = combined_search_keys
@@ -1869,7 +1869,7 @@ class FeaturesEnricher(TransformerMixin):
                     progress = self.get_progress(trace_id, validation_task)
             except KeyboardInterrupt as e:
                 print(bundle.get("search_stopping"))
-                get_rest_client(self.endpoint, self.api_key).stop_search_task_v2(
+                self.rest_client.stop_search_task_v2(
                     trace_id, validation_task.search_task_id
                 )
                 self.logger.warning(f"Search {validation_task.search_task_id} stopped by user")
@@ -2084,8 +2084,8 @@ class FeaturesEnricher(TransformerMixin):
             self.fit_generated_features.extend(converter.generated_features)
         else:
             self.logger.info("Input dataset hasn't date column")
-        email_column = self._get_email_column(self.fit_search_keys)
-        hem_column = self._get_hem_column(self.fit_search_keys)
+        email_column = self.__get_email_column(self.fit_search_keys)
+        hem_column = self.__get_hem_column(self.fit_search_keys)
         email_converted_to_hem = False
         if email_column:
             converter = EmailSearchKeyConverter(email_column, hem_column, self.fit_search_keys, self.logger)
@@ -2141,7 +2141,6 @@ class FeaturesEnricher(TransformerMixin):
             date_format=self.date_format,  # type: ignore
             random_state=self.random_state,  # type: ignore
             logger=self.logger,
-            client_ip=self.client_ip,
         )
         dataset.meaning_types = meaning_types
         dataset.search_keys = combined_search_keys
@@ -2198,7 +2197,7 @@ class FeaturesEnricher(TransformerMixin):
                 progress = self.get_progress(trace_id)
         except KeyboardInterrupt as e:
             print(bundle.get("search_stopping"))
-            get_rest_client(self.endpoint, self.api_key).stop_search_task_v2(trace_id, self._search_task.search_task_id)
+            self.rest_client.stop_search_task_v2(trace_id, self._search_task.search_task_id)
             self.logger.warning(f"Search {self._search_task.search_task_id} stopped by user")
             print(bundle.get("search_stopped"))
             raise e
@@ -2618,22 +2617,16 @@ class FeaturesEnricher(TransformerMixin):
         return [col for col, t in search_keys.items() if t not in [SearchKey.DATE, SearchKey.DATETIME]]
 
     @staticmethod
-    def _get_email_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
+    def __get_email_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
         for col, t in search_keys.items():
             if t == SearchKey.EMAIL:
                 return col
 
     @staticmethod
-    def _get_hem_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
+    def __get_hem_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
         for col, t in search_keys.items():
             if t == SearchKey.HEM:
                 return col
-    
-    @staticmethod
-    def _get_phone_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
-        for col, t in search_keys.items():
-            if t == SearchKey.PHONE:
-                return col
 
     def __add_fit_system_record_id(
         self, df: pd.DataFrame, meaning_types: Dict[str, FileColumnMeaningType], search_keys: Dict[str, SearchKey]
@@ -3190,7 +3183,7 @@ class FeaturesEnricher(TransformerMixin):
                 metrics_df=self.metrics,
                 autofe_descriptions_df=self.get_autofe_features_description(),
                 search_id=self._search_task.search_task_id,
-                email=get_rest_client(self.endpoint, self.api_key).get_current_email(),
+                email=self.rest_client.get_current_email(),
                 search_keys=[str(sk) for sk in self.search_keys.values()],
             )
         except Exception:
@@ -3374,7 +3367,7 @@ class FeaturesEnricher(TransformerMixin):
                                 pickle.dump(sample(eval_set[0][0], eval_xy_sample_index), eval_x_file)
                             with open(f"{tmp_dir}/eval_y.pickle", "wb") as eval_y_file:
                                 pickle.dump(sample(eval_set[0][1], eval_xy_sample_index), eval_y_file)
-                            get_rest_client(self.endpoint, self.api_key).dump_input_files(
+                            self.rest_client.dump_input_files(
                                 trace_id,
                                 f"{tmp_dir}/x.pickle",
                                 f"{tmp_dir}/y.pickle",
@@ -3382,13 +3375,13 @@ class FeaturesEnricher(TransformerMixin):
                                 f"{tmp_dir}/eval_y.pickle",
                             )
                         else:
-                            get_rest_client(self.endpoint, self.api_key).dump_input_files(
+                            self.rest_client.dump_input_files(
                                 trace_id,
                                 f"{tmp_dir}/x.pickle",
                                 f"{tmp_dir}/y.pickle",
                             )
                     else:
-                        get_rest_client(self.endpoint, self.api_key).dump_input_files(
+                        self.rest_client.dump_input_files(
                             trace_id,
                             f"{tmp_dir}/x.pickle",
                         )

upgini/http.py

@@ -289,7 +289,7 @@ class _RestClient:
     GET_ALL_ADS_DESCRIPTIONS_URI = "private/api/v2/ads/descriptions"
     GET_ACTIVE_ADS_DEFINITIONS_URI = "private/api/v2/ads/definitions"
     UPLOAD_ONLINE_URI = "private/api/v2/ads/upload-online"
-    UPLOAD_ONLINE_ALL_URI = "private/api/v2/ads/upload-online-all"
+    STOP_ADS_MANAGEMENT_TASK_URI_FMT = "private/api/v2/ads/management-task/{0}/stop"
 
     ACCESS_TOKEN_HEADER_NAME = "Authorization"
     CONTENT_TYPE_HEADER_NAME = "Content-Type"
@@ -301,11 +301,13 @@ class _RestClient:
     USER_AGENT_HEADER_VALUE = "pyupgini/" + __version__
     SEARCH_KEYS_HEADER_NAME = "Search-Keys"
 
-    def __init__(self, service_endpoint, refresh_token, silent_mode=False):
+    def __init__(self, service_endpoint, refresh_token, silent_mode=False, client_ip=None, client_visitorid=None):
         # debug_requests_on()
         self._service_endpoint = service_endpoint
         self._refresh_token = refresh_token
         self.silent_mode = silent_mode
+        self.client_ip = client_ip
+        self.client_visitorid = client_visitorid
         self._access_token = self._refresh_access_token()
         # self._access_token: Optional[str] = None  # self._refresh_access_token()
         self.last_refresh_time = time.time()
@@ -470,7 +472,7 @@ class _RestClient:
                     )
                 files["tracking"] = (
                     "tracking.json",
-                    dumps(get_track_metrics()).encode(),
+                    dumps(get_track_metrics(self.client_ip, self.client_visitorid)).encode(),
                     "application/json",
                 )
                 additional_headers = {self.SEARCH_KEYS_HEADER_NAME: ",".join(self.search_keys_meaning_types(metadata))}
@@ -554,7 +556,7 @@ class _RestClient:
                     )
                 files["tracking"] = (
                     "ide",
-                    dumps(get_track_metrics()).encode(),
+                    dumps(get_track_metrics(self.client_ip, self.client_visitorid)).encode(),
                     "application/json",
                 )
 
@@ -662,7 +664,7 @@ class _RestClient:
         return ProviderTaskMetadataV2.parse_obj(response)
 
     def get_current_transform_usage(self, trace_id) -> TransformUsage:
-        track_metrics = get_track_metrics()
+        track_metrics = get_track_metrics(self.client_ip, self.client_visitorid)
         visitor_id = track_metrics.get("visitorId")
         response = self._with_unauth_retry(
             lambda: self._send_get_req(
@@ -751,6 +753,10 @@ class _RestClient:
         response = self._with_unauth_retry(lambda: self._send_post_req(api_path, trace_id, request, result_format=None))
         return response["adsManagementTaskId"]
 
+    def stop_ads_management_task(self, ads_management_task_id: str, trace_id: str):
+        api_path = self.STOP_ADS_MANAGEMENT_TASK_URI_FMT.format(ads_management_task_id)
+        self._with_unauth_retry(lambda: self._send_post_req(api_path, trace_id))
+
     # ---
 
     def _send_get_req(self, api_path: str, trace_id: Optional[str], additional_headers: Optional[dict] = None):
@@ -901,11 +907,12 @@ def resolve_api_token(api_token: Optional[str]) -> str:
         return DEMO_API_KEY
 
 
-def get_rest_client(backend_url: Optional[str] = None, api_token: Optional[str] = None) -> _RestClient:
+def get_rest_client(backend_url: Optional[str] = None, api_token: Optional[str] = None,
+                    client_ip: Optional[str] = None, client_visitorid: Optional[str] = None) -> _RestClient:
     url = _resolve_backend_url(backend_url)
     token = resolve_api_token(api_token)
 
-    return _get_rest_client(url, token)
+    return _get_rest_client(url, token, client_ip, client_visitorid)
 
 
 def is_demo_api_key(api_token: Optional[str]) -> bool:
@@ -913,23 +920,27 @@ def is_demo_api_key(api_token: Optional[str]) -> bool:
 
 
 @lru_cache()
-def _get_rest_client(backend_url: str, api_token: str) -> _RestClient:
+def _get_rest_client(backend_url: str, api_token: str,
+                     client_ip: Optional[str] = None, client_visitorid: Optional[str] = None) -> _RestClient:
     return _RestClient(backend_url, api_token)
 
 
 class BackendLogHandler(logging.Handler):
-    def __init__(self, rest_client: _RestClient, client_ip: Optional[str] = None, *args, **kwargs) -> None:
+    def __init__(self, rest_client: _RestClient, 
+                 client_ip: Optional[str] = None, client_visitorid: Optional[str] = None, 
+                 *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self.rest_client = rest_client
         self.track_metrics = None
         self.hostname = "0.0.0.0"
         self.client_ip = client_ip
+        self.client_visitorid = client_visitorid
 
     def emit(self, record: logging.LogRecord) -> None:
         def task():
             try:
                 if self.track_metrics is None or len(self.track_metrics) == 0:
-                    self.track_metrics = get_track_metrics(self.client_ip)
+                    self.track_metrics = get_track_metrics(self.client_ip, self.client_visitorid)
                     self.hostname = self.track_metrics.get("ip") or "0.0.0.0"
                 text = self.format(record)
                 tags = self.track_metrics
@@ -971,7 +982,8 @@ class LoggerFactory:
         root.handlers.clear()
 
     def get_logger(
-        self, backend_url: Optional[str] = None, api_token: Optional[str] = None, client_ip: Optional[str] = None
+        self, backend_url: Optional[str] = None, api_token: Optional[str] = None, 
+        client_ip: Optional[str] = None, client_visitorid: Optional[str] = None
     ) -> logging.Logger:
         url = _resolve_backend_url(backend_url)
         token = resolve_api_token(api_token)
@@ -983,7 +995,7 @@ class LoggerFactory:
         upgini_logger = logging.getLogger(f"upgini.{hash(key)}")
         upgini_logger.handlers.clear()
         rest_client = get_rest_client(backend_url, api_token)
-        datadog_handler = BackendLogHandler(rest_client, client_ip)
+        datadog_handler = BackendLogHandler(rest_client, client_ip, client_visitorid)
         json_formatter = jsonlogger.JsonFormatter(
             "%(asctime)s %(threadName)s %(name)s %(levelname)s %(message)s",
             timestamp=True,

upgini/resource_bundle/strings.properties

@@ -142,7 +142,6 @@ dataset_empty_column_names=Some column names are empty. Add names please
 dataset_too_long_column_name=Column {} is too long: {} characters. Remove this column or trim length to 50 characters
 dataset_full_duplicates=\nWARNING: {:.5f}% of the rows are fully duplicated
 dataset_diff_target_duplicates=\nWARNING: {:.4f}% of rows ({}) in X and eval_set are duplicates with different y values. These rows will be deleted as incorrect\nIncorrect row indexes: {}
-dataset_diff_target_duplicates_fintech=\nWARNING: {:.4f}% of rows ({}) in X and eval_set are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
 dataset_drop_old_dates=\nWARNING: We don't have data before '2000-01-01' and removed all earlier records from the search dataset
 dataset_all_dates_old=There is empty train dataset after removing data before '2000-01-01'
 dataset_invalid_target_type=Unexpected dtype of target for binary task type: {}. Expected int or bool

upgini/search_task.py

@@ -1,3 +1,4 @@
+import logging
 import tempfile
 import time
 from functools import lru_cache
@@ -43,7 +44,7 @@ class SearchTask:
         task_type: Optional[ModelTaskType] = None,
         endpoint: Optional[str] = None,
         api_key: Optional[str] = None,
-        client_ip: Optional[str] = None,
+        logger: Optional[logging.Logger] = None,
     ):
         self.search_task_id = search_task_id
         self.initial_search_task_id = initial_search_task_id
@@ -55,7 +56,11 @@ class SearchTask:
         self.summary = None
         self.endpoint = endpoint
         self.api_key = api_key
-        self.logger = LoggerFactory().get_logger(endpoint, api_key, client_ip)
+        if logger is not None:
+            self.logger = logger
+        else:
+            self.logger = logging.getLogger()
+            self.logger.setLevel("FATAL")
         self.provider_metadata_v2: Optional[List[ProviderTaskMetadataV2]] = None
         self.unused_features_for_generation: Optional[List[str]] = None
 

upgini/utils/datetime_utils.py

@@ -61,22 +61,9 @@ class DateTimeSearchKeyConverter:
         elif is_period_dtype(df[self.date_column]):
             df[self.date_column] = pd.to_datetime(df[self.date_column].astype("string"))
         elif is_numeric_dtype(df[self.date_column]):
-            # 315532801 - 2524608001    - seconds
-            # 315532801000 - 2524608001000 - milliseconds
-            # 315532801000000 - 2524608001000000 - microseconds
-            # 315532801000000000 - 2524608001000000000 - nanoseconds
-            if df[self.date_column].apply(lambda x: 10**16 < x).all():
-                df[self.date_column] = pd.to_datetime(df[self.date_column], unit="ns")
-            elif df[self.date_column].apply(lambda x: 10**14 < x < 10**16).all():
-                df[self.date_column] = pd.to_datetime(df[self.date_column], unit="us")
-            elif df[self.date_column].apply(lambda x: 10**11 < x < 10**14).all():
-                df[self.date_column] = pd.to_datetime(df[self.date_column], unit="ms")
-            elif df[self.date_column].apply(lambda x: 0 < x < 10*11).all():
-                df[self.date_column] = pd.to_datetime(df[self.date_column], unit="s")
-            else:
-                msg = f"Unsupported type of date column {self.date_column}. Convert to datetime please."
-                self.logger.warning(msg)
-                raise ValidationError(msg)
+            msg = f"Unsupported type of date column {self.date_column}. Convert to datetime please."
+            self.logger.warning(msg)
+            raise ValidationError(msg)
 
         # If column with date is datetime then extract seconds of the day and minute of the hour
         # as additional features

upgini/utils/track_info.py

@@ -36,18 +36,22 @@ def _env_contains(envs) -> bool:
 
 
 def _get_execution_ide() -> str:
-    if "google.colab" in sys.modules and _env_contains(_ide_env_variables["colab"]):
-        return "colab"
-    elif os.path.exists("/kaggle") and _check_installed("kaggle") and _env_contains(_ide_env_variables["kaggle"]):
-        return "kaggle"
-    elif getuser() == "jovyan" and _env_contains(_ide_env_variables["binder"]):
-        return "binder"
-    else:
+    try:
+        if "google.colab" in sys.modules and _env_contains(_ide_env_variables["colab"]):
+            return "colab"
+        elif os.path.exists("/kaggle") and _check_installed("kaggle") and _env_contains(_ide_env_variables["kaggle"]):
+            return "kaggle"
+        elif getuser() == "jovyan" and _env_contains(_ide_env_variables["binder"]):
+            return "binder"
+        elif "widget" in socket.gethostname():
+            return "widget"
+        else:
+            return "other"
+    except Exception:
         return "other"
 
-
 @lru_cache()
-def get_track_metrics(client_ip: Optional[str] = None) -> dict:
+def get_track_metrics(client_ip: Optional[str] = None, client_visitorid: Optional[str] = None) -> dict:
     # default values
     track = {"ide": _get_execution_ide()}
     ident_res = "https://api.ipify.org"
@@ -66,24 +70,10 @@ def get_track_metrics(client_ip: Optional[str] = None) -> dict:
             from google.colab import output  # type: ignore
             from IPython.display import Javascript, display
 
-            # path_to_script = Path(__file__).parent.parent.resolve() / "fingerprint.js"
-            # with open(path_to_script) as f:
-            #     js_content = f.read()
-            # print(f"JS loaded. Length: {len(js_content)}")
-
             display(
                 Javascript(
-                    # """
-                    #     async function loadModuleFromString(code) {
-                    #         const blob = new Blob([code], { type: 'application/javascript' });
-                    #         const url = URL.createObjectURL(blob);
-                    #         const module = await import(url);
-                    #         URL.revokeObjectURL(url); // Clean URL-object after module load
-                    #         return module;
-                    #     }
-                    #     window.visitorId = loadModuleFromString(""" + js_content + """)
                     """
-                        window.visitorId = import('https://openfpcdn.io/fingerprintjs/v3')
+                        window.visitorId = import('https://upgini.github.io/upgini/js/visitorid.js')
                             .then(FingerprintJS => FingerprintJS.load())
                             .then(fp => fp.get())
                             .then(result => result.visitorId);
@@ -153,7 +143,10 @@ def get_track_metrics(client_ip: Optional[str] = None) -> dict:
                 track["ip"] = client_ip
             else:
                 track["ip"] = get(ident_res, timeout=10).text
-            track["visitorId"] = sha256(str(getnode()).encode()).hexdigest()
+            if client_visitorid:
+                track["visitorId"] = client_visitorid
+            else:
+                track["visitorId"] = sha256(str(getnode()).encode()).hexdigest()
         except Exception as e:
             track["err"] = str(e)
 

{upgini-1.1.237a2.dist-info → upgini-1.1.239a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.237a2
+Version: 1.1.239a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers
@@ -823,11 +823,11 @@ Requests and support, in preferred order
 - **scoped to a Single Bug** - one bug per report.
 
 ## 🧩 Contributing
-We are a **very** small team and this is a part-time project for us, thus most probably we won't be able:
+We are not a large team, so we probably won't be able to:
  - implement smooth integration with most common low-code ML libraries and platforms ([PyCaret](https://www.github.com/pycaret/pycaret), [H2O AutoML](https://github.com//h2oai/h2o-3/blob/master/h2o-docs/src/product/automl.rst), etc. )
- - implement all possible data verification and normalization capabilities for different types of search keys (we just started with current 6 types)
+ - implement all possible data verification and normalization capabilities for different types of search keys 
+And we need some help from the community!
 
-And we need some help from community)
 So, we'll be happy about every **pull request** you open and **issue** you find to make this library **more incredible**. Please note that it might sometimes take us a while to get back to you.
 **For major changes**, please open an issue first to discuss what you would like to change
 #### Developing

{upgini-1.1.237a2.dist-info → upgini-1.1.239a1.dist-info}/RECORD

@@ -1,13 +1,13 @@
 upgini/__init__.py,sha256=asENHgEVHQBIkV-e_0IhE_ZWqkCG6398U3ZLrNzAH6k,407
 upgini/ads.py,sha256=mre6xn44wcC_fg63iLT_kTh4mViZqR9AKRJZAtpQz8Y,2592
-upgini/dataset.py,sha256=4u9ziFdgPOqPn-jgdG6e2GqXmjJo34DKRhSft9W_H6s,50174
+upgini/dataset.py,sha256=qSjv09LKzCYayucb_JlhExw9uSRcscLWTaD8hqATE3s,49676
 upgini/errors.py,sha256=BqpvfhW2jJW5fa5KXj0alhXatGl-WK4xTl309-QNLp8,959
-upgini/features_enricher.py,sha256=mfm5rXHW2aesg4vmpeuq3Mz_VA05Jf70uQqwCNg-2WI,160756
+upgini/features_enricher.py,sha256=9RJi8NwYbXPK-vgWiMcYoD4I2wO0D91Uk-tvL_1nJ-8,160271
 upgini/fingerprint.js,sha256=VygVIQlN1v4NGZfjHqtRogOw8zjTnnMNJg_f7M5iGQU,33442
-upgini/http.py,sha256=HzUSZudCdISJGUqHC1gAT1v_x1n_dIFVDJW4z3Q7DCs,41204
+upgini/http.py,sha256=RG93QmV3mqKixQsSHqYeM1Mtucp-EpdavcpCuhufnGE,42141
 upgini/metadata.py,sha256=FZ5CQluLLWrfrBVThSIes1SW6wcs7n50aNZwzYnHiF0,9584
 upgini/metrics.py,sha256=YeYHJtEIs8OG-EzidG-nbSYB919pjZ4MMbdcZ_jfV2s,23639
-upgini/search_task.py,sha256=7YxH1zrUHMmePO0VbPBBCJjeoer7jAC0Gltc9EVAOIg,17126
+upgini/search_task.py,sha256=sqgb5MfwWXg6YAbVhLOPcVJ5tDCUyzxFRWfd9aWj8SM,17236
 upgini/spinner.py,sha256=yhakBaydMNS8E8TRAwTdCMdnWrHeWT0cR1M8c9hP6jA,1157
 upgini/version_validator.py,sha256=rDIncP6BEko4J2F2hUcMOtKm_vZbI4ICWcNcw8hrwM4,1400
 upgini/ads_management/__init__.py,sha256=qzyisOToVRP-tquAJD1PblZhNtMrOB8FiyF9JvfkvgE,50
@@ -17,18 +17,18 @@ upgini/autofe/all_operands.py,sha256=du44N6ISWe3ikb0y9ZzSOHNbLiyEYrJPwoBo0Z6xp2s
 upgini/autofe/binary.py,sha256=f8LQqZi9zyaMUAv-jASMmWNA_vT05ncYCjZq0qx3USs,3972
 upgini/autofe/feature.py,sha256=d_iikjQJYgTOkZrXON_IWY5S22OkSpCsk6lfbmVA9ts,11825
 upgini/autofe/groupby.py,sha256=iXRfOmOc84ooSzRhsh9GmmG7rTafX0-ekXko8s9Qs68,3089
-upgini/autofe/operand.py,sha256=8WqEoSIA5rEWCK1xuC303E4NW5a72GZ5jUMAEj4skII,2291
-upgini/autofe/unary.py,sha256=7TBe7PCt7l_XQEqu_G5g_TC2cW3tppL7uPDcX8xsqz0,2731
+upgini/autofe/operand.py,sha256=GpSx-nL2XKnTJ7kvRr_SIFoUMchqYian6SftJ82zsN4,2719
+upgini/autofe/unary.py,sha256=WB-Ovwaz2a-Jscpshg1Om7Ttx6DJ6gQ_fgqtXx_UHuw,2845
 upgini/autofe/vector.py,sha256=Qk7VmdwURNwVw7fIMEspWEo7HTiyUWCYIqu3hcWQQio,507
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/data_source/data_source_publisher.py,sha256=zFu0WMKwPM11gPZHq8dpsBP7s4wmTtBqYoDEakgNxoY,13725
+upgini/data_source/data_source_publisher.py,sha256=xvHi4N4m32eqB_h_qtY1wAt1dXekM5PdNL2T9JzFQD4,14051
 upgini/mdc/__init__.py,sha256=CuKmWYCqAnmiq1S7wgMzJhSCTsXuoeiZWXSfzw0lyig,1152
 upgini/mdc/context.py,sha256=eVNEubcgkiAP139Vna2qtUBZJWoy15rWWAuB0TFv54E,1484
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/normalizer/phone_normalizer.py,sha256=VIgLXuDuzzjPEXiy_LyDVLZKGaS7-le6Fh6T4D-TQDU,9930
 upgini/resource_bundle/__init__.py,sha256=M7GtS7KPQw9pinz8P2aQWXpSkD2YFwUPVGk1w92Pn84,7888
 upgini/resource_bundle/exceptions.py,sha256=KT-OnqA2J4OTfLjhbEl3KFZM2ci7EOPjqJuY_rXp3vs,622
-upgini/resource_bundle/strings.properties,sha256=2Lad26Y4spPt_i5EYfOPg5XInBU7CuQxH8mDUWKzbDo,24829
+upgini/resource_bundle/strings.properties,sha256=1mpOkd_wkKIJGwWRBgfXz0mLx4lqdDro5IUoj8BBxuE,24527
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=X2PVsfZ3Rl7twpFDh5UWyxqY2K_jcMGxZ2NcHLwFRj4,6489
 upgini/sampler/random_under_sampler.py,sha256=whX_f_TtalHH8Seyn_7n3sX_TSiDHeYfALmme9saqDg,4082
@@ -39,8 +39,7 @@ upgini/utils/blocked_time_series.py,sha256=dMz5ewk3PsoeOrc3lDzInCVPS9u_2XQkV0W6P
 upgini/utils/country_utils.py,sha256=9BXSXoGm3nVoOZE_bRENY-KMkwMUFvAF3Au0zxUNA1o,6436
 upgini/utils/custom_loss_utils.py,sha256=DBslpjWGPt7xTeypt78baR59012SYphbPsO_YLKdilo,3972
 upgini/utils/cv_utils.py,sha256=6pSSL_Ft_8C6n6aInJeiyeSBD7McjsMxKZpHqSBV0uY,2491
-upgini/utils/datetime_utils.py,sha256=awsLpnFjBNcrsCDyyiiJLicHgHiGCNAwi0UOwRKGD7s,8645
-upgini/utils/deduplicate_utils.py,sha256=o-XY0hbqikQTzwpX0nyl34j_oiBQTefCvRgLHkZRkTE,2795
+upgini/utils/datetime_utils.py,sha256=P56e7gcgAogJYfs2Blzk1uypxb9yrFzNaeJpMCRm6Zc,7716
 upgini/utils/display_utils.py,sha256=tiq5sFOfMwkKCjQ7OGdyK_twe0Qdr9F3mzkW1QXSDog,10664
 upgini/utils/email_utils.py,sha256=MhCLUAWqbp81xRyKizauNhVx6t_MFeJQRQ8pFM7EpFo,3480
 upgini/utils/fallback_progress_bar.py,sha256=f-VzVbiO6oU9WoKzEgoegYotixdiKanGlvdQCOGC-NY,1128
@@ -52,10 +51,10 @@ upgini/utils/postal_code_utils.py,sha256=_8CR9tBqsPptQsmMUvnrCAmBaMIQSWH3JfJ4ly3
 upgini/utils/progress_bar.py,sha256=iNXyqT3vKCeHpfiG5HHwr7Lk2cTtKViM93Fl8iZnjGc,1564
 upgini/utils/sklearn_ext.py,sha256=IMx2La70AXAggApVpT7sMEjWqVWon5AMZt4MARDsIMQ,43847
 upgini/utils/target_utils.py,sha256=cu52icjhDIPpEStHYMXrD2hIl9gzvfnxZr0Ra5osV0k,1616
-upgini/utils/track_info.py,sha256=DVNVZmXUb4f25DSPEuUNEFx49hNEBfmuY9iSW5jkMnI,5708
+upgini/utils/track_info.py,sha256=jPOiIGpAG_zvHgeiFe_pQ4TWC9ZPjnd_5hSOu5tzLi4,5207
 upgini/utils/warning_counter.py,sha256=vnmdFo5-7GBkU2bK9h_uC0K0Y_wtfcYstxOdeRfacO0,228
-upgini-1.1.237a2.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.237a2.dist-info/METADATA,sha256=mBtZeL3wFKNUwe4HYcOPXpcb4fhMFGmIcX-b_torfuo,48346
-upgini-1.1.237a2.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-upgini-1.1.237a2.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
-upgini-1.1.237a2.dist-info/RECORD,,
+upgini-1.1.239a1.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.239a1.dist-info/METADATA,sha256=Sl4XSdmxJTR9080xw55QKFkoMDFMHuspXT_54E07mm0,48264
+upgini-1.1.239a1.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+upgini-1.1.239a1.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
+upgini-1.1.239a1.dist-info/RECORD,,

upgini/utils/deduplicate_utils.py

@@ -1,72 +0,0 @@
-from logging import Logger
-from typing import Dict, List, Optional, Union
-
-import pandas as pd
-
-from upgini.metadata import TARGET, ModelTaskType, SearchKey
-from upgini.resource_bundle import bundle
-from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
-from upgini.utils.target_utils import define_task
-
-
-def remove_fintech_duplicates(df: pd.DataFrame, 
-                              search_keys: Dict[str, SearchKey], 
-                              logger: Optional[Logger] = None) -> pd.DataFrame:
-    if define_task(df.target, silent=True) != ModelTaskType.BINARY:
-        return df
-    
-    date_col = _get_column_by_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
-    if date_col is None:
-        return df
-    
-    personal_cols = []
-    phone_col = _get_column_by_key(search_keys, SearchKey.PHONE)
-    if phone_col:
-        personal_cols.append(phone_col)
-    email_col = _get_column_by_key(search_keys, SearchKey.EMAIL)
-    if email_col:
-        personal_cols.append(email_col)
-    hem_col = _get_column_by_key(search_keys, SearchKey.HEM)
-    if hem_col:
-        personal_cols.append(hem_col)
-    if len(personal_cols) == 0:
-        return df
-    
-    duplicates = df.duplicated(personal_cols, keep=False)
-    duplicate_rows = df[duplicates]
-    if len(duplicate_rows) == 0:
-        return df
-    
-    grouped_by_personal_cols = df.groupby(personal_cols, group_keys=False)
-    
-    uniques = grouped_by_personal_cols[date_col].nunique()
-    total = len(uniques)
-    diff_dates = len(uniques[uniques > 1])
-    if diff_dates / total >= 0.6:
-        return df
-    
-    if grouped_by_personal_cols[TARGET].apply(lambda x: len(x.unique()) == 1).all():
-        return df
-
-    def has_diff_target_within_60_days(rows):
-        rows = rows.sort_values(by=date_col)
-        return len(rows[rows[TARGET].ne(rows[TARGET].shift()) & (rows[date_col].diff() < 60 * 24 * 60 * 60 * 1000)]) > 0
-    
-    df = DateTimeSearchKeyConverter(date_col).convert(df)
-    grouped_by_personal_cols = df.groupby(personal_cols, group_keys=False)
-    rows_with_diff_target = grouped_by_personal_cols.filter(has_diff_target_within_60_days)
-    if len(rows_with_diff_target) > 0:
-        perc = len(rows_with_diff_target) * 100 / len(df)
-        msg = bundle.get("dataset_diff_target_duplicates_fintech").format(perc, len(rows_with_diff_target), rows_with_diff_target.index.to_list())
-        print(msg)
-        if logger:
-            logger.warning(msg)
-        df = df[~df.index.isin(rows_with_diff_target.index)]
-    
-    return df
-
-
-def _get_column_by_key(search_keys: Dict[str, SearchKey], keys: Union[SearchKey, List[SearchKey]]) -> Optional[str]:
-    for col, key_type in search_keys.items():
-        if (isinstance(keys, list) and key_type in keys) or key_type == keys:
-            return col