upgini 1.1.275a1__py3-none-any.whl → 1.1.275a99__py3-none-any.whl

upgini/autofe/date.py

@@ -2,6 +2,7 @@ from typing import Any, Optional, Union
 import numpy as np
 import pandas as pd
 from pydantic import BaseModel
+from pandas.core.arrays.timedeltas import TimedeltaArray
 
 from upgini.autofe.operand import PandasOperand
 
@@ -46,6 +47,7 @@ class DateDiffType2(PandasOperand, DateDiffMixin):
         future = right + (left.dt.year - right.dt.year).apply(
             lambda y: np.datetime64("NaT") if np.isnan(y) else pd.tseries.offsets.DateOffset(years=y)
         )
+        future = pd.to_datetime(future)
         before = future[future < left]
         future[future < left] = before + pd.tseries.offsets.DateOffset(years=1)
         diff = (future - left) / np.timedelta64(1, self.diff_unit)
@@ -72,8 +74,13 @@ class DateListDiff(PandasOperand, DateDiffMixin):
 
         return pd.Series(left - right.values).apply(lambda x: self._agg(self._diff(x)))
 
-    def _diff(self, x):
-        x = x / np.timedelta64(1, self.diff_unit)
+    def _diff(self, x: TimedeltaArray):
+        if self.diff_unit == "Y":
+            x = (x / 365 / 24 / 60 / 60 / 10**9).astype(int)
+        elif self.diff_unit == "M":
+            raise Exception("Unsupported difference unit: Month")
+        else:
+            x = x / np.timedelta64(1, self.diff_unit)
         return x[x > 0]
 
     def _agg(self, x):

upgini/data_source/data_source_publisher.py

@@ -48,6 +48,7 @@ class DataSourcePublisher:
         data_table_uri: str,
         search_keys: Dict[str, SearchKey],
         update_frequency: str,
+        exclude_from_autofe_generation: Optional[List[str]],
         secondary_search_keys: Optional[Dict[str, SearchKey]] = None,
         sort_column: Optional[str] = None,
         date_format: Optional[str] = None,
@@ -57,7 +58,6 @@ class DataSourcePublisher:
         join_date_abs_limit_days: Optional[int] = None,
         features_for_embeddings: Optional[List[str]] = DEFAULT_GENERATE_EMBEDDINGS,
         data_table_id_to_replace: Optional[str] = None,
-        exclude_from_autofe_generation: Optional[List[str]] = None,
         _force_generation=False,
         _silent=False,
     ) -> str:

upgini/dataset.py

@@ -22,9 +22,7 @@ from pandas.api.types import (
 from upgini.errors import ValidationError
 from upgini.http import ProgressStage, SearchProgress, _RestClient
 from upgini.metadata import (
-    ENTITY_SYSTEM_RECORD_ID,
     EVAL_SET_INDEX,
-    SEARCH_KEY_UNNEST,
     SYSTEM_COLUMNS,
     SYSTEM_RECORD_ID,
     TARGET,
@@ -80,7 +78,6 @@ class Dataset:  # (pd.DataFrame):
         path: Optional[str] = None,
         meaning_types: Optional[Dict[str, FileColumnMeaningType]] = None,
         search_keys: Optional[List[Tuple[str, ...]]] = None,
-        unnest_search_keys: Optional[List[str]] = None,
         model_task_type: Optional[ModelTaskType] = None,
         random_state: Optional[int] = None,
         rest_client: Optional[_RestClient] = None,
@@ -115,7 +112,6 @@ class Dataset:  # (pd.DataFrame):
         self.description = description
         self.meaning_types = meaning_types
         self.search_keys = search_keys
-        self.unnest_search_keys = unnest_search_keys
         self.ignore_columns = []
         self.hierarchical_group_keys = []
         self.hierarchical_subgroup_keys = []
@@ -175,7 +171,7 @@ class Dataset:  # (pd.DataFrame):
         new_columns = []
         dup_counter = 0
         for column in self.data.columns:
-            if column in [TARGET, EVAL_SET_INDEX, SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST]:
+            if column in [TARGET, EVAL_SET_INDEX, SYSTEM_RECORD_ID]:
                 self.columns_renaming[column] = column
                 new_columns.append(column)
                 continue
@@ -356,9 +352,7 @@ class Dataset:  # (pd.DataFrame):
 
             if is_string_dtype(self.data[postal_code]):
                 try:
-                    self.data[postal_code] = (
-                        self.data[postal_code].astype("string").astype("Float64").astype("Int64").astype("string")
-                    )
+                    self.data[postal_code] = self.data[postal_code].astype("float64").astype("Int64").astype("string")
                 except Exception:
                     pass
             elif is_float_dtype(self.data[postal_code]):
@@ -808,8 +802,6 @@ class Dataset:  # (pd.DataFrame):
                     meaningType=meaning_type,
                     minMaxValues=min_max_values,
                 )
-                if self.unnest_search_keys and column_meta.originalName in self.unnest_search_keys:
-                    column_meta.isUnnest = True
 
                 columns.append(column_meta)
 

upgini/features_enricher.py

@@ -1,4 +1,5 @@
 import dataclasses
+import datetime
 import gc
 import hashlib
 import itertools
@@ -10,7 +11,6 @@ import sys
 import tempfile
 import time
 import uuid
-from collections import Counter
 from dataclasses import dataclass
 from threading import Thread
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
@@ -44,11 +44,9 @@ from upgini.mdc import MDC
 from upgini.metadata import (
     COUNTRY,
     DEFAULT_INDEX,
-    ENTITY_SYSTEM_RECORD_ID,
     EVAL_SET_INDEX,
     ORIGINAL_INDEX,
     RENAMED_INDEX,
-    SEARCH_KEY_UNNEST,
     SORT_ID,
     SYSTEM_RECORD_ID,
     TARGET,
@@ -149,6 +147,7 @@ class FeaturesEnricher(TransformerMixin):
     """
 
     TARGET_NAME = "target"
+    CURRENT_DATE = "current_date"
     RANDOM_STATE = 42
     CALCULATE_METRICS_THRESHOLD = 50_000_000
     CALCULATE_METRICS_MIN_THRESHOLD = 500
@@ -210,6 +209,7 @@ class FeaturesEnricher(TransformerMixin):
         client_ip: Optional[str] = None,
         client_visitorid: Optional[str] = None,
         custom_bundle_config: Optional[str] = None,
+        add_date_if_missing: bool = True,
         **kwargs,
     ):
         self.bundle = get_custom_bundle(custom_bundle_config)
@@ -320,6 +320,7 @@ class FeaturesEnricher(TransformerMixin):
         self.raise_validation_error = raise_validation_error
         self.exclude_columns = exclude_columns
         self.baseline_score_column = baseline_score_column
+        self.add_date_if_missing = add_date_if_missing
 
     def _get_api_key(self):
         return self._api_key
@@ -423,6 +424,9 @@ class FeaturesEnricher(TransformerMixin):
 
             self.__validate_search_keys(self.search_keys, self.search_id)
 
+            # Validate client estimator params
+            self._get_client_cat_features(estimator, X, self.search_keys)
+
             try:
                 self.X = X
                 self.y = y
@@ -816,6 +820,7 @@ class FeaturesEnricher(TransformerMixin):
         trace_id = trace_id or str(uuid.uuid4())
         start_time = time.time()
         with MDC(trace_id=trace_id):
+            self.logger.info("Start calculate metrics")
             if len(args) > 0:
                 msg = f"WARNING: Unsupported positional arguments for calculate_metrics: {args}"
                 self.logger.warning(msg)
@@ -867,22 +872,9 @@ class FeaturesEnricher(TransformerMixin):
                     self.__display_support_link(msg)
                     return None
 
-                cat_features = None
-                search_keys_for_metrics = []
-                if (
-                    estimator is not None
-                    and hasattr(estimator, "get_param")
-                    and estimator.get_param("cat_features") is not None
-                ):
-                    cat_features = estimator.get_param("cat_features")
-                    if len(cat_features) > 0 and isinstance(cat_features[0], int):
-                        cat_features = [effective_X.columns[i] for i in cat_features]
-                        for cat_feature in cat_features:
-                            if cat_feature in self.search_keys:
-                                if self.search_keys[cat_feature] in [SearchKey.COUNTRY, SearchKey.POSTAL_CODE]:
-                                    search_keys_for_metrics.append(cat_feature)
-                                else:
-                                    raise ValidationError(self.bundle.get("cat_feature_search_key").format(cat_feature))
+                cat_features, search_keys_for_metrics = self._get_client_cat_features(
+                    estimator, effective_X, self.search_keys
+                )
 
                 prepared_data = self._prepare_data_for_metrics(
                     trace_id=trace_id,
@@ -897,6 +889,7 @@ class FeaturesEnricher(TransformerMixin):
                     search_keys_for_metrics=search_keys_for_metrics,
                     progress_bar=progress_bar,
                     progress_callback=progress_callback,
+                    cat_features=cat_features,
                 )
                 if prepared_data is None:
                     return None
@@ -1184,8 +1177,6 @@ class FeaturesEnricher(TransformerMixin):
         search_keys = self.search_keys.copy()
         search_keys = self.__prepare_search_keys(x, search_keys, is_demo_dataset, is_transform=True, silent_mode=True)
 
-        unnest_search_keys = []
-
         extended_X = x.copy()
         generated_features = []
         date_column = self._get_date_column(search_keys)
@@ -1196,7 +1187,7 @@ class FeaturesEnricher(TransformerMixin):
         email_column = self._get_email_column(search_keys)
         hem_column = self._get_hem_column(search_keys)
         if email_column:
-            converter = EmailSearchKeyConverter(email_column, hem_column, search_keys, unnest_search_keys, self.logger)
+            converter = EmailSearchKeyConverter(email_column, hem_column, search_keys, self.logger)
             extended_X = converter.convert(extended_X)
             generated_features.extend(converter.generated_features)
         if (
@@ -1274,6 +1265,29 @@ class FeaturesEnricher(TransformerMixin):
 
         return _cv, groups
 
+    def _get_client_cat_features(
+        self, estimator: Optional[Any], X: pd.DataFrame, search_keys: Dict[str, SearchKey]
+    ) -> Optional[List[str]]:
+        cat_features = None
+        search_keys_for_metrics = []
+        if (
+            estimator is not None
+            and hasattr(estimator, "get_param")
+            and estimator.get_param("cat_features") is not None
+        ):
+            cat_features = estimator.get_param("cat_features")
+            if len(cat_features) > 0:
+                if all([isinstance(f, int) for f in cat_features]):
+                    cat_features = [X.columns[i] for i in cat_features]
+                self.logger.info(f"Collected categorical features {cat_features} from user estimator")
+                for cat_feature in cat_features:
+                    if cat_feature in search_keys:
+                        if search_keys[cat_feature] in [SearchKey.COUNTRY, SearchKey.POSTAL_CODE]:
+                            search_keys_for_metrics.append(cat_feature)
+                        else:
+                            raise ValidationError(self.bundle.get("cat_feature_search_key").format(cat_feature))
+        return cat_features, search_keys_for_metrics
+
     def _prepare_data_for_metrics(
         self,
         trace_id: str,
@@ -1288,6 +1302,7 @@ class FeaturesEnricher(TransformerMixin):
         search_keys_for_metrics: Optional[List[str]] = None,
         progress_bar: Optional[ProgressBar] = None,
         progress_callback: Optional[Callable[[SearchProgress], Any]] = None,
+        cat_features: Optional[List[str]] = None,
     ):
         is_input_same_as_fit, X, y, eval_set = self._is_input_same_as_fit(X, y, eval_set)
         is_demo_dataset = hash_input(X, y, eval_set) in DEMO_DATASET_HASHES
@@ -1345,9 +1360,8 @@ class FeaturesEnricher(TransformerMixin):
 
         # Detect and drop high cardinality columns in train
         columns_with_high_cardinality = FeaturesValidator.find_high_cardinality(fitting_X)
-        columns_with_high_cardinality = [
-            c for c in columns_with_high_cardinality if c not in (self.generate_features or [])
-        ]
+        non_excluding_columns = (self.generate_features or []) + (cat_features or [])
+        columns_with_high_cardinality = [c for c in columns_with_high_cardinality if c not in non_excluding_columns]
         if len(columns_with_high_cardinality) > 0:
             self.logger.warning(
                 f"High cardinality columns {columns_with_high_cardinality} will be dropped for metrics calculation"
@@ -1809,10 +1823,11 @@ class FeaturesEnricher(TransformerMixin):
         else:
             features_section = ""
 
-        api_example = f"""curl 'https://inference-upgini.azurewebsites.net/api/http_inference_trigger' \\
+        search_id = self._search_task.search_task_id
+        api_example = f"""curl 'https://search.upgini.com/online/api/http_inference_trigger?search_id={search_id}' \\
     -H 'Authorization: {self.api_key}' \\
     -H 'Content-Type: application/json' \\
-    -d '{{"search_id": "{self._search_task.search_task_id}", "search_keys": {keys}{features_section}}}'"""
+    -d '{{"search_keys": {keys}{features_section}}}'"""
         return api_example
 
     def _get_copy_of_runtime_parameters(self) -> RuntimeParameters:
@@ -1907,38 +1922,13 @@ class FeaturesEnricher(TransformerMixin):
                 generated_features.extend(converter.generated_features)
             else:
                 self.logger.info("Input dataset hasn't date column")
-
-            # Don't pass all features in backend on transform
-            original_features_for_transform = []
-            runtime_parameters = self._get_copy_of_runtime_parameters()
-            features_not_to_pass = [column for column in df.columns if column not in search_keys.keys()]
-            if len(features_not_to_pass) > 0:
-                # Pass only features that need for transform
-                features_for_transform = self._search_task.get_features_for_transform()
-                if features_for_transform is not None and len(features_for_transform) > 0:
-                    file_metadata = self._search_task.get_file_metadata(trace_id)
-                    original_features_for_transform = [
-                        c.originalName or c.name for c in file_metadata.columns if c.name in features_for_transform
-                    ]
-
-                    runtime_parameters.properties["features_for_embeddings"] = ",".join(features_for_transform)
-
-            columns_for_system_record_id = sorted(list(search_keys.keys()) + (original_features_for_transform))
-
-            df[ENTITY_SYSTEM_RECORD_ID] = pd.util.hash_pandas_object(
-                df[columns_for_system_record_id], index=False
-            ).astype("Float64")
-
-            # Explode multiple search keys
-            df, unnest_search_keys = self._explode_multiple_search_keys(df, search_keys)
-
+                if self.add_date_if_missing:
+                    df = self._add_current_date_as_key(df, search_keys, self.logger, self.bundle)
             email_column = self._get_email_column(search_keys)
             hem_column = self._get_hem_column(search_keys)
             email_converted_to_hem = False
             if email_column:
-                converter = EmailSearchKeyConverter(
-                    email_column, hem_column, search_keys, unnest_search_keys, self.logger
-                )
+                converter = EmailSearchKeyConverter(email_column, hem_column, search_keys, self.logger)
                 df = converter.convert(df)
                 generated_features.extend(converter.generated_features)
                 email_converted_to_hem = converter.email_converted_to_hem
@@ -1952,21 +1942,30 @@ class FeaturesEnricher(TransformerMixin):
             generated_features = [f for f in generated_features if f in self.fit_generated_features]
 
             meaning_types = {col: key.value for col, key in search_keys.items()}
-            # non_keys_columns = [column for column in df.columns if column not in search_keys.keys()]
-            for col in original_features_for_transform:
-                meaning_types[col] = FileColumnMeaningType.FEATURE
-            features_not_to_pass = [column for column in features_not_to_pass if column not in search_keys.keys()]
+            non_keys_columns = [column for column in df.columns if column not in search_keys.keys()]
 
             if email_converted_to_hem:
-                features_not_to_pass.append(email_column)
+                non_keys_columns.append(email_column)
+
+            # Don't pass features in backend on transform
+            original_features_for_transform = None
+            runtime_parameters = self._get_copy_of_runtime_parameters()
+            if len(non_keys_columns) > 0:
+                # Pass only features that need for transform
+                features_for_transform = self._search_task.get_features_for_transform()
+                if features_for_transform is not None and len(features_for_transform) > 0:
+                    file_metadata = self._search_task.get_file_metadata(trace_id)
+                    original_features_for_transform = [
+                        c.originalName or c.name for c in file_metadata.columns if c.name in features_for_transform
+                    ]
+                    non_keys_columns = [c for c in non_keys_columns if c not in original_features_for_transform]
 
-            features_not_to_pass = [c for c in features_not_to_pass if c not in original_features_for_transform]
-            columns_for_system_record_id = sorted(list(search_keys.keys()) + (original_features_for_transform))
+                    runtime_parameters.properties["features_for_embeddings"] = ",".join(features_for_transform)
 
             if add_fit_system_record_id:
                 df = self.__add_fit_system_record_id(df, dict(), search_keys)
                 df = df.rename(columns={SYSTEM_RECORD_ID: SORT_ID})
-                features_not_to_pass.append(SORT_ID)
+                non_keys_columns.append(SORT_ID)
 
             columns_for_system_record_id = sorted(list(search_keys.keys()) + (original_features_for_transform or []))
 
@@ -1974,19 +1973,16 @@ class FeaturesEnricher(TransformerMixin):
                 "Float64"
             )
             meaning_types[SYSTEM_RECORD_ID] = FileColumnMeaningType.SYSTEM_RECORD_ID
-            meaning_types[ENTITY_SYSTEM_RECORD_ID] = FileColumnMeaningType.ENTITY_SYSTEM_RECORD_ID
-            if SEARCH_KEY_UNNEST in df.columns:
-                meaning_types[SEARCH_KEY_UNNEST] = FileColumnMeaningType.UNNEST_KEY
 
             df = df.reset_index(drop=True)
-            system_columns_with_original_index = [SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID] + generated_features
+            system_columns_with_original_index = [SYSTEM_RECORD_ID] + generated_features
             if add_fit_system_record_id:
                 system_columns_with_original_index.append(SORT_ID)
             df_with_original_index = df[system_columns_with_original_index].copy()
 
             combined_search_keys = combine_search_keys(search_keys.keys())
 
-            df_without_features = df.drop(columns=features_not_to_pass)
+            df_without_features = df.drop(columns=non_keys_columns)
 
             df_without_features = clean_full_duplicates(
                 df_without_features, self.logger, silent=silent_mode, bundle=self.bundle
@@ -2142,14 +2138,6 @@ class FeaturesEnricher(TransformerMixin):
 
         key_types = search_keys.values()
 
-        # Multiple search keys allowed only for PHONE, IP, POSTAL_CODE, EMAIL, HEM
-        multi_keys = [key for key, count in Counter(key_types).items() if count > 1]
-        for multi_key in multi_keys:
-            if multi_key not in [SearchKey.PHONE, SearchKey.IP, SearchKey.POSTAL_CODE, SearchKey.EMAIL, SearchKey.HEM]:
-                msg = self.bundle.get("unsupported_multi_key").format(multi_key)
-                self.logger.warning(msg)
-                raise ValidationError(msg)
-
         if SearchKey.DATE in key_types and SearchKey.DATETIME in key_types:
             msg = self.bundle.get("date_and_datetime_simultanious")
             self.logger.warning(msg)
@@ -2165,11 +2153,11 @@ class FeaturesEnricher(TransformerMixin):
             self.logger.warning(msg)
             raise ValidationError(msg)
 
-        # for key_type in SearchKey.__members__.values():
-        #     if key_type != SearchKey.CUSTOM_KEY and list(key_types).count(key_type) > 1:
-        #         msg = self.bundle.get("multiple_search_key").format(key_type)
-        #         self.logger.warning(msg)
-        #         raise ValidationError(msg)
+        for key_type in SearchKey.__members__.values():
+            if key_type != SearchKey.CUSTOM_KEY and list(key_types).count(key_type) > 1:
+                msg = self.bundle.get("multiple_search_key").format(key_type)
+                self.logger.warning(msg)
+                raise ValidationError(msg)
 
         # non_personal_keys = set(SearchKey.__members__.values()) - set(SearchKey.personal_keys())
         # if (
@@ -2305,7 +2293,16 @@ class FeaturesEnricher(TransformerMixin):
             self.fit_generated_features.extend(converter.generated_features)
         else:
             self.logger.info("Input dataset hasn't date column")
-
+            if self.add_date_if_missing:
+                df = self._add_current_date_as_key(df, self.fit_search_keys, self.logger, self.bundle)
+        email_column = self._get_email_column(self.fit_search_keys)
+        hem_column = self._get_hem_column(self.fit_search_keys)
+        email_converted_to_hem = False
+        if email_column:
+            converter = EmailSearchKeyConverter(email_column, hem_column, self.fit_search_keys, self.logger)
+            df = converter.convert(df)
+            self.fit_generated_features.extend(converter.generated_features)
+            email_converted_to_hem = converter.email_converted_to_hem
         if (
             self.detect_missing_search_keys
             and list(self.fit_search_keys.values()) == [SearchKey.DATE]
@@ -2314,37 +2311,7 @@ class FeaturesEnricher(TransformerMixin):
             converter = IpToCountrySearchKeyConverter(self.fit_search_keys, self.logger)
             df = converter.convert(df)
 
-        # Explode multiple search keys
         non_feature_columns = [self.TARGET_NAME, EVAL_SET_INDEX] + list(self.fit_search_keys.keys())
-        meaning_types = {
-            **{col: key.value for col, key in self.fit_search_keys.items()},
-            **{str(c): FileColumnMeaningType.FEATURE for c in df.columns if c not in non_feature_columns},
-        }
-        meaning_types[self.TARGET_NAME] = FileColumnMeaningType.TARGET
-        if eval_set is not None and len(eval_set) > 0:
-            meaning_types[EVAL_SET_INDEX] = FileColumnMeaningType.EVAL_SET_INDEX
-        df = self.__add_fit_system_record_id(df, meaning_types, self.fit_search_keys, ENTITY_SYSTEM_RECORD_ID)
-
-        # TODO check that this is correct for enrichment
-        self.df_with_original_index = df.copy()
-
-        df, unnest_search_keys = self._explode_multiple_search_keys(df, self.fit_search_keys)
-
-        # Convert EMAIL to HEM after unnesting to do it only with one column
-        email_column = self._get_email_column(self.fit_search_keys)
-        hem_column = self._get_hem_column(self.fit_search_keys)
-        email_converted_to_hem = False
-        if email_column:
-            converter = EmailSearchKeyConverter(
-                email_column, hem_column, self.fit_search_keys, unnest_search_keys, self.logger
-            )
-            df = converter.convert(df)
-            self.fit_generated_features.extend(converter.generated_features)
-            email_converted_to_hem = converter.email_converted_to_hem
-
-        non_feature_columns = [self.TARGET_NAME, EVAL_SET_INDEX, ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST] + list(
-            self.fit_search_keys.keys()
-        )
         if email_converted_to_hem:
             non_feature_columns.append(email_column)
         if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
@@ -2368,14 +2335,12 @@ class FeaturesEnricher(TransformerMixin):
             **{str(c): FileColumnMeaningType.FEATURE for c in df.columns if c not in non_feature_columns},
         }
         meaning_types[self.TARGET_NAME] = FileColumnMeaningType.TARGET
-        meaning_types[ENTITY_SYSTEM_RECORD_ID] = FileColumnMeaningType.ENTITY_SYSTEM_RECORD_ID
-        if SEARCH_KEY_UNNEST in df.columns:
-            meaning_types[SEARCH_KEY_UNNEST] = FileColumnMeaningType.UNNEST_KEY
         if eval_set is not None and len(eval_set) > 0:
             meaning_types[EVAL_SET_INDEX] = FileColumnMeaningType.EVAL_SET_INDEX
 
-        df = self.__add_fit_system_record_id(df, meaning_types, self.fit_search_keys, SYSTEM_RECORD_ID)
+        df = self.__add_fit_system_record_id(df, meaning_types, self.fit_search_keys)
 
+        self.df_with_original_index = df.copy()
         df = df.reset_index(drop=True).sort_values(by=SYSTEM_RECORD_ID).reset_index(drop=True)
 
         combined_search_keys = combine_search_keys(self.fit_search_keys.keys())
@@ -2383,15 +2348,14 @@ class FeaturesEnricher(TransformerMixin):
         dataset = Dataset(
             "tds_" + str(uuid.uuid4()),
             df=df,
-            meaning_types=meaning_types,
-            search_keys=combined_search_keys,
-            unnest_search_keys=unnest_search_keys,
             model_task_type=model_task_type,
             date_format=self.date_format,
             random_state=self.random_state,
             rest_client=self.rest_client,
             logger=self.logger,
         )
+        dataset.meaning_types = meaning_types
+        dataset.search_keys = combined_search_keys
         if email_converted_to_hem:
             dataset.ignore_columns = [email_column]
 
@@ -2911,6 +2875,25 @@ class FeaturesEnricher(TransformerMixin):
             if t in [SearchKey.DATE, SearchKey.DATETIME]:
                 return col
 
+    @staticmethod
+    def _add_current_date_as_key(
+        df: pd.DataFrame, search_keys: Dict[str, SearchKey], logger: logging.Logger, bundle: ResourceBundle
+    ) -> pd.DataFrame:
+        if (
+            set(search_keys.values()) == {SearchKey.PHONE}
+            or set(search_keys.values()) == {SearchKey.EMAIL}
+            or set(search_keys.values()) == {SearchKey.HEM}
+            or set(search_keys.values()) == {SearchKey.COUNTRY, SearchKey.POSTAL_CODE}
+        ):
+            msg = bundle.get("current_date_added")
+            print(msg)
+            logger.warning(msg)
+            df[FeaturesEnricher.CURRENT_DATE] = datetime.date.today()
+            search_keys[FeaturesEnricher.CURRENT_DATE] = SearchKey.DATE
+            converter = DateTimeSearchKeyConverter(FeaturesEnricher.CURRENT_DATE, None, logger, bundle)
+            df = converter.convert(df)
+        return df
+
     @staticmethod
     def _get_group_columns(df: pd.DataFrame, search_keys: Dict[str, SearchKey]) -> List[str]:
         return [
@@ -2921,19 +2904,15 @@ class FeaturesEnricher(TransformerMixin):
 
     @staticmethod
     def _get_email_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
-        cols = [col for col, t in search_keys.items() if t == SearchKey.EMAIL]
-        if len(cols) > 1:
-            raise Exception("More than one email column found after unnest")
-        if len(cols) == 1:
-            return cols[0]
+        for col, t in search_keys.items():
+            if t == SearchKey.EMAIL:
+                return col
 
     @staticmethod
     def _get_hem_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
-        cols = [col for col, t in search_keys.items() if t == SearchKey.HEM]
-        if len(cols) > 1:
-            raise Exception("More than one hem column found after unnest")
-        if len(cols) == 1:
-            return cols[0]
+        for col, t in search_keys.items():
+            if t == SearchKey.HEM:
+                return col
 
     @staticmethod
     def _get_phone_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
@@ -2941,42 +2920,8 @@ class FeaturesEnricher(TransformerMixin):
             if t == SearchKey.PHONE:
                 return col
 
-    def _explode_multiple_search_keys(self, df: pd.DataFrame, search_keys: Dict[str, SearchKey]) -> pd.DataFrame:
-        # find groups of multiple search keys
-        search_key_names_by_type: Dict[SearchKey, str] = dict()
-        for key_name, key_type in search_keys.items():
-            search_key_names_by_type[key_type] = search_key_names_by_type.get(key_type, []) + [key_name]
-        search_key_names_by_type = {
-            key_type: key_names for key_type, key_names in search_key_names_by_type.items() if len(key_names) > 1
-        }
-        if len(search_key_names_by_type) == 0:
-            return df, []
-
-        multiple_keys_columns = [col for cols in search_key_names_by_type.values() for col in cols]
-        other_columns = [col for col in df.columns if col not in multiple_keys_columns]
-        exploded_dfs = []
-        unnest_search_keys = []
-
-        for key_type, key_names in search_key_names_by_type.items():
-            new_search_key = f"upgini_{key_type.name.lower()}_unnest"
-            exploded_df = pd.melt(
-                df, id_vars=other_columns, value_vars=key_names, var_name=SEARCH_KEY_UNNEST, value_name=new_search_key
-            )
-            exploded_dfs.append(exploded_df)
-            for old_key in key_names:
-                del search_keys[old_key]
-            search_keys[new_search_key] = key_type
-            unnest_search_keys.append(new_search_key)
-
-        df = pd.concat(exploded_dfs, ignore_index=True)
-        return df, unnest_search_keys
-
     def __add_fit_system_record_id(
-        self,
-        df: pd.DataFrame,
-        meaning_types: Dict[str, FileColumnMeaningType],
-        search_keys: Dict[str, SearchKey],
-        id_name: str,
+        self, df: pd.DataFrame, meaning_types: Dict[str, FileColumnMeaningType], search_keys: Dict[str, SearchKey]
     ) -> pd.DataFrame:
         # save original order or rows
         original_index_name = df.index.name
@@ -3025,18 +2970,14 @@ class FeaturesEnricher(TransformerMixin):
 
         df = df.reset_index(drop=True).reset_index()
         # system_record_id saves correct order for fit
-        df = df.rename(columns={DEFAULT_INDEX: id_name})
+        df = df.rename(columns={DEFAULT_INDEX: SYSTEM_RECORD_ID})
 
         # return original order
         df = df.set_index(ORIGINAL_INDEX)
         df.index.name = original_index_name
         df = df.sort_values(by=original_order_name).drop(columns=original_order_name)
 
-        meaning_types[id_name] = (
-            FileColumnMeaningType.SYSTEM_RECORD_ID 
-            if id_name == SYSTEM_RECORD_ID 
-            else FileColumnMeaningType.ENTITY_SYSTEM_RECORD_ID
-        )
+        meaning_types[SYSTEM_RECORD_ID] = FileColumnMeaningType.SYSTEM_RECORD_ID
         return df
 
     def __correct_target(self, df: pd.DataFrame) -> pd.DataFrame:
@@ -3091,10 +3032,7 @@ class FeaturesEnricher(TransformerMixin):
         )
 
         comparing_columns = X.columns if is_transform else df_with_original_index.columns
-        dup_features = [
-            c for c in comparing_columns 
-            if c in result_features.columns and c not in [SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID]
-        ]
+        dup_features = [c for c in comparing_columns if c in result_features.columns and c != SYSTEM_RECORD_ID]
         if len(dup_features) > 0:
             self.logger.warning(f"X contain columns with same name as returned from backend: {dup_features}")
             raise ValidationError(self.bundle.get("returned_features_same_as_passed").format(dup_features))
@@ -3105,7 +3043,8 @@ class FeaturesEnricher(TransformerMixin):
         result_features = pd.merge(
             df_with_original_index,
             result_features,
-            on=ENTITY_SYSTEM_RECORD_ID,
+            left_on=SYSTEM_RECORD_ID,
+            right_on=SYSTEM_RECORD_ID,
             how="left" if is_transform else "inner",
         )
         result_features = result_features.set_index(original_index_name or DEFAULT_INDEX)
@@ -3485,13 +3424,13 @@ class FeaturesEnricher(TransformerMixin):
                 self.warning_counter.increment()
 
         if len(valid_search_keys) == 1:
-            key, value = list(valid_search_keys.items())[0]
-            # Show warning for country only if country is the only key
-            if x[key].nunique() == 1:
-                msg = self.bundle.get("single_constant_search_key").format(value, x[key].values[0])
-                print(msg)
-                self.logger.warning(msg)
-                self.warning_counter.increment()
+            for k, v in valid_search_keys.items():
+                # Show warning for country only if country is the only key
+                if x[k].nunique() == 1 and (v != SearchKey.COUNTRY or len(valid_search_keys) == 1):
+                    msg = self.bundle.get("single_constant_search_key").format(v, x[k].values[0])
+                    print(msg)
+                    self.logger.warning(msg)
+                    self.warning_counter.increment()
 
         self.logger.info(f"Prepared search keys: {valid_search_keys}")
 
@@ -3601,68 +3540,61 @@ class FeaturesEnricher(TransformerMixin):
         def check_need_detect(search_key: SearchKey):
             return not is_transform or search_key in self.fit_search_keys.values()
 
-        # if SearchKey.POSTAL_CODE not in search_keys.values() and check_need_detect(SearchKey.POSTAL_CODE):
-        if check_need_detect(SearchKey.POSTAL_CODE):
-            maybe_keys = PostalCodeSearchKeyDetector().get_search_key_columns(sample, search_keys)
-            if maybe_keys:
-                new_keys = {key: SearchKey.POSTAL_CODE for key in maybe_keys}
-                search_keys.update(new_keys)
-                self.autodetected_search_keys.update(new_keys)
-                self.logger.info(f"Autodetected search key POSTAL_CODE in column {maybe_keys}")
+        if SearchKey.POSTAL_CODE not in search_keys.values() and check_need_detect(SearchKey.POSTAL_CODE):
+            maybe_key = PostalCodeSearchKeyDetector().get_search_key_column(sample)
+            if maybe_key is not None:
+                search_keys[maybe_key] = SearchKey.POSTAL_CODE
+                self.autodetected_search_keys[maybe_key] = SearchKey.POSTAL_CODE
+                self.logger.info(f"Autodetected search key POSTAL_CODE in column {maybe_key}")
                 if not silent_mode:
-                    print(self.bundle.get("postal_code_detected").format(maybe_keys))
+                    print(self.bundle.get("postal_code_detected").format(maybe_key))
 
         if (
             SearchKey.COUNTRY not in search_keys.values()
             and self.country_code is None
             and check_need_detect(SearchKey.COUNTRY)
         ):
-            maybe_key = CountrySearchKeyDetector().get_search_key_columns(sample, search_keys)
-            if maybe_key:
-                search_keys[maybe_key[0]] = SearchKey.COUNTRY
-                self.autodetected_search_keys[maybe_key[0]] = SearchKey.COUNTRY
+            maybe_key = CountrySearchKeyDetector().get_search_key_column(sample)
+            if maybe_key is not None:
+                search_keys[maybe_key] = SearchKey.COUNTRY
+                self.autodetected_search_keys[maybe_key] = SearchKey.COUNTRY
                 self.logger.info(f"Autodetected search key COUNTRY in column {maybe_key}")
                 if not silent_mode:
                     print(self.bundle.get("country_detected").format(maybe_key))
 
         if (
-            # SearchKey.EMAIL not in search_keys.values()
-            SearchKey.HEM not in search_keys.values()
+            SearchKey.EMAIL not in search_keys.values()
+            and SearchKey.HEM not in search_keys.values()
             and check_need_detect(SearchKey.HEM)
         ):
-            maybe_keys = EmailSearchKeyDetector().get_search_key_columns(sample, search_keys)
-            if maybe_keys:
+            maybe_key = EmailSearchKeyDetector().get_search_key_column(sample)
+            if maybe_key is not None and maybe_key not in search_keys.keys():
                 if self.__is_registered or is_demo_dataset:
-                    new_keys = {key: SearchKey.EMAIL for key in maybe_keys}
-                    search_keys.update(new_keys)
-                    self.autodetected_search_keys.update(new_keys)
-                    self.logger.info(f"Autodetected search key EMAIL in column {maybe_keys}")
+                    search_keys[maybe_key] = SearchKey.EMAIL
+                    self.autodetected_search_keys[maybe_key] = SearchKey.EMAIL
+                    self.logger.info(f"Autodetected search key EMAIL in column {maybe_key}")
                     if not silent_mode:
-                        print(self.bundle.get("email_detected").format(maybe_keys))
+                        print(self.bundle.get("email_detected").format(maybe_key))
                 else:
                     self.logger.warning(
-                        f"Autodetected search key EMAIL in column {maybe_keys}."
-                        " But not used because not registered user"
+                        f"Autodetected search key EMAIL in column {maybe_key}. But not used because not registered user"
                     )
                     if not silent_mode:
-                        print(self.bundle.get("email_detected_not_registered").format(maybe_keys))
+                        print(self.bundle.get("email_detected_not_registered").format(maybe_key))
                     self.warning_counter.increment()
 
-        # if SearchKey.PHONE not in search_keys.values() and check_need_detect(SearchKey.PHONE):
-        if check_need_detect(SearchKey.PHONE):
-            maybe_keys = PhoneSearchKeyDetector().get_search_key_columns(sample, search_keys)
-            if maybe_keys:
+        if SearchKey.PHONE not in search_keys.values() and check_need_detect(SearchKey.PHONE):
+            maybe_key = PhoneSearchKeyDetector().get_search_key_column(sample)
+            if maybe_key is not None and maybe_key not in search_keys.keys():
                 if self.__is_registered or is_demo_dataset:
-                    new_keys = {key: SearchKey.PHONE for key in maybe_keys}
-                    search_keys.update(new_keys)
-                    self.autodetected_search_keys.update(new_keys)
-                    self.logger.info(f"Autodetected search key PHONE in column {maybe_keys}")
+                    search_keys[maybe_key] = SearchKey.PHONE
+                    self.autodetected_search_keys[maybe_key] = SearchKey.PHONE
+                    self.logger.info(f"Autodetected search key PHONE in column {maybe_key}")
                     if not silent_mode:
-                        print(self.bundle.get("phone_detected").format(maybe_keys))
+                        print(self.bundle.get("phone_detected").format(maybe_key))
                 else:
                     self.logger.warning(
-                        f"Autodetected search key PHONE in column {maybe_keys}. "
-                        "But not used because not registered user"
+                        f"Autodetected search key PHONE in column {maybe_key}. But not used because not registered user"
                     )
                     if not silent_mode:
                         print(self.bundle.get("phone_detected_not_registered"))

upgini/metadata.py

@@ -4,8 +4,6 @@ from typing import Dict, List, Optional, Set
 from pydantic import BaseModel
 
 SYSTEM_RECORD_ID = "system_record_id"
-ENTITY_SYSTEM_RECORD_ID = "entity_system_record_id"
-SEARCH_KEY_UNNEST = "search_key_unnest"
 SORT_ID = "sort_id"
 EVAL_SET_INDEX = "eval_set_index"
 TARGET = "target"
@@ -13,7 +11,7 @@ COUNTRY = "country_iso_code"
 RENAMED_INDEX = "index_col"
 DEFAULT_INDEX = "index"
 ORIGINAL_INDEX = "original_index"
-SYSTEM_COLUMNS = {SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST, EVAL_SET_INDEX, TARGET, COUNTRY}
+SYSTEM_COLUMNS = {SYSTEM_RECORD_ID, EVAL_SET_INDEX, TARGET, COUNTRY, SORT_ID}
 
 
 class FileColumnMeaningType(Enum):
@@ -39,8 +37,6 @@ class FileColumnMeaningType(Enum):
     POSTAL_CODE = "POSTAL_CODE"
     SYSTEM_RECORD_ID = "SYSTEM_RECORD_ID"
     EVAL_SET_INDEX = "EVAL_SET_INDEX"
-    ENTITY_SYSTEM_RECORD_ID = "ENTITY_SYSTEM_RECORD_ID"
-    UNNEST_KEY = "UNNEST_KEY"
 
 
 class SearchKey(Enum):
@@ -186,10 +182,6 @@ class FileColumnMetadata(BaseModel):
     meaningType: FileColumnMeaningType
     minMaxValues: Optional[NumericInterval] = None
     originalName: Optional[str]
-    # is this column contains keys from multiple key columns like msisdn1, msisdn2
-    isUnnest: bool = False,
-    # list of original etalon key column names like msisdn1, msisdn2
-    unnestKeyNames: Optional[list[str]]
 
 
 class FileMetadata(BaseModel):

upgini/metrics.py

@@ -1,3 +1,4 @@
+import inspect
 import logging
 import re
 from copy import deepcopy
@@ -381,6 +382,11 @@ class EstimatorWrapper:
             kwargs["estimator"] = estimator_copy
             if isinstance(estimator, CatBoostClassifier) or isinstance(estimator, CatBoostRegressor):
                 if cat_features is not None:
+                    for cat_feature in cat_features:
+                        if cat_feature not in X.columns:
+                            logger.error(
+                                f"Client cat_feature `{cat_feature}` not found in X columns: {X.columns.to_list()}"
+                            )
                     estimator_copy.set_params(
                         cat_features=[X.columns.get_loc(cat_feature) for cat_feature in cat_features]
                     )
@@ -647,6 +653,12 @@ class OtherEstimatorWrapper(EstimatorWrapper):
 def validate_scoring_argument(scoring: Union[Callable, str, None]):
     if isinstance(scoring, str) and scoring is not None:
         _get_scorer_by_name(scoring)
+    elif isinstance(scoring, Callable):
+        spec = inspect.getfullargspec(scoring)
+        if len(spec.args) < 3:
+            raise ValidationError(
+                f"Invalid scoring function passed {scoring}. It should accept 3 input arguments: estimator, X, y"
+            )
 
 
 def _get_scorer_by_name(scoring: str) -> Tuple[Callable, str, int]:

upgini/normalizer/phone_normalizer.py

@@ -1,7 +1,7 @@
 from typing import Optional
 
 import pandas as pd
-from pandas.api.types import is_float_dtype, is_int64_dtype, is_string_dtype
+from pandas.api.types import is_float_dtype, is_int64_dtype, is_string_dtype, is_object_dtype
 
 from upgini.errors import ValidationError
 
@@ -44,7 +44,7 @@ class PhoneNormalizer:
         Method will remove all non numeric chars from string and convert it to int.
         None will be set for phone numbers that couldn"t be converted to int
         """
-        if is_string_dtype(self.df[self.phone_column_name]):
+        if is_string_dtype(self.df[self.phone_column_name]) or is_object_dtype(self.df[self.phone_column_name]):
             convert_func = self.phone_str_to_int_safe
         elif is_float_dtype(self.df[self.phone_column_name]):
             convert_func = self.phone_float_to_int_safe

upgini/resource_bundle/strings.properties

@@ -38,6 +38,7 @@ loss_selection_warn=\nWARNING: Loss `{0}` is not supported for feature selection
 loss_calc_metrics_warn=\nWARNING: Loss `{0}` is not supported for metrics calculation with {1}
 multivariate_timeseries_detected=\nWARNING: Multivariate TimeSeries detected. Blocked time series cross-validation split selected.\nMore details: https://github.com/upgini/upgini#-time-series-prediction-support
 group_k_fold_in_classification=\nWARNING: Using group K-fold cross-validation split for classification task.
+current_date_added=\nWARNING: No date/datetime column was detected in X to be used as a search key. The current date will be used to match the latest version of data sources
 
 # Errors
 failed_search_by_task_id=Failed to retrieve the specified search results
@@ -87,7 +88,6 @@ unsupported_search_key_type=Unsupported type of key in search_keys: {}
 search_key_country_and_country_code=\nWARNING: SearchKey.COUNTRY and country_code parameter were passed simultaniously. Parameter country_code will be ignored
 empty_search_key=Search key {} is empty. Please fill values or remove this search key
 single_constant_search_key=\nWARNING: Constant value detected for the {} search key in the X dataframe: {}.\nThat search key will add constant features for different y values.\nPlease add extra search keys with non constant values, like the COUNTRY, POSTAL_CODE, DATE, PHONE NUMBER, EMAIL/HEM or IPv4
-unsupported_multi_key=Search key {} cannot be used multiple times
 unsupported_index_column=\nWARNING: Your column with name `index` was dropped because it's reserved name is booked for system needs.
 date_string_without_format=Date column `{}` has string type, but date_format is not specified. Convert column to datetime type or pass date_format
 invalid_date_format=Failed to parse date in column `{}`. Try to pass explicit date format in date_format argument of FeaturesEnricher constructor
@@ -159,7 +159,7 @@ dataset_invalid_multiclass_target=Unexpected dtype of target for multiclass task
 dataset_invalid_regression_target=Unexpected dtype of target for regression task type: {}. Expected float
 dataset_invalid_timeseries_target=Unexpected dtype of target for timeseries task type: {}. Expected float
 dataset_to_many_multiclass_targets=The number of target classes {} exceeds the allowed threshold: {}. Please, correct your data and try again
-dataset_rarest_class_less_min=Frequency of the rarest class `{}` is {}, minimum frequency must be > {} for each class\nPlease, remove rows with rarest class from your dataframe
+dataset_rarest_class_less_min=Count of rows with the rarest class `{}` is {}, minimum count must be > {} for each class\nPlease, remove rows with rarest class from your dataframe
 dataset_rarest_class_less_threshold=\nWARNING: Target is imbalanced and will be undersampled to the rarest class. Frequency of the rarest class `{}` is {}\nMinimum number of observations for each class to avoid undersampling {} ({}%)
 dataset_date_features=\nWARNING: Columns {} is a datetime or period type but not used as a search key, removed from X
 dataset_too_many_features=Too many features. Maximum number of features is {}

upgini/utils/base_search_key_detector.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Optional
 
 import pandas as pd
 
@@ -10,18 +10,16 @@ class BaseSearchKeyDetector:
     def _is_search_key_by_values(self, column: pd.Series) -> bool:
         raise NotImplementedError()
 
-    def _get_search_keys_by_name(self, column_names: List[str]) -> List[str]:
-        return [
-            column_name
-            for column_name in column_names
-            if self._is_search_key_by_name(column_name)
-        ]
+    def _get_search_key_by_name(self, column_names: List[str]) -> Optional[str]:
+        for column_name in column_names:
+            if self._is_search_key_by_name(column_name):
+                return column_name
 
-    def get_search_key_columns(self, df: pd.DataFrame, existing_search_keys: List[str]) -> List[str]:
-        other_columns = [col for col in df.columns if col not in existing_search_keys]
-        columns_by_names = self._get_search_keys_by_name(other_columns)
-        columns_by_values = []
-        for column_name in other_columns:
+    def get_search_key_column(self, df: pd.DataFrame) -> Optional[str]:
+        maybe_column = self._get_search_key_by_name(df.columns.to_list())
+        if maybe_column is not None:
+            return maybe_column
+
+        for column_name in df.columns:
             if self._is_search_key_by_values(df[column_name]):
-                columns_by_values.append(column_name)
-        return list(set(columns_by_names + columns_by_values))
+                return column_name

upgini/utils/datetime_utils.py

@@ -100,6 +100,9 @@ class DateTimeSearchKeyConverter:
                 msg = self.bundle.get("unsupported_date_type").format(self.date_column)
                 self.logger.warning(msg)
                 raise ValidationError(msg)
+        else:
+            df[self.date_column] = df[self.date_column].astype("string").apply(self.clean_date)
+            df[self.date_column] = self.parse_date(df)
 
         # If column with date is datetime then extract seconds of the day and minute of the hour
         # as additional features

upgini/utils/deduplicate_utils.py

@@ -3,15 +3,7 @@ from typing import Dict, List, Optional, Union
 
 import pandas as pd
 
-from upgini.metadata import (
-    ENTITY_SYSTEM_RECORD_ID,
-    EVAL_SET_INDEX,
-    SORT_ID,
-    SYSTEM_RECORD_ID,
-    TARGET,
-    ModelTaskType,
-    SearchKey,
-)
+from upgini.metadata import EVAL_SET_INDEX, SORT_ID, SYSTEM_RECORD_ID, TARGET, ModelTaskType, SearchKey
 from upgini.resource_bundle import ResourceBundle
 from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
 from upgini.utils.target_utils import define_task
@@ -151,8 +143,6 @@ def clean_full_duplicates(
     unique_columns = df.columns.tolist()
     if SYSTEM_RECORD_ID in unique_columns:
         unique_columns.remove(SYSTEM_RECORD_ID)
-    if ENTITY_SYSTEM_RECORD_ID in unique_columns:
-        unique_columns.remove(ENTITY_SYSTEM_RECORD_ID)
     if SORT_ID in unique_columns:
         unique_columns.remove(SORT_ID)
     if EVAL_SET_INDEX in unique_columns:

upgini/utils/email_utils.py

@@ -38,13 +38,11 @@ class EmailSearchKeyConverter:
         email_column: str,
         hem_column: Optional[str],
         search_keys: Dict[str, SearchKey],
-        unnest_search_keys: Optional[List[str]] = None,
         logger: Optional[logging.Logger] = None,
     ):
         self.email_column = email_column
         self.hem_column = hem_column
         self.search_keys = search_keys
-        self.unnest_search_keys = unnest_search_keys
         if logger is not None:
             self.logger = logger
         else:
@@ -82,12 +80,9 @@ class EmailSearchKeyConverter:
                 del self.search_keys[self.email_column]
                 return df
             self.search_keys[self.HEM_COLUMN_NAME] = SearchKey.HEM
-            self.unnest_search_keys.append(self.HEM_COLUMN_NAME)
             self.email_converted_to_hem = True
 
         del self.search_keys[self.email_column]
-        if self.email_column in self.unnest_search_keys:
-            self.unnest_search_keys.remove(self.email_column)
 
         df[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = df[self.email_column].apply(self._email_to_one_domain)
 

upgini/utils/features_validator.py

@@ -81,7 +81,8 @@ class FeaturesValidator:
         return [
             i
             for i in df
-            if (is_string_dtype(df[i]) or is_integer_dtype(df[i])) and (df[i].nunique(dropna=False) / row_count >= 0.95)
+            if (is_object_dtype(df[i]) or is_string_dtype(df[i]) or is_integer_dtype(df[i]))
+            and (df[i].nunique(dropna=False) / row_count >= 0.85)
         ]
 
     @staticmethod

upgini/utils/track_info.py

@@ -55,7 +55,7 @@ def _get_execution_ide() -> str:
 def get_track_metrics(client_ip: Optional[str] = None, client_visitorid: Optional[str] = None) -> dict:
     # default values
     track = {"ide": _get_execution_ide()}
-    ident_res = "https://api.ipify.org"
+    ident_res = "https://api64.ipify.org"
 
     try:
         track["hostname"] = socket.gethostname()
@@ -74,17 +74,20 @@ def get_track_metrics(client_ip: Optional[str] = None, client_visitorid: Optiona
             display(
                 Javascript(
                     """
-                        import('https://upgini.github.io/upgini/js/a.js')
+                    async function getVisitorId() {
+                        return import('https://upgini.github.io/upgini/js/a.js')
                             .then(FingerprintJS => FingerprintJS.load())
                             .then(fp => fp.get())
-                            .then(result => window.visitorId = result.visitorId);
+                            .then(result => result.visitorId);
+                    }
                     """
                 )
             )
-            track["visitorId"] = output.eval_js("window.visitorId", timeout_sec=10)
+            track["visitorId"] = output.eval_js("getVisitorId()", timeout_sec=30)
         except Exception as e:
             track["err"] = str(e)
-            track["visitorId"] = "None"
+            if "visitorId" not in track:
+                track["visitorId"] = "None"
         if client_ip:
             track["ip"] = client_ip
         else:
@@ -95,16 +98,19 @@ def get_track_metrics(client_ip: Optional[str] = None, client_visitorid: Optiona
                 display(
                     Javascript(
                         f"""
-                            fetch("{ident_res}")
+                        async function getIP() {{
+                            return fetch("{ident_res}")
                                 .then(response => response.text())
-                                .then(data => window.clientIP = data);
+                                .then(data => data);
+                        }}
                         """
                     )
                 )
-                track["ip"] = output.eval_js("window.clientIP", timeout_sec=10)
+                track["ip"] = output.eval_js("getIP()", timeout_sec=10)
             except Exception as e:
                 track["err"] = str(e)
-                track["ip"] = "0.0.0.0"
+                if "ip" not in track:
+                    track["ip"] = "0.0.0.0"
 
     elif track["ide"] == "binder":
         try:
@@ -116,8 +122,10 @@ def get_track_metrics(client_ip: Optional[str] = None, client_visitorid: Optiona
                 track["visitorId"] = sha256(os.environ["CLIENT_IP"].encode()).hexdigest()
         except Exception as e:
             track["err"] = str(e)
-            track["ip"] = "0.0.0.0"
-            track["visitorId"] = "None"
+            if "ip" not in track:
+                track["ip"] = "0.0.0.0"
+            if "visitorId" not in track:
+                track["visitorId"] = "None"
 
     elif track["ide"] == "kaggle":
         try:
@@ -136,8 +144,8 @@ def get_track_metrics(client_ip: Optional[str] = None, client_visitorid: Optiona
                     raise Exception(err)
         except Exception as e:
             track["err"] = str(e)
-            track["ip"] = "0.0.0.0"
-            track["visitorId"] = "None"
+            if "visitorId" not in track:
+                track["visitorId"] = "None"
     else:
         try:
             if client_ip:
@@ -150,5 +158,9 @@ def get_track_metrics(client_ip: Optional[str] = None, client_visitorid: Optiona
                 track["visitorId"] = sha256(str(getnode()).encode()).hexdigest()
         except Exception as e:
             track["err"] = str(e)
+            if "visitorId" not in track:
+                track["visitorId"] = "None"
+            if "ip" not in track:
+                track["ip"] = "0.0.0.0"
 
     return track

{upgini-1.1.275a1.dist-info → upgini-1.1.275a99.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.275a1
+Version: 1.1.275a99
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers
@@ -28,7 +28,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: python-dateutil >=2.8.0
 Requires-Dist: requests >=2.8.0
-Requires-Dist: pandas <2.0.0,>=1.1.0
+Requires-Dist: pandas <3.0.0,>=1.1.0
 Requires-Dist: numpy >=1.19.0
 Requires-Dist: scikit-learn >=1.3.0
 Requires-Dist: pydantic <2.0.0,>=1.8.2

{upgini-1.1.275a1.dist-info → upgini-1.1.275a99.dist-info}/RECORD

@@ -1,12 +1,12 @@
 upgini/__init__.py,sha256=asENHgEVHQBIkV-e_0IhE_ZWqkCG6398U3ZLrNzAH6k,407
 upgini/ads.py,sha256=mre6xn44wcC_fg63iLT_kTh4mViZqR9AKRJZAtpQz8Y,2592
-upgini/dataset.py,sha256=g10BnbayclZMno9mAabpz_Zu0iyMiW0f_jOwt_xJr8U,45947
+upgini/dataset.py,sha256=xb4gIANyGbdcuM8Awyq2pJPiH_3k_LEbETApJgAoRBA,45529
 upgini/errors.py,sha256=pdzQl3MKuK52yvncxMWMRWeSIOGhUFzpQoszoRFBOk0,958
-upgini/features_enricher.py,sha256=CgUBRCPW_itgBfaup3Tg_yfPYMbQpufoOqu4yYvn6VU,179316
+upgini/features_enricher.py,sha256=A03SPhpJNxpZiAq6aSKiVOG6mqo3YrZ9MQRwkk8_OSg,176071
 upgini/fingerprint.js,sha256=VygVIQlN1v4NGZfjHqtRogOw8zjTnnMNJg_f7M5iGQU,33442
 upgini/http.py,sha256=zaO86LBBLmkieGbgYifk29eVoPCxXimZQ8YkQtKcM0I,42244
-upgini/metadata.py,sha256=FFwTnoMxdJ-7oKXbRgght1yk7e2u90WpeqljKDWUj18,10106
-upgini/metrics.py,sha256=VmxVc-plbRPZ1U3Ve3E-FZkhYqi0X2r7x8H5L-shux4,29058
+upgini/metadata.py,sha256=fwVxtkR6Mn4iRoOqV6BfMJvJrx65I3YwZUMbZjhPyOI,9673
+upgini/metrics.py,sha256=tGzdn0jgup86OlH_GS4eoza8ZJZ9wgaJr7SaX3Upwzo,29652
 upgini/search_task.py,sha256=tmJ17WUxv3J5NWrYUJB_NKdZ792Ifz8Z8UnDXeQnpss,17077
 upgini/spinner.py,sha256=Dm1dQ5F_z_Ua2odLxZX7OypcOX9tSx_vE5MGaKtUmfw,1118
 upgini/version_validator.py,sha256=rDIncP6BEko4J2F2hUcMOtKm_vZbI4ICWcNcw8hrwM4,1400
@@ -15,38 +15,38 @@ upgini/ads_management/ads_manager.py,sha256=fP4Yqx3h2Snw5X335TbXEwFoupq1RYsE7y0P
 upgini/autofe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/autofe/all_operands.py,sha256=H66wqVLD-H9k8A4-q2wslhV9QaNxlb49f8YiT0Xfkps,2356
 upgini/autofe/binary.py,sha256=f8LQqZi9zyaMUAv-jASMmWNA_vT05ncYCjZq0qx3USs,3972
-upgini/autofe/date.py,sha256=cc0GMAJR0QZOI_Qp2V5UDklaXLNS_79O1GhU6GlOYzg,3895
+upgini/autofe/date.py,sha256=408p8P2OTPM2D3LsEGGtaiCepKGgM1BbOCQNRzAmI6c,4223
 upgini/autofe/feature.py,sha256=2FQRGtIumNz60hFAjfLReaY18SI7HxzYZOoC5avzSjQ,11847
 upgini/autofe/groupby.py,sha256=iXRfOmOc84ooSzRhsh9GmmG7rTafX0-ekXko8s9Qs68,3089
 upgini/autofe/operand.py,sha256=dhtToPDGWtP_0u_RjayUpezJJZAgq_TzNbPH0bI9OXI,2805
 upgini/autofe/unary.py,sha256=YRTzQLttbDdOnkogWBPnBexpu7uHWSLSFAxSCu3iFdY,3145
 upgini/autofe/vector.py,sha256=5qhI_bdwaWM1l7fgCkx1tMt9R9gxWzoYCl-7WO4KiOs,604
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/data_source/data_source_publisher.py,sha256=J2lrpPuysUHPeqTSfoybBtPRTBCFu7R5KzaakhjaRDc,16485
+upgini/data_source/data_source_publisher.py,sha256=taRzyGgrPrTTSGw4Y-Ca5k4bf30aiTa68rxqT9zfqeI,16478
 upgini/mdc/__init__.py,sha256=ETDh3JKbrDdPMOECiYLAa8lvKYe68mv4IY6fZa9FimA,1126
 upgini/mdc/context.py,sha256=Sl1S_InKlzzRxYqwJ2k24lawJdCKWgGJ-RIRfvzWJrk,1468
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/normalizer/phone_normalizer.py,sha256=lhwsPEnfyjeIsndW2EcQGZksXYsfxaQ1ghAzVYoDRKM,9927
+upgini/normalizer/phone_normalizer.py,sha256=_SYMX4GTgwzRXArK54Jp3vUBE5d4jZxSVyze-0tqzg0,9996
 upgini/resource_bundle/__init__.py,sha256=hdvbqL0b0xMWbY6-kiYGsW1ro2GMiWpxxsO9uCv-h9Q,8379
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=AK5xktWWYa0smEa_ZVT7BFlXPSx7M_NTMIfXhgsnE2Y,26177
+upgini/resource_bundle/strings.properties,sha256=1O779a0-Ai0j7W-Z5AznvjuV69YkJvgGhJda-6VMLOQ,26287
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=CC-DvPbrN7zp5--SVFuUqkVmdWM_5F7R0Do98ETV82U,6421
 upgini/sampler/random_under_sampler.py,sha256=XU4c2swPIFxVXHOPpxgM2bUao0Xm-aoMmd6fKjIuV5s,4068
 upgini/sampler/utils.py,sha256=PYOk3kKSnFlyxcpdtDNLBEEhTB4lO_iP7pQHqeUcmAc,20211
 upgini/utils/__init__.py,sha256=dQ4-s8-sZ5eOBZ-mH3gEwDHTdI0wI1bUAVgVqUKKPx4,786
-upgini/utils/base_search_key_detector.py,sha256=VvEdamjJT1wypsH6NAfOkPp7dHo7nxhl7LhwX7Z9N5w,1025
+upgini/utils/base_search_key_detector.py,sha256=DGwhXLvc8i5VZWMDr0rncFfV5GEHdsCSnLGon_W9TPs,859
 upgini/utils/blocked_time_series.py,sha256=dMz5ewk3PsoeOrc3lDzInCVPS9u_2XQkV0W6PuMMjPg,3380
 upgini/utils/country_utils.py,sha256=1KXhLSNqkNYVL3on8-zK0Arc_SspUH7AMZvGZICysOU,6462
 upgini/utils/custom_loss_utils.py,sha256=DBslpjWGPt7xTeypt78baR59012SYphbPsO_YLKdilo,3972
 upgini/utils/cv_utils.py,sha256=Tn01RJvpZGZh0PUQUimlBkV-AXwe7s6yjCNFtw352Uc,3525
-upgini/utils/datetime_utils.py,sha256=4ii5WphAHlb_NRmdJx35VZpTarJbAr-AnDw3XSzUSow,10346
-upgini/utils/deduplicate_utils.py,sha256=Zvs7zW4QzaERQmJNPrTVf2ZTVBkBLOycFCzyMwtXuV8,8770
+upgini/utils/datetime_utils.py,sha256=XciFOIYI4Zi7PqQS8dHxuPDEtdtwXbOrWsiAa04v2J4,10511
+upgini/utils/deduplicate_utils.py,sha256=6AbARehUCghJZ4PppFtrej2s3gFRruh41MEm6mzakHs,8607
 upgini/utils/display_utils.py,sha256=LKoSwjrE0xgS5_cqVhc2og2CQ1UCZ1nTI2VKboIhoQA,10858
-upgini/utils/email_utils.py,sha256=0EPCxMU-huzTgb_vySiAQ8tmSUhS31Mz2BpaHGwwYO4,3772
+upgini/utils/email_utils.py,sha256=3CvHXTSzlgLyGsQOXfRYVfFhfPy6OXG4uXOBWRaLfHg,3479
 upgini/utils/fallback_progress_bar.py,sha256=cdbd1XGcWm4Ed4eAqV2_St3z7uC_kkH22gEyrN5ub6M,1090
-upgini/utils/features_validator.py,sha256=P-dfjBLAMxgzOcUX1Jo1bhVp8-8WyTyF3Ef0YZ5nfRI,3269
+upgini/utils/features_validator.py,sha256=PgKNt5dyqfErTvjtRNNUS9g7GFqHBtAtnsfA-V5UO1A,3307
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
 upgini/utils/ip_utils.py,sha256=Zf3F2cnQmOCH09QLQHetpjMFu1PnD0cTmDymn0SnSy8,1672
 upgini/utils/phone_utils.py,sha256=JNSkF8G6mgsN8Czy11pamaJdsY6rBINEMpi7jbVt_RA,408
@@ -54,10 +54,10 @@ upgini/utils/postal_code_utils.py,sha256=_8CR9tBqsPptQsmMUvnrCAmBaMIQSWH3JfJ4ly3
 upgini/utils/progress_bar.py,sha256=iNXyqT3vKCeHpfiG5HHwr7Lk2cTtKViM93Fl8iZnjGc,1564
 upgini/utils/sklearn_ext.py,sha256=e1aMNXk1zUt7uFnl0FcUF0zOnaXSE7z5xBHmJPknUVs,44014
 upgini/utils/target_utils.py,sha256=9K67tkY7LWhQMO-vbbPqBaO-KriAmg_6fVz5RQRaLQc,7802
-upgini/utils/track_info.py,sha256=EPcJ13Jqa17_T0JjM37Ac9kWDz5Zk0GVsIZKutOb8aU,5207
+upgini/utils/track_info.py,sha256=p8gmuHhLamZF5JG7K9DeK-PcytQhlFCR29lyRr-wq_U,5665
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.1.275a1.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.275a1.dist-info/METADATA,sha256=ocZUhdmjsYXKoCXt0W3M4gfPGQ8UlFtQlYIjdD_6_w0,48158
-upgini-1.1.275a1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-upgini-1.1.275a1.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
-upgini-1.1.275a1.dist-info/RECORD,,
+upgini-1.1.275a99.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.275a99.dist-info/METADATA,sha256=6wFwtaOYKQ4o9mZBpQlJqST1_r1YaTkwqgYAi7zkkHM,48159
+upgini-1.1.275a99.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+upgini-1.1.275a99.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
+upgini-1.1.275a99.dist-info/RECORD,,