upgini 1.1.299a3511.dev10__py3-none-any.whl → 1.1.300__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.1.299a3511.dev10"
+__version__ = "1.1.300"

upgini/autofe/all_operands.py

@@ -1,20 +1,6 @@
 from typing import Dict
 
-from upgini.autofe.binary import (
-    Add,
-    Combine,
-    CombineThenFreq,
-    Distance,
-    Divide,
-    JaroWinklerSim1,
-    JaroWinklerSim2,
-    LevenshteinSim,
-    Max,
-    Min,
-    Multiply,
-    Sim,
-    Subtract,
-)
+from upgini.autofe.binary import Add, Divide, Max, Min, Multiply, Sim, Subtract
 from upgini.autofe.date import (
     DateDiff,
     DateDiffType2,
@@ -23,9 +9,9 @@ from upgini.autofe.date import (
     DatePercentile,
     DatePercentileMethod2,
 )
-from upgini.autofe.groupby import GroupByThenAgg, GroupByThenFreq, GroupByThenNUnique, GroupByThenRank
+from upgini.autofe.groupby import GroupByThenAgg, GroupByThenRank
 from upgini.autofe.operand import Operand
-from upgini.autofe.unary import Abs, Embeddings, Floor, Freq, Log, Residual, Norm, Sigmoid, Sqrt, Square
+from upgini.autofe.unary import Abs, Floor, Freq, Log, Residual, Norm, Sigmoid, Sqrt, Square
 from upgini.autofe.vector import Mean, Sum
 
 ALL_OPERANDS: Dict[str, Operand] = {
@@ -53,10 +39,10 @@ ALL_OPERANDS: Dict[str, Operand] = {
         GroupByThenAgg(name="GroupByThenMedian", agg="median"),
         GroupByThenAgg(name="GroupByThenStd", output_type="float", agg="std"),
         GroupByThenRank(),
-        Combine(),
-        CombineThenFreq(),
-        GroupByThenNUnique(),
-        GroupByThenFreq(),
+        Operand(name="Combine", has_symmetry_importance=True, output_type="object", is_categorical=True),
+        Operand(name="CombineThenFreq", has_symmetry_importance=True, output_type="float"),
+        Operand(name="GroupByThenNUnique", output_type="int", is_vectorizable=True, is_grouping=True),
+        Operand(name="GroupByThenFreq", output_type="float", is_grouping=True),
         Sim(),
         DateDiff(),
         DateDiffType2(),
@@ -73,11 +59,6 @@ ALL_OPERANDS: Dict[str, Operand] = {
         DatePercentile(),
         DatePercentileMethod2(),
         Norm(),
-        JaroWinklerSim1(),
-        JaroWinklerSim2(),
-        LevenshteinSim(),
-        Distance(),
-        Embeddings(),
     ]
 }
 

upgini/autofe/binary.py

@@ -1,11 +1,7 @@
-import abc
-from typing import Optional
-import Levenshtein
 import numpy as np
 import pandas as pd
 from numpy import dot
 from numpy.linalg import norm
-from jarowinkler import jarowinkler_similarity
 
 from upgini.autofe.operand import PandasOperand, VectorizableMixin
 
@@ -134,27 +130,7 @@ class CombineThenFreq(PandasOperand):
         self._loc(temp, value_counts)
 
 
-class Distance(PandasOperand):
-    name = "dist"
-    is_binary = True
-    output_type = "float"
-    is_symmetrical = True
-    has_symmetry_importance = True
-
-    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        return pd.Series(
-            1 - self.__dot(left, right) / (self.__dot(left, left) * self.__dot(right, right)), index=left.index
-        )
-
-    # row-wise dot product
-    def __dot(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        res = (left.dropna() * right.dropna()).apply(np.sum)
-        res = res.reindex(left.index.union(right.index))
-        return res
-
-
-# Left for backward compatibility
-class Sim(Distance):
+class Sim(PandasOperand):
     name = "sim"
     is_binary = True
     output_type = "float"
@@ -162,71 +138,4 @@ class Sim(Distance):
     has_symmetry_importance = True
 
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        return 1 - super().calculate_binary(left, right)
-
-
-class StringSim(PandasOperand, abc.ABC):
-    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        sims = []
-        for i in left.index:
-            left_i = self._prepare_value(left.get(i))
-            right_i = self._prepare_value(right.get(i))
-            if left_i is not None and right_i is not None:
-                sims.append(self._similarity(left_i, right_i))
-            else:
-                sims.append(None)
-
-        return pd.Series(sims, index=left.index)
-
-    @abc.abstractmethod
-    def _prepare_value(self, value: Optional[str]) -> Optional[str]:
-        pass
-
-    @abc.abstractmethod
-    def _similarity(self, left: str, right: str) -> float:
-        pass
-
-
-class JaroWinklerSim1(StringSim):
-    name = "sim_jw1"
-    is_binary = True
-    input_type = "string"
-    output_type = "float"
-    is_symmetrical = True
-    has_symmetry_importance = True
-
-    def _prepare_value(self, value: Optional[str]) -> Optional[str]:
-        return value
-
-    def _similarity(self, left: str, right: str) -> float:
-        return jarowinkler_similarity(left, right)
-
-
-class JaroWinklerSim2(StringSim):
-    name = "sim_jw2"
-    is_binary = True
-    input_type = "string"
-    output_type = "float"
-    is_symmetrical = True
-    has_symmetry_importance = True
-
-    def _prepare_value(self, value: Optional[str]) -> Optional[str]:
-        return value[::-1] if value is not None else None
-
-    def _similarity(self, left: str, right: str) -> float:
-        return jarowinkler_similarity(left, right)
-
-
-class LevenshteinSim(StringSim):
-    name = "sim_lv"
-    is_binary = True
-    input_type = "string"
-    output_type = "float"
-    is_symmetrical = True
-    has_symmetry_importance = True
-
-    def _prepare_value(self, value: Optional[str]) -> Optional[str]:
-        return value
-
-    def _similarity(self, left: str, right: str) -> float:
-        return 1 - Levenshtein.distance(left, right) / max(len(left), len(right))
+        return dot(left, right) / (norm(left) * norm(right))

upgini/autofe/date.py

@@ -43,8 +43,6 @@ class DateDiff(PandasOperand, DateDiffMixin):
     is_binary = True
     has_symmetry_importance = True
 
-    replace_negative: bool = False
-
     def get_params(self) -> Dict[str, Optional[str]]:
         res = super().get_params()
         res.update(
@@ -52,7 +50,6 @@ class DateDiff(PandasOperand, DateDiffMixin):
                 "diff_unit": self.diff_unit,
                 "left_unit": self.left_unit,
                 "right_unit": self.right_unit,
-                "replace_negative": self.replace_negative,
             }
         )
         return res
@@ -64,8 +61,7 @@ class DateDiff(PandasOperand, DateDiffMixin):
         return self.__replace_negative(diff)
 
     def __replace_negative(self, x: Union[pd.DataFrame, pd.Series]):
-        if self.replace_negative:
-            x[x < 0] = None
+        x[x < 0] = None
         return x
 
 
@@ -89,7 +85,7 @@ class DateDiffType2(PandasOperand, DateDiffMixin):
         left = self._convert_to_date(left, self.left_unit)
         right = self._convert_to_date(right, self.right_unit)
         future = right + (left.dt.year - right.dt.year).apply(
-            lambda y: pd.tseries.offsets.DateOffset(years=0 if np.isnan(y) else y)
+            lambda y: np.datetime64("NaT") if np.isnan(y) else pd.tseries.offsets.DateOffset(years=y)
         )
         future = pd.to_datetime(future)
         before = future[future < left]
@@ -105,19 +101,13 @@ _ext_aggregations = {"nunique": (lambda x: len(np.unique(x)), 0), "count": (len,
 class DateListDiff(PandasOperand, DateDiffMixin):
     is_binary = True
     has_symmetry_importance = True
-
     aggregation: str
-    replace_negative: bool = False
 
     def get_params(self) -> Dict[str, Optional[str]]:
         res = super().get_params()
         res.update(
             {
                 "aggregation": self.aggregation,
-                "diff_unit": self.diff_unit,
-                "left_unit": self.left_unit,
-                "right_unit": self.right_unit,
-                "replace_negative": self.replace_negative,
             }
         )
         return res
@@ -135,7 +125,7 @@ class DateListDiff(PandasOperand, DateDiffMixin):
 
     def _diff(self, x: TimedeltaArray):
         x = self._convert_diff_to_unit(x)
-        return x[x > 0] if self.replace_negative else x
+        return x[x > 0]
 
     def _agg(self, x):
         method = getattr(np, self.aggregation, None)
@@ -167,10 +157,7 @@ class DateListDiffBounded(DateListDiff):
         super().__init__(**data)
 
     def _agg(self, x):
-        x = x[
-            (x >= (self.lower_bound if self.lower_bound is not None else -np.inf))
-            & (x < (self.upper_bound if self.upper_bound is not None else np.inf))
-        ]
+        x = x[(x >= (self.lower_bound or -np.inf)) & (x < (self.upper_bound or np.inf))]
         return super()._agg(x)
 
 

upgini/autofe/feature.py

@@ -138,17 +138,15 @@ class Feature:
         if self.cached_display_name is not None and cache:
             return self.cached_display_name
 
-        should_stack_op = not isinstance(self.children[0], Column) if self.op.is_unary else False
-        prev_name = [self.children[0].get_op_display_name()] if should_stack_op else []
-
         if self.alias:
             components = ["f_autofe", self.alias]
-        elif shorten and (not self.op.is_unary or should_stack_op):
-            components = ["f_autofe"] + prev_name + [self.get_op_display_name()]
+        elif shorten and not self.op.is_unary:
+            components = ["f_autofe", self.get_op_display_name()]
         else:
-            components = (
-                ["f_" + "_f_".join(self.get_columns(**kwargs))] + ["autofe"] + prev_name + [self.get_op_display_name()]
-            )
+            components = ["f_" + "_f_".join(self.get_columns(**kwargs))] + [
+                "autofe",
+                self.get_op_display_name(),
+            ]
         components.extend([str(self.display_index)] if self.display_index is not None else [])
         display_name = "_".join(components)
 
@@ -323,10 +321,10 @@ class FeatureGroup:
             lower_order_names = [ch.get_display_name() for ch in lower_order_children]
             if any(isinstance(f, Feature) for f in lower_order_children):
                 child_data = pd.concat(
-                    [data[main_column or []]] + [ch.calculate(data) for ch in lower_order_children],
+                    [data[main_column]] + [ch.calculate(data) for ch in lower_order_children],
                     axis=1,
                 )
-                child_data.columns = ([main_column] if main_column is not None else []) + lower_order_names
+                child_data.columns = [main_column] + lower_order_names
             else:
                 child_data = data[columns]
 

upgini/autofe/unary.py

@@ -125,10 +125,3 @@ class Norm(PandasOperand):
         normalized_data = pd.Series(normalized_data[:, 0], index=data_dropna.index, name=data.name)
         normalized_data = normalized_data.reindex(data.index)
         return normalized_data
-
-
-class Embeddings(PandasOperand):
-    name = "emb"
-    is_unary = True
-    input_type = "string"
-    output_type = "vector"

upgini/dataset.py

@@ -23,7 +23,9 @@ from pandas.api.types import (
 from upgini.errors import ValidationError
 from upgini.http import ProgressStage, SearchProgress, _RestClient
 from upgini.metadata import (
+    ENTITY_SYSTEM_RECORD_ID,
     EVAL_SET_INDEX,
+    SEARCH_KEY_UNNEST,
     SYSTEM_COLUMNS,
     SYSTEM_RECORD_ID,
     TARGET,
@@ -79,6 +81,7 @@ class Dataset:  # (pd.DataFrame):
         path: Optional[str] = None,
         meaning_types: Optional[Dict[str, FileColumnMeaningType]] = None,
         search_keys: Optional[List[Tuple[str, ...]]] = None,
+        unnest_search_keys: Optional[Dict[str, str]] = None,
         model_task_type: Optional[ModelTaskType] = None,
         random_state: Optional[int] = None,
         rest_client: Optional[_RestClient] = None,
@@ -113,6 +116,7 @@ class Dataset:  # (pd.DataFrame):
         self.description = description
         self.meaning_types = meaning_types
         self.search_keys = search_keys
+        self.unnest_search_keys = unnest_search_keys
         self.ignore_columns = []
         self.hierarchical_group_keys = []
         self.hierarchical_subgroup_keys = []
@@ -172,7 +176,7 @@ class Dataset:  # (pd.DataFrame):
         new_columns = []
         dup_counter = 0
         for column in self.data.columns:
-            if column in [TARGET, EVAL_SET_INDEX, SYSTEM_RECORD_ID]:
+            if column in [TARGET, EVAL_SET_INDEX, SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST]:
                 self.columns_renaming[column] = column
                 new_columns.append(column)
                 continue
@@ -353,7 +357,9 @@ class Dataset:  # (pd.DataFrame):
 
             if is_string_dtype(self.data[postal_code]) or is_object_dtype(self.data[postal_code]):
                 try:
-                    self.data[postal_code] = self.data[postal_code].astype("float64").astype("Int64").astype("string")
+                    self.data[postal_code] = (
+                        self.data[postal_code].astype("string").astype("Float64").astype("Int64").astype("string")
+                    )
                 except Exception:
                     pass
             elif is_float_dtype(self.data[postal_code]):
@@ -803,6 +809,9 @@ class Dataset:  # (pd.DataFrame):
                     meaningType=meaning_type,
                     minMaxValues=min_max_values,
                 )
+                if self.unnest_search_keys and column_meta.originalName in self.unnest_search_keys:
+                    column_meta.isUnnest = True
+                    column_meta.unnestKeyNames = self.unnest_search_keys[column_meta.originalName]
 
                 columns.append(column_meta)
 

upgini/features_enricher.py

@@ -11,6 +11,7 @@ import sys
 import tempfile
 import time
 import uuid
+from collections import Counter
 from dataclasses import dataclass
 from threading import Thread
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
@@ -45,9 +46,11 @@ from upgini.mdc import MDC
 from upgini.metadata import (
     COUNTRY,
     DEFAULT_INDEX,
+    ENTITY_SYSTEM_RECORD_ID,
     EVAL_SET_INDEX,
     ORIGINAL_INDEX,
     RENAMED_INDEX,
+    SEARCH_KEY_UNNEST,
     SORT_ID,
     SYSTEM_RECORD_ID,
     TARGET,
@@ -248,7 +251,7 @@ class FeaturesEnricher(TransformerMixin):
         self.__cached_sampled_datasets: Optional[Tuple[pd.DataFrame, pd.DataFrame, pd.Series, Dict, Dict]] = None
 
         validate_version(self.logger)
-        self.search_keys = search_keys or dict()
+        self.search_keys = search_keys or {}
         self.country_code = country_code
         self.__validate_search_keys(search_keys, search_id)
         self.model_task_type = model_task_type
@@ -1200,7 +1203,7 @@ class FeaturesEnricher(TransformerMixin):
         email_column = self._get_email_column(search_keys)
         hem_column = self._get_hem_column(search_keys)
         if email_column:
-            converter = EmailSearchKeyConverter(email_column, hem_column, search_keys, self.logger)
+            converter = EmailSearchKeyConverter(email_column, hem_column, search_keys, [], self.logger)
             extended_X = converter.convert(extended_X)
             generated_features.extend(converter.generated_features)
         if (
@@ -1353,7 +1356,7 @@ class FeaturesEnricher(TransformerMixin):
             not in (
                 excluding_search_keys
                 + list(self.fit_dropped_features)
-                + [DateTimeSearchKeyConverter.DATETIME_COL, SYSTEM_RECORD_ID]
+                + [DateTimeSearchKeyConverter.DATETIME_COL, SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID]
             )
         ]
 
@@ -1417,7 +1420,7 @@ class FeaturesEnricher(TransformerMixin):
                     fitting_enriched_X[col].astype("string").str.replace(",", ".", regex=False).astype(np.float64)
                 )
 
-        fitting_eval_set_dict = dict()
+        fitting_eval_set_dict = {}
         for idx, eval_tuple in eval_set_sampled_dict.items():
             eval_X_sampled, enriched_eval_X, eval_y_sampled = eval_tuple
             eval_X_sorted, eval_y_sorted = self._sort_by_system_record_id(eval_X_sampled, eval_y_sampled, self.cv)
@@ -1534,7 +1537,7 @@ class FeaturesEnricher(TransformerMixin):
     def __sample_only_input(
         self, validated_X: pd.DataFrame, validated_y: pd.Series, eval_set: Optional[List[tuple]], is_demo_dataset: bool
     ) -> _SampledDataForMetrics:
-        eval_set_sampled_dict = dict()
+        eval_set_sampled_dict = {}
 
         df = validated_X.copy()
         df[TARGET] = validated_y
@@ -1560,7 +1563,7 @@ class FeaturesEnricher(TransformerMixin):
             df = df.sample(n=sample_rows, random_state=self.random_state)
 
         df_extended, search_keys = self._extend_x(df, is_demo_dataset)
-        df_extended = self.__add_fit_system_record_id(df_extended, dict(), search_keys)
+        df_extended = self.__add_fit_system_record_id(df_extended, {}, search_keys, SYSTEM_RECORD_ID)
 
         train_df = df_extended.query(f"{EVAL_SET_INDEX} == 0") if eval_set is not None else df_extended
         X_sampled = train_df.drop(columns=[TARGET, EVAL_SET_INDEX], errors="ignore")
@@ -1584,7 +1587,7 @@ class FeaturesEnricher(TransformerMixin):
         trace_id: str,
         remove_outliers_calc_metrics: Optional[bool],
     ) -> _SampledDataForMetrics:
-        eval_set_sampled_dict = dict()
+        eval_set_sampled_dict = {}
         search_keys = self.fit_search_keys
 
         rows_to_drop = None
@@ -1598,8 +1601,7 @@ class FeaturesEnricher(TransformerMixin):
                 outliers = pd.merge(
                     self.df_with_original_index,
                     target_outliers_df,
-                    left_on=SYSTEM_RECORD_ID,
-                    right_on=SYSTEM_RECORD_ID,
+                    on=ENTITY_SYSTEM_RECORD_ID,
                     how="inner",
                 )
                 top_outliers = outliers.sort_values(by=TARGET, ascending=False)[TARGET].head(3)
@@ -1658,7 +1660,7 @@ class FeaturesEnricher(TransformerMixin):
         progress_bar: Optional[ProgressBar],
         progress_callback: Optional[Callable[[SearchProgress], Any]],
     ) -> _SampledDataForMetrics:
-        eval_set_sampled_dict = dict()
+        eval_set_sampled_dict = {}
         if eval_set is not None:
             self.logger.info("Transform with eval_set")
             # concatenate X and eval_set with eval_set_index
@@ -1680,7 +1682,7 @@ class FeaturesEnricher(TransformerMixin):
                 self.logger.info(f"Downsampling from {num_samples} to {Dataset.FIT_SAMPLE_WITH_EVAL_SET_ROWS}")
                 df = df.sample(n=Dataset.FIT_SAMPLE_WITH_EVAL_SET_ROWS, random_state=self.random_state)
 
-            eval_set_sampled_dict = dict()
+            eval_set_sampled_dict = {}
 
             tmp_target_name = "__target"
             df = df.rename(columns={TARGET: tmp_target_name})
@@ -1943,11 +1945,38 @@ class FeaturesEnricher(TransformerMixin):
                 self.logger.info("Input dataset hasn't date column")
                 if self.add_date_if_missing:
                     df = self._add_current_date_as_key(df, search_keys, self.logger, self.bundle)
+
+            # Don't pass all features in backend on transform
+            original_features_for_transform = []
+            runtime_parameters = self._get_copy_of_runtime_parameters()
+            features_not_to_pass = [column for column in df.columns if column not in search_keys.keys()]
+            if len(features_not_to_pass) > 0:
+                # Pass only features that need for transform
+                features_for_transform = self._search_task.get_features_for_transform()
+                if features_for_transform is not None and len(features_for_transform) > 0:
+                    file_metadata = self._search_task.get_file_metadata(trace_id)
+                    original_features_for_transform = [
+                        c.originalName or c.name for c in file_metadata.columns if c.name in features_for_transform
+                    ]
+
+                    runtime_parameters.properties["features_for_embeddings"] = ",".join(features_for_transform)
+
+            columns_for_system_record_id = sorted(list(search_keys.keys()) + (original_features_for_transform))
+
+            df[ENTITY_SYSTEM_RECORD_ID] = pd.util.hash_pandas_object(
+                df[columns_for_system_record_id], index=False
+            ).astype("Float64")
+
+            # Explode multiple search keys
+            df, unnest_search_keys = self._explode_multiple_search_keys(df, search_keys)
+
             email_column = self._get_email_column(search_keys)
             hem_column = self._get_hem_column(search_keys)
             email_converted_to_hem = False
             if email_column:
-                converter = EmailSearchKeyConverter(email_column, hem_column, search_keys, self.logger)
+                converter = EmailSearchKeyConverter(
+                    email_column, hem_column, search_keys, list(unnest_search_keys.keys()), self.logger
+                )
                 df = converter.convert(df)
                 generated_features.extend(converter.generated_features)
                 email_converted_to_hem = converter.email_converted_to_hem
@@ -1961,30 +1990,21 @@ class FeaturesEnricher(TransformerMixin):
             generated_features = [f for f in generated_features if f in self.fit_generated_features]
 
             meaning_types = {col: key.value for col, key in search_keys.items()}
-            non_keys_columns = [column for column in df.columns if column not in search_keys.keys()]
+            # non_keys_columns = [column for column in df.columns if column not in search_keys.keys()]
+            for col in original_features_for_transform:
+                meaning_types[col] = FileColumnMeaningType.FEATURE
+            features_not_to_pass = [column for column in features_not_to_pass if column not in search_keys.keys()]
 
             if email_converted_to_hem:
-                non_keys_columns.append(email_column)
+                features_not_to_pass.append(email_column)
 
-            # Don't pass features in backend on transform
-            original_features_for_transform = None
-            runtime_parameters = self._get_copy_of_runtime_parameters()
-            if len(non_keys_columns) > 0:
-                # Pass only features that need for transform
-                features_for_transform = self._search_task.get_features_for_transform()
-                if features_for_transform is not None and len(features_for_transform) > 0:
-                    file_metadata = self._search_task.get_file_metadata(trace_id)
-                    original_features_for_transform = [
-                        c.originalName or c.name for c in file_metadata.columns if c.name in features_for_transform
-                    ]
-                    non_keys_columns = [c for c in non_keys_columns if c not in original_features_for_transform]
-
-                    runtime_parameters.properties["features_for_embeddings"] = ",".join(features_for_transform)
+            features_not_to_pass = [c for c in features_not_to_pass if c not in original_features_for_transform]
+            columns_for_system_record_id = sorted(list(search_keys.keys()) + (original_features_for_transform))
 
             if add_fit_system_record_id:
-                df = self.__add_fit_system_record_id(df, dict(), search_keys)
+                df = self.__add_fit_system_record_id(df, {}, search_keys, SYSTEM_RECORD_ID)
                 df = df.rename(columns={SYSTEM_RECORD_ID: SORT_ID})
-                non_keys_columns.append(SORT_ID)
+                features_not_to_pass.append(SORT_ID)
 
             columns_for_system_record_id = sorted(list(search_keys.keys()) + (original_features_for_transform or []))
 
@@ -1992,16 +2012,19 @@ class FeaturesEnricher(TransformerMixin):
                 "Float64"
             )
             meaning_types[SYSTEM_RECORD_ID] = FileColumnMeaningType.SYSTEM_RECORD_ID
+            meaning_types[ENTITY_SYSTEM_RECORD_ID] = FileColumnMeaningType.ENTITY_SYSTEM_RECORD_ID
+            if SEARCH_KEY_UNNEST in df.columns:
+                meaning_types[SEARCH_KEY_UNNEST] = FileColumnMeaningType.UNNEST_KEY
 
             df = df.reset_index(drop=True)
-            system_columns_with_original_index = [SYSTEM_RECORD_ID] + generated_features
+            system_columns_with_original_index = [SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID] + generated_features
             if add_fit_system_record_id:
                 system_columns_with_original_index.append(SORT_ID)
             df_with_original_index = df[system_columns_with_original_index].copy()
 
             combined_search_keys = combine_search_keys(search_keys.keys())
 
-            df_without_features = df.drop(columns=non_keys_columns)
+            df_without_features = df.drop(columns=features_not_to_pass)
 
             df_without_features = clean_full_duplicates(
                 df_without_features, self.logger, silent=silent_mode, bundle=self.bundle
@@ -2013,12 +2036,13 @@ class FeaturesEnricher(TransformerMixin):
             dataset = Dataset(
                 "sample_" + str(uuid.uuid4()),
                 df=df_without_features,
+                meaning_types=meaning_types,
+                search_keys=combined_search_keys,
+                unnest_search_keys=unnest_search_keys,
                 date_format=self.date_format,
                 rest_client=self.rest_client,
                 logger=self.logger,
             )
-            dataset.meaning_types = meaning_types
-            dataset.search_keys = combined_search_keys
             if email_converted_to_hem:
                 dataset.ignore_columns = [email_column]
 
@@ -2157,6 +2181,14 @@ class FeaturesEnricher(TransformerMixin):
 
         key_types = search_keys.values()
 
+        # Multiple search keys allowed only for PHONE, IP, POSTAL_CODE, EMAIL, HEM
+        multi_keys = [key for key, count in Counter(key_types).items() if count > 1]
+        for multi_key in multi_keys:
+            if multi_key not in [SearchKey.PHONE, SearchKey.IP, SearchKey.POSTAL_CODE, SearchKey.EMAIL, SearchKey.HEM]:
+                msg = self.bundle.get("unsupported_multi_key").format(multi_key)
+                self.logger.warning(msg)
+                raise ValidationError(msg)
+
         if SearchKey.DATE in key_types and SearchKey.DATETIME in key_types:
             msg = self.bundle.get("date_and_datetime_simultanious")
             self.logger.warning(msg)
@@ -2172,11 +2204,11 @@ class FeaturesEnricher(TransformerMixin):
             self.logger.warning(msg)
             raise ValidationError(msg)
 
-        for key_type in SearchKey.__members__.values():
-            if key_type != SearchKey.CUSTOM_KEY and list(key_types).count(key_type) > 1:
-                msg = self.bundle.get("multiple_search_key").format(key_type)
-                self.logger.warning(msg)
-                raise ValidationError(msg)
+        # for key_type in SearchKey.__members__.values():
+        #     if key_type != SearchKey.CUSTOM_KEY and list(key_types).count(key_type) > 1:
+        #         msg = self.bundle.get("multiple_search_key").format(key_type)
+        #         self.logger.warning(msg)
+        #         raise ValidationError(msg)
 
         # non_personal_keys = set(SearchKey.__members__.values()) - set(SearchKey.personal_keys())
         # if (
@@ -2314,14 +2346,7 @@ class FeaturesEnricher(TransformerMixin):
             self.logger.info("Input dataset hasn't date column")
             if self.add_date_if_missing:
                 df = self._add_current_date_as_key(df, self.fit_search_keys, self.logger, self.bundle)
-        email_column = self._get_email_column(self.fit_search_keys)
-        hem_column = self._get_hem_column(self.fit_search_keys)
-        email_converted_to_hem = False
-        if email_column:
-            converter = EmailSearchKeyConverter(email_column, hem_column, self.fit_search_keys, self.logger)
-            df = converter.convert(df)
-            self.fit_generated_features.extend(converter.generated_features)
-            email_converted_to_hem = converter.email_converted_to_hem
+
         if (
             self.detect_missing_search_keys
             and list(self.fit_search_keys.values()) == [SearchKey.DATE]
@@ -2330,7 +2355,37 @@ class FeaturesEnricher(TransformerMixin):
             converter = IpToCountrySearchKeyConverter(self.fit_search_keys, self.logger)
             df = converter.convert(df)
 
+        # Explode multiple search keys
         non_feature_columns = [self.TARGET_NAME, EVAL_SET_INDEX] + list(self.fit_search_keys.keys())
+        meaning_types = {
+            **{col: key.value for col, key in self.fit_search_keys.items()},
+            **{str(c): FileColumnMeaningType.FEATURE for c in df.columns if c not in non_feature_columns},
+        }
+        meaning_types[self.TARGET_NAME] = FileColumnMeaningType.TARGET
+        if eval_set is not None and len(eval_set) > 0:
+            meaning_types[EVAL_SET_INDEX] = FileColumnMeaningType.EVAL_SET_INDEX
+        df = self.__add_fit_system_record_id(df, meaning_types, self.fit_search_keys, ENTITY_SYSTEM_RECORD_ID)
+
+        # TODO check that this is correct for enrichment
+        self.df_with_original_index = df.copy()
+
+        df, unnest_search_keys = self._explode_multiple_search_keys(df, self.fit_search_keys)
+
+        # Convert EMAIL to HEM after unnesting to do it only with one column
+        email_column = self._get_email_column(self.fit_search_keys)
+        hem_column = self._get_hem_column(self.fit_search_keys)
+        email_converted_to_hem = False
+        if email_column:
+            converter = EmailSearchKeyConverter(
+                email_column, hem_column, self.fit_search_keys, list(unnest_search_keys.keys()), self.logger
+            )
+            df = converter.convert(df)
+            self.fit_generated_features.extend(converter.generated_features)
+            email_converted_to_hem = converter.email_converted_to_hem
+
+        non_feature_columns = [self.TARGET_NAME, EVAL_SET_INDEX, ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST] + list(
+            self.fit_search_keys.keys()
+        )
         if email_converted_to_hem:
             non_feature_columns.append(email_column)
         if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
@@ -2354,12 +2409,14 @@ class FeaturesEnricher(TransformerMixin):
             **{str(c): FileColumnMeaningType.FEATURE for c in df.columns if c not in non_feature_columns},
         }
         meaning_types[self.TARGET_NAME] = FileColumnMeaningType.TARGET
+        meaning_types[ENTITY_SYSTEM_RECORD_ID] = FileColumnMeaningType.ENTITY_SYSTEM_RECORD_ID
+        if SEARCH_KEY_UNNEST in df.columns:
+            meaning_types[SEARCH_KEY_UNNEST] = FileColumnMeaningType.UNNEST_KEY
         if eval_set is not None and len(eval_set) > 0:
             meaning_types[EVAL_SET_INDEX] = FileColumnMeaningType.EVAL_SET_INDEX
 
-        df = self.__add_fit_system_record_id(df, meaning_types, self.fit_search_keys)
+        df = self.__add_fit_system_record_id(df, meaning_types, self.fit_search_keys, SYSTEM_RECORD_ID)
 
-        self.df_with_original_index = df.copy()
         df = df.reset_index(drop=True).sort_values(by=SYSTEM_RECORD_ID).reset_index(drop=True)
 
         combined_search_keys = combine_search_keys(self.fit_search_keys.keys())
@@ -2367,14 +2424,15 @@ class FeaturesEnricher(TransformerMixin):
         dataset = Dataset(
             "tds_" + str(uuid.uuid4()),
             df=df,
+            meaning_types=meaning_types,
+            search_keys=combined_search_keys,
+            unnest_search_keys=unnest_search_keys,
             model_task_type=model_task_type,
             date_format=self.date_format,
             random_state=self.random_state,
             rest_client=self.rest_client,
             logger=self.logger,
         )
-        dataset.meaning_types = meaning_types
-        dataset.search_keys = combined_search_keys
         if email_converted_to_hem:
             dataset.ignore_columns = [email_column]
 
@@ -2744,9 +2802,10 @@ class FeaturesEnricher(TransformerMixin):
         X: pd.DataFrame, y: pd.Series, cv: Optional[CVType]
     ) -> Tuple[pd.DataFrame, pd.Series]:
         if cv not in [CVType.time_series, CVType.blocked_time_series]:
+            record_id_column = ENTITY_SYSTEM_RECORD_ID if ENTITY_SYSTEM_RECORD_ID in X else SYSTEM_RECORD_ID
             Xy = X.copy()
             Xy[TARGET] = y
-            Xy = Xy.sort_values(by=SYSTEM_RECORD_ID).reset_index(drop=True)
+            Xy = Xy.sort_values(by=record_id_column).reset_index(drop=True)
             X = Xy.drop(columns=TARGET)
             y = Xy[TARGET].copy()
 
@@ -2925,15 +2984,19 @@ class FeaturesEnricher(TransformerMixin):
 
     @staticmethod
     def _get_email_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
-        for col, t in search_keys.items():
-            if t == SearchKey.EMAIL:
-                return col
+        cols = [col for col, t in search_keys.items() if t == SearchKey.EMAIL]
+        if len(cols) > 1:
+            raise Exception("More than one email column found after unnest")
+        if len(cols) == 1:
+            return cols[0]
 
     @staticmethod
     def _get_hem_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
-        for col, t in search_keys.items():
-            if t == SearchKey.HEM:
-                return col
+        cols = [col for col, t in search_keys.items() if t == SearchKey.HEM]
+        if len(cols) > 1:
+            raise Exception("More than one hem column found after unnest")
+        if len(cols) == 1:
+            return cols[0]
 
     @staticmethod
     def _get_phone_column(search_keys: Dict[str, SearchKey]) -> Optional[str]:
@@ -2941,8 +3004,44 @@ class FeaturesEnricher(TransformerMixin):
             if t == SearchKey.PHONE:
                 return col
 
+    def _explode_multiple_search_keys(
+        self, df: pd.DataFrame, search_keys: Dict[str, SearchKey]
+    ) -> Tuple[pd.DataFrame, Dict[str, List[str]]]:
+        # find groups of multiple search keys
+        search_key_names_by_type: Dict[SearchKey, str] = {}
+        for key_name, key_type in search_keys.items():
+            search_key_names_by_type[key_type] = search_key_names_by_type.get(key_type, []) + [key_name]
+        search_key_names_by_type = {
+            key_type: key_names for key_type, key_names in search_key_names_by_type.items() if len(key_names) > 1
+        }
+        if len(search_key_names_by_type) == 0:
+            return df, {}
+
+        multiple_keys_columns = [col for cols in search_key_names_by_type.values() for col in cols]
+        other_columns = [col for col in df.columns if col not in multiple_keys_columns]
+        exploded_dfs = []
+        unnest_search_keys = {}
+
+        for key_type, key_names in search_key_names_by_type.items():
+            new_search_key = f"upgini_{key_type.name.lower()}_unnest"
+            exploded_df = pd.melt(
+                df, id_vars=other_columns, value_vars=key_names, var_name=SEARCH_KEY_UNNEST, value_name=new_search_key
+            )
+            exploded_dfs.append(exploded_df)
+            for old_key in key_names:
+                del search_keys[old_key]
+            search_keys[new_search_key] = key_type
+            unnest_search_keys[new_search_key] = key_names
+
+        df = pd.concat(exploded_dfs, ignore_index=True)
+        return df, unnest_search_keys
+
     def __add_fit_system_record_id(
-        self, df: pd.DataFrame, meaning_types: Dict[str, FileColumnMeaningType], search_keys: Dict[str, SearchKey]
+        self,
+        df: pd.DataFrame,
+        meaning_types: Dict[str, FileColumnMeaningType],
+        search_keys: Dict[str, SearchKey],
+        id_name: str,
     ) -> pd.DataFrame:
         # save original order or rows
         original_index_name = df.index.name
@@ -2953,7 +3052,14 @@ class FeaturesEnricher(TransformerMixin):
 
         # order by date and idempotent order by other keys
         if self.cv not in [CVType.time_series, CVType.blocked_time_series]:
-            sort_exclude_columns = [original_order_name, ORIGINAL_INDEX, EVAL_SET_INDEX, TARGET, "__target"]
+            sort_exclude_columns = [
+                original_order_name,
+                ORIGINAL_INDEX,
+                EVAL_SET_INDEX,
+                TARGET,
+                "__target",
+                ENTITY_SYSTEM_RECORD_ID,
+            ]
             if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
                 date_column = DateTimeSearchKeyConverter.DATETIME_COL
                 sort_exclude_columns.append(self._get_date_column(search_keys))
@@ -2991,14 +3097,18 @@ class FeaturesEnricher(TransformerMixin):
 
         df = df.reset_index(drop=True).reset_index()
         # system_record_id saves correct order for fit
-        df = df.rename(columns={DEFAULT_INDEX: SYSTEM_RECORD_ID})
+        df = df.rename(columns={DEFAULT_INDEX: id_name})
 
         # return original order
         df = df.set_index(ORIGINAL_INDEX)
         df.index.name = original_index_name
         df = df.sort_values(by=original_order_name).drop(columns=original_order_name)
 
-        meaning_types[SYSTEM_RECORD_ID] = FileColumnMeaningType.SYSTEM_RECORD_ID
+        meaning_types[id_name] = (
+            FileColumnMeaningType.SYSTEM_RECORD_ID
+            if id_name == SYSTEM_RECORD_ID
+            else FileColumnMeaningType.ENTITY_SYSTEM_RECORD_ID
+        )
         return df
 
     def __correct_target(self, df: pd.DataFrame) -> pd.DataFrame:
@@ -3053,7 +3163,11 @@ class FeaturesEnricher(TransformerMixin):
         )
 
         comparing_columns = X.columns if is_transform else df_with_original_index.columns
-        dup_features = [c for c in comparing_columns if c in result_features.columns and c != SYSTEM_RECORD_ID]
+        dup_features = [
+            c
+            for c in comparing_columns
+            if c in result_features.columns and c not in [SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID]
+        ]
         if len(dup_features) > 0:
             self.logger.warning(f"X contain columns with same name as returned from backend: {dup_features}")
             raise ValidationError(self.bundle.get("returned_features_same_as_passed").format(dup_features))
@@ -3064,8 +3178,7 @@ class FeaturesEnricher(TransformerMixin):
         result_features = pd.merge(
             df_with_original_index,
             result_features,
-            left_on=SYSTEM_RECORD_ID,
-            right_on=SYSTEM_RECORD_ID,
+            on=ENTITY_SYSTEM_RECORD_ID,
             how="left" if is_transform else "inner",
         )
         result_features = result_features.set_index(original_index_name or DEFAULT_INDEX)
@@ -3076,7 +3189,7 @@ class FeaturesEnricher(TransformerMixin):
             result_features = result_features[~result_features[SYSTEM_RECORD_ID].isin(rows_to_drop[SYSTEM_RECORD_ID])]
             self.logger.info(f"After dropping target outliers size: {len(result_features)}")
 
-        result_eval_sets = dict()
+        result_eval_sets = {}
         if not is_transform and EVAL_SET_INDEX in result_features.columns:
             result_train_features = result_features.loc[result_features[EVAL_SET_INDEX] == 0].copy()
             eval_set_indices = list(result_features[EVAL_SET_INDEX].unique())
@@ -3288,7 +3401,7 @@ class FeaturesEnricher(TransformerMixin):
                 if autofe_feature.op.is_vector:
                     continue
 
-                description = dict()
+                description = {}
 
                 feature_meta = get_feature_by_name(autofe_feature.get_display_name(shorten=True))
                 if feature_meta is None:
@@ -3454,13 +3567,13 @@ class FeaturesEnricher(TransformerMixin):
                 self.warning_counter.increment()
 
         if len(valid_search_keys) == 1:
-            for k, v in valid_search_keys.items():
-                # Show warning for country only if country is the only key
-                if x[k].nunique() == 1 and (v != SearchKey.COUNTRY or len(valid_search_keys) == 1):
-                    msg = self.bundle.get("single_constant_search_key").format(v, x[k].values[0])
-                    print(msg)
-                    self.logger.warning(msg)
-                    self.warning_counter.increment()
+            key, value = list(valid_search_keys.items())[0]
+            # Show warning for country only if country is the only key
+            if x[key].nunique() == 1:
+                msg = self.bundle.get("single_constant_search_key").format(value, x[key].values[0])
+                print(msg)
+                self.logger.warning(msg)
+                self.warning_counter.increment()
 
         self.logger.info(f"Prepared search keys: {valid_search_keys}")
 
@@ -3570,61 +3683,68 @@ class FeaturesEnricher(TransformerMixin):
         def check_need_detect(search_key: SearchKey):
             return not is_transform or search_key in self.fit_search_keys.values()
 
-        if SearchKey.POSTAL_CODE not in search_keys.values() and check_need_detect(SearchKey.POSTAL_CODE):
-            maybe_key = PostalCodeSearchKeyDetector().get_search_key_column(sample)
-            if maybe_key is not None:
-                search_keys[maybe_key] = SearchKey.POSTAL_CODE
-                self.autodetected_search_keys[maybe_key] = SearchKey.POSTAL_CODE
-                self.logger.info(f"Autodetected search key POSTAL_CODE in column {maybe_key}")
+        # if SearchKey.POSTAL_CODE not in search_keys.values() and check_need_detect(SearchKey.POSTAL_CODE):
+        if check_need_detect(SearchKey.POSTAL_CODE):
+            maybe_keys = PostalCodeSearchKeyDetector().get_search_key_columns(sample, search_keys)
+            if maybe_keys:
+                new_keys = {key: SearchKey.POSTAL_CODE for key in maybe_keys}
+                search_keys.update(new_keys)
+                self.autodetected_search_keys.update(new_keys)
+                self.logger.info(f"Autodetected search key POSTAL_CODE in column {maybe_keys}")
                 if not silent_mode:
-                    print(self.bundle.get("postal_code_detected").format(maybe_key))
+                    print(self.bundle.get("postal_code_detected").format(maybe_keys))
 
         if (
             SearchKey.COUNTRY not in search_keys.values()
             and self.country_code is None
             and check_need_detect(SearchKey.COUNTRY)
         ):
-            maybe_key = CountrySearchKeyDetector().get_search_key_column(sample)
-            if maybe_key is not None:
-                search_keys[maybe_key] = SearchKey.COUNTRY
-                self.autodetected_search_keys[maybe_key] = SearchKey.COUNTRY
+            maybe_key = CountrySearchKeyDetector().get_search_key_columns(sample, search_keys)
+            if maybe_key:
+                search_keys[maybe_key[0]] = SearchKey.COUNTRY
+                self.autodetected_search_keys[maybe_key[0]] = SearchKey.COUNTRY
                 self.logger.info(f"Autodetected search key COUNTRY in column {maybe_key}")
                 if not silent_mode:
                     print(self.bundle.get("country_detected").format(maybe_key))
 
         if (
-            SearchKey.EMAIL not in search_keys.values()
-            and SearchKey.HEM not in search_keys.values()
+            # SearchKey.EMAIL not in search_keys.values()
+            SearchKey.HEM not in search_keys.values()
             and check_need_detect(SearchKey.HEM)
         ):
-            maybe_key = EmailSearchKeyDetector().get_search_key_column(sample)
-            if maybe_key is not None and maybe_key not in search_keys.keys():
+            maybe_keys = EmailSearchKeyDetector().get_search_key_columns(sample, search_keys)
+            if maybe_keys:
                 if self.__is_registered or is_demo_dataset:
-                    search_keys[maybe_key] = SearchKey.EMAIL
-                    self.autodetected_search_keys[maybe_key] = SearchKey.EMAIL
-                    self.logger.info(f"Autodetected search key EMAIL in column {maybe_key}")
+                    new_keys = {key: SearchKey.EMAIL for key in maybe_keys}
+                    search_keys.update(new_keys)
+                    self.autodetected_search_keys.update(new_keys)
+                    self.logger.info(f"Autodetected search key EMAIL in column {maybe_keys}")
                     if not silent_mode:
-                        print(self.bundle.get("email_detected").format(maybe_key))
+                        print(self.bundle.get("email_detected").format(maybe_keys))
                 else:
                     self.logger.warning(
-                        f"Autodetected search key EMAIL in column {maybe_key}. But not used because not registered user"
+                        f"Autodetected search key EMAIL in column {maybe_keys}."
+                        " But not used because not registered user"
                     )
                     if not silent_mode:
-                        print(self.bundle.get("email_detected_not_registered").format(maybe_key))
+                        print(self.bundle.get("email_detected_not_registered").format(maybe_keys))
                     self.warning_counter.increment()
 
-        if SearchKey.PHONE not in search_keys.values() and check_need_detect(SearchKey.PHONE):
-            maybe_key = PhoneSearchKeyDetector().get_search_key_column(sample)
-            if maybe_key is not None and maybe_key not in search_keys.keys():
+        # if SearchKey.PHONE not in search_keys.values() and check_need_detect(SearchKey.PHONE):
+        if check_need_detect(SearchKey.PHONE):
+            maybe_keys = PhoneSearchKeyDetector().get_search_key_columns(sample, search_keys)
+            if maybe_keys:
                 if self.__is_registered or is_demo_dataset:
-                    search_keys[maybe_key] = SearchKey.PHONE
-                    self.autodetected_search_keys[maybe_key] = SearchKey.PHONE
-                    self.logger.info(f"Autodetected search key PHONE in column {maybe_key}")
+                    new_keys = {key: SearchKey.PHONE for key in maybe_keys}
+                    search_keys.update(new_keys)
+                    self.autodetected_search_keys.update(new_keys)
+                    self.logger.info(f"Autodetected search key PHONE in column {maybe_keys}")
                     if not silent_mode:
-                        print(self.bundle.get("phone_detected").format(maybe_key))
+                        print(self.bundle.get("phone_detected").format(maybe_keys))
                 else:
                     self.logger.warning(
-                        f"Autodetected search key PHONE in column {maybe_key}. But not used because not registered user"
+                        f"Autodetected search key PHONE in column {maybe_keys}. "
+                        "But not used because not registered user"
                     )
                     if not silent_mode:
                         print(self.bundle.get("phone_detected_not_registered"))

upgini/metadata.py

@@ -6,6 +6,8 @@ from typing import Dict, List, Optional, Set
 from pydantic import BaseModel
 
 SYSTEM_RECORD_ID = "system_record_id"
+ENTITY_SYSTEM_RECORD_ID = "entity_system_record_id"
+SEARCH_KEY_UNNEST = "search_key_unnest"
 SORT_ID = "sort_id"
 EVAL_SET_INDEX = "eval_set_index"
 TARGET = "target"
@@ -13,7 +15,7 @@ COUNTRY = "country_iso_code"
 RENAMED_INDEX = "index_col"
 DEFAULT_INDEX = "index"
 ORIGINAL_INDEX = "original_index"
-SYSTEM_COLUMNS = {SYSTEM_RECORD_ID, EVAL_SET_INDEX, TARGET, COUNTRY, SORT_ID}
+SYSTEM_COLUMNS = {SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST, EVAL_SET_INDEX, TARGET, COUNTRY}
 
 
 class FileColumnMeaningType(Enum):
@@ -39,6 +41,8 @@ class FileColumnMeaningType(Enum):
     POSTAL_CODE = "POSTAL_CODE"
     SYSTEM_RECORD_ID = "SYSTEM_RECORD_ID"
     EVAL_SET_INDEX = "EVAL_SET_INDEX"
+    ENTITY_SYSTEM_RECORD_ID = "ENTITY_SYSTEM_RECORD_ID"
+    UNNEST_KEY = "UNNEST_KEY"
 
 
 class SearchKey(Enum):
@@ -184,6 +188,10 @@ class FileColumnMetadata(BaseModel):
     meaningType: FileColumnMeaningType
     minMaxValues: Optional[NumericInterval] = None
     originalName: Optional[str]
+    # is this column contains keys from multiple key columns like msisdn1, msisdn2
+    isUnnest: bool = False
+    # list of original etalon key column names like msisdn1, msisdn2
+    unnestKeyNames: Optional[list[str]]
 
 
 class FileMetadata(BaseModel):
@@ -281,7 +289,7 @@ class FeaturesFilter(BaseModel):
 
 
 class RuntimeParameters(BaseModel):
-    properties: Dict[str, str] = dict()
+    properties: Dict[str, str] = {}
 
 
 class SearchCustomization(BaseModel):

upgini/metrics.py

@@ -369,7 +369,7 @@ class EstimatorWrapper:
             "logger": logger,
         }
         if estimator is None:
-            params = dict()
+            params = {}
             params["has_time"] = has_date
             # if metric_name.upper() in SUPPORTED_CATBOOST_METRICS:
             #     params["eval_metric"] = SUPPORTED_CATBOOST_METRICS[metric_name.upper()]

upgini/resource_bundle/strings.properties

@@ -88,6 +88,7 @@ unsupported_search_key_type=Unsupported type of key in search_keys: {}
 search_key_country_and_country_code=\nWARNING: SearchKey.COUNTRY and country_code parameter were passed simultaniously. Parameter country_code will be ignored
 empty_search_key=Search key {} is empty. Please fill values or remove this search key
 single_constant_search_key=\nWARNING: Constant value detected for the {} search key in the X dataframe: {}.\nThat search key will add constant features for different y values.\nPlease add extra search keys with non constant values, like the COUNTRY, POSTAL_CODE, DATE, PHONE NUMBER, EMAIL/HEM or IPv4
+unsupported_multi_key=Search key {} cannot be used multiple times
 unsupported_index_column=\nWARNING: Your column with name `index` was dropped because it's reserved name is booked for system needs.
 date_string_without_format=Date column `{}` has string type, but date_format is not specified. Convert column to datetime type or pass date_format
 invalid_date_format=Failed to parse date in column `{}`. Try to pass explicit date format in date_format argument of FeaturesEnricher constructor

upgini/utils/base_search_key_detector.py

@@ -1,4 +1,4 @@
-from typing import List, Optional
+from typing import List
 
 import pandas as pd
 
@@ -10,16 +10,18 @@ class BaseSearchKeyDetector:
     def _is_search_key_by_values(self, column: pd.Series) -> bool:
         raise NotImplementedError
 
-    def _get_search_key_by_name(self, column_names: List[str]) -> Optional[str]:
-        for column_name in column_names:
-            if self._is_search_key_by_name(column_name):
-                return column_name
+    def _get_search_keys_by_name(self, column_names: List[str]) -> List[str]:
+        return [
+            column_name
+            for column_name in column_names
+            if self._is_search_key_by_name(column_name)
+        ]
 
-    def get_search_key_column(self, df: pd.DataFrame) -> Optional[str]:
-        maybe_column = self._get_search_key_by_name(df.columns.to_list())
-        if maybe_column is not None:
-            return maybe_column
-
-        for column_name in df.columns:
+    def get_search_key_columns(self, df: pd.DataFrame, existing_search_keys: List[str]) -> List[str]:
+        other_columns = [col for col in df.columns if col not in existing_search_keys]
+        columns_by_names = self._get_search_keys_by_name(other_columns)
+        columns_by_values = []
+        for column_name in other_columns:
             if self._is_search_key_by_values(df[column_name]):
-                return column_name
+                columns_by_values.append(column_name)
+        return list(set(columns_by_names + columns_by_values))

upgini/utils/deduplicate_utils.py

@@ -3,7 +3,15 @@ from typing import Dict, List, Optional, Union
 
 import pandas as pd
 
-from upgini.metadata import EVAL_SET_INDEX, SORT_ID, SYSTEM_RECORD_ID, TARGET, ModelTaskType, SearchKey
+from upgini.metadata import (
+    ENTITY_SYSTEM_RECORD_ID,
+    EVAL_SET_INDEX,
+    SORT_ID,
+    SYSTEM_RECORD_ID,
+    TARGET,
+    ModelTaskType,
+    SearchKey,
+)
 from upgini.resource_bundle import ResourceBundle
 from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
 from upgini.utils.target_utils import define_task
@@ -143,6 +151,8 @@ def clean_full_duplicates(
     unique_columns = df.columns.tolist()
     if SYSTEM_RECORD_ID in unique_columns:
         unique_columns.remove(SYSTEM_RECORD_ID)
+    if ENTITY_SYSTEM_RECORD_ID in unique_columns:
+        unique_columns.remove(ENTITY_SYSTEM_RECORD_ID)
     if SORT_ID in unique_columns:
         unique_columns.remove(SORT_ID)
     if EVAL_SET_INDEX in unique_columns:

upgini/utils/email_utils.py

@@ -38,11 +38,13 @@ class EmailSearchKeyConverter:
         email_column: str,
         hem_column: Optional[str],
         search_keys: Dict[str, SearchKey],
+        unnest_search_keys: Optional[List[str]] = None,
         logger: Optional[logging.Logger] = None,
     ):
         self.email_column = email_column
         self.hem_column = hem_column
         self.search_keys = search_keys
+        self.unnest_search_keys = unnest_search_keys
         if logger is not None:
             self.logger = logger
         else:
@@ -80,9 +82,12 @@ class EmailSearchKeyConverter:
                 del self.search_keys[self.email_column]
                 return df
             self.search_keys[self.HEM_COLUMN_NAME] = SearchKey.HEM
+            self.unnest_search_keys.append(self.HEM_COLUMN_NAME)
             self.email_converted_to_hem = True
 
         del self.search_keys[self.email_column]
+        if self.email_column in self.unnest_search_keys:
+            self.unnest_search_keys.remove(self.email_column)
 
         df[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = df[self.email_column].apply(self._email_to_one_domain)
 

{upgini-1.1.299a3511.dev10.dist-info → upgini-1.1.300.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.1.299a3511.dev10
+Version: 1.1.300
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -26,8 +26,6 @@ Requires-Python: <3.11,>=3.8
 Requires-Dist: catboost>=1.0.3
 Requires-Dist: fastparquet>=0.8.1
 Requires-Dist: ipywidgets>=8.1.0
-Requires-Dist: jarowinkler>=2.0.0
-Requires-Dist: levenshtein>=0.25.1
 Requires-Dist: lightgbm>=3.3.2
 Requires-Dist: numpy>=1.19.0
 Requires-Dist: pandas<3.0.0,>=1.1.0
@@ -133,7 +131,7 @@ Description-Content-Type: text/markdown
 |Consumer Confidence index| 44 |22|-|Monthly|date, country|No
 |World economic indicators|191 |41|-|Monthly|date, country|No
 |Markets data|-|17|-|Monthly|date, datetime|No
-|World mobile & fixed broadband network coverage and perfomance |167|-|3|Monthly|country, postal/ZIP code|No
+|World mobile & fixed broadband network coverage and performance |167|-|3|Monthly|country, postal/ZIP code|No
 |World demographic data |90|-|2|Annual|country, postal/ZIP code|No
 |World house prices |44|-|3|Annual|country, postal/ZIP code|No
 |Public social media profile data |104|-|-|Monthly|date, email/HEM, phone |Yes
@@ -842,4 +840,4 @@ Some convenient ways to start contributing are:
 - [More perks for registered users](https://profile.upgini.com)
 
 <sup>😔 Found mistype or a bug in code snippet? Our bad! <a href="https://github.com/upgini/upgini/issues/new?assignees=&title=readme%2Fbug">
-Please report it here</a></sup>
+Please report it here</a></sup>

{upgini-1.1.299a3511.dev10.dist-info → upgini-1.1.300.dist-info}/RECORD

@@ -1,26 +1,26 @@
-upgini/__about__.py,sha256=I0j7AWy1_EKycImUvMKvXo5YCpu6evDNqtJKxoVOX5c,35
+upgini/__about__.py,sha256=IcfjUPTVJrTs-NV6tcVvLWMSPbW14GYerq2VeSrrzc0,24
 upgini/__init__.py,sha256=ObEtjFkIssl83qeKNMLpIQygfwK8TzztwiI43YTsAP0,353
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
-upgini/dataset.py,sha256=7TLVVhGtjgx_9yaiaIUK3kZSe_R9wg5dY0d4F5qCGM4,45636
+upgini/dataset.py,sha256=MOzBVsvzlHLxNfPWtMaXC_jIPeW7_gUvbSGeXnsPgNI,46158
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=HQFLw3VyEsZfAt4xFnIYOnp3fzQSHAsyHzIm0gTJpOI,177543
+upgini/features_enricher.py,sha256=nweJzEV8ZbDN5wvf5Gdf-HgcNz7711gNgmRxz4ZUopI,183112
 upgini/http.py,sha256=bp6jWl422Icy3AhHMdCcJv5NjExE45gSMmzMTPJjPuk,42600
 upgini/lazy_import.py,sha256=EwoM0msNGbSmWBhGbrLDny1DSnOlvTxCjmMKPxYlDms,610
-upgini/metadata.py,sha256=qDAIO7NLSSQp_XiXCv3U4XJTLO0KH3YuQ8lvCLYPqzs,9781
-upgini/metrics.py,sha256=DiDgdFvYu64ArlPEgjppZShK6yybWtIEbdPAhI3yO1I,30930
+upgini/metadata.py,sha256=wOFCJruDBhC4Hiiiqf8GeHZnnm6rhJy8t6fg5B0Z4TQ,10209
+upgini/metrics.py,sha256=Tu5cN8RlhOSSMWUTXRSkdl8SWBqR1N_2eJpBum9pZxc,30926
 upgini/search_task.py,sha256=LtRJ9bCPjMo1gJ-sUDKERhDwGcWKImrzwVFHjkMSQHQ,17071
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
 upgini/version_validator.py,sha256=ddSKUK_-eGJB3NgrqOMoWJU-OxQ253WsNLp8aqJkaIM,1389
 upgini/ads_management/__init__.py,sha256=qzyisOToVRP-tquAJD1PblZhNtMrOB8FiyF9JvfkvgE,50
 upgini/ads_management/ads_manager.py,sha256=igVbN2jz80Umb2BUJixmJVj-zx8unoKpecVo-R-nGdw,2648
 upgini/autofe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/autofe/all_operands.py,sha256=3LiH9iU-ArGmYpS8FHWH7yCFx40ILfvlSXJlKIa75BQ,2542
-upgini/autofe/binary.py,sha256=9W1DL2kZEmgV1P-0BEy8JYj9u_xhiDPKfeEsFQfrlkU,6860
-upgini/autofe/date.py,sha256=y_ien-z_nyPQX7FyZFcrB8dshyWLwA-8bSR9c3mCZZQ,8423
-upgini/autofe/feature.py,sha256=Rs-1_6UhObbA2UMz-_QeHjV1u2hTEY0h43HQjnA2OQo,13689
+upgini/autofe/all_operands.py,sha256=XbvgX2IU4aee9rJZ--d5MdmrfKhON_emle5-RU1qlEY,2506
+upgini/autofe/binary.py,sha256=8FXPJxN7fnC5wphO0Dp1tQCa0lFMSDGQGvBMkSIVAcE,4155
+upgini/autofe/date.py,sha256=8zYVhjl7jVS4xt-IjCgk9px2LHnACX2YlMlmDELlRTc,7943
+upgini/autofe/feature.py,sha256=ayxiF8Ip1ww_pt_BC9Pk127fAHZ_3fuluulS1EYLolk,13423
 upgini/autofe/groupby.py,sha256=4WjDzQxqpZxB79Ih4ihMMI5GDxaFqiH6ZelfV82ClT4,3091
 upgini/autofe/operand.py,sha256=MKEsl3zxpWzRDpTkE0sNJxTu62U20sWOvEKhPjUWS6s,2915
-upgini/autofe/unary.py,sha256=B4wp8oKnlJ0nUng-DRMKSiF8MHlhAFYbgmo9Nd_0ZaA,3777
+upgini/autofe/unary.py,sha256=ZWjLd-CUkNt_PpM8YuWLLipW1v_RdBlsl4JxXIVo9aM,3652
 upgini/autofe/vector.py,sha256=dLxfAstJs-gw_OQ1xxoxcM6pVzORlV0HVzdzt7cLXVQ,606
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/data_source/data_source_publisher.py,sha256=1cQZrK630VztwGGDp41ec9gqIeUtkefaqSSQEitVWiM,19581
@@ -30,22 +30,22 @@ upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 upgini/normalizer/phone_normalizer.py,sha256=EzTaahk6myRv6ZXgbyVFGY4kpo_2VlQgOrm5_lfbmNI,9996
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=1oHurL4I83P2lXIavx9vSdKM8ZqncAPXH2IZf76bD6g,26292
+upgini/resource_bundle/strings.properties,sha256=6jYqcxj06ZopXwr5YYMGXX1QiNNJNFo2SuwAR0qleRk,26358
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=7GpjYqjOp58vYcJLiX__1R5wjUlyQbxvHJ2klFnup_M,6389
 upgini/sampler/random_under_sampler.py,sha256=TIbm7ATo-bCMF-IiS5sZeDC1ad1SYg0eY_rRmg84yIQ,4024
 upgini/sampler/utils.py,sha256=PYOk3kKSnFlyxcpdtDNLBEEhTB4lO_iP7pQHqeUcmAc,20211
 upgini/utils/__init__.py,sha256=O_KgzKiJjW3g4NoqZ7lAxUpoHcBi_gze6r3ndEjCH74,842
-upgini/utils/base_search_key_detector.py,sha256=UNs2uxEcD1N_mOtkx3k6U70DCajW-QEO2vZp41GF0mU,855
+upgini/utils/base_search_key_detector.py,sha256=Inc6iGG-VXQdejWFfbekIkZk2ahC4k7CdGqzOkie6Bs,1021
 upgini/utils/blocked_time_series.py,sha256=Uqr3vp4YqNclj2-PzEYqVy763GSXHn86sbpIl1UOB4s,3382
 upgini/utils/country_utils.py,sha256=yE8oRgMpXuJxPfQm4fioY6dg6700HgVnHSk4Cv9sUyM,6511
 upgini/utils/custom_loss_utils.py,sha256=kieNZYBYZm5ZGBltF1F_jOSF4ea6C29rYuCyiDcqVNY,3857
 upgini/utils/cv_utils.py,sha256=w6FQb9nO8BWDx88EF83NpjPLarK4eR4ia0Wg0kLBJC4,3525
 upgini/utils/datetime_utils.py,sha256=Ujmu1ouwSFtG5SywQXJlmtDnGigAnIWPdE5Vx5NvgUM,10951
-upgini/utils/deduplicate_utils.py,sha256=6AbARehUCghJZ4PppFtrej2s3gFRruh41MEm6mzakHs,8607
+upgini/utils/deduplicate_utils.py,sha256=Zvs7zW4QzaERQmJNPrTVf2ZTVBkBLOycFCzyMwtXuV8,8770
 upgini/utils/display_utils.py,sha256=A2ouB5eiZ-Kyt9ykYxkLQwyoRPrdYeJymwNTiajtFXs,10990
-upgini/utils/email_utils.py,sha256=PLufTO97Pg9PPsNqB9agcM6M98MIxKUgIgNn2mVwSQ0,3520
+upgini/utils/email_utils.py,sha256=aKHa4xVBSsEsiZtFCPj_DrUaFupceYfvJeP_e8w_D5E,3813
 upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0--cOFTwJ0,1074
 upgini/utils/features_validator.py,sha256=PgKNt5dyqfErTvjtRNNUS9g7GFqHBtAtnsfA-V5UO1A,3307
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
@@ -57,7 +57,7 @@ upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,4
 upgini/utils/target_utils.py,sha256=Y96_PJ5cC-WsEbeqg20v9uqywDQobLoTb-xoP7S3o4E,7807
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.1.299a3511.dev10.dist-info/METADATA,sha256=qdPriI7TNVOSVvPHj-0S4IibaSoOojqgyq6tjDH3OjE,48231
-upgini-1.1.299a3511.dev10.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-upgini-1.1.299a3511.dev10.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.299a3511.dev10.dist-info/RECORD,,
+upgini-1.1.300.dist-info/METADATA,sha256=hLE9o5ZxxN1PnP4WR6ZUnp5yFpEb1cbVSwWEOiQDZBE,48153
+upgini-1.1.300.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.1.300.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.300.dist-info/RECORD,,

{upgini-1.1.299a3511.dev10.dist-info → upgini-1.1.300.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.24.2
+Generator: hatchling 1.25.0
 Root-Is-Purelib: true
 Tag: py3-none-any