upgini 1.1.312a1__py3-none-any.whl → 1.1.312a3__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.1.312a1"
+__version__ = "1.1.312a3"

upgini/dataset.py

@@ -302,7 +302,7 @@ class Dataset:  # (pd.DataFrame):
             key
             for search_group in self.search_keys_checked
             for key in search_group
-            if self.columns_renaming.get(key) != EmailSearchKeyConverter.EMAIL_ONE_DOMAIN_COLUMN_NAME
+            if not self.columns_renaming.get(key).endswith(EmailSearchKeyConverter.ONE_DOMAIN_SUFFIX)
         }
         ipv4_column = self.etalon_def_checked.get(FileColumnMeaningType.IP_ADDRESS.value)
         if (

upgini/features_enricher.py

@@ -91,7 +91,7 @@ from upgini.utils.display_utils import (
     prepare_and_show_report,
     show_request_quote_button,
 )
-from upgini.utils.email_utils import EmailSearchKeyConverter, EmailSearchKeyDetector
+from upgini.utils.email_utils import EmailDomainGenerator, EmailSearchKeyConverter, EmailSearchKeyDetector
 from upgini.utils.features_validator import FeaturesValidator
 from upgini.utils.format import Format
 from upgini.utils.ip_utils import IpSearchKeyConverter
@@ -1212,29 +1212,6 @@ class FeaturesEnricher(TransformerMixin):
     def _has_paid_features(self, exclude_features_sources: Optional[List[str]]) -> bool:
         return self._has_features_with_commercial_schema(CommercialSchema.PAID.value, exclude_features_sources)
 
-    def _extend_x(self, x: pd.DataFrame, is_demo_dataset: bool) -> Tuple[pd.DataFrame, Dict[str, SearchKey]]:
-        search_keys = self.search_keys.copy()
-        search_keys = self.__prepare_search_keys(x, search_keys, is_demo_dataset, is_transform=True, silent_mode=True)
-
-        extended_X = x.copy()
-        generated_features = []
-        date_column = SearchKey.find_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
-        if date_column is not None:
-            converter = DateTimeSearchKeyConverter(
-                date_column, self.date_format, self.logger, self.bundle, silent_mode=True
-            )
-            extended_X = converter.convert(extended_X, keep_time=True)
-            generated_features.extend(converter.generated_features)
-        email_column = self._get_email_column(search_keys)
-        hem_column = self._get_hem_column(search_keys)
-        if email_column:
-            converter = EmailSearchKeyConverter(email_column, hem_column, search_keys, [], self.logger)
-            extended_X = converter.convert(extended_X)
-            generated_features.extend(converter.generated_features)
-        generated_features = [f for f in generated_features if f in self.fit_generated_features]
-
-        return extended_X, search_keys
-
     def _is_input_same_as_fit(
         self,
         X: Union[pd.DataFrame, pd.Series, np.ndarray, None] = None,
@@ -1591,6 +1568,12 @@ class FeaturesEnricher(TransformerMixin):
             df = converter.convert(df, keep_time=True)
             generated_features = converter.generated_features
 
+        email_columns = SearchKey.find_all_keys(search_keys, SearchKey.EMAIL)
+        if email_columns:
+            generator = EmailDomainGenerator(email_columns)
+            df = generator.generate(df)
+            generated_features.extend(generator.generated_features)
+
         normalizer = Normalizer(self.search_keys, generated_features, self.bundle, self.logger, self.warning_counter)
         df = normalizer.normalize(df)
         columns_renaming = normalizer.columns_renaming
@@ -1607,13 +1590,6 @@ class FeaturesEnricher(TransformerMixin):
             self.logger.info(f"Downsampling from {num_samples} to {sample_rows}")
             df = df.sample(n=sample_rows, random_state=self.random_state)
 
-        email_column = self._get_email_column(search_keys)
-        hem_column = self._get_hem_column(search_keys)
-        if email_column:
-            converter = EmailSearchKeyConverter(
-                email_column, hem_column, search_keys, columns_renaming, [], self.bundle, self.logger
-            )
-            df = converter.convert(df)
         df = self.__add_fit_system_record_id(df, search_keys, SYSTEM_RECORD_ID)
         if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
             df = df.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL)
@@ -2030,6 +2006,12 @@ class FeaturesEnricher(TransformerMixin):
                 if self.add_date_if_missing:
                     df = self._add_current_date_as_key(df, search_keys, self.logger, self.bundle)
 
+            email_columns = SearchKey.find_all_keys(search_keys, SearchKey.EMAIL)
+            if email_columns:
+                generator = EmailDomainGenerator(email_columns)
+                df = generator.generate(df)
+                generated_features.extend(generator.generated_features)
+
             normalizer = Normalizer(
                 search_keys, generated_features, self.bundle, self.logger, self.warning_counter, silent_mode
             )
@@ -2053,7 +2035,6 @@ class FeaturesEnricher(TransformerMixin):
 
             email_column = self._get_email_column(search_keys)
             hem_column = self._get_hem_column(search_keys)
-            # email_converted_to_hem = False
             if email_column:
                 converter = EmailSearchKeyConverter(
                     email_column,
@@ -2064,7 +2045,6 @@ class FeaturesEnricher(TransformerMixin):
                     self.logger,
                 )
                 df = converter.convert(df)
-                generated_features.extend(converter.generated_features)
 
             ip_column = self._get_ip_column(search_keys)
             if ip_column:
@@ -2099,7 +2079,9 @@ class FeaturesEnricher(TransformerMixin):
             for col in features_for_transform:
                 meaning_types[col] = FileColumnMeaningType.FEATURE
             features_not_to_pass = [
-                c for c in df.columns if c not in search_keys.keys() and c not in features_for_transform
+                c
+                for c in df.columns
+                if c not in search_keys.keys() and c not in features_for_transform and c != ENTITY_SYSTEM_RECORD_ID
             ]
 
             if add_fit_system_record_id:
@@ -2446,6 +2428,14 @@ class FeaturesEnricher(TransformerMixin):
             if self.add_date_if_missing:
                 df = self._add_current_date_as_key(df, self.fit_search_keys, self.logger, self.bundle)
 
+        email_columns = SearchKey.find_all_keys(self.fit_search_keys, SearchKey.EMAIL)
+        if email_columns:
+            generator = EmailDomainGenerator(
+                email_columns
+            )
+            df = generator.generate(df)
+            self.fit_generated_features.extend(generator.generated_features)
+
         # Checks that need validated date
         validate_dates_distribution(df, self.fit_search_keys, self.logger, self.bundle, self.warning_counter)
 
@@ -2488,7 +2478,6 @@ class FeaturesEnricher(TransformerMixin):
                 self.logger,
             )
             df = converter.convert(df)
-            self.fit_generated_features.extend(converter.generated_features)
 
         ip_column = self._get_ip_column(self.fit_search_keys)
         if ip_column:

upgini/utils/email_utils.py

@@ -28,10 +28,31 @@ class EmailSearchKeyDetector(BaseSearchKeyDetector):
         return is_email_count / all_count > 0.1
 
 
+class EmailDomainGenerator:
+    DOMAIN_SUFFIX = "_domain"
+
+    def __init__(self, email_columns: List[str]):
+        self.email_columns = email_columns
+        self.generated_features = []
+
+    def generate(self, df: pd.DataFrame) -> pd.DataFrame:
+        for email_col in self.email_columns:
+            domain_feature = email_col + self.DOMAIN_SUFFIX
+            df[domain_feature] = df[email_col].apply(self._email_to_domain)
+            self.generated_features.append(domain_feature)
+        return df
+
+    @staticmethod
+    def _email_to_domain(email: str) -> Optional[str]:
+        if email is not None and isinstance(email, str) and "@" in email:
+            name_and_domain = email.split("@")
+            if len(name_and_domain) == 2 and len(name_and_domain[1]) > 0:
+                return name_and_domain[1]
+
+
 class EmailSearchKeyConverter:
-    HEM_COLUMN_NAME = "hashed_email"
-    DOMAIN_COLUMN_NAME = "email_domain"
-    EMAIL_ONE_DOMAIN_COLUMN_NAME = "email_one_domain"
+    HEM_SUFFIX = "_hem"
+    ONE_DOMAIN_SUFFIX = "_one_domain"
 
     def __init__(
         self,
@@ -54,7 +75,6 @@ class EmailSearchKeyConverter:
         else:
             self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
-        self.generated_features: List[str] = []
         self.email_converted_to_hem = False
 
     @staticmethod
@@ -78,18 +98,19 @@ class EmailSearchKeyConverter:
         df = df.copy()
         original_email_column = self.columns_renaming[self.email_column]
         if self.hem_column is None:
-            df[self.HEM_COLUMN_NAME] = df[self.email_column].apply(self._email_to_hem)
-            if df[self.HEM_COLUMN_NAME].isna().all():
+            hem_name = self.email_column + self.HEM_SUFFIX
+            df[hem_name] = df[self.email_column].apply(self._email_to_hem)
+            if df[hem_name].isna().all():
                 msg = self.bundle.get("all_emails_invalid").format(self.email_column)
                 print(msg)
                 self.logger.warning(msg)
-                df = df.drop(columns=self.HEM_COLUMN_NAME)
+                df = df.drop(columns=hem_name)
                 del self.search_keys[self.email_column]
                 return df
-            self.search_keys[self.HEM_COLUMN_NAME] = SearchKey.HEM
+            self.search_keys[hem_name] = SearchKey.HEM
             if self.email_column in self.unnest_search_keys:
-                self.unnest_search_keys.append(self.HEM_COLUMN_NAME)
-            self.columns_renaming[self.HEM_COLUMN_NAME] = original_email_column  # it could be upgini_email_unnest...
+                self.unnest_search_keys.append(hem_name)
+            self.columns_renaming[hem_name] = original_email_column  # it could be upgini_email_unnest...
             self.email_converted_to_hem = True
         else:
             df[self.hem_column] = df[self.hem_column].astype("string").str.lower()
@@ -98,16 +119,13 @@ class EmailSearchKeyConverter:
         if self.email_column in self.unnest_search_keys:
             self.unnest_search_keys.remove(self.email_column)
 
-        df[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = df[self.email_column].apply(self._email_to_one_domain)
-        self.columns_renaming[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = original_email_column
-        self.search_keys[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = SearchKey.EMAIL_ONE_DOMAIN
+        one_domain_name = self.email_column + self.ONE_DOMAIN_SUFFIX
+        df[one_domain_name] = df[self.email_column].apply(self._email_to_one_domain)
+        self.columns_renaming[one_domain_name] = original_email_column
+        self.search_keys[one_domain_name] = SearchKey.EMAIL_ONE_DOMAIN
 
         if self.email_converted_to_hem:
             df = df.drop(columns=self.email_column)
             del self.columns_renaming[self.email_column]
 
-        df[self.DOMAIN_COLUMN_NAME] = df[self.EMAIL_ONE_DOMAIN_COLUMN_NAME].str[1:]
-        self.generated_features.append(self.DOMAIN_COLUMN_NAME)
-        self.columns_renaming[self.DOMAIN_COLUMN_NAME] = original_email_column
-
         return df

{upgini-1.1.312a1.dist-info → upgini-1.1.312a3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.1.312a1
+Version: 1.1.312a3
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.1.312a1.dist-info → upgini-1.1.312a3.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=bgCPs5_xce8_o9bI-etowHUS0QExeDzykUqEC39xHMs,26
+upgini/__about__.py,sha256=IaDaRN1MPzK9IEvOazFTrqDhFeyxseC5mkVDu1NRrYc,26
 upgini/__init__.py,sha256=Xs0YFVBu1KUdtZzbStGRPQtLt3YLzJnjx5nIUBlX8BE,415
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
-upgini/dataset.py,sha256=TZkG1PBSnbvUt6j12GdUwdhXWE2mxbFks2wJdNm7ioM,30758
+upgini/dataset.py,sha256=CdOE1h94E1YgStslQIPfvMp5z_ODt7QfXfxqpmYL5Xs,30758
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=RYApDsW7z_6yIBgK26rFHgU187tOWD75s75nbRRHg58,187789
+upgini/features_enricher.py,sha256=FfFlFW3BArv2rQWGCs-SXrDDDcjQTwwJxzRysZlJfq0,186961
 upgini/http.py,sha256=a4Epc9YLIJBuYk4t8E_2-QDLBtJFqKO35jn2SnYQZCg,42920
 upgini/lazy_import.py,sha256=EwoM0msNGbSmWBhGbrLDny1DSnOlvTxCjmMKPxYlDms,610
 upgini/metadata.py,sha256=YQ-1HZGyPOksP2iM50ff_pMHXLyzvpChqSfNh8Z0ke4,10833
@@ -45,7 +45,7 @@ upgini/utils/cv_utils.py,sha256=w6FQb9nO8BWDx88EF83NpjPLarK4eR4ia0Wg0kLBJC4,3525
 upgini/utils/datetime_utils.py,sha256=O-IQbWtWJs6xTAr3m9FMRHyT-fL_28vCMrrt4eqfpa0,12025
 upgini/utils/deduplicate_utils.py,sha256=Zvs7zW4QzaERQmJNPrTVf2ZTVBkBLOycFCzyMwtXuV8,8770
 upgini/utils/display_utils.py,sha256=A2ouB5eiZ-Kyt9ykYxkLQwyoRPrdYeJymwNTiajtFXs,10990
-upgini/utils/email_utils.py,sha256=Ge4oRlpYpBA5ZLXWfXrMHx-DJWsVLglzWvpQOU6iZBw,4715
+upgini/utils/email_utils.py,sha256=j0Ug1R_0AnCg1Y92zIZ4XMwvKo3G5_pcOlBN1OH_gZs,5191
 upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0--cOFTwJ0,1074
 upgini/utils/features_validator.py,sha256=PgKNt5dyqfErTvjtRNNUS9g7GFqHBtAtnsfA-V5UO1A,3307
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
@@ -57,7 +57,7 @@ upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,4
 upgini/utils/target_utils.py,sha256=Y96_PJ5cC-WsEbeqg20v9uqywDQobLoTb-xoP7S3o4E,7807
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.1.312a1.dist-info/METADATA,sha256=263n_P2QL9zSfoNG0s4OTIUYgK3j3kwEsbCBacnpOm0,48155
-upgini-1.1.312a1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.1.312a1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.312a1.dist-info/RECORD,,
+upgini-1.1.312a3.dist-info/METADATA,sha256=133zxblkCtyOblc05Qe0mfRcwYYU9qLcPbNNBjHl-mY,48155
+upgini-1.1.312a3.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.1.312a3.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.312a3.dist-info/RECORD,,