upgini 1.1.312a2__tar.gz → 1.1.312a4__tar.gz

{upgini-1.1.312a2 → upgini-1.1.312a4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.1.312a2
+Version: 1.1.312a4
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.1.312a4/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.1.312a4"

{upgini-1.1.312a2 → upgini-1.1.312a4}/src/upgini/dataset.py

@@ -302,7 +302,7 @@ class Dataset:  # (pd.DataFrame):
             key
             for search_group in self.search_keys_checked
             for key in search_group
-            if self.columns_renaming.get(key) != EmailSearchKeyConverter.EMAIL_ONE_DOMAIN_COLUMN_NAME
+            if not self.columns_renaming.get(key).endswith(EmailSearchKeyConverter.ONE_DOMAIN_SUFFIX)
         }
         ipv4_column = self.etalon_def_checked.get(FileColumnMeaningType.IP_ADDRESS.value)
         if (
@@ -440,9 +440,11 @@ class Dataset:  # (pd.DataFrame):
                 FileColumnMeaningType.DATETIME,
                 # FileColumnMeaningType.IP_ADDRESS,
             }:
+                min_value = self.data[column_name].astype("Int64").min()
+                max_value = self.data[column_name].astype("Int64").max()
                 min_max_values = NumericInterval(
-                    minValue=self.data[column_name].astype("Int64").min(),
-                    maxValue=self.data[column_name].astype("Int64").max(),
+                    minValue=min_value,
+                    maxValue=max_value,
                 )
             else:
                 min_max_values = None

{upgini-1.1.312a2 → upgini-1.1.312a4}/src/upgini/features_enricher.py

@@ -91,7 +91,7 @@ from upgini.utils.display_utils import (
     prepare_and_show_report,
     show_request_quote_button,
 )
-from upgini.utils.email_utils import EmailSearchKeyConverter, EmailSearchKeyDetector
+from upgini.utils.email_utils import EmailDomainGenerator, EmailSearchKeyConverter, EmailSearchKeyDetector
 from upgini.utils.features_validator import FeaturesValidator
 from upgini.utils.format import Format
 from upgini.utils.ip_utils import IpSearchKeyConverter
@@ -1212,29 +1212,6 @@ class FeaturesEnricher(TransformerMixin):
     def _has_paid_features(self, exclude_features_sources: Optional[List[str]]) -> bool:
         return self._has_features_with_commercial_schema(CommercialSchema.PAID.value, exclude_features_sources)
 
-    def _extend_x(self, x: pd.DataFrame, is_demo_dataset: bool) -> Tuple[pd.DataFrame, Dict[str, SearchKey]]:
-        search_keys = self.search_keys.copy()
-        search_keys = self.__prepare_search_keys(x, search_keys, is_demo_dataset, is_transform=True, silent_mode=True)
-
-        extended_X = x.copy()
-        generated_features = []
-        date_column = SearchKey.find_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
-        if date_column is not None:
-            converter = DateTimeSearchKeyConverter(
-                date_column, self.date_format, self.logger, self.bundle, silent_mode=True
-            )
-            extended_X = converter.convert(extended_X, keep_time=True)
-            generated_features.extend(converter.generated_features)
-        email_column = self._get_email_column(search_keys)
-        hem_column = self._get_hem_column(search_keys)
-        if email_column:
-            converter = EmailSearchKeyConverter(email_column, hem_column, search_keys, [], self.logger)
-            extended_X = converter.convert(extended_X)
-            generated_features.extend(converter.generated_features)
-        generated_features = [f for f in generated_features if f in self.fit_generated_features]
-
-        return extended_X, search_keys
-
     def _is_input_same_as_fit(
         self,
         X: Union[pd.DataFrame, pd.Series, np.ndarray, None] = None,
@@ -1591,6 +1568,12 @@ class FeaturesEnricher(TransformerMixin):
             df = converter.convert(df, keep_time=True)
             generated_features = converter.generated_features
 
+        email_columns = SearchKey.find_all_keys(search_keys, SearchKey.EMAIL)
+        if email_columns:
+            generator = EmailDomainGenerator(email_columns)
+            df = generator.generate(df)
+            generated_features.extend(generator.generated_features)
+
         normalizer = Normalizer(self.search_keys, generated_features, self.bundle, self.logger, self.warning_counter)
         df = normalizer.normalize(df)
         columns_renaming = normalizer.columns_renaming
@@ -1607,13 +1590,6 @@ class FeaturesEnricher(TransformerMixin):
             self.logger.info(f"Downsampling from {num_samples} to {sample_rows}")
             df = df.sample(n=sample_rows, random_state=self.random_state)
 
-        email_column = self._get_email_column(search_keys)
-        hem_column = self._get_hem_column(search_keys)
-        if email_column:
-            converter = EmailSearchKeyConverter(
-                email_column, hem_column, search_keys, columns_renaming, [], self.bundle, self.logger
-            )
-            df = converter.convert(df)
         df = self.__add_fit_system_record_id(df, search_keys, SYSTEM_RECORD_ID)
         if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
             df = df.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL)
@@ -2030,6 +2006,12 @@ class FeaturesEnricher(TransformerMixin):
                 if self.add_date_if_missing:
                     df = self._add_current_date_as_key(df, search_keys, self.logger, self.bundle)
 
+            email_columns = SearchKey.find_all_keys(search_keys, SearchKey.EMAIL)
+            if email_columns:
+                generator = EmailDomainGenerator(email_columns)
+                df = generator.generate(df)
+                generated_features.extend(generator.generated_features)
+
             normalizer = Normalizer(
                 search_keys, generated_features, self.bundle, self.logger, self.warning_counter, silent_mode
             )
@@ -2053,7 +2035,6 @@ class FeaturesEnricher(TransformerMixin):
 
             email_column = self._get_email_column(search_keys)
             hem_column = self._get_hem_column(search_keys)
-            # email_converted_to_hem = False
             if email_column:
                 converter = EmailSearchKeyConverter(
                     email_column,
@@ -2064,7 +2045,6 @@ class FeaturesEnricher(TransformerMixin):
                     self.logger,
                 )
                 df = converter.convert(df)
-                generated_features.extend(converter.generated_features)
 
             ip_column = self._get_ip_column(search_keys)
             if ip_column:
@@ -2099,7 +2079,9 @@ class FeaturesEnricher(TransformerMixin):
             for col in features_for_transform:
                 meaning_types[col] = FileColumnMeaningType.FEATURE
             features_not_to_pass = [
-                c for c in df.columns if c not in search_keys.keys() and c not in features_for_transform and c != ENTITY_SYSTEM_RECORD_ID
+                c
+                for c in df.columns
+                if c not in search_keys.keys() and c not in features_for_transform and c != ENTITY_SYSTEM_RECORD_ID
             ]
 
             if add_fit_system_record_id:
@@ -2446,6 +2428,14 @@ class FeaturesEnricher(TransformerMixin):
             if self.add_date_if_missing:
                 df = self._add_current_date_as_key(df, self.fit_search_keys, self.logger, self.bundle)
 
+        email_columns = SearchKey.find_all_keys(self.fit_search_keys, SearchKey.EMAIL)
+        if email_columns:
+            generator = EmailDomainGenerator(
+                email_columns
+            )
+            df = generator.generate(df)
+            self.fit_generated_features.extend(generator.generated_features)
+
         # Checks that need validated date
         validate_dates_distribution(df, self.fit_search_keys, self.logger, self.bundle, self.warning_counter)
 
@@ -2488,7 +2478,6 @@ class FeaturesEnricher(TransformerMixin):
                 self.logger,
             )
             df = converter.convert(df)
-            self.fit_generated_features.extend(converter.generated_features)
 
         ip_column = self._get_ip_column(self.fit_search_keys)
         if ip_column:

{upgini-1.1.312a2 → upgini-1.1.312a4}/src/upgini/utils/datetime_utils.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 import re
+import pytz
 from typing import Dict, List, Optional
 
 import numpy as np
@@ -28,12 +29,13 @@ DATE_FORMATS = [
     "%Y-%m-%dT%H:%M:%S.%f",
 ]
 
-DATETIME_PATTERN = r"^[\d\s\.\-:T/]+$"
+DATETIME_PATTERN = r"^[\d\s\.\-:T/+]+$"
 
 
 class DateTimeSearchKeyConverter:
     DATETIME_COL = "_date_time"
-    MIN_SUPPORTED_DATE_TS = datetime.datetime(1999, 12, 31)  # 946684800000  # 2000-01-01
+    # MIN_SUPPORTED_DATE_TS = datetime.datetime(1999, 12, 31)  # 946684800000  # 2000-01-01
+    MIN_SUPPORTED_DATE_TS = pd.to_datetime(datetime.datetime(1999, 12, 31)).tz_localize(None)
 
     def __init__(
         self,
@@ -106,12 +108,13 @@ class DateTimeSearchKeyConverter:
             df[self.date_column] = df[self.date_column].astype("string").apply(self.clean_date)
             df[self.date_column] = self.parse_date(df)
 
-        df = self.clean_old_dates(df)
-
         # If column with date is datetime then extract seconds of the day and minute of the hour
         # as additional features
         seconds = "datetime_seconds"
         df[self.date_column] = df[self.date_column].dt.tz_localize(None)
+
+        df = self.clean_old_dates(df)
+
         df[seconds] = (df[self.date_column] - df[self.date_column].dt.floor("D")).dt.seconds
 
         seconds_without_na = df[seconds].dropna()

{upgini-1.1.312a2 → upgini-1.1.312a4}/src/upgini/utils/email_utils.py

@@ -28,10 +28,31 @@ class EmailSearchKeyDetector(BaseSearchKeyDetector):
         return is_email_count / all_count > 0.1
 
 
+class EmailDomainGenerator:
+    DOMAIN_SUFFIX = "_domain"
+
+    def __init__(self, email_columns: List[str]):
+        self.email_columns = email_columns
+        self.generated_features = []
+
+    def generate(self, df: pd.DataFrame) -> pd.DataFrame:
+        for email_col in self.email_columns:
+            domain_feature = email_col + self.DOMAIN_SUFFIX
+            df[domain_feature] = df[email_col].apply(self._email_to_domain)
+            self.generated_features.append(domain_feature)
+        return df
+
+    @staticmethod
+    def _email_to_domain(email: str) -> Optional[str]:
+        if email is not None and isinstance(email, str) and "@" in email:
+            name_and_domain = email.split("@")
+            if len(name_and_domain) == 2 and len(name_and_domain[1]) > 0:
+                return name_and_domain[1]
+
+
 class EmailSearchKeyConverter:
-    HEM_COLUMN_NAME = "hashed_email"
-    DOMAIN_COLUMN_NAME = "email_domain"
-    EMAIL_ONE_DOMAIN_COLUMN_NAME = "email_one_domain"
+    HEM_SUFFIX = "_hem"
+    ONE_DOMAIN_SUFFIX = "_one_domain"
 
     def __init__(
         self,
@@ -54,7 +75,6 @@ class EmailSearchKeyConverter:
         else:
             self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
-        self.generated_features: List[str] = []
         self.email_converted_to_hem = False
 
     @staticmethod
@@ -78,18 +98,19 @@ class EmailSearchKeyConverter:
         df = df.copy()
         original_email_column = self.columns_renaming[self.email_column]
         if self.hem_column is None:
-            df[self.HEM_COLUMN_NAME] = df[self.email_column].apply(self._email_to_hem)
-            if df[self.HEM_COLUMN_NAME].isna().all():
+            hem_name = self.email_column + self.HEM_SUFFIX
+            df[hem_name] = df[self.email_column].apply(self._email_to_hem)
+            if df[hem_name].isna().all():
                 msg = self.bundle.get("all_emails_invalid").format(self.email_column)
                 print(msg)
                 self.logger.warning(msg)
-                df = df.drop(columns=self.HEM_COLUMN_NAME)
+                df = df.drop(columns=hem_name)
                 del self.search_keys[self.email_column]
                 return df
-            self.search_keys[self.HEM_COLUMN_NAME] = SearchKey.HEM
+            self.search_keys[hem_name] = SearchKey.HEM
             if self.email_column in self.unnest_search_keys:
-                self.unnest_search_keys.append(self.HEM_COLUMN_NAME)
-            self.columns_renaming[self.HEM_COLUMN_NAME] = original_email_column  # it could be upgini_email_unnest...
+                self.unnest_search_keys.append(hem_name)
+            self.columns_renaming[hem_name] = original_email_column  # it could be upgini_email_unnest...
             self.email_converted_to_hem = True
         else:
             df[self.hem_column] = df[self.hem_column].astype("string").str.lower()
@@ -98,16 +119,13 @@ class EmailSearchKeyConverter:
         if self.email_column in self.unnest_search_keys:
             self.unnest_search_keys.remove(self.email_column)
 
-        df[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = df[self.email_column].apply(self._email_to_one_domain)
-        self.columns_renaming[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = original_email_column
-        self.search_keys[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = SearchKey.EMAIL_ONE_DOMAIN
+        one_domain_name = self.email_column + self.ONE_DOMAIN_SUFFIX
+        df[one_domain_name] = df[self.email_column].apply(self._email_to_one_domain)
+        self.columns_renaming[one_domain_name] = original_email_column
+        self.search_keys[one_domain_name] = SearchKey.EMAIL_ONE_DOMAIN
 
         if self.email_converted_to_hem:
             df = df.drop(columns=self.email_column)
             del self.columns_renaming[self.email_column]
 
-        df[self.DOMAIN_COLUMN_NAME] = df[self.EMAIL_ONE_DOMAIN_COLUMN_NAME].str[1:]
-        self.generated_features.append(self.DOMAIN_COLUMN_NAME)
-        self.columns_renaming[self.DOMAIN_COLUMN_NAME] = original_email_column
-
         return df

{upgini-1.1.312a2 → upgini-1.1.312a4}/src/upgini/utils/phone_utils.py

@@ -29,21 +29,21 @@ class PhoneSearchKeyConverter:
     def convert(self, df: pd.DataFrame) -> pd.DataFrame:
         df = self.phone_to_int(df)
         if self.country_column is not None:
-            df = df.apply(self.add_prefix, axis=1)
+            df[self.phone_column] = df.apply(self.add_prefix, axis=1)
         df[self.phone_column] = df[self.phone_column].astype("Int64")
         return df
 
     def add_prefix(self, row):
         phone = row[self.phone_column]
         if pd.isna(phone):
-            return row
+            return phone
         country = row[self.country_column]
         country_prefix_tuple = self.COUNTRIES_PREFIXES.get(country)
         if country_prefix_tuple is not None:
             country_prefix, number_of_digits = country_prefix_tuple
             if len(str(phone)) == number_of_digits:
-                row[self.phone_column] = int(country_prefix + str(phone))
-        return row
+                return int(country_prefix + str(phone))
+        return phone
 
     def phone_to_int(self, df: pd.DataFrame) -> pd.DataFrame:
         """

upgini-1.1.312a2/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.1.312a2"