upgini 1.1.315a3579.dev1__py3-none-any.whl → 1.1.316a1__py3-none-any.whl

upgini/utils/datetime_utils.py

@@ -6,13 +6,10 @@ from typing import Dict, List, Optional
 import numpy as np
 import pandas as pd
 from dateutil.relativedelta import relativedelta
-from pandas.api.types import (
-    is_numeric_dtype,
-    is_period_dtype,
-)
+from pandas.api.types import is_numeric_dtype
 
 from upgini.errors import ValidationError
-from upgini.metadata import SearchKey
+from upgini.metadata import EVAL_SET_INDEX, SearchKey
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.utils.warning_counter import WarningCounter
 
@@ -31,18 +28,22 @@ DATE_FORMATS = [
     "%Y-%m-%dT%H:%M:%S.%f",
 ]
 
-DATETIME_PATTERN = r"^[\d\s\.\-:T/]+$"
+DATETIME_PATTERN = r"^[\d\s\.\-:T/+]+$"
 
 
 class DateTimeSearchKeyConverter:
     DATETIME_COL = "_date_time"
+    # MIN_SUPPORTED_DATE_TS = datetime.datetime(1999, 12, 31)  # 946684800000  # 2000-01-01
+    MIN_SUPPORTED_DATE_TS = pd.to_datetime(datetime.datetime(1999, 12, 31)).tz_localize(None)
 
     def __init__(
         self,
         date_column: str,
         date_format: Optional[str] = None,
         logger: Optional[logging.Logger] = None,
-        bundle: ResourceBundle = None,
+        bundle: Optional[ResourceBundle] = None,
+        warnings_counter: Optional[WarningCounter] = None,
+        silent_mode=False,
     ):
         self.date_column = date_column
         self.date_format = date_format
@@ -53,6 +54,8 @@ class DateTimeSearchKeyConverter:
             self.logger.setLevel("FATAL")
         self.generated_features: List[str] = []
         self.bundle = bundle or get_custom_bundle()
+        self.warnings_counter = warnings_counter or WarningCounter()
+        self.silent_mode = silent_mode
 
     @staticmethod
     def _int_to_opt(i: int) -> Optional[int]:
@@ -81,20 +84,20 @@ class DateTimeSearchKeyConverter:
             df[self.date_column] = df[self.date_column].apply(lambda x: x.replace(tzinfo=None))
         elif isinstance(df[self.date_column].values[0], datetime.date):
             df[self.date_column] = pd.to_datetime(df[self.date_column], errors="coerce")
-        elif is_period_dtype(df[self.date_column]):
+        elif isinstance(df[self.date_column].dtype, pd.PeriodDtype):
             df[self.date_column] = df[self.date_column].dt.to_timestamp()
         elif is_numeric_dtype(df[self.date_column]):
             # 315532801 - 2524608001    - seconds
             # 315532801000 - 2524608001000 - milliseconds
             # 315532801000000 - 2524608001000000 - microseconds
             # 315532801000000000 - 2524608001000000000 - nanoseconds
-            if df[self.date_column].apply(lambda x: 10 ** 16 < x).all():
+            if df[self.date_column].apply(lambda x: 10**16 < x).all():
                 df[self.date_column] = pd.to_datetime(df[self.date_column], unit="ns")
-            elif df[self.date_column].apply(lambda x: 10 ** 14 < x < 10 ** 16).all():
+            elif df[self.date_column].apply(lambda x: 10**14 < x < 10**16).all():
                 df[self.date_column] = pd.to_datetime(df[self.date_column], unit="us")
-            elif df[self.date_column].apply(lambda x: 10 ** 11 < x < 10 ** 14).all():
+            elif df[self.date_column].apply(lambda x: 10**11 < x < 10**14).all():
                 df[self.date_column] = pd.to_datetime(df[self.date_column], unit="ms")
-            elif df[self.date_column].apply(lambda x: 0 < x < 10 ** 11).all():
+            elif df[self.date_column].apply(lambda x: 0 < x < 10**11).all():
                 df[self.date_column] = pd.to_datetime(df[self.date_column], unit="s")
             else:
                 msg = self.bundle.get("unsupported_date_type").format(self.date_column)
@@ -108,6 +111,9 @@ class DateTimeSearchKeyConverter:
         # as additional features
         seconds = "datetime_seconds"
         df[self.date_column] = df[self.date_column].dt.tz_localize(None)
+
+        df = self.clean_old_dates(df)
+
         df[seconds] = (df[self.date_column] - df[self.date_column].dt.floor("D")).dt.seconds
 
         seconds_without_na = df[seconds].dropna()
@@ -152,6 +158,19 @@ class DateTimeSearchKeyConverter:
             except ValueError:
                 raise ValidationError(self.bundle.get("invalid_date_format").format(self.date_column))
 
+    def clean_old_dates(self, df: pd.DataFrame) -> pd.DataFrame:
+        condition = df[self.date_column] <= self.MIN_SUPPORTED_DATE_TS
+        old_subset = df[condition]
+        if len(old_subset) > 0:
+            df.loc[condition, self.date_column] = None
+            self.logger.info(f"Set to None: {len(old_subset)} of {len(df)} rows because they are before 2000-01-01")
+            msg = self.bundle.get("dataset_drop_old_dates")
+            self.logger.warning(msg)
+            if not self.silent_mode:
+                print(msg)
+            self.warnings_counter.increment()
+        return df
+
 
 def is_time_series(df: pd.DataFrame, date_col: str) -> bool:
     try:
@@ -188,7 +207,7 @@ def is_time_series(df: pd.DataFrame, date_col: str) -> bool:
 def is_blocked_time_series(df: pd.DataFrame, date_col: str, search_keys: List[str]) -> bool:
     df = df.copy()
     seconds = "datetime_seconds"
-    if is_period_dtype(df[date_col]):
+    if isinstance(df[date_col].dtype, pd.PeriodDtype):
         df[date_col] = df[date_col].dt.to_timestamp()
     else:
         df[date_col] = pd.to_datetime(df[date_col])
@@ -238,23 +257,25 @@ def is_blocked_time_series(df: pd.DataFrame, date_col: str, search_keys: List[st
 
 
 def validate_dates_distribution(
-    X: pd.DataFrame,
+    df: pd.DataFrame,
     search_keys: Dict[str, SearchKey],
     logger: Optional[logging.Logger] = None,
     bundle: Optional[ResourceBundle] = None,
     warning_counter: Optional[WarningCounter] = None,
 ):
-    maybe_date_col = None
-    for key, key_type in search_keys.items():
-        if key_type in [SearchKey.DATE, SearchKey.DATETIME]:
-            maybe_date_col = key
+    maybe_date_col = SearchKey.find_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
+
+    if EVAL_SET_INDEX in df.columns:
+        X = df.query(f"{EVAL_SET_INDEX} == 0")
+    else:
+        X = df
 
     if maybe_date_col is None:
         for col in X.columns:
             if col in search_keys:
                 continue
             try:
-                if is_period_dtype(X[col]):
+                if isinstance(X[col].dtype, pd.PeriodDtype):
                     pass
                 elif pd.__version__ >= "2.0.0":
                     # Format mixed to avoid massive warnings
@@ -269,7 +290,7 @@ def validate_dates_distribution(
     if maybe_date_col is None:
         return
 
-    if is_period_dtype(X[maybe_date_col]):
+    if isinstance(X[maybe_date_col].dtype, pd.PeriodDtype):
         dates = X[maybe_date_col].dt.to_timestamp().dt.date
     elif pd.__version__ >= "2.0.0":
         dates = pd.to_datetime(X[maybe_date_col], format="mixed").dt.date

upgini/utils/email_utils.py

@@ -7,7 +7,7 @@ import pandas as pd
 from pandas.api.types import is_object_dtype, is_string_dtype
 
 from upgini.metadata import SearchKey
-from upgini.resource_bundle import bundle
+from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.utils.base_search_key_detector import BaseSearchKeyDetector
 
 EMAIL_REGEX = re.compile(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$")
@@ -28,29 +28,53 @@ class EmailSearchKeyDetector(BaseSearchKeyDetector):
         return is_email_count / all_count > 0.1
 
 
+class EmailDomainGenerator:
+    DOMAIN_SUFFIX = "_domain"
+
+    def __init__(self, email_columns: List[str]):
+        self.email_columns = email_columns
+        self.generated_features = []
+
+    def generate(self, df: pd.DataFrame) -> pd.DataFrame:
+        for email_col in self.email_columns:
+            domain_feature = email_col + self.DOMAIN_SUFFIX
+            df[domain_feature] = df[email_col].apply(self._email_to_domain)
+            self.generated_features.append(domain_feature)
+        return df
+
+    @staticmethod
+    def _email_to_domain(email: str) -> Optional[str]:
+        if email is not None and isinstance(email, str) and "@" in email:
+            name_and_domain = email.split("@")
+            if len(name_and_domain) == 2 and len(name_and_domain[1]) > 0:
+                return name_and_domain[1]
+
+
 class EmailSearchKeyConverter:
-    HEM_COLUMN_NAME = "hashed_email"
-    DOMAIN_COLUMN_NAME = "email_domain"
-    EMAIL_ONE_DOMAIN_COLUMN_NAME = "email_one_domain"
+    HEM_SUFFIX = "_hem"
+    ONE_DOMAIN_SUFFIX = "_one_domain"
 
     def __init__(
         self,
         email_column: str,
         hem_column: Optional[str],
         search_keys: Dict[str, SearchKey],
+        columns_renaming: Dict[str, str],
         unnest_search_keys: Optional[List[str]] = None,
+        bundle: Optional[ResourceBundle] = None,
         logger: Optional[logging.Logger] = None,
     ):
         self.email_column = email_column
         self.hem_column = hem_column
         self.search_keys = search_keys
+        self.columns_renaming = columns_renaming
         self.unnest_search_keys = unnest_search_keys
+        self.bundle = bundle or get_custom_bundle()
         if logger is not None:
             self.logger = logger
         else:
             self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
-        self.generated_features: List[str] = []
         self.email_converted_to_hem = False
 
     @staticmethod
@@ -61,7 +85,7 @@ class EmailSearchKeyConverter:
         if not EMAIL_REGEX.fullmatch(email):
             return None
 
-        return sha256(email.lower().encode("utf-8")).hexdigest()
+        return sha256(email.lower().encode("utf-8")).hexdigest().lower()
 
     @staticmethod
     def _email_to_one_domain(email: str) -> Optional[str]:
@@ -72,28 +96,36 @@ class EmailSearchKeyConverter:
 
     def convert(self, df: pd.DataFrame) -> pd.DataFrame:
         df = df.copy()
+        original_email_column = self.columns_renaming[self.email_column]
         if self.hem_column is None:
-            df[self.HEM_COLUMN_NAME] = df[self.email_column].apply(self._email_to_hem)
-            if df[self.HEM_COLUMN_NAME].isna().all():
-                msg = bundle.get("all_emails_invalid").format(self.email_column)
+            hem_name = self.email_column + self.HEM_SUFFIX
+            df[hem_name] = df[self.email_column].apply(self._email_to_hem)
+            if df[hem_name].isna().all():
+                msg = self.bundle.get("all_emails_invalid").format(self.email_column)
                 print(msg)
                 self.logger.warning(msg)
-                df = df.drop(columns=self.HEM_COLUMN_NAME)
+                df = df.drop(columns=hem_name)
                 del self.search_keys[self.email_column]
                 return df
-            self.search_keys[self.HEM_COLUMN_NAME] = SearchKey.HEM
-            self.unnest_search_keys.append(self.HEM_COLUMN_NAME)
+            self.search_keys[hem_name] = SearchKey.HEM
+            if self.email_column in self.unnest_search_keys:
+                self.unnest_search_keys.append(hem_name)
+            self.columns_renaming[hem_name] = original_email_column  # it could be upgini_email_unnest...
             self.email_converted_to_hem = True
+        else:
+            df[self.hem_column] = df[self.hem_column].astype("string").str.lower()
 
         del self.search_keys[self.email_column]
         if self.email_column in self.unnest_search_keys:
             self.unnest_search_keys.remove(self.email_column)
 
-        df[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = df[self.email_column].apply(self._email_to_one_domain)
-
-        self.search_keys[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = SearchKey.EMAIL_ONE_DOMAIN
+        one_domain_name = self.email_column + self.ONE_DOMAIN_SUFFIX
+        df[one_domain_name] = df[self.email_column].apply(self._email_to_one_domain)
+        self.columns_renaming[one_domain_name] = original_email_column
+        self.search_keys[one_domain_name] = SearchKey.EMAIL_ONE_DOMAIN
 
-        df[self.DOMAIN_COLUMN_NAME] = df[self.EMAIL_ONE_DOMAIN_COLUMN_NAME].str[1:]
-        self.generated_features.append(self.DOMAIN_COLUMN_NAME)
+        if self.email_converted_to_hem:
+            df = df.drop(columns=self.email_column)
+            del self.columns_renaming[self.email_column]
 
         return df

upgini/utils/ip_utils.py

@@ -1,15 +1,114 @@
 import logging
-from typing import Dict, List, Optional
+from ipaddress import IPv4Address, IPv6Address, _BaseAddress, ip_address
+from typing import Dict, List, Optional, Union
 
 import pandas as pd
 from requests import get
 
+from upgini.errors import ValidationError
 from upgini.metadata import SearchKey
+from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 
 # from upgini.resource_bundle import bundle
 # from upgini.utils.track_info import get_track_metrics
 
 
+class IpSearchKeyConverter:
+    def __init__(
+        self,
+        ip_column: str,
+        search_keys: Dict[str, SearchKey],
+        columns_renaming: Dict[str, str],
+        unnest_search_keys: Optional[List[str]] = None,
+        bundle: Optional[ResourceBundle] = None,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self.ip_column = ip_column
+        self.search_keys = search_keys
+        self.columns_renaming = columns_renaming
+        self.unnest_search_keys = unnest_search_keys
+        self.bundle = bundle or get_custom_bundle()
+        if logger is not None:
+            self.logger = logger
+        else:
+            self.logger = logging.getLogger()
+            self.logger.setLevel("FATAL")
+
+    @staticmethod
+    def _ip_to_int(ip: Optional[_BaseAddress]) -> Optional[int]:
+        try:
+            if isinstance(ip, (IPv4Address, IPv6Address)):
+                return int(ip)
+        except Exception:
+            pass
+
+    @staticmethod
+    def _ip_to_int_str(ip: Optional[_BaseAddress]) -> Optional[str]:
+        try:
+            if isinstance(ip, (IPv4Address, IPv6Address)):
+                return str(int(ip))
+        except Exception:
+            pass
+
+    @staticmethod
+    def _safe_ip_parse(ip: Union[str, int, IPv4Address, IPv6Address]) -> Optional[_BaseAddress]:
+        try:
+            return ip_address(ip)
+        except ValueError:
+            pass
+
+    @staticmethod
+    def _is_ipv4(ip: Optional[_BaseAddress]):
+        return ip is not None and (
+            isinstance(ip, IPv4Address) or (isinstance(ip, IPv6Address) and ip.ipv4_mapped is not None)
+        )
+
+    @staticmethod
+    def _to_ipv4(ip: Optional[_BaseAddress]) -> Optional[IPv4Address]:
+        if isinstance(ip, IPv4Address):
+            return ip
+        return None
+
+    @staticmethod
+    def _to_ipv6(ip: Optional[_BaseAddress]) -> Optional[IPv6Address]:
+        if isinstance(ip, IPv6Address):
+            return ip
+        if isinstance(ip, IPv4Address):
+            return IPv6Address("::ffff:" + str(ip))
+        return None
+
+    def convert(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Convert ip address to int"""
+        self.logger.info("Convert ip address to int")
+        original_ip = self.columns_renaming[self.ip_column]
+
+        df[self.ip_column] = df[self.ip_column].apply(self._safe_ip_parse)
+        if df[self.ip_column].isnull().all():
+            raise ValidationError(self.bundle.get("invalid_ip").format(self.ip_column))
+
+        # legacy support
+        ipv4 = self.ip_column + "_v4"
+        df[ipv4] = df[self.ip_column].apply(self._to_ipv4).apply(self._ip_to_int).astype("Int64")
+        self.search_keys[ipv4] = SearchKey.IP
+        self.columns_renaming[ipv4] = original_ip
+
+        ipv6 = self.ip_column + "_v6"
+        df[ipv6] = (
+            df[self.ip_column]
+            .apply(self._to_ipv6)
+            .apply(self._ip_to_int_str)
+            .astype("string")
+            # .str.replace(".0", "", regex=False)
+        )
+        df = df.drop(columns=self.ip_column)
+        del self.search_keys[self.ip_column]
+        del self.columns_renaming[self.ip_column]
+        self.search_keys[ipv6] = SearchKey.IPV6_ADDRESS
+        self.columns_renaming[ipv6] = original_ip  # could be __unnest_ip...
+
+        return df
+
+
 class IpToCountrySearchKeyConverter:
     url = "http://ip-api.com/json/{}"