upgini 1.1.311__py3-none-any.whl → 1.1.312a1__py3-none-any.whl

upgini/metadata.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from enum import Enum
-from typing import Dict, List, Optional, Set
+from typing import Dict, List, Optional, Set, Union
 
 from pydantic import BaseModel
 
@@ -113,6 +113,21 @@ class SearchKey(Enum):
         if meaning_type == FileColumnMeaningType.MSISDN_RANGE_TO:
             return SearchKey.MSISDN_RANGE_TO
 
+    @staticmethod
+    def find_key(search_keys: Dict[str, SearchKey], keys: Union[SearchKey, List[SearchKey]]) -> Optional[SearchKey]:
+        if isinstance(keys, SearchKey):
+            keys = [keys]
+        for col, key_type in search_keys.items():
+            if key_type in keys:
+                return col
+        return None
+
+    @staticmethod
+    def find_all_keys(search_keys: Dict[str, SearchKey], keys: Union[SearchKey, List[SearchKey]]) -> List[SearchKey]:
+        if isinstance(keys, SearchKey):
+            keys = [keys]
+        return [col for col, key_type in search_keys.items() if key_type in keys]
+
 
 class DataType(Enum):
     INT = "INT"

upgini/normalizer/normalize_utils.py

@@ -0,0 +1,203 @@
+import hashlib
+from logging import Logger, getLogger
+from typing import Dict, List
+
+import numpy as np
+import pandas as pd
+from pandas.api.types import is_bool_dtype as is_bool
+from pandas.api.types import is_datetime64_any_dtype as is_datetime
+from pandas.api.types import (
+    is_float_dtype,
+    is_numeric_dtype,
+    is_object_dtype,
+    is_period_dtype,
+    is_string_dtype,
+)
+
+from upgini.errors import ValidationError
+from upgini.metadata import (
+    ENTITY_SYSTEM_RECORD_ID,
+    EVAL_SET_INDEX,
+    SEARCH_KEY_UNNEST,
+    SYSTEM_RECORD_ID,
+    TARGET,
+    SearchKey,
+)
+from upgini.resource_bundle import ResourceBundle, get_custom_bundle
+from upgini.utils import find_numbers_with_decimal_comma
+from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
+from upgini.utils.phone_utils import PhoneSearchKeyConverter
+from upgini.utils.warning_counter import WarningCounter
+
+
+class Normalizer:
+
+    MAX_STRING_FEATURE_LENGTH = 24573
+
+    def __init__(
+        self,
+        search_keys: Dict[str, SearchKey],
+        generated_features: List[str],
+        bundle: ResourceBundle = None,
+        logger: Logger = None,
+        warnings_counter: WarningCounter = None,
+        silent_mode=False,
+    ):
+        self.search_keys = search_keys
+        self.generated_features = generated_features
+        self.bundle = bundle or get_custom_bundle()
+        self.logger = logger or getLogger()
+        self.warnings_counter = warnings_counter or WarningCounter()
+        self.silent_mode = silent_mode
+        self.columns_renaming = {}
+
+    def normalize(self, df: pd.DataFrame) -> pd.DataFrame:
+        df = df.copy()
+        df = self._rename_columns(df)
+
+        df = self._remove_dates_from_features(df)
+
+        df = self._cut_too_long_string_values(df)
+
+        df = self._convert_bools(df)
+
+        df = self._convert_float16(df)
+
+        df = self._correct_decimal_comma(df)
+
+        df = self._convert_phone_numbers(df)
+
+        df = self.__convert_features_types(df)
+
+        return df
+
+    def _rename_columns(self, df: pd.DataFrame):
+        # logger.info("Replace restricted symbols in column names")
+        new_columns = []
+        dup_counter = 0
+        for column in df.columns:
+            if column in [
+                TARGET,
+                EVAL_SET_INDEX,
+                SYSTEM_RECORD_ID,
+                ENTITY_SYSTEM_RECORD_ID,
+                SEARCH_KEY_UNNEST,
+                DateTimeSearchKeyConverter.DATETIME_COL,
+            ] + self.generated_features:
+                self.columns_renaming[column] = column
+                new_columns.append(column)
+                continue
+
+            new_column = str(column)
+            suffix = hashlib.sha256(new_column.encode()).hexdigest()[:6]
+            if len(new_column) == 0:
+                raise ValidationError(self.bundle.get("dataset_empty_column_names"))
+            # db limit for column length
+            if len(new_column) > 250:
+                new_column = new_column[:250]
+
+            # make column name unique relative to server features
+            new_column = f"{new_column}_{suffix}"
+
+            new_column = new_column.lower()
+
+            # if column starts with non alphabetic symbol then add "a" to the beginning of string
+            if ord(new_column[0]) not in range(ord("a"), ord("z") + 1):
+                new_column = "a" + new_column
+
+            # replace unsupported characters to "_"
+            for idx, c in enumerate(new_column):
+                if ord(c) not in range(ord("a"), ord("z") + 1) and ord(c) not in range(ord("0"), ord("9") + 1):
+                    new_column = new_column[:idx] + "_" + new_column[idx + 1 :]
+
+            if new_column in new_columns:
+                new_column = f"{new_column}_{dup_counter}"
+                dup_counter += 1
+            new_columns.append(new_column)
+
+            # df.columns.values[col_idx] = new_column
+            # rename(columns={column: new_column}, inplace=True)
+
+            if new_column != column and column in self.search_keys:
+                self.search_keys[new_column] = self.search_keys[column]
+                del self.search_keys[column]
+            self.columns_renaming[new_column] = str(column)
+        df.columns = new_columns
+        return df
+
+    def _get_features(self, df: pd.DataFrame) -> List[str]:
+        system_columns = [ENTITY_SYSTEM_RECORD_ID, EVAL_SET_INDEX, SEARCH_KEY_UNNEST, SYSTEM_RECORD_ID, TARGET]
+        features = set(df.columns) - set(self.search_keys.keys()) - set(system_columns)
+        return sorted(list(features))
+
+    def _remove_dates_from_features(self, df: pd.DataFrame):
+        features = self._get_features(df)
+
+        removed_features = []
+        for f in features:
+            if is_datetime(df[f]) or is_period_dtype(df[f]):
+                removed_features.append(f)
+                df.drop(columns=f, inplace=True)
+
+        if removed_features:
+            msg = self.bundle.get("dataset_date_features").format(removed_features)
+            self.logger.warning(msg)
+            if not self.silent_mode:
+                print(msg)
+            self.warnings_counter.increment()
+
+        return df
+
+    def _cut_too_long_string_values(self, df: pd.DataFrame):
+        """Check that string values less than maximum characters for LLM"""
+        # logger.info("Validate too long string values")
+        for col in df.columns:
+            if is_string_dtype(df[col]) or is_object_dtype(df[col]):
+                max_length: int = df[col].astype("str").str.len().max()
+                if max_length > self.MAX_STRING_FEATURE_LENGTH:
+                    df[col] = df[col].astype("str").str.slice(stop=self.MAX_STRING_FEATURE_LENGTH)
+
+        return df
+
+    @staticmethod
+    def _convert_bools(df: pd.DataFrame):
+        """Convert bool columns to string"""
+        # logger.info("Converting bool to int")
+        for col in df.columns:
+            if is_bool(df[col]):
+                df[col] = df[col].astype("str")
+        return df
+
+    @staticmethod
+    def _convert_float16(df: pd.DataFrame):
+        """Convert float16 to float"""
+        # logger.info("Converting float16 to float")
+        for col in df.columns:
+            if is_float_dtype(df[col]):
+                df[col] = df[col].astype("float64")
+        return df
+
+    def _correct_decimal_comma(self, df: pd.DataFrame):
+        """Check DataSet for decimal commas and fix them"""
+        # logger.info("Correct decimal commas")
+        columns_to_fix = find_numbers_with_decimal_comma(df)
+        if len(columns_to_fix) > 0:
+            self.logger.warning(f"Convert strings with decimal comma to float: {columns_to_fix}")
+            for col in columns_to_fix:
+                df[col] = df[col].astype("string").str.replace(",", ".", regex=False).astype(np.float64)
+        return df
+
+    def _convert_phone_numbers(self, df: pd.DataFrame) -> pd.DataFrame:
+        maybe_country_col = SearchKey.find_key(self.search_keys, SearchKey.COUNTRY)
+        for phone_col in SearchKey.find_all_keys(self.search_keys, SearchKey.PHONE):
+            converter = PhoneSearchKeyConverter(phone_col, maybe_country_col)
+            df = converter.convert(df)
+        return df
+
+    def __convert_features_types(self, df: pd.DataFrame):
+        # self.logger.info("Convert features to supported data types")
+
+        for f in self._get_features(df):
+            if not is_numeric_dtype(df[f]):
+                df[f] = df[f].astype("string")
+        return df

upgini/utils/country_utils.py

@@ -4,6 +4,22 @@ from pandas.api.types import is_object_dtype, is_string_dtype
 from upgini.utils.base_search_key_detector import BaseSearchKeyDetector
 
 
+class CountrySearchKeyConverter:
+
+    def __init__(self, country_col: str):
+        self.country_col = country_col
+
+    def convert(self, df: pd.DataFrame) -> pd.DataFrame:
+        df[self.country_col] = (
+            df[self.country_col]
+            .astype("string")
+            .str.upper()
+            .str.replace(r"[^A-Z]", "", regex=True)
+            .str.replace("UK", "GB", regex=False)
+        )
+        return df
+
+
 class CountrySearchKeyDetector(BaseSearchKeyDetector):
     def _is_search_key_by_name(self, column_name: str) -> bool:
         return "country" in str(column_name).lower()

upgini/utils/datetime_utils.py

@@ -6,13 +6,10 @@ from typing import Dict, List, Optional
 import numpy as np
 import pandas as pd
 from dateutil.relativedelta import relativedelta
-from pandas.api.types import (
-    is_numeric_dtype,
-    is_period_dtype,
-)
+from pandas.api.types import is_numeric_dtype, is_period_dtype
 
 from upgini.errors import ValidationError
-from upgini.metadata import SearchKey
+from upgini.metadata import EVAL_SET_INDEX, SearchKey
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.utils.warning_counter import WarningCounter
 
@@ -36,13 +33,16 @@ DATETIME_PATTERN = r"^[\d\s\.\-:T/]+$"
 
 class DateTimeSearchKeyConverter:
     DATETIME_COL = "_date_time"
+    MIN_SUPPORTED_DATE_TS = datetime.datetime(1999, 12, 31)  # 946684800000  # 2000-01-01
 
     def __init__(
         self,
         date_column: str,
         date_format: Optional[str] = None,
         logger: Optional[logging.Logger] = None,
-        bundle: ResourceBundle = None,
+        bundle: Optional[ResourceBundle] = None,
+        warnings_counter: Optional[WarningCounter] = None,
+        silent_mode=False,
     ):
         self.date_column = date_column
         self.date_format = date_format
@@ -53,6 +53,8 @@ class DateTimeSearchKeyConverter:
             self.logger.setLevel("FATAL")
         self.generated_features: List[str] = []
         self.bundle = bundle or get_custom_bundle()
+        self.warnings_counter = warnings_counter or WarningCounter()
+        self.silent_mode = silent_mode
 
     @staticmethod
     def _int_to_opt(i: int) -> Optional[int]:
@@ -88,13 +90,13 @@ class DateTimeSearchKeyConverter:
             # 315532801000 - 2524608001000 - milliseconds
             # 315532801000000 - 2524608001000000 - microseconds
             # 315532801000000000 - 2524608001000000000 - nanoseconds
-            if df[self.date_column].apply(lambda x: 10 ** 16 < x).all():
+            if df[self.date_column].apply(lambda x: 10**16 < x).all():
                 df[self.date_column] = pd.to_datetime(df[self.date_column], unit="ns")
-            elif df[self.date_column].apply(lambda x: 10 ** 14 < x < 10 ** 16).all():
+            elif df[self.date_column].apply(lambda x: 10**14 < x < 10**16).all():
                 df[self.date_column] = pd.to_datetime(df[self.date_column], unit="us")
-            elif df[self.date_column].apply(lambda x: 10 ** 11 < x < 10 ** 14).all():
+            elif df[self.date_column].apply(lambda x: 10**11 < x < 10**14).all():
                 df[self.date_column] = pd.to_datetime(df[self.date_column], unit="ms")
-            elif df[self.date_column].apply(lambda x: 0 < x < 10 ** 11).all():
+            elif df[self.date_column].apply(lambda x: 0 < x < 10**11).all():
                 df[self.date_column] = pd.to_datetime(df[self.date_column], unit="s")
             else:
                 msg = self.bundle.get("unsupported_date_type").format(self.date_column)
@@ -104,6 +106,8 @@ class DateTimeSearchKeyConverter:
             df[self.date_column] = df[self.date_column].astype("string").apply(self.clean_date)
             df[self.date_column] = self.parse_date(df)
 
+        df = self.clean_old_dates(df)
+
         # If column with date is datetime then extract seconds of the day and minute of the hour
         # as additional features
         seconds = "datetime_seconds"
@@ -152,6 +156,19 @@ class DateTimeSearchKeyConverter:
             except ValueError:
                 raise ValidationError(self.bundle.get("invalid_date_format").format(self.date_column))
 
+    def clean_old_dates(self, df: pd.DataFrame) -> pd.DataFrame:
+        condition = df[self.date_column] <= self.MIN_SUPPORTED_DATE_TS
+        old_subset = df[condition]
+        if len(old_subset) > 0:
+            df.loc[condition, self.date_column] = None
+            self.logger.info(f"Set to None: {len(old_subset)} of {len(df)} rows because they are before 2000-01-01")
+            msg = self.bundle.get("dataset_drop_old_dates")
+            self.logger.warning(msg)
+            if not self.silent_mode:
+                print(msg)
+            self.warnings_counter.increment()
+        return df
+
 
 def is_time_series(df: pd.DataFrame, date_col: str) -> bool:
     try:
@@ -238,16 +255,18 @@ def is_blocked_time_series(df: pd.DataFrame, date_col: str, search_keys: List[st
 
 
 def validate_dates_distribution(
-    X: pd.DataFrame,
+    df: pd.DataFrame,
     search_keys: Dict[str, SearchKey],
     logger: Optional[logging.Logger] = None,
     bundle: Optional[ResourceBundle] = None,
     warning_counter: Optional[WarningCounter] = None,
 ):
-    maybe_date_col = None
-    for key, key_type in search_keys.items():
-        if key_type in [SearchKey.DATE, SearchKey.DATETIME]:
-            maybe_date_col = key
+    maybe_date_col = SearchKey.find_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
+
+    if EVAL_SET_INDEX in df.columns:
+        X = df.query(f"{EVAL_SET_INDEX} == 0")
+    else:
+        X = df
 
     if maybe_date_col is None:
         for col in X.columns:

upgini/utils/email_utils.py

@@ -7,7 +7,7 @@ import pandas as pd
 from pandas.api.types import is_object_dtype, is_string_dtype
 
 from upgini.metadata import SearchKey
-from upgini.resource_bundle import bundle
+from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.utils.base_search_key_detector import BaseSearchKeyDetector
 
 EMAIL_REGEX = re.compile(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$")
@@ -38,13 +38,17 @@ class EmailSearchKeyConverter:
         email_column: str,
         hem_column: Optional[str],
         search_keys: Dict[str, SearchKey],
+        columns_renaming: Dict[str, str],
         unnest_search_keys: Optional[List[str]] = None,
+        bundle: Optional[ResourceBundle] = None,
         logger: Optional[logging.Logger] = None,
     ):
         self.email_column = email_column
         self.hem_column = hem_column
         self.search_keys = search_keys
+        self.columns_renaming = columns_renaming
         self.unnest_search_keys = unnest_search_keys
+        self.bundle = bundle or get_custom_bundle()
         if logger is not None:
             self.logger = logger
         else:
@@ -61,7 +65,7 @@ class EmailSearchKeyConverter:
         if not EMAIL_REGEX.fullmatch(email):
             return None
 
-        return sha256(email.lower().encode("utf-8")).hexdigest()
+        return sha256(email.lower().encode("utf-8")).hexdigest().lower()
 
     @staticmethod
     def _email_to_one_domain(email: str) -> Optional[str]:
@@ -72,28 +76,38 @@ class EmailSearchKeyConverter:
 
     def convert(self, df: pd.DataFrame) -> pd.DataFrame:
         df = df.copy()
+        original_email_column = self.columns_renaming[self.email_column]
         if self.hem_column is None:
             df[self.HEM_COLUMN_NAME] = df[self.email_column].apply(self._email_to_hem)
             if df[self.HEM_COLUMN_NAME].isna().all():
-                msg = bundle.get("all_emails_invalid").format(self.email_column)
+                msg = self.bundle.get("all_emails_invalid").format(self.email_column)
                 print(msg)
                 self.logger.warning(msg)
                 df = df.drop(columns=self.HEM_COLUMN_NAME)
                 del self.search_keys[self.email_column]
                 return df
             self.search_keys[self.HEM_COLUMN_NAME] = SearchKey.HEM
-            self.unnest_search_keys.append(self.HEM_COLUMN_NAME)
+            if self.email_column in self.unnest_search_keys:
+                self.unnest_search_keys.append(self.HEM_COLUMN_NAME)
+            self.columns_renaming[self.HEM_COLUMN_NAME] = original_email_column  # it could be upgini_email_unnest...
             self.email_converted_to_hem = True
+        else:
+            df[self.hem_column] = df[self.hem_column].astype("string").str.lower()
 
         del self.search_keys[self.email_column]
         if self.email_column in self.unnest_search_keys:
             self.unnest_search_keys.remove(self.email_column)
 
         df[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = df[self.email_column].apply(self._email_to_one_domain)
-
+        self.columns_renaming[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = original_email_column
         self.search_keys[self.EMAIL_ONE_DOMAIN_COLUMN_NAME] = SearchKey.EMAIL_ONE_DOMAIN
 
+        if self.email_converted_to_hem:
+            df = df.drop(columns=self.email_column)
+            del self.columns_renaming[self.email_column]
+
         df[self.DOMAIN_COLUMN_NAME] = df[self.EMAIL_ONE_DOMAIN_COLUMN_NAME].str[1:]
         self.generated_features.append(self.DOMAIN_COLUMN_NAME)
+        self.columns_renaming[self.DOMAIN_COLUMN_NAME] = original_email_column
 
         return df

upgini/utils/ip_utils.py

@@ -1,15 +1,114 @@
 import logging
-from typing import Dict, List, Optional
+from ipaddress import IPv4Address, IPv6Address, _BaseAddress, ip_address
+from typing import Dict, List, Optional, Union
 
 import pandas as pd
 from requests import get
 
+from upgini.errors import ValidationError
 from upgini.metadata import SearchKey
+from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 
 # from upgini.resource_bundle import bundle
 # from upgini.utils.track_info import get_track_metrics
 
 
+class IpSearchKeyConverter:
+    def __init__(
+        self,
+        ip_column: str,
+        search_keys: Dict[str, SearchKey],
+        columns_renaming: Dict[str, str],
+        unnest_search_keys: Optional[List[str]] = None,
+        bundle: Optional[ResourceBundle] = None,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self.ip_column = ip_column
+        self.search_keys = search_keys
+        self.columns_renaming = columns_renaming
+        self.unnest_search_keys = unnest_search_keys
+        self.bundle = bundle or get_custom_bundle()
+        if logger is not None:
+            self.logger = logger
+        else:
+            self.logger = logging.getLogger()
+            self.logger.setLevel("FATAL")
+
+    @staticmethod
+    def _ip_to_int(ip: Optional[_BaseAddress]) -> Optional[int]:
+        try:
+            if isinstance(ip, (IPv4Address, IPv6Address)):
+                return int(ip)
+        except Exception:
+            pass
+
+    @staticmethod
+    def _ip_to_int_str(ip: Optional[_BaseAddress]) -> Optional[str]:
+        try:
+            if isinstance(ip, (IPv4Address, IPv6Address)):
+                return str(int(ip))
+        except Exception:
+            pass
+
+    @staticmethod
+    def _safe_ip_parse(ip: Union[str, int, IPv4Address, IPv6Address]) -> Optional[_BaseAddress]:
+        try:
+            return ip_address(ip)
+        except ValueError:
+            pass
+
+    @staticmethod
+    def _is_ipv4(ip: Optional[_BaseAddress]):
+        return ip is not None and (
+            isinstance(ip, IPv4Address) or (isinstance(ip, IPv6Address) and ip.ipv4_mapped is not None)
+        )
+
+    @staticmethod
+    def _to_ipv4(ip: Optional[_BaseAddress]) -> Optional[IPv4Address]:
+        if isinstance(ip, IPv4Address):
+            return ip
+        return None
+
+    @staticmethod
+    def _to_ipv6(ip: Optional[_BaseAddress]) -> Optional[IPv6Address]:
+        if isinstance(ip, IPv6Address):
+            return ip
+        if isinstance(ip, IPv4Address):
+            return IPv6Address("::ffff:" + str(ip))
+        return None
+
+    def convert(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Convert ip address to int"""
+        self.logger.info("Convert ip address to int")
+        original_ip = self.columns_renaming[self.ip_column]
+
+        df[self.ip_column] = df[self.ip_column].apply(self._safe_ip_parse)
+        if df[self.ip_column].isnull().all():
+            raise ValidationError(self.bundle.get("invalid_ip").format(self.ip_column))
+
+        # legacy support
+        ipv4 = self.ip_column + "_v4"
+        df[ipv4] = df[self.ip_column].apply(self._to_ipv4).apply(self._ip_to_int).astype("Int64")
+        self.search_keys[ipv4] = SearchKey.IP
+        self.columns_renaming[ipv4] = original_ip
+
+        ipv6 = self.ip_column + "_v6"
+        df[ipv6] = (
+            df[self.ip_column]
+            .apply(self._to_ipv6)
+            .apply(self._ip_to_int_str)
+            .astype("string")
+            # .str.replace(".0", "", regex=False)
+        )
+        df = df.drop(columns=self.ip_column)
+        del self.search_keys[self.ip_column]
+        del self.columns_renaming[self.ip_column]
+        self.search_keys[ipv6] = SearchKey.IPV6_ADDRESS
+        self.columns_renaming[ipv6] = original_ip  # could be __unnest_ip...
+
+        return df
+
+
 class IpToCountrySearchKeyConverter:
     url = "http://ip-api.com/json/{}"