upgini 1.1.311__py3-none-any.whl → 1.1.312a1__py3-none-any.whl

upgini/utils/phone_utils.py

@@ -1,5 +1,14 @@
+from typing import Optional
+
 import pandas as pd
+from pandas.api.types import (
+    is_float_dtype,
+    is_int64_dtype,
+    is_object_dtype,
+    is_string_dtype,
+)
 
+from upgini.errors import ValidationError
 from upgini.utils.base_search_key_detector import BaseSearchKeyDetector
 
 
@@ -9,3 +18,339 @@ class PhoneSearchKeyDetector(BaseSearchKeyDetector):
 
     def _is_search_key_by_values(self, column: pd.Series) -> bool:
         return False
+
+
+class PhoneSearchKeyConverter:
+
+    def __init__(self, phone_column: str, country_column: Optional[str] = None):
+        self.phone_column = phone_column
+        self.country_column = country_column
+
+    def convert(self, df: pd.DataFrame) -> pd.DataFrame:
+        df = self.phone_to_int(df)
+        if self.country_column is not None:
+            df = df.apply(self.add_prefix, axis=1)
+        df[self.phone_column] = df[self.phone_column].astype("Int64")
+        return df
+
+    def add_prefix(self, row):
+        phone = row[self.phone_column]
+        if pd.isna(phone):
+            return row
+        country = row[self.country_column]
+        country_prefix_tuple = self.COUNTRIES_PREFIXES.get(country)
+        if country_prefix_tuple is not None:
+            country_prefix, number_of_digits = country_prefix_tuple
+            if len(str(phone)) == number_of_digits:
+                row[self.phone_column] = int(country_prefix + str(phone))
+        return row
+
+    def phone_to_int(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Convention: phone number is always presented as int number.
+        phone_number = Country code + National Destination Code + Subscriber Number.
+        Examples:
+        41793834315     for Switzerland
+        46767040672     for Sweden
+        861065529988    for China
+        18143008198     for the USA
+        Inplace conversion of phone to int.
+
+        Method will remove all non numeric chars from string and convert it to int.
+        None will be set for phone numbers that couldn"t be converted to int
+        """
+        if is_string_dtype(df[self.phone_column]) or is_object_dtype(df[self.phone_column]):
+            convert_func = self.phone_str_to_int_safe
+        elif is_float_dtype(df[self.phone_column]):
+            convert_func = self.phone_float_to_int_safe
+        elif is_int64_dtype(df[self.phone_column]):
+            convert_func = self.phone_int_to_int_safe
+        else:
+            raise ValidationError(
+                f"phone_column_name {self.phone_column} doesn't have supported dtype. "
+                f"Dataset dtypes: {df.dtypes}. "
+                f"Contact developer and request to implement conversion of {self.phone_column} to int"
+            )
+        df[self.phone_column] = df[self.phone_column].apply(convert_func).astype("Int64")
+        return df
+
+    @staticmethod
+    def phone_float_to_int_safe(value: float) -> Optional[int]:
+        try:
+            return PhoneSearchKeyConverter.validate_length(int(value))
+        except Exception:
+            return None
+
+    @staticmethod
+    def phone_int_to_int_safe(value: int) -> Optional[int]:
+        try:
+            return PhoneSearchKeyConverter.validate_length(int(value))
+        except Exception:
+            return None
+
+    @staticmethod
+    def phone_str_to_int_safe(value: str) -> Optional[int]:
+        try:
+            value = str(value)
+            if value.endswith(".0"):
+                value = value[: len(value) - 2]
+            numeric_filter = filter(str.isdigit, value)
+            numeric_string = "".join(numeric_filter)
+            return PhoneSearchKeyConverter.validate_length(int(numeric_string))
+        except Exception:
+            return None
+
+    @staticmethod
+    def validate_length(value: int) -> Optional[int]:
+        if value < 10000000 or value > 999999999999999:
+            return None
+        else:
+            return value
+
+    COUNTRIES_PREFIXES = {
+        "US": ("1", 10),
+        "CA": ("1", 10),
+        "AI": ("1", 10),
+        "AG": ("1", 10),
+        "AS": ("1", 10),
+        "BB": ("1", 10),
+        "BS": ("1", 10),
+        "VG": ("1", 10),
+        "VI": ("1", 10),
+        "KY": ("1", 10),
+        "BM": ("1", 10),
+        "GD": ("1", 10),
+        "TC": ("1", 10),
+        "MS": ("1", 10),
+        "MP": ("1", 10),
+        "GU": ("1", 10),
+        "SX": ("1", 10),
+        "LC": ("1", 10),
+        "DM": ("1", 10),
+        "VC": ("1", 10),
+        "PR": ("1", 10),
+        "TT": ("1", 10),
+        "KN": ("1", 10),
+        "JM": ("1", 10),
+        "EG": ("20", 9),
+        "SS": ("211", 9),
+        "MA": ("212", 9),
+        "EH": ("212", 4),
+        "DZ": ("213", 8),
+        "TN": ("216", 8),
+        "LY": ("218", 9),
+        "GM": ("220", 6),
+        "SN": ("221", 9),
+        "MR": ("222", 7),
+        "ML": ("223", 8),
+        "GN": ("224", 9),
+        "CI": ("225", 7),
+        "BF": ("226", 8),
+        "NE": ("227", 8),
+        "TG": ("228", 8),
+        "BJ": ("229", 8),
+        "MU": ("230", 7),
+        "LR": ("231", 9),
+        "SL": ("232", 8),
+        "GH": ("233", 9),
+        "NG": ("234", 9),
+        "TD": ("235", 8),
+        "CF": ("236", 7),
+        "CM": ("237", 9),
+        "CV": ("238", 7),
+        "ST": ("239", 7),
+        "GQ": ("240", 9),
+        "GA": ("241", 8),
+        "CG": ("242", 7),
+        "CD": ("243", 9),
+        "AO": ("244", 9),
+        "GW": ("245", 6),
+        "IO": ("246", 7),
+        "AC": ("247", 5),
+        "SC": ("248", 7),
+        "SD": ("249", 9),
+        "RW": ("250", 9),
+        "ET": ("251", 9),
+        "SO": ("252", 9),
+        "DJ": ("253", 8),
+        "KE": ("254", 9),
+        "TZ": ("255", 9),
+        "UG": ("256", 9),
+        "BI": ("257", 8),
+        "MZ": ("258", 8),
+        "ZM": ("260", 9),
+        "MG": ("261", 9),
+        "RE": ("262", 9),
+        "YT": ("262", 9),
+        "TF": ("262", 9),
+        "ZW": ("263", 9),
+        "NA": ("264", 9),
+        "MW": ("265", 7),
+        "LS": ("266", 8),
+        "BW": ("267", 7),
+        "SZ": ("268", 8),
+        "KM": ("269", 7),
+        "ZA": ("27", 10),
+        "SH": ("290", 5),
+        "TA": ("290", 5),
+        "ER": ("291", 7),
+        "AT": ("43", 10),
+        "AW": ("297", 7),
+        "FO": ("298", 6),
+        "GL": ("299", 6),
+        "GR": ("30", 10),
+        "BE": ("32", 8),
+        "FR": ("33", 9),
+        "ES": ("34", 9),
+        "GI": ("350", 8),
+        "PE": ("51", 8),
+        "MX": ("52", 10),
+        "CU": ("53", 8),
+        "AR": ("54", 10),
+        "BR": ("55", 10),
+        "CL": ("56", 9),
+        "CO": ("57", 8),
+        "VE": ("58", 10),
+        "PT": ("351", 9),
+        "LU": ("352", 8),
+        "IE": ("353", 8),
+        "IS": ("354", 7),
+        "AL": ("355", 8),
+        "MT": ("356", 8),
+        "CY": ("357", 8),
+        "FI": ("358", 9),
+        "BG": ("359", 8),
+        "HU": ("36", 8),
+        "LT": ("370", 8),
+        "LV": ("371", 8),
+        "EE": ("372", 7),
+        "MD": ("373", 8),
+        "AM": ("374", 8),
+        "BY": ("375", 9),
+        "AD": ("376", 6),
+        "MC": ("377", 8),
+        "SM": ("378", 9),
+        "VA": ("3906698", 5),
+        "UA": ("380", 9),
+        "RS": ("381", 9),
+        "ME": ("382", 8),
+        "HR": ("385", 8),
+        "SI": ("386", 8),
+        "BA": ("387", 8),
+        "MK": ("389", 8),
+        "MY": ("60", 9),
+        "AU": ("61", 9),
+        "CX": ("61", 9),
+        "CC": ("61", 9),
+        "ID": ("62", 9),
+        "PH": ("632", 7),
+        "NZ": ("64", 8),
+        "PN": ("64", 8),
+        "SG": ("65", 8),
+        "TH": ("66", 8),
+        "IT": ("39", 10),
+        "RO": ("40", 9),
+        "CH": ("41", 9),
+        "CZ": ("420", 9),
+        "SK": ("421", 9),
+        "GB": ("44", 10),
+        "LI": ("423", 7),
+        "GG": ("44", 10),
+        "IM": ("44", 10),
+        "JE": ("44", 10),
+        "DK": ("45", 8),
+        "SE": ("46", 8),
+        "BD": ("880", 8),
+        "TW": ("886", 9),
+        "JP": ("81", 9),
+        "KR": ("82", 9),
+        "VN": ("84", 10),
+        "KP": ("850", 8),
+        "HK": ("852", 8),
+        "MO": ("853", 8),
+        "KH": ("855", 8),
+        "LA": ("856", 8),
+        "NO": ("47", 8),
+        "SJ": ("47", 8),
+        "BV": ("47", 8),
+        "PL": ("48", 9),
+        "DE": ("49", 10),
+        "TR": ("90", 10),
+        "IN": ("91", 10),
+        "PK": ("92", 9),
+        "AF": ("93", 9),
+        "LK": ("94", 9),
+        "MM": ("95", 7),
+        "IR": ("98", 10),
+        "MV": ("960", 7),
+        "LB": ("961", 7),
+        "JO": ("962", 9),
+        "SY": ("963", 10),
+        "IQ": ("964", 10),
+        "KW": ("965", 7),
+        "SA": ("966", 9),
+        "YE": ("967", 7),
+        "OM": ("968", 8),
+        "PS": ("970", 8),
+        "AE": ("971", 8),
+        "IL": ("972", 9),
+        "BH": ("973", 8),
+        "QA": ("974", 8),
+        "BT": ("975", 7),
+        "MN": ("976", 8),
+        "NP": ("977", 8),
+        "TJ": ("992", 9),
+        "TM": ("993", 8),
+        "AZ": ("994", 9),
+        "GE": ("995", 9),
+        "KG": ("996", 9),
+        "UZ": ("998", 9),
+        "FK": ("500", 5),
+        "BZ": ("501", 7),
+        "GT": ("502", 8),
+        "SV": ("503", 8),
+        "HN": ("504", 8),
+        "NI": ("505", 8),
+        "CR": ("506", 8),
+        "PA": ("507", 7),
+        "PM": ("508", 6),
+        "HT": ("509", 8),
+        "GS": ("500", 5),
+        "MF": ("590", 9),
+        "BL": ("590", 9),
+        "GP": ("590", 9),
+        "BO": ("591", 9),
+        "GY": ("592", 9),
+        "EC": ("593", 9),
+        "GF": ("594", 9),
+        "PY": ("595", 9),
+        "MQ": ("596", 9),
+        "SR": ("597", 9),
+        "UY": ("598", 9),
+        "CW": ("599", 9),
+        "BQ": ("599", 9),
+        "RU": ("7", 10),
+        "KZ": ("7", 10),
+        "TL": ("670", 7),
+        "NF": ("672", 7),
+        "HM": ("672", 7),
+        "BN": ("673", 7),
+        "NR": ("674", 7),
+        "PG": ("675", 7),
+        "TO": ("676", 7),
+        "SB": ("677", 7),
+        "VU": ("678", 7),
+        "FJ": ("679", 7),
+        "PW": ("680", 7),
+        "WF": ("681", 7),
+        "CK": ("682", 5),
+        "NU": ("683", 7),
+        "WS": ("685", 7),
+        "KI": ("686", 7),
+        "NC": ("687", 7),
+        "TV": ("688", 7),
+        "PF": ("689", 7),
+        "TK": ("690", 7),
+        "FM": ("691", 7),
+        "MH": ("692", 7),
+    }

upgini/utils/postal_code_utils.py

@@ -1,4 +1,9 @@
 import pandas as pd
+from pandas.api.types import (
+    is_float_dtype,
+    is_object_dtype,
+    is_string_dtype,
+)
 
 from upgini.utils.base_search_key_detector import BaseSearchKeyDetector
 
@@ -9,3 +14,32 @@ class PostalCodeSearchKeyDetector(BaseSearchKeyDetector):
 
     def _is_search_key_by_values(self, column: pd.Series) -> bool:
         return False
+
+
+class PostalCodeSearchKeyConverter:
+
+    def __init__(self, postal_code_column: str):
+        self.postal_code_column = postal_code_column
+
+    def convert(self, df: pd.DataFrame) -> pd.DataFrame:
+        if is_string_dtype(df[self.postal_code_column]) or is_object_dtype(df[self.postal_code_column]):
+            try:
+                df[self.postal_code_column] = (
+                    df[self.postal_code_column].astype("string").astype("Float64").astype("Int64").astype("string")
+                )
+            except Exception:
+                pass
+        elif is_float_dtype(df[self.postal_code_column]):
+            df[self.postal_code_column] = df[self.postal_code_column].astype("Int64").astype("string")
+
+        df[self.postal_code_column] = (
+            df[self.postal_code_column]
+            .astype("string")
+            .str.upper()
+            .str.replace(r"[^0-9A-Z]", "", regex=True)  # remove non alphanumeric characters
+            .str.replace(r"^0+\B", "", regex=True)  # remove leading zeros
+        )
+        # if (df[self.postal_code_column] == "").all():
+        #     raise ValidationError(self.bundle.get("invalid_postal_code").format(self.postal_code_column))
+
+        return df

{upgini-1.1.311.dist-info → upgini-1.1.312a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.1.311
+Version: 1.1.312a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.1.311.dist-info → upgini-1.1.312a1.dist-info}/RECORD

@@ -1,12 +1,12 @@
-upgini/__about__.py,sha256=smEELyNW16dHm6X3OGsBE5lKltq2roLEcEmmrhihCwA,24
+upgini/__about__.py,sha256=bgCPs5_xce8_o9bI-etowHUS0QExeDzykUqEC39xHMs,26
 upgini/__init__.py,sha256=Xs0YFVBu1KUdtZzbStGRPQtLt3YLzJnjx5nIUBlX8BE,415
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
-upgini/dataset.py,sha256=MOzBVsvzlHLxNfPWtMaXC_jIPeW7_gUvbSGeXnsPgNI,46158
+upgini/dataset.py,sha256=TZkG1PBSnbvUt6j12GdUwdhXWE2mxbFks2wJdNm7ioM,30758
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=JzSnwqxUKuYqBC4DHgPcG4MxQzvnCfKuOgihTllwRis,182583
+upgini/features_enricher.py,sha256=RYApDsW7z_6yIBgK26rFHgU187tOWD75s75nbRRHg58,187789
 upgini/http.py,sha256=a4Epc9YLIJBuYk4t8E_2-QDLBtJFqKO35jn2SnYQZCg,42920
 upgini/lazy_import.py,sha256=EwoM0msNGbSmWBhGbrLDny1DSnOlvTxCjmMKPxYlDms,610
-upgini/metadata.py,sha256=E5WWZ_MkjGyYNQh_LnwMIBHyqPx1fxk-qhEfQIJnzq8,10209
+upgini/metadata.py,sha256=YQ-1HZGyPOksP2iM50ff_pMHXLyzvpChqSfNh8Z0ke4,10833
 upgini/metrics.py,sha256=Tu5cN8RlhOSSMWUTXRSkdl8SWBqR1N_2eJpBum9pZxc,30926
 upgini/search_task.py,sha256=LtRJ9bCPjMo1gJ-sUDKERhDwGcWKImrzwVFHjkMSQHQ,17071
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
@@ -27,7 +27,7 @@ upgini/data_source/data_source_publisher.py,sha256=kTewGmdoxTVkZEqDdbhWbmIKIvb7W
 upgini/mdc/__init__.py,sha256=aM08nIWFc2gWdWUa3_IuEnNND0cQPkBGnYpRMnfFN8k,1019
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/normalizer/phone_normalizer.py,sha256=EzTaahk6myRv6ZXgbyVFGY4kpo_2VlQgOrm5_lfbmNI,9996
+upgini/normalizer/normalize_utils.py,sha256=8gH1oabPNZrC1kHSRFxGGcO0o6yNDlOJXCLzzExq-3s,7451
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
 upgini/resource_bundle/strings.properties,sha256=WZAuYPX2Dpn6BHoA3RX8uvMNMr-yJE2fF7Gz0i24x2s,26459
@@ -39,25 +39,25 @@ upgini/sampler/utils.py,sha256=PYOk3kKSnFlyxcpdtDNLBEEhTB4lO_iP7pQHqeUcmAc,20211
 upgini/utils/__init__.py,sha256=O_KgzKiJjW3g4NoqZ7lAxUpoHcBi_gze6r3ndEjCH74,842
 upgini/utils/base_search_key_detector.py,sha256=Inc6iGG-VXQdejWFfbekIkZk2ahC4k7CdGqzOkie6Bs,1021
 upgini/utils/blocked_time_series.py,sha256=Uqr3vp4YqNclj2-PzEYqVy763GSXHn86sbpIl1UOB4s,3382
-upgini/utils/country_utils.py,sha256=yE8oRgMpXuJxPfQm4fioY6dg6700HgVnHSk4Cv9sUyM,6511
+upgini/utils/country_utils.py,sha256=lY-eXWwFVegdVENFttbvLcgGDjFO17Sex8hd2PyJaRk,6937
 upgini/utils/custom_loss_utils.py,sha256=kieNZYBYZm5ZGBltF1F_jOSF4ea6C29rYuCyiDcqVNY,3857
 upgini/utils/cv_utils.py,sha256=w6FQb9nO8BWDx88EF83NpjPLarK4eR4ia0Wg0kLBJC4,3525
-upgini/utils/datetime_utils.py,sha256=uJ3wJNr4KQvDJ-gSOLcmP85hLtASK271o6mob4aZT90,11064
+upgini/utils/datetime_utils.py,sha256=O-IQbWtWJs6xTAr3m9FMRHyT-fL_28vCMrrt4eqfpa0,12025
 upgini/utils/deduplicate_utils.py,sha256=Zvs7zW4QzaERQmJNPrTVf2ZTVBkBLOycFCzyMwtXuV8,8770
 upgini/utils/display_utils.py,sha256=A2ouB5eiZ-Kyt9ykYxkLQwyoRPrdYeJymwNTiajtFXs,10990
-upgini/utils/email_utils.py,sha256=aKHa4xVBSsEsiZtFCPj_DrUaFupceYfvJeP_e8w_D5E,3813
+upgini/utils/email_utils.py,sha256=Ge4oRlpYpBA5ZLXWfXrMHx-DJWsVLglzWvpQOU6iZBw,4715
 upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0--cOFTwJ0,1074
 upgini/utils/features_validator.py,sha256=PgKNt5dyqfErTvjtRNNUS9g7GFqHBtAtnsfA-V5UO1A,3307
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
-upgini/utils/ip_utils.py,sha256=Zf3F2cnQmOCH09QLQHetpjMFu1PnD0cTmDymn0SnSy8,1672
-upgini/utils/phone_utils.py,sha256=JNSkF8G6mgsN8Czy11pamaJdsY6rBINEMpi7jbVt_RA,408
-upgini/utils/postal_code_utils.py,sha256=_8CR9tBqsPptQsmMUvnrCAmBaMIQSWH3JfJ4ly3x_zs,409
+upgini/utils/ip_utils.py,sha256=ZZj_uQFTHhagzt-MRew__ZBOp2DdnkMrachS7PElkSE,5143
+upgini/utils/phone_utils.py,sha256=c8oNajhT7Z1hXpiRAEH828vX7SoALBJKUun_M5qu9vg,10363
+upgini/utils/postal_code_utils.py,sha256=C899tJS8qM_ps4I3g-Ve6qzIa22O_UqwNmGFoyy9sO8,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
 upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,44511
 upgini/utils/target_utils.py,sha256=Y96_PJ5cC-WsEbeqg20v9uqywDQobLoTb-xoP7S3o4E,7807
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.1.311.dist-info/METADATA,sha256=P5UASl-w8f_9qvUslKiNUe7JFnL9yyR618xh5H9hZzk,48153
-upgini-1.1.311.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.1.311.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.311.dist-info/RECORD,,
+upgini-1.1.312a1.dist-info/METADATA,sha256=263n_P2QL9zSfoNG0s4OTIUYgK3j3kwEsbCBacnpOm0,48155
+upgini-1.1.312a1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.1.312a1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.312a1.dist-info/RECORD,,