upgini 1.2.96a2__py3-none-any.whl → 1.2.96a3__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.96a2"
+__version__ = "1.2.96a3"

upgini/features_enricher.py

@@ -71,10 +71,7 @@ from upgini.search_task import SearchTask
 from upgini.spinner import Spinner
 from upgini.utils import combine_search_keys, find_numbers_with_decimal_comma
 from upgini.utils.blocked_time_series import BlockedTimeSeriesSplit
-from upgini.utils.country_utils import (
-    CountrySearchKeyConverter,
-    CountrySearchKeyDetector,
-)
+from upgini.utils.country_utils import CountrySearchKeyDetector
 from upgini.utils.custom_loss_utils import (
     get_additional_params_custom_loss,
     get_runtime_params_custom_loss,
@@ -105,11 +102,8 @@ from upgini.utils.feature_info import FeatureInfo, _round_shap_value
 from upgini.utils.features_validator import FeaturesValidator
 from upgini.utils.format import Format
 from upgini.utils.ip_utils import IpSearchKeyConverter
-from upgini.utils.phone_utils import PhoneSearchKeyConverter, PhoneSearchKeyDetector
-from upgini.utils.postal_code_utils import (
-    PostalCodeSearchKeyConverter,
-    PostalCodeSearchKeyDetector,
-)
+from upgini.utils.phone_utils import PhoneSearchKeyDetector
+from upgini.utils.postal_code_utils import PostalCodeSearchKeyDetector
 
 try:
     from upgini.utils.progress_bar import CustomProgressBar as ProgressBar
@@ -2505,21 +2499,6 @@ if response.status_code == 200:
                 )
                 df = converter.convert(df)
 
-            phone_column = self._get_phone_column(search_keys)
-            country_column = self._get_country_column(search_keys)
-            if phone_column:
-                converter = PhoneSearchKeyConverter(phone_column, country_column)
-                df = converter.convert(df)
-
-            if country_column:
-                converter = CountrySearchKeyConverter(country_column)
-                df = converter.convert(df)
-
-            postal_code = self._get_postal_column(search_keys)
-            if postal_code:
-                converter = PostalCodeSearchKeyConverter(postal_code)
-                df = converter.convert(df)
-
             meaning_types = {}
             meaning_types.update({col: FileColumnMeaningType.FEATURE for col in features_for_transform})
             meaning_types.update({col: key.value for col, key in search_keys.items()})
@@ -2914,6 +2893,7 @@ if response.status_code == 200:
             self.fit_generated_features.extend(converter.generated_features)
         else:
             self.logger.info("Input dataset hasn't date column")
+            # TODO remove when this logic will be implemented on the back
             if self.__should_add_date_column():
                 df = self._add_current_date_as_key(df, self.fit_search_keys, self.logger, self.bundle)
 
@@ -2945,6 +2925,26 @@ if response.status_code == 200:
         if normalizer.removed_features:
             self.__log_warning(self.bundle.get("dataset_date_features").format(normalizer.removed_features))
 
+        non_feature_columns = [
+            self.TARGET_NAME,
+            EVAL_SET_INDEX,
+        ] + list(self.fit_search_keys.keys())
+        if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
+            non_feature_columns.append(DateTimeSearchKeyConverter.DATETIME_COL)
+
+        features_columns = [c for c in df.columns if c not in non_feature_columns]
+
+        features_to_drop, feature_validator_warnings = FeaturesValidator(self.logger).validate(
+            df, features_columns, self.generate_features, self.fit_columns_renaming
+        )
+        if feature_validator_warnings:
+            for warning in feature_validator_warnings:
+                self.__log_warning(warning)
+        self.fit_dropped_features.update(features_to_drop)
+        df = df.drop(columns=features_to_drop)
+
+        self.fit_generated_features = [f for f in self.fit_generated_features if f not in self.fit_dropped_features]
+
         self.__adjust_cv(df)
 
         if self.id_columns is not None and self.cv is not None and self.cv.is_time_series():
@@ -2984,6 +2984,7 @@ if response.status_code == 200:
         # Convert EMAIL to HEM etc after unnesting to do it only with one column
         df = self.__convert_unnestable_keys(df, unnest_search_keys)
 
+        # refresh features columns
         non_feature_columns = [
             self.TARGET_NAME,
             EVAL_SET_INDEX,
@@ -2995,17 +2996,6 @@ if response.status_code == 200:
 
         features_columns = [c for c in df.columns if c not in non_feature_columns]
 
-        features_to_drop, feature_validator_warnings = FeaturesValidator(self.logger).validate(
-            df, features_columns, self.generate_features, self.fit_columns_renaming
-        )
-        if feature_validator_warnings:
-            for warning in feature_validator_warnings:
-                self.__log_warning(warning)
-        self.fit_dropped_features.update(features_to_drop)
-        df = df.drop(columns=features_to_drop)
-
-        self.fit_generated_features = [f for f in self.fit_generated_features if f not in self.fit_dropped_features]
-
         meaning_types = {
             **{col: key.value for col, key in self.fit_search_keys.items()},
             **{str(c): FileColumnMeaningType.FEATURE for c in df.columns if c not in non_feature_columns},
@@ -3235,20 +3225,6 @@ if response.status_code == 200:
                 self.logger,
             )
             df = converter.convert(df)
-        phone_column = self._get_phone_column(self.fit_search_keys)
-        country_column = self._get_country_column(self.fit_search_keys)
-        if phone_column:
-            converter = PhoneSearchKeyConverter(phone_column, country_column)
-            df = converter.convert(df)
-
-        if country_column:
-            converter = CountrySearchKeyConverter(country_column)
-            df = converter.convert(df)
-
-        postal_code = self._get_postal_column(self.fit_search_keys)
-        if postal_code:
-            converter = PostalCodeSearchKeyConverter(postal_code)
-            df = converter.convert(df)
 
         return df
 

upgini/normalizer/normalize_utils.py

@@ -24,8 +24,11 @@ from upgini.metadata import (
 )
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.utils import find_numbers_with_decimal_comma
+from upgini.utils.country_utils import CountrySearchKeyConverter
 from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
+from upgini.utils.ip_utils import IpSearchKeyConverter
 from upgini.utils.phone_utils import PhoneSearchKeyConverter
+from upgini.utils.postal_code_utils import PostalCodeSearchKeyConverter
 
 
 class Normalizer:
@@ -65,6 +68,12 @@ class Normalizer:
 
         df = self._convert_phone_numbers(df)
 
+        df = self._convert_ip_addresses(df)
+
+        df = self._convert_postal_codes(df)
+
+        df = self._convert_countries(df)
+
         df = self.__convert_features_types(df)
 
         return df, self.search_keys, self.generated_features
@@ -191,6 +200,22 @@ class Normalizer:
             df = converter.convert(df)
         return df
 
+    def _convert_ip_addresses(self, df: pd.DataFrame) -> pd.DataFrame:
+        for ip_col in SearchKey.find_all_keys(self.search_keys, SearchKey.IP):
+            df[ip_col] = df[ip_col].apply(IpSearchKeyConverter.safe_ip_parse)
+        return df
+
+    def _convert_postal_codes(self, df: pd.DataFrame) -> pd.DataFrame:
+        for postal_code_col in SearchKey.find_all_keys(self.search_keys, SearchKey.POSTAL_CODE):
+            df = PostalCodeSearchKeyConverter(postal_code_col).convert(df)
+        return df
+
+    def _convert_countries(self, df: pd.DataFrame) -> pd.DataFrame:
+        maybe_country_col = SearchKey.find_key(self.search_keys, SearchKey.COUNTRY)
+        if maybe_country_col:
+            df = CountrySearchKeyConverter(maybe_country_col).convert(df)
+        return df
+
     def __convert_features_types(self, df: pd.DataFrame):
         # self.logger.info("Convert features to supported data types")
 

upgini/utils/ip_utils.py

@@ -79,7 +79,7 @@ class IpSearchKeyConverter:
             pass
 
     @staticmethod
-    def _safe_ip_parse(ip: Union[str, int, IPv4Address, IPv6Address]) -> Optional[_BaseAddress]:
+    def safe_ip_parse(ip: Union[str, int, IPv4Address, IPv6Address, bytes]) -> Optional[_BaseAddress]:
         try:
             return ip_address(ip)
         except ValueError:
@@ -110,7 +110,7 @@ class IpSearchKeyConverter:
         self.logger.info("Convert ip address to int")
         original_ip = self.columns_renaming[self.ip_column]
 
-        df[self.ip_column] = df[self.ip_column].apply(self._safe_ip_parse)
+        df[self.ip_column] = df[self.ip_column].apply(self.safe_ip_parse)
         if df[self.ip_column].isnull().all():
             raise ValidationError(self.bundle.get("invalid_ip").format(self.ip_column))
 

{upgini-1.2.96a2.dist-info → upgini-1.2.96a3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.96a2
+Version: 1.2.96a3
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.96a2.dist-info → upgini-1.2.96a3.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=TCqVix1Z4Eze5phtCvjrIOxc4bqjoaN5fjIL4q_OmWs,25
+upgini/__about__.py,sha256=euNfF4usca0xoZtvXArti-DB6GH8wgwXSYVRRbwzMkE,25
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=e6JDYTZ2AwC5aF-dqclKZKkiKrHo2f6cFmMQO2ZZmjM,32724
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=gqXsghM1TjtTUXtrsqDQYFEkNA6gf4zqOdCfPYKnDv8,219124
+upgini/features_enricher.py,sha256=dGB4utdlCtgILwX1Hnchg066fwwoyItPBSyzoPe64Z8,218244
 upgini/http.py,sha256=4i7fQwrwU3WzDUOWzrgR-4C8eJwj_5dBwRAR-UjUtlc,44345
 upgini/metadata.py,sha256=vsbbHyPCP3Rs8WkeDgQg99uAA_zmsbDStAT-NwDYhO4,12455
 upgini/metrics.py,sha256=UbKEsHB7XDzoyGNqDx846zbh1t65GpqdnnhViccdoKU,45615
@@ -35,7 +35,7 @@ upgini/data_source/data_source_publisher.py,sha256=ufL8qK1vg8iUKd5bLWz6hEMGiC3Je
 upgini/mdc/__init__.py,sha256=iHJlXQg6xRM1-ZOUtaPSJqw5SpQDszvxp4LyqviNLIQ,1027
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/normalizer/normalize_utils.py,sha256=g2TcDXZeJp9kAFO2sTqZ4CAsN4J1qHNgoJHZ8gtzUWo,7376
+upgini/normalizer/normalize_utils.py,sha256=BSP0vIjRPrupL1sziAudPXJ-qsO4UE9Pyhwiqa1MZV8,8484
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
 upgini/resource_bundle/strings.properties,sha256=UO6K0wwvutyOyClOnJYlFYAETzMSen6hHnj3--5AIAs,28497
@@ -59,7 +59,7 @@ upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0-
 upgini/utils/feature_info.py,sha256=b3RvAeOHSEu-ZXWTrf42Dll_3ZUBL0pw7sdk7hgUKD0,7284
 upgini/utils/features_validator.py,sha256=lEfmk4DoxZ4ooOE1HC0ZXtUb_lFKRFHIrnFULZ4_rL8,3746
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
-upgini/utils/ip_utils.py,sha256=TSQ_qDsLlVnm09X1HacpabEf_HNqSWpxBF4Sdc2xs08,6580
+upgini/utils/ip_utils.py,sha256=wmnnwVQdjX9o1cNQw6VQMk6maHhvsq6hNsZBYf9knrw,6585
 upgini/utils/mstats.py,sha256=u3gQVUtDRbyrOQK6V1UJ2Rx1QbkSNYGjXa6m3Z_dPVs,6286
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
@@ -71,7 +71,7 @@ upgini/utils/target_utils.py,sha256=i3Xt5l9ybB2_nF_ma5cfPuL3OeFTs2dY2xDI0p4Azpg,
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.96a2.dist-info/METADATA,sha256=UxJo1kNyV-BOPTrcplRgnnFraIMqzEZmML8RUR8hO-8,49530
-upgini-1.2.96a2.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.96a2.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.96a2.dist-info/RECORD,,
+upgini-1.2.96a3.dist-info/METADATA,sha256=HdJ2Ptri3J0BuSdc6tXhBZ79vpGvbKIuPx5PVLrtTeY,49530
+upgini-1.2.96a3.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.96a3.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.96a3.dist-info/RECORD,,