upgini 1.2.49__py3-none-any.whl → 1.2.51__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.49"
+__version__ = "1.2.51"

upgini/dataset.py

@@ -37,12 +37,18 @@ from upgini.metadata import (
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.search_task import SearchTask
 from upgini.utils.email_utils import EmailSearchKeyConverter
-from upgini.utils.target_utils import balance_undersample, balance_undersample_forced, balance_undersample_time_series
+from upgini.utils.target_utils import (
+    balance_undersample,
+    balance_undersample_forced,
+    balance_undersample_time_series,
+)
 
 try:
     from upgini.utils.progress_bar import CustomProgressBar as ProgressBar
 except Exception:
-    from upgini.utils.fallback_progress_bar import CustomFallbackProgressBar as ProgressBar
+    from upgini.utils.fallback_progress_bar import (
+        CustomFallbackProgressBar as ProgressBar,
+    )
 
 
 class Dataset:  # (pd.DataFrame):
@@ -347,7 +353,8 @@ class Dataset:  # (pd.DataFrame):
             key
             for search_group in self.search_keys_checked
             for key in search_group
-            if not self.columns_renaming.get(key).endswith(EmailSearchKeyConverter.ONE_DOMAIN_SUFFIX)
+            if key in self.columns_renaming
+            and not self.columns_renaming.get(key).endswith(EmailSearchKeyConverter.ONE_DOMAIN_SUFFIX)
         }
         ipv4_column = self.etalon_def_checked.get(FileColumnMeaningType.IP_ADDRESS.value)
         if (

upgini/features_enricher.py

@@ -2270,6 +2270,7 @@ if response.status_code == 200:
                 df = converter.convert(df)
 
             ip_column = self._get_ip_column(search_keys)
+            ip_prefix_column = None
             if ip_column:
                 converter = IpSearchKeyConverter(
                     ip_column,
@@ -2280,6 +2281,7 @@ if response.status_code == 200:
                     self.logger,
                 )
                 df = converter.convert(df)
+                ip_prefix_column = converter.ip_prefix_column
 
             phone_column = self._get_phone_column(search_keys)
             country_column = self._get_country_column(search_keys)
@@ -2299,12 +2301,15 @@ if response.status_code == 200:
             # generated_features = [f for f in generated_features if f in self.fit_generated_features]
 
             meaning_types = {col: key.value for col, key in search_keys.items()}
+            if ip_prefix_column:
+                meaning_types[ip_prefix_column] = FileColumnMeaningType.IP_PREFIX
             for col in features_for_transform:
                 meaning_types[col] = FileColumnMeaningType.FEATURE
             features_not_to_pass = [
                 c
                 for c in df.columns
                 if c not in search_keys.keys()
+                and c != ip_prefix_column
                 and c not in features_for_transform
                 and c not in [ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST]
             ]
@@ -2619,6 +2624,11 @@ if response.status_code == 200:
             self.generate_features = checked_generate_features
             self.runtime_parameters.properties["generate_features"] = ",".join(self.generate_features)
 
+        if self.id_columns is not None:
+            for id_column in self.id_columns:
+                if id_column not in validated_X.columns:
+                    raise ValidationError(self.bundle.get("missing_id_column").format(id_column))
+
         validate_scoring_argument(scoring)
 
         self.__log_debug_information(
@@ -2761,6 +2771,7 @@ if response.status_code == 200:
             df = converter.convert(df)
 
         ip_column = self._get_ip_column(self.fit_search_keys)
+        ip_prefix_column = None
         if ip_column:
             converter = IpSearchKeyConverter(
                 ip_column,
@@ -2771,7 +2782,7 @@ if response.status_code == 200:
                 self.logger,
             )
             df = converter.convert(df)
-
+            ip_prefix_column = converter.ip_prefix_column
         phone_column = self._get_phone_column(self.fit_search_keys)
         country_column = self._get_country_column(self.fit_search_keys)
         if phone_column:
@@ -2787,9 +2798,13 @@ if response.status_code == 200:
             converter = PostalCodeSearchKeyConverter(postal_code)
             df = converter.convert(df)
 
-        non_feature_columns = [self.TARGET_NAME, EVAL_SET_INDEX, ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST] + list(
-            self.fit_search_keys.keys()
-        )
+        non_feature_columns = [
+            self.TARGET_NAME,
+            EVAL_SET_INDEX,
+            ENTITY_SYSTEM_RECORD_ID,
+            SEARCH_KEY_UNNEST,
+            ip_prefix_column,
+        ] + list(self.fit_search_keys.keys())
         if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
             non_feature_columns.append(DateTimeSearchKeyConverter.DATETIME_COL)
 
@@ -2810,6 +2825,8 @@ if response.status_code == 200:
             **{col: key.value for col, key in self.fit_search_keys.items()},
             **{str(c): FileColumnMeaningType.FEATURE for c in df.columns if c not in non_feature_columns},
         }
+        if ip_prefix_column:
+            meaning_types[ip_prefix_column] = FileColumnMeaningType.IP_PREFIX
         meaning_types[self.TARGET_NAME] = FileColumnMeaningType.TARGET
         meaning_types[ENTITY_SYSTEM_RECORD_ID] = FileColumnMeaningType.ENTITY_SYSTEM_RECORD_ID
         if SEARCH_KEY_UNNEST in df.columns:

upgini/metadata.py

@@ -44,6 +44,7 @@ class FileColumnMeaningType(Enum):
     ENTITY_SYSTEM_RECORD_ID = "ENTITY_SYSTEM_RECORD_ID"
     UNNEST_KEY = "UNNEST_KEY"
     IP_BINARY = "IP_BINARY"
+    IP_PREFIX = "IP_PREFIX"
     IP_RANGE_FROM_BINARY = "IP_RANGE_FROM_BINARY"
     IP_RANGE_TO_BINARY = "IP_RANGE_TO_BINARY"
 

upgini/resource_bundle/strings.properties

@@ -134,6 +134,7 @@ x_and_eval_x_diff_types=X and eval_set X has different types: {} and {}
 baseline_score_column_not_exists=baseline_score_column {} doesn't exist in input dataframe
 baseline_score_column_has_na=baseline_score_column contains NaN. Clear it and and retry
 missing_features_for_transform=Missing some features for transform that were presented on fit: {}
+missing_id_column=Id column {} not found in X
     # target validation
 empty_target=Target is empty in all rows
 # non_numeric_target=Binary target should be numerical type

upgini/utils/ip_utils.py

@@ -33,9 +33,12 @@ class IpSearchKeyConverter:
         else:
             self.logger = logging.getLogger()
             self.logger.setLevel("FATAL")
+        self.ip_prefix_column = None
 
     @staticmethod
     def _ip_to_int(ip: Optional[_BaseAddress]) -> Optional[int]:
+        if ip is None:
+            return None
         try:
             if isinstance(ip, (IPv4Address, IPv6Address)):
                 return int(ip)
@@ -44,6 +47,8 @@ class IpSearchKeyConverter:
 
     @staticmethod
     def _ip_to_binary(ip: Optional[_BaseAddress]) -> Optional[bytes]:
+        if ip is None:
+            return None
         try:
             if isinstance(ip, IPv6Address) and ip.ipv4_mapped is not None:
                 return ip.ipv4_mapped.packed
@@ -52,6 +57,20 @@ class IpSearchKeyConverter:
         except Exception:
             pass
 
+    @staticmethod
+    def _ip_to_prefix(ip: Optional[_BaseAddress]) -> Optional[str]:
+        if ip is None:
+            return None
+        try:
+            if isinstance(ip, IPv6Address):
+                if ip.ipv4_mapped is not None:
+                    return ".".join(ip.ipv4_mapped.exploded.split(".")[:2])
+                return ":".join(ip.exploded.split(":")[:2])  # TODO use 3 in future
+            else:
+                return ".".join(ip.exploded.split(".")[:2])
+        except Exception:
+            pass
+
     @staticmethod
     def _ip_to_int_str(ip: Optional[_BaseAddress]) -> Optional[str]:
         try:
@@ -102,24 +121,26 @@ class IpSearchKeyConverter:
         # self.search_keys[ipv4] = SearchKey.IP
         # self.columns_renaming[ipv4] = original_ip
 
-        ipv6 = self.ip_column + "_v6"
-        df[ipv6] = (
-            df[self.ip_column]
-            .apply(self._to_ipv6)
-            .apply(self._ip_to_int_str)
-            .astype("string")
-            # .str.replace(".0", "", regex=False)
-        )
-        # ip_binary = self.ip_column + "_binary"
-        # df[ip_binary] = df[self.ip_column].apply(self._ip_to_binary)
+        # ipv6 = self.ip_column + "_v6"
+        # df[ipv6] = (
+        #     df[self.ip_column]
+        #     .apply(self._to_ipv6)
+        #     .apply(self._ip_to_int_str)
+        #     .astype("string")
+        #     # .str.replace(".0", "", regex=False)
+        # )
+        ip_binary = self.ip_column + "_binary"
+        df[ip_binary] = df[self.ip_column].apply(self._ip_to_binary)
+        self.ip_prefix_column = self.ip_column + "_prefix"
+        df[self.ip_prefix_column] = df[self.ip_column].apply(self._ip_to_prefix)
 
         df = df.drop(columns=self.ip_column)
         del self.search_keys[self.ip_column]
         del self.columns_renaming[self.ip_column]
-        self.search_keys[ipv6] = SearchKey.IPV6_ADDRESS
-        # self.search_keys[ip_binary] = SearchKey.IP_BINARY
-        self.columns_renaming[ipv6] = original_ip
-        # self.columns_renaming[ip_binary] = original_ip
+        # self.search_keys[ipv6] = SearchKey.IPV6_ADDRESS
+        self.search_keys[ip_binary] = SearchKey.IP_BINARY
+        # self.columns_renaming[ipv6] = original_ip
+        self.columns_renaming[ip_binary] = original_ip
 
         return df
 

{upgini-1.2.49.dist-info → upgini-1.2.51.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.49
+Version: 1.2.51
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.49.dist-info → upgini-1.2.51.dist-info}/RECORD

@@ -1,12 +1,12 @@
-upgini/__about__.py,sha256=qkDVgmmc_W7WSA8UFQzKzULlTX27bnHrBTAlSeTYqYs,23
+upgini/__about__.py,sha256=kgsz9u_lLDc3N0akch6v9PpMXz_PW7_aEHXRb1pWgHg,23
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
-upgini/dataset.py,sha256=QC3jncWS3wHe4CY7pWWDMO_3HKxGbi0EyPHXMdBtoQM,33456
+upgini/dataset.py,sha256=vT4JyHmafLNbj54SySXr93f5hNS6-t94aFslbBy-7No,33535
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=8bhABqZ3aXIc_5WBVNqFnRNT_0fCNbLyPwXv7VXdygs,200350
+upgini/features_enricher.py,sha256=80h-1a-UxhuknvuEk1tQk5q5dckqlD_DHzNfufNuaPI,201110
 upgini/http.py,sha256=danPeX7nTMa_70S-pk-4UUm5yOvXYlR84jgyjoHYBkU,43367
 upgini/lazy_import.py,sha256=74gQ8JuA48BGRLxAo7lNHNKY2D2emMxrUxKGdxVGhuY,1012
-upgini/metadata.py,sha256=-ibqiNjD7dTagqg53FoEJNEqvAYbwgfyn9PGTRQ_YKU,12054
+upgini/metadata.py,sha256=zuLdt5XyO_ZH4VsUNshzRHgv6VfYiXy0M8jeohloFBw,12082
 upgini/metrics.py,sha256=hr7UwLphbZ_FEglLuO2lzr_pFgxOJ4c3WBeg7H-fNqY,35521
 upgini/search_task.py,sha256=qxUxAD-bed-FpZYmTB_4orW7YJsW_O6a1TcgnZIRFr4,17307
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
@@ -30,7 +30,7 @@ upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 upgini/normalizer/normalize_utils.py,sha256=Ft2MwSgVoBilXAORAOYAuwPD79GOLfwn4qQE3IUFzzg,7218
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=0jZC0HjyQHeqFCHt6nn1kz7vV0oq92AYQJvy-soAwe4,27304
+upgini/resource_bundle/strings.properties,sha256=0_KAExIi1u48N1CQ13LKJS3bgDlRs-MPOyU3VxcE-qY,27350
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=7GpjYqjOp58vYcJLiX__1R5wjUlyQbxvHJ2klFnup_M,6389
@@ -51,7 +51,7 @@ upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0-
 upgini/utils/feature_info.py,sha256=0rOXSyCj-sw-8migWP0ge8qrOzGU50dQvH0JUJUrDfQ,6766
 upgini/utils/features_validator.py,sha256=lEfmk4DoxZ4ooOE1HC0ZXtUb_lFKRFHIrnFULZ4_rL8,3746
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
-upgini/utils/ip_utils.py,sha256=VORRmtKlItcbBVVK5SiwXD7J-6Y5rn7UQ5m6WcBXt7E,5698
+upgini/utils/ip_utils.py,sha256=GZqBaV-nky-_Yb9KclmTrYovCG4kawYcbdjEpw1e5Mo,6500
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
@@ -59,7 +59,7 @@ upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,4
 upgini/utils/target_utils.py,sha256=RlpKGss9kMibVSlA8iZuO_qxmyeplqzn7X8g6hiGGGs,14341
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.49.dist-info/METADATA,sha256=XZ11OqCR6UqQrqo2RsKAIZYdPIVmIcn61GxRJn6f9Ys,49055
-upgini-1.2.49.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.49.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.49.dist-info/RECORD,,
+upgini-1.2.51.dist-info/METADATA,sha256=WXti81Fx4H5NawX2D7XvQ5cPEUVi4mlMkykbn94gXKI,49055
+upgini-1.2.51.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.51.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.51.dist-info/RECORD,,