upgini 1.2.96a3__py3-none-any.whl → 1.2.96a3906.dev2__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.96a3"
+__version__ = "1.2.96a3906.dev2"

upgini/autofe/unary.py

@@ -190,3 +190,11 @@ class Bin(PandasOperator):
         if isinstance(value, str):
             return json.loads(value)
         return value
+
+
+class Cluster(PandasOperator):
+    name: str = "cluster"
+    is_unary: bool = True
+    input_type: Optional[str] = "vector"
+    output_type: Optional[str] = "category"
+    is_categorical: bool = True

upgini/features_enricher.py

@@ -71,7 +71,10 @@ from upgini.search_task import SearchTask
 from upgini.spinner import Spinner
 from upgini.utils import combine_search_keys, find_numbers_with_decimal_comma
 from upgini.utils.blocked_time_series import BlockedTimeSeriesSplit
-from upgini.utils.country_utils import CountrySearchKeyDetector
+from upgini.utils.country_utils import (
+    CountrySearchKeyConverter,
+    CountrySearchKeyDetector,
+)
 from upgini.utils.custom_loss_utils import (
     get_additional_params_custom_loss,
     get_runtime_params_custom_loss,
@@ -102,8 +105,11 @@ from upgini.utils.feature_info import FeatureInfo, _round_shap_value
 from upgini.utils.features_validator import FeaturesValidator
 from upgini.utils.format import Format
 from upgini.utils.ip_utils import IpSearchKeyConverter
-from upgini.utils.phone_utils import PhoneSearchKeyDetector
-from upgini.utils.postal_code_utils import PostalCodeSearchKeyDetector
+from upgini.utils.phone_utils import PhoneSearchKeyConverter, PhoneSearchKeyDetector
+from upgini.utils.postal_code_utils import (
+    PostalCodeSearchKeyConverter,
+    PostalCodeSearchKeyDetector,
+)
 
 try:
     from upgini.utils.progress_bar import CustomProgressBar as ProgressBar
@@ -1116,7 +1122,6 @@ class FeaturesEnricher(TransformerMixin):
                     # and calculate final metric (and uplift)
                     enriched_metric = None
                     uplift = None
-                    uplift_perc = None
                     enriched_estimator = None
                     if set(fitting_X.columns) != set(fitting_enriched_X.columns):
                         self.logger.info(
@@ -1148,7 +1153,6 @@ class FeaturesEnricher(TransformerMixin):
                             self.logger.info(f"Enriched {metric} on train combined features: {enriched_metric}")
                         if baseline_metric is not None and enriched_metric is not None:
                             uplift = (enriched_cv_result.metric - baseline_cv_result.metric) * multiplier
-                            uplift_perc = uplift / abs(baseline_cv_result.metric) * 100
 
                     train_metrics = {
                         self.bundle.get("quality_metrics_segment_header"): self.bundle.get(
@@ -1175,10 +1179,7 @@ class FeaturesEnricher(TransformerMixin):
                             enriched_metric
                         )
                     if uplift is not None:
-                        train_metrics[self.bundle.get("quality_metrics_uplift_header")] = round(uplift, 3)
-                        train_metrics[self.bundle.get("quality_metrics_uplift_perc_header")] = (
-                            f"{round(uplift_perc, 1)}%"
-                        )
+                        train_metrics[self.bundle.get("quality_metrics_uplift_header")] = uplift
                     metrics = [train_metrics]
 
                     # 3 If eval_set is presented - fit final model on train enriched data and score each
@@ -1227,10 +1228,8 @@ class FeaturesEnricher(TransformerMixin):
 
                             if etalon_eval_metric is not None and enriched_eval_metric is not None:
                                 eval_uplift = (enriched_eval_results.metric - etalon_eval_results.metric) * multiplier
-                                eval_uplift_perc = eval_uplift / abs(etalon_eval_results.metric) * 100
                             else:
                                 eval_uplift = None
-                                eval_uplift_perc = None
 
                             eval_metrics = {
                                 self.bundle.get("quality_metrics_segment_header"): self.bundle.get(
@@ -1261,10 +1260,7 @@ class FeaturesEnricher(TransformerMixin):
                                     enriched_eval_metric
                                 )
                             if eval_uplift is not None:
-                                eval_metrics[self.bundle.get("quality_metrics_uplift_header")] = round(eval_uplift, 3)
-                                eval_metrics[self.bundle.get("quality_metrics_uplift_perc_header")] = (
-                                    f"{round(eval_uplift_perc, 1)}%"
-                                )
+                                eval_metrics[self.bundle.get("quality_metrics_uplift_header")] = eval_uplift
 
                             metrics.append(eval_metrics)
 
@@ -2499,6 +2495,21 @@ if response.status_code == 200:
                 )
                 df = converter.convert(df)
 
+            phone_column = self._get_phone_column(search_keys)
+            country_column = self._get_country_column(search_keys)
+            if phone_column:
+                converter = PhoneSearchKeyConverter(phone_column, country_column)
+                df = converter.convert(df)
+
+            if country_column:
+                converter = CountrySearchKeyConverter(country_column)
+                df = converter.convert(df)
+
+            postal_code = self._get_postal_column(search_keys)
+            if postal_code:
+                converter = PostalCodeSearchKeyConverter(postal_code)
+                df = converter.convert(df)
+
             meaning_types = {}
             meaning_types.update({col: FileColumnMeaningType.FEATURE for col in features_for_transform})
             meaning_types.update({col: key.value for col, key in search_keys.items()})
@@ -2893,7 +2904,6 @@ if response.status_code == 200:
             self.fit_generated_features.extend(converter.generated_features)
         else:
             self.logger.info("Input dataset hasn't date column")
-            # TODO remove when this logic will be implemented on the back
             if self.__should_add_date_column():
                 df = self._add_current_date_as_key(df, self.fit_search_keys, self.logger, self.bundle)
 
@@ -2925,26 +2935,6 @@ if response.status_code == 200:
         if normalizer.removed_features:
             self.__log_warning(self.bundle.get("dataset_date_features").format(normalizer.removed_features))
 
-        non_feature_columns = [
-            self.TARGET_NAME,
-            EVAL_SET_INDEX,
-        ] + list(self.fit_search_keys.keys())
-        if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
-            non_feature_columns.append(DateTimeSearchKeyConverter.DATETIME_COL)
-
-        features_columns = [c for c in df.columns if c not in non_feature_columns]
-
-        features_to_drop, feature_validator_warnings = FeaturesValidator(self.logger).validate(
-            df, features_columns, self.generate_features, self.fit_columns_renaming
-        )
-        if feature_validator_warnings:
-            for warning in feature_validator_warnings:
-                self.__log_warning(warning)
-        self.fit_dropped_features.update(features_to_drop)
-        df = df.drop(columns=features_to_drop)
-
-        self.fit_generated_features = [f for f in self.fit_generated_features if f not in self.fit_dropped_features]
-
         self.__adjust_cv(df)
 
         if self.id_columns is not None and self.cv is not None and self.cv.is_time_series():
@@ -2984,7 +2974,6 @@ if response.status_code == 200:
         # Convert EMAIL to HEM etc after unnesting to do it only with one column
         df = self.__convert_unnestable_keys(df, unnest_search_keys)
 
-        # refresh features columns
         non_feature_columns = [
             self.TARGET_NAME,
             EVAL_SET_INDEX,
@@ -2996,6 +2985,17 @@ if response.status_code == 200:
 
         features_columns = [c for c in df.columns if c not in non_feature_columns]
 
+        features_to_drop, feature_validator_warnings = FeaturesValidator(self.logger).validate(
+            df, features_columns, self.generate_features, self.fit_columns_renaming
+        )
+        if feature_validator_warnings:
+            for warning in feature_validator_warnings:
+                self.__log_warning(warning)
+        self.fit_dropped_features.update(features_to_drop)
+        df = df.drop(columns=features_to_drop)
+
+        self.fit_generated_features = [f for f in self.fit_generated_features if f not in self.fit_dropped_features]
+
         meaning_types = {
             **{col: key.value for col, key in self.fit_search_keys.items()},
             **{str(c): FileColumnMeaningType.FEATURE for c in df.columns if c not in non_feature_columns},
@@ -3225,6 +3225,20 @@ if response.status_code == 200:
                 self.logger,
             )
             df = converter.convert(df)
+        phone_column = self._get_phone_column(self.fit_search_keys)
+        country_column = self._get_country_column(self.fit_search_keys)
+        if phone_column:
+            converter = PhoneSearchKeyConverter(phone_column, country_column)
+            df = converter.convert(df)
+
+        if country_column:
+            converter = CountrySearchKeyConverter(country_column)
+            df = converter.convert(df)
+
+        postal_code = self._get_postal_column(self.fit_search_keys)
+        if postal_code:
+            converter = PostalCodeSearchKeyConverter(postal_code)
+            df = converter.convert(df)
 
         return df
 

upgini/normalizer/normalize_utils.py

@@ -24,11 +24,8 @@ from upgini.metadata import (
 )
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.utils import find_numbers_with_decimal_comma
-from upgini.utils.country_utils import CountrySearchKeyConverter
 from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
-from upgini.utils.ip_utils import IpSearchKeyConverter
 from upgini.utils.phone_utils import PhoneSearchKeyConverter
-from upgini.utils.postal_code_utils import PostalCodeSearchKeyConverter
 
 
 class Normalizer:
@@ -68,12 +65,6 @@ class Normalizer:
 
         df = self._convert_phone_numbers(df)
 
-        df = self._convert_ip_addresses(df)
-
-        df = self._convert_postal_codes(df)
-
-        df = self._convert_countries(df)
-
         df = self.__convert_features_types(df)
 
         return df, self.search_keys, self.generated_features
@@ -83,33 +74,19 @@ class Normalizer:
         new_columns = []
         dup_counter = 0
         for column in df.columns:
-            if (
-                column
-                in [
-                    TARGET,
-                    EVAL_SET_INDEX,
-                    SYSTEM_RECORD_ID,
-                    ENTITY_SYSTEM_RECORD_ID,
-                    SEARCH_KEY_UNNEST,
-                    DateTimeSearchKeyConverter.DATETIME_COL,
-                ]
-            ):
+            if column in [
+                TARGET,
+                EVAL_SET_INDEX,
+                SYSTEM_RECORD_ID,
+                ENTITY_SYSTEM_RECORD_ID,
+                SEARCH_KEY_UNNEST,
+                DateTimeSearchKeyConverter.DATETIME_COL,
+            ]:
                 self.columns_renaming[column] = column
                 new_columns.append(column)
                 continue
 
-            new_column = str(column)
-            suffix = hashlib.sha256(new_column.encode()).hexdigest()[:6]
-            if len(new_column) == 0:
-                raise ValidationError(self.bundle.get("dataset_empty_column_names"))
-            # db limit for column length
-            if len(new_column) > 250:
-                new_column = new_column[:250]
-
-            # make column name unique relative to server features
-            new_column = f"{new_column}_{suffix}"
-
-            new_column = new_column.lower()
+            new_column = add_hash_suffix(column, self.bundle)
 
             # if column starts with non alphabetic symbol then add "a" to the beginning of string
             if ord(new_column[0]) not in range(ord("a"), ord("z") + 1):
@@ -200,22 +177,6 @@ class Normalizer:
             df = converter.convert(df)
         return df
 
-    def _convert_ip_addresses(self, df: pd.DataFrame) -> pd.DataFrame:
-        for ip_col in SearchKey.find_all_keys(self.search_keys, SearchKey.IP):
-            df[ip_col] = df[ip_col].apply(IpSearchKeyConverter.safe_ip_parse)
-        return df
-
-    def _convert_postal_codes(self, df: pd.DataFrame) -> pd.DataFrame:
-        for postal_code_col in SearchKey.find_all_keys(self.search_keys, SearchKey.POSTAL_CODE):
-            df = PostalCodeSearchKeyConverter(postal_code_col).convert(df)
-        return df
-
-    def _convert_countries(self, df: pd.DataFrame) -> pd.DataFrame:
-        maybe_country_col = SearchKey.find_key(self.search_keys, SearchKey.COUNTRY)
-        if maybe_country_col:
-            df = CountrySearchKeyConverter(maybe_country_col).convert(df)
-        return df
-
     def __convert_features_types(self, df: pd.DataFrame):
         # self.logger.info("Convert features to supported data types")
 
@@ -223,3 +184,19 @@ class Normalizer:
             if not is_numeric_dtype(df[f]):
                 df[f] = df[f].astype("string")
         return df
+
+
+def add_hash_suffix(column: str, bundle: ResourceBundle | None = None) -> str:
+    new_column = str(column)
+    suffix = hashlib.sha256(new_column.encode()).hexdigest()[:6]
+    if bundle is not None and len(new_column) == 0:
+        raise ValidationError(bundle.get("dataset_empty_column_names"))
+    # db limit for column length
+    if len(new_column) > 250:
+        new_column = new_column[:250]
+
+    # make column name unique relative to server features
+    new_column = f"{new_column}_{suffix}"
+
+    new_column = new_column.lower()
+    return new_column

upgini/resource_bundle/strings.properties

@@ -284,8 +284,8 @@ quality_metrics_segment_header=Dataset type
 quality_metrics_match_rate_header=Match rate
 quality_metrics_baseline_header=Baseline {}
 quality_metrics_enriched_header=Enriched {}
-quality_metrics_uplift_header=Uplift, abs
-quality_metrics_uplift_perc_header=Uplift, %
+quality_metrics_uplift_header=Uplift
+quality_metrics_uplift_prc_header=Uplift, %
 
 # Legacy native api messages
 dataset_dataframe_or_path_empty=Either `df` or `path` must be provided

upgini/utils/ip_utils.py

@@ -79,7 +79,7 @@ class IpSearchKeyConverter:
             pass
 
     @staticmethod
-    def safe_ip_parse(ip: Union[str, int, IPv4Address, IPv6Address, bytes]) -> Optional[_BaseAddress]:
+    def _safe_ip_parse(ip: Union[str, int, IPv4Address, IPv6Address]) -> Optional[_BaseAddress]:
         try:
             return ip_address(ip)
         except ValueError:
@@ -110,7 +110,7 @@ class IpSearchKeyConverter:
         self.logger.info("Convert ip address to int")
         original_ip = self.columns_renaming[self.ip_column]
 
-        df[self.ip_column] = df[self.ip_column].apply(self.safe_ip_parse)
+        df[self.ip_column] = df[self.ip_column].apply(self._safe_ip_parse)
         if df[self.ip_column].isnull().all():
             raise ValidationError(self.bundle.get("invalid_ip").format(self.ip_column))
 

{upgini-1.2.96a3.dist-info → upgini-1.2.96a3906.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.96a3
+Version: 1.2.96a3906.dev2
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.96a3.dist-info → upgini-1.2.96a3906.dev2.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=euNfF4usca0xoZtvXArti-DB6GH8wgwXSYVRRbwzMkE,25
+upgini/__about__.py,sha256=8ZaMc0M4yKUigIQciHTdkff0EFfiqt8pmRDvJz70MsQ,33
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=e6JDYTZ2AwC5aF-dqclKZKkiKrHo2f6cFmMQO2ZZmjM,32724
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=dGB4utdlCtgILwX1Hnchg066fwwoyItPBSyzoPe64Z8,218244
+upgini/features_enricher.py,sha256=DFBA-3_yZSDcvJnfZjPCvNFFSC8OZwDl992-dlathm0,218432
 upgini/http.py,sha256=4i7fQwrwU3WzDUOWzrgR-4C8eJwj_5dBwRAR-UjUtlc,44345
 upgini/metadata.py,sha256=vsbbHyPCP3Rs8WkeDgQg99uAA_zmsbDStAT-NwDYhO4,12455
 upgini/metrics.py,sha256=UbKEsHB7XDzoyGNqDx846zbh1t65GpqdnnhViccdoKU,45615
@@ -19,7 +19,7 @@ upgini/autofe/date.py,sha256=MM1S-6imNSzCDOhbNnmsc_bwSqUWBcS8vWAdHF8j1kY,11134
 upgini/autofe/feature.py,sha256=cu4xXjzVVF13ZV4RxuTrysK2qCfezlRCMOzCKRo1rNs,15558
 upgini/autofe/groupby.py,sha256=IYmQV9uoCdRcpkeWZj_kI3ObzoNCNx3ff3h8sTL01tk,3603
 upgini/autofe/operator.py,sha256=EOffJw6vKXpEh5yymqb1RFNJPxGxmnHdFRo9dB5SCFo,4969
-upgini/autofe/unary.py,sha256=Sx11IoHRh5nwyALzjgG9GQOrVNIs8NZ1JzunAJuN66A,5731
+upgini/autofe/unary.py,sha256=N76Pehn-hO8FWlSdqJ2Wm-yoU1MSR7m6yb2GWYBcumU,5933
 upgini/autofe/utils.py,sha256=dYrtyAM8Vcc_R8u4dNo54IsGrHKagTHDJTKhGho0bRg,2967
 upgini/autofe/vector.py,sha256=jHs0nNTOaHspYUlxW7fjQepk4cvr_JDQ65L1OCiVsds,1360
 upgini/autofe/timeseries/__init__.py,sha256=PGwwDAMwvkXl3el12tXVEmZUgDUvlmIPlXtROm6bD18,738
@@ -35,10 +35,10 @@ upgini/data_source/data_source_publisher.py,sha256=ufL8qK1vg8iUKd5bLWz6hEMGiC3Je
 upgini/mdc/__init__.py,sha256=iHJlXQg6xRM1-ZOUtaPSJqw5SpQDszvxp4LyqviNLIQ,1027
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/normalizer/normalize_utils.py,sha256=BSP0vIjRPrupL1sziAudPXJ-qsO4UE9Pyhwiqa1MZV8,8484
+upgini/normalizer/normalize_utils.py,sha256=hMHi5u6Oleqp885UW0Q0Uf1F8fRdZ5jJ7NYoY52SqaI,7403
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=UO6K0wwvutyOyClOnJYlFYAETzMSen6hHnj3--5AIAs,28497
+upgini/resource_bundle/strings.properties,sha256=Hfpr2-I5Ws6ugIN1QSz549OHayZeLYglRsbrGDT6g9g,28491
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=7GpjYqjOp58vYcJLiX__1R5wjUlyQbxvHJ2klFnup_M,6389
@@ -59,7 +59,7 @@ upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0-
 upgini/utils/feature_info.py,sha256=b3RvAeOHSEu-ZXWTrf42Dll_3ZUBL0pw7sdk7hgUKD0,7284
 upgini/utils/features_validator.py,sha256=lEfmk4DoxZ4ooOE1HC0ZXtUb_lFKRFHIrnFULZ4_rL8,3746
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
-upgini/utils/ip_utils.py,sha256=wmnnwVQdjX9o1cNQw6VQMk6maHhvsq6hNsZBYf9knrw,6585
+upgini/utils/ip_utils.py,sha256=TSQ_qDsLlVnm09X1HacpabEf_HNqSWpxBF4Sdc2xs08,6580
 upgini/utils/mstats.py,sha256=u3gQVUtDRbyrOQK6V1UJ2Rx1QbkSNYGjXa6m3Z_dPVs,6286
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
@@ -71,7 +71,7 @@ upgini/utils/target_utils.py,sha256=i3Xt5l9ybB2_nF_ma5cfPuL3OeFTs2dY2xDI0p4Azpg,
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.96a3.dist-info/METADATA,sha256=HdJ2Ptri3J0BuSdc6tXhBZ79vpGvbKIuPx5PVLrtTeY,49530
-upgini-1.2.96a3.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.96a3.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.96a3.dist-info/RECORD,,
+upgini-1.2.96a3906.dev2.dist-info/METADATA,sha256=IiF040yVE4K20_1kus6Y4tzj6rDeA2X8kYkhJ0_Kxr8,49538
+upgini-1.2.96a3906.dev2.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+upgini-1.2.96a3906.dev2.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.96a3906.dev2.dist-info/RECORD,,

{upgini-1.2.96a3.dist-info → upgini-1.2.96a3906.dev2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: hatchling 1.24.2
 Root-Is-Purelib: true
 Tag: py3-none-any