upgini 1.2.96a3906.dev2__py3-none-any.whl → 1.2.98__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.96a3906.dev2"
+__version__ = "1.2.98"

upgini/autofe/feature.py

@@ -18,10 +18,7 @@ class Column:
         self.data = data
         self.calculate_all = calculate_all
 
-    def get_display_name(self, cache: bool = True, shorten: bool = False, **kwargs) -> str:
-        return self.name
-
-    def set_op_params(self, params: Dict[str, str]) -> "Column":
+    def set_op_params(self, params: Dict[str, str], **kwargs) -> "Column":
         return self
 
     def get_op_params(self, **kwargs):
@@ -37,8 +34,21 @@ class Column:
     def get_column_nodes(self) -> List["Column"]:
         return [self]
 
-    def get_columns(self, **kwargs) -> List[str]:
-        return [self.name]
+    def get_columns(self, unhash=False, **kwargs):
+        name = self.name
+        return [self._unhash(name) if unhash else name]
+
+    def get_display_name(self, cache: bool = True, shorten: bool = False, **kwargs) -> str:
+        return self.get_columns(**kwargs)[0]
+
+    def _unhash(self, feature_name: str) -> str:
+        last_component_idx = feature_name.rfind("_")
+        if not feature_name.startswith("f_"):
+            return feature_name  # etalon feature
+        elif last_component_idx == 1:
+            return feature_name[2:]  # fully hashed name, cannot unhash
+        else:
+            return feature_name[2:last_component_idx]
 
     @property
     def children(self) -> List[Union["Feature", "Column"]]:
@@ -81,7 +91,7 @@ class Feature:
         self.cached_display_name = cached_display_name
         self.alias = alias
 
-    def set_op_params(self, params: Optional[Dict[str, str]]) -> "Feature":
+    def set_op_params(self, params: Optional[Dict[str, str]], **kwargs) -> "Feature":
         obj_dict = pydantic_dump_method(self.op)().copy()
         obj_dict.update(params or {})
         self.op = pydantic_parse_method(self.op.__class__)(obj_dict)
@@ -89,13 +99,13 @@ class Feature:
 
         for child in self.children:
             child_params = {
-                k[len(child.get_display_name()) + 1 :]: v
+                k[len(child.get_display_name(**kwargs)) + 1 :]: v
                 for k, v in params.items()
-                if k.startswith(child.get_display_name())
+                if k.startswith(child.get_display_name(**kwargs))
             }
             if not child_params:
                 child_params = params
-            child.set_op_params(child_params)
+            child.set_op_params(child_params, **kwargs)
         return self
 
     def get_op_params(self, **kwargs) -> Dict[str, str]:

upgini/autofe/unary.py

@@ -3,7 +3,7 @@ from typing import Dict, List, Optional
 import numpy as np
 import pandas as pd
 
-from upgini.autofe.operator import PandasOperator, VectorizableMixin
+from upgini.autofe.operator import PandasOperator, ParametrizedOperator, VectorizableMixin
 from upgini.autofe.utils import pydantic_validator
 
 
@@ -198,3 +198,24 @@ class Cluster(PandasOperator):
     input_type: Optional[str] = "vector"
     output_type: Optional[str] = "category"
     is_categorical: bool = True
+
+
+class OutlierDistance(PandasOperator, ParametrizedOperator):
+    name: str = "outlier_dist"
+    is_unary: bool = True
+    input_type: Optional[str] = "vector"
+    output_type: Optional[str] = "float"
+    class_value: Optional[str] = None
+
+    def to_formula(self) -> str:
+        return f"outlier_dist_{self.class_value if self.class_value is not None else 'all'}"
+
+    @classmethod
+    def from_formula(cls, formula: str) -> Optional["OutlierDistance"]:
+        if formula == "outlier_dist":
+            return cls()
+
+        if formula.startswith("outlier_dist_"):
+            class_value = formula.split("_")[-1]
+            return cls(class_value=None if class_value == "all" else class_value)
+        return None

upgini/data_source/data_source_publisher.py

@@ -149,9 +149,8 @@ class DataSourcePublisher:
                         existing_secondary_keys = {item for sublist in row["secondarySearchKeys"] for item in sublist}
                         if existing_secondary_keys == {v.value.name for v in secondary_search_keys.values()}:
                             existing_search_keys = {item for sublist in row["searchKeys"] for item in sublist}
-                            if (
-                                existing_search_keys == {v.value.name for v in search_keys.values()}
-                                or ("IP" in str(existing_search_keys) and "IP" in str(search_keys.values()))
+                            if existing_search_keys == {v.value.name for v in search_keys.values()} or (
+                                "IP" in str(existing_search_keys) and "IP" in str(search_keys.values())
                             ):
                                 raise ValidationError(
                                     "ADS with the same PRIMARY_KEYS -> SECONDARY_KEYS mapping "
@@ -494,3 +493,21 @@ class DataSourcePublisher:
                     raise Exception("Failed to reannounce all ADS: " + status_response["errorMessage"])
             except Exception:
                 self.logger.exception("Failed to reannounce all ADS-es")
+
+    def upload_autofe_model(
+        self, file_path: str, name: str, model_type: Optional[Literal["ONNX"]] = None, description: str = ""
+    ):
+        if model_type is not None and model_type not in ["ONNX"]:
+            raise ValueError(f"Invalid model type: {model_type}. Available values: ONNX")
+        metadata = {
+            "modelName": name,
+            "modelType": model_type or "ONNX",
+            "description": description,
+        }
+        trace_id = str(uuid.uuid4())
+        with MDC(trace_id=trace_id):
+            try:
+                self._rest_client.upload_autofe_model(file_path, metadata, trace_id)
+            except Exception:
+                self.logger.exception("Failed to upload autofe model")
+                raise

upgini/features_enricher.py

@@ -71,10 +71,7 @@ from upgini.search_task import SearchTask
 from upgini.spinner import Spinner
 from upgini.utils import combine_search_keys, find_numbers_with_decimal_comma
 from upgini.utils.blocked_time_series import BlockedTimeSeriesSplit
-from upgini.utils.country_utils import (
-    CountrySearchKeyConverter,
-    CountrySearchKeyDetector,
-)
+from upgini.utils.country_utils import CountrySearchKeyDetector
 from upgini.utils.custom_loss_utils import (
     get_additional_params_custom_loss,
     get_runtime_params_custom_loss,
@@ -105,11 +102,8 @@ from upgini.utils.feature_info import FeatureInfo, _round_shap_value
 from upgini.utils.features_validator import FeaturesValidator
 from upgini.utils.format import Format
 from upgini.utils.ip_utils import IpSearchKeyConverter
-from upgini.utils.phone_utils import PhoneSearchKeyConverter, PhoneSearchKeyDetector
-from upgini.utils.postal_code_utils import (
-    PostalCodeSearchKeyConverter,
-    PostalCodeSearchKeyDetector,
-)
+from upgini.utils.phone_utils import PhoneSearchKeyDetector
+from upgini.utils.postal_code_utils import PostalCodeSearchKeyDetector
 
 try:
     from upgini.utils.progress_bar import CustomProgressBar as ProgressBar
@@ -1122,6 +1116,7 @@ class FeaturesEnricher(TransformerMixin):
                     # and calculate final metric (and uplift)
                     enriched_metric = None
                     uplift = None
+                    uplift_perc = None
                     enriched_estimator = None
                     if set(fitting_X.columns) != set(fitting_enriched_X.columns):
                         self.logger.info(
@@ -1153,6 +1148,7 @@ class FeaturesEnricher(TransformerMixin):
                             self.logger.info(f"Enriched {metric} on train combined features: {enriched_metric}")
                         if baseline_metric is not None and enriched_metric is not None:
                             uplift = (enriched_cv_result.metric - baseline_cv_result.metric) * multiplier
+                            uplift_perc = uplift / abs(baseline_cv_result.metric) * 100
 
                     train_metrics = {
                         self.bundle.get("quality_metrics_segment_header"): self.bundle.get(
@@ -1179,7 +1175,10 @@ class FeaturesEnricher(TransformerMixin):
                             enriched_metric
                         )
                     if uplift is not None:
-                        train_metrics[self.bundle.get("quality_metrics_uplift_header")] = uplift
+                        train_metrics[self.bundle.get("quality_metrics_uplift_header")] = round(uplift, 3)
+                        train_metrics[self.bundle.get("quality_metrics_uplift_perc_header")] = (
+                            f"{round(uplift_perc, 1)}%"
+                        )
                     metrics = [train_metrics]
 
                     # 3 If eval_set is presented - fit final model on train enriched data and score each
@@ -1228,8 +1227,10 @@ class FeaturesEnricher(TransformerMixin):
 
                             if etalon_eval_metric is not None and enriched_eval_metric is not None:
                                 eval_uplift = (enriched_eval_results.metric - etalon_eval_results.metric) * multiplier
+                                eval_uplift_perc = eval_uplift / abs(etalon_eval_results.metric) * 100
                             else:
                                 eval_uplift = None
+                                eval_uplift_perc = None
 
                             eval_metrics = {
                                 self.bundle.get("quality_metrics_segment_header"): self.bundle.get(
@@ -1260,7 +1261,10 @@ class FeaturesEnricher(TransformerMixin):
                                     enriched_eval_metric
                                 )
                             if eval_uplift is not None:
-                                eval_metrics[self.bundle.get("quality_metrics_uplift_header")] = eval_uplift
+                                eval_metrics[self.bundle.get("quality_metrics_uplift_header")] = round(eval_uplift, 3)
+                                eval_metrics[self.bundle.get("quality_metrics_uplift_perc_header")] = (
+                                    f"{round(eval_uplift_perc, 1)}%"
+                                )
 
                             metrics.append(eval_metrics)
 
@@ -2495,21 +2499,6 @@ if response.status_code == 200:
                 )
                 df = converter.convert(df)
 
-            phone_column = self._get_phone_column(search_keys)
-            country_column = self._get_country_column(search_keys)
-            if phone_column:
-                converter = PhoneSearchKeyConverter(phone_column, country_column)
-                df = converter.convert(df)
-
-            if country_column:
-                converter = CountrySearchKeyConverter(country_column)
-                df = converter.convert(df)
-
-            postal_code = self._get_postal_column(search_keys)
-            if postal_code:
-                converter = PostalCodeSearchKeyConverter(postal_code)
-                df = converter.convert(df)
-
             meaning_types = {}
             meaning_types.update({col: FileColumnMeaningType.FEATURE for col in features_for_transform})
             meaning_types.update({col: key.value for col, key in search_keys.items()})
@@ -2904,6 +2893,7 @@ if response.status_code == 200:
             self.fit_generated_features.extend(converter.generated_features)
         else:
             self.logger.info("Input dataset hasn't date column")
+            # TODO remove when this logic will be implemented on the back
             if self.__should_add_date_column():
                 df = self._add_current_date_as_key(df, self.fit_search_keys, self.logger, self.bundle)
 
@@ -2935,6 +2925,26 @@ if response.status_code == 200:
         if normalizer.removed_features:
             self.__log_warning(self.bundle.get("dataset_date_features").format(normalizer.removed_features))
 
+        non_feature_columns = [
+            self.TARGET_NAME,
+            EVAL_SET_INDEX,
+        ] + list(self.fit_search_keys.keys())
+        if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
+            non_feature_columns.append(DateTimeSearchKeyConverter.DATETIME_COL)
+
+        features_columns = [c for c in df.columns if c not in non_feature_columns]
+
+        features_to_drop, feature_validator_warnings = FeaturesValidator(self.logger).validate(
+            df, features_columns, self.generate_features, self.fit_columns_renaming
+        )
+        if feature_validator_warnings:
+            for warning in feature_validator_warnings:
+                self.__log_warning(warning)
+        self.fit_dropped_features.update(features_to_drop)
+        df = df.drop(columns=features_to_drop)
+
+        self.fit_generated_features = [f for f in self.fit_generated_features if f not in self.fit_dropped_features]
+
         self.__adjust_cv(df)
 
         if self.id_columns is not None and self.cv is not None and self.cv.is_time_series():
@@ -2974,6 +2984,7 @@ if response.status_code == 200:
         # Convert EMAIL to HEM etc after unnesting to do it only with one column
         df = self.__convert_unnestable_keys(df, unnest_search_keys)
 
+        # refresh features columns
         non_feature_columns = [
             self.TARGET_NAME,
             EVAL_SET_INDEX,
@@ -2985,17 +2996,6 @@ if response.status_code == 200:
 
         features_columns = [c for c in df.columns if c not in non_feature_columns]
 
-        features_to_drop, feature_validator_warnings = FeaturesValidator(self.logger).validate(
-            df, features_columns, self.generate_features, self.fit_columns_renaming
-        )
-        if feature_validator_warnings:
-            for warning in feature_validator_warnings:
-                self.__log_warning(warning)
-        self.fit_dropped_features.update(features_to_drop)
-        df = df.drop(columns=features_to_drop)
-
-        self.fit_generated_features = [f for f in self.fit_generated_features if f not in self.fit_dropped_features]
-
         meaning_types = {
             **{col: key.value for col, key in self.fit_search_keys.items()},
             **{str(c): FileColumnMeaningType.FEATURE for c in df.columns if c not in non_feature_columns},
@@ -3225,20 +3225,6 @@ if response.status_code == 200:
                 self.logger,
             )
             df = converter.convert(df)
-        phone_column = self._get_phone_column(self.fit_search_keys)
-        country_column = self._get_country_column(self.fit_search_keys)
-        if phone_column:
-            converter = PhoneSearchKeyConverter(phone_column, country_column)
-            df = converter.convert(df)
-
-        if country_column:
-            converter = CountrySearchKeyConverter(country_column)
-            df = converter.convert(df)
-
-        postal_code = self._get_postal_column(self.fit_search_keys)
-        if postal_code:
-            converter = PostalCodeSearchKeyConverter(postal_code)
-            df = converter.convert(df)
 
         return df
 
@@ -4188,7 +4174,7 @@ if response.status_code == 200:
 
                 description = {}
 
-                feature_meta = get_feature_by_name(autofe_feature.get_display_name(shorten=True))
+                feature_meta = get_feature_by_name(autofe_feature.get_display_name(shorten=True, unhash=True))
                 if feature_meta is None:
                     self.logger.warning(f"Feature meta for display index {m.display_index} not found")
                     continue
@@ -4642,42 +4628,59 @@ if response.status_code == 200:
                 if isinstance(X_, pd.Series):
                     X_ = X_.to_frame()
 
-                # TODO check that this file was already uploaded
-
                 with tempfile.TemporaryDirectory() as tmp_dir:
                     X_.to_parquet(f"{tmp_dir}/x.parquet", compression="zstd")
+                    x_digest_sha256 = self.rest_client.compute_file_digest(f"{tmp_dir}/x.parquet")
+                    if self.rest_client.is_file_uploaded(trace_id, x_digest_sha256):
+                        self.logger.info(f"File x.parquet was already uploaded with digest {x_digest_sha256}, skipping")
+                    else:
+                        self.rest_client.dump_input_file(trace_id, f"{tmp_dir}/x.parquet", "x.parquet")
 
                     if y_ is not None:
                         if isinstance(y_, pd.Series):
                             y_ = y_.to_frame()
                         y_.to_parquet(f"{tmp_dir}/y.parquet", compression="zstd")
-                        if eval_set_ and _num_samples(eval_set_[0][0]) > 0:
-                            eval_x_ = eval_set_[0][0]
-                            eval_y_ = eval_set_[0][1]
-                            if isinstance(eval_x_, pd.Series):
-                                eval_x_ = eval_x_.to_frame()
-                            eval_x_.to_parquet(f"{tmp_dir}/eval_x.parquet", compression="zstd")
-                            if isinstance(eval_y_, pd.Series):
-                                eval_y_ = eval_y_.to_frame()
-                            eval_y_.to_parquet(f"{tmp_dir}/eval_y.parquet", compression="zstd")
-                            self.rest_client.dump_input_files(
-                                trace_id,
-                                f"{tmp_dir}/x.parquet",
-                                f"{tmp_dir}/y.parquet",
-                                f"{tmp_dir}/eval_x.parquet",
-                                f"{tmp_dir}/eval_y.parquet",
+                        y_digest_sha256 = self.rest_client.compute_file_digest(f"{tmp_dir}/y.parquet")
+                        if self.rest_client.is_file_uploaded(trace_id, y_digest_sha256):
+                            self.logger.info(
+                                f"File y.parquet was already uploaded with digest {y_digest_sha256}, skipping"
                             )
                         else:
-                            self.rest_client.dump_input_files(
-                                trace_id,
-                                f"{tmp_dir}/x.parquet",
-                                f"{tmp_dir}/y.parquet",
-                            )
-                    else:
-                        self.rest_client.dump_input_files(
-                            trace_id,
-                            f"{tmp_dir}/x.parquet",
-                        )
+                            self.rest_client.dump_input_file(trace_id, f"{tmp_dir}/y.parquet", "y.parquet")
+
+                        if eval_set_ is not None and len(eval_set_) > 0:
+                            for idx, (eval_x_, eval_y_) in enumerate(eval_set_):
+                                if isinstance(eval_x_, pd.Series):
+                                    eval_x_ = eval_x_.to_frame()
+                                eval_x_.to_parquet(f"{tmp_dir}/eval_x_{idx}.parquet", compression="zstd")
+                                eval_x_digest_sha256 = self.rest_client.compute_file_digest(
+                                    f"{tmp_dir}/eval_x_{idx}.parquet"
+                                )
+                                if self.rest_client.is_file_uploaded(trace_id, eval_x_digest_sha256):
+                                    self.logger.info(
+                                        f"File eval_x_{idx}.parquet was already uploaded with"
+                                        f" digest {eval_x_digest_sha256}, skipping"
+                                    )
+                                else:
+                                    self.rest_client.dump_input_file(
+                                        trace_id, f"{tmp_dir}/eval_x_{idx}.parquet", f"eval_x_{idx}.parquet"
+                                    )
+
+                                if isinstance(eval_y_, pd.Series):
+                                    eval_y_ = eval_y_.to_frame()
+                                eval_y_.to_parquet(f"{tmp_dir}/eval_y_{idx}.parquet", compression="zstd")
+                                eval_y_digest_sha256 = self.rest_client.compute_file_digest(
+                                    f"{tmp_dir}/eval_y_{idx}.parquet"
+                                )
+                                if self.rest_client.is_file_uploaded(trace_id, eval_y_digest_sha256):
+                                    self.logger.info(
+                                        f"File eval_y_{idx}.parquet was already uploaded"
+                                        f" with digest {eval_y_digest_sha256}, skipping"
+                                    )
+                                else:
+                                    self.rest_client.dump_input_file(
+                                        trace_id, f"{tmp_dir}/eval_y_{idx}.parquet", f"eval_y_{idx}.parquet"
+                                    )
             except Exception:
                 self.logger.warning("Failed to dump input files", exc_info=True)
 

upgini/http.py

@@ -12,6 +12,7 @@ from enum import Enum
 from functools import lru_cache
 from http.client import HTTPConnection
 from json import dumps
+from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 from urllib.parse import urljoin
 
@@ -292,6 +293,7 @@ class _RestClient:
     UPLOAD_ONLINE_URI = "private/api/v2/ads/upload-online"
     STOP_ADS_MANAGEMENT_TASK_URI_FMT = "private/api/v2/ads/management-task/{0}/stop"
     UNION_SEARCH_TASKS_URI_FMT = SERVICE_ROOT_V2 + "search/merge"
+    UPLOAD_AUTOFE_MODEL_URI_FMT = "private/api/v2/autofe/model/upload"
 
     ACCESS_TOKEN_HEADER_NAME = "Authorization"
     CONTENT_TYPE_HEADER_NAME = "Content-Type"
@@ -404,6 +406,16 @@ class _RestClient:
         meaning_types = [_RestClient.meaning_type_by_name(name, metadata) for name in search_key_names]
         return [meaning_type.value for meaning_type in meaning_types if meaning_type is not None]
 
+    def dump_input_file(self, trace_id: str, path: str, file_name: str):
+        api_path = self.SEARCH_DUMP_INPUT_FILE_FMT
+        with open(path, "rb") as file:
+            files = {"file": (file_name, file, "application/octet-stream")}
+            self._with_unauth_retry(
+                lambda: self._send_post_file_req_v2(
+                    api_path, files, trace_id=trace_id, need_json_response=False
+                )
+            )
+
     def dump_input_files(
         self,
         trace_id: str,
@@ -811,6 +823,17 @@ class _RestClient:
         api_path = self.UNION_SEARCH_TASKS_URI_FMT
         return self._with_unauth_retry(lambda: self._send_post_req(api_path, trace_id, request, result_format=None))
 
+    def upload_autofe_model(self, file_path: str, metadata: dict, trace_id: str):
+        api_path = self.UPLOAD_AUTOFE_MODEL_URI_FMT
+        with open(file_path, "rb") as file:
+            files = {
+                "meta": ("metadata.json", dumps(metadata).encode(), "application/json"),
+                "model": (Path(file_path).name, file, "application/octet-stream"),
+            }
+            return self._with_unauth_retry(
+                lambda: self._send_post_file_req_v2(api_path, files, trace_id=trace_id, need_json_response=False)
+            )
+
     # ---
 
     def _send_get_req(self, api_path: str, trace_id: Optional[str], additional_headers: Optional[dict] = None):

upgini/metrics.py

@@ -399,14 +399,14 @@ class EstimatorWrapper:
                 self.converted_to_str.append(c)
             elif c in self.cat_features:
                 if x[c].dtype == "bool" or (x[c].dtype == "category" and x[c].cat.categories.dtype == "bool"):
-                    x[c] = x[c].astype(np.int64)
+                    x[c] = x[c].astype(pd.Int64Dtype())
                     self.converted_to_int.append(c)
                 elif x[c].dtype == "category" and is_integer_dtype(x[c].cat.categories):
                     self.logger.info(
                         f"Convert categorical feature {c} with integer categories"
                         " to int64 and remove from cat_features"
                     )
-                    x[c] = x[c].astype(np.int64)
+                    x[c] = x[c].astype(pd.Int64Dtype())
                     self.converted_to_int.append(c)
                     self.cat_features.remove(c)
                 elif is_float_dtype(x[c]) or (x[c].dtype == "category" and is_float_dtype(x[c].cat.categories)):
@@ -419,7 +419,7 @@ class EstimatorWrapper:
             else:
                 if x[c].dtype == "bool" or (x[c].dtype == "category" and x[c].cat.categories.dtype == "bool"):
                     self.logger.info(f"Convert bool feature {c} to int64")
-                    x[c] = x[c].astype(np.int64)
+                    x[c] = x[c].astype(pd.Int64Dtype())
                     self.converted_to_int.append(c)
                 elif not is_valid_numeric_array_data(x[c]) and not is_numeric_dtype(x[c]):
                     try:
@@ -442,7 +442,7 @@ class EstimatorWrapper:
         if self.converted_to_int:
             self.logger.info(f"Convert to int features on calculate metrics: {self.converted_to_int}")
             for c in self.converted_to_int:
-                x[c] = x[c].astype(np.int64)
+                x[c] = x[c].astype(pd.Int64Dtype())
 
         if self.converted_to_str:
             self.logger.info(f"Convert to str features on calculate metrics: {self.converted_to_str}")
@@ -896,7 +896,7 @@ class LightGBMWrapper(EstimatorWrapper):
                     x[c] = x[c].astype("category")
 
         for c in x.columns:
-            if x[c].dtype not in ["category", "int64", "float64", "bool"]:
+            if x[c].dtype not in ["category", "int64", "float64", "bool", "Int64"]:
                 self.logger.warning(f"Feature {c} is not numeric and will be dropped")
                 self.dropped_features.append(c)
                 x = x.drop(columns=c, errors="ignore")
@@ -987,7 +987,7 @@ class OtherEstimatorWrapper(EstimatorWrapper):
                     x[c] = x[c].astype("category")
             params["cat_features"] = self.cat_features
         for c in x.columns:
-            if x[c].dtype not in ["category", "int64", "float64", "bool"]:
+            if x[c].dtype not in ["category", "int64", "float64", "bool", "Int64"]:
                 self.logger.warning(f"Feature {c} is not numeric and will be dropped")
                 self.dropped_features.append(c)
                 x = x.drop(columns=c, errors="ignore")

upgini/normalizer/normalize_utils.py

@@ -24,8 +24,11 @@ from upgini.metadata import (
 )
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.utils import find_numbers_with_decimal_comma
+from upgini.utils.country_utils import CountrySearchKeyConverter
 from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
+from upgini.utils.ip_utils import IpSearchKeyConverter
 from upgini.utils.phone_utils import PhoneSearchKeyConverter
+from upgini.utils.postal_code_utils import PostalCodeSearchKeyConverter
 
 
 class Normalizer:
@@ -65,6 +68,12 @@ class Normalizer:
 
         df = self._convert_phone_numbers(df)
 
+        df = self._convert_ip_addresses(df)
+
+        df = self._convert_postal_codes(df)
+
+        df = self._convert_countries(df)
+
         df = self.__convert_features_types(df)
 
         return df, self.search_keys, self.generated_features
@@ -177,6 +186,22 @@ class Normalizer:
             df = converter.convert(df)
         return df
 
+    def _convert_ip_addresses(self, df: pd.DataFrame) -> pd.DataFrame:
+        for ip_col in SearchKey.find_all_keys(self.search_keys, SearchKey.IP):
+            df[ip_col] = df[ip_col].apply(IpSearchKeyConverter.safe_ip_parse)
+        return df
+
+    def _convert_postal_codes(self, df: pd.DataFrame) -> pd.DataFrame:
+        for postal_code_col in SearchKey.find_all_keys(self.search_keys, SearchKey.POSTAL_CODE):
+            df = PostalCodeSearchKeyConverter(postal_code_col).convert(df)
+        return df
+
+    def _convert_countries(self, df: pd.DataFrame) -> pd.DataFrame:
+        maybe_country_col = SearchKey.find_key(self.search_keys, SearchKey.COUNTRY)
+        if maybe_country_col:
+            df = CountrySearchKeyConverter(maybe_country_col).convert(df)
+        return df
+
     def __convert_features_types(self, df: pd.DataFrame):
         # self.logger.info("Convert features to supported data types")
 

upgini/resource_bundle/strings.properties

@@ -284,8 +284,8 @@ quality_metrics_segment_header=Dataset type
 quality_metrics_match_rate_header=Match rate
 quality_metrics_baseline_header=Baseline {}
 quality_metrics_enriched_header=Enriched {}
-quality_metrics_uplift_header=Uplift
-quality_metrics_uplift_prc_header=Uplift, %
+quality_metrics_uplift_header=Uplift, abs
+quality_metrics_uplift_perc_header=Uplift, %
 
 # Legacy native api messages
 dataset_dataframe_or_path_empty=Either `df` or `path` must be provided

upgini/utils/ip_utils.py

@@ -79,7 +79,7 @@ class IpSearchKeyConverter:
             pass
 
     @staticmethod
-    def _safe_ip_parse(ip: Union[str, int, IPv4Address, IPv6Address]) -> Optional[_BaseAddress]:
+    def safe_ip_parse(ip: Union[str, int, IPv4Address, IPv6Address, bytes]) -> Optional[_BaseAddress]:
         try:
             return ip_address(ip)
         except ValueError:
@@ -110,7 +110,7 @@ class IpSearchKeyConverter:
         self.logger.info("Convert ip address to int")
         original_ip = self.columns_renaming[self.ip_column]
 
-        df[self.ip_column] = df[self.ip_column].apply(self._safe_ip_parse)
+        df[self.ip_column] = df[self.ip_column].apply(self.safe_ip_parse)
         if df[self.ip_column].isnull().all():
             raise ValidationError(self.bundle.get("invalid_ip").format(self.ip_column))
 

{upgini-1.2.96a3906.dev2.dist-info → upgini-1.2.98.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.96a3906.dev2
+Version: 1.2.98
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.96a3906.dev2.dist-info → upgini-1.2.98.dist-info}/RECORD

@@ -1,12 +1,12 @@
-upgini/__about__.py,sha256=8ZaMc0M4yKUigIQciHTdkff0EFfiqt8pmRDvJz70MsQ,33
+upgini/__about__.py,sha256=lf7CSxsvho_1d6v9vU7WGiP7TC1XTdLXyd8LGFkkMoU,23
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=e6JDYTZ2AwC5aF-dqclKZKkiKrHo2f6cFmMQO2ZZmjM,32724
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=DFBA-3_yZSDcvJnfZjPCvNFFSC8OZwDl992-dlathm0,218432
-upgini/http.py,sha256=4i7fQwrwU3WzDUOWzrgR-4C8eJwj_5dBwRAR-UjUtlc,44345
+upgini/features_enricher.py,sha256=KSOEzO29nY79RIW0hdbf1qXQGxa3itKZ0PkcwVPPf9U,219954
+upgini/http.py,sha256=DNcoS7qdxG0mOJn6I8r6O5I6XdIJTdzDzW3hkz3NgG4,45443
 upgini/metadata.py,sha256=vsbbHyPCP3Rs8WkeDgQg99uAA_zmsbDStAT-NwDYhO4,12455
-upgini/metrics.py,sha256=UbKEsHB7XDzoyGNqDx846zbh1t65GpqdnnhViccdoKU,45615
+upgini/metrics.py,sha256=gXr2aiw5j9QBWBo1hZp40Is679hef5q8MrT6LJfjsBk,45661
 upgini/search_task.py,sha256=Q5HjBpLIB3OCxAD1zNv5yQ3ZNJx696WCK_-H35_y7Rs,17912
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
 upgini/version_validator.py,sha256=DvbaAvuYFoJqYt0fitpsk6Xcv-H1BYDJYHUMxaKSH_Y,1509
@@ -16,10 +16,10 @@ upgini/autofe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/autofe/all_operators.py,sha256=rdjF5eaE4bC6Q4eu_el5Z7ekYt8DjOFermz2bePPbUc,333
 upgini/autofe/binary.py,sha256=oOEECc4nRzZN2tYaiqx8F2XHnfWpk1bVvb7ZkZJ0lO8,7709
 upgini/autofe/date.py,sha256=MM1S-6imNSzCDOhbNnmsc_bwSqUWBcS8vWAdHF8j1kY,11134
-upgini/autofe/feature.py,sha256=cu4xXjzVVF13ZV4RxuTrysK2qCfezlRCMOzCKRo1rNs,15558
+upgini/autofe/feature.py,sha256=1jiy9_aiaQdVGIh5UbnIGF8St5BkiikOUh5KywMLYRY,16056
 upgini/autofe/groupby.py,sha256=IYmQV9uoCdRcpkeWZj_kI3ObzoNCNx3ff3h8sTL01tk,3603
 upgini/autofe/operator.py,sha256=EOffJw6vKXpEh5yymqb1RFNJPxGxmnHdFRo9dB5SCFo,4969
-upgini/autofe/unary.py,sha256=N76Pehn-hO8FWlSdqJ2Wm-yoU1MSR7m6yb2GWYBcumU,5933
+upgini/autofe/unary.py,sha256=FFtvkQaT0cu_zPZ1jCLcsjik-UUh12qQFF3tUW8NqsE,6675
 upgini/autofe/utils.py,sha256=dYrtyAM8Vcc_R8u4dNo54IsGrHKagTHDJTKhGho0bRg,2967
 upgini/autofe/vector.py,sha256=jHs0nNTOaHspYUlxW7fjQepk4cvr_JDQ65L1OCiVsds,1360
 upgini/autofe/timeseries/__init__.py,sha256=PGwwDAMwvkXl3el12tXVEmZUgDUvlmIPlXtROm6bD18,738
@@ -31,14 +31,14 @@ upgini/autofe/timeseries/roll.py,sha256=zADKXU-eYWQnQ5R3am1yEal8uU6Tm0jLAixwPb_a
 upgini/autofe/timeseries/trend.py,sha256=K1_iw2ko_LIUU8YCUgrvN3n0MkHtsi7-63-8x9er1k4,2129
 upgini/autofe/timeseries/volatility.py,sha256=SvZfhM_ZAWCNpTf87WjSnZsnlblARgruDlu4By4Zvhc,8078
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/data_source/data_source_publisher.py,sha256=ufL8qK1vg8iUKd5bLWz6hEMGiC3JepUaWYx-nBKVqjA,24294
+upgini/data_source/data_source_publisher.py,sha256=iqcDx2oRiVyeuzQqqPBnfqWhzQSXTyya0wk2ltibBAA,25010
 upgini/mdc/__init__.py,sha256=iHJlXQg6xRM1-ZOUtaPSJqw5SpQDszvxp4LyqviNLIQ,1027
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/normalizer/normalize_utils.py,sha256=hMHi5u6Oleqp885UW0Q0Uf1F8fRdZ5jJ7NYoY52SqaI,7403
+upgini/normalizer/normalize_utils.py,sha256=mDh2mBW3aQMB4EFP2aHbf2dGMVkOcWnp4sKKvKDBh8w,8511
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=Hfpr2-I5Ws6ugIN1QSz549OHayZeLYglRsbrGDT6g9g,28491
+upgini/resource_bundle/strings.properties,sha256=UO6K0wwvutyOyClOnJYlFYAETzMSen6hHnj3--5AIAs,28497
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=7GpjYqjOp58vYcJLiX__1R5wjUlyQbxvHJ2klFnup_M,6389
@@ -59,7 +59,7 @@ upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0-
 upgini/utils/feature_info.py,sha256=b3RvAeOHSEu-ZXWTrf42Dll_3ZUBL0pw7sdk7hgUKD0,7284
 upgini/utils/features_validator.py,sha256=lEfmk4DoxZ4ooOE1HC0ZXtUb_lFKRFHIrnFULZ4_rL8,3746
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
-upgini/utils/ip_utils.py,sha256=TSQ_qDsLlVnm09X1HacpabEf_HNqSWpxBF4Sdc2xs08,6580
+upgini/utils/ip_utils.py,sha256=wmnnwVQdjX9o1cNQw6VQMk6maHhvsq6hNsZBYf9knrw,6585
 upgini/utils/mstats.py,sha256=u3gQVUtDRbyrOQK6V1UJ2Rx1QbkSNYGjXa6m3Z_dPVs,6286
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
@@ -71,7 +71,7 @@ upgini/utils/target_utils.py,sha256=i3Xt5l9ybB2_nF_ma5cfPuL3OeFTs2dY2xDI0p4Azpg,
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.96a3906.dev2.dist-info/METADATA,sha256=IiF040yVE4K20_1kus6Y4tzj6rDeA2X8kYkhJ0_Kxr8,49538
-upgini-1.2.96a3906.dev2.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-upgini-1.2.96a3906.dev2.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.96a3906.dev2.dist-info/RECORD,,
+upgini-1.2.98.dist-info/METADATA,sha256=3-L9c9p3ul3Cz1l1zFF2XTxlkN21bygcCWccOd6KbZo,49528
+upgini-1.2.98.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+upgini-1.2.98.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.98.dist-info/RECORD,,