upgini 1.2.55a1__tar.gz → 1.2.56__tar.gz

{upgini-1.2.55a1 → upgini-1.2.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.55a1
+Version: 1.2.56
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.56/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.56"

{upgini-1.2.55a1 → upgini-1.2.56}/src/upgini/dataset.py

@@ -587,23 +587,15 @@ class Dataset:  # (pd.DataFrame):
         if (
             runtime_parameters is not None
             and runtime_parameters.properties is not None
+            and "generate_features" in runtime_parameters.properties
         ):
-            if "generate_features" in runtime_parameters.properties:
-                generate_features = runtime_parameters.properties["generate_features"].split(",")
-                renamed_generate_features = []
-                for f in generate_features:
-                    for new_column, orig_column in self.columns_renaming.items():
-                        if f == orig_column:
-                            renamed_generate_features.append(new_column)
-                runtime_parameters.properties["generate_features"] = ",".join(renamed_generate_features)
-            if "columns_for_online_api" in runtime_parameters.properties:
-                columns_for_online_api = runtime_parameters.properties["columns_for_online_api"].split(",")
-                renamed_columns_for_online_api = []
-                for f in columns_for_online_api:
-                    for new_column, orig_column in self.columns_renaming.items():
-                        if f == orig_column:
-                            renamed_columns_for_online_api.append(new_column)
-                runtime_parameters.properties["columns_for_online_api"] = ",".join(renamed_columns_for_online_api)
+            generate_features = runtime_parameters.properties["generate_features"].split(",")
+            renamed_generate_features = []
+            for f in generate_features:
+                for new_column, orig_column in self.columns_renaming.items():
+                    if f == orig_column:
+                        renamed_generate_features.append(new_column)
+            runtime_parameters.properties["generate_features"] = ",".join(renamed_generate_features)
 
         return runtime_parameters
 

{upgini-1.2.55a1 → upgini-1.2.56}/src/upgini/features_enricher.py

@@ -222,7 +222,6 @@ class FeaturesEnricher(TransformerMixin):
         loss: Optional[str] = None,
         detect_missing_search_keys: bool = True,
         generate_features: Optional[List[str]] = None,
-        columns_for_online_api: Optional[List[str]] = None,
         round_embeddings: Optional[int] = None,
         logs_enabled: bool = True,
         raise_validation_error: bool = True,
@@ -346,9 +345,6 @@ class FeaturesEnricher(TransformerMixin):
                     self.logger.error(msg)
                     raise ValidationError(msg)
                 self.runtime_parameters.properties["round_embeddings"] = round_embeddings
-        self.columns_for_online_api = columns_for_online_api
-        if columns_for_online_api is not None:
-            self.runtime_parameters.properties["columns_for_online_api"] = ",".join(columns_for_online_api)
         maybe_downsampling_limit = self.runtime_parameters.properties.get("downsampling_limit")
         if maybe_downsampling_limit is not None:
             Dataset.FIT_SAMPLE_THRESHOLD = int(maybe_downsampling_limit)
@@ -2464,7 +2460,19 @@ if response.status_code == 200:
             if add_fit_system_record_id:
                 selecting_columns.append(SORT_ID)
 
-            result = result[selecting_columns]
+            selecting_columns = list(set(selecting_columns))
+            # sorting: first columns from X, then generated features, then enriched features
+            sorted_selecting_columns = [c for c in validated_X.columns if c in selecting_columns]
+            for c in generated_features:
+                if c in selecting_columns and c not in sorted_selecting_columns:
+                    sorted_selecting_columns.append(c)
+            for c in result.columns:
+                if c in selecting_columns and c not in sorted_selecting_columns:
+                    sorted_selecting_columns.append(c)
+
+            self.logger.info(f"Transform sorted_selecting_columns: {sorted_selecting_columns}")
+
+            result = result[sorted_selecting_columns]
 
             if self.country_added:
                 result = result.drop(columns=COUNTRY, errors="ignore")
@@ -2612,18 +2620,17 @@ if response.status_code == 200:
             checked_generate_features = []
             for gen_feature in self.generate_features:
                 if gen_feature not in x_columns:
-                    msg = self.bundle.get("missing_generate_feature").format(gen_feature, x_columns)
-                    self.__log_warning(msg)
+                    if gen_feature == self._get_phone_column(self.search_keys):
+                        raise ValidationError(
+                            self.bundle.get("missing_generate_feature").format(gen_feature, x_columns)
+                        )
+                    else:
+                        self.__log_warning(self.bundle.get("missing_generate_feature").format(gen_feature, x_columns))
                 else:
                     checked_generate_features.append(gen_feature)
             self.generate_features = checked_generate_features
             self.runtime_parameters.properties["generate_features"] = ",".join(self.generate_features)
 
-        if self.columns_for_online_api is not None and len(self.columns_for_online_api) > 0:
-            for column in self.columns_for_online_api:
-                if column not in validated_X.columns:
-                    raise ValidationError(self.bundle.get("missing_column_for_online_api").format(column))
-
         if self.id_columns is not None:
             for id_column in self.id_columns:
                 if id_column not in validated_X.columns:
@@ -3726,6 +3733,8 @@ if response.status_code == 200:
         features_info_without_links = []
         internal_features_info = []
 
+        original_shaps = {original_names_dict.get(fm.name, fm.name): fm.shap_value for fm in features_meta}
+
         if updated_shaps is not None:
             for fm in features_meta:
                 fm.shap_value = updated_shaps.get(fm.name, 0.0)
@@ -3737,15 +3746,16 @@ if response.status_code == 200:
 
             is_client_feature = feature_meta.name in x_columns
 
-            if feature_meta.shap_value == 0.0:
+            # TODO make a decision about selected features based on special flag from mlb
+            if original_shaps.get(feature_meta.name, 0.0) == 0.0:
                 if self.fit_select_features:
                     self.dropped_client_feature_names_.append(feature_meta.name)
                 continue
 
             # Use only important features
             if (
-                feature_meta.name in self.fit_generated_features
-                or feature_meta.name == COUNTRY
+                # feature_meta.name in self.fit_generated_features or
+                feature_meta.name == COUNTRY
                 # In select_features mode we select also from etalon features and need to show them
                 or (not self.fit_select_features and is_client_feature)
             ):

{upgini-1.2.55a1 → upgini-1.2.56}/src/upgini/metadata.py

@@ -89,7 +89,7 @@ class SearchKey(Enum):
         if meaning_type == FileColumnMeaningType.EMAIL:
             return SearchKey.EMAIL
         if meaning_type == FileColumnMeaningType.HEM:
-            return SearchKey.HEM
+            return SearchKey.HEM  # TODO check that it wasn't EMAIL
         if meaning_type == FileColumnMeaningType.IP_ADDRESS:
             return SearchKey.IP
         if meaning_type == FileColumnMeaningType.MSISDN:
@@ -105,27 +105,27 @@ class SearchKey(Enum):
         if meaning_type == FileColumnMeaningType.POSTAL_CODE:
             return SearchKey.POSTAL_CODE
         if meaning_type == FileColumnMeaningType.IPV6_ADDRESS:
-            return SearchKey.IPV6_ADDRESS
-        if meaning_type == FileColumnMeaningType.IPV6_RANGE_FROM:
-            return SearchKey.IPV6_RANGE_FROM
-        if meaning_type == FileColumnMeaningType.IPV6_RANGE_TO:
-            return SearchKey.IPV6_RANGE_TO
-        if meaning_type == FileColumnMeaningType.EMAIL_ONE_DOMAIN:
-            return SearchKey.EMAIL_ONE_DOMAIN
-        if meaning_type == FileColumnMeaningType.IP_RANGE_FROM:
-            return SearchKey.IP_RANGE_FROM
-        if meaning_type == FileColumnMeaningType.IP_RANGE_TO:
-            return SearchKey.IP_RANGE_TO
-        if meaning_type == FileColumnMeaningType.MSISDN_RANGE_FROM:
-            return SearchKey.MSISDN_RANGE_FROM
-        if meaning_type == FileColumnMeaningType.MSISDN_RANGE_TO:
-            return SearchKey.MSISDN_RANGE_TO
+            return SearchKey.IP
+        # if meaning_type == FileColumnMeaningType.IPV6_RANGE_FROM:
+        #     return SearchKey.IPV6_RANGE_FROM
+        # if meaning_type == FileColumnMeaningType.IPV6_RANGE_TO:
+        #     return SearchKey.IPV6_RANGE_TO
+        # if meaning_type == FileColumnMeaningType.EMAIL_ONE_DOMAIN:
+        #     return SearchKey.EMAIL_ONE_DOMAIN
+        # if meaning_type == FileColumnMeaningType.IP_RANGE_FROM:
+        #     return SearchKey.IP_RANGE_FROM
+        # if meaning_type == FileColumnMeaningType.IP_RANGE_TO:
+        #     return SearchKey.IP_RANGE_TO
+        # if meaning_type == FileColumnMeaningType.MSISDN_RANGE_FROM:
+        #     return SearchKey.MSISDN_RANGE_FROM
+        # if meaning_type == FileColumnMeaningType.MSISDN_RANGE_TO:
+        #     return SearchKey.MSISDN_RANGE_TO
         if meaning_type == FileColumnMeaningType.IP_BINARY:
-            return SearchKey.IP_BINARY
-        if meaning_type == FileColumnMeaningType.IP_RANGE_FROM_BINARY:
-            return SearchKey.IP_RANGE_FROM_BINARY
-        if meaning_type == FileColumnMeaningType.IP_RANGE_TO_BINARY:
-            return SearchKey.IP_RANGE_TO_BINARY
+            return SearchKey.IP
+        # if meaning_type == FileColumnMeaningType.IP_RANGE_FROM_BINARY:
+        #     return SearchKey.IP_RANGE_FROM_BINARY
+        # if meaning_type == FileColumnMeaningType.IP_RANGE_TO_BINARY:
+        #     return SearchKey.IP_RANGE_TO_BINARY
 
     @staticmethod
     def find_key(search_keys: Dict[str, SearchKey], keys: Union[SearchKey, List[SearchKey]]) -> Optional[SearchKey]:
@@ -249,7 +249,9 @@ class FileMetadata(BaseModel):
             for key in keys_group:
                 column = self.column_by_name(key)
                 if column:
-                    search_keys[SearchKey.from_meaning_type(column.meaningType)] = column.name
+                    search_key = SearchKey.from_meaning_type(column.meaningType)
+                    if search_key is not None:
+                        search_keys[search_key] = column.name
         return search_keys
 
 

{upgini-1.2.55a1 → upgini-1.2.56}/src/upgini/resource_bundle/strings.properties

@@ -111,7 +111,6 @@ x_is_empty=X is empty
 y_is_empty=y is empty
 x_contains_reserved_column_name=Column name {} is reserved. Please rename column and try again
 missing_generate_feature=Feature {} specified in `generate_features` is not present in input columns: {}
-missing_column_for_online_api=Column {} specified in `columns_for_online_api` is not present in input columns: {}
 x_unstable_by_date=Your training sample is unstable in number of rows per date. It is recommended to redesign the training sample
 train_unstable_target=Your training sample contains an unstable target event, PSI = {}. This will lead to unstable scoring on deferred samples. It is recommended to redesign the training sample
 eval_unstable_target=Your training and evaluation samples have a difference in target distribution. PSI = {}. The results will be unstable. It is recommended to redesign the training and evaluation samples

{upgini-1.2.55a1 → upgini-1.2.56}/src/upgini/utils/email_utils.py

@@ -116,17 +116,17 @@ class EmailSearchKeyConverter:
         else:
             df[self.hem_column] = df[self.hem_column].astype("string").str.lower()
 
-        # del self.search_keys[self.email_column]
-        # if self.email_column in self.unnest_search_keys:
-        #     self.unnest_search_keys.remove(self.email_column)
+        del self.search_keys[self.email_column]
+        if self.email_column in self.unnest_search_keys:
+            self.unnest_search_keys.remove(self.email_column)
 
         one_domain_name = self.email_column + self.ONE_DOMAIN_SUFFIX
         df[one_domain_name] = df[self.email_column].apply(self._email_to_one_domain)
         self.columns_renaming[one_domain_name] = original_email_column
         self.search_keys[one_domain_name] = SearchKey.EMAIL_ONE_DOMAIN
 
-        # if self.email_converted_to_hem:
-        #     df = df.drop(columns=self.email_column)
-        #     del self.columns_renaming[self.email_column]
+        if self.email_converted_to_hem:
+            df = df.drop(columns=self.email_column)
+            del self.columns_renaming[self.email_column]
 
         return df

upgini-1.2.55a1/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.55a1"