upgini 1.2.125__py3-none-any.whl → 1.2.127__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.125"
+__version__ = "1.2.127"

upgini/data_source/data_source_publisher.py

@@ -123,17 +123,9 @@ class DataSourcePublisher:
                     set(search_keys.values()) == {SearchKey.IP_RANGE_FROM, SearchKey.IP_RANGE_TO}
                     or set(search_keys.values()) == {SearchKey.IPV6_RANGE_FROM, SearchKey.IPV6_RANGE_TO}
                     or set(search_keys.values()) == {SearchKey.MSISDN_RANGE_FROM, SearchKey.MSISDN_RANGE_TO}
+                    or snapshot_frequency_days is not None or join_date_abs_limit_days is not None
                 ) and sort_column is None:
                     raise ValidationError("Sort column is required for passed search keys")
-                if (
-                    set(search_keys.values()) == {SearchKey.PHONE, SearchKey.DATE}
-                    and snapshot_frequency_days is None
-                    and join_date_abs_limit_days is None
-                ):
-                    raise ValidationError(
-                        "With MSISDN and DATE keys one of the snapshot_frequency_days or"
-                        " join_date_abs_limit_days parameters is required"
-                    )
                 if (
                     set(search_keys.values()) == {SearchKey.PHONE, SearchKey.DATE}
                     or set(search_keys.values()) == {SearchKey.HEM, SearchKey.DATE}

upgini/features_enricher.py

@@ -1479,11 +1479,7 @@ class FeaturesEnricher(TransformerMixin):
 
         self.logger.info(f"PSI values by sparsity: {psi_values_sparse}")
 
-        unstable_by_sparsity = [
-            feature
-            for feature, psi in psi_values_sparse.items()
-            if psi > stability_threshold
-        ]
+        unstable_by_sparsity = [feature for feature, psi in psi_values_sparse.items() if psi > stability_threshold]
         if unstable_by_sparsity:
             self.logger.info(f"Unstable by sparsity features ({stability_threshold}): {sorted(unstable_by_sparsity)}")
 
@@ -1493,11 +1489,7 @@ class FeaturesEnricher(TransformerMixin):
 
         self.logger.info(f"PSI values by value: {psi_values}")
 
-        unstable_by_value = [
-            feature
-            for feature, psi in psi_values.items()
-            if psi > stability_threshold
-        ]
+        unstable_by_value = [feature for feature, psi in psi_values.items() if psi > stability_threshold]
         if unstable_by_value:
             self.logger.info(f"Unstable by value features ({stability_threshold}): {sorted(unstable_by_value)}")
 
@@ -2561,10 +2553,15 @@ if response.status_code == 200:
                 if transform_usage.has_limit:
                     if len(X) > transform_usage.rest_rows:
                         rest_rows = max(transform_usage.rest_rows, 0)
-                        msg = self.bundle.get("transform_usage_warning").format(len(X), rest_rows)
+                        bundle_msg = (
+                            "transform_usage_warning_registered"
+                            if self.__is_registered
+                            else "transform_usage_warning_demo"
+                        )
+                        msg = self.bundle.get(bundle_msg).format(len(X), rest_rows)
                         self.logger.warning(msg)
                         print(msg)
-                        show_request_quote_button()
+                        show_request_quote_button(is_registered=self.__is_registered)
                         return None, {}, [], {}
                     else:
                         msg = self.bundle.get("transform_usage_info").format(
@@ -2749,8 +2746,13 @@ if response.status_code == 200:
 
             meaning_types = {}
             meaning_types.update(
-                {col: FileColumnMeaningType.FEATURE for col in features_for_transform if col not in date_features}
+                {
+                    col: FileColumnMeaningType.FEATURE
+                    for col in features_for_transform
+                    if col not in date_features and col not in generated_features
+                }
             )
+            meaning_types.update({col: FileColumnMeaningType.GENERATED_FEATURE for col in generated_features})
             meaning_types.update({col: FileColumnMeaningType.DATE_FEATURE for col in date_features})
             meaning_types.update({col: key.value for col, key in search_keys.items()})
 
@@ -3304,9 +3306,10 @@ if response.status_code == 200:
             **{
                 str(c): FileColumnMeaningType.FEATURE
                 for c in df.columns
-                if c not in non_feature_columns and c not in date_features
+                if c not in non_feature_columns and c not in date_features and c not in self.fit_generated_features
             },
         }
+        meaning_types.update({col: FileColumnMeaningType.GENERATED_FEATURE for col in self.fit_generated_features})
         meaning_types.update({col: FileColumnMeaningType.DATE_FEATURE for col in date_features})
         meaning_types[self.TARGET_NAME] = FileColumnMeaningType.TARGET
         meaning_types[ENTITY_SYSTEM_RECORD_ID] = FileColumnMeaningType.ENTITY_SYSTEM_RECORD_ID
@@ -3368,7 +3371,12 @@ if response.status_code == 200:
         self.passed_features = [
             column
             for column, meaning_type in meaning_types.items()
-            if meaning_type in [FileColumnMeaningType.FEATURE, FileColumnMeaningType.DATE_FEATURE]
+            if meaning_type
+            in [
+                FileColumnMeaningType.FEATURE,
+                FileColumnMeaningType.DATE_FEATURE,
+                FileColumnMeaningType.GENERATED_FEATURE,
+            ]
         ]
 
         self._search_task = dataset.search(
@@ -4433,7 +4441,9 @@ if response.status_code == 200:
             raise Exception(self.bundle.get("missing_features_meta"))
         features_meta = deepcopy(features_meta)
 
-        original_names_dict = {c.name: c.originalName for c in self._search_task.get_file_metadata(trace_id).columns}
+        file_metadata_columns = self._search_task.get_file_metadata(trace_id).columns
+        file_meta_by_orig_name = {c.originalName: c for c in file_metadata_columns}
+        original_names_dict = {c.name: c.originalName for c in file_metadata_columns}
         features_df = self._search_task.get_all_initial_raw_features(trace_id, metrics_calculation=True)
 
         # To be sure that names with hash suffixes
@@ -4453,7 +4463,11 @@ if response.status_code == 200:
             original_name = original_names_dict.get(feature_meta.name, feature_meta.name)
             feature_meta.name = original_name
 
-            is_client_feature = original_name in clients_features_df.columns
+            file_meta = file_meta_by_orig_name.get(original_name)
+            is_generated_feature = (
+                file_meta is not None and file_meta.meaningType == FileColumnMeaningType.GENERATED_FEATURE
+            )
+            is_client_feature = original_name in clients_features_df.columns and not is_generated_feature
 
             if selected_features is not None and feature_meta.name not in selected_features:
                 self.logger.info(f"Feature {feature_meta.name} is not selected before and skipped")
@@ -4476,9 +4490,13 @@ if response.status_code == 200:
 
         for feature_meta in selected_features_meta:
             original_name = original_names_dict.get(feature_meta.name, feature_meta.name)
-            is_client_feature = original_name in clients_features_df.columns
+            file_meta = file_meta_by_orig_name.get(original_name)
+            is_generated_feature = (
+                file_meta is not None and file_meta.meaningType == FileColumnMeaningType.GENERATED_FEATURE
+            )
+            is_client_feature = original_name in clients_features_df.columns and not is_generated_feature
 
-            if not is_client_feature:
+            if not is_client_feature and not is_generated_feature:
                 self.external_source_feature_names.append(original_name)
 
             if self.psi_values is not None:
@@ -4509,9 +4527,10 @@ if response.status_code == 200:
 
             self.feature_names_.append(feature_meta.name)
             self.feature_importances_.append(_round_shap_value(feature_meta.shap_value))
-
             df_for_sample = features_df if feature_meta.name in features_df.columns else clients_features_df
-            feature_info = FeatureInfo.from_metadata(feature_meta, df_for_sample, is_client_feature)
+            feature_info = FeatureInfo.from_metadata(
+                feature_meta, df_for_sample, is_client_feature, is_generated_feature
+            )
             features_info.append(feature_info.to_row(self.bundle))
             features_info_without_links.append(feature_info.to_row_without_links(self.bundle))
             internal_features_info.append(feature_info.to_internal_row(self.bundle))
@@ -4522,7 +4541,7 @@ if response.status_code == 200:
         if len(features_info) > 0:
             self.features_info = pd.DataFrame(features_info)
             # If all psi values are 0 or null, drop psi column
-            if self.features_info[self.bundle.get("features_info_psi")].fillna(0.0).eq(0.0).all():
+            if self.features_info[self.bundle.get("features_info_psi")].astype(np.float64).fillna(0.0).eq(0.0).all():
                 self.features_info.drop(columns=[self.bundle.get("features_info_psi")], inplace=True)
             self._features_info_without_links = pd.DataFrame(features_info_without_links)
             self._internal_features_info = pd.DataFrame(internal_features_info)

upgini/metadata.py

@@ -36,6 +36,7 @@ class FileColumnMeaningType(Enum):
     SCORE = "SCORE"
     TARGET = "TARGET"
     FEATURE = "FEATURE"
+    GENERATED_FEATURE = "GENERATED_FEATURE"
     DATE_FEATURE = "DATE_FEATURE"
     CUSTOM_KEY = "CUSTOM_KEY"
     COUNTRY = "COUNTRY"

upgini/resource_bundle/strings.properties

@@ -12,7 +12,8 @@ polling_unregister_information=We'll send email notification once it's completed
 ads_upload_finish=Thank you for your submission!\nWe'll check your data sharing proposal and get back to you
 demo_dataset_info=Demo training dataset detected. Registration for an API key is not required.\n
 transform_usage_info=You use Trial access to Upgini data enrichment. Limit for Trial: {} rows. You have already enriched: {} rows.
-transform_usage_warning=You are trying to launch enrichment for {} rows, which will exceed the rest limit {}.
+transform_usage_warning_demo=Unregistered-user limit: {} rows remaining; you requested {}.
+transform_usage_warning_registered=Free tier limit: {} rows remaining; you requested {}.
 
 # Warnings
 support_link=https://upgini.com/support

upgini/utils/display_utils.py

@@ -339,17 +339,54 @@ def show_button_download_pdf(
             return display(HTML(html), display_id=display_id)
 
 
-def show_request_quote_button():
+def show_request_quote_button(is_registered: bool):
     if not ipython_available():
-        print("https://upgini.com/request-a-quote")
+        if is_registered:
+            print("https://upgini.com/request-a-quote")
+        else:
+            print("https://profile.upgini.com/login")
     else:
-        import ipywidgets as widgets
-        from IPython.display import Javascript, display
-
-        button = widgets.Button(description="Request a quote", button_style="danger")
+        from IPython.display import HTML, display, Javascript
+        from ipywidgets import Layout, Button
+
+        if is_registered:
+            display(HTML("""
+                <style>
+                    button.custom-button {
+                        border: 1px solid black !important;
+                        background: white !important;
+                        color: black !important;
+                        white-space: nowrap;
+                    }
+                </style>
+            """))
+            description = "Request a quote"
+            tooltip = "Ask a quote"
+            url = "https://upgini.com/request-a-quote"
+        else:
+            display(HTML("""
+                <style>
+                    button.custom-button {
+                        border: 1px solid #d00 !important;
+                        background: #fff !important;
+                        color: #d00 !important;
+                        white-space: nowrap;
+                    }
+                </style>
+            """))
+            description = "Get an API KEY"
+            tooltip = "Register"
+            url = "https://profile.upgini.com/login"
+
+        button = Button(
+            description=description,
+            layout=Layout(width='auto'),
+            tooltip=tooltip
+        )
+        button.add_class("custom-button")
 
         def on_button_clicked(b):
-            display(Javascript('window.open("https://upgini.com/request-a-quote");'))
+            display(Javascript('window.open("' + url + '");'))
 
         button.on_click(on_button_clicked)
 

upgini/utils/feature_info.py

@@ -31,7 +31,10 @@ class FeatureInfo:
 
     @staticmethod
     def from_metadata(
-        feature_meta: FeaturesMetadataV2, data: Optional[pd.DataFrame], is_client_feature: bool
+        feature_meta: FeaturesMetadataV2,
+        data: Optional[pd.DataFrame],
+        is_client_feature: bool,
+        is_generated_feature: bool,
     ) -> "FeatureInfo":
         return FeatureInfo(
             name=_get_name(feature_meta),
@@ -41,8 +44,8 @@ class FeatureInfo:
             value_preview=_get_feature_sample(feature_meta, data),
             provider=_get_provider(feature_meta, is_client_feature),
             internal_provider=_get_internal_provider(feature_meta, is_client_feature),
-            source=_get_source(feature_meta, is_client_feature),
-            internal_source=_get_internal_source(feature_meta, is_client_feature),
+            source=_get_source(feature_meta, is_client_feature, is_generated_feature),
+            internal_source=_get_internal_source(feature_meta, is_client_feature, is_generated_feature),
             update_frequency=feature_meta.update_frequency,
             commercial_schema=feature_meta.commercial_schema,
             doc_link=feature_meta.doc_link,
@@ -139,22 +142,30 @@ def _get_internal_provider(feature_meta: FeaturesMetadataV2, is_client_feature:
         return "" if is_client_feature else (feature_meta.data_provider or "Upgini")
 
 
-def _get_source(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
+def _get_source(feature_meta: FeaturesMetadataV2, is_client_feature: bool, is_generated_feature: bool) -> str:
+    if is_generated_feature:
+        return "AutoFE: features from Training dataset"
+
     sources = _list_or_single(feature_meta.data_sources, feature_meta.data_source)
     source_links = _list_or_single(feature_meta.data_source_links, feature_meta.data_source_link)
     if sources:
         source = _make_links(sources, source_links)
     else:
-        source = _get_internal_source(feature_meta, is_client_feature)
+        source = _get_internal_source(feature_meta, is_client_feature, is_generated_feature)
     return source
 
 
-def _get_internal_source(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
+def _get_internal_source(feature_meta: FeaturesMetadataV2, is_client_feature: bool, is_generated_feature: bool) -> str:
+    if is_generated_feature:
+        return "AutoFE: features from Training dataset"
+
     sources = _list_or_single(feature_meta.data_sources, feature_meta.data_source)
     if sources:
         return ", ".join(sources)
+    elif feature_meta.data_source:
+        return feature_meta.data_source
     else:
-        return feature_meta.data_source or (
+        return (
             LLM_SOURCE
             if not feature_meta.name.endswith("_country")
             and not feature_meta.name.endswith("_postal_code")

{upgini-1.2.125.dist-info → upgini-1.2.127.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.125
+Version: 1.2.127
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.125.dist-info → upgini-1.2.127.dist-info}/RECORD

@@ -1,11 +1,11 @@
-upgini/__about__.py,sha256=khvL6Ma3KHnaaXtUCPR9kKBJFG5qg7emKoKVlrbEt0k,24
+upgini/__about__.py,sha256=h491OIJG19TxwsLIKSTYrHLAOPuj31b_J7sUaPKFa6c,24
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=Nm2ZmwyQqvTnymYpGUwyJWy7y2ebXlHMyYmGeGcyA_s,31652
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=tmKeERG2b0YfJ47g-UXQQ3S-9tyagwUOhI4oqN3kG2w,233058
+upgini/features_enricher.py,sha256=wC9hWu47gdn-dXs5yLHO9etjm3t7XVF-xpafF1gakWI,234470
 upgini/http.py,sha256=-J_wOpnwVnT0ebPC6sOs6fN3AWtCD0LJLu6nlYmxaqk,44348
-upgini/metadata.py,sha256=CL9bFytdUZlbQYtTgNgAkt_sxO9klARQtULDBgb2Hlg,12575
+upgini/metadata.py,sha256=H3wiN37k-yqWZgbPD0tJzx8DzaCIkgmX5cybhByQWLg,12619
 upgini/metrics.py,sha256=KCPE_apPN-9BIdv6GqASbJVaB_gBcy8wzNApAcyaGo4,46020
 upgini/search_task.py,sha256=5mL_qV5mVtDkIumM9xCOgfa9Lc2B8mxJ1qI21iaScnQ,18656
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
@@ -31,14 +31,14 @@ upgini/autofe/timeseries/roll.py,sha256=zADKXU-eYWQnQ5R3am1yEal8uU6Tm0jLAixwPb_a
 upgini/autofe/timeseries/trend.py,sha256=K1_iw2ko_LIUU8YCUgrvN3n0MkHtsi7-63-8x9er1k4,2129
 upgini/autofe/timeseries/volatility.py,sha256=SvZfhM_ZAWCNpTf87WjSnZsnlblARgruDlu4By4Zvhc,8078
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/data_source/data_source_publisher.py,sha256=qXQUYErhCmkWHm2FWgTL0FYZ2aJbxtSDV94OCM3eqUU,26653
+upgini/data_source/data_source_publisher.py,sha256=CQi3fEukaStV-RiadSEvEFLThOlZJzA6PzleQQgGfGk,26286
 upgini/mdc/__init__.py,sha256=iHJlXQg6xRM1-ZOUtaPSJqw5SpQDszvxp4LyqviNLIQ,1027
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/normalizer/normalize_utils.py,sha256=w9f_9udrwqbhXgFMTs2keuce-6X_j6h3D7EdNo_2X7g,8493
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=KcXm1Nl6c3zswL91tIbG0DjuuNpzxUdCg1cY9f2-9cg,29283
+upgini/resource_bundle/strings.properties,sha256=3aK2sxXYuvSLuoOyLq8IcyekfINH0Il5nLvVXMsuEpY,29353
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=Fva2FEhLiNRPZ9Q6uOtJRtRzwsayjv7aphalAZO_4lc,6452
@@ -54,10 +54,10 @@ upgini/utils/custom_loss_utils.py,sha256=kieNZYBYZm5ZGBltF1F_jOSF4ea6C29rYuCyiDc
 upgini/utils/cv_utils.py,sha256=w6FQb9nO8BWDx88EF83NpjPLarK4eR4ia0Wg0kLBJC4,3525
 upgini/utils/datetime_utils.py,sha256=l85UzSQLhtMeI2G6m-m8y8bCColCLSXNHb2-G6fKpLM,16988
 upgini/utils/deduplicate_utils.py,sha256=6czbn1q0p-lOmrNvbAzueBpDHmfIP4TfV4poWqbjX5w,11255
-upgini/utils/display_utils.py,sha256=uSG3JwpwCIgRJXsp-8ktuJ0Dh-WFti7IrRLMUfHfoDc,11973
+upgini/utils/display_utils.py,sha256=p6o0VlYtGpU6bXv3B-fjQM9PeZEkl05OylHXSRyP0us,13219
 upgini/utils/email_utils.py,sha256=pZ2vCfNxLIPUhxr0-OlABNXm12jjU44isBk8kGmqQzA,5277
 upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0--cOFTwJ0,1074
-upgini/utils/feature_info.py,sha256=6vihytwKma_TlXtTn4l6Aj4kqlOj0ouLy-yWVV6VUw8,7551
+upgini/utils/feature_info.py,sha256=SQTRbSxJDkh2G2c0KGBmOv8f69gVzWbTtcXn0_2Qb-8,7945
 upgini/utils/features_validator.py,sha256=A_3AX7X5u5AH7RLgkTiS6dHxaOiq5vm8w4ijQWLGcMY,4871
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
 upgini/utils/hash_utils.py,sha256=mP2yHyzvDNdpa5g3B4MHzulxBeEz_ZSoGl1YF_VnAyE,5538
@@ -74,7 +74,7 @@ upgini/utils/target_utils.py,sha256=GCPn4QeJ83JJ_vyBJ3IhY5fyIRkLC9q9BE59S2FRO1I,
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.125.dist-info/METADATA,sha256=CAoP8m15syLZEVmnYuUjUMI1Jo-XvMCGhz-CZnRYwy4,50781
-upgini-1.2.125.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.125.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.125.dist-info/RECORD,,
+upgini-1.2.127.dist-info/METADATA,sha256=KaZiSMDjzxqjhOoh3zY_EH9-kwLTMy71Us_ge2j-YyM,50781
+upgini-1.2.127.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.127.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.127.dist-info/RECORD,,