upgini 1.2.77__py3-none-any.whl → 1.2.78__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.77"
+__version__ = "1.2.78"

upgini/features_enricher.py

@@ -757,7 +757,8 @@ class FeaturesEnricher(TransformerMixin):
             if new_progress:
                 progress_bar.display()
         trace_id = trace_id or str(uuid.uuid4())
-        with MDC(trace_id=trace_id):
+        search_id = self.search_id or (self._search_task.search_task_id if self._search_task is not None else None)
+        with MDC(trace_id=trace_id, search_id=search_id):
             self.dump_input(trace_id, X)
             if len(args) > 0:
                 msg = f"WARNING: Unsupported positional arguments for transform: {args}"
@@ -887,7 +888,8 @@ class FeaturesEnricher(TransformerMixin):
 
         trace_id = trace_id or str(uuid.uuid4())
         start_time = time.time()
-        with MDC(trace_id=trace_id):
+        search_id = self.search_id or (self._search_task.search_task_id if self._search_task is not None else None)
+        with MDC(trace_id=trace_id, search_id=search_id):
             self.logger.info("Start calculate metrics")
             if len(args) > 0:
                 msg = f"WARNING: Unsupported positional arguments for calculate_metrics: {args}"
@@ -2199,7 +2201,8 @@ if response.status_code == 200:
             raise NotFittedError(self.bundle.get("transform_unfitted_enricher"))
 
         start_time = time.time()
-        with MDC(trace_id=trace_id):
+        search_id = self.search_id or (self._search_task.search_task_id if self._search_task is not None else None)
+        with MDC(trace_id=trace_id, search_id=search_id):
             self.logger.info("Start transform")
 
             validated_X = self._validate_X(X, is_transform=True)
@@ -2473,7 +2476,7 @@ if response.status_code == 200:
             gc.collect()
 
             if not silent_mode:
-                print(self.bundle.get("polling_search_task").format(validation_task.search_task_id))
+                print(self.bundle.get("polling_transform_task").format(validation_task.search_task_id))
                 if not self.__is_registered:
                     print(self.bundle.get("polling_unregister_information"))
 
@@ -2989,137 +2992,141 @@ if response.status_code == 200:
             auto_fe_parameters=auto_fe_parameters,
         )
 
-        if search_id_callback is not None:
-            search_id_callback(self._search_task.search_task_id)
+        with MDC(search_id=self._search_task.search_task_id):
 
-        print(self.bundle.get("polling_search_task").format(self._search_task.search_task_id))
-        if not self.__is_registered:
-            print(self.bundle.get("polling_unregister_information"))
+            if search_id_callback is not None:
+                search_id_callback(self._search_task.search_task_id)
 
-        progress = self.get_progress(trace_id)
-        prev_progress = None
-        progress.recalculate_eta(time.time() - start_time)
-        if progress_bar is not None:
-            progress_bar.progress = progress.to_progress_bar()
-        if progress_callback is not None:
-            progress_callback(progress)
-        poll_period_seconds = 1
-        try:
-            while progress.stage != ProgressStage.GENERATING_REPORT.value:
-                if prev_progress is None or prev_progress.percent != progress.percent:
-                    progress.recalculate_eta(time.time() - start_time)
-                else:
-                    progress.update_eta(prev_progress.eta - poll_period_seconds)
-                prev_progress = progress
-                if progress_bar is not None:
-                    progress_bar.progress = progress.to_progress_bar()
-                if progress_callback is not None:
-                    progress_callback(progress)
-                if progress.stage == ProgressStage.FAILED.value:
-                    self.logger.error(
-                        f"Search {self._search_task.search_task_id} failed with error {progress.error}"
-                        f" and message {progress.error_message}"
-                    )
-                    raise RuntimeError(self.bundle.get("search_task_failed_status"))
-                time.sleep(poll_period_seconds)
-                progress = self.get_progress(trace_id)
-        except KeyboardInterrupt as e:
-            print(self.bundle.get("search_stopping"))
-            self.rest_client.stop_search_task_v2(trace_id, self._search_task.search_task_id)
-            self.logger.warning(f"Search {self._search_task.search_task_id} stopped by user")
-            print(self.bundle.get("search_stopped"))
-            raise e
-
-        self._search_task.poll_result(trace_id, quiet=True)
-
-        seconds_left = time.time() - start_time
-        progress = SearchProgress(97.0, ProgressStage.GENERATING_REPORT, seconds_left)
-        if progress_bar is not None:
-            progress_bar.progress = progress.to_progress_bar()
-        if progress_callback is not None:
-            progress_callback(progress)
+            print(self.bundle.get("polling_search_task").format(self._search_task.search_task_id))
+            if not self.__is_registered:
+                print(self.bundle.get("polling_unregister_information"))
 
-        self.imbalanced = dataset.imbalanced
+            progress = self.get_progress(trace_id)
+            prev_progress = None
+            progress.recalculate_eta(time.time() - start_time)
+            if progress_bar is not None:
+                progress_bar.progress = progress.to_progress_bar()
+            if progress_callback is not None:
+                progress_callback(progress)
+            poll_period_seconds = 1
+            try:
+                while progress.stage != ProgressStage.GENERATING_REPORT.value:
+                    if prev_progress is None or prev_progress.percent != progress.percent:
+                        progress.recalculate_eta(time.time() - start_time)
+                    else:
+                        progress.update_eta(prev_progress.eta - poll_period_seconds)
+                    prev_progress = progress
+                    if progress_bar is not None:
+                        progress_bar.progress = progress.to_progress_bar()
+                    if progress_callback is not None:
+                        progress_callback(progress)
+                    if progress.stage == ProgressStage.FAILED.value:
+                        self.logger.error(
+                            f"Search {self._search_task.search_task_id} failed with error {progress.error}"
+                            f" and message {progress.error_message}"
+                        )
+                        raise RuntimeError(self.bundle.get("search_task_failed_status"))
+                    time.sleep(poll_period_seconds)
+                    progress = self.get_progress(trace_id)
+            except KeyboardInterrupt as e:
+                print(self.bundle.get("search_stopping"))
+                self.rest_client.stop_search_task_v2(trace_id, self._search_task.search_task_id)
+                self.logger.warning(f"Search {self._search_task.search_task_id} stopped by user")
+                print(self.bundle.get("search_stopped"))
+                raise e
 
-        zero_hit_search_keys = self._search_task.get_zero_hit_rate_search_keys()
-        if zero_hit_search_keys:
-            self.logger.warning(
-                f"Intersections with this search keys are empty for all datasets: {zero_hit_search_keys}"
-            )
-            zero_hit_columns = self.get_columns_by_search_keys(zero_hit_search_keys)
-            if zero_hit_columns:
-                msg = self.bundle.get("features_info_zero_hit_rate_search_keys").format(zero_hit_columns)
-                self.__log_warning(msg, show_support_link=True)
+            self._search_task.poll_result(trace_id, quiet=True)
 
-        if (
-            self._search_task.unused_features_for_generation is not None
-            and len(self._search_task.unused_features_for_generation) > 0
-        ):
-            unused_features_for_generation = [
-                dataset.columns_renaming.get(col) or col for col in self._search_task.unused_features_for_generation
-            ]
-            msg = self.bundle.get("features_not_generated").format(unused_features_for_generation)
-            self.__log_warning(msg)
+            seconds_left = time.time() - start_time
+            progress = SearchProgress(97.0, ProgressStage.GENERATING_REPORT, seconds_left)
+            if progress_bar is not None:
+                progress_bar.progress = progress.to_progress_bar()
+            if progress_callback is not None:
+                progress_callback(progress)
 
-        self.__prepare_feature_importances(trace_id, df)
+            self.imbalanced = dataset.imbalanced
 
-        self.__show_selected_features(self.fit_search_keys)
+            zero_hit_search_keys = self._search_task.get_zero_hit_rate_search_keys()
+            if zero_hit_search_keys:
+                self.logger.warning(
+                    f"Intersections with this search keys are empty for all datasets: {zero_hit_search_keys}"
+                )
+                zero_hit_columns = self.get_columns_by_search_keys(zero_hit_search_keys)
+                if zero_hit_columns:
+                    msg = self.bundle.get("features_info_zero_hit_rate_search_keys").format(zero_hit_columns)
+                    self.__log_warning(msg, show_support_link=True)
 
-        autofe_description = self.get_autofe_features_description()
-        if autofe_description is not None:
-            self.logger.info(f"AutoFE descriptions: {autofe_description}")
-            self.autofe_features_display_handle = display_html_dataframe(
-                df=autofe_description,
-                internal_df=autofe_description,
-                header=self.bundle.get("autofe_descriptions_header"),
-                display_id=f"autofe_descriptions_{uuid.uuid4()}",
-            )
+            if (
+                self._search_task.unused_features_for_generation is not None
+                and len(self._search_task.unused_features_for_generation) > 0
+            ):
+                unused_features_for_generation = [
+                    dataset.columns_renaming.get(col) or col for col in self._search_task.unused_features_for_generation
+                ]
+                msg = self.bundle.get("features_not_generated").format(unused_features_for_generation)
+                self.__log_warning(msg)
 
-        if self._has_paid_features(exclude_features_sources):
-            if calculate_metrics is not None and calculate_metrics:
-                msg = self.bundle.get("metrics_with_paid_features")
-                self.logger.warning(msg)
-                self.__display_support_link(msg)
-        else:
-            if (scoring is not None or estimator is not None) and calculate_metrics is None:
-                calculate_metrics = True
+            self.__prepare_feature_importances(trace_id, df)
 
-            if calculate_metrics is None:
-                if len(validated_X) < self.CALCULATE_METRICS_MIN_THRESHOLD or any(
-                    [len(eval_X) < self.CALCULATE_METRICS_MIN_THRESHOLD for eval_X, _ in validated_eval_set]
-                ):
-                    msg = self.bundle.get("too_small_for_metrics")
+            self.__show_selected_features(self.fit_search_keys)
+
+            autofe_description = self.get_autofe_features_description()
+            if autofe_description is not None:
+                self.logger.info(f"AutoFE descriptions: {autofe_description}")
+                self.autofe_features_display_handle = display_html_dataframe(
+                    df=autofe_description,
+                    internal_df=autofe_description,
+                    header=self.bundle.get("autofe_descriptions_header"),
+                    display_id=f"autofe_descriptions_{uuid.uuid4()}",
+                )
+
+            if self._has_paid_features(exclude_features_sources):
+                if calculate_metrics is not None and calculate_metrics:
+                    msg = self.bundle.get("metrics_with_paid_features")
                     self.logger.warning(msg)
-                    calculate_metrics = False
-                elif len(dataset) * len(dataset.columns) > self.CALCULATE_METRICS_THRESHOLD:
-                    self.logger.warning("Too big dataset for automatic metrics calculation")
-                    calculate_metrics = False
-                else:
+                    self.__display_support_link(msg)
+            else:
+                if (scoring is not None or estimator is not None) and calculate_metrics is None:
                     calculate_metrics = True
 
-            del df, validated_X, validated_y, dataset
-            gc.collect()
+                if calculate_metrics is None:
+                    if len(validated_X) < self.CALCULATE_METRICS_MIN_THRESHOLD or any(
+                        [len(eval_X) < self.CALCULATE_METRICS_MIN_THRESHOLD for eval_X, _ in validated_eval_set]
+                    ):
+                        msg = self.bundle.get("too_small_for_metrics")
+                        self.logger.warning(msg)
+                        calculate_metrics = False
+                    elif len(dataset) * len(dataset.columns) > self.CALCULATE_METRICS_THRESHOLD:
+                        self.logger.warning("Too big dataset for automatic metrics calculation")
+                        calculate_metrics = False
+                    else:
+                        calculate_metrics = True
 
-            if calculate_metrics:
-                try:
-                    self.__show_metrics(
-                        scoring,
-                        estimator,
-                        importance_threshold,
-                        max_features,
-                        remove_outliers_calc_metrics,
-                        trace_id,
-                        progress_bar,
-                        progress_callback,
-                    )
-                except Exception:
-                    self.report_button_handle = self.__show_report_button(display_id=f"report_button_{uuid.uuid4()}")
-                    raise
+                del df, validated_X, validated_y, dataset
+                gc.collect()
+
+                if calculate_metrics:
+                    try:
+                        self.__show_metrics(
+                            scoring,
+                            estimator,
+                            importance_threshold,
+                            max_features,
+                            remove_outliers_calc_metrics,
+                            trace_id,
+                            progress_bar,
+                            progress_callback,
+                        )
+                    except Exception:
+                        self.report_button_handle = self.__show_report_button(
+                            display_id=f"report_button_{uuid.uuid4()}"
+                        )
+                        raise
 
-        self.report_button_handle = self.__show_report_button(display_id=f"report_button_{uuid.uuid4()}")
+            self.report_button_handle = self.__show_report_button(display_id=f"report_button_{uuid.uuid4()}")
 
-        if not self.warning_counter.has_warnings():
-            self.__display_support_link(self.bundle.get("all_ok_community_invite"))
+            if not self.warning_counter.has_warnings():
+                self.__display_support_link(self.bundle.get("all_ok_community_invite"))
 
     def __should_add_date_column(self):
         return self.add_date_if_missing or (self.cv is not None and self.cv.is_time_series())
@@ -3443,7 +3450,8 @@ if response.status_code == 200:
                 f"Estimator: {estimator}\n"
                 f"Remove target outliers: {remove_outliers_calc_metrics}\n"
                 f"Exclude columns: {self.exclude_columns}\n"
-                f"Search id: {self.search_id}\n"
+                f"Passed search id: {self.search_id}\n"
+                f"Search id from search task: {self._search_task.search_task_id if self._search_task else None}\n"
                 f"Custom loss: {self.loss}\n"
                 f"Logs enabled: {self.logs_enabled}\n"
                 f"Raise validation error: {self.raise_validation_error}\n"

upgini/http.py

@@ -125,6 +125,8 @@ class SearchTaskSummary:
         self.search_task_id = response["searchTaskId"]
         self.file_upload_id = response["fileUploadTaskId"]
         self.status = response["searchTaskStatus"]
+        self.task_type = response.get("searchTaskType")
+        self.initial_search_task_id = response.get("initialSearchTaskId")
         # self.features_found = response["featuresFoundCount"]
         # self.providers_checked = response["providersCheckedCount"]
         # self.important_providers_count = response["importantProvidersCount"]

upgini/resource_bundle/strings.properties

@@ -7,6 +7,7 @@ transform_start=Retrieving selected features from data sources...
 search_stopping=Search interrupted. Stopping search request
 search_stopped=Search request stopped
 polling_search_task=\nRunning search request, search_id={}
+polling_transform_task=\nRunning transform request, id={}
 polling_unregister_information=We'll send email notification once it's completed, just use your personal api_key from profile.upgini.com
 ads_upload_finish=Thank you for your submission!\nWe'll check your data sharing proposal and get back to you
 demo_dataset_info=Demo training dataset detected. Registration for an API key is not required.\n
@@ -67,6 +68,7 @@ unsupported_search_key=Search key {} not supported
 too_many_generate_features=Too many columns passed in `generate_features` argument. Only {} columns supported to generate features now
 invalid_round_embeddings=Argument `round_embeddings` should be non negative integer
 no_important_features_for_transform=There are no important features for transform. Return input as transformed
+search_task_not_initial=Passed search_id {} is transform id. Please use search task id of fit call: {}.
 
 # Validation errors
     # params validation

upgini/search_task.py

@@ -1,9 +1,9 @@
 import logging
 import tempfile
 import time
+import uuid
 from functools import lru_cache
 from typing import Dict, List, Optional
-import uuid
 
 import pandas as pd
 
@@ -116,6 +116,11 @@ class SearchTask:
                 self.logger.error(f"Search failed with errors: {','.join(error_messages)}")
                 raise RuntimeError(bundle.get("all_providers_failed_with_error").format(",".join(error_messages)))
 
+        if check_fit and self.summary.task_type != "INITIAL":
+            raise RuntimeError(
+                bundle.get("search_task_not_initial").format(self.search_task_id, self.summary.initial_search_task_id)
+            )
+
         if self.summary.status in ["COMPLETED", "VALIDATION_COMPLETED"] or (
             check_fit and "VALIDATION" in self.summary.status
         ):

{upgini-1.2.77.dist-info → upgini-1.2.78.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.77
+Version: 1.2.78
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.77.dist-info → upgini-1.2.78.dist-info}/RECORD

@@ -1,13 +1,13 @@
-upgini/__about__.py,sha256=O1xqGt2rgTjvj07mjY1zhjZkS-9n0t1f6_sESHXFb5A,23
+upgini/__about__.py,sha256=um-GnDBOmvXVFDzFeMIfwSF9jG0V0HqCNhYlIkfUoOQ,23
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=aspri7ZAgwkNNUiIgQ1GRXvw8XQii3F4RfNXSrF4wrw,35365
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=M_QxI7-yrDTXGpVEjMhjwR8yvIZh8Vxj9_63VaBZLWM,207132
-upgini/http.py,sha256=RvzcShpDXssLs6ycGN8xilkKi8ZV9XGUrrk8bwdUzbw,43607
+upgini/features_enricher.py,sha256=_UkJS35uGaYtI7dR6Xd9Q28nmiPzTjhK3y8v3IjJTfQ,208245
+upgini/http.py,sha256=UH7nswcZ221un3O_VW9limCBO5oRsyg1eKUHiVslRPs,43737
 upgini/metadata.py,sha256=Yd6iW2f7Wz6vUkg5uvR4xylN16ANnCKVKqAsAkap7p8,12354
 upgini/metrics.py,sha256=1AgIPEzd4sGJ8AlMrJziQruT-M1-lGgpnDIXUYeQJ6Q,39171
-upgini/search_task.py,sha256=EuCGp0iCWz2fpuJgN6M47aP_CtIi3Oq9zw78w0mkKiU,17595
+upgini/search_task.py,sha256=RcvAE785yksWTsTNWuZFVNlk32jHElMoEna1T_C5N8Q,17823
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
 upgini/version_validator.py,sha256=DvbaAvuYFoJqYt0fitpsk6Xcv-H1BYDJYHUMxaKSH_Y,1509
 upgini/ads_management/__init__.py,sha256=qzyisOToVRP-tquAJD1PblZhNtMrOB8FiyF9JvfkvgE,50
@@ -38,7 +38,7 @@ upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 upgini/normalizer/normalize_utils.py,sha256=Ft2MwSgVoBilXAORAOYAuwPD79GOLfwn4qQE3IUFzzg,7218
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=mwQrerdJj3adzT-fHqvs6Qjf-rqDccsUzELDIXJKAmY,27791
+upgini/resource_bundle/strings.properties,sha256=o9iLgAJSxXm6mkQgtNwBVQKIRVwLF1__Dn9gSXb1kLY,27953
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=7GpjYqjOp58vYcJLiX__1R5wjUlyQbxvHJ2klFnup_M,6389
@@ -70,7 +70,7 @@ upgini/utils/target_utils.py,sha256=P0cCVRaakWLydYwFjk3TEaQfr0p0hfsJCvKRD8qcxiE,
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.77.dist-info/METADATA,sha256=75-l6Mbzg2HoUaocvJJO79ZL6rb54E42qmZ5Ucf88Cs,49091
-upgini-1.2.77.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.77.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.77.dist-info/RECORD,,
+upgini-1.2.78.dist-info/METADATA,sha256=9NDKzhMak5vKoLrxoAAfTjU-HsTJByudWztn1gxxJyA,49091
+upgini-1.2.78.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.78.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.78.dist-info/RECORD,,