upgini 1.2.45__py3-none-any.whl → 1.2.47__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.45"
+__version__ = "1.2.47"

upgini/dataset.py

@@ -646,7 +646,7 @@ class Dataset:  # (pd.DataFrame):
                 parquet_file_path = self.prepare_uploading_file(tmp_dir)
                 time.sleep(1)  # this is neccesary to avoid requests rate limit restrictions
                 # If previous steps were too fast, time estimation could be calculated incorrectly
-                time_left = max(time.time() - start_time, 20)
+                time_left = max(time.time() - start_time, 20.0)
                 search_progress = SearchProgress(1.0, ProgressStage.CREATING_FIT, time_left)
                 if progress_bar is not None:
                     progress_bar.progress = search_progress.to_progress_bar()
@@ -699,7 +699,7 @@ class Dataset:  # (pd.DataFrame):
             runtime_parameters=runtime_parameters,
             metrics_calculation=metrics_calculation,
         )
-        seconds_left = max(time.time() - start_time, 20)
+        seconds_left = max(time.time() - start_time, 20.0)
         search_progress = SearchProgress(1.0, ProgressStage.CREATING_TRANSFORM, seconds_left)
         if progress_bar is not None:
             progress_bar.progress = search_progress.to_progress_bar()

upgini/features_enricher.py

@@ -165,10 +165,6 @@ class FeaturesEnricher(TransformerMixin):
 
     shared_datasets: list of str, optional (default=None)
         List of private shared dataset ids for custom search
-
-    select_features: bool, optional (default=False)
-        If True, return only selected features both from input and data sources.
-        Otherwise, return all features from input and only selected features from data sources.
     """
 
     TARGET_NAME = "target"
@@ -235,7 +231,6 @@ class FeaturesEnricher(TransformerMixin):
         client_visitorid: Optional[str] = None,
         custom_bundle_config: Optional[str] = None,
         add_date_if_missing: bool = True,
-        select_features: bool = False,
         disable_force_downsampling: bool = False,
         id_columns: Optional[List[str]] = None,
         **kwargs,
@@ -273,6 +268,7 @@ class FeaturesEnricher(TransformerMixin):
         self.eval_set: Optional[List[Tuple]] = None
         self.autodetected_search_keys: Dict[str, SearchKey] = {}
         self.imbalanced = False
+        self.fit_select_features = False
         self.__cached_sampled_datasets: Dict[str, Tuple[pd.DataFrame, pd.DataFrame, pd.Series, Dict, Dict, Dict]] = (
             dict()
         )
@@ -297,7 +293,6 @@ class FeaturesEnricher(TransformerMixin):
         self.dropped_client_feature_names_ = []
         self.feature_importances_ = []
         self.search_id = search_id
-        self.select_features = select_features
         self.disable_force_downsampling = disable_force_downsampling
 
         if search_id:
@@ -405,6 +400,7 @@ class FeaturesEnricher(TransformerMixin):
         remove_outliers_calc_metrics: Optional[bool] = None,
         progress_callback: Optional[Callable[[SearchProgress], Any]] = None,
         search_id_callback: Optional[Callable[[str], Any]] = None,
+        select_features: bool = False,
         **kwargs,
     ):
         """Fit to data.
@@ -440,6 +436,10 @@ class FeaturesEnricher(TransformerMixin):
 
         remove_outliers_calc_metrics, optional (default=True)
             If True then rows with target ouliers will be dropped on metrics calculation
+
+        select_features: bool, optional (default=False)
+            If True, return only selected features both from input and data sources.
+            Otherwise, return all features from input and only selected features from data sources.
         """
         trace_id = str(uuid.uuid4())
         start_time = time.time()
@@ -474,6 +474,7 @@ class FeaturesEnricher(TransformerMixin):
                 self.y = y
                 self.eval_set = self._check_eval_set(eval_set, X, self.bundle)
                 self.dump_input(trace_id, X, y, self.eval_set)
+                self.__set_select_features(select_features)
                 self.__inner_fit(
                     trace_id,
                     X,
@@ -523,6 +524,10 @@ class FeaturesEnricher(TransformerMixin):
             finally:
                 self.logger.info(f"Fit elapsed time: {time.time() - start_time}")
 
+    def __set_select_features(self, select_features: bool):
+        self.fit_select_features = select_features
+        self.runtime_parameters.properties["select_features"] = select_features
+
     def fit_transform(
         self,
         X: Union[pd.DataFrame, pd.Series, np.ndarray],
@@ -538,6 +543,7 @@ class FeaturesEnricher(TransformerMixin):
         estimator: Optional[Any] = None,
         remove_outliers_calc_metrics: Optional[bool] = None,
         progress_callback: Optional[Callable[[SearchProgress], Any]] = None,
+        select_features: bool = False,
         **kwargs,
     ) -> pd.DataFrame:
         """Fit to data, then transform it.
@@ -578,6 +584,10 @@ class FeaturesEnricher(TransformerMixin):
         remove_outliers_calc_metrics, optional (default=True)
             If True then rows with target ouliers will be dropped on metrics calculation
 
+        select_features: bool, optional (default=False)
+            If True, return only selected features both from input and data sources.
+            Otherwise, return all features from input and only selected features from data sources.
+
         Returns
         -------
         X_new: pandas.DataFrame of shape (n_samples, n_features_new)
@@ -613,6 +623,7 @@ class FeaturesEnricher(TransformerMixin):
                 self.X = X
                 self.y = y
                 self.eval_set = self._check_eval_set(eval_set, X, self.bundle)
+                self.__set_select_features(select_features)
                 self.dump_input(trace_id, X, y, self.eval_set)
 
                 if _num_samples(drop_duplicates(X)) > Dataset.MAX_ROWS:
@@ -1096,7 +1107,8 @@ class FeaturesEnricher(TransformerMixin):
                     ):
                         train_metrics[self.bundle.get("quality_metrics_mean_target_header")] = round(
                             # np.mean(validated_y), 4
-                            np.mean(y_sorted), 4
+                            np.mean(y_sorted),
+                            4,
                         )
                     if etalon_metric is not None:
                         train_metrics[self.bundle.get("quality_metrics_baseline_header").format(metric)] = etalon_metric
@@ -1174,7 +1186,8 @@ class FeaturesEnricher(TransformerMixin):
                                 eval_metrics[self.bundle.get("quality_metrics_mean_target_header")] = round(
                                     # np.mean(validated_eval_set[idx][1]), 4
                                     # Use actually used for metrics dataset
-                                    np.mean(eval_y_sorted), 4
+                                    np.mean(eval_y_sorted),
+                                    4,
                                 )
                             if etalon_eval_metric is not None:
                                 eval_metrics[self.bundle.get("quality_metrics_baseline_header").format(metric)] = (
@@ -1238,8 +1251,11 @@ class FeaturesEnricher(TransformerMixin):
                 self.logger.info(f"Calculating metrics elapsed time: {time.time() - start_time}")
 
     def _update_shap_values(self, trace_id: str, x_columns: List[str], new_shaps: Dict[str, float]):
+        renaming = self.fit_columns_renaming or {}
         new_shaps = {
-            feature: _round_shap_value(shap) for feature, shap in new_shaps.items() if feature in self.feature_names_
+            renaming.get(feature, feature): _round_shap_value(shap)
+            for feature, shap in new_shaps.items()
+            if feature in self.feature_names_ or renaming.get(feature, feature) in self.feature_names_
         }
         self.__prepare_feature_importances(trace_id, x_columns, new_shaps, silent=True)
 
@@ -1458,7 +1474,8 @@ class FeaturesEnricher(TransformerMixin):
         if search_keys_for_metrics is not None and len(search_keys_for_metrics) > 0:
             excluded = set()
             for sk in excluding_search_keys:
-                if columns_renaming.get(sk) in search_keys_for_metrics:
+                renamed_sk = columns_renaming.get(sk)
+                if renamed_sk in search_keys_for_metrics or renamed_sk in self.feature_names_:
                     excluded.add(sk)
             excluding_search_keys = [sk for sk in excluding_search_keys if sk not in excluded]
 
@@ -1468,7 +1485,7 @@ class FeaturesEnricher(TransformerMixin):
             c
             for c in X_sampled.columns.to_list()
             if (
-                not self.select_features
+                not self.fit_select_features
                 or c in self.feature_names_
                 or (self.fit_columns_renaming is not None and self.fit_columns_renaming.get(c) in self.feature_names_)
             )
@@ -3315,8 +3332,8 @@ if response.status_code == 200:
                 f"Client ip: {self.client_ip}\n"
                 f"Client visitorId: {self.client_visitorid}\n"
                 f"Add date if missing: {self.add_date_if_missing}\n"
-                f"Select features: {self.select_features}\n"
                 f"Disable force downsampling: {self.disable_force_downsampling}\n"
+                f"Id columns: {self.id_columns}\n"
             )
 
             def sample(df):
@@ -3703,7 +3720,7 @@ if response.status_code == 200:
             is_client_feature = feature_meta.name in x_columns
 
             if feature_meta.shap_value == 0.0:
-                if self.select_features:
+                if self.fit_select_features:
                     self.dropped_client_feature_names_.append(feature_meta.name)
                 continue
 
@@ -3712,7 +3729,7 @@ if response.status_code == 200:
                 feature_meta.name in self.fit_generated_features
                 or feature_meta.name == COUNTRY
                 # In select_features mode we select also from etalon features and need to show them
-                or (not self.select_features and is_client_feature)
+                or (not self.fit_select_features and is_client_feature)
             ):
                 continue
 

upgini/http.py

@@ -16,7 +16,7 @@ from typing import Any, Dict, List, Optional, Tuple
 from urllib.parse import urljoin
 
 import jwt
-import pandas as pd
+# import pandas as pd
 import requests
 from pydantic import BaseModel
 from pythonjsonlogger import jsonlogger
@@ -422,6 +422,16 @@ class _RestClient:
                     lambda: self._send_post_file_req_v2(api_path, files, trace_id=trace_id, need_json_response=False)
                 )
 
+    @staticmethod
+    def compute_file_digest(filepath: str, algorithm="sha256", chunk_size=4096) -> str:
+        hash_func = getattr(hashlib, algorithm)()
+
+        with open(filepath, "rb") as f:
+            for chunk in iter(lambda: f.read(chunk_size), b""):
+                hash_func.update(chunk)
+
+        return hash_func.hexdigest()
+
     def initial_search_v2(
         self,
         trace_id: str,
@@ -442,9 +452,10 @@ class _RestClient:
                 digest = md5_hash.hexdigest()
                 metadata_with_md5 = metadata.copy(update={"checksumMD5": digest})
 
-            digest_sha256 = hashlib.sha256(
-                pd.util.hash_pandas_object(pd.read_parquet(file_path, engine="fastparquet")).values
-            ).hexdigest()
+            # digest_sha256 = hashlib.sha256(
+            #     pd.util.hash_pandas_object(pd.read_parquet(file_path, engine="fastparquet")).values
+            # ).hexdigest()
+            digest_sha256 = self.compute_file_digest(file_path)
             metadata_with_md5 = metadata_with_md5.copy(update={"digest": digest_sha256})
 
             with open(file_path, "rb") as file:
@@ -530,9 +541,10 @@ class _RestClient:
                 digest = md5_hash.hexdigest()
                 metadata_with_md5 = metadata.copy(update={"checksumMD5": digest})
 
-            digest_sha256 = hashlib.sha256(
-                pd.util.hash_pandas_object(pd.read_parquet(file_path, engine="fastparquet")).values
-            ).hexdigest()
+            # digest_sha256 = hashlib.sha256(
+            #     pd.util.hash_pandas_object(pd.read_parquet(file_path, engine="fastparquet")).values
+            # ).hexdigest()
+            digest_sha256 = self.compute_file_digest(file_path)
             metadata_with_md5 = metadata_with_md5.copy(update={"digest": digest_sha256})
 
             with open(file_path, "rb") as file:

{upgini-1.2.45.dist-info → upgini-1.2.47.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.45
+Version: 1.2.47
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.45.dist-info → upgini-1.2.47.dist-info}/RECORD

@@ -1,10 +1,10 @@
-upgini/__about__.py,sha256=TdZZy40X_55kxtHKjwbGqPrDr5qQe_uUz_Fvup_EuNI,23
+upgini/__about__.py,sha256=o2NRe9gScRz1I1oB_R5MjkQ4w7BrDovQP2Z_Mq2c6bo,23
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
-upgini/dataset.py,sha256=WfCg7x_HREGwbFNlF-CTSk-KZzkGYlT7PtZdiJfuzLM,33452
+upgini/dataset.py,sha256=QC3jncWS3wHe4CY7pWWDMO_3HKxGbi0EyPHXMdBtoQM,33456
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=QQaK682uFjrkvDOt-ub7UFMAEy6SVjPQubb6dY_7moE,199109
-upgini/http.py,sha256=plZGTGoi1h2edd8Cnjt4eYB8t4NbBGnZz7DtPTByiNc,42885
+upgini/features_enricher.py,sha256=NWYNZtSgAR05zOZp_Wq1ltVGThCttTbVN_TP2RaWFSI,200008
+upgini/http.py,sha256=danPeX7nTMa_70S-pk-4UUm5yOvXYlR84jgyjoHYBkU,43367
 upgini/lazy_import.py,sha256=74gQ8JuA48BGRLxAo7lNHNKY2D2emMxrUxKGdxVGhuY,1012
 upgini/metadata.py,sha256=-ibqiNjD7dTagqg53FoEJNEqvAYbwgfyn9PGTRQ_YKU,12054
 upgini/metrics.py,sha256=hr7UwLphbZ_FEglLuO2lzr_pFgxOJ4c3WBeg7H-fNqY,35521
@@ -59,7 +59,7 @@ upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,4
 upgini/utils/target_utils.py,sha256=RlpKGss9kMibVSlA8iZuO_qxmyeplqzn7X8g6hiGGGs,14341
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.45.dist-info/METADATA,sha256=nv_TxkRQegnVyCPp8Wj5iyVbA08rQ2FWTKdoPTm20Gg,49055
-upgini-1.2.45.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.45.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.45.dist-info/RECORD,,
+upgini-1.2.47.dist-info/METADATA,sha256=4pKaboM3TxupdS6iw1Uh_IW9Dw0X88LnDh1pGjsc3fs,49055
+upgini-1.2.47.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.47.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.47.dist-info/RECORD,,