upgini 1.2.113a3974.dev2__py3-none-any.whl → 1.2.114__py3-none-any.whl

upgini/http.py

@@ -45,6 +45,7 @@ from upgini.metadata import (
     SearchCustomization,
 )
 from upgini.resource_bundle import bundle
+from upgini.utils.hash_utils import file_hash
 from upgini.utils.track_info import get_track_metrics
 
 UPGINI_URL: str = "UPGINI_URL"
@@ -276,6 +277,7 @@ class _RestClient:
     SEARCH_DUMP_INPUT_FMT_V2 = SERVICE_ROOT_V2 + "search/dump-input"
     SEARCH_DUMP_INPUT_FILE_FMT = SERVICE_ROOT_V2 + "search/dump-input-file?digest={0}"
     TRANSFORM_USAGE_FMT = SERVICE_ROOT_V2 + "user/transform-usage"
+    SEARCH_SELECTED_FEATURES_URI_FMT = SERVICE_ROOT_V2 + "search/{0}/selected-features"
 
     UPLOAD_USER_ADS_URI = SERVICE_ROOT + "ads/upload"
     SEND_LOG_EVENT_URI = "private/api/v2/events/send"
@@ -427,7 +429,7 @@ class _RestClient:
         api_path = self.SEARCH_DUMP_INPUT_FILE_FMT
 
         def upload_with_check(path: str, file_name: str):
-            digest_sha256 = self.compute_file_digest(path)
+            digest_sha256 = file_hash(path)
             if self.is_file_uploaded(trace_id, digest_sha256):
                 # print(f"File {path} was already uploaded with digest {digest_sha256}, skipping")
                 return
@@ -448,16 +450,6 @@ class _RestClient:
         if eval_y_path:
             upload_with_check(eval_y_path, "eval_y.parquet")
 
-    @staticmethod
-    def compute_file_digest(filepath: str, algorithm="sha256", chunk_size=4096) -> str:
-        hash_func = getattr(hashlib, algorithm)()
-
-        with open(filepath, "rb") as f:
-            for chunk in iter(lambda: f.read(chunk_size), b""):
-                hash_func.update(chunk)
-
-        return hash_func.hexdigest()
-
     def initial_search_v2(
         self,
         trace_id: str,
@@ -478,10 +470,7 @@ class _RestClient:
                 digest = md5_hash.hexdigest()
                 metadata_with_md5 = pydantic_copy_method(metadata)(update={"checksumMD5": digest})
 
-            # digest_sha256 = hashlib.sha256(
-            #     pd.util.hash_pandas_object(pd.read_parquet(file_path, engine="fastparquet")).values
-            # ).hexdigest()
-            digest_sha256 = self.compute_file_digest(file_path)
+            digest_sha256 = file_hash(file_path)
             metadata_with_md5 = pydantic_copy_method(metadata_with_md5)(update={"digest": digest_sha256})
 
             with open(file_path, "rb") as file:
@@ -576,10 +565,7 @@ class _RestClient:
                 digest = md5_hash.hexdigest()
                 metadata_with_md5 = pydantic_copy_method(metadata)(update={"checksumMD5": digest})
 
-            # digest_sha256 = hashlib.sha256(
-            #     pd.util.hash_pandas_object(pd.read_parquet(file_path, engine="fastparquet")).values
-            # ).hexdigest()
-            digest_sha256 = self.compute_file_digest(file_path)
+            digest_sha256 = file_hash(file_path)
             metadata_with_md5 = pydantic_copy_method(metadata_with_md5)(update={"digest": digest_sha256})
 
             with open(file_path, "rb") as file:
@@ -729,6 +715,16 @@ class _RestClient:
         )
         return TransformUsage(response)
 
+    def update_selected_features(self, trace_id: str, search_task_id: str, selected_features: list[str]):
+        api_path = self.SEARCH_SELECTED_FEATURES_URI_FMT.format(search_task_id)
+        request = {"features": selected_features}
+        self._with_unauth_retry(lambda: self._send_post_req(api_path, trace_id, request, result_format=None))
+
+    def get_selected_features(self, trace_id: str, search_task_id: str) -> list[str] | None:
+        api_path = self.SEARCH_SELECTED_FEATURES_URI_FMT.format(search_task_id)
+        response = self._with_unauth_retry(lambda: self._send_get_req(api_path, trace_id))
+        return response.get("features")
+
     def send_log_event(self, log_event: LogEvent):
         api_path = self.SEND_LOG_EVENT_URI
         try:

upgini/metadata.py

@@ -285,6 +285,7 @@ class FeaturesMetadataV2(BaseModel):
     doc_link: Optional[str] = None
     update_frequency: Optional[str] = None
     from_online_api: Optional[bool] = None
+    psi_value: Optional[float] = None
 
 
 class HitRateMetrics(BaseModel):
@@ -326,13 +327,6 @@ class ProviderTaskMetadataV2(BaseModel):
     generated_features: Optional[List[GeneratedFeatureMetadata]] = None
 
 
-class FeaturesFilter(BaseModel):
-    minImportance: Optional[float] = None
-    maxPSI: Optional[float] = None
-    maxCount: Optional[int] = None
-    selectedFeatures: Optional[List[str]] = None
-
-
 class RuntimeParameters(BaseModel):
     properties: Dict[str, Any] = {}
 
@@ -342,11 +336,8 @@ class AutoFEParameters(BaseModel):
 
 
 class SearchCustomization(BaseModel):
-    featuresFilter: Optional[FeaturesFilter] = None
     extractFeatures: Optional[bool] = None
     accurateModel: Optional[bool] = None
-    importanceThreshold: Optional[float] = None
-    maxFeatures: Optional[int] = None
     returnScores: Optional[bool] = None
     runtimeParameters: Optional[RuntimeParameters] = None
     metricsCalculation: Optional[bool] = None

upgini/metrics.py

@@ -816,7 +816,8 @@ class CatBoostWrapper(EstimatorWrapper):
                 else:
                     encoded = cat_encoder.transform(x[self.cat_features])
                     cat_features = encoded.columns.to_list()
-                x.loc[:, self.cat_features] = encoded
+                x.drop(columns=encoded.columns, inplace=True, errors="ignore")
+                x[encoded.columns] = encoded
             else:
                 cat_features = self.cat_features
 
@@ -1175,7 +1176,10 @@ def _ext_mean_squared_log_error(y_true, y_pred, *, sample_weight=None, multioutp
     >>> mean_squared_log_error(y_true, y_pred, multioutput=[0.3, 0.7])
     0.060...
     """
-    _, y_true, y_pred, multioutput = _check_reg_targets(y_true, y_pred, multioutput)
+    try:
+        _, y_true, y_pred, multioutput = _check_reg_targets(y_true, y_pred, multioutput)
+    except TypeError:
+        _, y_true, y_pred, sample_weight, multioutput = _check_reg_targets(y_true, y_pred, sample_weight, multioutput)
     check_consistent_length(y_true, y_pred, sample_weight)
 
     if (y_true < 0).any():

upgini/resource_bundle/strings.properties

@@ -72,9 +72,6 @@ binary_target_unique_count_not_2=Binary target should contain only 2 unique valu
 binary_target_eval_unique_count_not_2=Binary target should contain only 2 unique values, but {} found in eval_set
 
 # Validation errors
-    # params validation
-invalid_importance_threshold=importance_threshold must be float
-invalid_max_features=max_features must be int
     # search keys validation
 search_key_differ_from_fit=With search_id passed as a parameter, search_keys should same as for fit call\nSee docs https://github.com/upgini/upgini#61-reuse-completed-search-for-enrichment-without-fit-run
 empty_search_keys=At least one column with a search key required\nSee docs https://github.com/upgini/upgini#3--choose-one-or-multiple-columns-as-a-search-keys
@@ -123,7 +120,7 @@ train_unstable_target=Your training sample contains an unstable target event, PS
 eval_unstable_target=Your training and evaluation samples have a difference in target distribution. PSI = {}. The results will be unstable. It is recommended to redesign the training and evaluation samples
     # eval set validation
 unsupported_type_eval_set=Unsupported type of eval_set: {}. It should be list of tuples with two elements: X and y
-eval_set_invalid_tuple_size=eval_set contains a tuple of size {}. It should contain only pairs of X and y
+eval_set_invalid_tuple_size=eval_set contains a tuple of size {}. It should contain only pairs of X and y or X only
 unsupported_x_type_eval_set=Unsupported type of X in eval_set: {}. Use pandas.DataFrame, pandas.Series or numpy.ndarray or list.
 eval_x_and_x_diff_shape=The column set in eval_set are differ from the column set in X
 unsupported_y_type_eval_set=Unsupported type of y in eval_set: {}. Use pandas.Series, numpy.ndarray or list
@@ -139,6 +136,8 @@ eval_x_is_empty=X in eval_set is empty.
 eval_y_is_empty=y in eval_set is empty.
 x_and_eval_x_diff_types=X and eval_set X has different types: {} and {}
 eval_x_has_train_samples=Eval set X has rows that are present in train set X
+oot_without_date_not_supported=Eval set {} provided as OOT but date column is missing. It will be ignored for stability check
+oot_with_online_sources_not_supported=Eval set {} provided as OOT and also provided columns for online API. It will be ignored for stability check
 
 baseline_score_column_not_exists=baseline_score_column {} doesn't exist in input dataframe
 baseline_score_column_has_na=baseline_score_column contains NaN. Clear it and and retry
@@ -163,6 +162,7 @@ dataset_too_many_rows_registered=X rows limit for transform is {}. Please sample
 dataset_empty_column_names=Some column names are empty. Add names please
 dataset_full_duplicates={:.5f}% of the rows are fully duplicated
 dataset_diff_target_duplicates={:.4f}% of rows ({}) in X and eval_set are duplicates with different y values. These rows will be deleted as incorrect\nSample of incorrect row indexes: {}
+dataset_diff_target_duplicates_oot={:.4f}% of rows ({}) in OOT eval_set are duplicates with train or another eval_set. These rows will be deleted from OOT\nSample of incorrect row indexes: {}
 dataset_train_diff_target_duplicates_fintech={:.4f}% of rows ({}) in X are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
 dataset_eval_diff_target_duplicates_fintech={:.4f}% of rows ({}) in eval{} X are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
 dataset_drop_old_dates=We don't have data before '2000-01-01' and removed all earlier records from the search dataset
@@ -183,6 +183,7 @@ dataset_invalid_column_type=Unsupported data type of column {}: {}
 dataset_invalid_filter=Unknown field in filter_features. Should be {'min_importance', 'max_psi', 'max_count', 'selected_features'}.
 dataset_too_big_file=Too big size of dataframe X for processing. Please reduce number of rows or columns
 dataset_transform_diff_fit=You try to enrich dataset that column names are different from the train dataset column names that you used on the fit stage. Please make the column names the same as in the train dataset and restart.
+oot_eval_set_too_small_after_dedup=OOT eval set {} has less than 1000 rows after deduplication. It will be ignored for stability check
 binary_small_dataset=The least populated class in Target contains less than 1000 rows.\nSmall numbers of observations may negatively affect the number of selected features and quality of your ML model.\nUpgini recommends you increase the number of observations in the least populated class.\n
 all_search_keys_invalid=All search keys are invalid
 all_emails_invalid=All values in column {} are invalid emails    # Metrics validation
@@ -240,7 +241,7 @@ validation_all_valid_status=All valid
 validation_all_valid_message= -
 validation_drop_message= Invalid rows will be dropped.
 validation_some_invalid_status=Some invalid
-validation_invalid_message={:.1f}% values failed validation and removed from dataframe, invalid values: {}
+validation_invalid_message={:.2f}% values failed validation and removed from dataframe, invalid values: {}
 validation_all_invalid_status=All invalid
 validation_all_valid_color=#DAF7A6
 validation_some_invalid_color=#FFC300
@@ -250,11 +251,12 @@ validation_text_color=black
 
 # Features info table
 features_info_header=\n{} relevant feature(s) found with the search keys: {}
-relevant_features_header=Relevant features
+relevant_features_header=Relevant features ({})
 features_info_provider=Provider
 features_info_source=Source
 features_info_name=Feature name
 features_info_shap=SHAP value
+features_info_psi=PSI value
 features_info_hitrate=Coverage %
 features_info_type=Type
 # Deprecated

upgini/sampler/base.py

@@ -1,6 +1,7 @@
 """
 Base class for the under-sampling method.
 """
+
 # Authors: Guillaume Lemaitre <g.lemaitre58@gmail.com>
 # License: MIT
 
@@ -12,6 +13,7 @@ import numpy as np
 from sklearn.base import BaseEstimator
 from sklearn.preprocessing import label_binarize
 from sklearn.utils.multiclass import check_classification_targets
+from sklearn.utils.validation import check_X_y
 
 from .utils import ArraysTransformer, check_sampling_strategy, check_target_type
 
@@ -125,7 +127,7 @@ class BaseSampler(SamplerMixin):
         if accept_sparse is None:
             accept_sparse = ["csr", "csc"]
         y, binarize_y = check_target_type(y, indicate_one_vs_all=True)
-        X, y = self._validate_data(X, y, reset=True, accept_sparse=accept_sparse)
+        X, y = check_X_y(X, y, accept_sparse=accept_sparse, dtype=None, ensure_all_finite=False)
         return X, y, binarize_y
 
     def _more_tags(self):

upgini/sampler/random_under_sampler.py

@@ -80,14 +80,24 @@ RandomUnderSampler # doctest: +NORMALIZE_WHITESPACE
 
     def _check_X_y(self, X, y):
         y, binarize_y = check_target_type(y, indicate_one_vs_all=True)
-        X, y = self._validate_data(
-            X,
-            y,
-            reset=True,
-            accept_sparse=["csr", "csc"],
-            dtype=None,
-            force_all_finite=False,
-        )
+        try:
+            X, y = self._validate_data(
+                X,
+                y,
+                reset=True,
+                accept_sparse=["csr", "csc"],
+                dtype=None,
+                force_all_finite=False,
+            )
+        except AttributeError:
+            from sklearn.utils.validation import check_X_y
+            X, y = check_X_y(
+                X,
+                y,
+                accept_sparse=["csr", "csc"],
+                dtype=None,
+                ensure_all_finite=False,
+            )
         return X, y, binarize_y
 
     def _fit_resample(self, X, y):

upgini/search_task.py

@@ -312,6 +312,12 @@ class SearchTask:
     def get_file_metadata(self, trace_id: str) -> FileMetadata:
         return self.rest_client.get_search_file_metadata(self.search_task_id, trace_id)
 
+    def update_selected_features(self, trace_id: str, selected_features: list[str]):
+        self.rest_client.update_selected_features(trace_id, self.search_task_id, selected_features)
+
+    def get_selected_features(self, trace_id: str) -> list[str] | None:
+        return self.rest_client.get_selected_features(trace_id, self.search_task_id)
+
 
 @lru_cache
 def _get_all_initial_raw_features_cached(

upgini/utils/config.py

@@ -0,0 +1,43 @@
+from dataclasses import dataclass, field
+from typing import List
+
+import pandas as pd
+
+# Constants for SampleConfig
+TS_MIN_DIFFERENT_IDS_RATIO = 0.2
+TS_DEFAULT_HIGH_FREQ_TRUNC_LENGTHS = [pd.DateOffset(years=2, months=6), pd.DateOffset(years=2, days=7)]
+TS_DEFAULT_LOW_FREQ_TRUNC_LENGTHS = [pd.DateOffset(years=7), pd.DateOffset(years=5)]
+TS_DEFAULT_TIME_UNIT_THRESHOLD = pd.Timedelta(weeks=4)
+FIT_SAMPLE_ROWS_TS = 100_000
+
+BINARY_MIN_SAMPLE_THRESHOLD = 5_000
+MULTICLASS_MIN_SAMPLE_THRESHOLD = 25_000
+BINARY_BOOTSTRAP_LOOPS = 5
+MULTICLASS_BOOTSTRAP_LOOPS = 2
+
+FIT_SAMPLE_THRESHOLD = 100_000
+FIT_SAMPLE_ROWS = 100_000
+FIT_SAMPLE_ROWS_WITH_EVAL_SET = 100_000
+FIT_SAMPLE_THRESHOLD_WITH_EVAL_SET = 100_000
+
+
+@dataclass
+class SampleConfig:
+    force_sample_size: int = 7000
+    ts_min_different_ids_ratio: float = TS_MIN_DIFFERENT_IDS_RATIO
+    ts_default_high_freq_trunc_lengths: List[pd.DateOffset] = field(
+        default_factory=TS_DEFAULT_HIGH_FREQ_TRUNC_LENGTHS.copy
+    )
+    ts_default_low_freq_trunc_lengths: List[pd.DateOffset] = field(
+        default_factory=TS_DEFAULT_LOW_FREQ_TRUNC_LENGTHS.copy
+    )
+    ts_default_time_unit_threshold: pd.Timedelta = TS_DEFAULT_TIME_UNIT_THRESHOLD
+    binary_min_sample_threshold: int = BINARY_MIN_SAMPLE_THRESHOLD
+    multiclass_min_sample_threshold: int = MULTICLASS_MIN_SAMPLE_THRESHOLD
+    binary_bootstrap_loops: int = BINARY_BOOTSTRAP_LOOPS
+    multiclass_bootstrap_loops: int = MULTICLASS_BOOTSTRAP_LOOPS
+    fit_sample_threshold: int = FIT_SAMPLE_THRESHOLD
+    fit_sample_rows: int = FIT_SAMPLE_ROWS
+    fit_sample_rows_with_eval_set: int = FIT_SAMPLE_ROWS_WITH_EVAL_SET
+    fit_sample_threshold_with_eval_set: int = FIT_SAMPLE_THRESHOLD_WITH_EVAL_SET
+    fit_sample_rows_ts: int = FIT_SAMPLE_ROWS_TS

upgini/utils/deduplicate_utils.py

@@ -134,8 +134,13 @@ def remove_fintech_duplicates(
     logger.info(f"Train dataset shape after clean fintech duplicates: {train_df.shape}")
 
     # Process each eval_set part separately
+    oot_eval_dfs = []
     new_eval_dfs = []
     for i, eval_df in enumerate(eval_dfs, 1):
+        # Skip OOT
+        if eval_df[TARGET].isna().all():
+            oot_eval_dfs.append(eval_df)
+            continue
         logger.info(f"Eval {i} dataset shape before clean fintech duplicates: {eval_df.shape}")
         cleaned_eval_df, eval_warning = process_df(eval_df, i)
         if eval_warning:
@@ -145,8 +150,8 @@ def remove_fintech_duplicates(
 
     # Combine the processed train and eval parts back into one dataset
     logger.info(f"Dataset shape before clean fintech duplicates: {df.shape}")
-    if new_eval_dfs:
-        df = pd.concat([train_df] + new_eval_dfs)
+    if new_eval_dfs or oot_eval_dfs:
+        df = pd.concat([train_df] + new_eval_dfs + oot_eval_dfs, ignore_index=False)
     else:
         df = train_df
     logger.info(f"Dataset shape after clean fintech duplicates: {df.shape}")
@@ -190,16 +195,59 @@ def clean_full_duplicates(
     msg = None
     if TARGET in df.columns:
         unique_columns.remove(TARGET)
-        marked_duplicates = df.duplicated(subset=unique_columns, keep=False)
+
+        # Separate rows to exclude from deduplication:
+        # for each eval_set_index != 0 check separately, all TARGET values are NaN
+        df_for_dedup = df
+        oot_df = None
+
+        if EVAL_SET_INDEX in df.columns:
+            oot_eval_dfs = []
+            other_dfs = []
+            for eval_idx in df[EVAL_SET_INDEX].unique():
+                eval_subset = df[df[EVAL_SET_INDEX] == eval_idx]
+                # Check that all TARGET values for this specific eval_set_index are NaN
+                if eval_idx != 0 and eval_subset[TARGET].isna().all():
+                    oot_eval_dfs.append(eval_subset)
+                    logger.info(
+                        f"Excluded {len(eval_subset)} rows from deduplication "
+                        f"(eval_set_index={eval_idx} and all TARGET values are NaN)"
+                    )
+                else:
+                    other_dfs.append(eval_subset)
+
+            if oot_eval_dfs:
+                oot_df = pd.concat(oot_eval_dfs, ignore_index=False)
+                df_for_dedup = pd.concat(other_dfs, ignore_index=False)
+            else:
+                df_for_dedup = df
+
+        marked_duplicates = df_for_dedup.duplicated(subset=unique_columns, keep=False)
         if marked_duplicates.sum() > 0:
-            dups_indices = df[marked_duplicates].index.to_list()[:100]
-            nrows_after_tgt_dedup = len(df.drop_duplicates(subset=unique_columns, keep=False))
-            num_dup_rows = nrows_after_full_dedup - nrows_after_tgt_dedup
-            share_tgt_dedup = 100 * num_dup_rows / nrows_after_full_dedup
+            dups_indices = df_for_dedup[marked_duplicates].index.to_list()[:100]
+            nrows_after_tgt_dedup = len(df_for_dedup.drop_duplicates(subset=unique_columns, keep=False))
+            num_dup_rows = len(df_for_dedup) - nrows_after_tgt_dedup
+            share_tgt_dedup = 100 * num_dup_rows / len(df_for_dedup)
 
             msg = bundle.get("dataset_diff_target_duplicates").format(share_tgt_dedup, num_dup_rows, dups_indices)
-            df = df.drop_duplicates(subset=unique_columns, keep=False)
-            logger.info(f"Dataset shape after clean invalid target duplicates: {df.shape}")
+            df_for_dedup = df_for_dedup.drop_duplicates(subset=unique_columns, keep=False)
+            logger.info(f"Dataset shape after clean invalid target duplicates: {df_for_dedup.shape}")
+        # Combine back excluded rows
+        if oot_df is not None:
+            df = pd.concat([df_for_dedup, oot_df], ignore_index=False)
+            marked_duplicates = df.duplicated(subset=unique_columns, keep=False)
+            if marked_duplicates.sum() > 0:
+                dups_indices = df[marked_duplicates].index.to_list()[:100]
+                nrows_after_tgt_dedup = len(df.drop_duplicates(subset=unique_columns, keep=False))
+                num_dup_rows = len(df) - nrows_after_tgt_dedup
+                share_tgt_dedup = 100 * num_dup_rows / len(df)
+                msg = bundle.get("dataset_diff_target_duplicates_oot").format(
+                    share_tgt_dedup, num_dup_rows, dups_indices
+                )
+                df = df.drop_duplicates(subset=unique_columns, keep="first")
+            logger.info(f"Final dataset shape after adding back excluded rows: {df.shape}")
+        else:
+            df = df_for_dedup
 
     return df, msg
 

upgini/utils/display_utils.py

@@ -137,7 +137,7 @@ def display_html_dataframe(
             {table_html}
         </div>
         """
-    if display_handle:
+    if display_handle is not None:
         return display_handle.update(HTML(result_html))
     else:
         return display(HTML(result_html), display_id=display_id)

upgini/utils/feature_info.py

@@ -27,6 +27,7 @@ class FeatureInfo:
     doc_link: str
     data_provider_link: str
     data_source_link: str
+    psi_value: Optional[float] = None
 
     @staticmethod
     def from_metadata(
@@ -47,12 +48,14 @@ class FeatureInfo:
             doc_link=feature_meta.doc_link,
             data_provider_link=feature_meta.data_provider_link,
             data_source_link=feature_meta.data_source_link,
+            psi_value=feature_meta.psi_value,
         )
 
     def to_row(self, bundle: ResourceBundle) -> Dict[str, str]:
         return {
             bundle.get("features_info_name"): self.name,
             bundle.get("features_info_shap"): self.rounded_shap,
+            bundle.get("features_info_psi"): self.psi_value,
             bundle.get("features_info_hitrate"): self.hitrate,
             bundle.get("features_info_value_preview"): self.value_preview,
             bundle.get("features_info_provider"): self.provider,
@@ -64,6 +67,7 @@ class FeatureInfo:
         return {
             bundle.get("features_info_name"): self.internal_name,
             bundle.get("features_info_shap"): self.rounded_shap,
+            bundle.get("features_info_psi"): self.psi_value,
             bundle.get("features_info_hitrate"): self.hitrate,
             bundle.get("features_info_value_preview"): self.value_preview,
             bundle.get("features_info_provider"): self.internal_provider,
@@ -76,6 +80,7 @@ class FeatureInfo:
             bundle.get("features_info_name"): self.internal_name,
             "feature_link": self.doc_link,
             bundle.get("features_info_shap"): self.rounded_shap,
+            bundle.get("features_info_psi"): self.psi_value,
             bundle.get("features_info_hitrate"): self.hitrate,
             bundle.get("features_info_value_preview"): self.value_preview,
             bundle.get("features_info_provider"): self.internal_provider,

upgini/utils/hash_utils.py

@@ -0,0 +1,159 @@
+import hashlib
+import os
+import platform
+import shutil
+import subprocess
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+import numpy as np
+import pandas as pd
+
+
+def file_hash(path: str | os.PathLike, algo: str = "sha256") -> str:
+    """
+    Returns file hash using system utilities, working consistently on Windows/macOS/Linux.
+    If no suitable utility is found, gracefully falls back to hashlib.
+
+    Supported algo values (depend on OS and available utilities):
+      - "md5", "sha1", "sha224", "sha256", "sha384", "sha512"
+    On Windows uses `certutil`.
+    On Linux uses `sha*sum` (e.g., sha256sum) or `shasum -a N`.
+    On macOS uses `shasum -a N` or `md5` for MD5.
+    """
+    p = str(Path(path))
+
+    sysname = platform.system().lower()
+    algo = algo.lower()
+
+    # -------- command attempts depending on OS --------
+    candidates: list[list[str]] = []
+
+    if sysname == "windows":
+        # certutil supports: MD5, SHA1, SHA256, SHA384, SHA512
+        name_map = {
+            "md5": "MD5",
+            "sha1": "SHA1",
+            "sha224": None,  # certutil doesn't support
+            "sha256": "SHA256",
+            "sha384": "SHA384",
+            "sha512": "SHA512",
+        }
+        cert_name = name_map.get(algo)
+        if cert_name:
+            candidates.append(["certutil", "-hashfile", p, cert_name])
+    else:
+        # Unix-like systems
+        # 1) specialized *sum utility if available (usually present on Linux)
+        sum_cmd = f"{algo}sum"  # md5sum, sha256sum, etc.
+        if shutil.which(sum_cmd):
+            candidates.append([sum_cmd, p])
+
+        # 2) universal shasum with -a parameter (available on macOS and often on Linux)
+        shasum_bits = {
+            "sha1": "1",
+            "sha224": "224",
+            "sha256": "256",
+            "sha384": "384",
+            "sha512": "512",
+        }
+        if algo in shasum_bits and shutil.which("shasum"):
+            candidates.append(["shasum", "-a", shasum_bits[algo], p])
+
+        # 3) for MD5 on macOS there's often a separate `md5` utility
+        if algo == "md5" and shutil.which("md5"):
+            candidates.append(["md5", p])
+
+    # -------- try system utilities --------
+    for cmd in candidates:
+        try:
+            out = subprocess.check_output(cmd, text=True, stderr=subprocess.STDOUT)
+            digest = _parse_hash_output(out, cmd[0])
+            if digest:
+                return digest.lower()
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            continue  # try next candidate
+
+    # -------- reliable fallback to hashlib --------
+    import hashlib
+
+    try:
+        h = getattr(hashlib, algo)
+    except AttributeError:
+        raise ValueError(f"Algorithm not supported: {algo}")
+
+    hasher = h()
+    with open(p, "rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest().lower()
+
+
+def _parse_hash_output(output: str, tool: str) -> Optional[str]:
+    """
+    Converts output from different utilities to clean hash.
+    Supports:
+      - sha*sum / shasum: '<hex>  <filename>'
+      - certutil (Windows): line with second element as hash (spaces inside are removed)
+      - md5 (macOS): 'MD5 (file) = <hex>'
+    """
+    tool = tool.lower()
+    lines = [ln.strip() for ln in output.splitlines() if ln.strip()]
+
+    if not lines:
+        return None
+
+    if tool in {"sha1sum", "sha224sum", "sha256sum", "sha384sum", "sha512sum", "md5sum", "shasum"}:
+        # format: '<hex>  <filename>'
+        first = lines[0]
+        parts = first.split()
+        return parts[0] if parts else None
+
+    if tool == "certutil":
+        # format:
+        # SHA256 hash of file <path>:
+        # <AA BB CC ...>
+        # CertUtil: -hashfile command completed successfully.
+        if len(lines) >= 2:
+            # Second line contains hex with spaces
+            candidate = lines[1].replace(" ", "")
+            # ensure it's hex
+            if all(c in "0123456789abcdefABCDEF" for c in candidate):
+                return candidate
+        return None
+
+    if tool == "md5":
+        # format: 'MD5 (<file>) = <hex>'
+        last = lines[-1]
+        if "=" in last:
+            return last.split("=", 1)[1].strip()
+        # sometimes md5 can return just the hash
+        parts = last.split()
+        if parts and all(c in "0123456789abcdefABCDEF" for c in parts[-1]):
+            return parts[-1]
+        return None
+
+    # as a last resort: take the first "looks like hash" word
+    for ln in lines:
+        for token in ln.split():
+            if all(c in "0123456789abcdefABCDEF" for c in token) and len(token) >= 32:
+                return token
+    return None
+
+
+def hash_input(X: pd.DataFrame, y: Optional[pd.Series] = None, eval_set: Optional[List[Tuple]] = None) -> str:
+    hashed_objects = []
+    try:
+        hashed_objects.append(pd.util.hash_pandas_object(X, index=False).values)
+        if y is not None:
+            hashed_objects.append(pd.util.hash_pandas_object(y, index=False).values)
+        if eval_set is not None:
+            if isinstance(eval_set, tuple):
+                eval_set = [eval_set]
+            for eval_X, eval_y in eval_set:
+                hashed_objects.append(pd.util.hash_pandas_object(eval_X, index=False).values)
+                hashed_objects.append(pd.util.hash_pandas_object(eval_y, index=False).values)
+        common_hash = hashlib.sha256(np.concatenate(hashed_objects)).hexdigest()
+        return common_hash
+    except Exception:
+        return ""