upgini 1.2.114a2__tar.gz → 1.2.114a4__tar.gz

{upgini-1.2.114a2 → upgini-1.2.114a4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.114a2
+Version: 1.2.114a4
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.114a4/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.114a4"

{upgini-1.2.114a2 → upgini-1.2.114a4}/src/upgini/features_enricher.py

@@ -101,6 +101,7 @@ from upgini.utils.email_utils import (
 from upgini.utils.feature_info import FeatureInfo, _round_shap_value
 from upgini.utils.features_validator import FeaturesValidator
 from upgini.utils.format import Format
+from upgini.utils.hash_utils import file_hash
 from upgini.utils.ip_utils import IpSearchKeyConverter
 from upgini.utils.phone_utils import PhoneSearchKeyDetector
 from upgini.utils.postal_code_utils import PostalCodeSearchKeyDetector
@@ -1516,6 +1517,8 @@ class FeaturesEnricher(TransformerMixin):
             checking_eval_set_df, cat_features, date_column, self.logger, model_task_type
         )
 
+        self.logger.info(f"PSI values by sparsity: {psi_values_sparse}")
+
         unstable_by_sparsity = [feature for feature, psi in psi_values_sparse.items() if psi > stability_threshold]
         if unstable_by_sparsity:
             self.logger.info(f"Unstable by sparsity features: {sorted(unstable_by_sparsity)}")
@@ -1524,6 +1527,8 @@ class FeaturesEnricher(TransformerMixin):
             checking_eval_set_df, cat_features, date_column, self.logger, model_task_type
         )
 
+        self.logger.info(f"PSI values by value: {psi_values}")
+
         unstable_by_value = [feature for feature, psi in psi_values.items() if psi > stability_threshold]
         if unstable_by_value:
             self.logger.info(f"Unstable by value features: {sorted(unstable_by_value)}")
@@ -2105,7 +2110,18 @@ class FeaturesEnricher(TransformerMixin):
         columns_renaming = normalizer.columns_renaming
 
         df, _ = clean_full_duplicates(df, logger=self.logger, bundle=self.bundle)
-        df = self.__add_fit_system_record_id(df, search_keys, SYSTEM_RECORD_ID, TARGET, columns_renaming, silent=True)
+        df = self._add_fit_system_record_id(
+            df,
+            search_keys,
+            SYSTEM_RECORD_ID,
+            TARGET,
+            columns_renaming,
+            self.id_columns,
+            self.cv,
+            self.model_task_type,
+            self.logger,
+            self.bundle,
+        )
 
         # Sample after sorting by system_record_id for idempotency
         df.sort_values(by=SYSTEM_RECORD_ID, inplace=True)
@@ -2717,13 +2733,17 @@ if response.status_code == 200:
 
             features_not_to_pass = []
             if add_fit_system_record_id:
-                df = self.__add_fit_system_record_id(
+                df = self._add_fit_system_record_id(
                     df,
                     search_keys,
                     SYSTEM_RECORD_ID,
                     TARGET,
                     columns_renaming,
-                    silent=True,
+                    self.id_columns,
+                    self.cv,
+                    self.model_task_type,
+                    self.logger,
+                    self.bundle,
                 )
                 df = df.rename(columns={SYSTEM_RECORD_ID: SORT_ID})
                 features_not_to_pass.append(SORT_ID)
@@ -3263,8 +3283,17 @@ if response.status_code == 200:
                     self.__log_warning(self.bundle.get("oot_eval_set_too_small_after_dedup").format(eval_set_index + 1))
 
         # Explode multiple search keys
-        df = self.__add_fit_system_record_id(
-            df, self.fit_search_keys, ENTITY_SYSTEM_RECORD_ID, TARGET, self.fit_columns_renaming
+        df = self._add_fit_system_record_id(
+            df,
+            self.fit_search_keys,
+            ENTITY_SYSTEM_RECORD_ID,
+            TARGET,
+            self.fit_columns_renaming,
+            self.id_columns,
+            self.cv,
+            self.model_task_type,
+            self.logger,
+            self.bundle,
         )
 
         # TODO check that this is correct for enrichment
@@ -3298,8 +3327,17 @@ if response.status_code == 200:
         if eval_set is not None and len(eval_set) > 0:
             meaning_types[EVAL_SET_INDEX] = FileColumnMeaningType.EVAL_SET_INDEX
 
-        df = self.__add_fit_system_record_id(
-            df, self.fit_search_keys, SYSTEM_RECORD_ID, TARGET, self.fit_columns_renaming, silent=True
+        df = self._add_fit_system_record_id(
+            df,
+            self.fit_search_keys,
+            SYSTEM_RECORD_ID,
+            TARGET,
+            self.fit_columns_renaming,
+            self.id_columns,
+            self.cv,
+            self.model_task_type,
+            self.logger,
+            self.bundle,
         )
 
         if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
@@ -4130,14 +4168,18 @@ if response.status_code == 200:
         self.logger.info(f"Finished explosion. Size after: {len(df)}")
         return df, unnest_search_keys
 
-    def __add_fit_system_record_id(
-        self,
+    @staticmethod
+    def _add_fit_system_record_id(
         df: pd.DataFrame,
         search_keys: Dict[str, SearchKey],
         id_name: str,
         target_name: str,
         columns_renaming: Dict[str, str],
-        silent: bool = False,
+        id_columns: Optional[List[str]],
+        cv: Optional[CVType],
+        model_task_type: ModelTaskType,
+        logger: Optional[logging.Logger] = None,
+        bundle: ResourceBundle = bundle,
     ) -> pd.DataFrame:
         original_index_name = df.index.name
         index_name = df.index.name or DEFAULT_INDEX
@@ -4166,32 +4208,33 @@ if response.status_code == 200:
         columns_to_sort = [date_column] if date_column is not None else []
 
         do_sorting = True
-        if self.id_columns and self.cv is not None and self.cv.is_time_series():
+        if id_columns and cv is not None and cv.is_time_series():
             # Check duplicates by date and id_columns
             reversed_columns_renaming = {v: k for k, v in columns_renaming.items()}
-            renamed_id_columns = [reversed_columns_renaming.get(c, c) for c in self.id_columns]
+            renamed_id_columns = [reversed_columns_renaming.get(c, c) for c in id_columns]
             duplicate_check_columns = [c for c in renamed_id_columns if c in df.columns]
             if date_column is not None:
                 duplicate_check_columns.append(date_column)
 
             duplicates = df.duplicated(subset=duplicate_check_columns, keep=False)
             if duplicates.any():
-                raise ValueError(self.bundle.get("date_and_id_columns_duplicates").format(duplicates.sum()))
+                raise ValueError(bundle.get("date_and_id_columns_duplicates").format(duplicates.sum()))
             else:
                 columns_to_hash = list(set(list(search_keys.keys()) + renamed_id_columns + [target_name]))
                 columns_to_hash = sort_columns(
                     df[columns_to_hash],
                     target_name,
                     search_keys,
-                    self.model_task_type,
+                    model_task_type,
                     sort_exclude_columns,
-                    logger=self.logger,
+                    logger=logger,
                 )
         else:
             columns_to_hash = sort_columns(
-                df, target_name, search_keys, self.model_task_type, sort_exclude_columns, logger=self.logger
+                df, target_name, search_keys, model_task_type, sort_exclude_columns, logger=logger
             )
-        if do_sorting:
+
+        def sort_df(df: pd.DataFrame) -> pd.DataFrame:
             search_keys_hash = "search_keys_hash"
             if len(columns_to_hash) > 0:
                 factorized_df = df.copy()
@@ -4205,6 +4248,24 @@ if response.status_code == 200:
 
             if search_keys_hash in df.columns:
                 df.drop(columns=search_keys_hash, inplace=True)
+            return df
+
+        if do_sorting:
+            sorted_dfs = []
+            if EVAL_SET_INDEX in df.columns:
+                # Sort train and eval sets separately
+                train = df[df[EVAL_SET_INDEX] == 0].copy()
+                sorted_dfs.append(sort_df(train))
+
+                for eval_set_index in df[EVAL_SET_INDEX].unique():
+                    if eval_set_index == 0:
+                        continue
+                    eval_set_df = df[df[EVAL_SET_INDEX] == eval_set_index].copy()
+                    sorted_dfs.append(sort_df(eval_set_df))
+
+                df = pd.concat(sorted_dfs)
+            else:
+                df = sort_df(df)
 
         df = df.reset_index(drop=True).reset_index()
         # system_record_id saves correct order for fit
@@ -4215,11 +4276,6 @@ if response.status_code == 200:
         df.index.name = original_index_name
         df = df.sort_values(by=original_order_name).drop(columns=original_order_name)
 
-        # meaning_types[id_name] = (
-        #     FileColumnMeaningType.SYSTEM_RECORD_ID
-        #     if id_name == SYSTEM_RECORD_ID
-        #     else FileColumnMeaningType.ENTITY_SYSTEM_RECORD_ID
-        # )
         return df
 
     def __correct_target(self, df: pd.DataFrame) -> pd.DataFrame:
@@ -4266,6 +4322,7 @@ if response.status_code == 200:
             self.logger.error(f"result features not found by search_task_id: {self.get_search_id()}")
             raise RuntimeError(self.bundle.get("features_wasnt_returned"))
 
+        result_features = result_features.copy()
         if EVAL_SET_INDEX in result_features.columns:
             result_features = result_features.drop(columns=EVAL_SET_INDEX)
 
@@ -4993,7 +5050,7 @@ if response.status_code == 200:
 
                     with tempfile.TemporaryDirectory() as tmp_dir:
                         X_.to_parquet(f"{tmp_dir}/x.parquet", compression="zstd")
-                        x_digest_sha256 = self.rest_client.compute_file_digest(f"{tmp_dir}/x.parquet")
+                        x_digest_sha256 = file_hash(f"{tmp_dir}/x.parquet")
                         if self.rest_client.is_file_uploaded(trace_id, x_digest_sha256):
                             self.logger.info(
                                 f"File x.parquet was already uploaded with digest {x_digest_sha256}, skipping"
@@ -5007,7 +5064,7 @@ if response.status_code == 200:
                             if isinstance(y_, pd.Series):
                                 y_ = y_.to_frame()
                             y_.to_parquet(f"{tmp_dir}/y.parquet", compression="zstd")
-                            y_digest_sha256 = self.rest_client.compute_file_digest(f"{tmp_dir}/y.parquet")
+                            y_digest_sha256 = file_hash(f"{tmp_dir}/y.parquet")
                             if self.rest_client.is_file_uploaded(trace_id, y_digest_sha256):
                                 self.logger.info(
                                     f"File y.parquet was already uploaded with digest {y_digest_sha256}, skipping"
@@ -5022,9 +5079,7 @@ if response.status_code == 200:
                                     if isinstance(eval_x_, pd.Series):
                                         eval_x_ = eval_x_.to_frame()
                                     eval_x_.to_parquet(f"{tmp_dir}/eval_x_{idx}.parquet", compression="zstd")
-                                    eval_x_digest_sha256 = self.rest_client.compute_file_digest(
-                                        f"{tmp_dir}/eval_x_{idx}.parquet"
-                                    )
+                                    eval_x_digest_sha256 = file_hash(f"{tmp_dir}/eval_x_{idx}.parquet")
                                     if self.rest_client.is_file_uploaded(trace_id, eval_x_digest_sha256):
                                         self.logger.info(
                                             f"File eval_x_{idx}.parquet was already uploaded with"
@@ -5041,9 +5096,7 @@ if response.status_code == 200:
                                     if isinstance(eval_y_, pd.Series):
                                         eval_y_ = eval_y_.to_frame()
                                     eval_y_.to_parquet(f"{tmp_dir}/eval_y_{idx}.parquet", compression="zstd")
-                                    eval_y_digest_sha256 = self.rest_client.compute_file_digest(
-                                        f"{tmp_dir}/eval_y_{idx}.parquet"
-                                    )
+                                    eval_y_digest_sha256 = file_hash(f"{tmp_dir}/eval_y_{idx}.parquet")
                                     if self.rest_client.is_file_uploaded(trace_id, eval_y_digest_sha256):
                                         self.logger.info(
                                             f"File eval_y_{idx}.parquet was already uploaded"

{upgini-1.2.114a2 → upgini-1.2.114a4}/src/upgini/http.py

@@ -45,6 +45,7 @@ from upgini.metadata import (
     SearchCustomization,
 )
 from upgini.resource_bundle import bundle
+from upgini.utils.hash_utils import file_hash
 from upgini.utils.track_info import get_track_metrics
 
 UPGINI_URL: str = "UPGINI_URL"
@@ -427,7 +428,7 @@ class _RestClient:
         api_path = self.SEARCH_DUMP_INPUT_FILE_FMT
 
         def upload_with_check(path: str, file_name: str):
-            digest_sha256 = self.compute_file_digest(path)
+            digest_sha256 = file_hash(path)
             if self.is_file_uploaded(trace_id, digest_sha256):
                 # print(f"File {path} was already uploaded with digest {digest_sha256}, skipping")
                 return
@@ -448,16 +449,6 @@ class _RestClient:
         if eval_y_path:
             upload_with_check(eval_y_path, "eval_y.parquet")
 
-    @staticmethod
-    def compute_file_digest(filepath: str, algorithm="sha256", chunk_size=4096) -> str:
-        hash_func = getattr(hashlib, algorithm)()
-
-        with open(filepath, "rb") as f:
-            for chunk in iter(lambda: f.read(chunk_size), b""):
-                hash_func.update(chunk)
-
-        return hash_func.hexdigest()
-
     def initial_search_v2(
         self,
         trace_id: str,
@@ -478,10 +469,7 @@ class _RestClient:
                 digest = md5_hash.hexdigest()
                 metadata_with_md5 = pydantic_copy_method(metadata)(update={"checksumMD5": digest})
 
-            # digest_sha256 = hashlib.sha256(
-            #     pd.util.hash_pandas_object(pd.read_parquet(file_path, engine="fastparquet")).values
-            # ).hexdigest()
-            digest_sha256 = self.compute_file_digest(file_path)
+            digest_sha256 = file_hash(file_path)
             metadata_with_md5 = pydantic_copy_method(metadata_with_md5)(update={"digest": digest_sha256})
 
             with open(file_path, "rb") as file:
@@ -576,10 +564,7 @@ class _RestClient:
                 digest = md5_hash.hexdigest()
                 metadata_with_md5 = pydantic_copy_method(metadata)(update={"checksumMD5": digest})
 
-            # digest_sha256 = hashlib.sha256(
-            #     pd.util.hash_pandas_object(pd.read_parquet(file_path, engine="fastparquet")).values
-            # ).hexdigest()
-            digest_sha256 = self.compute_file_digest(file_path)
+            digest_sha256 = file_hash(file_path)
             metadata_with_md5 = pydantic_copy_method(metadata_with_md5)(update={"digest": digest_sha256})
 
             with open(file_path, "rb") as file:

{upgini-1.2.114a2 → upgini-1.2.114a4}/src/upgini/resource_bundle/strings.properties

@@ -244,7 +244,7 @@ validation_all_valid_status=All valid
 validation_all_valid_message= -
 validation_drop_message= Invalid rows will be dropped.
 validation_some_invalid_status=Some invalid
-validation_invalid_message={:.1f}% values failed validation and removed from dataframe, invalid values: {}
+validation_invalid_message={:.2f}% values failed validation and removed from dataframe, invalid values: {}
 validation_all_invalid_status=All invalid
 validation_all_valid_color=#DAF7A6
 validation_some_invalid_color=#FFC300

upgini-1.2.114a4/src/upgini/utils/hash_utils.py

@@ -0,0 +1,137 @@
+import os
+import platform
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+
+def file_hash(path: str | os.PathLike, algo: str = "sha256") -> str:
+    """
+    Returns file hash using system utilities, working consistently on Windows/macOS/Linux.
+    If no suitable utility is found, gracefully falls back to hashlib.
+
+    Supported algo values (depend on OS and available utilities):
+      - "md5", "sha1", "sha224", "sha256", "sha384", "sha512"
+    On Windows uses `certutil`.
+    On Linux uses `sha*sum` (e.g., sha256sum) or `shasum -a N`.
+    On macOS uses `shasum -a N` or `md5` for MD5.
+    """
+    p = str(Path(path))
+
+    sysname = platform.system().lower()
+    algo = algo.lower()
+
+    # -------- command attempts depending on OS --------
+    candidates: list[list[str]] = []
+
+    if sysname == "windows":
+        # certutil supports: MD5, SHA1, SHA256, SHA384, SHA512
+        name_map = {
+            "md5": "MD5",
+            "sha1": "SHA1",
+            "sha224": None,  # certutil doesn't support
+            "sha256": "SHA256",
+            "sha384": "SHA384",
+            "sha512": "SHA512",
+        }
+        cert_name = name_map.get(algo)
+        if cert_name:
+            candidates.append(["certutil", "-hashfile", p, cert_name])
+    else:
+        # Unix-like systems
+        # 1) specialized *sum utility if available (usually present on Linux)
+        sum_cmd = f"{algo}sum"  # md5sum, sha256sum, etc.
+        if shutil.which(sum_cmd):
+            candidates.append([sum_cmd, p])
+
+        # 2) universal shasum with -a parameter (available on macOS and often on Linux)
+        shasum_bits = {
+            "sha1": "1",
+            "sha224": "224",
+            "sha256": "256",
+            "sha384": "384",
+            "sha512": "512",
+        }
+        if algo in shasum_bits and shutil.which("shasum"):
+            candidates.append(["shasum", "-a", shasum_bits[algo], p])
+
+        # 3) for MD5 on macOS there's often a separate `md5` utility
+        if algo == "md5" and shutil.which("md5"):
+            candidates.append(["md5", p])
+
+    # -------- try system utilities --------
+    for cmd in candidates:
+        try:
+            out = subprocess.check_output(cmd, text=True, stderr=subprocess.STDOUT)
+            digest = _parse_hash_output(out, cmd[0])
+            if digest:
+                return digest.lower()
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            continue  # try next candidate
+
+    # -------- reliable fallback to hashlib --------
+    import hashlib
+
+    try:
+        h = getattr(hashlib, algo)
+    except AttributeError:
+        raise ValueError(f"Algorithm not supported: {algo}")
+
+    hasher = h()
+    with open(p, "rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest().lower()
+
+
+def _parse_hash_output(output: str, tool: str) -> Optional[str]:
+    """
+    Converts output from different utilities to clean hash.
+    Supports:
+      - sha*sum / shasum: '<hex>  <filename>'
+      - certutil (Windows): line with second element as hash (spaces inside are removed)
+      - md5 (macOS): 'MD5 (file) = <hex>'
+    """
+    tool = tool.lower()
+    lines = [ln.strip() for ln in output.splitlines() if ln.strip()]
+
+    if not lines:
+        return None
+
+    if tool in {"sha1sum", "sha224sum", "sha256sum", "sha384sum", "sha512sum", "md5sum", "shasum"}:
+        # format: '<hex>  <filename>'
+        first = lines[0]
+        parts = first.split()
+        return parts[0] if parts else None
+
+    if tool == "certutil":
+        # format:
+        # SHA256 hash of file <path>:
+        # <AA BB CC ...>
+        # CertUtil: -hashfile command completed successfully.
+        if len(lines) >= 2:
+            # Second line contains hex with spaces
+            candidate = lines[1].replace(" ", "")
+            # ensure it's hex
+            if all(c in "0123456789abcdefABCDEF" for c in candidate):
+                return candidate
+        return None
+
+    if tool == "md5":
+        # format: 'MD5 (<file>) = <hex>'
+        last = lines[-1]
+        if "=" in last:
+            return last.split("=", 1)[1].strip()
+        # sometimes md5 can return just the hash
+        parts = last.split()
+        if parts and all(c in "0123456789abcdefABCDEF" for c in parts[-1]):
+            return parts[-1]
+        return None
+
+    # as a last resort: take the first "looks like hash" word
+    for ln in lines:
+        for token in ln.split():
+            if all(c in "0123456789abcdefABCDEF" for c in token) and len(token) >= 32:
+                return token
+    return None

{upgini-1.2.114a2 → upgini-1.2.114a4}/src/upgini/utils/psi.py

@@ -77,11 +77,14 @@ def calculate_features_psi(
     psi_features_params: StabilityParams = DEFAULT_FEATURES_PARAMS,
     psi_target_params: StabilityParams = DEFAULT_TARGET_PARAMS,
 ) -> Dict[str, float]:
-    empty_res = pd.Series(index=df.columns, data=0)
+    empty_res = {col: 0.0 for col in df.columns if col not in [TARGET, date_column]}
 
     if not is_numeric_dtype(df[date_column]):
         df[date_column] = pd.to_datetime(df[date_column]).dt.floor("D").astype(np.int64) / 10**6
 
+    # Filter out rows with missing dates
+    df = df[df[date_column].notna()].copy()
+
     n_months = pd.to_datetime(df[date_column], unit="ms").dt.month.nunique()
 
     if TARGET in df.columns:
@@ -113,9 +116,9 @@ def calculate_features_psi(
             cat_top_pct=psi_target_params.cat_top_pct,
             agg_func=target_agg_func,
         )
-        if target_psi is None:
+        if target_psi is None or np.isnan(target_psi):
             logger.info("Cannot determine target PSI. Skip feature PSI check")
-            return pd.Series(index=df.columns, data=0)
+            return empty_res
 
         if target_psi > psi_target_params.threshold:
             logger.info(
@@ -221,7 +224,7 @@ def _stability_agg(
 
     psi_value = agg_func([_psi(reference, c) for c in current])
 
-    return psi_value
+    return float(psi_value)
 
 
 def _get_binned_data(

upgini-1.2.114a2/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.114a2"