upgini 1.2.68a3832.dev11__tar.gz → 1.2.69__tar.gz

{upgini-1.2.68a3832.dev11 → upgini-1.2.69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.68a3832.dev11
+Version: 1.2.69
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -23,12 +23,12 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Requires-Python: <3.12,>=3.8
+Requires-Dist: catboost>=1.0.3
 Requires-Dist: fastparquet>=0.8.1
 Requires-Dist: ipywidgets>=8.1.0
 Requires-Dist: jarowinkler>=2.0.0
 Requires-Dist: levenshtein>=0.25.1
-Requires-Dist: lightgbm>=4.6.0
-Requires-Dist: numpy<3.0.0,>=1.19.0
+Requires-Dist: numpy<=1.26.4,>=1.19.0
 Requires-Dist: pandas<3.0.0,>=1.1.0
 Requires-Dist: psutil>=6.0.0
 Requires-Dist: pydantic<3.0.0,>1.0.0
@@ -39,7 +39,6 @@ Requires-Dist: python-json-logger>=3.3.0
 Requires-Dist: requests>=2.8.0
 Requires-Dist: scikit-learn>=1.3.0
 Requires-Dist: scipy>=1.10.0
-Requires-Dist: shap>=0.44.0
 Requires-Dist: xhtml2pdf<0.3.0,>=0.2.11
 Description-Content-Type: text/markdown
 

{upgini-1.2.68a3832.dev11 → upgini-1.2.69}/pyproject.toml

@@ -35,11 +35,10 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Information Analysis",
 ]
 dependencies = [
-    "lightgbm>=4.6.0",
-    "shap>=0.44.0",
+    "catboost>=1.0.3",
     "fastparquet>=0.8.1",
     "ipywidgets>=8.1.0",
-    "numpy>=1.19.0,<3.0.0",
+    "numpy>=1.19.0,<=1.26.4",
     "pandas>=1.1.0,<3.0.0",
     "pydantic>1.0.0,<3.0.0",
     "pyjwt>=2.8.0",

upgini-1.2.69/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.69"

{upgini-1.2.68a3832.dev11 → upgini-1.2.69}/src/upgini/dataset.py

@@ -388,7 +388,7 @@ class Dataset:  # (pd.DataFrame):
         for col in columns_to_validate:
             self.data[f"{col}_is_valid"] = ~self.data[col].isnull()
             if validate_target and target is not None and col == target:
-                self.data.loc[self.data[target] == np.inf, f"{col}_is_valid"] = False
+                self.data.loc[self.data[target] == np.Inf, f"{col}_is_valid"] = False
 
             if col in mandatory_columns:
                 self.data["valid_mandatory"] = self.data["valid_mandatory"] & self.data[f"{col}_is_valid"]

{upgini-1.2.68a3832.dev11 → upgini-1.2.69}/src/upgini/features_enricher.py

@@ -3845,6 +3845,11 @@ if response.status_code == 200:
             ):
                 continue
 
+            # Temporary workaround for duplicate features metadata
+            if feature_meta.name in self.feature_names_:
+                self.logger.warning(f"WARNING: Duplicate feature metadata: {feature_meta}")
+                continue
+
             self.feature_names_.append(feature_meta.name)
             self.feature_importances_.append(_round_shap_value(feature_meta.shap_value))
 
@@ -4070,10 +4075,7 @@ if response.status_code == 200:
             )
 
         if all(k == SearchKey.CUSTOM_KEY for k in valid_search_keys.values()):
-            if self.__is_registered:
-                msg = self.bundle.get("only_custom_keys")
-            else:
-                msg = self.bundle.get("unregistered_only_personal_keys")
+            msg = self.bundle.get("unregistered_only_personal_keys")
             self.logger.warning(msg + f" Provided search keys: {search_keys}")
             raise ValidationError(msg)
 

{upgini-1.2.68a3832.dev11 → upgini-1.2.69}/src/upgini/http.py

@@ -342,7 +342,7 @@ class _RestClient:
         else:
             return self._syncronized_refresh_access_token()
 
-    def _with_unauth_retry(self, request, try_number: int = 0, need_connection_retry: bool = True, silent: bool = False):
+    def _with_unauth_retry(self, request, try_number: int = 0, need_connection_retry: bool = True):
         try:
             return request()
         except RequestException as e:
@@ -373,9 +373,8 @@ class _RestClient:
             elif "more than one concurrent search request" in e.message.lower():
                 raise ValidationError(bundle.get("concurrent_request"))
             else:
-                if not silent:
-                    print(e)
-                    show_status_error()
+                print(e)
+                show_status_error()
                 raise e
 
     @staticmethod
@@ -707,7 +706,6 @@ class _RestClient:
                     silent=True,
                 ),
                 need_connection_retry=False,
-                silent=True,
             )
         except Exception:
             self.send_log_event_unauth(log_event)
@@ -718,7 +716,7 @@ class _RestClient:
         try:
             requests.post(
                 url=urljoin(_RestClient.PROD_BACKEND_URL, api_path),
-                json=log_event.model_dump(exclude_none=True),
+                json=log_event.dict(exclude_none=True),
                 headers=_RestClient._get_base_headers(content_type="application/json"),
             )
         except Exception:

{upgini-1.2.68a3832.dev11 → upgini-1.2.69}/src/upgini/metrics.py

@@ -1,16 +1,17 @@
 from __future__ import annotations
 
+from dataclasses import dataclass
 import inspect
 import logging
 import re
 from collections import defaultdict
 from copy import deepcopy
-from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
+import catboost
 import numpy as np
 import pandas as pd
-from lightgbm import LGBMClassifier, LGBMRegressor
+from catboost import CatBoost, CatBoostClassifier, CatBoostRegressor, Pool
 from numpy import log1p
 from pandas.api.types import is_numeric_dtype
 from sklearn.metrics import check_scoring, get_scorer, make_scorer, roc_auc_score
@@ -26,8 +27,11 @@ except ImportError:
     from sklearn.metrics._scorer import SCORERS
 
     available_scorers = SCORERS
+from sklearn.metrics._regression import (
+    _check_reg_targets,
+    check_consistent_length,
+)
 from sklearn.metrics import mean_squared_error
-from sklearn.metrics._regression import _check_reg_targets, check_consistent_length
 from sklearn.model_selection import BaseCrossValidator
 
 from upgini.errors import ValidationError
@@ -84,73 +88,11 @@ CATBOOST_MULTICLASS_PARAMS = {
 
 LIGHTGBM_PARAMS = {
     "random_state": DEFAULT_RANDOM_STATE,
-    # "num_leaves": 16,
-    # "n_estimators": 150,
-    # "min_child_weight": 1,
+    "num_leaves": 16,
     "max_depth": 4,
-    "max_cat_threshold": 80,
-    "min_data_per_group": 25,
-    "num_boost_round": 150,
-    "cat_l2": 10,
-    "cat_smooth": 12,
-    "learning_rate": 0.05,
-    "feature_fraction": 1.0,
-    "min_sum_hessian_in_leaf": 0.01,
-}
-
-LIGHTGBM_REGRESSION_PARAMS = {
-    "random_state": DEFAULT_RANDOM_STATE,
-    "deterministic": True,
-    "min_gain_to_split": 0.001,
-    "n_estimators": 275,
-    "max_depth": 5,
-    "max_cat_threshold": 80,
-    "min_data_per_group": 25,
-    "cat_l2": 10,
-    "cat_smooth": 12,
-    "learning_rate": 0.05,
-    "feature_fraction": 1.0,
-    "min_sum_hessian_in_leaf": 0.01,
-    "objective": "huber",
-    "verbosity": -1,
-}
-
-LIGHTGBM_MULTICLASS_PARAMS = {
-    "random_state": DEFAULT_RANDOM_STATE,
-    "deterministic": True,
-    "min_gain_to_split": 0.001,
-    "n_estimators": 275,
-    "max_depth": 3,
-    "max_cat_threshold": 80,
-    "min_data_per_group": 25,
-    "cat_l2": 10,
-    "cat_smooth": 12,
-    "learning_rate": 0.25,  # CatBoost 0.25
-    "min_sum_hessian_in_leaf": 0.01,
-    "class_weight": "balanced",  # TODO pass dict with weights for each class
-    "objective": "multiclass",
-    "use_quantized_grad": "true",
-    "num_grad_quant_bins": "8",
-    "stochastic_rounding": "true",
-    "verbosity": -1,
-}
-
-LIGHTGBM_BINARY_PARAMS = {
-    "random_state": DEFAULT_RANDOM_STATE,
-    "deterministic": True,
-    "min_gain_to_split": 0.001,
-    "n_estimators": 275,
-    "max_depth": 5,
-    "max_cat_threshold": 80,
-    "min_data_per_group": 25,
-    "cat_l2": 10,
-    "cat_smooth": 12,
+    "n_estimators": 150,
     "learning_rate": 0.05,
-    "feature_fraction": 1.0,
-    "min_sum_hessian_in_leaf": 0.01,
-    "objective": "binary",
-    "class_weight": "balanced",  # TODO pass dict with weights for each class
-    "verbosity": -1,
+    "min_child_weight": 1,
 }
 
 N_FOLDS = 5
@@ -269,15 +211,6 @@ SUPPORTED_CATBOOST_METRICS = {
 }
 
 
-def is_catboost_estimator(estimator):
-    try:
-        from catboost import CatBoostClassifier, CatBoostRegressor
-
-        return isinstance(estimator, (CatBoostClassifier, CatBoostRegressor))
-    except ImportError:
-        return False
-
-
 @dataclass
 class _CrossValResults:
     metric: Optional[float]
@@ -418,7 +351,7 @@ class EstimatorWrapper:
                 if shaps is not None:
                     for feature, shap_value in shaps.items():
                         # shap_values_all_folds[feature] = shap_values_all_folds.get(feature, []) + shap_value.tolist()
-                        shap_values_all_folds[feature].append(shap_value)
+                        shap_values_all_folds[feature].extend(shap_value.tolist())
 
         if shap_values_all_folds:
             average_shap_values = {
@@ -494,18 +427,21 @@ class EstimatorWrapper:
         }
         if estimator is None:
             params = {}
+            params["has_time"] = has_date
+            # if metric_name.upper() in SUPPORTED_CATBOOST_METRICS:
+            #     params["eval_metric"] = SUPPORTED_CATBOOST_METRICS[metric_name.upper()]
             if target_type == ModelTaskType.MULTICLASS:
-                params = _get_add_params(params, LIGHTGBM_MULTICLASS_PARAMS)
+                params = _get_add_params(params, CATBOOST_MULTICLASS_PARAMS)
                 params = _get_add_params(params, add_params)
-                estimator = LightGBMWrapper(LGBMClassifier(**params), **kwargs)
+                estimator = CatBoostWrapper(CatBoostClassifier(**params), **kwargs)
             elif target_type == ModelTaskType.BINARY:
-                params = _get_add_params(params, LIGHTGBM_BINARY_PARAMS)
+                params = _get_add_params(params, CATBOOST_BINARY_PARAMS)
                 params = _get_add_params(params, add_params)
-                estimator = LightGBMWrapper(LGBMClassifier(**params), **kwargs)
+                estimator = CatBoostWrapper(CatBoostClassifier(**params), **kwargs)
             elif target_type == ModelTaskType.REGRESSION:
-                params = _get_add_params(params, LIGHTGBM_REGRESSION_PARAMS)
+                params = _get_add_params(params, CATBOOST_REGRESSION_PARAMS)
                 params = _get_add_params(params, add_params)
-                estimator = LightGBMWrapper(LGBMRegressor(**params), **kwargs)
+                estimator = CatBoostWrapper(CatBoostRegressor(**params), **kwargs)
             else:
                 raise Exception(bundle.get("metrics_unsupported_target_type").format(target_type))
         else:
@@ -514,21 +450,31 @@ class EstimatorWrapper:
             else:
                 estimator_copy = deepcopy(estimator)
             kwargs["estimator"] = estimator_copy
-            if is_catboost_estimator(estimator):
+            if isinstance(estimator, (CatBoostClassifier, CatBoostRegressor)):
                 if cat_features is not None:
                     for cat_feature in cat_features:
                         if cat_feature not in x.columns:
                             logger.error(
                                 f"Client cat_feature `{cat_feature}` not found in x columns: {x.columns.to_list()}"
                             )
-                    estimator_copy.set_params(cat_features=cat_features, has_time=has_date)
+                    estimator_copy.set_params(
+                        # cat_features=[x.columns.get_loc(cat_feature) for cat_feature in cat_features]
+                        cat_features=cat_features
+                    )
                 estimator = CatBoostWrapper(**kwargs)
             else:
-                if isinstance(estimator, (LGBMClassifier, LGBMRegressor)):
-                    estimator = LightGBMWrapper(**kwargs)
-                elif is_catboost_estimator(estimator):
-                    estimator = CatBoostWrapper(**kwargs)
-                else:
+                try:
+                    from lightgbm import LGBMClassifier, LGBMRegressor
+
+                    if isinstance(estimator, (LGBMClassifier, LGBMRegressor)):
+                        estimator = LightGBMWrapper(**kwargs)
+                    else:
+                        logger.warning(
+                            f"Unexpected estimator is used for metrics: {estimator}. "
+                            "Default strategy for category features will be used"
+                        )
+                        estimator = OtherEstimatorWrapper(**kwargs)
+                except ModuleNotFoundError:
                     logger.warning(
                         f"Unexpected estimator is used for metrics: {estimator}. "
                         "Default strategy for category features will be used"
@@ -541,7 +487,7 @@ class EstimatorWrapper:
 class CatBoostWrapper(EstimatorWrapper):
     def __init__(
         self,
-        estimator,
+        estimator: Union[CatBoostClassifier, CatBoostRegressor],
         scorer: Callable,
         metric_name: str,
         multiplier: int,
@@ -571,9 +517,6 @@ class CatBoostWrapper(EstimatorWrapper):
         x, y, groups, params = super()._prepare_to_fit(x, y)
 
         # Find embeddings
-        import catboost
-        from catboost import CatBoostClassifier
-
         if hasattr(CatBoostClassifier, "get_embedding_feature_indices"):
             emb_pattern = r"(.+)_emb\d+"
             self.emb_features = [c for c in x.columns if re.match(emb_pattern, c) and is_numeric_dtype(x[c])]
@@ -694,10 +637,8 @@ class CatBoostWrapper(EstimatorWrapper):
             else:
                 raise e
 
-    def calculate_shap(self, x: pd.DataFrame, y: pd.Series, estimator) -> Optional[Dict[str, float]]:
+    def calculate_shap(self, x: pd.DataFrame, y: pd.Series, estimator: CatBoost) -> Optional[Dict[str, float]]:
         try:
-            from catboost import Pool
-
             # Create Pool for fold data, if need (for example, when categorical features are present)
             fold_pool = Pool(
                 x,
@@ -754,12 +695,12 @@ class LightGBMWrapper(EstimatorWrapper):
         self.cat_features = None
 
     def _prepare_to_fit(self, x: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, pd.Series, np.ndarray, dict]:
-        x, y_numpy, groups, params = super()._prepare_to_fit(x, y)
+        x, y, groups, params = super()._prepare_to_fit(x, y)
         self.cat_features = _get_cat_features(x)
         x = fill_na_cat_features(x, self.cat_features)
         for feature in self.cat_features:
             x[feature] = x[feature].astype("category").cat.codes
-        if not is_numeric_dtype(y_numpy):
+        if not is_numeric_dtype(y):
             y = correct_string_target(y)
 
         return x, y, groups, params
@@ -774,34 +715,6 @@ class LightGBMWrapper(EstimatorWrapper):
             y = correct_string_target(y)
         return x, y, params
 
-    def calculate_shap(self, x: pd.DataFrame, y: pd.Series, estimator) -> Optional[Dict[str, float]]:
-        try:
-            import lightgbm as lgb
-            import shap
-
-            if not isinstance(estimator, (lgb.LGBMRegressor, lgb.LGBMClassifier)):
-                return None
-
-            explainer = shap.TreeExplainer(estimator)
-
-            shap_values = explainer.shap_values(x)
-
-            # For classification, shap_values is returned as a list for each class
-            # Take values for the positive class
-            if isinstance(shap_values, list):
-                shap_values = shap_values[1]
-
-            # Calculate mean absolute SHAP value for each feature
-            feature_importance = {}
-            for i, col in enumerate(x.columns):
-                feature_importance[col] = np.mean(np.abs(shap_values[:, i]))
-
-            return feature_importance
-
-        except Exception as e:
-            self.logger.warning(f"Failed to calculate SHAP values: {str(e)}")
-            return None
-
 
 class OtherEstimatorWrapper(EstimatorWrapper):
     def __init__(

{upgini-1.2.68a3832.dev11 → upgini-1.2.69}/src/upgini/resource_bundle/strings.properties

@@ -80,7 +80,6 @@ email_and_hem_simultanious=EMAIL and HEM search keys cannot be used simultaneous
 postal_code_without_country=COUNTRY search key required if POSTAL_CODE is present
 multiple_search_key=Search key {} passed multiple times
 unregistered_only_personal_keys=Only personal search keys used. Api_key from profile.upgini.com required for EMAIL/HEM, PHONE NUMBER or IPv4/IPv6 search keys\nSee docs https://github.com/upgini/upgini#-open-up-all-capabilities-of-upgini
-only_custom_keys=Only CUSTOM_KEY search keys were provided. At least one of DATE, COUNTRY, POSTAL_CODE, PHONE, EMAIL, HEM, IP should be provided
 search_key_not_found=Column `{}` from search_keys was not found in X dataframe: {}
 numeric_search_key_not_found=Index {} in search_keys is out of bounds for {} columns of X dataframe
 unsupported_search_key_type=Unsupported type of key in search_keys: {}

{upgini-1.2.68a3832.dev11 → upgini-1.2.69}/src/upgini/utils/deduplicate_utils.py

@@ -74,8 +74,6 @@ def remove_fintech_duplicates(
         # Checking for different dates by the same personal keys
         uniques = grouped_by_personal_cols[date_col].nunique()
         total = len(uniques)
-        if total == 0:
-            return segment_df, None
         diff_dates = len(uniques[uniques > 1])
         if diff_dates / total >= 0.6:
             return segment_df, None

{upgini-1.2.68a3832.dev11 → upgini-1.2.69}/src/upgini/utils/feature_info.py

@@ -90,8 +90,7 @@ class FeatureInfo:
 def _get_feature_sample(feature_meta: FeaturesMetadataV2, data: Optional[pd.DataFrame]) -> str:
     if data is not None and len(data) > 0 and feature_meta.name in data.columns:
         if len(data) > 3:
-            rand = np.random.RandomState(42)
-            feature_sample = rand.choice(data[feature_meta.name].dropna().unique(), 3).tolist()
+            feature_sample = np.random.choice(data[feature_meta.name].dropna().unique(), 3).tolist()
         else:
             feature_sample = data[feature_meta.name].dropna().unique().tolist()
         if len(feature_sample) > 0 and isinstance(feature_sample[0], float):

{upgini-1.2.68a3832.dev11 → upgini-1.2.69}/src/upgini/utils/sklearn_ext.py

@@ -9,6 +9,7 @@ from traceback import format_exc
 
 import numpy as np
 import scipy.sparse as sp
+from catboost import CatBoostClassifier, CatBoostRegressor
 from joblib import Parallel, logger
 from scipy.sparse import issparse
 from sklearn import config_context, get_config
@@ -341,14 +342,6 @@ def cross_validate(
         raise e
 
 
-def is_catboost_estimator(estimator):
-    try:
-        from catboost import CatBoostClassifier, CatBoostRegressor
-        return isinstance(estimator, (CatBoostClassifier, CatBoostRegressor))
-    except ImportError:
-        return False
-
-
 def _fit_and_score(
     estimator,
     X,
@@ -504,7 +497,7 @@ def _fit_and_score(
         if y_train is None:
             estimator.fit(X_train, **fit_params)
         else:
-            if is_catboost_estimator(estimator):
+            if isinstance(estimator, (CatBoostClassifier, CatBoostRegressor)):
                 fit_params = fit_params.copy()
                 fit_params["eval_set"] = [(X_test, y_test)]
             estimator.fit(X_train, y_train, **fit_params)

{upgini-1.2.68a3832.dev11 → upgini-1.2.69}/src/upgini/utils/sort.py

@@ -39,6 +39,11 @@ def sort_columns(
     sorted_keys = sorted(search_keys.keys(), key=lambda x: str(search_keys.get(x)))
     sorted_keys = [k for k in sorted_keys if k in df.columns and k not in exclude_columns]
 
+    duplicate_names = df.columns[df.columns.duplicated()].unique()
+    if len(duplicate_names) > 0:
+        logger.warning(f"WARNING: Found columns with duplicate names: {list(duplicate_names)}")
+        df = df[list(set(df.columns))]
+
     other_columns = sorted(
         [
             c

upgini-1.2.68a3832.dev11/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.68a3832.dev11"