upgini 1.1.280.dev0__py3-none-any.whl → 1.2.31a1__py3-none-any.whl

upgini/metrics.py

@@ -1,16 +1,17 @@
 from __future__ import annotations
 
+from dataclasses import dataclass
 import inspect
 import logging
 import re
+from collections import defaultdict
 from copy import deepcopy
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import catboost
 import numpy as np
 import pandas as pd
-from catboost import CatBoostClassifier, CatBoostRegressor
-from lightgbm import LGBMClassifier, LGBMRegressor
+from catboost import CatBoost, CatBoostClassifier, CatBoostRegressor, Pool
 from numpy import log1p
 from pandas.api.types import is_numeric_dtype
 from sklearn.metrics import check_scoring, get_scorer, make_scorer, roc_auc_score
@@ -210,6 +211,21 @@ SUPPORTED_CATBOOST_METRICS = {
 }
 
 
+@dataclass
+class _CrossValResults:
+    metric: Optional[float]
+    metric_std: Optional[float]
+    shap_values: Optional[Dict[str, float]]
+
+    def get_display_metric(self) -> Optional[str]:
+        if self.metric is None:
+            return None
+        elif self.metric_std is None:
+            return f"{self.metric:.3f}"
+        else:
+            return f"{self.metric:.3f} ± {self.metric_std:.3f}"
+
+
 class EstimatorWrapper:
     def __init__(
         self,
@@ -254,6 +270,7 @@ class EstimatorWrapper:
     def _prepare_data(
         self, x: pd.DataFrame, y: pd.Series, groups: Optional[np.ndarray] = None
     ) -> Tuple[pd.DataFrame, np.ndarray, np.ndarray]:
+        self.logger.info(f"Before preparing data columns: {x.columns.to_list()}")
         for c in x.columns:
             if is_numeric_dtype(x[c]):
                 x[c] = x[c].astype(float)
@@ -272,6 +289,10 @@ class EstimatorWrapper:
         else:
             x, y = self._remove_empty_target_rows(x, y)
 
+        # Make order of columns idempotent
+        x = x[sorted(x.columns)]
+
+        self.logger.info(f"After preparing data columns: {x.columns.to_list()}")
         return x, y, groups
 
     def _remove_empty_target_rows(self, x: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, pd.Series]:
@@ -287,17 +308,22 @@ class EstimatorWrapper:
         x, y, _ = self._prepare_data(x, y)
         return x, y, {}
 
+    def calculate_shap(self, x: pd.DataFrame, y: pd.Series, estimator) -> Optional[Dict[str, float]]:
+        return None
+
     def cross_val_predict(
         self, x: pd.DataFrame, y: np.ndarray, baseline_score_column: Optional[Any] = None
-    ) -> Optional[float]:
+    ) -> _CrossValResults:
         x, y, groups, fit_params = self._prepare_to_fit(x, y)
 
         if x.shape[1] == 0:
-            return None
+            return _CrossValResults(metric=None, metric_std=None, shap_values=None)
 
         scorer = check_scoring(self.estimator, scoring=self.scorer)
 
+        shap_values_all_folds = defaultdict(list)
         if baseline_score_column is not None and self.metric_name == "GINI":
+            self.logger.info("Calculate baseline GINI on passed baseline_score_column and target")
             metric = roc_auc_score(y, x[baseline_score_column])
         else:
             cv_results = cross_validate(
@@ -314,25 +340,68 @@ class EstimatorWrapper:
             metrics_by_fold = cv_results["test_score"]
             self.cv_estimators = cv_results["estimator"]
 
-            metric = np.mean(metrics_by_fold) * self.multiplier
-        return self.post_process_metric(metric)
+            self.check_fold_metrics(metrics_by_fold)
+
+            metric, metric_std = self._calculate_metric_from_folds(metrics_by_fold)
+
+            splits = self.cv.split(x, y, groups)
+
+            for estimator, split in zip(self.cv_estimators, splits):
+                _, validation_idx = split
+                cv_x = x.iloc[validation_idx]
+                cv_y = y[validation_idx]
+                shaps = self.calculate_shap(cv_x, cv_y, estimator)
+                if shaps is not None:
+                    for feature, shap_value in shaps.items():
+                        # shap_values_all_folds[feature] = shap_values_all_folds.get(feature, []) + shap_value.tolist()
+                        shap_values_all_folds[feature].extend(shap_value.tolist())
+
+        if shap_values_all_folds:
+            average_shap_values = {
+                feature: np.mean(np.array(shaps)) for feature, shaps in shap_values_all_folds.items() if len(shaps) > 0
+            }
+            if len(average_shap_values) == 0:
+                average_shap_values = None
+            else:
+                average_shap_values = self.process_shap_values(average_shap_values)
+        else:
+            average_shap_values = None
+
+        return _CrossValResults(metric=metric, metric_std=metric_std, shap_values=average_shap_values)
+
+    def process_shap_values(self, shap_values: Dict[str, float]) -> Dict[str, float]:
+        return shap_values
+
+    def check_fold_metrics(self, metrics_by_fold: List[float]):
+        first_metric_sign = 1 if metrics_by_fold[0] >= 0 else -1
+        for metric in metrics_by_fold[1:]:
+            if first_metric_sign * metric < 0:
+                self.logger.warning(f"Sign of metrics differs between folds: {metrics_by_fold}")
 
     def post_process_metric(self, metric: float) -> float:
         if self.metric_name == "GINI":
             metric = 2 * metric - 1
         return metric
 
-    def calculate_metric(self, x: pd.DataFrame, y: np.ndarray, baseline_score_column: Optional[Any] = None) -> float:
+    def calculate_metric(
+        self, x: pd.DataFrame, y: np.ndarray, baseline_score_column: Optional[Any] = None
+    ) -> _CrossValResults:
         x, y, _ = self._prepare_to_calculate(x, y)
         if baseline_score_column is not None and self.metric_name == "GINI":
-            metric = roc_auc_score(y, x[baseline_score_column])
+            metric, metric_std = roc_auc_score(y, x[baseline_score_column]), None
         else:
             metrics = []
             for est in self.cv_estimators:
                 metrics.append(self.scorer(est, x, y))
 
-            metric = np.mean(metrics) * self.multiplier
-        return self.post_process_metric(metric)
+            metric, metric_std = self._calculate_metric_from_folds(metrics)
+        return _CrossValResults(metric=metric, metric_std=metric_std, shap_values=None)
+
+    def _calculate_metric_from_folds(self, metrics_by_fold: List[float]) -> Tuple[float, float]:
+        metrics_by_fold = [self.post_process_metric(m) for m in metrics_by_fold]
+        metric = np.mean(metrics_by_fold) * self.multiplier
+        metric_std = np.std(metrics_by_fold) * np.abs(self.multiplier)
+        return metric, metric_std
 
     @staticmethod
     def create(
@@ -346,6 +415,7 @@ class EstimatorWrapper:
         text_features: Optional[List[str]] = None,
         add_params: Optional[Dict[str, Any]] = None,
         groups: Optional[List[str]] = None,
+        has_date: Optional[bool] = None,
     ) -> EstimatorWrapper:
         scorer, metric_name, multiplier = _get_scorer(target_type, scoring)
         kwargs = {
@@ -359,7 +429,8 @@ class EstimatorWrapper:
             "logger": logger,
         }
         if estimator is None:
-            params = dict()
+            params = {}
+            params["has_time"] = has_date
             # if metric_name.upper() in SUPPORTED_CATBOOST_METRICS:
             #     params["eval_metric"] = SUPPORTED_CATBOOST_METRICS[metric_name.upper()]
             if target_type == ModelTaskType.MULTICLASS:
@@ -390,11 +461,14 @@ class EstimatorWrapper:
                                 f"Client cat_feature `{cat_feature}` not found in x columns: {x.columns.to_list()}"
                             )
                     estimator_copy.set_params(
-                        cat_features=[x.columns.get_loc(cat_feature) for cat_feature in cat_features]
+                        # cat_features=[x.columns.get_loc(cat_feature) for cat_feature in cat_features]
+                        cat_features=cat_features
                     )
                 estimator = CatBoostWrapper(**kwargs)
             else:
                 try:
+                    from lightgbm import LGBMClassifier, LGBMRegressor
+
                     if isinstance(estimator, (LGBMClassifier, LGBMRegressor)):
                         estimator = LightGBMWrapper(**kwargs)
                     else:
@@ -439,6 +513,7 @@ class CatBoostWrapper(EstimatorWrapper):
         )
         self.cat_features = None
         self.emb_features = None
+        self.grouped_embedding_features = None
         self.exclude_features = []
 
     def _prepare_to_fit(self, x: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, np.ndarray, np.ndarray, dict]:
@@ -448,17 +523,16 @@ class CatBoostWrapper(EstimatorWrapper):
         if hasattr(CatBoostClassifier, "get_embedding_feature_indices"):
             emb_pattern = r"(.+)_emb\d+"
             self.emb_features = [c for c in x.columns if re.match(emb_pattern, c) and is_numeric_dtype(x[c])]
-            embedding_features = []
             if len(self.emb_features) > 3:  # There is no reason to reduce embeddings dimension with less than 4
                 self.logger.info(
                     "Embedding features count more than 3, so group them into one vector for CatBoost: "
                     f"{self.emb_features}"
                 )
-                x, embedding_features = self.group_embeddings(x)
-                params["embedding_features"] = embedding_features
+                x, self.grouped_embedding_features = self.group_embeddings(x)
+                params["embedding_features"] = self.grouped_embedding_features
             else:
                 self.logger.info(f"Embedding features count less than 3, so use them separately: {self.emb_features}")
-                self.emb_features = []
+                self.grouped_embedding_features = None
         else:
             self.logger.warning(f"Embedding features are not supported by Catboost version {catboost.__version__}")
 
@@ -474,15 +548,17 @@ class CatBoostWrapper(EstimatorWrapper):
             self.logger.warning(f"Text features are not supported by this Catboost version {catboost.__version__}")
 
         # Find rest categorical features
-        self.cat_features = _get_cat_features(x, self.text_features, embedding_features)
-        x = fill_na_cat_features(x, self.cat_features)
+        self.cat_features = _get_cat_features(x, self.text_features, self.grouped_embedding_features)
+        # x = fill_na_cat_features(x, self.cat_features)
         unique_cat_features = []
         for name in self.cat_features:
             # Remove constant categorical features
             if x[name].nunique() > 1:
                 unique_cat_features.append(name)
             else:
+                self.logger.info(f"Drop column {name} on preparing data for fit")
                 x = x.drop(columns=name)
+                self.exclude_features.append(name)
         self.cat_features = unique_cat_features
         if (
             hasattr(self.estimator, "get_param")
@@ -510,46 +586,90 @@ class CatBoostWrapper(EstimatorWrapper):
         emb_name = "__grouped_embeddings"
         df = df.copy()
         df[self.emb_features] = df[self.emb_features].fillna(0.0)
-        df[emb_name] = df[self.emb_features].values.tolist()
+        df[emb_name] = pd.Series(df[self.emb_features].values.tolist())
         df = df.drop(columns=self.emb_features)
 
         return df, [emb_name]
 
+    def process_shap_values(self, shap_values: Dict[str, float]) -> Dict[str, float]:
+        if "__grouped_embeddings" in shap_values:
+            for emb_feature in self.emb_features:
+                shap_values[emb_feature] = shap_values["__grouped_embeddings"]
+            del shap_values["__grouped_embeddings"]
+        return shap_values
+
     def _prepare_to_calculate(self, x: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, np.ndarray, dict]:
         if self.exclude_features:
             x = x.drop(columns=self.exclude_features)
         x, y, params = super()._prepare_to_calculate(x, y)
         if self.text_features:
             params["text_features"] = self.text_features
-        if self.emb_features:
+        if self.grouped_embedding_features:
             x, emb_columns = self.group_embeddings(x)
             params["embedding_features"] = emb_columns
         if self.cat_features:
-            x = fill_na_cat_features(x, self.cat_features)
+            # x = fill_na_cat_features(x, self.cat_features)
             params["cat_features"] = self.cat_features
 
         return x, y, params
 
     def cross_val_predict(
         self, x: pd.DataFrame, y: np.ndarray, baseline_score_column: Optional[Any] = None
-    ) -> Optional[float]:
+    ) -> _CrossValResults:
         try:
             return super().cross_val_predict(x, y, baseline_score_column)
         except Exception as e:
             if "Dictionary size is 0" in e.args[0] and self.text_features:
                 high_cardinality_features = FeaturesValidator.find_high_cardinality(x[self.text_features])
-                self.logger.warning(
-                    "Calculate metrics has problem with CatBoost text features. Try to remove high cardinality"
-                    f" text features {high_cardinality_features} and retry"
-                )
+                if len(high_cardinality_features) == 0:
+                    high_cardinality_features = self.text_features
+                    self.logger.warning(
+                        "Calculate metrics has problem with CatBoost text features. High cardinality features not found"
+                        f". Try to remove all text features {high_cardinality_features} and retry"
+                    )
+                else:
+                    self.logger.warning(
+                        "Calculate metrics has problem with CatBoost text features. Try to remove high cardinality"
+                        f" text features {high_cardinality_features} and retry"
+                    )
                 for f in high_cardinality_features:
                     self.text_features.remove(f)
                     self.exclude_features.append(f)
-                    x = x.drop(columns=f)
+                    x = x.drop(columns=f, errors="ignore")
                 return super().cross_val_predict(x, y, baseline_score_column)
             else:
                 raise e
 
+    def calculate_shap(self, x: pd.DataFrame, y: pd.Series, estimator: CatBoost) -> Optional[Dict[str, float]]:
+        try:
+            # Create Pool for fold data, if need (for example, when categorical features are present)
+            fold_pool = Pool(
+                x,
+                y,
+                cat_features=self.cat_features,
+                text_features=self.text_features,
+                embedding_features=self.grouped_embedding_features,
+            )
+
+            # Get SHAP values of current estimator
+            shap_values_fold = estimator.get_feature_importance(data=fold_pool, type="ShapValues")
+
+            # Remove last columns (base value) and flatten
+            if self.target_type == ModelTaskType.MULTICLASS:
+                all_shaps = shap_values_fold[:, :, :-1]
+                all_shaps = [all_shaps[:, :, k].flatten() for k in range(all_shaps.shape[2])]
+            else:
+                all_shaps = shap_values_fold[:, :-1]
+                all_shaps = [all_shaps[:, k].flatten() for k in range(all_shaps.shape[1])]
+
+            all_shaps = np.abs(all_shaps)
+
+            return dict(zip(estimator.feature_names_, all_shaps))
+
+        except Exception:
+            self.logger.exception("Failed to recalculate new SHAP values")
+            return None
+
 
 class LightGBMWrapper(EstimatorWrapper):
     def __init__(
@@ -653,14 +773,24 @@ class OtherEstimatorWrapper(EstimatorWrapper):
 
 
 def validate_scoring_argument(scoring: Union[Callable, str, None]):
-    if isinstance(scoring, str) and scoring is not None:
+    if scoring is None:
+        return
+
+    if isinstance(scoring, str):
         _get_scorer_by_name(scoring)
-    elif isinstance(scoring, Callable):
-        spec = inspect.getfullargspec(scoring)
-        if len(spec.args) < 3:
-            raise ValidationError(
-                f"Invalid scoring function passed {scoring}. It should accept 3 input arguments: estimator, x, y"
-            )
+        return
+
+    if not isinstance(scoring, Callable):
+        raise ValidationError(
+            f"Invalid scoring argument passed {scoring}. It should be string with scoring name or function"
+            " that accepts 3 input arguments: estimator, x, y"
+        )
+
+    spec = inspect.getfullargspec(scoring)
+    if len(spec.args) < 3:
+        raise ValidationError(
+            f"Invalid scoring function passed {scoring}. It should accept 3 input arguments: estimator, x, y"
+        )
 
 
 def _get_scorer_by_name(scoring: str) -> Tuple[Callable, str, int]:

upgini/normalizer/normalize_utils.py

@@ -0,0 +1,197 @@
+import hashlib
+from logging import Logger, getLogger
+from typing import Dict, List, Tuple
+
+import numpy as np
+import pandas as pd
+from pandas.api.types import is_bool_dtype as is_bool
+from pandas.api.types import is_datetime64_any_dtype as is_datetime
+from pandas.api.types import (
+    is_float_dtype,
+    is_numeric_dtype,
+    is_object_dtype,
+    is_string_dtype,
+)
+
+from upgini.errors import ValidationError
+from upgini.metadata import (
+    ENTITY_SYSTEM_RECORD_ID,
+    EVAL_SET_INDEX,
+    SEARCH_KEY_UNNEST,
+    SYSTEM_RECORD_ID,
+    TARGET,
+    SearchKey,
+)
+from upgini.resource_bundle import ResourceBundle, get_custom_bundle
+from upgini.utils import find_numbers_with_decimal_comma
+from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
+from upgini.utils.phone_utils import PhoneSearchKeyConverter
+
+
+class Normalizer:
+
+    MAX_STRING_FEATURE_LENGTH = 24573
+
+    def __init__(
+        self,
+        bundle: ResourceBundle = None,
+        logger: Logger = None,
+    ):
+        self.bundle = bundle or get_custom_bundle()
+        self.logger = logger or getLogger()
+        self.columns_renaming = {}
+        self.search_keys = {}
+        self.generated_features = []
+        self.removed_features = []
+
+    def normalize(
+        self, df: pd.DataFrame, search_keys: Dict[str, SearchKey], generated_features: List[str]
+    ) -> Tuple[pd.DataFrame, Dict[str, SearchKey], List[str]]:
+        self.search_keys = search_keys.copy()
+        self.generated_features = generated_features.copy()
+
+        df = df.copy()
+        df = self._rename_columns(df)
+
+        df = self._remove_dates_from_features(df)
+
+        df = self._cut_too_long_string_values(df)
+
+        df = self._convert_bools(df)
+
+        df = self._convert_float16(df)
+
+        df = self._correct_decimal_comma(df)
+
+        df = self._convert_phone_numbers(df)
+
+        df = self.__convert_features_types(df)
+
+        return df, self.search_keys, self.generated_features
+
+    def _rename_columns(self, df: pd.DataFrame):
+        # logger.info("Replace restricted symbols in column names")
+        new_columns = []
+        dup_counter = 0
+        for column in df.columns:
+            if (
+                column
+                in [
+                    TARGET,
+                    EVAL_SET_INDEX,
+                    SYSTEM_RECORD_ID,
+                    ENTITY_SYSTEM_RECORD_ID,
+                    SEARCH_KEY_UNNEST,
+                    DateTimeSearchKeyConverter.DATETIME_COL,
+                ]
+                + self.generated_features
+            ):
+                self.columns_renaming[column] = column
+                new_columns.append(column)
+                continue
+
+            new_column = str(column)
+            suffix = hashlib.sha256(new_column.encode()).hexdigest()[:6]
+            if len(new_column) == 0:
+                raise ValidationError(self.bundle.get("dataset_empty_column_names"))
+            # db limit for column length
+            if len(new_column) > 250:
+                new_column = new_column[:250]
+
+            # make column name unique relative to server features
+            new_column = f"{new_column}_{suffix}"
+
+            new_column = new_column.lower()
+
+            # if column starts with non alphabetic symbol then add "a" to the beginning of string
+            if ord(new_column[0]) not in range(ord("a"), ord("z") + 1):
+                new_column = "a" + new_column
+
+            # replace unsupported characters to "_"
+            for idx, c in enumerate(new_column):
+                if ord(c) not in range(ord("a"), ord("z") + 1) and ord(c) not in range(ord("0"), ord("9") + 1):
+                    new_column = new_column[:idx] + "_" + new_column[idx + 1 :]
+
+            if new_column in new_columns:
+                new_column = f"{new_column}_{dup_counter}"
+                dup_counter += 1
+            new_columns.append(new_column)
+
+            # df.columns.values[col_idx] = new_column
+            # rename(columns={column: new_column}, inplace=True)
+
+            if new_column != column and column in self.search_keys:
+                self.search_keys[new_column] = self.search_keys[column]
+                del self.search_keys[column]
+            self.columns_renaming[new_column] = str(column)
+        df.columns = new_columns
+        return df
+
+    def _get_features(self, df: pd.DataFrame) -> List[str]:
+        system_columns = [ENTITY_SYSTEM_RECORD_ID, EVAL_SET_INDEX, SEARCH_KEY_UNNEST, SYSTEM_RECORD_ID, TARGET]
+        features = set(df.columns) - set(self.search_keys.keys()) - set(system_columns)
+        return sorted(list(features))
+
+    def _remove_dates_from_features(self, df: pd.DataFrame):
+        features = self._get_features(df)
+
+        for f in features:
+            if is_datetime(df[f]) or isinstance(df[f].dtype, pd.PeriodDtype):
+                self.removed_features.append(f)
+                df.drop(columns=f, inplace=True)
+
+        return df
+
+    def _cut_too_long_string_values(self, df: pd.DataFrame):
+        """Check that string values less than maximum characters for LLM"""
+        # logger.info("Validate too long string values")
+        for col in df.columns:
+            if is_string_dtype(df[col]) or is_object_dtype(df[col]):
+                max_length: int = df[col].astype("str").str.len().max()
+                if max_length > self.MAX_STRING_FEATURE_LENGTH:
+                    df[col] = df[col].astype("str").str.slice(stop=self.MAX_STRING_FEATURE_LENGTH)
+
+        return df
+
+    @staticmethod
+    def _convert_bools(df: pd.DataFrame):
+        """Convert bool columns to string"""
+        # logger.info("Converting bool to int")
+        for col in df.columns:
+            if is_bool(df[col]):
+                df[col] = df[col].astype("str")
+        return df
+
+    @staticmethod
+    def _convert_float16(df: pd.DataFrame):
+        """Convert float16 to float"""
+        # logger.info("Converting float16 to float")
+        for col in df.columns:
+            if is_float_dtype(df[col]):
+                df[col] = df[col].astype("float64")
+        return df
+
+    def _correct_decimal_comma(self, df: pd.DataFrame):
+        """Check DataSet for decimal commas and fix them"""
+        # logger.info("Correct decimal commas")
+        columns_to_fix = find_numbers_with_decimal_comma(df)
+        if len(columns_to_fix) > 0:
+            self.logger.warning(f"Convert strings with decimal comma to float: {columns_to_fix}")
+            for col in columns_to_fix:
+                df[col] = df[col].astype("string").str.replace(",", ".", regex=False).astype(np.float64)
+        return df
+
+    def _convert_phone_numbers(self, df: pd.DataFrame) -> pd.DataFrame:
+        maybe_country_col = SearchKey.find_key(self.search_keys, SearchKey.COUNTRY)
+        for phone_col in SearchKey.find_all_keys(self.search_keys, SearchKey.PHONE):
+            converter = PhoneSearchKeyConverter(phone_col, maybe_country_col)
+            df = converter.convert(df)
+        return df
+
+    def __convert_features_types(self, df: pd.DataFrame):
+        # self.logger.info("Convert features to supported data types")
+
+        for f in self._get_features(df):
+            if not is_numeric_dtype(df[f]):
+                df[f] = df[f].astype("string")
+        return df