upgini 1.1.282a3418.post3__py3-none-any.whl → 1.1.284__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.1.282a3418-3"
+__version__ = "1.1.284"

upgini/autofe/all_operands.py

@@ -1,7 +1,7 @@
 from typing import Dict
 
 from upgini.autofe.binary import Add, Divide, Max, Min, Multiply, Sim, Subtract
-from upgini.autofe.date import DateDiff, DateDiffType2, DateListDiff, DateListDiffBounded, DatePercentile
+from upgini.autofe.date import DateDiff, DateDiffType2, DateListDiff, DateListDiffBounded
 from upgini.autofe.groupby import GroupByThenAgg, GroupByThenRank
 from upgini.autofe.operand import Operand
 from upgini.autofe.unary import Abs, Floor, Freq, Log, Residual, Sigmoid, Sqrt, Square
@@ -49,7 +49,6 @@ ALL_OPERANDS: Dict[str, Operand] = {
         DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=30, upper_bound=45),
         DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=45, upper_bound=60),
         DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=60),
-        DatePercentile(),
     ]
 }
 

upgini/autofe/date.py

@@ -1,9 +1,9 @@
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Optional, Union
 
 import numpy as np
 import pandas as pd
 from pandas.core.arrays.timedeltas import TimedeltaArray
-from pydantic import BaseModel, validator
+from pydantic import BaseModel
 
 from upgini.autofe.operand import PandasOperand
 
@@ -27,17 +27,6 @@ class DateDiff(PandasOperand, DateDiffMixin):
     is_binary = True
     has_symmetry_importance = True
 
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = super().get_params()
-        res.update(
-            {
-                "diff_unit": self.diff_unit,
-                "left_unit": self.left_unit,
-                "right_unit": self.right_unit,
-            }
-        )
-        return res
-
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
         left = self._convert_to_date(left, self.left_unit)
         right = self._convert_to_date(right, self.right_unit)
@@ -53,17 +42,6 @@ class DateDiffType2(PandasOperand, DateDiffMixin):
     is_binary = True
     has_symmetry_importance = True
 
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = super().get_params()
-        res.update(
-            {
-                "diff_unit": self.diff_unit,
-                "left_unit": self.left_unit,
-                "right_unit": self.right_unit,
-            }
-        )
-        return res
-
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
         left = self._convert_to_date(left, self.left_unit)
         right = self._convert_to_date(right, self.right_unit)
@@ -86,15 +64,6 @@ class DateListDiff(PandasOperand, DateDiffMixin):
     has_symmetry_importance = True
     aggregation: str
 
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = super().get_params()
-        res.update(
-            {
-                "aggregation": self.aggregation,
-            }
-        )
-        return res
-
     def __init__(self, **data: Any) -> None:
         if "name" not in data:
             data["name"] = f"date_diff_{data.get('aggregation')}"
@@ -147,55 +116,3 @@ class DateListDiffBounded(DateListDiff):
     def _agg(self, x):
         x = x[(x >= (self.lower_bound or -np.inf)) & (x < (self.upper_bound or np.inf))]
         return super()._agg(x)
-
-
-class DatePercentile(PandasOperand):
-    name = "date_per"
-    is_binary = True
-    output_type = "float"
-
-    date_unit: Optional[str] = None
-    zero_month: Optional[int]
-    zero_year: Optional[int]
-    zero_bounds: Optional[List[float]]
-    step: int = 30
-
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = super().get_params()
-        res.update(
-            {
-                "date_unit": self.date_unit,
-                "zero_month": self.zero_month,
-                "zero_year": self.zero_year,
-                "zero_bounds": self.zero_bounds,
-                "step": self.step,
-            }
-        )
-        return res
-
-    @validator("zero_bounds", pre=True)
-    def validate_bounds(cls, value):
-        if value is None or isinstance(value, list):
-            return value
-        elif isinstance(value, str):
-            return value[1:-1].split(", ")
-
-    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        # Assuming that left is a date column, right is a feature column
-        left = pd.to_datetime(left, unit=self.date_unit)
-        months = left.dt.month
-        years = left.dt.year
-
-        month_diffs = 12 * (years - (self.zero_year or 0)) + (months - (self.zero_month or 0))
-        bounds = month_diffs.apply(
-            lambda d: np.array(self.zero_bounds if self.zero_bounds is not None else []) + d * 30
-        )
-
-        return right.index.to_series().apply(lambda i: self.__perc(right[i], bounds[i]))
-
-    def __perc(self, f, bounds):
-        hit = np.where(f >= bounds)[0]
-        if hit.size > 0:
-            return np.max(hit) * 10
-        else:
-            return np.nan

upgini/autofe/feature.py

@@ -1,6 +1,6 @@
 import hashlib
 import itertools
-from typing import Dict, List, Optional, Set, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 import numpy as np
 import pandas as pd
@@ -16,12 +16,6 @@ class Column:
         self.data = data
         self.calculate_all = calculate_all
 
-    def get_display_name(self, cache: bool = True, shorten: bool = False, **kwargs) -> str:
-        return self.name
-
-    def set_op_params(self, params: Dict[str, str]) -> "Column":
-        return self
-
     def rename_columns(self, mapping: Dict[str, str]) -> "Column":
         self.name = self._unhash(mapping.get(self.name) or self.name)
         return self
@@ -75,30 +69,19 @@ class Feature:
         self.cached_display_name = cached_display_name
         self.alias = alias
 
-    def set_op_params(self, params: Optional[Dict[str, str]]) -> "Feature":
-        obj_dict = self.op.dict().copy()
-        obj_dict.update(params or {})
-        self.op = self.op.__class__.parse_obj(obj_dict)
+    def set_op_params(self, params: Dict[str, str]) -> "Feature":
         self.op.set_params(params)
-
-        for child in self.children:
-            child.set_op_params(params)
         return self
 
     def get_hash(self) -> str:
-        return hashlib.sha256(
-            "_".join([self.op.name] + [ch.get_display_name() for ch in self.children]).encode("utf-8")
-        ).hexdigest()[:8]
+        return hashlib.sha256("_".join([self.op.name] + [ch.name for ch in self.children]).encode("utf-8")).hexdigest()[
+            :8
+        ]
 
     def set_alias(self, alias: str) -> "Feature":
         self.alias = alias
         return self
 
-    def get_all_operand_names(self) -> Set[str]:
-        return {self.op.name}.union(
-            {n for f in self.children if isinstance(f, Feature) for n in f.get_all_operand_names()}
-        )
-
     def rename_columns(self, mapping: Dict[str, str]) -> "Feature":
         for child in self.children:
             child.rename_columns(mapping)

upgini/autofe/operand.py

@@ -25,10 +25,8 @@ class Operand(BaseModel):
         self.params = params
         return self
 
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = {"alias": self.alias}
-        res.update(self.params or {})
-        return res
+    def get_params(self) -> Dict[str, str]:
+        return self.params
 
 
 MAIN_COLUMN = "main_column"

upgini/features_enricher.py

@@ -935,7 +935,7 @@ class FeaturesEnricher(TransformerMixin):
                     metric = wrapper.metric_name
                     multiplier = wrapper.multiplier
 
-                    # 1 If client features are presented - fit and predict with KFold CatBoost model
+                    # 1 If client features are presented - fit and predict with KFold estimator
                     # on etalon features and calculate baseline metric
                     etalon_metric = None
                     baseline_estimator = None
@@ -962,9 +962,15 @@ class FeaturesEnricher(TransformerMixin):
                         etalon_metric = baseline_estimator.cross_val_predict(
                             fitting_X, y_sorted, self.baseline_score_column
                         )
-                        self.logger.info(f"Baseline {metric} on train client features: {etalon_metric}")
+                        if etalon_metric is None:
+                            self.logger.info(
+                                f"Baseline {metric} on train client features is None (maybe all features was removed)"
+                            )
+                            baseline_estimator = None
+                        else:
+                            self.logger.info(f"Baseline {metric} on train client features: {etalon_metric}")
 
-                    # 2 Fit and predict with KFold Catboost model on enriched tds
+                    # 2 Fit and predict with KFold estimator on enriched tds
                     # and calculate final metric (and uplift)
                     enriched_estimator = None
                     if set(fitting_X.columns) != set(fitting_enriched_X.columns):
@@ -986,11 +992,15 @@ class FeaturesEnricher(TransformerMixin):
                             has_date=has_date,
                         )
                         enriched_metric = enriched_estimator.cross_val_predict(fitting_enriched_X, enriched_y_sorted)
-                        self.logger.info(f"Enriched {metric} on train combined features: {enriched_metric}")
-                        if etalon_metric is not None:
-                            uplift = (enriched_metric - etalon_metric) * multiplier
-                        else:
+                        if etalon_metric is None:
+                            self.logger.warning(
+                                f"Enriched {metric} on train combined features is None (maybe all features was removed)"
+                            )
+                            enriched_estimator = None
                             uplift = None
+                        else:
+                            self.logger.info(f"Enriched {metric} on train combined features: {enriched_metric}")
+                            uplift = (enriched_metric - etalon_metric) * multiplier
                     else:
                         enriched_metric = None
                         uplift = None
@@ -1442,12 +1452,15 @@ class FeaturesEnricher(TransformerMixin):
             if len(decimal_columns_to_fix) > 0:
                 for col in decimal_columns_to_fix:
                     fitting_eval_X[col] = (
-                        fitting_eval_X[col].astype("string").str.replace(",", ".", regex=False).astype(np.float64)
+                        fitting_eval_X[col]
+                        .astype("string").str
+                        .replace(",", ".", regex=False)
+                        .astype(np.float64)
                     )
                     fitting_enriched_eval_X[col] = (
                         fitting_enriched_eval_X[col]
-                        .astype("string")
-                        .str.replace(",", ".", regex=False)
+                        .astype("string").str
+                        .replace(",", ".", regex=False)
                         .astype(np.float64)
                     )
 
@@ -3266,16 +3279,10 @@ class FeaturesEnricher(TransformerMixin):
 
             descriptions = []
             for m in autofe_meta:
+                autofe_feature = Feature.from_formula(m.formula)
                 orig_to_hashed = {base_column.original_name: base_column.hashed_name for base_column in m.base_columns}
-
-                autofe_feature = (
-                    Feature.from_formula(m.formula)
-                    .set_display_index(m.display_index)
-                    .set_alias(m.alias)
-                    .set_op_params(m.operator_params or {})
-                    .rename_columns(orig_to_hashed)
-                )
-
+                autofe_feature.rename_columns(orig_to_hashed)
+                autofe_feature.set_display_index(m.display_index)
                 if autofe_feature.op.is_vector:
                     continue
 
@@ -3296,7 +3303,7 @@ class FeaturesEnricher(TransformerMixin):
                     description[f"Feature {feature_idx}"] = bc.hashed_name
                     feature_idx += 1
 
-                description["Function"] = ",".join(sorted(autofe_feature.get_all_operand_names()))
+                description["Function"] = autofe_feature.op.name
 
                 descriptions.append(description)
 

upgini/metadata.py

@@ -256,11 +256,9 @@ class BaseColumnMetadata(BaseModel):
 
 
 class GeneratedFeatureMetadata(BaseModel):
-    alias: Optional[str]
-    formula: str
+    formula: str  # on hashed names
     display_index: str
     base_columns: List[BaseColumnMetadata]
-    operator_params: Optional[Dict[str, str]]
 
 
 class ProviderTaskMetadataV2(BaseModel):

upgini/metrics.py

@@ -298,6 +298,7 @@ class EstimatorWrapper:
         scorer = check_scoring(self.estimator, scoring=self.scorer)
 
         if baseline_score_column is not None and self.metric_name == "GINI":
+            self.logger.info("Calculate baseline GINI on passed baseline_score_column and target")
             metric = roc_auc_score(y, x[baseline_score_column])
         else:
             cv_results = cross_validate(
@@ -548,14 +549,21 @@ class CatBoostWrapper(EstimatorWrapper):
         except Exception as e:
             if "Dictionary size is 0" in e.args[0] and self.text_features:
                 high_cardinality_features = FeaturesValidator.find_high_cardinality(x[self.text_features])
-                self.logger.warning(
-                    "Calculate metrics has problem with CatBoost text features. Try to remove high cardinality"
-                    f" text features {high_cardinality_features} and retry"
-                )
+                if len(high_cardinality_features) == 0:
+                    high_cardinality_features = self.text_features
+                    self.logger.warning(
+                        "Calculate metrics has problem with CatBoost text features. High cardinality features not found"
+                        f". Try to remove all text features {high_cardinality_features} and retry"
+                    )
+                else:
+                    self.logger.warning(
+                        "Calculate metrics has problem with CatBoost text features. Try to remove high cardinality"
+                        f" text features {high_cardinality_features} and retry"
+                    )
                 for f in high_cardinality_features:
                     self.text_features.remove(f)
                     self.exclude_features.append(f)
-                    x = x.drop(columns=f)
+                    x = x.drop(columns=f, errors="ignore")
                 return super().cross_val_predict(x, y, baseline_score_column)
             else:
                 raise e

upgini/utils/sklearn_ext.py

@@ -17,7 +17,7 @@ from sklearn.base import clone, is_classifier
 from sklearn.exceptions import FitFailedWarning, NotFittedError
 from sklearn.metrics import check_scoring
 from sklearn.metrics._scorer import _MultimetricScorer
-from sklearn.model_selection import check_cv
+from sklearn.model_selection import StratifiedKFold, check_cv
 from sklearn.utils.fixes import np_version, parse_version
 from sklearn.utils.validation import indexable
 
@@ -312,25 +312,34 @@ def cross_validate(
                 ret[key] = train_scores_dict[name]
 
         return ret
-    except Exception:
+    except ValueError as e:
         # logging.exception("Failed to execute overriden cross_validate. Fallback to original")
-        raise
-        # fit_params["use_best_model"] = False
-        # return original_cross_validate(
-        #     estimator,
-        #     X,
-        #     y,
-        #     groups=groups,
-        #     scoring=scoring,
-        #     cv=cv,
-        #     n_jobs=n_jobs,
-        #     verbose=verbose,
-        #     fit_params=fit_params,
-        #     pre_dispatch=pre_dispatch,
-        #     return_train_score=return_train_score,
-        #     return_estimator=return_estimator,
-        #     error_score=error_score,
-        # )
+        if hasattr(e, "args") and len(e.args) > 0 and "Only one class present in y_true" in e.args[0]:
+            # Try change CV to StratifiedKFold and retry
+            if hasattr(cv, "shuffle"):
+                shuffle = cv.shuffle
+            else:
+                shuffle = False
+            if hasattr(cv, "random_state"):
+                random_state = cv.random_state
+            else:
+                random_state = None
+            return cross_validate(
+                estimator,
+                x,
+                y,
+                groups=groups,
+                scoring=scoring,
+                cv=StratifiedKFold(n_splits=cv.get_n_splits(), shuffle=shuffle, random_state=random_state),
+                n_jobs=n_jobs,
+                verbose=verbose,
+                fit_params=fit_params,
+                pre_dispatch=pre_dispatch,
+                return_train_score=return_train_score,
+                return_estimator=return_estimator,
+                error_score=error_score,
+            )
+        raise e
 
 
 def _fit_and_score(

{upgini-1.1.282a3418.post3.dist-info → upgini-1.1.284.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.1.282a3418.post3
+Version: 1.1.284
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -839,4 +839,4 @@ Some convenient ways to start contributing are:
 - [More perks for registered users](https://profile.upgini.com)
 
 <sup>😔 Found mistype or a bug in code snippet? Our bad! <a href="https://github.com/upgini/upgini/issues/new?assignees=&title=readme%2Fbug">
-Please report it here.</a></sup>
+Please report it here</a></sup>

{upgini-1.1.282a3418.post3.dist-info → upgini-1.1.284.dist-info}/RECORD

@@ -1,24 +1,24 @@
-upgini/__about__.py,sha256=UWj0vKkKvP9Cu_tYbFWFIdRBjiAcKLIfcNwhmbKYIHI,31
+upgini/__about__.py,sha256=nDI-Yx5FbPR6Ypp9ywFECEIUTCUuFhHhMahhqU8Larw,24
 upgini/__init__.py,sha256=asENHgEVHQBIkV-e_0IhE_ZWqkCG6398U3ZLrNzAH6k,407
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=7TLVVhGtjgx_9yaiaIUK3kZSe_R9wg5dY0d4F5qCGM4,45636
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=cEv1veAlJe0iz9-myz46Ecz8ii6I-v7Qr2ksaFxEsZw,176966
+upgini/features_enricher.py,sha256=PU9i_5yUIF3ANxe6kJBC2GrDX9UNDQ0u8nJNSFkHxx8,177462
 upgini/http.py,sha256=khrYSldpY-HbVLCcApfV1BjBFK6Uyuatb4colKybxgY,42301
-upgini/metadata.py,sha256=qDAIO7NLSSQp_XiXCv3U4XJTLO0KH3YuQ8lvCLYPqzs,9781
-upgini/metrics.py,sha256=tTXAgjEuoo_vDe4n-R0AFK95IIx_7kugIJJJv2Hr_1o,30128
+upgini/metadata.py,sha256=CFJekYGD7Ep7pRFH7wCEcsXS4bz83do33FNmtcCY9P4,9729
+upgini/metrics.py,sha256=DLvA2YLV4f7lnzBCcfZ5T4NkqAv3pbstbjTepavuT7U,30688
 upgini/search_task.py,sha256=LtRJ9bCPjMo1gJ-sUDKERhDwGcWKImrzwVFHjkMSQHQ,17071
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
 upgini/version_validator.py,sha256=RGg87VweujTNlibgsOuqPLIEiBgIOkuXNVTGuNCD234,1405
 upgini/ads_management/__init__.py,sha256=qzyisOToVRP-tquAJD1PblZhNtMrOB8FiyF9JvfkvgE,50
 upgini/ads_management/ads_manager.py,sha256=igVbN2jz80Umb2BUJixmJVj-zx8unoKpecVo-R-nGdw,2648
 upgini/autofe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/autofe/all_operands.py,sha256=7UyvmmqGSqQu4kDgoFwQRKY__b9xKDk3Fpp2-H8A7AA,2399
+upgini/autofe/all_operands.py,sha256=SyKVU-xGMHgoRZvHrCmba2u2Ygc73c1mXFolNSWe8Uo,2357
 upgini/autofe/binary.py,sha256=441BRuqMsxlxuw4c8rMZB6h5EpRdVMk-bVa03U7T5Hg,3973
-upgini/autofe/date.py,sha256=ZRFfZHKttBWSM_NDp7kgpMahli91_HVpnZeBDbm8aeI,6732
-upgini/autofe/feature.py,sha256=_V9B74B3ue7eAYXSOt9JKhVC9klkAKks22MwnBRye_w,12487
+upgini/autofe/date.py,sha256=Vy1I92fLLYLhuYKJmtuPBMI8cPxE4Uwk40hqE2F2e1A,4224
+upgini/autofe/feature.py,sha256=ChSuuIbRPGIWnPjKAgZbeAEi7Y_PjSVRyxxx41MyFp0,11845
 upgini/autofe/groupby.py,sha256=4WjDzQxqpZxB79Ih4ihMMI5GDxaFqiH6ZelfV82ClT4,3091
-upgini/autofe/operand.py,sha256=JjEVT1U3kY9NDjUPMdoki7Oa8hMDG0-_h_NklVjIFyc,2882
+upgini/autofe/operand.py,sha256=xgEIZuFCfckc6LpBqVu1OVK3JEabm1O-LHUsp83EHKA,2806
 upgini/autofe/unary.py,sha256=v-l3aiE5hj6kurvh6adCQL8W3X9u9a7RVbS_WPR2qlw,3146
 upgini/autofe/vector.py,sha256=dLxfAstJs-gw_OQ1xxoxcM6pVzORlV0HVzdzt7cLXVQ,606
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -52,11 +52,11 @@ upgini/utils/ip_utils.py,sha256=Zf3F2cnQmOCH09QLQHetpjMFu1PnD0cTmDymn0SnSy8,1672
 upgini/utils/phone_utils.py,sha256=JNSkF8G6mgsN8Czy11pamaJdsY6rBINEMpi7jbVt_RA,408
 upgini/utils/postal_code_utils.py,sha256=_8CR9tBqsPptQsmMUvnrCAmBaMIQSWH3JfJ4ly3x_zs,409
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
-upgini/utils/sklearn_ext.py,sha256=c23MGSUVfxLnaDWKAxavHgnOtm5dGKkF3YswdWQcFzs,43984
+upgini/utils/sklearn_ext.py,sha256=N-eJrfAJxYpDPc85sKQyMFcIeL9Ug2lwlqDyS4jFOdE,44499
 upgini/utils/target_utils.py,sha256=Y96_PJ5cC-WsEbeqg20v9uqywDQobLoTb-xoP7S3o4E,7807
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.1.282a3418.post3.dist-info/METADATA,sha256=PEPCHswhtmofXHXwBfboDN-RUzok5rp2yBkczC4Sgac,48129
-upgini-1.1.282a3418.post3.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-upgini-1.1.282a3418.post3.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.282a3418.post3.dist-info/RECORD,,
+upgini-1.1.284.dist-info/METADATA,sha256=dDV3y75wuSFvxeGRlh_IwhfMWn4qKG6YE_-Um4Drtmo,48117
+upgini-1.1.284.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+upgini-1.1.284.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.284.dist-info/RECORD,,