upgini 1.1.244a6__tar.gz → 1.1.244a8__tar.gz

{upgini-1.1.244a6/src/upgini.egg-info → upgini-1.1.244a8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.244a6
+Version: 1.1.244a8
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.244a6 → upgini-1.1.244a8}/setup.py

@@ -40,7 +40,7 @@ def send_log(msg: str):
 
 
 here = Path(__file__).parent.resolve()
-version = "1.1.244a6"
+version = "1.1.244a8"
 try:
     send_log(f"Start setup PyLib version {version}")
     setup(

{upgini-1.1.244a6 → upgini-1.1.244a8}/src/upgini/dataset.py

@@ -61,7 +61,7 @@ class Dataset:  # (pd.DataFrame):
     FIT_SAMPLE_THRESHOLD = 200_000
     FIT_SAMPLE_WITH_EVAL_SET_ROWS = 200_000
     FIT_SAMPLE_WITH_EVAL_SET_THRESHOLD = 200_000
-    MIN_SAMPLE_THRESHOLD = 20_000
+    MIN_SAMPLE_THRESHOLD = 5_000
     IMBALANCE_THESHOLD = 0.4
     MIN_TARGET_CLASS_ROWS = 100
     MAX_MULTICLASS_CLASS_COUNT = 100

{upgini-1.1.244a6 → upgini-1.1.244a8}/src/upgini/features_enricher.py

@@ -955,6 +955,7 @@ class FeaturesEnricher(TransformerMixin):
                         fitting_enriched_X,
                         scoring,
                         groups=groups,
+                        text_features=self.generate_features,
                     )
                     metric = wrapper.metric_name
                     multiplier = wrapper.multiplier
@@ -980,6 +981,7 @@ class FeaturesEnricher(TransformerMixin):
                             cat_features,
                             add_params=custom_loss_add_params,
                             groups=groups,
+                            text_features=self.generate_features,
                         )
                         etalon_metric = baseline_estimator.cross_val_predict(
                             fitting_X, y_sorted, self.baseline_score_column
@@ -1004,6 +1006,7 @@ class FeaturesEnricher(TransformerMixin):
                             cat_features,
                             add_params=custom_loss_add_params,
                             groups=groups,
+                            text_features=self.generate_features,
                         )
                         enriched_metric = enriched_estimator.cross_val_predict(fitting_enriched_X, enriched_y_sorted)
                         self.logger.info(f"Enriched {metric} on train combined features: {enriched_metric}")
@@ -1394,7 +1397,7 @@ class FeaturesEnricher(TransformerMixin):
         elif not self.imbalanced and not exclude_features_sources and is_input_same_as_fit:
             self.logger.info("Dataset is not imbalanced, so use enriched_X from fit")
             return self.__sample_balanced(
-                validated_X, validated_y, eval_set, trace_id, remove_outliers_calc_metrics, is_demo_dataset
+                eval_set, trace_id, remove_outliers_calc_metrics
             )
         else:
             self.logger.info("Dataset is imbalanced or exclude_features_sources or X was passed. Run transform")
@@ -1421,13 +1424,43 @@ class FeaturesEnricher(TransformerMixin):
         self, validated_X: pd.DataFrame, validated_y: pd.Series, eval_set: Optional[List[tuple]], is_demo_dataset: bool
     ) -> _SampledDataForMetrics:
         eval_set_sampled_dict = dict()
-        X_sampled, search_keys = self._extend_x(validated_X, is_demo_dataset)
-        y_sampled = validated_y
+
+        df = validated_X.copy()
+        df[TARGET] = validated_y
+
+        if eval_set is not None:
+            df[EVAL_SET_INDEX] = 0
+            for idx, (eval_x, eval_y) in enumerate(eval_set):
+                eval_xy = eval_x.copy()
+                eval_xy[TARGET] = eval_y
+                eval_xy[EVAL_SET_INDEX] = idx + 1
+                df = pd.concat([df, eval_xy])
+
+        num_samples = _num_samples(df)
+        sample_threshold, sample_rows = (
+            (Dataset.FIT_SAMPLE_WITH_EVAL_SET_THRESHOLD, Dataset.FIT_SAMPLE_WITH_EVAL_SET_ROWS)
+            if eval_set is not None
+            else (Dataset.FIT_SAMPLE_THRESHOLD, Dataset.FIT_SAMPLE_ROWS)
+        )
+        if num_samples > sample_threshold:
+            self.logger.info(f"Downsampling from {num_samples} to {sample_rows}")
+            df = df.sample(
+                n=sample_rows, random_state=self.random_state
+            )
+
+        df_extended, search_keys = self._extend_x(df, is_demo_dataset)
+        df_extended = self.__add_fit_system_record_id(df_extended, dict(), search_keys)
+
+        train_df = df_extended.query(f"{EVAL_SET_INDEX} == 0") if eval_set is not None else df_extended
+        X_sampled = train_df.drop(columns=[TARGET, EVAL_SET_INDEX], errors="ignore")
+        y_sampled = train_df[TARGET].copy()
         enriched_X = X_sampled
+
         if eval_set is not None:
             for idx in range(len(eval_set)):
-                eval_X_sampled, _ = self._extend_x(eval_set[idx][0], is_demo_dataset)
-                eval_y_sampled = eval_set[idx][1]
+                eval_xy_sampled = df_extended.query(f"{EVAL_SET_INDEX} == {idx + 1}")
+                eval_X_sampled = eval_xy_sampled.drop(columns=[TARGET, EVAL_SET_INDEX], errors="ignore")
+                eval_y_sampled = eval_xy_sampled[TARGET].copy()
                 enriched_eval_X = eval_X_sampled
                 eval_set_sampled_dict[idx] = (eval_X_sampled, enriched_eval_X, eval_y_sampled)
         self.__cached_sampled_datasets = (X_sampled, y_sampled, enriched_X, eval_set_sampled_dict, search_keys)
@@ -1436,18 +1469,15 @@ class FeaturesEnricher(TransformerMixin):
 
     def __sample_balanced(
         self,
-        validated_X: pd.DataFrame,
-        validated_y: pd.Series,
         eval_set: Optional[List[tuple]],
         trace_id: str,
         remove_outliers_calc_metrics: Optional[bool],
-        is_demo_dataset: bool,
     ) -> _SampledDataForMetrics:
         eval_set_sampled_dict = dict()
         search_keys = self.fit_search_keys
 
         rows_to_drop = None
-        task_type = self.model_task_type or define_task(validated_y, self.logger, silent=True)
+        task_type = self.model_task_type or define_task(self.df_with_original_index[TARGET], self.logger, silent=True)
         if task_type == ModelTaskType.REGRESSION:
             target_outliers_df = self._search_task.get_target_outliers(trace_id)
             if target_outliers_df is not None and len(target_outliers_df) > 0:
@@ -2853,11 +2883,12 @@ class FeaturesEnricher(TransformerMixin):
         else:
             result_train = result_train_features
 
-        if SYSTEM_RECORD_ID in result_train.columns:
-            result_train = result_train.drop(columns=SYSTEM_RECORD_ID)
-        for eval_set_index in result_eval_sets.keys():
-            if SYSTEM_RECORD_ID in result_eval_sets[eval_set_index].columns:
-                result_eval_sets[eval_set_index] = result_eval_sets[eval_set_index].drop(columns=SYSTEM_RECORD_ID)
+        if drop_system_record_id:
+            if SYSTEM_RECORD_ID in result_train.columns:
+                result_train = result_train.drop(columns=SYSTEM_RECORD_ID)
+            for eval_set_index in result_eval_sets.keys():
+                if SYSTEM_RECORD_ID in result_eval_sets[eval_set_index].columns:
+                    result_eval_sets[eval_set_index] = result_eval_sets[eval_set_index].drop(columns=SYSTEM_RECORD_ID)
 
         return result_train, result_eval_sets
 

{upgini-1.1.244a6 → upgini-1.1.244a8}/src/upgini/metrics.py

@@ -1,4 +1,5 @@
 import logging
+import re
 from copy import deepcopy
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
@@ -201,6 +202,7 @@ class EstimatorWrapper:
         target_type: ModelTaskType,
         add_params: Optional[Dict[str, Any]] = None,
         groups: Optional[np.ndarray] = None,
+        text_features: Optional[List[str]] = None,
     ):
         self.estimator = estimator
         self.scorer = scorer
@@ -213,6 +215,7 @@ class EstimatorWrapper:
         self.add_params = add_params
         self.cv_estimators = None
         self.groups = groups
+        self.text_features = text_features
 
     def fit(self, X: pd.DataFrame, y: np.ndarray, **kwargs):
         X, y, _, fit_params = self._prepare_to_fit(X, y)
@@ -285,6 +288,7 @@ class EstimatorWrapper:
                 groups=groups,
                 fit_params=fit_params,
                 return_estimator=True,
+                error_score="raise",
             )
             metrics_by_fold = cv_results["test_score"]
             self.cv_estimators = cv_results["estimator"]
@@ -330,6 +334,7 @@ class EstimatorWrapper:
             "cv": cv,
             "target_type": target_type,
             "groups": groups,
+            "text_features": text_features,
         }
         if estimator is None:
             params = dict()
@@ -391,27 +396,56 @@ class CatBoostWrapper(EstimatorWrapper):
         cv: BaseCrossValidator,
         target_type: ModelTaskType,
         groups: Optional[List[str]] = None,
+        text_features: Optional[List[str]] = None,
     ):
         super(CatBoostWrapper, self).__init__(
-            estimator, scorer, metric_name, multiplier, cv, target_type, groups=groups
+            estimator, scorer, metric_name, multiplier, cv, target_type, groups=groups, text_features=text_features
         )
         self.cat_features = None
         self.cat_features_idx = None
+        self.emb_groups = None
 
     def _prepare_to_fit(self, X: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, np.ndarray, np.ndarray, dict]:
         X, y, groups, params = super()._prepare_to_fit(X, y)
+
+        # Find embeddings
+        emb_pattern = r"(.+)_emb\d+"
+        emb_features = [c for c in X.columns if re.match(emb_pattern, c) and is_numeric_dtype(X[c])]
+        embedding_features = []
+        if len(emb_features) > 0:
+            # group by source feature
+            self.emb_groups = dict()
+            for emb in emb_features:
+                source_feature = re.match(emb_pattern, emb).group(1)
+                embs = self.emb_groups.get(source_feature, [])
+                embs.append(emb)
+                self.emb_groups[source_feature] = embs
+            self.emb_groups = {
+                source_feature: embs for source_feature, embs in self.emb_groups.items() if len(embs) > 1
+            }
+            X, embedding_features = self.group_embeddings(X)
+            params["embedding_features"] = embedding_features
+
+        # Find text features from passed in generate_features
+        if self.text_features is not None:
+            self.text_features = [f for f in self.text_features if not is_numeric_dtype(X[f])]
+            params["text_features"] = self.text_features
+
+        # Find rest categorical features
         self.cat_features = _get_cat_features(X)
+        if self.text_features is not None:
+            self.cat_features = [
+                f for f in self.cat_features if f not in self.text_features and f not in embedding_features
+            ]
         X = fill_na_cat_features(X, self.cat_features)
-        # unique_cat_features = []
-        # # TODO try to remove this condition because now we remove constant features earlier
-        # for name in cat_features:
-        #     # Remove constant categorical features
-        #     if X[name].nunique() > 1:
-        #         unique_cat_features.append(name)
-        #     else:
-        #         X = X.drop(columns=name)
-        # cat_features_idx = [X.columns.get_loc(c) for c in unique_cat_features]
-        self.cat_features_idx = [X.columns.get_loc(c) for c in self.cat_features]
+        unique_cat_features = []
+        for name in self.cat_features:
+            # Remove constant categorical features
+            if X[name].nunique() > 1:
+                unique_cat_features.append(name)
+            else:
+                X = X.drop(columns=name)
+        self.cat_features_idx = [X.columns.get_loc(c) for c in unique_cat_features]
         if (
             hasattr(self.estimator, "get_param")
             and hasattr(self.estimator, "_init_params")
@@ -422,15 +456,32 @@ class CatBoostWrapper(EstimatorWrapper):
             self.cat_features_idx = list(cat_features_set)
             del self.estimator._init_params["cat_features"]
 
-        params.update({"cat_features": self.cat_features_idx})
+        params["cat_features"] = self.cat_features_idx
+
         return X, y, groups, params
 
+    def group_embeddings(self, df: pd.DataFrame):
+        emb_columns = []
+        for source_feature, embs in self.emb_groups.items():
+            emb_name = f"{source_feature}_emb"
+            df[embs] = df[embs].fillna(0.0)
+            df[emb_name] = df[embs].values.tolist()
+            df = df.drop(columns=embs)
+            emb_columns.append(emb_name)
+        return df, emb_columns
+
     def _prepare_to_calculate(self, X: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, np.ndarray, dict]:
         X, y, params = super()._prepare_to_calculate(X, y)
+        if self.text_features is not None:
+            params["text_features"] = self.text_features
+        if self.emb_groups is not None:
+            X, emb_columns = self.group_embeddings(X)
+            params["embedding_features"] = emb_columns
         if self.cat_features is not None:
             X = fill_na_cat_features(X, self.cat_features)
         if self.cat_features_idx is not None:
-            params.update({"cat_features": self.cat_features_idx})
+            params["cat_features"] = self.cat_features_idx
+
         return X, y, params
 
 
@@ -444,9 +495,10 @@ class LightGBMWrapper(EstimatorWrapper):
         cv: BaseCrossValidator,
         target_type: ModelTaskType,
         groups: Optional[List[str]] = None,
+        text_features: Optional[List[str]] = None,
     ):
         super(LightGBMWrapper, self).__init__(
-            estimator, scorer, metric_name, multiplier, cv, target_type, groups=groups
+            estimator, scorer, metric_name, multiplier, cv, target_type, groups=groups, text_features=text_features
         )
         self.cat_features = None
 
@@ -482,9 +534,10 @@ class OtherEstimatorWrapper(EstimatorWrapper):
         cv: BaseCrossValidator,
         target_type: ModelTaskType,
         groups: Optional[List[str]] = None,
+        text_features: Optional[List[str]] = None,
     ):
         super(OtherEstimatorWrapper, self).__init__(
-            estimator, scorer, metric_name, multiplier, cv, target_type, groups=groups
+            estimator, scorer, metric_name, multiplier, cv, target_type, groups=groups, text_features=text_features
         )
         self.cat_features = None
 

{upgini-1.1.244a6 → upgini-1.1.244a8}/src/upgini/utils/sklearn_ext.py

@@ -21,6 +21,7 @@ from sklearn.metrics._scorer import _MultimetricScorer
 from sklearn.model_selection import check_cv
 from sklearn.utils.fixes import np_version, parse_version
 from sklearn.utils.validation import indexable
+from sklearn.model_selection import cross_validate as original_cross_validate
 
 _DEFAULT_TAGS = {
     "non_deterministic": False,
@@ -313,21 +314,23 @@ def cross_validate(
         return ret
     except Exception:
         logging.exception("Failed to execute overriden cross_validate. Fallback to original")
-        return cross_validate(
-            estimator,
-            X,
-            y,
-            groups=groups,
-            scoring=scoring,
-            cv=cv,
-            n_jobs=n_jobs,
-            verbose=verbose,
-            fit_params=fit_params,
-            pre_dispatch=pre_dispatch,
-            return_train_score=return_train_score,
-            return_estimator=return_estimator,
-            error_score=error_score,
-        )
+        raise
+        # fit_params["use_best_model"] = False
+        # return original_cross_validate(
+        #     estimator,
+        #     X,
+        #     y,
+        #     groups=groups,
+        #     scoring=scoring,
+        #     cv=cv,
+        #     n_jobs=n_jobs,
+        #     verbose=verbose,
+        #     fit_params=fit_params,
+        #     pre_dispatch=pre_dispatch,
+        #     return_train_score=return_train_score,
+        #     return_estimator=return_estimator,
+        #     error_score=error_score,
+        # )
 
 
 def _fit_and_score(

{upgini-1.1.244a6 → upgini-1.1.244a8/src/upgini.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.244a6
+Version: 1.1.244a8
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.244a6 → upgini-1.1.244a8}/tests/test_continuous_dataset.py

@@ -1,4 +1,5 @@
 import os
+from typing import Dict
 
 import pandas as pd
 import pytest
@@ -13,7 +14,7 @@ FIXTURE_DIR = os.path.join(
 
 
 @pytest.fixture
-def etalon_definition():
+def etalon_definition() -> Dict[str, FileColumnMeaningType]:
     return {
         "phone_num": FileColumnMeaningType.MSISDN,
         "rep_date": FileColumnMeaningType.DATE,
@@ -27,9 +28,11 @@ def etalon_search_keys():
 
 
 @pytest.mark.datafiles(os.path.join(FIXTURE_DIR, "data.csv.gz"))
-def test_continuous_dataset(datafiles, etalon_definition, etalon_search_keys):
+def test_continuous_dataset(datafiles, etalon_definition: Dict[str, FileColumnMeaningType], etalon_search_keys):
     df = pd.read_csv(datafiles / "data.csv.gz")
-    df = df.reset_index().rename(columns={"index": "system_record_id"})
+    df = df.reset_index().rename(columns={"index": "system_record_id", "score": "target"})
+    del etalon_definition["score"]
+    etalon_definition["target"] = FileColumnMeaningType.TARGET
     converter = DateTimeSearchKeyConverter("rep_date")
     df = converter.convert(df)
     ds = Dataset(

{upgini-1.1.244a6 → upgini-1.1.244a8}/tests/test_etalon_validation.py

@@ -25,7 +25,7 @@ def test_etalon_validation(etalon: Dataset):
     valid_count = len(etalon)
     valid_rate = 100 * valid_count / count
 
-    assert valid_count == 4
+    assert valid_count == 5
     valid_rate_expected = 100 * (valid_count / 10)
     assert valid_rate == pytest.approx(valid_rate_expected, abs=0.01)
 
@@ -497,8 +497,9 @@ def test_time_cutoff_from_period():
 def test_time_cutoff_from_timestamp():
     df = pd.DataFrame({"date": [1577836800000000000, 1577840400000000000, 1577844000000000000]})
     converter = DateTimeSearchKeyConverter("date")
-    with pytest.raises(Exception, match="Unsupported type of date column date.*"):
-        converter.convert(df)
+    # with pytest.raises(Exception, match="Unsupported type of date column date.*"):
+    df = converter.convert(df)
+    assert len(df) == 3
 
 
 def test_time_cutoff_with_different_timezones():

{upgini-1.1.244a6 → upgini-1.1.244a8}/tests/test_features_enricher.py

@@ -2161,7 +2161,6 @@ def test_idempotent_order_with_imbalanced_dataset(requests_mock: Mocker):
                 pass
 
             actual_result_df = result_wrapper.df.sort_values(by="system_record_id").reset_index(drop=True)
-
             assert_frame_equal(actual_result_df, expected_result_df)
 
         for i in range(5):

{upgini-1.1.244a6 → upgini-1.1.244a8}/tests/test_metrics.py

@@ -147,6 +147,7 @@ def test_real_case_metric_binary(requests_mock: Mocker):
     mock_raw_features(requests_mock, url, search_task_id, path_to_mock_features)
 
     train = pd.read_parquet(os.path.join(BASE_DIR, "real_train.parquet"))
+    train.sort_index()
     X = train[["request_date", "score"]]
     y = train["target1"].rename("target")
     test = pd.read_parquet(os.path.join(BASE_DIR, "real_test.parquet"))
@@ -168,18 +169,25 @@ def test_real_case_metric_binary(requests_mock: Mocker):
     enricher.eval_set = eval_set
 
     enriched_X = pd.read_parquet(os.path.join(BASE_DIR, "real_enriched_x.parquet"))
+
+    # TODO join enriched_X and X and y by index and then sort and add system_record_id
+    
+    enriched_X = enriched_X.sort_values(by="request_date").reset_index().rename(columns={"index": "system_record_id"})
     enriched_eval_x = pd.read_parquet(os.path.join(BASE_DIR, "real_enriched_eval_x.parquet"))
+    enriched_eval_x = enriched_eval_x.sort_values(by="request_date").reset_index().rename(columns={"index": "system_record_id"})
 
     sampled_Xy = X.copy()
     sampled_Xy["target"] = y
     sampled_Xy = sampled_Xy[sampled_Xy.index.isin(enriched_X.index)]
     sampled_X = sampled_Xy.drop(columns="target")
+    sampled_X = sampled_X.reset_index().rename(columns={"index": "system_record_id"})
     sampled_y = sampled_Xy["target"]
+    sampled_eval_x = eval_set[0][0].reset_index().rename(columns={"index": "system_record_id"})
     enricher._FeaturesEnricher__cached_sampled_datasets = (
         sampled_X,
         sampled_y,
         enriched_X,
-        {0: (eval_set[0][0], enriched_eval_x, eval_set[0][1])},
+        {0: (sampled_eval_x, enriched_eval_x, eval_set[0][1])},
         search_keys,
     )