upgini 1.2.86a2__py3-none-any.whl → 1.2.87.dev2__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.86a2"
+__version__ = "1.2.87.dev2"

upgini/features_enricher.py

@@ -30,7 +30,7 @@ from pandas.api.types import (
 from scipy.stats import ks_2samp
 from sklearn.base import TransformerMixin
 from sklearn.exceptions import NotFittedError
-from sklearn.model_selection import BaseCrossValidator
+from sklearn.model_selection import BaseCrossValidator, TimeSeriesSplit
 
 from upgini.autofe.feature import Feature
 from upgini.autofe.timeseries import TimeSeriesBase
@@ -71,6 +71,7 @@ from upgini.resource_bundle import ResourceBundle, bundle, get_custom_bundle
 from upgini.search_task import SearchTask
 from upgini.spinner import Spinner
 from upgini.utils import combine_search_keys, find_numbers_with_decimal_comma
+from upgini.utils.blocked_time_series import BlockedTimeSeriesSplit
 from upgini.utils.country_utils import (
     CountrySearchKeyConverter,
     CountrySearchKeyDetector,
@@ -114,7 +115,9 @@ from upgini.utils.postal_code_utils import (
 try:
     from upgini.utils.progress_bar import CustomProgressBar as ProgressBar
 except Exception:
-    from upgini.utils.fallback_progress_bar import CustomFallbackProgressBar as ProgressBar
+    from upgini.utils.fallback_progress_bar import (
+        CustomFallbackProgressBar as ProgressBar,
+    )
 
 from upgini.utils.sort import sort_columns
 from upgini.utils.target_utils import (
@@ -239,6 +242,7 @@ class FeaturesEnricher(TransformerMixin):
         add_date_if_missing: bool = True,
         disable_force_downsampling: bool = False,
         id_columns: Optional[List[str]] = None,
+        generate_search_key_features: bool = True,
         **kwargs,
     ):
         self.bundle = get_custom_bundle(custom_bundle_config)
@@ -365,6 +369,8 @@ class FeaturesEnricher(TransformerMixin):
         self.exclude_columns = exclude_columns
         self.baseline_score_column = baseline_score_column
         self.add_date_if_missing = add_date_if_missing
+        self.generate_search_key_features = generate_search_key_features
+
         self.features_info_display_handle = None
         self.data_sources_display_handle = None
         self.autofe_features_display_handle = None
@@ -1045,6 +1051,7 @@ class FeaturesEnricher(TransformerMixin):
                     self._check_train_and_eval_target_distribution(y_sorted, fitting_eval_set_dict)
 
                     has_date = self._get_date_column(search_keys) is not None
+                    has_time = has_date and isinstance(_cv, TimeSeriesSplit) or isinstance(_cv, BlockedTimeSeriesSplit)
                     model_task_type = self.model_task_type or define_task(y_sorted, has_date, self.logger, silent=True)
                     cat_features = list(set(client_cat_features + cat_features_from_backend))
                     baseline_cat_features = [f for f in cat_features if f in fitting_X.columns]
@@ -1077,7 +1084,7 @@ class FeaturesEnricher(TransformerMixin):
                             add_params=custom_loss_add_params,
                             groups=groups,
                             text_features=text_features,
-                            has_date=has_date,
+                            has_time=has_time,
                         )
                         baseline_cv_result = baseline_estimator.cross_val_predict(
                             fitting_X, y_sorted, baseline_score_column
@@ -1112,7 +1119,7 @@ class FeaturesEnricher(TransformerMixin):
                             add_params=custom_loss_add_params,
                             groups=groups,
                             text_features=text_features,
-                            has_date=has_date,
+                            has_time=has_time,
                         )
                         enriched_cv_result = enriched_estimator.cross_val_predict(fitting_enriched_X, enriched_y_sorted)
                         enriched_metric = enriched_cv_result.get_display_metric()
@@ -1773,7 +1780,13 @@ class FeaturesEnricher(TransformerMixin):
         date_column = self._get_date_column(search_keys)
         generated_features = []
         if date_column is not None:
-            converter = DateTimeSearchKeyConverter(date_column, self.date_format, self.logger, self.bundle)
+            converter = DateTimeSearchKeyConverter(
+                date_column,
+                self.date_format,
+                self.logger,
+                self.bundle,
+                generate_cyclical_features=self.generate_search_key_features,
+            )
             # Leave original date column values
             df_with_date_features = converter.convert(df, keep_time=True)
             df_with_date_features[date_column] = df[date_column]
@@ -1781,7 +1794,7 @@ class FeaturesEnricher(TransformerMixin):
             generated_features = converter.generated_features
 
         email_columns = SearchKey.find_all_keys(search_keys, SearchKey.EMAIL)
-        if email_columns:
+        if email_columns and self.generate_search_key_features:
             generator = EmailDomainGenerator(email_columns)
             df = generator.generate(df)
             generated_features.extend(generator.generated_features)
@@ -2204,10 +2217,12 @@ class FeaturesEnricher(TransformerMixin):
                         {"name": name, "value": key_example(sk_type)} for name in sk_meta.unnestKeyNames
                     ]
                 else:
-                    search_keys_with_values[sk_type.name] = [{
-                        "name": sk_meta.originalName,
-                        "value": key_example(sk_type),
-                    }]
+                    search_keys_with_values[sk_type.name] = [
+                        {
+                            "name": sk_meta.originalName,
+                            "value": key_example(sk_type),
+                        }
+                    ]
 
         keys_section = json.dumps(search_keys_with_values)
         features_for_transform = self._search_task.get_features_for_transform()
@@ -2360,7 +2375,13 @@ if response.status_code == 200:
             generated_features = []
             date_column = self._get_date_column(search_keys)
             if date_column is not None:
-                converter = DateTimeSearchKeyConverter(date_column, self.date_format, self.logger, bundle=self.bundle)
+                converter = DateTimeSearchKeyConverter(
+                    date_column,
+                    self.date_format,
+                    self.logger,
+                    bundle=self.bundle,
+                    generate_cyclical_features=self.generate_search_key_features,
+                )
                 df = converter.convert(df, keep_time=True)
                 self.logger.info(f"Date column after convertion: {df[date_column]}")
                 generated_features.extend(converter.generated_features)
@@ -2370,7 +2391,7 @@ if response.status_code == 200:
                     df = self._add_current_date_as_key(df, search_keys, self.logger, self.bundle)
 
             email_columns = SearchKey.find_all_keys(search_keys, SearchKey.EMAIL)
-            if email_columns:
+            if email_columns and self.generate_search_key_features:
                 generator = EmailDomainGenerator(email_columns)
                 df = generator.generate(df)
                 generated_features.extend(generator.generated_features)
@@ -2860,6 +2881,7 @@ if response.status_code == 200:
                 self.date_format,
                 self.logger,
                 bundle=self.bundle,
+                generate_cyclical_features=self.generate_search_key_features,
             )
             df = converter.convert(df, keep_time=True)
             if converter.has_old_dates:
@@ -2872,7 +2894,7 @@ if response.status_code == 200:
                 df = self._add_current_date_as_key(df, self.fit_search_keys, self.logger, self.bundle)
 
         email_columns = SearchKey.find_all_keys(self.fit_search_keys, SearchKey.EMAIL)
-        if email_columns:
+        if email_columns and self.generate_search_key_features:
             generator = EmailDomainGenerator(email_columns)
             df = generator.generate(df)
             self.fit_generated_features.extend(generator.generated_features)
@@ -3564,7 +3586,9 @@ if response.status_code == 200:
             maybe_date_col = SearchKey.find_key(self.search_keys, [SearchKey.DATE, SearchKey.DATETIME])
             if X is not None and maybe_date_col is not None and maybe_date_col in X.columns:
                 # TODO cast date column to single dtype
-                date_converter = DateTimeSearchKeyConverter(maybe_date_col, self.date_format)
+                date_converter = DateTimeSearchKeyConverter(
+                    maybe_date_col, self.date_format, generate_cyclical_features=False
+                )
                 converted_X = date_converter.convert(X)
                 min_date = converted_X[maybe_date_col].min()
                 max_date = converted_X[maybe_date_col].max()
@@ -3603,7 +3627,7 @@ if response.status_code == 200:
             self.__log_warning(bundle.get("current_date_added"))
             df[FeaturesEnricher.CURRENT_DATE] = datetime.date.today()
             search_keys[FeaturesEnricher.CURRENT_DATE] = SearchKey.DATE
-            converter = DateTimeSearchKeyConverter(FeaturesEnricher.CURRENT_DATE)
+            converter = DateTimeSearchKeyConverter(FeaturesEnricher.CURRENT_DATE, generate_cyclical_features=False)
             df = converter.convert(df)
         return df
 

upgini/metrics.py

@@ -15,7 +15,7 @@ from catboost import CatBoostClassifier, CatBoostRegressor
 from category_encoders.cat_boost import CatBoostEncoder
 from lightgbm import LGBMClassifier, LGBMRegressor
 from numpy import log1p
-from pandas.api.types import is_numeric_dtype, is_integer_dtype, is_float_dtype
+from pandas.api.types import is_float_dtype, is_integer_dtype, is_numeric_dtype
 from sklearn.metrics import check_scoring, get_scorer, make_scorer, roc_auc_score
 
 from upgini.utils.blocked_time_series import BlockedTimeSeriesSplit
@@ -391,9 +391,7 @@ class EstimatorWrapper:
                     self.converted_to_int.append(c)
                     self.cat_features.remove(c)
                 elif is_float_dtype(x[c]) or (x[c].dtype == "category" and is_float_dtype(x[c].cat.categories)):
-                    self.logger.info(
-                        f"Convert float cat feature {c} to string"
-                    )
+                    self.logger.info(f"Convert float cat feature {c} to string")
                     x[c] = x[c].astype(str)
                     self.converted_to_str.append(c)
                 elif x[c].dtype not in ["category", "int64"]:
@@ -551,7 +549,7 @@ class EstimatorWrapper:
         text_features: Optional[List[str]] = None,
         add_params: Optional[Dict[str, Any]] = None,
         groups: Optional[List[str]] = None,
-        has_date: Optional[bool] = None,
+        has_time: Optional[bool] = None,
     ) -> EstimatorWrapper:
         scorer, metric_name, multiplier = define_scorer(target_type, scoring)
         kwargs = {
@@ -568,7 +566,7 @@ class EstimatorWrapper:
         if estimator is None:
             if EstimatorWrapper.default_estimator == "catboost":
                 logger.info("Using CatBoost as default estimator")
-                params = {"has_time": has_date}
+                params = {"has_time": has_time}
                 if target_type == ModelTaskType.MULTICLASS:
                     params = _get_add_params(params, CATBOOST_MULTICLASS_PARAMS)
                     params = _get_add_params(params, add_params)
@@ -610,8 +608,8 @@ class EstimatorWrapper:
                 estimator_copy = deepcopy(estimator)
             kwargs["estimator"] = estimator_copy
             if is_catboost_estimator(estimator):
-                if has_date is not None:
-                    estimator_copy.set_params(has_time=has_date)
+                if has_time is not None:
+                    estimator_copy.set_params(has_time=has_time)
                 estimator = CatBoostWrapper(**kwargs)
             else:
                 if isinstance(estimator, (LGBMClassifier, LGBMRegressor)):

upgini/utils/datetime_utils.py

@@ -41,6 +41,7 @@ class DateTimeSearchKeyConverter:
         date_format: Optional[str] = None,
         logger: Optional[logging.Logger] = None,
         bundle: Optional[ResourceBundle] = None,
+        generate_cyclical_features: bool = True,
     ):
         self.date_column = date_column
         self.date_format = date_format
@@ -51,6 +52,7 @@ class DateTimeSearchKeyConverter:
             self.logger.setLevel("FATAL")
         self.generated_features: List[str] = []
         self.bundle = bundle or get_custom_bundle()
+        self.generate_cyclical_features = generate_cyclical_features
         self.has_old_dates = False
 
     @staticmethod
@@ -121,61 +123,63 @@ class DateTimeSearchKeyConverter:
                 df[cos_feature] = np.cos(2 * np.pi * df[column] / period)
                 self.generated_features.append(cos_feature)
 
-        df["quarter"] = df[self.date_column].dt.quarter
+        if self.generate_cyclical_features:
 
-        # Calculate the start date of the quarter for each timestamp
-        df["quarter_start"] = df[self.date_column].dt.to_period("Q").dt.start_time
+            df["quarter"] = df[self.date_column].dt.quarter
 
-        # Calculate the day in the quarter
-        df["day_in_quarter"] = (df[self.date_column] - df["quarter_start"]).dt.days + 1
+            # Calculate the start date of the quarter for each timestamp
+            df["quarter_start"] = df[self.date_column].dt.to_period("Q").dt.start_time
 
-        # Vectorized calculation of days_in_quarter
-        quarter = df["quarter"]
-        start = df["quarter_start"]
-        year = start.dt.year
-        month = start.dt.month
+            # Calculate the day in the quarter
+            df["day_in_quarter"] = (df[self.date_column] - df["quarter_start"]).dt.days + 1
 
-        quarter_end_year = np.where(quarter == 4, year + 1, year)
-        quarter_end_month = np.where(quarter == 4, 1, month + 3)
+            # Vectorized calculation of days_in_quarter
+            quarter = df["quarter"]
+            start = df["quarter_start"]
+            year = start.dt.year
+            month = start.dt.month
 
-        end = pd.to_datetime({"year": quarter_end_year, "month": quarter_end_month, "day": 1})
-        end.index = df.index
+            quarter_end_year = np.where(quarter == 4, year + 1, year)
+            quarter_end_month = np.where(quarter == 4, 1, month + 3)
 
-        df["days_in_quarter"] = (end - start).dt.days
+            end = pd.to_datetime({"year": quarter_end_year, "month": quarter_end_month, "day": 1})
+            end.index = df.index
 
-        add_cyclical_features(df, "day_in_quarter", df["days_in_quarter"])  # Days in the quarter
+            df["days_in_quarter"] = (end - start).dt.days
 
-        df.drop(columns=["quarter", "quarter_start", "day_in_quarter", "days_in_quarter"], inplace=True)
+            add_cyclical_features(df, "day_in_quarter", df["days_in_quarter"])  # Days in the quarter
 
-        df[seconds] = (df[self.date_column] - df[self.date_column].dt.floor("D")).dt.seconds
+            df.drop(columns=["quarter", "quarter_start", "day_in_quarter", "days_in_quarter"], inplace=True)
 
-        seconds_without_na = df[seconds].dropna()
-        if (seconds_without_na != 0).any() and seconds_without_na.nunique() > 1:
-            self.logger.info("Time found in date search key. Add extra features based on time")
+            df[seconds] = (df[self.date_column] - df[self.date_column].dt.floor("D")).dt.seconds
 
-            # Extract basic components
-            df["second"] = df[self.date_column].dt.second
-            df["minute"] = df[self.date_column].dt.minute
-            df["hour"] = df[self.date_column].dt.hour
+            seconds_without_na = df[seconds].dropna()
+            if (seconds_without_na != 0).any() and seconds_without_na.nunique() > 1:
+                self.logger.info("Time found in date search key. Add extra features based on time")
 
-            # Apply cyclical transformations
-            add_cyclical_features(df, "second", 60)  # Seconds in a minute
-            add_cyclical_features(df, "minute", 60)  # Minutes in an hour
-            add_cyclical_features(df, "minute", 30)  # Minutes in half an hour
-            add_cyclical_features(df, "hour", 24)  # Hours in a day
+                # Extract basic components
+                df["second"] = df[self.date_column].dt.second
+                df["minute"] = df[self.date_column].dt.minute
+                df["hour"] = df[self.date_column].dt.hour
 
-            # Drop intermediate columns if not needed
-            df.drop(columns=["second", "minute", "hour"], inplace=True)
-        else:
-            keep_time = False
+                # Apply cyclical transformations
+                add_cyclical_features(df, "second", 60)  # Seconds in a minute
+                add_cyclical_features(df, "minute", 60)  # Minutes in an hour
+                add_cyclical_features(df, "minute", 30)  # Minutes in half an hour
+                add_cyclical_features(df, "hour", 24)  # Hours in a day
+
+                # Drop intermediate columns if not needed
+                df.drop(columns=["second", "minute", "hour"], inplace=True)
+            else:
+                keep_time = False
 
-        for generated_feature in self.generated_features[:]:
-            if df[generated_feature].dropna().nunique() <= 1:
-                self.logger.warning(f"Generated constant feature {generated_feature} will be dropped")
-                df.drop(columns=generated_feature, inplace=True)
-                self.generated_features.remove(generated_feature)
+            for generated_feature in self.generated_features[:]:
+                if df[generated_feature].dropna().nunique() <= 1:
+                    self.logger.warning(f"Generated constant feature {generated_feature} will be dropped")
+                    df.drop(columns=generated_feature, inplace=True)
+                    self.generated_features.remove(generated_feature)
 
-        df.drop(columns=seconds, inplace=True)
+            df.drop(columns=seconds, inplace=True)
 
         if keep_time:
             df[self.DATETIME_COL] = df[self.date_column].astype(np.int64) // 1_000_000

upgini/utils/deduplicate_utils.py

@@ -104,9 +104,9 @@ def remove_fintech_duplicates(
         sub_df = pd.merge(sub_df, nonunique_target_rows, on=personal_cols)
 
         # Convert date columns for further checks
-        sub_df = DateTimeSearchKeyConverter(date_col, date_format=date_format, logger=logger, bundle=bundle).convert(
-            sub_df
-        )
+        sub_df = DateTimeSearchKeyConverter(
+            date_col, date_format=date_format, logger=logger, bundle=bundle, generate_cyclical_features=False
+        ).convert(sub_df)
         grouped_by_personal_cols = sub_df.groupby(personal_cols, group_keys=False)
         rows_with_diff_target = grouped_by_personal_cols.filter(has_diff_target_within_60_days)
 

{upgini-1.2.86a2.dist-info → upgini-1.2.87.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.86a2
+Version: 1.2.87.dev2
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.86a2.dist-info → upgini-1.2.87.dev2.dist-info}/RECORD

@@ -1,12 +1,12 @@
-upgini/__about__.py,sha256=yGLa0SZe61T_OjwHem32zlqsP2f3eCCrsj4uwsanjlA,25
+upgini/__about__.py,sha256=lMGQTkNeO4UaNk9zHrLiiQXvmrSDuq_oo4cWNvv6Lho,28
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=fRtqSkXNONLnPe6cCL967GMt349FTIpXzy_u8LUKncw,35354
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=G0qbRPdlWe9p6cwYF3khP99-0kgAO8N0A2sfQxSLgmM,213446
+upgini/features_enricher.py,sha256=n8KBoBgJApLiRv4wXeSgfS-PfbB1D5aDOJfFnL0q6v8,214487
 upgini/http.py,sha256=6Qcepv0tDC72mBBJxYHnA2xqw6QwFaKrXN8o4vju8Es,44372
 upgini/metadata.py,sha256=zt_9k0iQbWXuiRZcel4ORNPdQKt6Ou69ucZD_E1Q46o,12341
-upgini/metrics.py,sha256=3cip0_L6-OFew74KsRwzxJDU6UFq05h2v7IsyHLcMRc,43164
+upgini/metrics.py,sha256=7j8wcU-5xh_jSzqGpY7SiWwo1FEAhcPelFVOVu0qwzs,43118
 upgini/search_task.py,sha256=Q5HjBpLIB3OCxAD1zNv5yQ3ZNJx696WCK_-H35_y7Rs,17912
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
 upgini/version_validator.py,sha256=DvbaAvuYFoJqYt0fitpsk6Xcv-H1BYDJYHUMxaKSH_Y,1509
@@ -51,8 +51,8 @@ upgini/utils/blocked_time_series.py,sha256=Uqr3vp4YqNclj2-PzEYqVy763GSXHn86sbpIl
 upgini/utils/country_utils.py,sha256=lY-eXWwFVegdVENFttbvLcgGDjFO17Sex8hd2PyJaRk,6937
 upgini/utils/custom_loss_utils.py,sha256=kieNZYBYZm5ZGBltF1F_jOSF4ea6C29rYuCyiDcqVNY,3857
 upgini/utils/cv_utils.py,sha256=w6FQb9nO8BWDx88EF83NpjPLarK4eR4ia0Wg0kLBJC4,3525
-upgini/utils/datetime_utils.py,sha256=_jq-kn_dGNFfs-DGXcWCGzy9bkplfAjrZ8SsmN28zXc,13535
-upgini/utils/deduplicate_utils.py,sha256=AcMLoObMjhOTQ_fMS1LWy0GKp6WXnZ-FNux_8V3nbZU,8914
+upgini/utils/datetime_utils.py,sha256=UpM2Ljri8rCqHBrSGhmjViGDheLiOYWARFcGaFvC7JE,13858
+upgini/utils/deduplicate_utils.py,sha256=jm9ARZ0fbJFF3aJqj-xm_T6lNh-WErM0H0h6B_L1xQc,8948
 upgini/utils/display_utils.py,sha256=hAeWEcJtPDg8fAVcMNrNB-azFD2WJp1nvbPAhR7SeP4,12071
 upgini/utils/email_utils.py,sha256=pZ2vCfNxLIPUhxr0-OlABNXm12jjU44isBk8kGmqQzA,5277
 upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0--cOFTwJ0,1074
@@ -70,7 +70,7 @@ upgini/utils/target_utils.py,sha256=LRN840dzx78-wg7ftdxAkp2c1eu8-JDvkACiRThm4HE,
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.86a2.dist-info/METADATA,sha256=xU87Vnwtiae10PnJMUIC5KiOMP_TUEZ8BeafznKJxCg,49164
-upgini-1.2.86a2.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.86a2.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.86a2.dist-info/RECORD,,
+upgini-1.2.87.dev2.dist-info/METADATA,sha256=7Mt_3MedPG9E1Ie9vtf8T67JlPCGd8PnISUSrN91ctU,49167
+upgini-1.2.87.dev2.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.87.dev2.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.87.dev2.dist-info/RECORD,,