upgini 1.2.133a1__py3-none-any.whl → 1.2.135a1__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.133a1"
+__version__ = "1.2.135a1"

upgini/features_enricher.py

@@ -751,7 +751,6 @@ class FeaturesEnricher(TransformerMixin):
         exclude_features_sources: list[str] | None = None,
         keep_input: bool = True,
         trace_id: str | None = None,
-        metrics_calculation: bool = False,
         silent_mode=False,
         progress_bar: ProgressBar | None = None,
         progress_callback: Callable[[SearchProgress], Any] | None = None,
@@ -810,11 +809,12 @@ class FeaturesEnricher(TransformerMixin):
                     X,
                     y=y,
                     exclude_features_sources=exclude_features_sources,
-                    metrics_calculation=metrics_calculation,
                     silent_mode=silent_mode,
                     progress_bar=progress_bar,
                     keep_input=keep_input,
                 )
+                if TARGET in result.columns:
+                    result.drop(columns=TARGET, inplace=True)
                 self.logger.info("Transform finished successfully")
                 search_progress = SearchProgress(100.0, ProgressStage.FINISHED)
                 if progress_bar is not None:
@@ -1637,7 +1637,7 @@ class FeaturesEnricher(TransformerMixin):
 
         if not isinstance(_cv, BaseCrossValidator):
             date_column = self._get_date_column(search_keys)
-            date_series = X[date_column] if date_column is not None else None
+            date_series = X[date_column] if date_column is not None and date_column in X.columns else None
             _cv, groups = CVConfig(
                 _cv, date_series, self.random_state, self._search_task.get_shuffle_kfold(), group_columns=group_columns
             ).get_cv_and_groups(X)
@@ -1738,7 +1738,7 @@ class FeaturesEnricher(TransformerMixin):
 
         client_features = [
             c
-            for c in (validated_X.columns.to_list() + generated_features)
+            for c in validated_X.columns.to_list()
             if (not self.fit_select_features or c in set(self.feature_names_).union(self.id_columns or []))
             and c
             not in (
@@ -1747,6 +1747,7 @@ class FeaturesEnricher(TransformerMixin):
                 + [DateTimeConverter.DATETIME_COL, SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID]
             )
         ]
+        client_features.extend(f for f in generated_features if f in self.feature_names_)
         if self.baseline_score_column is not None and self.baseline_score_column not in client_features:
             client_features.append(self.baseline_score_column)
         self.logger.info(f"Client features column on prepare data for metrics: {client_features}")
@@ -1847,7 +1848,7 @@ class FeaturesEnricher(TransformerMixin):
             enriched_eval_X_sorted, enriched_eval_y_sorted = self._sort_by_system_record_id(
                 enriched_eval_X, eval_y_sampled, self.cv
             )
-            if date_column is not None:
+            if date_column is not None and date_column in eval_X_sorted.columns:
                 eval_set_dates[idx] = eval_X_sorted[date_column]
             fitting_eval_X = eval_X_sorted[fitting_x_columns].copy()
             fitting_enriched_eval_X = enriched_eval_X_sorted[fitting_enriched_x_columns].copy()
@@ -1936,7 +1937,9 @@ class FeaturesEnricher(TransformerMixin):
             and self.df_with_original_index is not None
         ):
             self.logger.info("Dataset is not imbalanced, so use enriched_X from fit")
-            return self.__get_enriched_from_fit(eval_set, trace_id, remove_outliers_calc_metrics)
+            return self.__get_enriched_from_fit(
+                validated_X, validated_y, eval_set, trace_id, remove_outliers_calc_metrics
+            )
         else:
             self.logger.info(
                 "Dataset is imbalanced or exclude_features_sources or X was passed or this is saved search."
@@ -2074,6 +2077,8 @@ class FeaturesEnricher(TransformerMixin):
 
     def __get_enriched_from_fit(
         self,
+        validated_X: pd.DataFrame,
+        validated_y: pd.Series,
         eval_set: list[tuple] | None,
         trace_id: str,
         remove_outliers_calc_metrics: bool | None,
@@ -2124,6 +2129,24 @@ class FeaturesEnricher(TransformerMixin):
             drop_system_record_id=False,
         )
 
+        enriched_Xy.rename(columns=self.fit_columns_renaming, inplace=True)
+        search_keys = {self.fit_columns_renaming.get(k, k): v for k, v in search_keys.items()}
+        generated_features = [self.fit_columns_renaming.get(c, c) for c in self.fit_generated_features]
+
+        validated_Xy = validated_X.copy()
+        validated_Xy[TARGET] = validated_y
+
+        selecting_columns = self._selecting_input_and_generated_columns(
+            validated_Xy, self.fit_generated_features, keep_input=True, trace_id=trace_id
+        )
+        selecting_columns.extend(
+            c
+            for c in enriched_Xy.columns
+            if (c in self.feature_names_ and c not in selecting_columns and c not in validated_X.columns)
+            or c in [EVAL_SET_INDEX, ENTITY_SYSTEM_RECORD_ID, SYSTEM_RECORD_ID]
+        )
+        enriched_Xy = enriched_Xy[selecting_columns]
+
         # Handle eval sets extraction based on EVAL_SET_INDEX
         if EVAL_SET_INDEX in enriched_Xy.columns:
             eval_set_indices = list(enriched_Xy[EVAL_SET_INDEX].unique())
@@ -2135,7 +2158,11 @@ class FeaturesEnricher(TransformerMixin):
                 ].copy()
             enriched_Xy = enriched_Xy.loc[enriched_Xy[EVAL_SET_INDEX] == 0].copy()
 
-        x_columns = [c for c in self.df_with_original_index.columns if c not in [EVAL_SET_INDEX, TARGET]]
+        x_columns = [
+            c
+            for c in [self.fit_columns_renaming.get(k, k) for k in self.df_with_original_index.columns]
+            if c not in [EVAL_SET_INDEX, TARGET] and c in selecting_columns
+        ]
         X_sampled = enriched_Xy[x_columns].copy()
         y_sampled = enriched_Xy[TARGET].copy()
         enriched_X = enriched_Xy.drop(columns=[TARGET, EVAL_SET_INDEX], errors="ignore")
@@ -2157,15 +2184,6 @@ class FeaturesEnricher(TransformerMixin):
                 enriched_eval_X = enriched_eval_sets[idx + 1][enriched_X_columns].copy()
                 eval_set_sampled_dict[idx] = (eval_X_sampled, enriched_eval_X, eval_y_sampled)
 
-        # reversed_renaming = {v: k for k, v in self.fit_columns_renaming.items()}
-        X_sampled.rename(columns=self.fit_columns_renaming, inplace=True)
-        enriched_X.rename(columns=self.fit_columns_renaming, inplace=True)
-        for _, (eval_X_sampled, enriched_eval_X, _) in eval_set_sampled_dict.items():
-            eval_X_sampled.rename(columns=self.fit_columns_renaming, inplace=True)
-            enriched_eval_X.rename(columns=self.fit_columns_renaming, inplace=True)
-        search_keys = {self.fit_columns_renaming.get(k, k): v for k, v in search_keys.items()}
-        generated_features = [self.fit_columns_renaming.get(c, c) for c in self.fit_generated_features]
-
         datasets_hash = hash_input(self.X, self.y, self.eval_set)
         return self.__cache_and_return_results(
             datasets_hash,
@@ -2642,7 +2660,7 @@ if response.status_code == 200:
                 generated_features = [columns_renaming.get(c, c) for c in generated_features]
                 search_keys = {columns_renaming.get(c, c): t for c, t in search_keys.items()}
                 selecting_columns = self._selecting_input_and_generated_columns(
-                    validated_Xy, generated_features, keep_input, trace_id
+                    validated_Xy, generated_features, keep_input, trace_id, is_transform=True
                 )
                 self.logger.warning(f"Filtered columns by existance in dataframe: {selecting_columns}")
                 if add_fit_system_record_id:
@@ -2895,7 +2913,7 @@ if response.status_code == 200:
             )
 
             selecting_columns = self._selecting_input_and_generated_columns(
-                validated_Xy, generated_features, keep_input, trace_id
+                validated_Xy, generated_features, keep_input, trace_id, is_transform=True
             )
             selecting_columns.extend(
                 c
@@ -2933,20 +2951,19 @@ if response.status_code == 200:
         generated_features: list[str],
         keep_input: bool,
         trace_id: str,
+        is_transform: bool = False,
     ):
         fit_input_columns = [c.originalName for c in self._search_task.get_file_metadata(trace_id).columns]
         new_columns_on_transform = [c for c in validated_Xy.columns if c not in fit_input_columns]
 
-        selected_generated_features = [
-            c for c in generated_features if not self.fit_select_features or c in self.feature_names_
-        ]
+        selected_generated_features = [c for c in generated_features if c in self.feature_names_]
         if keep_input is True:
             selected_input_columns = [
                 c
                 for c in validated_Xy.columns
                 if not self.fit_select_features
                 or c in self.feature_names_
-                or c in new_columns_on_transform
+                or (c in new_columns_on_transform and is_transform)
                 or c in self.search_keys
                 or c in (self.id_columns or [])
                 or c in [EVAL_SET_INDEX, TARGET]  # transform for metrics calculation
@@ -3245,7 +3262,7 @@ if response.status_code == 200:
         if fintech_warnings:
             for fintech_warning in fintech_warnings:
                 self.__log_warning(fintech_warning)
-        df, full_duplicates_warning = clean_full_duplicates(df, self.logger, bundle=self.bundle)
+        df, full_duplicates_warning = clean_full_duplicates(df, logger=self.logger, bundle=self.bundle)
         if full_duplicates_warning:
             if len(df) == 0:
                 raise ValidationError(full_duplicates_warning)

upgini/utils/datetime_utils.py

@@ -1,6 +1,5 @@
 import datetime
 import logging
-import re
 from typing import Dict, List, Optional
 
 import numpy as np
@@ -67,7 +66,7 @@ class DateTimeConverter:
         try:
             if s is None or len(str(s).strip()) == 0:
                 return None
-            if not re.match(DATETIME_PATTERN, str(s)):
+            if sum(ch.isdigit() for ch in str(s)) < 6:
                 return None
             return s
         except Exception:
@@ -84,30 +83,31 @@ class DateTimeConverter:
         return parsed is not None and not parsed.isna().all()
 
     def parse_datetime(self, df: pd.DataFrame, raise_errors=True) -> pd.Series | None:
-        df = df.copy()
         if len(df) == 0 or df[self.date_column].isna().all():
             return None
 
+        date_col = df[self.date_column].copy()
+
         try:
-            if df[self.date_column].apply(lambda x: isinstance(x, datetime.datetime)).all():
-                parsed_datetime = df[self.date_column].apply(lambda x: x.replace(tzinfo=None))
-            elif isinstance(df[self.date_column].dropna().values[0], datetime.date):
-                parsed_datetime = pd.to_datetime(df[self.date_column], errors="coerce")
-            elif isinstance(df[self.date_column].dtype, pd.PeriodDtype):
-                parsed_datetime = df[self.date_column].dt.to_timestamp()
-            elif is_numeric_dtype(df[self.date_column]):
+            if date_col.apply(lambda x: isinstance(x, datetime.datetime)).all():
+                parsed_datetime = date_col.apply(lambda x: x.replace(tzinfo=None))
+            elif isinstance(date_col.dropna().values[0], datetime.date):
+                parsed_datetime = pd.to_datetime(date_col, errors="coerce")
+            elif isinstance(date_col.dtype, pd.PeriodDtype):
+                parsed_datetime = date_col.dt.to_timestamp()
+            elif is_numeric_dtype(date_col):
                 # 315532801 - 2524608001    - seconds
                 # 315532801000 - 2524608001000 - milliseconds
                 # 315532801000000 - 2524608001000000 - microseconds
                 # 315532801000000000 - 2524608001000000000 - nanoseconds
-                if df[self.date_column].apply(lambda x: 10**16 < x).all():
-                    parsed_datetime = pd.to_datetime(df[self.date_column], unit="ns")
-                elif df[self.date_column].apply(lambda x: 10**14 < x < 10**16).all():
-                    parsed_datetime = pd.to_datetime(df[self.date_column], unit="us")
-                elif df[self.date_column].apply(lambda x: 10**11 < x < 10**14).all():
-                    parsed_datetime = pd.to_datetime(df[self.date_column], unit="ms")
-                elif df[self.date_column].apply(lambda x: 10**8 < x < 10**11).all():
-                    parsed_datetime = pd.to_datetime(df[self.date_column], unit="s")
+                if date_col.apply(lambda x: 10**16 < x).all():
+                    parsed_datetime = pd.to_datetime(date_col, unit="ns")
+                elif date_col.apply(lambda x: 10**14 < x < 10**16).all():
+                    parsed_datetime = pd.to_datetime(date_col, unit="us")
+                elif date_col.apply(lambda x: 10**11 < x < 10**14).all():
+                    parsed_datetime = pd.to_datetime(date_col, unit="ms")
+                elif date_col.apply(lambda x: 10**8 < x < 10**11).all():
+                    parsed_datetime = pd.to_datetime(date_col, unit="s")
                 else:
                     msg = self.bundle.get("unsupported_date_type").format(self.date_column)
                     if raise_errors:
@@ -115,8 +115,10 @@ class DateTimeConverter:
                     else:
                         return None
             else:
-                df[self.date_column] = df[self.date_column].astype("string").apply(self.clean_date)
-                parsed_datetime = self.parse_string_date(df, raise_errors)
+                date_col = date_col.astype("string").apply(self.clean_date)
+                parsed_datetime = self.parse_string_date(date_col.to_frame(self.date_column), raise_errors)
+                if parsed_datetime.isna().all():
+                    raise ValidationError(self.bundle.get("invalid_date_format").format(self.date_column))
             parsed_datetime = parsed_datetime.dt.tz_localize(None)
             return parsed_datetime
         except Exception as e:

{upgini-1.2.133a1.dist-info → upgini-1.2.135a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: upgini
-Version: 1.2.133a1
+Version: 1.2.135a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.133a1.dist-info → upgini-1.2.135a1.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=2J0xRzQRVTKW9-UjHayhhp4WFFpAteaH5RVfrXavaz0,26
+upgini/__about__.py,sha256=ut2rbJ0xiGgZg547NgkkPDfF6sWBeWwDef4pISy7Ipc,26
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=Nm2ZmwyQqvTnymYpGUwyJWy7y2ebXlHMyYmGeGcyA_s,31652
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=Na-W1f5xQVUKF4_m2Bw8mM29kLD8N3U7p9-FU0J9bi8,234415
+upgini/features_enricher.py,sha256=YvVLu2Fa0XQb-J8sUKH0W0_k-WLyfwWQ7646raObva4,235033
 upgini/http.py,sha256=-J_wOpnwVnT0ebPC6sOs6fN3AWtCD0LJLu6nlYmxaqk,44348
 upgini/metadata.py,sha256=H3wiN37k-yqWZgbPD0tJzx8DzaCIkgmX5cybhByQWLg,12619
 upgini/metrics.py,sha256=KCPE_apPN-9BIdv6GqASbJVaB_gBcy8wzNApAcyaGo4,46020
@@ -52,7 +52,7 @@ upgini/utils/config.py,sha256=zFdnjchykfp_1Tm3Qep7phLzXBpXIOzr2tIuXchRBLw,1754
 upgini/utils/country_utils.py,sha256=lY-eXWwFVegdVENFttbvLcgGDjFO17Sex8hd2PyJaRk,6937
 upgini/utils/custom_loss_utils.py,sha256=kieNZYBYZm5ZGBltF1F_jOSF4ea6C29rYuCyiDcqVNY,3857
 upgini/utils/cv_utils.py,sha256=w6FQb9nO8BWDx88EF83NpjPLarK4eR4ia0Wg0kLBJC4,3525
-upgini/utils/datetime_utils.py,sha256=l85UzSQLhtMeI2G6m-m8y8bCColCLSXNHb2-G6fKpLM,16988
+upgini/utils/datetime_utils.py,sha256=3_FQoa_ywgEeznaEPN2kuH_ES-LZJWSN2AI39sM9NRg,16988
 upgini/utils/deduplicate_utils.py,sha256=CLX0QapRxB-ZVQT7yGvv1vSd2zac5SwRjCJavujdCps,11332
 upgini/utils/display_utils.py,sha256=MoTqXZJvC6pAqgOaI3V0FG-IU_LnMfrn4TDcNvUqsdg,13316
 upgini/utils/email_utils.py,sha256=pZ2vCfNxLIPUhxr0-OlABNXm12jjU44isBk8kGmqQzA,5277
@@ -74,7 +74,7 @@ upgini/utils/target_utils.py,sha256=CihpV6SC95HwtlMH60rGAUzVDa4Id0Bva8ySprmNHlE,
 upgini/utils/track_info.py,sha256=NDKeQTUlZaYp15UoP-xLKGoDoJQ0drbDMwB0g9R0PUg,6427
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.133a1.dist-info/METADATA,sha256=oveLN_pPi2K1BqqAnu5ZnGXVMl7TeD65Jg1biA1drE0,51135
-upgini-1.2.133a1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.133a1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.133a1.dist-info/RECORD,,
+upgini-1.2.135a1.dist-info/METADATA,sha256=0E6YorGA-6HKO5wnsp75qUKG-BxNKXOVEEH5snEVBvI,51135
+upgini-1.2.135a1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+upgini-1.2.135a1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.135a1.dist-info/RECORD,,

{upgini-1.2.133a1.dist-info → upgini-1.2.135a1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: hatchling 1.27.0
 Root-Is-Purelib: true
 Tag: py3-none-any