upgini 1.2.124__tar.gz → 1.2.127__tar.gz

{upgini-1.2.124 → upgini-1.2.127}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.124
+Version: 1.2.127
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -30,6 +30,7 @@ Requires-Dist: ipywidgets>=8.1.0
 Requires-Dist: jarowinkler>=2.0.0
 Requires-Dist: levenshtein>=0.25.1
 Requires-Dist: lightgbm>=4.6.0
+Requires-Dist: more-itertools==10.7.0
 Requires-Dist: numpy<3.0.0,>=1.19.0
 Requires-Dist: pandas<3.0.0,>=1.1.0
 Requires-Dist: psutil>=5.9.0

{upgini-1.2.124 → upgini-1.2.127}/pyproject.toml

@@ -55,6 +55,7 @@ dependencies = [
     "levenshtein>=0.25.1",
     "psutil>=5.9.0",
     "category-encoders>=2.8.1",
+    "more_itertools==10.7.0",
 ]
 
 [project.urls]

upgini-1.2.127/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.127"

{upgini-1.2.124 → upgini-1.2.127}/src/upgini/autofe/binary.py

@@ -1,5 +1,6 @@
 import abc
 from typing import Optional
+
 import Levenshtein
 import numpy as np
 import pandas as pd
@@ -201,7 +202,7 @@ class JaroWinklerSim1(StringSim):
     has_symmetry_importance: bool = True
 
     def _prepare_value(self, value: Optional[str]) -> Optional[str]:
-        return value
+        return value if value is not None and len(value) > 0 else None
 
     def _similarity(self, left: str, right: str) -> float:
         return jarowinkler_similarity(left, right)
@@ -216,7 +217,7 @@ class JaroWinklerSim2(StringSim):
     has_symmetry_importance: bool = True
 
     def _prepare_value(self, value: Optional[str]) -> Optional[str]:
-        return value[::-1] if value is not None else None
+        return value[::-1] if value is not None and len(value) > 0 else None
 
     def _similarity(self, left: str, right: str) -> float:
         return jarowinkler_similarity(left, right)
@@ -231,7 +232,7 @@ class LevenshteinSim(StringSim):
     has_symmetry_importance: bool = True
 
     def _prepare_value(self, value: Optional[str]) -> Optional[str]:
-        return value
+        return value if value is not None and len(value) > 0 else None
 
     def _similarity(self, left: str, right: str) -> float:
         return 1 - Levenshtein.distance(left, right) / max(len(left), len(right))

{upgini-1.2.124 → upgini-1.2.127}/src/upgini/data_source/data_source_publisher.py

@@ -123,17 +123,9 @@ class DataSourcePublisher:
                     set(search_keys.values()) == {SearchKey.IP_RANGE_FROM, SearchKey.IP_RANGE_TO}
                     or set(search_keys.values()) == {SearchKey.IPV6_RANGE_FROM, SearchKey.IPV6_RANGE_TO}
                     or set(search_keys.values()) == {SearchKey.MSISDN_RANGE_FROM, SearchKey.MSISDN_RANGE_TO}
+                    or snapshot_frequency_days is not None or join_date_abs_limit_days is not None
                 ) and sort_column is None:
                     raise ValidationError("Sort column is required for passed search keys")
-                if (
-                    set(search_keys.values()) == {SearchKey.PHONE, SearchKey.DATE}
-                    and snapshot_frequency_days is None
-                    and join_date_abs_limit_days is None
-                ):
-                    raise ValidationError(
-                        "With MSISDN and DATE keys one of the snapshot_frequency_days or"
-                        " join_date_abs_limit_days parameters is required"
-                    )
                 if (
                     set(search_keys.values()) == {SearchKey.PHONE, SearchKey.DATE}
                     or set(search_keys.values()) == {SearchKey.HEM, SearchKey.DATE}

{upgini-1.2.124 → upgini-1.2.127}/src/upgini/dataset.py

@@ -151,7 +151,9 @@ class Dataset:
     def etalon_def_checked(self) -> Dict[str, str]:
         if self.etalon_def is None:
             self.etalon_def = {
-                v.value: k for k, v in self.meaning_types_checked.items() if v != FileColumnMeaningType.FEATURE
+                v.value: k
+                for k, v in self.meaning_types_checked.items()
+                if v not in [FileColumnMeaningType.FEATURE, FileColumnMeaningType.DATE_FEATURE]
             }
 
         return self.etalon_def

{upgini-1.2.124 → upgini-1.2.127}/src/upgini/features_enricher.py

@@ -76,7 +76,7 @@ from upgini.utils.custom_loss_utils import (
 )
 from upgini.utils.cv_utils import CVConfig, get_groups
 from upgini.utils.datetime_utils import (
-    DateTimeSearchKeyConverter,
+    DateTimeConverter,
     is_blocked_time_series,
     is_dates_distribution_valid,
     is_time_series,
@@ -220,7 +220,9 @@ class FeaturesEnricher(TransformerMixin):
         cv: CVType | None = None,
         loss: str | None = None,
         autodetect_search_keys: bool = True,
+        # deprecated, use text_features instead
         generate_features: list[str] | None = None,
+        text_features: list[str] | None = None,
         columns_for_online_api: list[str] | None = None,
         round_embeddings: int | None = None,
         logs_enabled: bool = True,
@@ -305,10 +307,8 @@ class FeaturesEnricher(TransformerMixin):
             search_task = SearchTask(search_id, rest_client=self.rest_client, logger=self.logger)
 
             print(self.bundle.get("search_by_task_id_start"))
-            trace_id = str(uuid.uuid4())
-            if self.print_trace_id:
-                print(f"https://app.datadoghq.eu/logs?query=%40trace_id%3A{trace_id}")
-            with MDC(trace_id=trace_id):
+            trace_id = time.time_ns()
+            with MDC(correlation_id=trace_id):
                 try:
                     self.logger.debug(f"FeaturesEnricher created from existing search: {search_id}")
                     self._search_task = search_task.poll_result(trace_id, quiet=True, check_fit=True)
@@ -342,14 +342,14 @@ class FeaturesEnricher(TransformerMixin):
         self.shared_datasets = shared_datasets
         if shared_datasets is not None:
             self.runtime_parameters.properties["shared_datasets"] = ",".join(shared_datasets)
-        self.generate_features = generate_features
+        self.generate_features = text_features or generate_features
         self.round_embeddings = round_embeddings
-        if generate_features is not None:
-            if len(generate_features) > self.GENERATE_FEATURES_LIMIT:
+        if self.generate_features is not None:
+            if len(self.generate_features) > self.GENERATE_FEATURES_LIMIT:
                 msg = self.bundle.get("too_many_generate_features").format(self.GENERATE_FEATURES_LIMIT)
                 self.logger.error(msg)
                 raise ValidationError(msg)
-            self.runtime_parameters.properties["generate_features"] = ",".join(generate_features)
+            self.runtime_parameters.properties["generate_features"] = ",".join(self.generate_features)
             if round_embeddings is not None:
                 if not isinstance(round_embeddings, int) or round_embeddings < 0:
                     msg = self.bundle.get("invalid_round_embeddings")
@@ -484,9 +484,9 @@ class FeaturesEnricher(TransformerMixin):
         stability_agg_func: str, optional (default="max")
             Function to aggregate stability values. Can be "max", "min", "mean".
         """
-        trace_id = str(uuid.uuid4())
+        trace_id = time.time_ns()
         if self.print_trace_id:
-            print(f"https://app.datadoghq.eu/logs?query=%40trace_id%3A{trace_id}")
+            print(f"https://app.datadoghq.eu/logs?query=%40correlation_id%3A{trace_id}")
         start_time = time.time()
         auto_fe_parameters = AutoFEParameters() if auto_fe_parameters is None else auto_fe_parameters
         search_progress = SearchProgress(0.0, ProgressStage.START_FIT)
@@ -498,7 +498,7 @@ class FeaturesEnricher(TransformerMixin):
             progress_bar.progress = search_progress.to_progress_bar()
             progress_bar.display()
 
-        with MDC(trace_id=trace_id):
+        with MDC(correlation_id=trace_id):
             if len(args) > 0:
                 msg = f"WARNING: Unsupported positional arguments for fit: {args}"
                 self.logger.warning(msg)
@@ -643,11 +643,11 @@ class FeaturesEnricher(TransformerMixin):
 
         self.warning_counter.reset()
         auto_fe_parameters = AutoFEParameters() if auto_fe_parameters is None else auto_fe_parameters
-        trace_id = str(uuid.uuid4())
+        trace_id = time.time_ns()
         if self.print_trace_id:
-            print(f"https://app.datadoghq.eu/logs?query=%40trace_id%3A{trace_id}")
+            print(f"https://app.datadoghq.eu/logs?query=%40correlation_id%3A{trace_id}")
         start_time = time.time()
-        with MDC(trace_id=trace_id):
+        with MDC(correlation_id=trace_id):
             if len(args) > 0:
                 msg = f"WARNING: Unsupported positional arguments for fit_transform: {args}"
                 self.logger.warning(msg)
@@ -745,8 +745,8 @@ class FeaturesEnricher(TransformerMixin):
     def transform(
         self,
         X: pd.DataFrame,
-        *args,
         y: pd.Series | None = None,
+        *args,
         exclude_features_sources: list[str] | None = None,
         keep_input: bool = True,
         trace_id: str | None = None,
@@ -787,9 +787,11 @@ class FeaturesEnricher(TransformerMixin):
             progress_bar.progress = search_progress.to_progress_bar()
             if new_progress:
                 progress_bar.display()
-        trace_id = trace_id or str(uuid.uuid4())
+        trace_id = trace_id or time.time_ns()
+        if self.print_trace_id:
+            print(f"https://app.datadoghq.eu/logs?query=%40correlation_id%3A{trace_id}")
         search_id = self.search_id or (self._search_task.search_task_id if self._search_task is not None else None)
-        with MDC(trace_id=trace_id, search_id=search_id):
+        with MDC(correlation_id=trace_id, search_id=search_id):
             self.dump_input(trace_id, X)
             if len(args) > 0:
                 msg = f"WARNING: Unsupported positional arguments for transform: {args}"
@@ -904,10 +906,10 @@ class FeaturesEnricher(TransformerMixin):
             Dataframe with metrics calculated on train and validation datasets.
         """
 
-        trace_id = trace_id or str(uuid.uuid4())
+        trace_id = trace_id or time.time_ns()
         start_time = time.time()
         search_id = self.search_id or (self._search_task.search_task_id if self._search_task is not None else None)
-        with MDC(trace_id=trace_id, search_id=search_id):
+        with MDC(correlation_id=trace_id, search_id=search_id):
             self.logger.info("Start calculate metrics")
             if len(args) > 0:
                 msg = f"WARNING: Unsupported positional arguments for calculate_metrics: {args}"
@@ -1415,13 +1417,11 @@ class FeaturesEnricher(TransformerMixin):
         # Find latest eval set or earliest if all eval sets are before train set
         date_column = self._get_date_column(search_keys)
 
-        date_converter = DateTimeSearchKeyConverter(
+        date_converter = DateTimeConverter(
             date_column, self.date_format, self.logger, self.bundle, generate_cyclical_features=False
         )
 
-        X = date_converter.convert(X)
-
-        x_date = X[date_column].dropna()
+        x_date = date_converter.to_date_ms(X).dropna()
         if len(x_date) == 0:
             self.logger.warning("Empty date column in X")
             return []
@@ -1434,8 +1434,7 @@ class FeaturesEnricher(TransformerMixin):
             if date_column not in eval_x.columns:
                 self.logger.warning(f"Date column not found in eval_set {i + 1}")
                 continue
-            eval_x = date_converter.convert(eval_x)
-            eval_x_date = eval_x[date_column].dropna()
+            eval_x_date = date_converter.to_date_ms(eval_x).dropna()
             if len(eval_x_date) < 1000:
                 self.logger.warning(f"Eval_set {i} has less than 1000 rows. It will be ignored for stability check")
                 continue
@@ -1472,8 +1471,7 @@ class FeaturesEnricher(TransformerMixin):
         )
         checking_eval_set_df = checking_eval_set_df.copy()
 
-        checking_eval_set_df[date_column] = eval_set_dates[selected_eval_set_idx]
-        checking_eval_set_df = date_converter.convert(checking_eval_set_df)
+        checking_eval_set_df[date_column] = date_converter.to_date_ms(eval_set_dates[selected_eval_set_idx].to_frame())
 
         psi_values_sparse = calculate_sparsity_psi(
             checking_eval_set_df, cat_features, date_column, self.logger, model_task_type
@@ -1745,9 +1743,11 @@ class FeaturesEnricher(TransformerMixin):
             not in (
                 excluding_search_keys
                 + list(self.fit_dropped_features)
-                + [DateTimeSearchKeyConverter.DATETIME_COL, SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID]
+                + [DateTimeConverter.DATETIME_COL, SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID]
             )
         ]
+        if self.baseline_score_column is not None and self.baseline_score_column not in client_features:
+            client_features.append(self.baseline_score_column)
         self.logger.info(f"Client features column on prepare data for metrics: {client_features}")
 
         selected_enriched_features = [c for c in self.feature_names_ if c not in client_features]
@@ -1995,7 +1995,7 @@ class FeaturesEnricher(TransformerMixin):
         date_column = self._get_date_column(search_keys)
         generated_features = []
         if date_column is not None:
-            converter = DateTimeSearchKeyConverter(
+            converter = DateTimeConverter(
                 date_column,
                 self.date_format,
                 self.logger,
@@ -2004,6 +2004,7 @@ class FeaturesEnricher(TransformerMixin):
             )
             # Leave original date column values
             df_with_date_features = converter.convert(df, keep_time=True)
+            # TODO check if this is correct
             df_with_date_features[date_column] = df[date_column]
             df = df_with_date_features
             generated_features = converter.generated_features
@@ -2035,8 +2036,8 @@ class FeaturesEnricher(TransformerMixin):
         # Sample after sorting by system_record_id for idempotency
         df.sort_values(by=SYSTEM_RECORD_ID, inplace=True)
 
-        if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
-            df = df.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL)
+        if DateTimeConverter.DATETIME_COL in df.columns:
+            df = df.drop(columns=DateTimeConverter.DATETIME_COL)
 
         df = df.rename(columns=columns_renaming)
         generated_features = [columns_renaming.get(c, c) for c in generated_features]
@@ -2388,7 +2389,7 @@ class FeaturesEnricher(TransformerMixin):
     def get_progress(self, trace_id: str | None = None, search_task: SearchTask | None = None) -> SearchProgress:
         search_task = search_task or self._search_task
         if search_task is not None:
-            trace_id = trace_id or uuid.uuid4()
+            trace_id = trace_id or time.time_ns()
             return search_task.get_progress(trace_id)
 
     def display_transactional_transform_api(self, only_online_sources=False):
@@ -2416,7 +2417,7 @@ class FeaturesEnricher(TransformerMixin):
                 return "12345678"
             return "test_value"
 
-        file_metadata = self._search_task.get_file_metadata(str(uuid.uuid4()))
+        file_metadata = self._search_task.get_file_metadata(time.time_ns())
 
         def get_column_meta(column_name: str) -> FileColumnMetadata:
             for c in file_metadata.columns:
@@ -2510,7 +2511,7 @@ if response.status_code == 200:
 
         start_time = time.time()
         search_id = self.search_id or (self._search_task.search_task_id if self._search_task is not None else None)
-        with MDC(trace_id=trace_id, search_id=search_id):
+        with MDC(correlation_id=trace_id, search_id=search_id):
             self.logger.info("Start transform")
 
             validated_X, validated_y, validated_eval_set = self._validate_train_eval(
@@ -2552,10 +2553,15 @@ if response.status_code == 200:
                 if transform_usage.has_limit:
                     if len(X) > transform_usage.rest_rows:
                         rest_rows = max(transform_usage.rest_rows, 0)
-                        msg = self.bundle.get("transform_usage_warning").format(len(X), rest_rows)
+                        bundle_msg = (
+                            "transform_usage_warning_registered"
+                            if self.__is_registered
+                            else "transform_usage_warning_demo"
+                        )
+                        msg = self.bundle.get(bundle_msg).format(len(X), rest_rows)
                         self.logger.warning(msg)
                         print(msg)
-                        show_request_quote_button()
+                        show_request_quote_button(is_registered=self.__is_registered)
                         return None, {}, [], {}
                     else:
                         msg = self.bundle.get("transform_usage_info").format(
@@ -2599,7 +2605,7 @@ if response.status_code == 200:
             generated_features = []
             date_column = self._get_date_column(search_keys)
             if date_column is not None:
-                converter = DateTimeSearchKeyConverter(
+                converter = DateTimeConverter(
                     date_column,
                     self.date_format,
                     self.logger,
@@ -2656,8 +2662,8 @@ if response.status_code == 200:
 
             # Don't pass all features in backend on transform
             runtime_parameters = self._get_copy_of_runtime_parameters()
-            features_for_transform = self._search_task.get_features_for_transform() or []
-            if len(features_for_transform) > 0:
+            features_for_transform = self._search_task.get_features_for_transform()
+            if features_for_transform:
                 missing_features_for_transform = [
                     columns_renaming.get(f) or f for f in features_for_transform if f not in df.columns
                 ]
@@ -2668,7 +2674,10 @@ if response.status_code == 200:
                     raise ValidationError(
                         self.bundle.get("missing_features_for_transform").format(missing_features_for_transform)
                     )
-                runtime_parameters.properties["features_for_embeddings"] = ",".join(features_for_transform)
+            features_for_embeddings = self._search_task.get_features_for_embeddings()
+            if features_for_embeddings:
+                runtime_parameters.properties["features_for_embeddings"] = ",".join(features_for_embeddings)
+            features_for_transform = [f for f in features_for_transform if f not in search_keys.keys()]
 
             columns_for_system_record_id = sorted(list(search_keys.keys()) + features_for_transform)
 
@@ -2729,8 +2738,22 @@ if response.status_code == 200:
                 )
                 df = converter.convert(df)
 
+            date_features = []
+            for col in features_for_transform:
+                if DateTimeConverter(col).is_datetime(df):
+                    df[col] = DateTimeConverter(col).to_date_string(df)
+                    date_features.append(col)
+
             meaning_types = {}
-            meaning_types.update({col: FileColumnMeaningType.FEATURE for col in features_for_transform})
+            meaning_types.update(
+                {
+                    col: FileColumnMeaningType.FEATURE
+                    for col in features_for_transform
+                    if col not in date_features and col not in generated_features
+                }
+            )
+            meaning_types.update({col: FileColumnMeaningType.GENERATED_FEATURE for col in generated_features})
+            meaning_types.update({col: FileColumnMeaningType.DATE_FEATURE for col in date_features})
             meaning_types.update({col: key.value for col, key in search_keys.items()})
 
             features_not_to_pass.extend(
@@ -2743,8 +2766,8 @@ if response.status_code == 200:
                 ]
             )
 
-            if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
-                df = df.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL)
+            if DateTimeConverter.DATETIME_COL in df.columns:
+                df = df.drop(columns=DateTimeConverter.DATETIME_COL)
 
             # search keys might be changed after explode
             columns_for_system_record_id = sorted(list(search_keys.keys()) + features_for_transform)
@@ -2926,6 +2949,7 @@ if response.status_code == 200:
                 or c in self.search_keys
                 or c in (self.id_columns or [])
                 or c in [EVAL_SET_INDEX, TARGET]  # transform for metrics calculation
+                or c == self.baseline_score_column
             ]
         else:
             selected_input_columns = []
@@ -3124,7 +3148,7 @@ if response.status_code == 200:
         self.fit_generated_features = []
 
         if has_date:
-            converter = DateTimeSearchKeyConverter(
+            converter = DateTimeConverter(
                 maybe_date_column,
                 self.date_format,
                 self.logger,
@@ -3177,8 +3201,8 @@ if response.status_code == 200:
             self.TARGET_NAME,
             EVAL_SET_INDEX,
         ] + list(self.fit_search_keys.keys())
-        if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
-            non_feature_columns.append(DateTimeSearchKeyConverter.DATETIME_COL)
+        if DateTimeConverter.DATETIME_COL in df.columns:
+            non_feature_columns.append(DateTimeConverter.DATETIME_COL)
 
         features_columns = [c for c in df.columns if c not in non_feature_columns]
 
@@ -3265,15 +3289,28 @@ if response.status_code == 200:
             ENTITY_SYSTEM_RECORD_ID,
             SEARCH_KEY_UNNEST,
         ] + list(self.fit_search_keys.keys())
-        if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
-            non_feature_columns.append(DateTimeSearchKeyConverter.DATETIME_COL)
+        if DateTimeConverter.DATETIME_COL in df.columns:
+            non_feature_columns.append(DateTimeConverter.DATETIME_COL)
 
         features_columns = [c for c in df.columns if c not in non_feature_columns]
 
+        # find date features
+        date_features = []
+        for col in features_columns:
+            if DateTimeConverter(col).is_datetime(df):
+                df[col] = DateTimeConverter(col).to_date_string(df)
+                date_features.append(col)
+
         meaning_types = {
             **{col: key.value for col, key in self.fit_search_keys.items()},
-            **{str(c): FileColumnMeaningType.FEATURE for c in df.columns if c not in non_feature_columns},
+            **{
+                str(c): FileColumnMeaningType.FEATURE
+                for c in df.columns
+                if c not in non_feature_columns and c not in date_features and c not in self.fit_generated_features
+            },
         }
+        meaning_types.update({col: FileColumnMeaningType.GENERATED_FEATURE for col in self.fit_generated_features})
+        meaning_types.update({col: FileColumnMeaningType.DATE_FEATURE for col in date_features})
         meaning_types[self.TARGET_NAME] = FileColumnMeaningType.TARGET
         meaning_types[ENTITY_SYSTEM_RECORD_ID] = FileColumnMeaningType.ENTITY_SYSTEM_RECORD_ID
         if SEARCH_KEY_UNNEST in df.columns:
@@ -3294,8 +3331,8 @@ if response.status_code == 200:
             self.bundle,
         )
 
-        if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
-            df = df.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL)
+        if DateTimeConverter.DATETIME_COL in df.columns:
+            df = df.drop(columns=DateTimeConverter.DATETIME_COL)
 
         meaning_types[SYSTEM_RECORD_ID] = FileColumnMeaningType.SYSTEM_RECORD_ID
 
@@ -3332,7 +3369,14 @@ if response.status_code == 200:
         dataset.columns_renaming = self.fit_columns_renaming
 
         self.passed_features = [
-            column for column, meaning_type in meaning_types.items() if meaning_type == FileColumnMeaningType.FEATURE
+            column
+            for column, meaning_type in meaning_types.items()
+            if meaning_type
+            in [
+                FileColumnMeaningType.FEATURE,
+                FileColumnMeaningType.DATE_FEATURE,
+                FileColumnMeaningType.GENERATED_FEATURE,
+            ]
         ]
 
         self._search_task = dataset.search(
@@ -3860,8 +3904,8 @@ if response.status_code == 200:
             X = Xy.drop(columns=TARGET)
             y = Xy[TARGET].copy()
 
-        if DateTimeSearchKeyConverter.DATETIME_COL in X.columns:
-            X.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL, inplace=True)
+        if DateTimeConverter.DATETIME_COL in X.columns:
+            X.drop(columns=DateTimeConverter.DATETIME_COL, inplace=True)
 
         return X, y
 
@@ -3871,8 +3915,8 @@ if response.status_code == 200:
         X: pd.DataFrame, y: pd.Series, search_keys: dict[str, SearchKey], cv: CVType | None
     ) -> tuple[pd.DataFrame, pd.Series]:
         if cv not in [CVType.time_series, CVType.blocked_time_series]:
-            if DateTimeSearchKeyConverter.DATETIME_COL in X.columns:
-                date_column = DateTimeSearchKeyConverter.DATETIME_COL
+            if DateTimeConverter.DATETIME_COL in X.columns:
+                date_column = DateTimeConverter.DATETIME_COL
             else:
                 date_column = FeaturesEnricher._get_date_column(search_keys)
             sort_columns = [date_column] if date_column is not None else []
@@ -3900,8 +3944,8 @@ if response.status_code == 200:
 
             y = Xy[TARGET].copy()
 
-        if DateTimeSearchKeyConverter.DATETIME_COL in X.columns:
-            X.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL, inplace=True)
+        if DateTimeConverter.DATETIME_COL in X.columns:
+            X.drop(columns=DateTimeConverter.DATETIME_COL, inplace=True)
 
         return X, y
 
@@ -3980,12 +4024,10 @@ if response.status_code == 200:
             maybe_date_col = SearchKey.find_key(self.search_keys, [SearchKey.DATE, SearchKey.DATETIME])
             if X is not None and maybe_date_col is not None and maybe_date_col in X.columns:
                 # TODO cast date column to single dtype
-                date_converter = DateTimeSearchKeyConverter(
-                    maybe_date_col, self.date_format, generate_cyclical_features=False
-                )
-                converted_X = date_converter.convert(X)
-                min_date = converted_X[maybe_date_col].min()
-                max_date = converted_X[maybe_date_col].max()
+                date_converter = DateTimeConverter(maybe_date_col, self.date_format, generate_cyclical_features=False)
+                date_col_values = date_converter.to_date_ms(X)
+                min_date = date_col_values.min()
+                max_date = date_col_values.max()
                 self.logger.info(f"Dates interval is ({min_date}, {max_date})")
 
         except Exception:
@@ -4022,7 +4064,7 @@ if response.status_code == 200:
                 self.__log_warning(bundle.get("current_date_added"))
             df[FeaturesEnricher.CURRENT_DATE] = datetime.date.today()
             search_keys[FeaturesEnricher.CURRENT_DATE] = SearchKey.DATE
-            converter = DateTimeSearchKeyConverter(FeaturesEnricher.CURRENT_DATE, generate_cyclical_features=False)
+            converter = DateTimeConverter(FeaturesEnricher.CURRENT_DATE, generate_cyclical_features=False)
             df = converter.convert(df)
         return df
 
@@ -4153,8 +4195,8 @@ if response.status_code == 200:
             "__target",
             ENTITY_SYSTEM_RECORD_ID,
         ]
-        if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
-            date_column = DateTimeSearchKeyConverter.DATETIME_COL
+        if DateTimeConverter.DATETIME_COL in df.columns:
+            date_column = DateTimeConverter.DATETIME_COL
             sort_exclude_columns.append(FeaturesEnricher._get_date_column(search_keys))
         else:
             date_column = FeaturesEnricher._get_date_column(search_keys)
@@ -4399,7 +4441,9 @@ if response.status_code == 200:
             raise Exception(self.bundle.get("missing_features_meta"))
         features_meta = deepcopy(features_meta)
 
-        original_names_dict = {c.name: c.originalName for c in self._search_task.get_file_metadata(trace_id).columns}
+        file_metadata_columns = self._search_task.get_file_metadata(trace_id).columns
+        file_meta_by_orig_name = {c.originalName: c for c in file_metadata_columns}
+        original_names_dict = {c.name: c.originalName for c in file_metadata_columns}
         features_df = self._search_task.get_all_initial_raw_features(trace_id, metrics_calculation=True)
 
         # To be sure that names with hash suffixes
@@ -4419,7 +4463,11 @@ if response.status_code == 200:
             original_name = original_names_dict.get(feature_meta.name, feature_meta.name)
             feature_meta.name = original_name
 
-            is_client_feature = original_name in clients_features_df.columns
+            file_meta = file_meta_by_orig_name.get(original_name)
+            is_generated_feature = (
+                file_meta is not None and file_meta.meaningType == FileColumnMeaningType.GENERATED_FEATURE
+            )
+            is_client_feature = original_name in clients_features_df.columns and not is_generated_feature
 
             if selected_features is not None and feature_meta.name not in selected_features:
                 self.logger.info(f"Feature {feature_meta.name} is not selected before and skipped")
@@ -4442,9 +4490,13 @@ if response.status_code == 200:
 
         for feature_meta in selected_features_meta:
             original_name = original_names_dict.get(feature_meta.name, feature_meta.name)
-            is_client_feature = original_name in clients_features_df.columns
+            file_meta = file_meta_by_orig_name.get(original_name)
+            is_generated_feature = (
+                file_meta is not None and file_meta.meaningType == FileColumnMeaningType.GENERATED_FEATURE
+            )
+            is_client_feature = original_name in clients_features_df.columns and not is_generated_feature
 
-            if not is_client_feature:
+            if not is_client_feature and not is_generated_feature:
                 self.external_source_feature_names.append(original_name)
 
             if self.psi_values is not None:
@@ -4475,9 +4527,10 @@ if response.status_code == 200:
 
             self.feature_names_.append(feature_meta.name)
             self.feature_importances_.append(_round_shap_value(feature_meta.shap_value))
-
             df_for_sample = features_df if feature_meta.name in features_df.columns else clients_features_df
-            feature_info = FeatureInfo.from_metadata(feature_meta, df_for_sample, is_client_feature)
+            feature_info = FeatureInfo.from_metadata(
+                feature_meta, df_for_sample, is_client_feature, is_generated_feature
+            )
             features_info.append(feature_info.to_row(self.bundle))
             features_info_without_links.append(feature_info.to_row_without_links(self.bundle))
             internal_features_info.append(feature_info.to_internal_row(self.bundle))
@@ -4488,7 +4541,7 @@ if response.status_code == 200:
         if len(features_info) > 0:
             self.features_info = pd.DataFrame(features_info)
             # If all psi values are 0 or null, drop psi column
-            if self.features_info[self.bundle.get("features_info_psi")].fillna(0.0).eq(0.0).all():
+            if self.features_info[self.bundle.get("features_info_psi")].astype(np.float64).fillna(0.0).eq(0.0).all():
                 self.features_info.drop(columns=[self.bundle.get("features_info_psi")], inplace=True)
             self._features_info_without_links = pd.DataFrame(features_info_without_links)
             self._internal_features_info = pd.DataFrame(internal_features_info)
@@ -4954,7 +5007,7 @@ if response.status_code == 200:
         eval_set: tuple | None = None,
     ):
         def dump_task(X_, y_, eval_set_):
-            with MDC(trace_id=trace_id):
+            with MDC(correlation_id=trace_id):
                 try:
                     if isinstance(X_, pd.Series):
                         X_ = X_.to_frame()

{upgini-1.2.124 → upgini-1.2.127}/src/upgini/metadata.py

@@ -36,6 +36,8 @@ class FileColumnMeaningType(Enum):
     SCORE = "SCORE"
     TARGET = "TARGET"
     FEATURE = "FEATURE"
+    GENERATED_FEATURE = "GENERATED_FEATURE"
+    DATE_FEATURE = "DATE_FEATURE"
     CUSTOM_KEY = "CUSTOM_KEY"
     COUNTRY = "COUNTRY"
     POSTAL_CODE = "POSTAL_CODE"

{upgini-1.2.124 → upgini-1.2.127}/src/upgini/normalizer/normalize_utils.py

@@ -25,7 +25,7 @@ from upgini.metadata import (
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.utils import find_numbers_with_decimal_comma
 from upgini.utils.country_utils import CountrySearchKeyConverter
-from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
+from upgini.utils.datetime_utils import DateTimeConverter
 from upgini.utils.ip_utils import IpSearchKeyConverter
 from upgini.utils.phone_utils import PhoneSearchKeyConverter
 from upgini.utils.postal_code_utils import PostalCodeSearchKeyConverter
@@ -89,7 +89,7 @@ class Normalizer:
                 SYSTEM_RECORD_ID,
                 ENTITY_SYSTEM_RECORD_ID,
                 SEARCH_KEY_UNNEST,
-                DateTimeSearchKeyConverter.DATETIME_COL,
+                DateTimeConverter.DATETIME_COL,
             ]:
                 self.columns_renaming[column] = column
                 new_columns.append(column)