upgini 1.2.29__tar.gz → 1.2.29a1__tar.gz

{upgini-1.2.29 → upgini-1.2.29a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.29
+Version: 1.2.29a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.29a1/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.29a1"

{upgini-1.2.29 → upgini-1.2.29a1}/src/upgini/features_enricher.py

@@ -54,7 +54,6 @@ from upgini.metadata import (
     SYSTEM_RECORD_ID,
     TARGET,
     CVType,
-    FeaturesMetadataV2,
     FileColumnMeaningType,
     ModelTaskType,
     RuntimeParameters,
@@ -96,7 +95,6 @@ from upgini.utils.email_utils import (
     EmailSearchKeyConverter,
     EmailSearchKeyDetector,
 )
-from upgini.utils.feature_info import FeatureInfo, _round_shap_value
 from upgini.utils.features_validator import FeaturesValidator
 from upgini.utils.format import Format
 from upgini.utils.ip_utils import IpSearchKeyConverter
@@ -160,10 +158,6 @@ class FeaturesEnricher(TransformerMixin):
 
     shared_datasets: list of str, optional (default=None)
         List of private shared dataset ids for custom search
-
-    select_features: bool, optional (default=False)
-        If True, return only selected features both from input and data sources.
-        Otherwise, return all features from input and only selected features from data sources.
     """
 
     TARGET_NAME = "target"
@@ -230,7 +224,6 @@ class FeaturesEnricher(TransformerMixin):
         client_visitorid: Optional[str] = None,
         custom_bundle_config: Optional[str] = None,
         add_date_if_missing: bool = True,
-        select_features: bool = False,
         **kwargs,
     ):
         self.bundle = get_custom_bundle(custom_bundle_config)
@@ -284,11 +277,8 @@ class FeaturesEnricher(TransformerMixin):
         self._relevant_data_sources_wo_links: pd.DataFrame = self.EMPTY_DATA_SOURCES
         self.metrics: Optional[pd.DataFrame] = None
         self.feature_names_ = []
-        self.dropped_client_feature_names_ = []
         self.feature_importances_ = []
         self.search_id = search_id
-        self.select_features = select_features
-
         if search_id:
             search_task = SearchTask(search_id, rest_client=self.rest_client, logger=self.logger)
 
@@ -1009,10 +999,9 @@ class FeaturesEnricher(TransformerMixin):
                             text_features=self.generate_features,
                             has_date=has_date,
                         )
-                        etalon_cv_result = baseline_estimator.cross_val_predict(
+                        etalon_metric, _ = baseline_estimator.cross_val_predict(
                             fitting_X, y_sorted, self.baseline_score_column
                         )
-                        etalon_metric = etalon_cv_result.get_display_metric()
                         if etalon_metric is None:
                             self.logger.info(
                                 f"Baseline {metric} on train client features is None (maybe all features was removed)"
@@ -1044,9 +1033,9 @@ class FeaturesEnricher(TransformerMixin):
                             text_features=self.generate_features,
                             has_date=has_date,
                         )
-                        enriched_cv_result = enriched_estimator.cross_val_predict(fitting_enriched_X, enriched_y_sorted)
-                        enriched_metric = enriched_cv_result.get_display_metric()
-                        enriched_shaps = enriched_cv_result.shap_values
+                        enriched_metric, enriched_shaps = enriched_estimator.cross_val_predict(
+                            fitting_enriched_X, enriched_y_sorted
+                        )
 
                         if enriched_shaps is not None:
                             self._update_shap_values(enriched_shaps)
@@ -1059,7 +1048,7 @@ class FeaturesEnricher(TransformerMixin):
                         else:
                             self.logger.info(f"Enriched {metric} on train combined features: {enriched_metric}")
                         if etalon_metric is not None and enriched_metric is not None:
-                            uplift = (enriched_cv_result.metric - etalon_cv_result.metric) * multiplier
+                            uplift = (enriched_metric - etalon_metric) * multiplier
 
                     train_metrics = {
                         self.bundle.get("quality_metrics_segment_header"): self.bundle.get(
@@ -1102,10 +1091,9 @@ class FeaturesEnricher(TransformerMixin):
                                     f"Calculate baseline {metric} on eval set {idx + 1} "
                                     f"on client features: {eval_X_sorted.columns.to_list()}"
                                 )
-                                etalon_eval_results = baseline_estimator.calculate_metric(
+                                etalon_eval_metric = baseline_estimator.calculate_metric(
                                     eval_X_sorted, eval_y_sorted, self.baseline_score_column
                                 )
-                                etalon_eval_metric = etalon_eval_results.get_display_metric()
                                 self.logger.info(
                                     f"Baseline {metric} on eval set {idx + 1} client features: {etalon_eval_metric}"
                                 )
@@ -1117,10 +1105,9 @@ class FeaturesEnricher(TransformerMixin):
                                     f"Calculate enriched {metric} on eval set {idx + 1} "
                                     f"on combined features: {enriched_eval_X_sorted.columns.to_list()}"
                                 )
-                                enriched_eval_results = enriched_estimator.calculate_metric(
+                                enriched_eval_metric = enriched_estimator.calculate_metric(
                                     enriched_eval_X_sorted, enriched_eval_y_sorted
                                 )
-                                enriched_eval_metric = enriched_eval_results.get_display_metric()
                                 self.logger.info(
                                     f"Enriched {metric} on eval set {idx + 1} combined features: {enriched_eval_metric}"
                                 )
@@ -1128,7 +1115,7 @@ class FeaturesEnricher(TransformerMixin):
                                 enriched_eval_metric = None
 
                             if etalon_eval_metric is not None and enriched_eval_metric is not None:
-                                eval_uplift = (enriched_eval_results.metric - etalon_eval_results.metric) * multiplier
+                                eval_uplift = (enriched_eval_metric - etalon_eval_metric) * multiplier
                             else:
                                 eval_uplift = None
 
@@ -1211,7 +1198,9 @@ class FeaturesEnricher(TransformerMixin):
 
     def _update_shap_values(self, new_shaps: Dict[str, float]):
         new_shaps = {
-            feature: _round_shap_value(shap) for feature, shap in new_shaps.items() if feature in self.feature_names_
+            feature: self._round_shap_value(shap)
+            for feature, shap in new_shaps.items()
+            if feature in self.feature_names_
         }
         features_importances = list(new_shaps.items())
         features_importances.sort(key=lambda m: (-m[1], m[0]))
@@ -1260,7 +1249,7 @@ class FeaturesEnricher(TransformerMixin):
                 display_html_dataframe(
                     self.relevant_data_sources,
                     self._relevant_data_sources_wo_links,
-                    self.bundle.get("relevant_data_sources_header"),
+                    self.bundle.get("relevant_features_header"),
                     display_handle=self.data_sources_display_handle,
                 )
             except (ImportError, NameError):
@@ -1448,12 +1437,7 @@ class FeaturesEnricher(TransformerMixin):
         client_features = [
             c
             for c in X_sampled.columns.to_list()
-            if (
-                not self.select_features
-                or c in self.feature_names_
-                or (self.fit_columns_renaming is not None and self.fit_columns_renaming.get(c) in self.feature_names_)
-            )
-            and c
+            if c
             not in (
                 excluding_search_keys
                 + list(self.fit_dropped_features)
@@ -1669,10 +1653,7 @@ class FeaturesEnricher(TransformerMixin):
         generated_features = []
         if date_column is not None:
             converter = DateTimeSearchKeyConverter(date_column, self.date_format, self.logger, self.bundle)
-            # Leave original date column values
-            df_with_date_features = converter.convert(df, keep_time=True)
-            df_with_date_features[date_column] = df[date_column]
-            df = df_with_date_features
+            df = converter.convert(df, keep_time=True)
             generated_features = converter.generated_features
 
         email_columns = SearchKey.find_all_keys(search_keys, SearchKey.EMAIL)
@@ -1681,10 +1662,9 @@ class FeaturesEnricher(TransformerMixin):
             df = generator.generate(df)
             generated_features.extend(generator.generated_features)
 
-        # normalizer = Normalizer(self.bundle, self.logger)
-        # df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
-        # columns_renaming = normalizer.columns_renaming
-        columns_renaming = {c: c for c in df.columns}
+        normalizer = Normalizer(self.bundle, self.logger)
+        df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
+        columns_renaming = normalizer.columns_renaming
 
         df, _ = clean_full_duplicates(df, logger=self.logger, bundle=self.bundle)
 
@@ -2000,19 +1980,9 @@ class FeaturesEnricher(TransformerMixin):
         file_metadata = self._search_task.get_file_metadata(str(uuid.uuid4()))
         search_keys = file_metadata.search_types()
         if SearchKey.IPV6_ADDRESS in search_keys:
-            # search_keys.remove(SearchKey.IPV6_ADDRESS)
-            search_keys.pop(SearchKey.IPV6_ADDRESS, None)
+            search_keys.remove(SearchKey.IPV6_ADDRESS)
 
-        keys = (
-            "{"
-            + ", ".join(
-                [
-                    f'"{key.name}": {{"name": "{name}", "value": "{key_example(key)}"}}'
-                    for key, name in search_keys.items()
-                ]
-            )
-            + "}"
-        )
+        keys = "{" + ", ".join([f'"{key.name}": "{key_example(key)}"' for key in search_keys]) + "}"
         features_for_transform = self._search_task.get_features_for_transform()
         if features_for_transform:
             original_features_for_transform = [
@@ -2093,9 +2063,7 @@ class FeaturesEnricher(TransformerMixin):
 
             is_demo_dataset = hash_input(validated_X) in DEMO_DATASET_HASHES
 
-            columns_to_drop = [
-                c for c in validated_X.columns if c in self.feature_names_ and c in self.dropped_client_feature_names_
-            ]
+            columns_to_drop = [c for c in validated_X.columns if c in self.feature_names_]
             if len(columns_to_drop) > 0:
                 msg = self.bundle.get("x_contains_enriching_columns").format(columns_to_drop)
                 self.logger.warning(msg)
@@ -2124,7 +2092,7 @@ class FeaturesEnricher(TransformerMixin):
             date_column = SearchKey.find_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
             if date_column is not None:
                 converter = DateTimeSearchKeyConverter(date_column, self.date_format, self.logger, bundle=self.bundle)
-                df = converter.convert(df, keep_time=True)
+                df = converter.convert(df)
                 self.logger.info(f"Date column after convertion: {df[date_column]}")
                 generated_features.extend(converter.generated_features)
             else:
@@ -2219,12 +2187,11 @@ class FeaturesEnricher(TransformerMixin):
 
             if add_fit_system_record_id:
                 df = self.__add_fit_system_record_id(df, search_keys, SYSTEM_RECORD_ID)
+                if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
+                    df = df.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL)
                 df = df.rename(columns={SYSTEM_RECORD_ID: SORT_ID})
                 features_not_to_pass.append(SORT_ID)
 
-            if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
-                df = df.drop(columns=DateTimeSearchKeyConverter.DATETIME_COL)
-
             # search keys might be changed after explode
             columns_for_system_record_id = sorted(list(search_keys.keys()) + features_for_transform)
             df[SYSTEM_RECORD_ID] = pd.util.hash_pandas_object(df[columns_for_system_record_id], index=False).astype(
@@ -2243,7 +2210,7 @@ class FeaturesEnricher(TransformerMixin):
 
             combined_search_keys = combine_search_keys(search_keys.keys())
 
-            df_without_features = df.drop(columns=features_not_to_pass, errors="ignore")
+            df_without_features = df.drop(columns=features_not_to_pass)
 
             df_without_features, full_duplicates_warning = clean_full_duplicates(
                 df_without_features, self.logger, bundle=self.bundle
@@ -2352,15 +2319,11 @@ class FeaturesEnricher(TransformerMixin):
             else:
                 result = enrich()
 
-            selecting_columns = [
-                c
-                for c in itertools.chain(validated_X.columns.tolist(), generated_features)
-                if c not in self.dropped_client_feature_names_
-            ]
             filtered_columns = self.__filtered_enriched_features(importance_threshold, max_features)
-            selecting_columns.extend(
+            existing_filtered_columns = [
                 c for c in filtered_columns if c in result.columns and c not in validated_X.columns
-            )
+            ]
+            selecting_columns = validated_X.columns.tolist() + generated_features + existing_filtered_columns
             if add_fit_system_record_id:
                 selecting_columns.append(SORT_ID)
 
@@ -3527,7 +3490,15 @@ class FeaturesEnricher(TransformerMixin):
 
         return result_train, result_eval_sets
 
+    @staticmethod
+    def _round_shap_value(shap: float) -> float:
+        if shap > 0.0 and shap < 0.0001:
+            return 0.0001
+        else:
+            return round(shap, 4)
+
     def __prepare_feature_importances(self, trace_id: str, x_columns: List[str], silent=False):
+        llm_source = "LLM with external data augmentation"
         if self._search_task is None:
             raise NotFittedError(self.bundle.get("transform_unfitted_enricher"))
         features_meta = self._search_task.get_all_features_metadata_v2()
@@ -3538,40 +3509,116 @@ class FeaturesEnricher(TransformerMixin):
         features_df = self._search_task.get_all_initial_raw_features(trace_id, metrics_calculation=True)
 
         self.feature_names_ = []
-        self.dropped_client_feature_names_ = []
         self.feature_importances_ = []
         features_info = []
         features_info_without_links = []
         internal_features_info = []
 
+        def list_or_single(lst: List[str], single: str):
+            return lst or ([single] if single else [])
+
+        def to_anchor(link: str, value: str) -> str:
+            if not value:
+                return ""
+            elif not link:
+                return value
+            elif value == llm_source:
+                return value
+            else:
+                return f"<a href='{link}' target='_blank' rel='noopener noreferrer'>{value}</a>"
+
+        def make_links(names: List[str], links: List[str]):
+            all_links = [to_anchor(link, name) for name, link in itertools.zip_longest(names, links)]
+            return ",".join(all_links)
+
         features_meta.sort(key=lambda m: (-m.shap_value, m.name))
         for feature_meta in features_meta:
             if feature_meta.name in original_names_dict.keys():
                 feature_meta.name = original_names_dict[feature_meta.name]
-
-            is_client_feature = feature_meta.name in x_columns
-
-            if feature_meta.shap_value == 0.0:
-                if self.select_features:
-                    self.dropped_client_feature_names_.append(feature_meta.name)
-                continue
-
-            # Use only important features
+            # Use only enriched features
             if (
-                feature_meta.name in self.fit_generated_features
+                feature_meta.name in x_columns
                 or feature_meta.name == COUNTRY
-                # In select_features mode we select also from etalon features and need to show them
-                or (not self.select_features and is_client_feature)
+                or feature_meta.shap_value == 0.0
+                or feature_meta.name in self.fit_generated_features
             ):
                 continue
 
+            feature_sample = []
             self.feature_names_.append(feature_meta.name)
-            self.feature_importances_.append(_round_shap_value(feature_meta.shap_value))
+            self.feature_importances_.append(self._round_shap_value(feature_meta.shap_value))
+            if feature_meta.name in features_df.columns:
+                feature_sample = np.random.choice(features_df[feature_meta.name].dropna().unique(), 3).tolist()
+                if len(feature_sample) > 0 and isinstance(feature_sample[0], float):
+                    feature_sample = [round(f, 4) for f in feature_sample]
+                feature_sample = [str(f) for f in feature_sample]
+                feature_sample = ", ".join(feature_sample)
+                if len(feature_sample) > 30:
+                    feature_sample = feature_sample[:30] + "..."
+
+            internal_provider = feature_meta.data_provider or "Upgini"
+            providers = list_or_single(feature_meta.data_providers, feature_meta.data_provider)
+            provider_links = list_or_single(feature_meta.data_provider_links, feature_meta.data_provider_link)
+            if providers:
+                provider = make_links(providers, provider_links)
+            else:
+                provider = to_anchor("https://upgini.com", "Upgini")
 
-            feature_info = FeatureInfo.from_metadata(feature_meta, features_df, is_client_feature)
-            features_info.append(feature_info.to_row(self.bundle))
-            features_info_without_links.append(feature_info.to_row_without_links(self.bundle))
-            internal_features_info.append(feature_info.to_internal_row(self.bundle))
+            internal_source = feature_meta.data_source or (
+                llm_source
+                if not feature_meta.name.endswith("_country") and not feature_meta.name.endswith("_postal_code")
+                else ""
+            )
+            sources = list_or_single(feature_meta.data_sources, feature_meta.data_source)
+            source_links = list_or_single(feature_meta.data_source_links, feature_meta.data_source_link)
+            if sources:
+                source = make_links(sources, source_links)
+            else:
+                source = internal_source
+
+            internal_feature_name = feature_meta.name
+            if feature_meta.doc_link:
+                feature_name = to_anchor(feature_meta.doc_link, feature_meta.name)
+            else:
+                feature_name = internal_feature_name
+
+            features_info.append(
+                {
+                    self.bundle.get("features_info_name"): feature_name,
+                    self.bundle.get("features_info_shap"): self._round_shap_value(feature_meta.shap_value),
+                    self.bundle.get("features_info_hitrate"): feature_meta.hit_rate,
+                    self.bundle.get("features_info_value_preview"): feature_sample,
+                    self.bundle.get("features_info_provider"): provider,
+                    self.bundle.get("features_info_source"): source,
+                    self.bundle.get("features_info_update_frequency"): feature_meta.update_frequency,
+                }
+            )
+            features_info_without_links.append(
+                {
+                    self.bundle.get("features_info_name"): internal_feature_name,
+                    self.bundle.get("features_info_shap"): self._round_shap_value(feature_meta.shap_value),
+                    self.bundle.get("features_info_hitrate"): feature_meta.hit_rate,
+                    self.bundle.get("features_info_value_preview"): feature_sample,
+                    self.bundle.get("features_info_provider"): internal_provider,
+                    self.bundle.get("features_info_source"): internal_source,
+                    self.bundle.get("features_info_update_frequency"): feature_meta.update_frequency,
+                }
+            )
+            internal_features_info.append(
+                {
+                    self.bundle.get("features_info_name"): internal_feature_name,
+                    "feature_link": feature_meta.doc_link,
+                    self.bundle.get("features_info_shap"): self._round_shap_value(feature_meta.shap_value),
+                    self.bundle.get("features_info_hitrate"): feature_meta.hit_rate,
+                    self.bundle.get("features_info_value_preview"): feature_sample,
+                    self.bundle.get("features_info_provider"): internal_provider,
+                    "provider_link": feature_meta.data_provider_link,
+                    self.bundle.get("features_info_source"): internal_source,
+                    "source_link": feature_meta.data_source_link,
+                    self.bundle.get("features_info_commercial_schema"): feature_meta.commercial_schema or "",
+                    self.bundle.get("features_info_update_frequency"): feature_meta.update_frequency,
+                }
+            )
 
         if len(features_info) > 0:
             self.features_info = pd.DataFrame(features_info)
@@ -3596,22 +3643,7 @@ class FeaturesEnricher(TransformerMixin):
             autofe_meta = self._search_task.get_autofe_metadata()
             if autofe_meta is None:
                 return None
-            if len(self._internal_features_info) != 0:
-
-                def to_feature_meta(row):
-                    fm = FeaturesMetadataV2(
-                        name=row[bundle.get("features_info_name")],
-                        type="",
-                        source="",
-                        hit_rate=bundle.get("features_info_hitrate"),
-                        shap_value=bundle.get("features_info_shap"),
-                        data_source=bundle.get("features_info_source"),
-                    )
-                    return fm
-
-                features_meta = self._internal_features_info.apply(to_feature_meta).to_list()
-            else:
-                features_meta = self._search_task.get_all_features_metadata_v2()
+            features_meta = self._search_task.get_all_features_metadata_v2()
 
             def get_feature_by_name(name: str):
                 for m in features_meta:

{upgini-1.2.29 → upgini-1.2.29a1}/src/upgini/http.py

@@ -882,7 +882,7 @@ class _RestClient:
         if content_type:
             headers[_RestClient.CONTENT_TYPE_HEADER_NAME] = content_type
         if trace_id:
-            headers[_RestClient.TRACE_ID_HEADER_NAME] = str(trace_id)
+            headers[_RestClient.TRACE_ID_HEADER_NAME] = trace_id
         for header_key, header_value in additional_headers.items():
             headers[header_key] = header_value
         return headers

{upgini-1.2.29 → upgini-1.2.29a1}/src/upgini/metadata.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from enum import Enum
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Set, Union
 
 from pydantic import BaseModel
 
@@ -228,13 +228,13 @@ class FileMetadata(BaseModel):
                 return c
         return None
 
-    def search_types(self) -> Dict[SearchKey, str]:
-        search_keys = dict()
+    def search_types(self) -> Set[SearchKey]:
+        search_keys = set()
         for keys_group in self.searchKeys:
             for key in keys_group:
                 column = self.column_by_name(key)
                 if column:
-                    search_keys[SearchKey.from_meaning_type(column.meaningType)] = column.name
+                    search_keys.add(SearchKey.from_meaning_type(column.meaningType))
         return search_keys
 
 

{upgini-1.2.29 → upgini-1.2.29a1}/src/upgini/metrics.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-from dataclasses import dataclass
 import inspect
 import logging
 import re
@@ -211,21 +210,6 @@ SUPPORTED_CATBOOST_METRICS = {
 }
 
 
-@dataclass
-class _CrossValResults:
-    metric: Optional[float]
-    metric_std: Optional[float]
-    shap_values: Optional[Dict[str, float]]
-
-    def get_display_metric(self) -> Optional[str]:
-        if self.metric is None:
-            return None
-        elif self.metric_std is None:
-            return f"{self.metric:.3f}"
-        else:
-            return f"{self.metric:.3f} ± {self.metric_std:.3f}"
-
-
 class EstimatorWrapper:
     def __init__(
         self,
@@ -313,11 +297,11 @@ class EstimatorWrapper:
 
     def cross_val_predict(
         self, x: pd.DataFrame, y: np.ndarray, baseline_score_column: Optional[Any] = None
-    ) -> _CrossValResults:
+    ) -> Tuple[Optional[float], Optional[Dict[str, float]]]:
         x, y, groups, fit_params = self._prepare_to_fit(x, y)
 
         if x.shape[1] == 0:
-            return _CrossValResults(metric=None, metric_std=None, shap_values=None)
+            return None
 
         scorer = check_scoring(self.estimator, scoring=self.scorer)
 
@@ -342,7 +326,7 @@ class EstimatorWrapper:
 
             self.check_fold_metrics(metrics_by_fold)
 
-            metric, metric_std = self._calculate_metric_from_folds(metrics_by_fold)
+            metric = np.mean(metrics_by_fold) * self.multiplier
 
             splits = self.cv.split(x, y, groups)
 
@@ -367,7 +351,7 @@ class EstimatorWrapper:
         else:
             average_shap_values = None
 
-        return _CrossValResults(metric=metric, metric_std=metric_std, shap_values=average_shap_values)
+        return self.post_process_metric(metric), average_shap_values
 
     def process_shap_values(self, shap_values: Dict[str, float]) -> Dict[str, float]:
         return shap_values
@@ -383,25 +367,17 @@ class EstimatorWrapper:
             metric = 2 * metric - 1
         return metric
 
-    def calculate_metric(
-        self, x: pd.DataFrame, y: np.ndarray, baseline_score_column: Optional[Any] = None
-    ) -> _CrossValResults:
+    def calculate_metric(self, x: pd.DataFrame, y: np.ndarray, baseline_score_column: Optional[Any] = None) -> float:
         x, y, _ = self._prepare_to_calculate(x, y)
         if baseline_score_column is not None and self.metric_name == "GINI":
-            metric, metric_std = roc_auc_score(y, x[baseline_score_column]), None
+            metric = roc_auc_score(y, x[baseline_score_column])
         else:
             metrics = []
             for est in self.cv_estimators:
                 metrics.append(self.scorer(est, x, y))
 
-            metric, metric_std = self._calculate_metric_from_folds(metrics)
-        return _CrossValResults(metric=metric, metric_std=metric_std, shap_values=None)
-
-    def _calculate_metric_from_folds(self, metrics_by_fold: List[float]) -> Tuple[float, float]:
-        metrics_by_fold = [self.post_process_metric(m) for m in metrics_by_fold]
-        metric = np.mean(metrics_by_fold) * self.multiplier
-        metric_std = np.std(metrics_by_fold) * np.abs(self.multiplier)
-        return metric, metric_std
+            metric = np.mean(metrics) * self.multiplier
+        return self.post_process_metric(metric)
 
     @staticmethod
     def create(
@@ -615,7 +591,7 @@ class CatBoostWrapper(EstimatorWrapper):
 
     def cross_val_predict(
         self, x: pd.DataFrame, y: np.ndarray, baseline_score_column: Optional[Any] = None
-    ) -> _CrossValResults:
+    ) -> Tuple[Optional[float], Optional[Dict[str, float]]]:
         try:
             return super().cross_val_predict(x, y, baseline_score_column)
         except Exception as e:

{upgini-1.2.29 → upgini-1.2.29a1}/src/upgini/utils/datetime_utils.py

@@ -109,63 +109,54 @@ class DateTimeSearchKeyConverter:
 
         df = self.clean_old_dates(df)
 
-        # Define function to apply sine and cosine transformations
-        def add_cyclical_features(df, column, period):
-            period_suffix = f"_{period}" if column != "day_in_quarter" else ""
-            sin_feature = f"datetime_{column}_sin{period_suffix}"
-            cos_feature = f"datetime_{column}_cos{period_suffix}"
-            if sin_feature not in df.columns:
-                df[sin_feature] = np.sin(2 * np.pi * df[column] / period)
-                self.generated_features.append(sin_feature)
-            if cos_feature not in df.columns:
-                df[cos_feature] = np.cos(2 * np.pi * df[column] / period)
-                self.generated_features.append(cos_feature)
-
-        df["quarter"] = df[self.date_column].dt.quarter
-
-        # Calculate the start date of the quarter for each timestamp
-        df["quarter_start"] = df[self.date_column].dt.to_period("Q").dt.start_time
-
-        # Calculate the day in the quarter
-        df["day_in_quarter"] = (df[self.date_column] - df["quarter_start"]).dt.days + 1
-
-        # Vectorized calculation of days_in_quarter
-        quarter = df["quarter"]
-        start = df["quarter_start"]
-        year = start.dt.year
-        month = start.dt.month
-
-        quarter_end_year = np.where(quarter == 4, year + 1, year)
-        quarter_end_month = np.where(quarter == 4, 1, month + 3)
-
-        end = pd.to_datetime({"year": quarter_end_year, "month": quarter_end_month, "day": 1})
-        end.index = df.index
-
-        df["days_in_quarter"] = (end - start).dt.days
-
-        add_cyclical_features(df, "day_in_quarter", df["days_in_quarter"])  # Days in the quarter
-
-        df.drop(columns=["quarter", "quarter_start", "day_in_quarter", "days_in_quarter"], inplace=True)
-
         df[seconds] = (df[self.date_column] - df[self.date_column].dt.floor("D")).dt.seconds
 
         seconds_without_na = df[seconds].dropna()
         if (seconds_without_na != 0).any() and seconds_without_na.nunique() > 1:
             self.logger.info("Time found in date search key. Add extra features based on time")
 
-            # Extract basic components
+            # Extract time components
             df["second"] = df[self.date_column].dt.second
             df["minute"] = df[self.date_column].dt.minute
             df["hour"] = df[self.date_column].dt.hour
+            df["day"] = df[self.date_column].dt.day
+            df["month"] = df[self.date_column].dt.month
+
+            # Get the actual number of days in each month
+            df["days_in_month"] = df[self.date_column].dt.days_in_month
+
+            # Define function to apply sine and cosine transformations
+            def add_cyclical_features(df, column, period):
+                sin_feature = f"datetime_{column}_sin_{period}"
+                cos_feature = f"datetime_{column}_cos_{period}"
+                df[sin_feature] = np.sin(2 * np.pi * df[column] / period)
+                df[cos_feature] = np.cos(2 * np.pi * df[column] / period)
+                self.generated_features.append(sin_feature)
+                self.generated_features.append(cos_feature)
 
-            # Apply cyclical transformations
+            # Apply transformations using vectorized operations
             add_cyclical_features(df, "second", 60)  # Seconds in a minute
             add_cyclical_features(df, "minute", 60)  # Minutes in an hour
-            add_cyclical_features(df, "minute", 30)  # Minutes in half an hour
             add_cyclical_features(df, "hour", 24)  # Hours in a day
-
-            # Drop intermediate columns if not needed
-            df.drop(columns=["second", "minute", "hour"], inplace=True)
+            add_cyclical_features(df, "day", df["days_in_month"])  # Days in the specific month
+            add_cyclical_features(df, "month", 12)  # Months in a year
+
+            # Extract quarter information
+            df["quarter"] = df[self.date_column].dt.quarter
+
+            # Apply transformations for quarters
+            add_cyclical_features(df, "quarter", 4)  # Quarters in a year
+            # seconds_in_day = 60 * 60 * 24
+            # orders = [1, 2, 24, 48]
+            # for order in orders:
+            #     sin_feature = f"datetime_time_sin_{order}"
+            #     cos_feature = f"datetime_time_cos_{order}"
+            #     df[sin_feature] = np.round(np.sin(2 * np.pi * order * df[seconds] / seconds_in_day), 10)
+            #     df[cos_feature] = np.round(np.cos(2 * np.pi * order * df[seconds] / seconds_in_day), 10)
+            #     self.generated_features.append(sin_feature)
+            #     self.generated_features.append(cos_feature)
+
+            df.drop(columns=["second", "minute", "hour", "day", "month", "days_in_month", "quarter"])
 
         df.drop(columns=seconds, inplace=True)
 

{upgini-1.2.29 → upgini-1.2.29a1}/src/upgini/utils/email_utils.py

@@ -38,9 +38,8 @@ class EmailDomainGenerator:
     def generate(self, df: pd.DataFrame) -> pd.DataFrame:
         for email_col in self.email_columns:
             domain_feature = email_col + self.DOMAIN_SUFFIX
-            if domain_feature not in df.columns:
-                df[domain_feature] = df[email_col].apply(self._email_to_domain).astype("string")
-                self.generated_features.append(domain_feature)
+            df[domain_feature] = df[email_col].apply(self._email_to_domain)
+            self.generated_features.append(domain_feature)
         return df
 
     @staticmethod

{upgini-1.2.29 → upgini-1.2.29a1}/src/upgini/utils/features_validator.py

@@ -2,7 +2,6 @@ import logging
 from logging import Logger
 from typing import Dict, List, Optional, Tuple
 
-import numpy as np
 import pandas as pd
 from pandas.api.types import is_integer_dtype, is_object_dtype, is_string_dtype
 
@@ -84,21 +83,10 @@ class FeaturesValidator:
         return [
             i
             for i in df
-            if (is_object_dtype(df[i]) or is_string_dtype(df[i]) or FeaturesValidator.__is_integer(df[i]))
+            if (is_object_dtype(df[i]) or is_string_dtype(df[i]) or is_integer_dtype(df[i]))
             and (df[i].nunique(dropna=False) / row_count >= 0.85)
         ]
 
-    @staticmethod
-    def __is_integer(series: pd.Series) -> bool:
-        return (
-            is_integer_dtype(series)
-            or series.dropna()
-            .apply(
-                lambda f: (float.is_integer(f) and abs(f) < np.iinfo(np.int64).max) if isinstance(f, float) else False
-            )
-            .all()
-        )
-
     @staticmethod
     def find_constant_features(df: pd.DataFrame) -> List[str]:
         return [i for i in df if df[i].nunique() <= 1]

upgini-1.2.29/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.29"

upgini-1.2.29/src/upgini/utils/feature_info.py

@@ -1,172 +0,0 @@
-from dataclasses import dataclass
-import itertools
-from typing import Dict, List
-
-import numpy as np
-import pandas as pd
-
-from upgini.metadata import FeaturesMetadataV2
-from upgini.resource_bundle import ResourceBundle
-
-
-LLM_SOURCE = "LLM with external data augmentation"
-
-
-@dataclass
-class FeatureInfo:
-    name: str
-    internal_name: str
-    rounded_shap: float
-    hitrate: float
-    value_preview: str
-    provider: str
-    internal_provider: str
-    source: str
-    internal_source: str
-    update_frequency: str
-    commercial_schema: str
-    doc_link: str
-    data_provider_link: str
-    data_source_link: str
-
-    @staticmethod
-    def from_metadata(feature_meta: FeaturesMetadataV2, data: pd.DataFrame, is_client_feature: bool) -> "FeatureInfo":
-        return FeatureInfo(
-            name=_get_name(feature_meta),
-            internal_name=_get_internal_name(feature_meta),
-            rounded_shap=_round_shap_value(feature_meta.shap_value),
-            hitrate=feature_meta.hit_rate,
-            value_preview=_get_feature_sample(feature_meta, data),
-            provider=_get_provider(feature_meta, is_client_feature),
-            internal_provider=_get_internal_provider(feature_meta, is_client_feature),
-            source=_get_source(feature_meta, is_client_feature),
-            internal_source=_get_internal_source(feature_meta, is_client_feature),
-            update_frequency=feature_meta.update_frequency,
-            commercial_schema=feature_meta.commercial_schema,
-            doc_link=feature_meta.doc_link,
-            data_provider_link=feature_meta.data_provider_link,
-            data_source_link=feature_meta.data_source_link,
-        )
-
-    def to_row(self, bundle: ResourceBundle) -> Dict[str, str]:
-        return {
-            bundle.get("features_info_name"): self.name,
-            bundle.get("features_info_shap"): self.rounded_shap,
-            bundle.get("features_info_hitrate"): self.hitrate,
-            bundle.get("features_info_value_preview"): self.value_preview,
-            bundle.get("features_info_provider"): self.provider,
-            bundle.get("features_info_source"): self.source,
-            bundle.get("features_info_update_frequency"): self.update_frequency,
-        }
-
-    def to_row_without_links(self, bundle: ResourceBundle) -> Dict[str, str]:
-        return {
-            bundle.get("features_info_name"): self.internal_name,
-            bundle.get("features_info_shap"): self.rounded_shap,
-            bundle.get("features_info_hitrate"): self.hitrate,
-            bundle.get("features_info_value_preview"): self.value_preview,
-            bundle.get("features_info_provider"): self.internal_provider,
-            bundle.get("features_info_source"): self.internal_source,
-            bundle.get("features_info_update_frequency"): self.update_frequency,
-        }
-
-    def to_internal_row(self, bundle: ResourceBundle) -> Dict[str, str]:
-        return {
-            bundle.get("features_info_name"): self.internal_name,
-            "feature_link": self.doc_link,
-            bundle.get("features_info_shap"): self.rounded_shap,
-            bundle.get("features_info_hitrate"): self.hitrate,
-            bundle.get("features_info_value_preview"): self.value_preview,
-            bundle.get("features_info_provider"): self.internal_provider,
-            "provider_link": self.data_provider_link,
-            bundle.get("features_info_source"): self.internal_source,
-            "source_link": self.data_source_link,
-            bundle.get("features_info_commercial_schema"): self.commercial_schema or "",
-            bundle.get("features_info_update_frequency"): self.update_frequency,
-        }
-
-
-def _get_feature_sample(feature_meta: FeaturesMetadataV2, data: pd.DataFrame) -> str:
-    if feature_meta.name in data.columns:
-        feature_sample = np.random.choice(data[feature_meta.name].dropna().unique(), 3).tolist()
-        if len(feature_sample) > 0 and isinstance(feature_sample[0], float):
-            feature_sample = [round(f, 4) for f in feature_sample]
-        feature_sample = [str(f) for f in feature_sample]
-        feature_sample = ", ".join(feature_sample)
-        if len(feature_sample) > 30:
-            feature_sample = feature_sample[:30] + "..."
-    else:
-        feature_sample = ""
-    return feature_sample
-
-
-def _get_name(feature_meta: FeaturesMetadataV2) -> str:
-    if feature_meta.doc_link:
-        return _to_anchor(feature_meta.doc_link, feature_meta.name)
-    else:
-        return feature_meta.name
-
-
-def _get_internal_name(feature_meta: FeaturesMetadataV2) -> str:
-    return feature_meta.name
-
-
-def _get_provider(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
-    providers = _list_or_single(feature_meta.data_providers, feature_meta.data_provider)
-    provider_links = _list_or_single(feature_meta.data_provider_links, feature_meta.data_provider_link)
-    if providers:
-        provider = _make_links(providers, provider_links)
-    else:
-        provider = "" if is_client_feature else _to_anchor("https://upgini.com", "Upgini")
-    return provider
-
-
-def _get_internal_provider(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
-    return "" if is_client_feature else (feature_meta.data_provider or "Upgini")
-
-
-def _get_source(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
-    sources = _list_or_single(feature_meta.data_sources, feature_meta.data_source)
-    source_links = _list_or_single(feature_meta.data_source_links, feature_meta.data_source_link)
-    if sources:
-        source = _make_links(sources, source_links)
-    else:
-        source = _get_internal_source(feature_meta, is_client_feature)
-    return source
-
-
-def _get_internal_source(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
-    return feature_meta.data_source or (
-        LLM_SOURCE
-        if not feature_meta.name.endswith("_country")
-        and not feature_meta.name.endswith("_postal_code")
-        and not is_client_feature
-        else ""
-    )
-
-
-def _list_or_single(lst: List[str], single: str):
-    return lst or ([single] if single else [])
-
-
-def _to_anchor(link: str, value: str) -> str:
-    if not value:
-        return ""
-    elif not link:
-        return value
-    elif value == LLM_SOURCE:
-        return value
-    else:
-        return f"<a href='{link}' target='_blank' rel='noopener noreferrer'>{value}</a>"
-
-
-def _make_links(names: List[str], links: List[str]):
-    all_links = [_to_anchor(link, name) for name, link in itertools.zip_longest(names, links)]
-    return ",".join(all_links)
-
-
-def _round_shap_value(shap: float) -> float:
-    if shap > 0.0 and shap < 0.0001:
-        return 0.0001
-    else:
-        return round(shap, 4)