upgini 1.2.29a4__tar.gz → 1.2.29a6__tar.gz

{upgini-1.2.29a4 → upgini-1.2.29a6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.29a4
+Version: 1.2.29a6
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.29a6/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.29a6"

{upgini-1.2.29a4 → upgini-1.2.29a6}/src/upgini/features_enricher.py

@@ -54,6 +54,7 @@ from upgini.metadata import (
     SYSTEM_RECORD_ID,
     TARGET,
     CVType,
+    FeaturesMetadataV2,
     FileColumnMeaningType,
     ModelTaskType,
     RuntimeParameters,
@@ -95,6 +96,7 @@ from upgini.utils.email_utils import (
     EmailSearchKeyConverter,
     EmailSearchKeyDetector,
 )
+from upgini.utils.feature_info import FeatureInfo, _round_shap_value
 from upgini.utils.features_validator import FeaturesValidator
 from upgini.utils.format import Format
 from upgini.utils.ip_utils import IpSearchKeyConverter
@@ -158,6 +160,10 @@ class FeaturesEnricher(TransformerMixin):
 
     shared_datasets: list of str, optional (default=None)
         List of private shared dataset ids for custom search
+
+    select_features: bool, optional (default=False)
+        If True, return only selected features both from input and data sources.
+        Otherwise, return all features from input and only selected features from data sources.
     """
 
     TARGET_NAME = "target"
@@ -224,6 +230,7 @@ class FeaturesEnricher(TransformerMixin):
         client_visitorid: Optional[str] = None,
         custom_bundle_config: Optional[str] = None,
         add_date_if_missing: bool = True,
+        select_features: bool = False,
         **kwargs,
     ):
         self.bundle = get_custom_bundle(custom_bundle_config)
@@ -277,8 +284,11 @@ class FeaturesEnricher(TransformerMixin):
         self._relevant_data_sources_wo_links: pd.DataFrame = self.EMPTY_DATA_SOURCES
         self.metrics: Optional[pd.DataFrame] = None
         self.feature_names_ = []
+        self.dropped_client_feature_names_ = []
         self.feature_importances_ = []
         self.search_id = search_id
+        self.select_features = select_features
+
         if search_id:
             search_task = SearchTask(search_id, rest_client=self.rest_client, logger=self.logger)
 
@@ -1201,9 +1211,7 @@ class FeaturesEnricher(TransformerMixin):
 
     def _update_shap_values(self, new_shaps: Dict[str, float]):
         new_shaps = {
-            feature: self._round_shap_value(shap)
-            for feature, shap in new_shaps.items()
-            if feature in self.feature_names_
+            feature: _round_shap_value(shap) for feature, shap in new_shaps.items() if feature in self.feature_names_
         }
         features_importances = list(new_shaps.items())
         features_importances.sort(key=lambda m: (-m[1], m[0]))
@@ -1440,7 +1448,8 @@ class FeaturesEnricher(TransformerMixin):
         client_features = [
             c
             for c in X_sampled.columns.to_list()
-            if c
+            if (not self.select_features or c in self.feature_names_)
+            and c
             not in (
                 excluding_search_keys
                 + list(self.fit_dropped_features)
@@ -2066,7 +2075,9 @@ class FeaturesEnricher(TransformerMixin):
 
             is_demo_dataset = hash_input(validated_X) in DEMO_DATASET_HASHES
 
-            columns_to_drop = [c for c in validated_X.columns if c in self.feature_names_]
+            columns_to_drop = [
+                c for c in validated_X.columns if c in self.feature_names_ and c in self.dropped_client_feature_names_
+            ]
             if len(columns_to_drop) > 0:
                 msg = self.bundle.get("x_contains_enriching_columns").format(columns_to_drop)
                 self.logger.warning(msg)
@@ -2322,11 +2333,13 @@ class FeaturesEnricher(TransformerMixin):
             else:
                 result = enrich()
 
-            filtered_columns = self.__filtered_enriched_features(importance_threshold, max_features)
-            existing_filtered_columns = [
-                c for c in filtered_columns if c in result.columns and c not in validated_X.columns
+            selecting_columns = [
+                c
+                for c in itertools.chain(validated_X.columns.tolist(), generated_features)
+                if c not in self.dropped_client_feature_names_
             ]
-            selecting_columns = validated_X.columns.tolist() + generated_features + existing_filtered_columns
+            filtered_columns = self.__filtered_enriched_features(importance_threshold, max_features)
+            selecting_columns.extend(c for c in filtered_columns if c in result.columns and c not in validated_X.columns)
             if add_fit_system_record_id:
                 selecting_columns.append(SORT_ID)
 
@@ -3493,15 +3506,7 @@ class FeaturesEnricher(TransformerMixin):
 
         return result_train, result_eval_sets
 
-    @staticmethod
-    def _round_shap_value(shap: float) -> float:
-        if shap > 0.0 and shap < 0.0001:
-            return 0.0001
-        else:
-            return round(shap, 4)
-
     def __prepare_feature_importances(self, trace_id: str, x_columns: List[str], silent=False):
-        llm_source = "LLM with external data augmentation"
         if self._search_task is None:
             raise NotFittedError(self.bundle.get("transform_unfitted_enricher"))
         features_meta = self._search_task.get_all_features_metadata_v2()
@@ -3512,116 +3517,41 @@ class FeaturesEnricher(TransformerMixin):
         features_df = self._search_task.get_all_initial_raw_features(trace_id, metrics_calculation=True)
 
         self.feature_names_ = []
+        self.dropped_client_feature_names_ = []
         self.feature_importances_ = []
         features_info = []
         features_info_without_links = []
         internal_features_info = []
 
-        def list_or_single(lst: List[str], single: str):
-            return lst or ([single] if single else [])
-
-        def to_anchor(link: str, value: str) -> str:
-            if not value:
-                return ""
-            elif not link:
-                return value
-            elif value == llm_source:
-                return value
-            else:
-                return f"<a href='{link}' target='_blank' rel='noopener noreferrer'>{value}</a>"
-
-        def make_links(names: List[str], links: List[str]):
-            all_links = [to_anchor(link, name) for name, link in itertools.zip_longest(names, links)]
-            return ",".join(all_links)
-
         features_meta.sort(key=lambda m: (-m.shap_value, m.name))
         for feature_meta in features_meta:
             if feature_meta.name in original_names_dict.keys():
                 feature_meta.name = original_names_dict[feature_meta.name]
-            # Use only enriched features
+
+            is_client_feature = feature_meta.name in x_columns
+
+            if feature_meta.shap_value == 0.0:
+                if self.select_features:
+                    self.dropped_client_feature_names_.append(feature_meta.name)
+                continue
+
+            # Use only important features
             if (
-                feature_meta.name in x_columns
+                feature_meta.name in self.fit_generated_features
                 or feature_meta.name == COUNTRY
-                or feature_meta.shap_value == 0.0
-                or feature_meta.name in self.fit_generated_features
+                # In select_features mode we select also from etalon features and need to show them
+                or (not self.select_features and is_client_feature)
             ):
                 continue
 
-            feature_sample = []
-            self.feature_names_.append(feature_meta.name)
-            self.feature_importances_.append(self._round_shap_value(feature_meta.shap_value))
-            if feature_meta.name in features_df.columns:
-                feature_sample = np.random.choice(features_df[feature_meta.name].dropna().unique(), 3).tolist()
-                if len(feature_sample) > 0 and isinstance(feature_sample[0], float):
-                    feature_sample = [round(f, 4) for f in feature_sample]
-                feature_sample = [str(f) for f in feature_sample]
-                feature_sample = ", ".join(feature_sample)
-                if len(feature_sample) > 30:
-                    feature_sample = feature_sample[:30] + "..."
-
-            internal_provider = feature_meta.data_provider or "Upgini"
-            providers = list_or_single(feature_meta.data_providers, feature_meta.data_provider)
-            provider_links = list_or_single(feature_meta.data_provider_links, feature_meta.data_provider_link)
-            if providers:
-                provider = make_links(providers, provider_links)
-            else:
-                provider = to_anchor("https://upgini.com", "Upgini")
 
-            internal_source = feature_meta.data_source or (
-                llm_source
-                if not feature_meta.name.endswith("_country") and not feature_meta.name.endswith("_postal_code")
-                else ""
-            )
-            sources = list_or_single(feature_meta.data_sources, feature_meta.data_source)
-            source_links = list_or_single(feature_meta.data_source_links, feature_meta.data_source_link)
-            if sources:
-                source = make_links(sources, source_links)
-            else:
-                source = internal_source
+            self.feature_names_.append(feature_meta.name)
+            self.feature_importances_.append(_round_shap_value(feature_meta.shap_value))
 
-            internal_feature_name = feature_meta.name
-            if feature_meta.doc_link:
-                feature_name = to_anchor(feature_meta.doc_link, feature_meta.name)
-            else:
-                feature_name = internal_feature_name
-
-            features_info.append(
-                {
-                    self.bundle.get("features_info_name"): feature_name,
-                    self.bundle.get("features_info_shap"): self._round_shap_value(feature_meta.shap_value),
-                    self.bundle.get("features_info_hitrate"): feature_meta.hit_rate,
-                    self.bundle.get("features_info_value_preview"): feature_sample,
-                    self.bundle.get("features_info_provider"): provider,
-                    self.bundle.get("features_info_source"): source,
-                    self.bundle.get("features_info_update_frequency"): feature_meta.update_frequency,
-                }
-            )
-            features_info_without_links.append(
-                {
-                    self.bundle.get("features_info_name"): internal_feature_name,
-                    self.bundle.get("features_info_shap"): self._round_shap_value(feature_meta.shap_value),
-                    self.bundle.get("features_info_hitrate"): feature_meta.hit_rate,
-                    self.bundle.get("features_info_value_preview"): feature_sample,
-                    self.bundle.get("features_info_provider"): internal_provider,
-                    self.bundle.get("features_info_source"): internal_source,
-                    self.bundle.get("features_info_update_frequency"): feature_meta.update_frequency,
-                }
-            )
-            internal_features_info.append(
-                {
-                    self.bundle.get("features_info_name"): internal_feature_name,
-                    "feature_link": feature_meta.doc_link,
-                    self.bundle.get("features_info_shap"): self._round_shap_value(feature_meta.shap_value),
-                    self.bundle.get("features_info_hitrate"): feature_meta.hit_rate,
-                    self.bundle.get("features_info_value_preview"): feature_sample,
-                    self.bundle.get("features_info_provider"): internal_provider,
-                    "provider_link": feature_meta.data_provider_link,
-                    self.bundle.get("features_info_source"): internal_source,
-                    "source_link": feature_meta.data_source_link,
-                    self.bundle.get("features_info_commercial_schema"): feature_meta.commercial_schema or "",
-                    self.bundle.get("features_info_update_frequency"): feature_meta.update_frequency,
-                }
-            )
+            feature_info = FeatureInfo.from_metadata(feature_meta, features_df, is_client_feature)
+            features_info.append(feature_info.to_row(self.bundle))
+            features_info_without_links.append(feature_info.to_row_without_links(self.bundle))
+            internal_features_info.append(feature_info.to_internal_row(self.bundle))
 
         if len(features_info) > 0:
             self.features_info = pd.DataFrame(features_info)

{upgini-1.2.29a4 → upgini-1.2.29a6}/src/upgini/utils/datetime_utils.py

@@ -137,8 +137,9 @@ class DateTimeSearchKeyConverter:
         quarter_end_month = np.where(quarter == 4, 1, month + 3)
 
         end = pd.to_datetime({"year": quarter_end_year, "month": quarter_end_month, "day": 1})
+        end.index = df.index
 
-        df["days_in_quarter"] = (end.reset_index(drop=True) - start.reset_index(drop=True)).dt.days
+        df["days_in_quarter"] = (end - start).dt.days
 
         add_cyclical_features(df, "day_in_quarter", df["days_in_quarter"])  # Days in the quarter
 

upgini-1.2.29a6/src/upgini/utils/feature_info.py

@@ -0,0 +1,172 @@
+from dataclasses import dataclass
+import itertools
+from typing import Dict, List
+
+import numpy as np
+import pandas as pd
+
+from upgini.metadata import FeaturesMetadataV2
+from upgini.resource_bundle import ResourceBundle
+
+
+LLM_SOURCE = "LLM with external data augmentation"
+
+
+@dataclass
+class FeatureInfo:
+    name: str
+    internal_name: str
+    rounded_shap: float
+    hitrate: float
+    value_preview: str
+    provider: str
+    internal_provider: str
+    source: str
+    internal_source: str
+    update_frequency: str
+    commercial_schema: str
+    doc_link: str
+    data_provider_link: str
+    data_source_link: str
+
+    @staticmethod
+    def from_metadata(feature_meta: FeaturesMetadataV2, data: pd.DataFrame, is_client_feature: bool) -> "FeatureInfo":
+        return FeatureInfo(
+            name=_get_name(feature_meta),
+            internal_name=_get_internal_name(feature_meta),
+            rounded_shap=_round_shap_value(feature_meta.shap_value),
+            hitrate=feature_meta.hit_rate,
+            value_preview=_get_feature_sample(feature_meta, data),
+            provider=_get_provider(feature_meta, is_client_feature),
+            internal_provider=_get_internal_provider(feature_meta, is_client_feature),
+            source=_get_source(feature_meta, is_client_feature),
+            internal_source=_get_internal_source(feature_meta, is_client_feature),
+            update_frequency=feature_meta.update_frequency,
+            commercial_schema=feature_meta.commercial_schema,
+            doc_link=feature_meta.doc_link,
+            data_provider_link=feature_meta.data_provider_link,
+            data_source_link=feature_meta.data_source_link,
+        )
+
+    def to_row(self, bundle: ResourceBundle) -> Dict[str, str]:
+        return {
+            bundle.get("features_info_name"): self.name,
+            bundle.get("features_info_shap"): self.rounded_shap,
+            bundle.get("features_info_hitrate"): self.hitrate,
+            bundle.get("features_info_value_preview"): self.value_preview,
+            bundle.get("features_info_provider"): self.provider,
+            bundle.get("features_info_source"): self.source,
+            bundle.get("features_info_update_frequency"): self.update_frequency,
+        }
+
+    def to_row_without_links(self, bundle: ResourceBundle) -> Dict[str, str]:
+        return {
+            bundle.get("features_info_name"): self.internal_name,
+            bundle.get("features_info_shap"): self.rounded_shap,
+            bundle.get("features_info_hitrate"): self.hitrate,
+            bundle.get("features_info_value_preview"): self.value_preview,
+            bundle.get("features_info_provider"): self.internal_provider,
+            bundle.get("features_info_source"): self.internal_source,
+            bundle.get("features_info_update_frequency"): self.update_frequency,
+        }
+
+    def to_internal_row(self, bundle: ResourceBundle) -> Dict[str, str]:
+        return {
+            bundle.get("features_info_name"): self.internal_name,
+            "feature_link": self.doc_link,
+            bundle.get("features_info_shap"): self.rounded_shap,
+            bundle.get("features_info_hitrate"): self.hitrate,
+            bundle.get("features_info_value_preview"): self.value_preview,
+            bundle.get("features_info_provider"): self.internal_provider,
+            "provider_link": self.data_provider_link,
+            bundle.get("features_info_source"): self.internal_source,
+            "source_link": self.data_source_link,
+            bundle.get("features_info_commercial_schema"): self.commercial_schema or "",
+            bundle.get("features_info_update_frequency"): self.update_frequency,
+        }
+
+
+def _get_feature_sample(feature_meta: FeaturesMetadataV2, data: pd.DataFrame) -> str:
+    if feature_meta.name in data.columns:
+        feature_sample = np.random.choice(data[feature_meta.name].dropna().unique(), 3).tolist()
+        if len(feature_sample) > 0 and isinstance(feature_sample[0], float):
+            feature_sample = [round(f, 4) for f in feature_sample]
+        feature_sample = [str(f) for f in feature_sample]
+        feature_sample = ", ".join(feature_sample)
+        if len(feature_sample) > 30:
+            feature_sample = feature_sample[:30] + "..."
+    else:
+        feature_sample = ""
+    return feature_sample
+
+
+def _get_name(feature_meta: FeaturesMetadataV2) -> str:
+    if feature_meta.doc_link:
+        return _to_anchor(feature_meta.doc_link, feature_meta.name)
+    else:
+        return feature_meta.name
+
+
+def _get_internal_name(feature_meta: FeaturesMetadataV2) -> str:
+    return feature_meta.name
+
+
+def _get_provider(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
+    providers = _list_or_single(feature_meta.data_providers, feature_meta.data_provider)
+    provider_links = _list_or_single(feature_meta.data_provider_links, feature_meta.data_provider_link)
+    if providers:
+        provider = _make_links(providers, provider_links)
+    else:
+        provider = "" if is_client_feature else _to_anchor("https://upgini.com", "Upgini")
+    return provider
+
+
+def _get_internal_provider(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
+    return "" if is_client_feature else (feature_meta.data_provider or "Upgini")
+
+
+def _get_source(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
+    sources = _list_or_single(feature_meta.data_sources, feature_meta.data_source)
+    source_links = _list_or_single(feature_meta.data_source_links, feature_meta.data_source_link)
+    if sources:
+        source = _make_links(sources, source_links)
+    else:
+        source = _get_internal_source(feature_meta, is_client_feature)
+    return source
+
+
+def _get_internal_source(feature_meta: FeaturesMetadataV2, is_client_feature: bool) -> str:
+    return feature_meta.data_source or (
+        LLM_SOURCE
+        if not feature_meta.name.endswith("_country")
+        and not feature_meta.name.endswith("_postal_code")
+        and not is_client_feature
+        else ""
+    )
+
+
+def _list_or_single(lst: List[str], single: str):
+    return lst or ([single] if single else [])
+
+
+def _to_anchor(link: str, value: str) -> str:
+    if not value:
+        return ""
+    elif not link:
+        return value
+    elif value == LLM_SOURCE:
+        return value
+    else:
+        return f"<a href='{link}' target='_blank' rel='noopener noreferrer'>{value}</a>"
+
+
+def _make_links(names: List[str], links: List[str]):
+    all_links = [_to_anchor(link, name) for name, link in itertools.zip_longest(names, links)]
+    return ",".join(all_links)
+
+
+def _round_shap_value(shap: float) -> float:
+    if shap > 0.0 and shap < 0.0001:
+        return 0.0001
+    else:
+        return round(shap, 4)

upgini-1.2.29a4/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.29a4"