upgini 1.2.167a4220.dev11__tar.gz → 1.2.167a4220.dev13__tar.gz

{upgini-1.2.167a4220.dev11 → upgini-1.2.167a4220.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: upgini
-Version: 1.2.167a4220.dev11
+Version: 1.2.167a4220.dev13
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.167a4220.dev13/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.167.a4220.dev13"

{upgini-1.2.167a4220.dev11 → upgini-1.2.167a4220.dev13}/src/upgini/report/assemble.py

@@ -22,7 +22,7 @@ from upgini.report.data import (
 )
 from upgini.report.stats import PSI_CRITICAL, PSI_WARNING, compute_report_charts
 from upgini.resource_bundle import ResourceBundle
-from upgini.utils.feature_info import FeatureInfo
+from upgini.utils.feature_info import CLIENT_SOURCE, GENERATED_SOURCE, FeatureInfo
 
 UPGINI_REPORT_BRANDING_URL = "UPGINI_REPORT_BRANDING_URL"
 _STABILITY_PSI = 0.2
@@ -129,7 +129,6 @@ def build_search_results(
     ]
     sources = _source_rows(found_rows)
     summary = _overview_summary(
-        features_meta=features_meta,
         model_rows=model_rows,
         found_rows=found_rows,
         sources=sources,
@@ -304,7 +303,6 @@ def _has_nonzero_shap_value(shap: Optional[float]) -> bool:
 
 def _overview_summary(
     *,
-    features_meta: list[FeaturesMetadataV2],
     model_rows: list[FeatureRow],
     found_rows: list[FeatureRow],
     sources: list[SourceRow],
@@ -314,21 +312,16 @@ def _overview_summary(
     joined_ads_count: Optional[int],
 ) -> SearchResultsSummary:
     top_level = _unique_fold_model_inputs(generated_features)
-    features_by_name = {meta.name: meta for meta in features_meta}
     autofe_by_name = _autofe_generated_by_name(generated_features, is_ensemble)
-    columns_by_name = _base_columns_by_name(generated_features)
     ads_ids = _contributed_ads_ids(generated_features, autofe_by_name)
     if top_level:
         counts = {"external": 0, "original": 0, "autofe": 0}
-        psis: list[Optional[float]] = []
-        for node in top_level:
-            kind = _top_level_kind(node, features_by_name, columns_by_name, autofe_by_name, is_ensemble)
-            counts[kind] += 1
-            meta = _feature_meta_for_node(node, features_by_name, columns_by_name)
-            psis.append(None if meta is None else meta.psi_value)
-        model_features = len(top_level)
+        for row in model_rows:
+            counts[_model_row_kind(row)] += 1
+        model_features = len(model_rows)
         external, original, autofe = counts["external"], counts["original"], counts["autofe"]
         contributed = len(ads_ids or set())
+        psis = [row.psi for row in model_rows]
     else:
         model_features = len(model_rows) or None
         external = original = autofe = None
@@ -383,38 +376,6 @@ def _top_level_key(node: Column | Feature) -> str:
     return f"fe:{node.to_formula()}"
 
 
-def _base_columns_by_name(generated_features: list[GeneratedFeatureMetadata]) -> dict[str, BaseColumnMetadata]:
-    by_name: dict[str, BaseColumnMetadata] = {}
-    for meta in generated_features:
-        for column in meta.base_columns:
-            if column.hashed_name:
-                by_name.setdefault(column.hashed_name, column)
-            if column.original_name:
-                by_name.setdefault(column.original_name, column)
-    return by_name
-
-
-def _column_names(node: Column, columns_by_name: dict[str, BaseColumnMetadata]) -> list[str]:
-    names = [node.name]
-    column = columns_by_name.get(node.name)
-    if column is not None:
-        names.extend([column.hashed_name, column.original_name])
-    return [name for name in names if name]
-
-
-def _feature_meta_for_node(
-    node: Column | Feature,
-    features_by_name: dict[str, FeaturesMetadataV2],
-    columns_by_name: dict[str, BaseColumnMetadata],
-) -> Optional[FeaturesMetadataV2]:
-    if not isinstance(node, Column):
-        return None
-    return next(
-        (features_by_name[name] for name in _column_names(node, columns_by_name) if name in features_by_name),
-        None,
-    )
-
-
 def _autofe_generated_by_name(
     generated_features: list[GeneratedFeatureMetadata], is_ensemble: Callable[[str], bool]
 ) -> dict[str, GeneratedFeatureMetadata]:
@@ -452,30 +413,13 @@ def _generated_feature_names(meta: GeneratedFeatureMetadata) -> set[str]:
     return {feature.get_display_name(shorten=True, unhash=True, cache=False)}
 
 
-def _top_level_kind(
-    node: Column | Feature,
-    features_by_name: dict[str, FeaturesMetadataV2],
-    columns_by_name: dict[str, BaseColumnMetadata],
-    autofe_by_name: dict[str, GeneratedFeatureMetadata],
-    is_ensemble: Callable[[str], bool],
-) -> str:
-    if isinstance(node, Feature):
-        return "autofe"
-    names = _column_names(node, columns_by_name)
-    if any(name in autofe_by_name for name in names):
+def _model_row_kind(row: FeatureRow) -> str:
+    source = (row.source or "").lower()
+    if source.startswith("autofe") or source == GENERATED_SOURCE.lower():
         return "autofe"
-    meta = next((features_by_name[name] for name in names if name in features_by_name), None)
-    if meta is not None and not is_ensemble(meta.name):
-        if meta.source == "generated":
-            return "autofe"
-        if meta.source == "ads":
-            return "external"
-        if meta.source == "etalon":
-            return "original"
-    column = next((columns_by_name[name] for name in names if name in columns_by_name), None)
-    if column is not None and column.ads_definition_id:
-        return "external"
-    return "original"
+    if source == CLIENT_SOURCE.lower():
+        return "original"
+    return "external"
 
 
 def _contributed_ads_ids(

{upgini-1.2.167a4220.dev11 → upgini-1.2.167a4220.dev13}/src/upgini/report/html.py

@@ -41,7 +41,7 @@ def _report_payload(data: ReportData) -> dict:
         "features": _features_payload(data.features),
         "shap": {"topN": 5},
         "searchResults": {
-            "relevantFeaturesCount": _dash(data.summary.relevant_features),
+            "relevantFeaturesCount": _dash(len(data.features) if data.features else data.summary.relevant_features),
             "dataSourcesCount": _dash(data.summary.data_sources),
             "autofeCount": _dash(len(data.autofe)),
             "sources": _sources_payload(data.sources),
@@ -170,21 +170,27 @@ def _used_in_model_caption(summary: SearchResultsSummary) -> str:
 
 
 def _features_payload(features: list[FeatureRow]) -> list[dict]:
-    return [
-        {
-            "name": row.name,
-            "provider": row.provider,
-            "source": row.source,
-            "shapClass": _shap_fill_class(row),
-            "importance": abs(row.shap) if row.shap is not None else 0,
-            "shap": row.shap,
-            "coverage": row.coverage,
-            "status": row.stability_status,
-            "psi": row.psi,
-            "drift": row.drift,
-        }
-        for row in features
-    ]
+    rows = []
+    for row in features:
+        shap_class = _shap_fill_class(row)
+        provider = row.provider
+        if not provider and shap_class == "user":
+            provider = CLIENT_SOURCE
+        rows.append(
+            {
+                "name": row.name,
+                "provider": provider,
+                "source": row.source,
+                "shapClass": shap_class,
+                "importance": abs(row.shap) if row.shap is not None else 0,
+                "shap": row.shap,
+                "coverage": row.coverage,
+                "status": row.stability_status,
+                "psi": row.psi,
+                "drift": row.drift,
+            }
+        )
+    return rows
 
 
 def _shap_fill_class(row: FeatureRow) -> str:

{upgini-1.2.167a4220.dev11 → upgini-1.2.167a4220.dev13}/src/upgini/utils/deduplicate_utils.py

@@ -17,6 +17,15 @@ from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.utils.datetime_utils import DateTimeConverter
 from upgini.utils.target_utils import define_task
 
+_INDEX_SAMPLE = 3
+
+
+def _format_index_sample(indexes) -> str:
+    values = list(indexes)
+    if len(values) <= _INDEX_SAMPLE:
+        return str(values)
+    return str(values[:_INDEX_SAMPLE])[:-1] + ", ...]"
+
 
 def remove_fintech_duplicates(
     df: pd.DataFrame,
@@ -117,11 +126,11 @@ def remove_fintech_duplicates(
             perc = len(rows_to_remove) * 100 / len(segment_df)
             if eval_index == 0:
                 msg = bundle.get("dataset_train_diff_target_duplicates_fintech").format(
-                    perc, len(rows_to_remove), rows_to_remove.index.to_list()
+                    perc, len(rows_to_remove), _format_index_sample(rows_to_remove.index)
                 )
             else:
                 msg = bundle.get("dataset_eval_diff_target_duplicates_fintech").format(
-                    perc, len(rows_to_remove), eval_index, rows_to_remove.index.to_list()
+                    perc, len(rows_to_remove), eval_index, _format_index_sample(rows_to_remove.index)
                 )
             return segment_df[~segment_df.index.isin(rows_to_remove.index)], msg
         return segment_df, None
@@ -227,7 +236,7 @@ def clean_full_duplicates(
 
         marked_duplicates = df_for_dedup.duplicated(subset=unique_columns, keep=False)
         if marked_duplicates.sum() > 0:
-            dups_indices = df_for_dedup[marked_duplicates].index.to_list()[:100]
+            dups_indices = _format_index_sample(df_for_dedup[marked_duplicates].index)
             nrows_after_tgt_dedup = len(df_for_dedup.drop_duplicates(subset=unique_columns, keep=False))
             num_dup_rows = len(df_for_dedup) - nrows_after_tgt_dedup
             share_tgt_dedup = 100 * num_dup_rows / len(df_for_dedup)
@@ -240,7 +249,7 @@ def clean_full_duplicates(
             df = pd.concat([df_for_dedup, oot_df], ignore_index=False)
             marked_duplicates = df.duplicated(subset=unique_columns, keep=False)
             if marked_duplicates.sum() > 0:
-                dups_indices = df[marked_duplicates].index.to_list()[:100]
+                dups_indices = _format_index_sample(df[marked_duplicates].index)
                 nrows_after_tgt_dedup = len(df.drop_duplicates(subset=unique_columns, keep=False))
                 num_dup_rows = len(df) - nrows_after_tgt_dedup
                 share_tgt_dedup = 100 * num_dup_rows / len(df)

upgini-1.2.167a4220.dev11/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.167.a4220.dev11"