upgini 1.2.167a4220.dev3__tar.gz → 1.2.167a4220.dev5__tar.gz

{upgini-1.2.167a4220.dev3 → upgini-1.2.167a4220.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: upgini
-Version: 1.2.167a4220.dev3
+Version: 1.2.167a4220.dev5
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.167a4220.dev5/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.167.a4220.dev5"

{upgini-1.2.167a4220.dev3 → upgini-1.2.167a4220.dev5}/src/upgini/features_enricher.py

@@ -100,6 +100,7 @@ from upgini.utils.deduplicate_utils import (
 )
 from upgini.report.assemble import assemble_report_data
 from upgini.report.html import generate_html_report
+from upgini.report.stats import make_report_frame
 from upgini.utils.display_utils import (
     display_html_dataframe,
     do_without_pandas_limits,
@@ -1461,6 +1462,9 @@ class FeaturesEnricher(TransformerMixin):
         progress_bar: bool = True,
         progress_callback: Callable | None = None,
     ):
+        if self._has_single_ensemble_score():
+            self.logger.info("Skip OOT PSI stability check for single ensemble score")
+            return
         search_keys = self.search_keys.copy()
         search_keys.update(self._get_autodetected_search_keys())
         validated_X, _, validated_eval_set = self._validate_train_eval(X, y, eval_set)
@@ -2308,17 +2312,135 @@ class FeaturesEnricher(TransformerMixin):
             rows += _num_samples(eval_pair[0])
         return rows
 
+    def _search_key_names(self) -> list[str]:
+        return [key.name for key in self.search_keys.values()]
+
     def _report_sample_names(self) -> list[str]:
         samples = [self.bundle.get("quality_metrics_train_segment")]
         for idx, _ in enumerate(self.eval_set or []):
             samples.append(self.bundle.get("quality_metrics_eval_segment").format(idx + 1))
         return samples
 
+    def _report_dataset_samples(self) -> dict[str, pd.DataFrame]:
+        names = self._report_sample_names()
+        if self.df_with_original_index is not None:
+            return self._report_samples_from_combined(self.df_with_original_index, names, self.fit_search_keys)
+        if self.X is None or self.y is None:
+            return {}
+        date_col = self._get_date_column(self.search_keys)
+        frames = {names[0]: self._report_xy_frame(self.X, self.y, date_col)}
+        for idx, (eval_x, eval_y) in enumerate(self.eval_set or []):
+            if idx + 1 >= len(names):
+                break
+            frames[names[idx + 1]] = self._report_xy_frame(eval_x, eval_y, date_col)
+        return frames
+
+    def _report_samples_from_combined(
+        self, df: pd.DataFrame, names: list[str], search_keys: dict[str, SearchKey] | None
+    ) -> dict[str, pd.DataFrame]:
+        date_col = self._get_date_column(search_keys or {})
+        if EVAL_SET_INDEX not in df.columns:
+            return {names[0]: self._report_df_frame(df, date_col)} if names else {}
+        frames: dict[str, pd.DataFrame] = {}
+        for eval_idx in sorted(pd.unique(df[EVAL_SET_INDEX].dropna())):
+            name_idx = int(eval_idx)
+            if name_idx >= len(names):
+                continue
+            frames[names[name_idx]] = self._report_df_frame(df[df[EVAL_SET_INDEX] == eval_idx], date_col)
+        return frames
+
+    def _report_xy_frame(self, X, y, date_col: str | None) -> pd.DataFrame:
+        date = X[date_col] if isinstance(X, pd.DataFrame) and date_col and date_col in X.columns else None
+        return make_report_frame(y, date=date)
+
+    def _report_df_frame(self, df: pd.DataFrame, date_col: str | None) -> pd.DataFrame:
+        target = df[TARGET] if TARGET in df.columns else pd.Series(np.nan, index=df.index)
+        return self._report_xy_frame(df, target, date_col)
+
+    def _report_scored_samples(self) -> dict[str, pd.DataFrame]:
+        cached = self._peek_cached_sampled()
+        if cached is None:
+            return {}
+        X_sampled, y_sampled, enriched_X, eval_set_sampled_dict, search_keys, columns_renaming, _generated = cached
+        score_col = self._report_score_column(enriched_X)
+        if score_col is None:
+            return {}
+        date_col = self._get_date_column(search_keys or {})
+        names = self._report_sample_names()
+        frames = {
+            names[0]: self._report_scored_frame(
+                X_sampled, y_sampled, enriched_X, score_col, date_col, columns_renaming
+            )
+        }
+        for idx, (eval_X, enriched_eval_X, eval_y) in eval_set_sampled_dict.items():
+            name_idx = idx + 1
+            if name_idx >= len(names):
+                continue
+            frames[names[name_idx]] = self._report_scored_frame(
+                eval_X, eval_y, enriched_eval_X, score_col, date_col, columns_renaming
+            )
+        return frames
+
+    def _report_scored_frame(
+        self,
+        X: pd.DataFrame,
+        y,
+        enriched: pd.DataFrame,
+        score_col: str,
+        date_col: str | None,
+        columns_renaming: dict[str, str] | None,
+    ) -> pd.DataFrame:
+        target = y
+        if target is None and TARGET in enriched.columns:
+            target = enriched[TARGET]
+        date_name = self._report_existing_column(date_col, columns_renaming, X, enriched)
+        baseline_name = self._report_existing_column(self.baseline_score_column, columns_renaming, X, enriched)
+        return make_report_frame(
+            target,
+            date=self._report_column_values(date_name, X, enriched),
+            score=enriched[score_col] if score_col in enriched.columns else None,
+            baseline=self._report_column_values(baseline_name, X, enriched),
+        )
+
+    def _report_score_column(self, enriched_X: pd.DataFrame) -> str | None:
+        name = self._get_single_ensemble_score_name()
+        if name and name in enriched_X.columns:
+            return name
+        return None
+
+    def _report_existing_column(
+        self, name: str | None, renaming: dict[str, str] | None, *frames: pd.DataFrame
+    ) -> str | None:
+        if not name:
+            return None
+        aliases = self._column_name_aliases([name], renaming or {})
+        for frame in frames:
+            if not isinstance(frame, pd.DataFrame):
+                continue
+            for col in frame.columns:
+                if col in aliases:
+                    return col
+        return None
+
+    @staticmethod
+    def _report_column_values(column: str | None, *frames: pd.DataFrame):
+        if not column:
+            return None
+        for frame in frames:
+            if isinstance(frame, pd.DataFrame) and column in frame.columns:
+                return frame[column]
+        return None
+
+    def _peek_cached_sampled(self):
+        if not self.__cached_sampled_datasets:
+            return None
+        return max(self.__cached_sampled_datasets.values(), key=lambda item: len(item[0]) if item[0] is not None else 0)
+
     def _assemble_report_data(self):
         search_id = self._search_task.search_task_id if self._search_task is not None else (self.search_id or "")
         return assemble_report_data(
             search_id=search_id,
-            search_keys=[str(sk) for sk in self.search_keys.values()],
+            search_keys=self._search_key_names(),
             search_duration_seconds=self.search_duration_seconds,
             reference_rows=self._reference_rows(),
             samples=self._report_sample_names(),
@@ -2326,6 +2448,8 @@ class FeaturesEnricher(TransformerMixin):
             metric_name=self.metrics_metric_name,
             model_features=self._ensemble_model_feature_count(),
             is_binary=self.model_task_type == ModelTaskType.BINARY,
+            dataset_samples=self._report_dataset_samples(),
+            scored_samples=self._report_scored_samples(),
             bundle=self.bundle,
         )
 
@@ -6092,7 +6216,7 @@ if response.status_code == 200:
                 autofe_descriptions_df=self.get_autofe_features_description(),
                 search_id=self._search_task.search_task_id,
                 email=self.rest_client.get_current_email(),
-                search_keys=[str(sk) for sk in self.search_keys.values()],
+                search_keys=self._search_key_names(),
                 eval_sets_drift_df=eval_sets_drift_df,
                 display_id=display_id,
                 display_handle=display_handle,

upgini-1.2.167a4220.dev5/src/upgini/report/__init__.py

@@ -0,0 +1,11 @@
+from upgini.report.assemble import UPGINI_REPORT_BRANDING_URL, assemble_report_data, format_search_duration
+from upgini.report.data import ReportData
+from upgini.report.html import generate_html_report
+
+__all__ = [
+    "ReportData",
+    "UPGINI_REPORT_BRANDING_URL",
+    "assemble_report_data",
+    "format_search_duration",
+    "generate_html_report",
+]

{upgini-1.2.167a4220.dev3 → upgini-1.2.167a4220.dev5}/src/upgini/report/assemble.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 from datetime import datetime, timezone
 from typing import Optional
 
@@ -7,8 +8,11 @@ import pandas as pd
 
 from upgini.__about__ import __version__
 from upgini.report.data import QualitySample, ReportData, ReportMetadata, SampleStats, SearchResultsSummary
+from upgini.report.stats import compute_report_charts
 from upgini.resource_bundle import ResourceBundle
 
+UPGINI_REPORT_BRANDING_URL = "UPGINI_REPORT_BRANDING_URL"
+
 
 def format_search_duration(seconds: Optional[float]) -> Optional[str]:
     if seconds is None:
@@ -36,11 +40,19 @@ def assemble_report_data(
     metric_name: Optional[str] = None,
     model_features: Optional[int] = None,
     is_binary: bool = False,
-    logo_url: Optional[str] = None,
+    dataset_samples: Optional[dict[str, pd.DataFrame]] = None,
+    scored_samples: Optional[dict[str, pd.DataFrame]] = None,
     bundle: ResourceBundle,
 ) -> ReportData:
     generated_at = generated_at or datetime.now(timezone.utc)
     sample_names = list(samples or [])
+    chart_stats, charts = compute_report_charts(
+        sample_names=sample_names,
+        dataset_samples=dataset_samples,
+        scored_samples=scored_samples,
+        metric_name=metric_name,
+        is_binary=is_binary,
+    )
     return ReportData(
         metadata=ReportMetadata(
             search_id=search_id,
@@ -50,15 +62,23 @@ def assemble_report_data(
             search_duration=format_search_duration(search_duration_seconds),
             reference_rows=reference_rows,
             samples=sample_names,
-            logo_url=logo_url or None,
+            logo_url=_branding_url(),
         ),
         quality_by_sample=_quality_samples(metrics_df, metric_name, bundle),
         summary=SearchResultsSummary(model_features=model_features),
-        sample_stats=_sample_stats(sample_names, metrics_df, bundle),
+        sample_stats=chart_stats or _sample_stats(sample_names, metrics_df, bundle),
         is_binary=is_binary,
+        charts=charts,
     )
 
 
+def _branding_url() -> Optional[str]:
+    env_url = os.getenv(UPGINI_REPORT_BRANDING_URL)
+    if env_url and env_url.strip():
+        return env_url.strip()
+    return None
+
+
 def _quality_samples(
     metrics_df: Optional[pd.DataFrame], metric_name: Optional[str], bundle: ResourceBundle
 ) -> list[QualitySample]:

{upgini-1.2.167a4220.dev3 → upgini-1.2.167a4220.dev5}/src/upgini/report/data.py

@@ -79,6 +79,70 @@ class ModelFeatureShap:
     mean_abs_shap: Optional[float] = None
 
 
+@dataclass
+class MonthlySamplePoint:
+    month: str
+    sample: str
+    total: int
+    labeled: int
+    unlabeled: int
+    positive: int
+    mean: Optional[float] = None
+
+
+@dataclass
+class QualityMonthly:
+    evaluation_scope: str
+    metric: str
+    baseline: list[Optional[float]] = field(default_factory=list)
+    enriched: list[Optional[float]] = field(default_factory=list)
+
+
+@dataclass
+class ScoreMonthlyStats:
+    sample: str
+    mean_score: list[Optional[float]] = field(default_factory=list)
+    mean_target: list[Optional[float]] = field(default_factory=list)
+    labeled: list[Optional[int]] = field(default_factory=list)
+    unlabeled: list[Optional[int]] = field(default_factory=list)
+
+
+@dataclass
+class ScoreHistogram:
+    sample: str
+    target_0: list[float] = field(default_factory=list)
+    target_1: list[float] = field(default_factory=list)
+    n_target_0: int = 0
+    n_target_1: int = 0
+
+
+@dataclass
+class ScorePsi:
+    sample: str
+    psi: list[Optional[float]] = field(default_factory=list)
+    rows: list[Optional[int]] = field(default_factory=list)
+    coverage: list[Optional[float]] = field(default_factory=list)
+
+
+@dataclass
+class ReportCharts:
+    timeline_months: list[str] = field(default_factory=list)
+    period_options: list[int] = field(default_factory=list)
+    default_period: int = 0
+    sample_monthly: list[MonthlySamplePoint] = field(default_factory=list)
+    monthly_count_max: int = 0
+    quality_monthly: list[QualityMonthly] = field(default_factory=list)
+    score_monthly: list[ScoreMonthlyStats] = field(default_factory=list)
+    score_count_axis_max: int = 0
+    histogram_bin_edges: list[float] = field(default_factory=list)
+    histograms: list[ScoreHistogram] = field(default_factory=list)
+    histogram_density_max: float = 0.0
+    score_psi: list[ScorePsi] = field(default_factory=list)
+    psi_axis_max: float = 0.32
+    psi_warning: float = 0.1
+    psi_critical: float = 0.25
+
+
 @dataclass
 class ReportData:
     metadata: ReportMetadata
@@ -90,3 +154,4 @@ class ReportData:
     sources: list[SourceRow] = field(default_factory=list)
     autofe: list[dict[str, str]] = field(default_factory=list)
     model_feature_shap: list[ModelFeatureShap] = field(default_factory=list)
+    charts: ReportCharts = field(default_factory=ReportCharts)

{upgini-1.2.167a4220.dev3 → upgini-1.2.167a4220.dev5}/src/upgini/report/html.py

@@ -4,7 +4,8 @@ import json
 from pathlib import Path
 from typing import Optional
 
-from upgini.report.data import ReportData, SampleStats, SearchResultsSummary
+from upgini.report.data import ReportCharts, ReportData, SampleStats, SearchResultsSummary
+from upgini.report.stats import MEAN_AXIS_PADDING
 
 _TEMPLATE_PATH = Path(__file__).with_name("template.html")
 _SAMPLE_COLORS = ("#1645ee", "#20aab4", "#53cc61", "#e19132", "#d45b5b")
@@ -21,16 +22,21 @@ def generate_html_report(data: ReportData) -> str:
 
 def _report_payload(data: ReportData) -> dict:
     samples = _samples(data)
+    charts = data.charts or ReportCharts()
     return {
         "meta": _meta(data, samples),
-        "timeline": {"months": [], "periodOptions": [], "defaultPeriod": 0},
+        "timeline": {
+            "months": list(charts.timeline_months),
+            "periodOptions": list(charts.period_options),
+            "defaultPeriod": charts.default_period,
+        },
         "samples": samples,
-        "keyResult": _key_result(data),
+        "keyResult": _key_result(data, charts),
         "summaryCards": _summary_cards(data.summary),
-        "sampleStats": _sample_stats_payload(data, samples),
-        "scoreAnalysis": {"bySample": {}},
-        "scoreDistribution": {"bySample": {}},
-        "scoreStability": {"bySample": {}},
+        "sampleStats": _sample_stats_payload(data, samples, charts),
+        "scoreAnalysis": _score_analysis_payload(charts, samples),
+        "scoreDistribution": _score_distribution_payload(charts, samples),
+        "scoreStability": _score_stability_payload(charts, samples),
         "features": [],
         "featureStability": {"bySample": {}},
         "shap": {"topN": 5},
@@ -73,23 +79,43 @@ def _samples(data: ReportData) -> list[dict]:
         {
             "id": _slug(name),
             "label": name,
-            "caption": "",
+            "caption": "OOF validation" if index == 0 else "",
             "color": _SAMPLE_COLORS[index % len(_SAMPLE_COLORS)],
         }
         for index, name in enumerate(names)
     ]
 
 
-def _key_result(data: ReportData) -> dict:
+def _key_result(data: ReportData, charts: ReportCharts) -> dict:
+    series = {
+        (_slug(row.metric or "score"), _slug(row.evaluation_scope)): row for row in charts.quality_monthly
+    }
     metrics: dict[str, dict] = {}
     for sample in data.quality_by_sample:
         key = _slug(sample.metric or "score")
         bucket = metrics.setdefault(key, {"name": sample.metric or "score", "bySample": {}})
+        monthly = series.get((key, _slug(sample.evaluation_scope)))
         bucket["bySample"][_slug(sample.evaluation_scope)] = {
             "baseline": sample.baseline,
             "baselineCi": sample.baseline_std,
             "enriched": sample.enriched,
             "enrichedCi": sample.enriched_std,
+            "baselineSeries": list(monthly.baseline) if monthly else [],
+            "enrichedSeries": list(monthly.enriched) if monthly else [],
+        }
+    for monthly in charts.quality_monthly:
+        key = _slug(monthly.metric or "score")
+        sample_id = _slug(monthly.evaluation_scope)
+        bucket = metrics.setdefault(key, {"name": monthly.metric or "score", "bySample": {}})
+        if sample_id in bucket["bySample"]:
+            continue
+        bucket["bySample"][sample_id] = {
+            "baseline": None,
+            "baselineCi": None,
+            "enriched": None,
+            "enrichedCi": None,
+            "baselineSeries": list(monthly.baseline),
+            "enrichedSeries": list(monthly.enriched),
         }
     default_metric = next(iter(metrics), "")
     has_std = any(
@@ -123,7 +149,7 @@ def _summary_cards(summary: SearchResultsSummary) -> list[dict]:
     ]
 
 
-def _sample_stats_payload(data: ReportData, samples: list[dict]) -> dict:
+def _sample_stats_payload(data: ReportData, samples: list[dict], charts: ReportCharts) -> dict:
     by_id = {_slug(stats.sample): stats for stats in data.sample_stats}
     rows = [
         _stat_row("Date range", samples, by_id, lambda stats: stats.date_range),
@@ -138,7 +164,76 @@ def _sample_stats_payload(data: ReportData, samples: list[dict]) -> dict:
             _stat_row("Count (unlabeled)", samples, by_id, lambda stats: _format_int(stats.unlabeled)),
         ]
     )
-    return {"rows": rows, "monthlyAxis": [], "monthlyPoints": []}
+    return {
+        "rows": rows,
+        "monthlyAxis": list(charts.timeline_months),
+        "monthlyCountMax": charts.monthly_count_max or 1,
+        "monthlyMeanAxisPadding": MEAN_AXIS_PADDING,
+        "monthlyPoints": [
+            {
+                "month": point.month,
+                "sample": _slug(point.sample),
+                "total": point.total,
+                "labeled": point.labeled,
+                "unlabeled": point.unlabeled,
+                "positive": point.positive,
+                "mean": point.mean,
+            }
+            for point in charts.sample_monthly
+        ],
+    }
+
+
+def _score_analysis_payload(charts: ReportCharts, samples: list[dict]) -> dict:
+    by_id = {_slug(row.sample): row for row in charts.score_monthly}
+    return {
+        "countAxisMax": charts.score_count_axis_max or 1,
+        "bySample": {
+            sample["id"]: {
+                "meanScore": list(row.mean_score),
+                "meanTarget": list(row.mean_target),
+                "labeled": list(row.labeled),
+                "unlabeled": list(row.unlabeled),
+            }
+            for sample in samples
+            if (row := by_id.get(sample["id"])) is not None
+        },
+    }
+
+
+def _score_distribution_payload(charts: ReportCharts, samples: list[dict]) -> dict:
+    by_id = {_slug(row.sample): row for row in charts.histograms}
+    train = next((by_id.get(sample["id"]) for sample in samples if sample["id"] in by_id), None)
+    return {
+        "binEdges": list(charts.histogram_bin_edges),
+        "densityAxisMax": charts.histogram_density_max or 1,
+        "countScale": {
+            "target0": train.n_target_0 if train else 0,
+            "target1": train.n_target_1 if train else 0,
+        },
+        "bySample": {
+            sample["id"]: {"target0": list(row.target_0), "target1": list(row.target_1)}
+            for sample in samples
+            if (row := by_id.get(sample["id"])) is not None
+        },
+    }
+
+
+def _score_stability_payload(charts: ReportCharts, samples: list[dict]) -> dict:
+    by_id = {_slug(row.sample): row for row in charts.score_psi}
+    return {
+        "psiAxisMax": charts.psi_axis_max or charts.psi_critical,
+        "thresholds": {"warning": charts.psi_warning, "critical": charts.psi_critical},
+        "bySample": {
+            sample["id"]: {
+                "psi": list(row.psi),
+                "rows": list(row.rows),
+                "scoreCoverage": list(row.coverage),
+            }
+            for sample in samples
+            if (row := by_id.get(sample["id"])) is not None
+        },
+    }
 
 
 def _stat_row(label: str, samples: list[dict], by_id: dict[str, SampleStats], value) -> dict: