proxyml 0.5.0__tar.gz → 0.6.0__tar.gz

{proxyml-0.5.0 → proxyml-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxyml
-Version: 0.5.0
+Version: 0.6.0
 Summary: Python SDK for calling the ProxyML API
 Author-email: ProxyML <contact@proxyml.ai>
 License:                                  Apache License

{proxyml-0.5.0 → proxyml-0.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "proxyml"
-version = "0.5.0"
+version = "0.6.0"
 description = "Python SDK for calling the ProxyML API"
 readme = "README.md"
 license = {file = "LICENSE"}

{proxyml-0.5.0 → proxyml-0.6.0}/src/proxyml/local/challenger.py

@@ -118,6 +118,22 @@ class TrainedChallenger:
     metrics: dict[str, float]
     hyperparameters: dict[str, Any]
     export: SurrogateExport
+    n_samples_total: int
+    n_samples_dropped_unlabeled: int
+    population_note: str
+    champion_metrics: dict[str, float] | None = None
+
+
+def _population_note(target_name: str, n_total: int, n_labeled: int, n_dropped: int) -> str:
+    if n_dropped == 0:
+        return f"Evaluated on all {n_total} row(s) — '{target_name}' had no missing values."
+    return (
+        f"Evaluated on {n_labeled} of {n_total} row(s) with a non-null '{target_name}' value "
+        f"({n_dropped} unlabeled row(s) dropped before training/scoring). "
+        "Labeled-vs-unlabeled selection may not be random; treat this as a declared "
+        "scope limitation on the evaluation population, not a claim about performance "
+        "on the full dataset."
+    )
 
 
 def train_challenger(
@@ -129,6 +145,8 @@ def train_challenger(
     feature_names: list[str] | None = None,
     task: Literal["classification", "regression", "auto"] = "auto",
     test_size: float = 0.2,
+    target_name: str = "target",
+    champion_predictions: np.ndarray | list | None = None,
 ) -> TrainedChallenger:
     """Train a linear challenger model on ``df`` against ``target``, locally.
 
@@ -142,6 +160,15 @@ def train_challenger(
     can be compared with the same ``proxyml_core.export.predict_from_export``
     arithmetic.
 
+    Rows where ``target`` is missing (NaN/None) are dropped before training,
+    the CV split, and champion scoring — never silently included. The drop
+    count and a human-readable scope-limitation note are recorded on the
+    result (``n_samples_total``, ``n_samples_dropped_unlabeled``,
+    ``population_note``). If ``champion_predictions`` is given, it must have
+    one entry per row of ``df``/``target`` (same order) so the identical rows
+    are dropped from both sides — champion and challenger are always
+    evaluated on the same labeled population, never on different ones.
+
     Args:
         df: samples to train on, one column per schema feature.
         target: the value to predict for each row of ``df`` — ground-truth
@@ -151,7 +178,35 @@ def train_challenger(
         feature_names: subset of ``schema.features`` to train on; omit for all.
         task: "classification", "regression", or "auto" to infer from ``target``.
         test_size: fraction of data held out to compute fidelity metrics.
+        target_name: human-readable name for ``target``, used in
+            ``population_note`` (e.g. the column name, if known).
+        champion_predictions: a champion model's predictions, one per row of
+            ``df``/``target`` (same order). If given, scored via
+            ``score_champion()`` against the same (row-dropped) ``target``,
+            and the result is attached as ``TrainedChallenger.champion_metrics``.
     """
+    target_arr = np.asarray(target)
+    if champion_predictions is not None and len(champion_predictions) != len(target_arr):
+        raise ValueError(
+            f"champion_predictions must have one entry per row of target "
+            f"({len(target_arr)} rows, got {len(champion_predictions)}) — same order — so "
+            f"rows with a missing {target_name!r} value can be dropped from both the "
+            f"challenger and the champion, keeping them evaluated on the same population."
+        )
+
+    labeled_mask = ~pd.isna(target_arr)
+    n_total = len(target_arr)
+    n_labeled = int(labeled_mask.sum())
+    n_dropped = n_total - n_labeled
+    if n_labeled == 0:
+        raise ValueError(f"All {n_total} row(s) have a missing {target_name!r} value; nothing to train on")
+
+    df = df.iloc[labeled_mask].reset_index(drop=True)
+    target_arr = target_arr[labeled_mask]
+    champion_predictions_labeled = (
+        np.asarray(champion_predictions)[labeled_mask] if champion_predictions is not None else None
+    )
+
     rung = LADDERS[complexity]
 
     features: list[Feature] = schema.features
@@ -161,7 +216,7 @@ def train_challenger(
     col_order = [f.name for f in features]
 
     X = df[col_order].to_numpy(dtype=object)
-    y = np.asarray(target)
+    y = target_arr
 
     if task == "auto":
         classification = is_classification(y)
@@ -174,6 +229,10 @@ def train_challenger(
     if classification:
         y = binarize_if_probabilities(y)
 
+    champion_metrics = None
+    if champion_predictions_labeled is not None:
+        champion_metrics = score_champion(y, champion_predictions_labeled, task=resolved_task)
+
     preprocessor = build_preprocessor(features)
     estimator = rung.build_classifier() if classification else rung.build_regressor()
     pipeline = Pipeline(steps=[("preprocessor", preprocessor), ("estimator", estimator)])
@@ -202,6 +261,10 @@ def train_challenger(
         metrics=metrics,
         hyperparameters=hyperparameters,
         export=export,
+        n_samples_total=n_total,
+        n_samples_dropped_unlabeled=n_dropped,
+        population_note=_population_note(target_name, n_total, n_labeled, n_dropped),
+        champion_metrics=champion_metrics,
     )
 
 
@@ -230,7 +293,7 @@ def score_champion(
 def to_challenger_upload(
     result: TrainedChallenger,
     *,
-    n_samples: int,
+    n_samples: int | None = None,
     champion_metrics: dict[str, float] | None = None,
     sdk_version: str | None = None,
     proxyml_core_version: str | None = None,
@@ -245,27 +308,32 @@ def to_challenger_upload(
     ``json.dump`` — upload it either by POSTing it directly, or by saving it
     to a file and using the dashboard's "Upload challenger" button.
 
-    ``champion_metrics`` is optional: pass ``None`` (the default) to get a
-    self-contained export of the challenger alone — e.g. to save/share it
-    before you have a champion to compare against — and fill in
-    ``champion_metrics`` later. The upload endpoint itself still requires
-    ``champion_metrics`` at upload time; this function just doesn't force you
-    to have it up front.
+    ``champion_metrics`` is optional: pass ``None`` (the default) to fall
+    back to ``result.champion_metrics`` (populated automatically if you
+    passed ``champion_predictions`` to ``train_challenger()``/
+    ``train_auto_challenger()``) — or, if that's also ``None``, to get a
+    self-contained export of the challenger alone, e.g. to save/share it
+    before you have a champion to compare against. The upload endpoint
+    itself still requires ``champion_metrics`` at upload time; this function
+    just doesn't force you to have it up front.
 
     Args:
         result: output of ``train_challenger()``/``train_auto_challenger()``.
         n_samples: size of the evaluation set both ``result.metrics`` and
-            ``champion_metrics`` were scored on. Not derived automatically —
-            ``TrainedChallenger`` doesn't retain its internal held-out split,
-            and ``champion_metrics`` typically comes from a separate
-            ``score_champion()`` call the two need to share, so the caller is
-            the only one who actually knows this number.
+            ``champion_metrics`` were scored on. Defaults to
+            ``result.n_samples_total - result.n_samples_dropped_unlabeled``
+            (the labeled-row count) — override only if you scored on some
+            other population.
         champion_metrics: the champion's real-world performance, from
             ``score_champion()`` — same metric keys as ``result.metrics``.
-            Omit if you don't have it yet.
+            Defaults to ``result.champion_metrics``.
         sdk_version: defaults to the installed ``proxyml`` version.
         proxyml_core_version: defaults to the installed ``proxyml-core`` version.
     """
+    if n_samples is None:
+        n_samples = result.n_samples_total - result.n_samples_dropped_unlabeled
+    if champion_metrics is None:
+        champion_metrics = result.champion_metrics
     if sdk_version is None:
         sdk_version = _pkg_version("proxyml")
     if proxyml_core_version is None:
@@ -275,6 +343,9 @@ def to_challenger_upload(
         "export": result.export.to_dict(),
         "challenger_metrics": result.metrics,
         "n_samples": n_samples,
+        "n_samples_total": result.n_samples_total,
+        "n_samples_dropped_unlabeled": result.n_samples_dropped_unlabeled,
+        "population_note": result.population_note,
         "complexity": result.complexity.value,
         "sdk_version": sdk_version,
         "proxyml_core_version": proxyml_core_version,
@@ -293,6 +364,7 @@ def train_auto_challenger(
     feature_names: list[str] | None = None,
     task: Literal["classification", "regression", "auto"] = "auto",
     test_size: float = 0.2,
+    champion_predictions: np.ndarray | list | None = None,
 ) -> TrainedChallenger:
     """Load data, infer a schema, and train a linear challenger in one call.
 
@@ -302,6 +374,12 @@ def train_auto_challenger(
     remains overridable; this does not search across ``LADDERS`` to find the
     best-fitting rung.
 
+    Rows with a missing ``target_col`` value are dropped before training and
+    champion scoring — see ``train_challenger()`` for details. Schema
+    inference (feature means/stds/categories) still runs over every row,
+    including ones later dropped for a missing target — only training and
+    evaluation are restricted to the labeled subset.
+
     Args:
         data: a CSV path, or an already-loaded DataFrame containing both the
             feature columns and ``target_col``.
@@ -312,6 +390,8 @@ def train_auto_challenger(
         feature_names: subset of feature columns to train on; omit for all.
         task: "classification", "regression", or "auto" to infer from ``target_col``.
         test_size: fraction of data held out to compute fidelity metrics.
+        champion_predictions: a champion model's predictions, one per row of
+            ``data`` (same order) — see ``train_challenger()``.
     """
     df = data if isinstance(data, pd.DataFrame) else pd.read_csv(data)
     target = df[target_col]
@@ -326,4 +406,6 @@ def train_auto_challenger(
         feature_names=feature_names,
         task=task,
         test_size=test_size,
+        target_name=target_col,
+        champion_predictions=champion_predictions,
     )

{proxyml-0.5.0 → proxyml-0.6.0}/src/proxyml.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxyml
-Version: 0.5.0
+Version: 0.6.0
 Summary: Python SDK for calling the ProxyML API
 Author-email: ProxyML <contact@proxyml.ai>
 License:                                  Apache License

{proxyml-0.5.0 → proxyml-0.6.0}/tests/test_local_challenger.py

@@ -286,6 +286,108 @@ def test_to_challenger_upload_payload_is_json_serializable():
     json.dumps(payload)  # must not raise
 
 
+def _labeled_df_with_nan_target(n=200, n_nan=20, seed=20):
+    df = _labeled_df(n=n, seed=seed)
+    df["approved"] = df["approved"].astype(float)
+    df.loc[df.index[:n_nan], "approved"] = np.nan
+    return df
+
+
+def test_nan_target_rows_are_dropped_and_counted():
+    df = _labeled_df_with_nan_target(n=200, n_nan=20)
+    result = train_auto_challenger(df, "approved", task="classification")
+
+    assert result.n_samples_total == 200
+    assert result.n_samples_dropped_unlabeled == 20
+    assert "20" in result.population_note
+    assert "180" in result.population_note
+
+
+def test_no_nan_targets_reports_zero_dropped():
+    df = _labeled_df(seed=21)
+    result = train_auto_challenger(df, "approved", task="classification")
+
+    assert result.n_samples_total == len(df)
+    assert result.n_samples_dropped_unlabeled == 0
+    assert "no missing values" in result.population_note
+
+
+def test_all_nan_target_raises():
+    df = _labeled_df(n=20, seed=22)
+    df["approved"] = np.nan
+    with pytest.raises(ValueError, match="missing"):
+        train_auto_challenger(df, "approved", task="classification")
+
+
+def test_champion_predictions_wrong_length_raises():
+    df = _labeled_df(seed=23)
+    with pytest.raises(ValueError, match="one entry per row"):
+        train_auto_challenger(
+            df, "approved", task="classification", champion_predictions=[True, False]
+        )
+
+
+def test_champion_predictions_scored_only_on_labeled_rows():
+    # Champion predictions mirror the (possibly-NaN) target itself, except on
+    # rows that get dropped as unlabeled, where they're deliberately wrong.
+    # If those rows leaked into scoring, champion accuracy would come in
+    # under 1.0 instead of exactly 1.0 — proving the shared-drop guarantee,
+    # not just that nothing crashes.
+    df = _labeled_df_with_nan_target(n=200, n_nan=20, seed=24)
+    champion_predictions = [False if pd.isna(v) else v for v in df["approved"]]
+
+    result = train_auto_challenger(
+        df, "approved", task="classification", champion_predictions=champion_predictions
+    )
+
+    assert result.champion_metrics is not None
+    assert result.champion_metrics["accuracy"] == 1.0
+
+
+def test_champion_predictions_not_given_leaves_champion_metrics_none():
+    df = _labeled_df(seed=25)
+    result = train_auto_challenger(df, "approved", task="classification")
+    assert result.champion_metrics is None
+
+
+def test_to_challenger_upload_defaults_n_samples_from_result():
+    df = _labeled_df_with_nan_target(n=200, n_nan=20, seed=26)
+    result = train_auto_challenger(df, "approved", task="classification")
+
+    payload = to_challenger_upload(result)
+
+    assert payload["n_samples"] == 180
+    assert payload["n_samples_total"] == 200
+    assert payload["n_samples_dropped_unlabeled"] == 20
+    assert payload["population_note"] == result.population_note
+
+
+def test_to_challenger_upload_defaults_champion_metrics_from_result():
+    df = _labeled_df(seed=27)
+    champion_predictions = df["approved"].tolist()
+    result = train_auto_challenger(
+        df, "approved", task="classification", champion_predictions=champion_predictions
+    )
+
+    payload = to_challenger_upload(result)
+
+    assert payload["champion_metrics"] == result.champion_metrics
+
+
+def test_to_challenger_upload_explicit_args_override_result_defaults():
+    df = _labeled_df(seed=28)
+    champion_predictions = df["approved"].tolist()
+    result = train_auto_challenger(
+        df, "approved", task="classification", champion_predictions=champion_predictions
+    )
+
+    override_metrics = {"f1": 0.1, "accuracy": 0.1}
+    payload = to_challenger_upload(result, n_samples=999, champion_metrics=override_metrics)
+
+    assert payload["n_samples"] == 999
+    assert payload["champion_metrics"] == override_metrics
+
+
 def test_train_auto_challenger_passes_immutable_cols_to_get_schema():
     from unittest.mock import patch