proxyml 0.5.0__tar.gz → 0.7.0__tar.gz

{proxyml-0.5.0 → proxyml-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxyml
-Version: 0.5.0
+Version: 0.7.0
 Summary: Python SDK for calling the ProxyML API
 Author-email: ProxyML <contact@proxyml.ai>
 License:                                  Apache License

{proxyml-0.5.0 → proxyml-0.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "proxyml"
-version = "0.5.0"
+version = "0.7.0"
 description = "Python SDK for calling the ProxyML API"
 readme = "README.md"
 license = {file = "LICENSE"}

{proxyml-0.5.0 → proxyml-0.7.0}/src/proxyml/local/challenger.py

@@ -10,6 +10,8 @@ training target was real ground truth or a black box's predictions.
 
 from __future__ import annotations
 
+import hashlib
+import json
 from dataclasses import dataclass, replace
 from enum import Enum
 from importlib.metadata import version as _pkg_version
@@ -118,6 +120,36 @@ class TrainedChallenger:
     metrics: dict[str, float]
     hyperparameters: dict[str, Any]
     export: SurrogateExport
+    n_samples_total: int
+    n_samples_dropped_unlabeled: int
+    population_note: str
+    target_fingerprint: str
+    champion_metrics: dict[str, float] | None = None
+
+
+def _fingerprint_values(values: np.ndarray | list) -> str:
+    """Hash an array of labels, deterministically, without the data ever leaving this process.
+
+    ``.tolist()`` converts numpy scalars to native Python types before
+    serializing, so the hash doesn't drift across numpy versions with
+    different scalar repr behavior. Order is preserved (not sorted) since
+    it encodes row alignment — that's exactly what a "same data?" check
+    needs to be sensitive to.
+    """
+    canonical = json.dumps(np.asarray(values).tolist())
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _population_note(target_name: str, n_total: int, n_labeled: int, n_dropped: int) -> str:
+    if n_dropped == 0:
+        return f"Evaluated on all {n_total} row(s) — '{target_name}' had no missing values."
+    return (
+        f"Evaluated on {n_labeled} of {n_total} row(s) with a non-null '{target_name}' value "
+        f"({n_dropped} unlabeled row(s) dropped before training/scoring). "
+        "Labeled-vs-unlabeled selection may not be random; treat this as a declared "
+        "scope limitation on the evaluation population, not a claim about performance "
+        "on the full dataset."
+    )
 
 
 def train_challenger(
@@ -129,6 +161,8 @@ def train_challenger(
     feature_names: list[str] | None = None,
     task: Literal["classification", "regression", "auto"] = "auto",
     test_size: float = 0.2,
+    target_name: str = "target",
+    champion_predictions: np.ndarray | list | None = None,
 ) -> TrainedChallenger:
     """Train a linear challenger model on ``df`` against ``target``, locally.
 
@@ -142,6 +176,15 @@ def train_challenger(
     can be compared with the same ``proxyml_core.export.predict_from_export``
     arithmetic.
 
+    Rows where ``target`` is missing (NaN/None) are dropped before training,
+    the CV split, and champion scoring — never silently included. The drop
+    count and a human-readable scope-limitation note are recorded on the
+    result (``n_samples_total``, ``n_samples_dropped_unlabeled``,
+    ``population_note``). If ``champion_predictions`` is given, it must have
+    one entry per row of ``df``/``target`` (same order) so the identical rows
+    are dropped from both sides — champion and challenger are always
+    evaluated on the same labeled population, never on different ones.
+
     Args:
         df: samples to train on, one column per schema feature.
         target: the value to predict for each row of ``df`` — ground-truth
@@ -151,7 +194,36 @@ def train_challenger(
         feature_names: subset of ``schema.features`` to train on; omit for all.
         task: "classification", "regression", or "auto" to infer from ``target``.
         test_size: fraction of data held out to compute fidelity metrics.
+        target_name: human-readable name for ``target``, used in
+            ``population_note`` (e.g. the column name, if known).
+        champion_predictions: a champion model's predictions, one per row of
+            ``df``/``target`` (same order). If given, scored via
+            ``score_champion()`` against the same (row-dropped) ``target``,
+            and the result is attached as ``TrainedChallenger.champion_metrics``.
     """
+    target_arr = np.asarray(target)
+    target_fingerprint = _fingerprint_values(target_arr)
+    if champion_predictions is not None and len(champion_predictions) != len(target_arr):
+        raise ValueError(
+            f"champion_predictions must have one entry per row of target "
+            f"({len(target_arr)} rows, got {len(champion_predictions)}) — same order — so "
+            f"rows with a missing {target_name!r} value can be dropped from both the "
+            f"challenger and the champion, keeping them evaluated on the same population."
+        )
+
+    labeled_mask = ~pd.isna(target_arr)
+    n_total = len(target_arr)
+    n_labeled = int(labeled_mask.sum())
+    n_dropped = n_total - n_labeled
+    if n_labeled == 0:
+        raise ValueError(f"All {n_total} row(s) have a missing {target_name!r} value; nothing to train on")
+
+    df = df.iloc[labeled_mask].reset_index(drop=True)
+    target_arr = target_arr[labeled_mask]
+    champion_predictions_labeled = (
+        np.asarray(champion_predictions)[labeled_mask] if champion_predictions is not None else None
+    )
+
     rung = LADDERS[complexity]
 
     features: list[Feature] = schema.features
@@ -161,7 +233,7 @@ def train_challenger(
     col_order = [f.name for f in features]
 
     X = df[col_order].to_numpy(dtype=object)
-    y = np.asarray(target)
+    y = target_arr
 
     if task == "auto":
         classification = is_classification(y)
@@ -174,6 +246,10 @@ def train_challenger(
     if classification:
         y = binarize_if_probabilities(y)
 
+    champion_metrics = None
+    if champion_predictions_labeled is not None:
+        champion_metrics = score_champion(y, champion_predictions_labeled, task=resolved_task)
+
     preprocessor = build_preprocessor(features)
     estimator = rung.build_classifier() if classification else rung.build_regressor()
     pipeline = Pipeline(steps=[("preprocessor", preprocessor), ("estimator", estimator)])
@@ -202,6 +278,11 @@ def train_challenger(
         metrics=metrics,
         hyperparameters=hyperparameters,
         export=export,
+        n_samples_total=n_total,
+        n_samples_dropped_unlabeled=n_dropped,
+        population_note=_population_note(target_name, n_total, n_labeled, n_dropped),
+        target_fingerprint=target_fingerprint,
+        champion_metrics=champion_metrics,
     )
 
 
@@ -223,6 +304,13 @@ def score_champion(
 
     Returns ``{"f1":..., "accuracy":...}`` for classification or ``{"r2":...}``
     for regression — the same shape as ``TrainedChallenger.metrics``.
+
+    If you're calling this decoupled from ``train_challenger()`` (i.e. not
+    via its ``champion_predictions=`` param), pass the same ``labels`` you
+    used here as ``champion_labels=`` to ``to_challenger_upload()`` — that
+    lets the upload endpoint confirm the challenger and champion were
+    actually scored on the same data, catching an accidental mismatched
+    file before it silently produces a misleading comparison.
     """
     return score_predictions(np.asarray(labels), np.asarray(predictions), task=task)
 
@@ -230,8 +318,9 @@ def score_champion(
 def to_challenger_upload(
     result: TrainedChallenger,
     *,
-    n_samples: int,
+    n_samples: int | None = None,
     champion_metrics: dict[str, float] | None = None,
+    champion_labels: np.ndarray | list | None = None,
     sdk_version: str | None = None,
     proxyml_core_version: str | None = None,
 ) -> dict[str, Any]:
@@ -245,27 +334,42 @@ def to_challenger_upload(
     ``json.dump`` — upload it either by POSTing it directly, or by saving it
     to a file and using the dashboard's "Upload challenger" button.
 
-    ``champion_metrics`` is optional: pass ``None`` (the default) to get a
-    self-contained export of the challenger alone — e.g. to save/share it
-    before you have a champion to compare against — and fill in
-    ``champion_metrics`` later. The upload endpoint itself still requires
-    ``champion_metrics`` at upload time; this function just doesn't force you
-    to have it up front.
+    ``champion_metrics`` is optional: pass ``None`` (the default) to fall
+    back to ``result.champion_metrics`` (populated automatically if you
+    passed ``champion_predictions`` to ``train_challenger()``/
+    ``train_auto_challenger()``) — or, if that's also ``None``, to get a
+    self-contained export of the challenger alone, e.g. to save/share it
+    before you have a champion to compare against. The upload endpoint
+    itself still requires ``champion_metrics`` at upload time; this function
+    just doesn't force you to have it up front.
 
     Args:
         result: output of ``train_challenger()``/``train_auto_challenger()``.
         n_samples: size of the evaluation set both ``result.metrics`` and
-            ``champion_metrics`` were scored on. Not derived automatically —
-            ``TrainedChallenger`` doesn't retain its internal held-out split,
-            and ``champion_metrics`` typically comes from a separate
-            ``score_champion()`` call the two need to share, so the caller is
-            the only one who actually knows this number.
+            ``champion_metrics`` were scored on. Defaults to
+            ``result.n_samples_total - result.n_samples_dropped_unlabeled``
+            (the labeled-row count) — override only if you scored on some
+            other population.
         champion_metrics: the champion's real-world performance, from
             ``score_champion()`` — same metric keys as ``result.metrics``.
-            Omit if you don't have it yet.
+            Defaults to ``result.champion_metrics``.
+        champion_labels: the ``labels`` array you passed to a standalone
+            ``score_champion()`` call, if ``champion_metrics`` didn't come
+            from ``train_challenger()``'s internal ``champion_predictions=``
+            path. Used only to compute ``champion_data_fingerprint`` — the
+            labels themselves are never included in the payload. If
+            ``champion_metrics`` resolves from ``result.champion_metrics``
+            instead, the fingerprint defaults to ``result.target_fingerprint``
+            (guaranteed identical, since that internal path scores against
+            the exact same data).
         sdk_version: defaults to the installed ``proxyml`` version.
         proxyml_core_version: defaults to the installed ``proxyml-core`` version.
     """
+    if n_samples is None:
+        n_samples = result.n_samples_total - result.n_samples_dropped_unlabeled
+    used_internal_champion_metrics = champion_metrics is None
+    if champion_metrics is None:
+        champion_metrics = result.champion_metrics
     if sdk_version is None:
         sdk_version = _pkg_version("proxyml")
     if proxyml_core_version is None:
@@ -275,12 +379,20 @@ def to_challenger_upload(
         "export": result.export.to_dict(),
         "challenger_metrics": result.metrics,
         "n_samples": n_samples,
+        "n_samples_total": result.n_samples_total,
+        "n_samples_dropped_unlabeled": result.n_samples_dropped_unlabeled,
+        "population_note": result.population_note,
         "complexity": result.complexity.value,
         "sdk_version": sdk_version,
         "proxyml_core_version": proxyml_core_version,
     }
     if champion_metrics is not None:
         payload["champion_metrics"] = champion_metrics
+        payload["challenger_data_fingerprint"] = result.target_fingerprint
+        if champion_labels is not None:
+            payload["champion_data_fingerprint"] = _fingerprint_values(champion_labels)
+        elif used_internal_champion_metrics:
+            payload["champion_data_fingerprint"] = result.target_fingerprint
     return payload
 
 
@@ -293,6 +405,7 @@ def train_auto_challenger(
     feature_names: list[str] | None = None,
     task: Literal["classification", "regression", "auto"] = "auto",
     test_size: float = 0.2,
+    champion_predictions: np.ndarray | list | None = None,
 ) -> TrainedChallenger:
     """Load data, infer a schema, and train a linear challenger in one call.
 
@@ -302,6 +415,12 @@ def train_auto_challenger(
     remains overridable; this does not search across ``LADDERS`` to find the
     best-fitting rung.
 
+    Rows with a missing ``target_col`` value are dropped before training and
+    champion scoring — see ``train_challenger()`` for details. Schema
+    inference (feature means/stds/categories) still runs over every row,
+    including ones later dropped for a missing target — only training and
+    evaluation are restricted to the labeled subset.
+
     Args:
         data: a CSV path, or an already-loaded DataFrame containing both the
             feature columns and ``target_col``.
@@ -312,6 +431,8 @@ def train_auto_challenger(
         feature_names: subset of feature columns to train on; omit for all.
         task: "classification", "regression", or "auto" to infer from ``target_col``.
         test_size: fraction of data held out to compute fidelity metrics.
+        champion_predictions: a champion model's predictions, one per row of
+            ``data`` (same order) — see ``train_challenger()``.
     """
     df = data if isinstance(data, pd.DataFrame) else pd.read_csv(data)
     target = df[target_col]
@@ -326,4 +447,6 @@ def train_auto_challenger(
         feature_names=feature_names,
         task=task,
         test_size=test_size,
+        target_name=target_col,
+        champion_predictions=champion_predictions,
     )

{proxyml-0.5.0 → proxyml-0.7.0}/src/proxyml.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxyml
-Version: 0.5.0
+Version: 0.7.0
 Summary: Python SDK for calling the ProxyML API
 Author-email: ProxyML <contact@proxyml.ai>
 License:                                  Apache License

{proxyml-0.5.0 → proxyml-0.7.0}/tests/test_local_challenger.py

@@ -286,6 +286,108 @@ def test_to_challenger_upload_payload_is_json_serializable():
     json.dumps(payload)  # must not raise
 
 
+def _labeled_df_with_nan_target(n=200, n_nan=20, seed=20):
+    df = _labeled_df(n=n, seed=seed)
+    df["approved"] = df["approved"].astype(float)
+    df.loc[df.index[:n_nan], "approved"] = np.nan
+    return df
+
+
+def test_nan_target_rows_are_dropped_and_counted():
+    df = _labeled_df_with_nan_target(n=200, n_nan=20)
+    result = train_auto_challenger(df, "approved", task="classification")
+
+    assert result.n_samples_total == 200
+    assert result.n_samples_dropped_unlabeled == 20
+    assert "20" in result.population_note
+    assert "180" in result.population_note
+
+
+def test_no_nan_targets_reports_zero_dropped():
+    df = _labeled_df(seed=21)
+    result = train_auto_challenger(df, "approved", task="classification")
+
+    assert result.n_samples_total == len(df)
+    assert result.n_samples_dropped_unlabeled == 0
+    assert "no missing values" in result.population_note
+
+
+def test_all_nan_target_raises():
+    df = _labeled_df(n=20, seed=22)
+    df["approved"] = np.nan
+    with pytest.raises(ValueError, match="missing"):
+        train_auto_challenger(df, "approved", task="classification")
+
+
+def test_champion_predictions_wrong_length_raises():
+    df = _labeled_df(seed=23)
+    with pytest.raises(ValueError, match="one entry per row"):
+        train_auto_challenger(
+            df, "approved", task="classification", champion_predictions=[True, False]
+        )
+
+
+def test_champion_predictions_scored_only_on_labeled_rows():
+    # Champion predictions mirror the (possibly-NaN) target itself, except on
+    # rows that get dropped as unlabeled, where they're deliberately wrong.
+    # If those rows leaked into scoring, champion accuracy would come in
+    # under 1.0 instead of exactly 1.0 — proving the shared-drop guarantee,
+    # not just that nothing crashes.
+    df = _labeled_df_with_nan_target(n=200, n_nan=20, seed=24)
+    champion_predictions = [False if pd.isna(v) else v for v in df["approved"]]
+
+    result = train_auto_challenger(
+        df, "approved", task="classification", champion_predictions=champion_predictions
+    )
+
+    assert result.champion_metrics is not None
+    assert result.champion_metrics["accuracy"] == 1.0
+
+
+def test_champion_predictions_not_given_leaves_champion_metrics_none():
+    df = _labeled_df(seed=25)
+    result = train_auto_challenger(df, "approved", task="classification")
+    assert result.champion_metrics is None
+
+
+def test_to_challenger_upload_defaults_n_samples_from_result():
+    df = _labeled_df_with_nan_target(n=200, n_nan=20, seed=26)
+    result = train_auto_challenger(df, "approved", task="classification")
+
+    payload = to_challenger_upload(result)
+
+    assert payload["n_samples"] == 180
+    assert payload["n_samples_total"] == 200
+    assert payload["n_samples_dropped_unlabeled"] == 20
+    assert payload["population_note"] == result.population_note
+
+
+def test_to_challenger_upload_defaults_champion_metrics_from_result():
+    df = _labeled_df(seed=27)
+    champion_predictions = df["approved"].tolist()
+    result = train_auto_challenger(
+        df, "approved", task="classification", champion_predictions=champion_predictions
+    )
+
+    payload = to_challenger_upload(result)
+
+    assert payload["champion_metrics"] == result.champion_metrics
+
+
+def test_to_challenger_upload_explicit_args_override_result_defaults():
+    df = _labeled_df(seed=28)
+    champion_predictions = df["approved"].tolist()
+    result = train_auto_challenger(
+        df, "approved", task="classification", champion_predictions=champion_predictions
+    )
+
+    override_metrics = {"f1": 0.1, "accuracy": 0.1}
+    payload = to_challenger_upload(result, n_samples=999, champion_metrics=override_metrics)
+
+    assert payload["n_samples"] == 999
+    assert payload["champion_metrics"] == override_metrics
+
+
 def test_train_auto_challenger_passes_immutable_cols_to_get_schema():
     from unittest.mock import patch
 
@@ -300,3 +402,81 @@ def test_train_auto_challenger_passes_immutable_cols_to_get_schema():
     called_kwargs = mock_get_schema.call_args.kwargs
     assert list(called_df.columns) == list(features_df.columns)
     assert called_kwargs["immutable_cols"] == ["age"]
+
+
+def test_target_fingerprint_is_deterministic_for_identical_data():
+    df = _labeled_df(seed=29)
+    result_a = train_auto_challenger(df, "approved", task="classification")
+    result_b = train_auto_challenger(df, "approved", task="classification")
+
+    assert result_a.target_fingerprint == result_b.target_fingerprint
+
+
+def test_target_fingerprint_differs_for_different_data():
+    df_a = _labeled_df(seed=29)
+    df_b = _labeled_df(seed=30)
+    result_a = train_auto_challenger(df_a, "approved", task="classification")
+    result_b = train_auto_challenger(df_b, "approved", task="classification")
+
+    assert result_a.target_fingerprint != result_b.target_fingerprint
+
+
+def test_to_challenger_upload_includes_matching_fingerprints_on_internal_champion_path():
+    df = _labeled_df(seed=31)
+    champion_predictions = df["approved"].tolist()
+    result = train_auto_challenger(
+        df, "approved", task="classification", champion_predictions=champion_predictions
+    )
+
+    payload = to_challenger_upload(result)
+
+    assert payload["challenger_data_fingerprint"] == result.target_fingerprint
+    assert payload["champion_data_fingerprint"] == result.target_fingerprint
+
+
+def test_to_challenger_upload_champion_labels_fingerprint_for_decoupled_path():
+    df = _labeled_df(seed=32)
+    target = df["approved"]
+    result = train_challenger(df, target, _schema(), task="classification")
+    champion_metrics = score_champion(target, target, task="classification")
+
+    payload = to_challenger_upload(result, champion_metrics=champion_metrics, champion_labels=target)
+
+    assert payload["challenger_data_fingerprint"] == result.target_fingerprint
+    assert payload["champion_data_fingerprint"] == result.target_fingerprint
+
+
+def test_to_challenger_upload_champion_labels_fingerprint_differs_for_different_data():
+    df = _labeled_df(seed=33)
+    target = df["approved"]
+    result = train_challenger(df, target, _schema(), task="classification")
+    champion_metrics = score_champion(target, target, task="classification")
+    other_labels = ~target
+
+    payload = to_challenger_upload(
+        result, champion_metrics=champion_metrics, champion_labels=other_labels
+    )
+
+    assert payload["challenger_data_fingerprint"] != payload["champion_data_fingerprint"]
+
+
+def test_to_challenger_upload_omits_champion_fingerprint_without_labels_or_internal_path():
+    df = _labeled_df(seed=34)
+    target = df["approved"]
+    result = train_challenger(df, target, _schema(), task="classification")
+    champion_metrics = score_champion(target, target, task="classification")
+
+    payload = to_challenger_upload(result, champion_metrics=champion_metrics)
+
+    assert payload["challenger_data_fingerprint"] == result.target_fingerprint
+    assert "champion_data_fingerprint" not in payload
+
+
+def test_to_challenger_upload_without_champion_metrics_omits_fingerprints():
+    df = _labeled_df(seed=35)
+    result = train_auto_challenger(df, "approved", task="classification")
+
+    payload = to_challenger_upload(result)
+
+    assert "challenger_data_fingerprint" not in payload
+    assert "champion_data_fingerprint" not in payload