proxyml 0.4.1__tar.gz → 0.6.0__tar.gz

{proxyml-0.4.1 → proxyml-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxyml
-Version: 0.4.1
+Version: 0.6.0
 Summary: Python SDK for calling the ProxyML API
 Author-email: ProxyML <contact@proxyml.ai>
 License:                                  Apache License

{proxyml-0.4.1 → proxyml-0.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "proxyml"
-version = "0.4.1"
+version = "0.6.0"
 description = "Python SDK for calling the ProxyML API"
 readme = "README.md"
 license = {file = "LICENSE"}

{proxyml-0.4.1 → proxyml-0.6.0}/src/proxyml/local/__init__.py

@@ -11,6 +11,7 @@ from proxyml.local.challenger import (
     Rung,
     TrainedChallenger,
     score_champion,
+    to_challenger_upload,
     train_auto_challenger,
     train_challenger,
 )
@@ -19,6 +20,7 @@ __all__ = [
     "train_challenger",
     "train_auto_challenger",
     "score_champion",
+    "to_challenger_upload",
     "Complexity",
     "Rung",
     "TrainedChallenger",

{proxyml-0.4.1 → proxyml-0.6.0}/src/proxyml/local/challenger.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 from dataclasses import dataclass, replace
 from enum import Enum
+from importlib.metadata import version as _pkg_version
 from pathlib import Path
 from typing import Any, Callable, Literal
 
@@ -113,9 +114,26 @@ LADDERS: dict[Complexity, Rung] = {
 class TrainedChallenger:
     pipeline: Pipeline
     task: Literal["classification", "regression"]
+    complexity: Complexity
     metrics: dict[str, float]
     hyperparameters: dict[str, Any]
     export: SurrogateExport
+    n_samples_total: int
+    n_samples_dropped_unlabeled: int
+    population_note: str
+    champion_metrics: dict[str, float] | None = None
+
+
+def _population_note(target_name: str, n_total: int, n_labeled: int, n_dropped: int) -> str:
+    if n_dropped == 0:
+        return f"Evaluated on all {n_total} row(s) — '{target_name}' had no missing values."
+    return (
+        f"Evaluated on {n_labeled} of {n_total} row(s) with a non-null '{target_name}' value "
+        f"({n_dropped} unlabeled row(s) dropped before training/scoring). "
+        "Labeled-vs-unlabeled selection may not be random; treat this as a declared "
+        "scope limitation on the evaluation population, not a claim about performance "
+        "on the full dataset."
+    )
 
 
 def train_challenger(
@@ -127,6 +145,8 @@ def train_challenger(
     feature_names: list[str] | None = None,
     task: Literal["classification", "regression", "auto"] = "auto",
     test_size: float = 0.2,
+    target_name: str = "target",
+    champion_predictions: np.ndarray | list | None = None,
 ) -> TrainedChallenger:
     """Train a linear challenger model on ``df`` against ``target``, locally.
 
@@ -140,6 +160,15 @@ def train_challenger(
     can be compared with the same ``proxyml_core.export.predict_from_export``
     arithmetic.
 
+    Rows where ``target`` is missing (NaN/None) are dropped before training,
+    the CV split, and champion scoring — never silently included. The drop
+    count and a human-readable scope-limitation note are recorded on the
+    result (``n_samples_total``, ``n_samples_dropped_unlabeled``,
+    ``population_note``). If ``champion_predictions`` is given, it must have
+    one entry per row of ``df``/``target`` (same order) so the identical rows
+    are dropped from both sides — champion and challenger are always
+    evaluated on the same labeled population, never on different ones.
+
     Args:
         df: samples to train on, one column per schema feature.
         target: the value to predict for each row of ``df`` — ground-truth
@@ -149,7 +178,35 @@ def train_challenger(
         feature_names: subset of ``schema.features`` to train on; omit for all.
         task: "classification", "regression", or "auto" to infer from ``target``.
         test_size: fraction of data held out to compute fidelity metrics.
+        target_name: human-readable name for ``target``, used in
+            ``population_note`` (e.g. the column name, if known).
+        champion_predictions: a champion model's predictions, one per row of
+            ``df``/``target`` (same order). If given, scored via
+            ``score_champion()`` against the same (row-dropped) ``target``,
+            and the result is attached as ``TrainedChallenger.champion_metrics``.
     """
+    target_arr = np.asarray(target)
+    if champion_predictions is not None and len(champion_predictions) != len(target_arr):
+        raise ValueError(
+            f"champion_predictions must have one entry per row of target "
+            f"({len(target_arr)} rows, got {len(champion_predictions)}) — same order — so "
+            f"rows with a missing {target_name!r} value can be dropped from both the "
+            f"challenger and the champion, keeping them evaluated on the same population."
+        )
+
+    labeled_mask = ~pd.isna(target_arr)
+    n_total = len(target_arr)
+    n_labeled = int(labeled_mask.sum())
+    n_dropped = n_total - n_labeled
+    if n_labeled == 0:
+        raise ValueError(f"All {n_total} row(s) have a missing {target_name!r} value; nothing to train on")
+
+    df = df.iloc[labeled_mask].reset_index(drop=True)
+    target_arr = target_arr[labeled_mask]
+    champion_predictions_labeled = (
+        np.asarray(champion_predictions)[labeled_mask] if champion_predictions is not None else None
+    )
+
     rung = LADDERS[complexity]
 
     features: list[Feature] = schema.features
@@ -159,7 +216,7 @@ def train_challenger(
     col_order = [f.name for f in features]
 
     X = df[col_order].to_numpy(dtype=object)
-    y = np.asarray(target)
+    y = target_arr
 
     if task == "auto":
         classification = is_classification(y)
@@ -172,6 +229,10 @@ def train_challenger(
     if classification:
         y = binarize_if_probabilities(y)
 
+    champion_metrics = None
+    if champion_predictions_labeled is not None:
+        champion_metrics = score_champion(y, champion_predictions_labeled, task=resolved_task)
+
     preprocessor = build_preprocessor(features)
     estimator = rung.build_classifier() if classification else rung.build_regressor()
     pipeline = Pipeline(steps=[("preprocessor", preprocessor), ("estimator", estimator)])
@@ -196,9 +257,14 @@ def train_challenger(
     return TrainedChallenger(
         pipeline=pipeline,
         task=resolved_task,
+        complexity=complexity,
         metrics=metrics,
         hyperparameters=hyperparameters,
         export=export,
+        n_samples_total=n_total,
+        n_samples_dropped_unlabeled=n_dropped,
+        population_note=_population_note(target_name, n_total, n_labeled, n_dropped),
+        champion_metrics=champion_metrics,
     )
 
 
@@ -224,6 +290,71 @@ def score_champion(
     return score_predictions(np.asarray(labels), np.asarray(predictions), task=task)
 
 
+def to_challenger_upload(
+    result: TrainedChallenger,
+    *,
+    n_samples: int | None = None,
+    champion_metrics: dict[str, float] | None = None,
+    sdk_version: str | None = None,
+    proxyml_core_version: str | None = None,
+) -> dict[str, Any]:
+    """Assemble the JSON-serializable payload for a challenger upload.
+
+    Matches the shape ProxyML's dashboard/API expects at
+    ``POST /app/projects/{id}/challenger`` — handles the mechanical assembly
+    (serializing the export, stamping SDK/core versions, converting
+    ``complexity`` to a plain string) so callers don't have to hand-roll it.
+    The result is plain ``dict``/``str``/``float`` data, ready for
+    ``json.dump`` — upload it either by POSTing it directly, or by saving it
+    to a file and using the dashboard's "Upload challenger" button.
+
+    ``champion_metrics`` is optional: pass ``None`` (the default) to fall
+    back to ``result.champion_metrics`` (populated automatically if you
+    passed ``champion_predictions`` to ``train_challenger()``/
+    ``train_auto_challenger()``) — or, if that's also ``None``, to get a
+    self-contained export of the challenger alone, e.g. to save/share it
+    before you have a champion to compare against. The upload endpoint
+    itself still requires ``champion_metrics`` at upload time; this function
+    just doesn't force you to have it up front.
+
+    Args:
+        result: output of ``train_challenger()``/``train_auto_challenger()``.
+        n_samples: size of the evaluation set both ``result.metrics`` and
+            ``champion_metrics`` were scored on. Defaults to
+            ``result.n_samples_total - result.n_samples_dropped_unlabeled``
+            (the labeled-row count) — override only if you scored on some
+            other population.
+        champion_metrics: the champion's real-world performance, from
+            ``score_champion()`` — same metric keys as ``result.metrics``.
+            Defaults to ``result.champion_metrics``.
+        sdk_version: defaults to the installed ``proxyml`` version.
+        proxyml_core_version: defaults to the installed ``proxyml-core`` version.
+    """
+    if n_samples is None:
+        n_samples = result.n_samples_total - result.n_samples_dropped_unlabeled
+    if champion_metrics is None:
+        champion_metrics = result.champion_metrics
+    if sdk_version is None:
+        sdk_version = _pkg_version("proxyml")
+    if proxyml_core_version is None:
+        proxyml_core_version = _pkg_version("proxyml-core")
+
+    payload: dict[str, Any] = {
+        "export": result.export.to_dict(),
+        "challenger_metrics": result.metrics,
+        "n_samples": n_samples,
+        "n_samples_total": result.n_samples_total,
+        "n_samples_dropped_unlabeled": result.n_samples_dropped_unlabeled,
+        "population_note": result.population_note,
+        "complexity": result.complexity.value,
+        "sdk_version": sdk_version,
+        "proxyml_core_version": proxyml_core_version,
+    }
+    if champion_metrics is not None:
+        payload["champion_metrics"] = champion_metrics
+    return payload
+
+
 def train_auto_challenger(
     data: str | Path | pd.DataFrame,
     target_col: str,
@@ -233,6 +364,7 @@ def train_auto_challenger(
     feature_names: list[str] | None = None,
     task: Literal["classification", "regression", "auto"] = "auto",
     test_size: float = 0.2,
+    champion_predictions: np.ndarray | list | None = None,
 ) -> TrainedChallenger:
     """Load data, infer a schema, and train a linear challenger in one call.
 
@@ -242,6 +374,12 @@ def train_auto_challenger(
     remains overridable; this does not search across ``LADDERS`` to find the
     best-fitting rung.
 
+    Rows with a missing ``target_col`` value are dropped before training and
+    champion scoring — see ``train_challenger()`` for details. Schema
+    inference (feature means/stds/categories) still runs over every row,
+    including ones later dropped for a missing target — only training and
+    evaluation are restricted to the labeled subset.
+
     Args:
         data: a CSV path, or an already-loaded DataFrame containing both the
             feature columns and ``target_col``.
@@ -252,6 +390,8 @@ def train_auto_challenger(
         feature_names: subset of feature columns to train on; omit for all.
         task: "classification", "regression", or "auto" to infer from ``target_col``.
         test_size: fraction of data held out to compute fidelity metrics.
+        champion_predictions: a champion model's predictions, one per row of
+            ``data`` (same order) — see ``train_challenger()``.
     """
     df = data if isinstance(data, pd.DataFrame) else pd.read_csv(data)
     target = df[target_col]
@@ -266,4 +406,6 @@ def train_auto_challenger(
         feature_names=feature_names,
         task=task,
         test_size=test_size,
+        target_name=target_col,
+        champion_predictions=champion_predictions,
     )

{proxyml-0.4.1 → proxyml-0.6.0}/src/proxyml.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxyml
-Version: 0.4.1
+Version: 0.6.0
 Summary: Python SDK for calling the ProxyML API
 Author-email: ProxyML <contact@proxyml.ai>
 License:                                  Apache License

{proxyml-0.4.1 → proxyml-0.6.0}/tests/test_local_challenger.py

@@ -9,6 +9,7 @@ from proxyml.local import (
     LADDERS,
     TrainedChallenger,
     score_champion,
+    to_challenger_upload,
     train_auto_challenger,
     train_challenger,
 )
@@ -197,6 +198,196 @@ def test_score_champion_uses_same_scoring_as_train_challenger():
     assert reproduced_metrics["r2"] == pytest.approx(r2_score(target, y_pred))
 
 
+def test_train_challenger_result_carries_complexity():
+    schema = _schema()
+    df = _df(seed=13)
+    target = df["age"] * 0.5 + df["income"] * 0.0001
+
+    result = train_challenger(df, target, schema, complexity=Complexity.FLEXIBLE, task="regression")
+
+    assert result.complexity is Complexity.FLEXIBLE
+
+
+def test_train_auto_challenger_result_carries_complexity():
+    df = _labeled_df(seed=14)
+    result = train_auto_challenger(df, "approved", task="classification", complexity=Complexity.SIMPLE)
+
+    assert result.complexity is Complexity.SIMPLE
+
+
+def test_to_challenger_upload_shape_without_champion_metrics():
+    schema = _schema()
+    df = _df(seed=15)
+    target = df["age"] * 0.5 + df["income"] * 0.0001
+    result = train_challenger(df, target, schema, complexity=Complexity.MODERATE, task="regression")
+
+    payload = to_challenger_upload(result, n_samples=40)
+
+    assert payload["export"] == result.export.to_dict()
+    assert payload["challenger_metrics"] == result.metrics
+    assert payload["n_samples"] == 40
+    assert payload["complexity"] == "moderate"
+    assert "champion_metrics" not in payload
+    assert isinstance(payload["sdk_version"], str) and payload["sdk_version"]
+    assert isinstance(payload["proxyml_core_version"], str) and payload["proxyml_core_version"]
+
+
+def test_to_challenger_upload_includes_champion_metrics_when_given():
+    schema = _schema()
+    df = _df(seed=16)
+    target = df["age"] * 0.5 + df["income"] * 0.0001
+    result = train_challenger(df, target, schema, complexity=Complexity.MODERATE, task="regression")
+    champion_metrics = score_champion(target, target, task="regression")
+
+    payload = to_challenger_upload(result, n_samples=40, champion_metrics=champion_metrics)
+
+    assert payload["champion_metrics"] == champion_metrics
+
+
+def test_to_challenger_upload_defaults_versions_to_installed_packages():
+    from importlib.metadata import version as pkg_version
+
+    schema = _schema()
+    df = _df(seed=17)
+    target = df["age"] * 0.5 + df["income"] * 0.0001
+    result = train_challenger(df, target, schema, task="regression")
+
+    payload = to_challenger_upload(result, n_samples=10)
+
+    assert payload["sdk_version"] == pkg_version("proxyml")
+    assert payload["proxyml_core_version"] == pkg_version("proxyml-core")
+
+
+def test_to_challenger_upload_allows_version_override():
+    schema = _schema()
+    df = _df(seed=18)
+    target = df["age"] * 0.5 + df["income"] * 0.0001
+    result = train_challenger(df, target, schema, task="regression")
+
+    payload = to_challenger_upload(
+        result, n_samples=10, sdk_version="9.9.9", proxyml_core_version="8.8.8"
+    )
+
+    assert payload["sdk_version"] == "9.9.9"
+    assert payload["proxyml_core_version"] == "8.8.8"
+
+
+def test_to_challenger_upload_payload_is_json_serializable():
+    import json
+
+    schema = _schema()
+    df = _df(seed=19, n=300)
+    target = np.where(df["age"] > 50, "senior", "junior")
+    result = train_challenger(df, target, schema, task="classification")
+    champion_metrics = score_champion(target, target, task="classification")
+
+    payload = to_challenger_upload(result, n_samples=300, champion_metrics=champion_metrics)
+
+    json.dumps(payload)  # must not raise
+
+
+def _labeled_df_with_nan_target(n=200, n_nan=20, seed=20):
+    df = _labeled_df(n=n, seed=seed)
+    df["approved"] = df["approved"].astype(float)
+    df.loc[df.index[:n_nan], "approved"] = np.nan
+    return df
+
+
+def test_nan_target_rows_are_dropped_and_counted():
+    df = _labeled_df_with_nan_target(n=200, n_nan=20)
+    result = train_auto_challenger(df, "approved", task="classification")
+
+    assert result.n_samples_total == 200
+    assert result.n_samples_dropped_unlabeled == 20
+    assert "20" in result.population_note
+    assert "180" in result.population_note
+
+
+def test_no_nan_targets_reports_zero_dropped():
+    df = _labeled_df(seed=21)
+    result = train_auto_challenger(df, "approved", task="classification")
+
+    assert result.n_samples_total == len(df)
+    assert result.n_samples_dropped_unlabeled == 0
+    assert "no missing values" in result.population_note
+
+
+def test_all_nan_target_raises():
+    df = _labeled_df(n=20, seed=22)
+    df["approved"] = np.nan
+    with pytest.raises(ValueError, match="missing"):
+        train_auto_challenger(df, "approved", task="classification")
+
+
+def test_champion_predictions_wrong_length_raises():
+    df = _labeled_df(seed=23)
+    with pytest.raises(ValueError, match="one entry per row"):
+        train_auto_challenger(
+            df, "approved", task="classification", champion_predictions=[True, False]
+        )
+
+
+def test_champion_predictions_scored_only_on_labeled_rows():
+    # Champion predictions mirror the (possibly-NaN) target itself, except on
+    # rows that get dropped as unlabeled, where they're deliberately wrong.
+    # If those rows leaked into scoring, champion accuracy would come in
+    # under 1.0 instead of exactly 1.0 — proving the shared-drop guarantee,
+    # not just that nothing crashes.
+    df = _labeled_df_with_nan_target(n=200, n_nan=20, seed=24)
+    champion_predictions = [False if pd.isna(v) else v for v in df["approved"]]
+
+    result = train_auto_challenger(
+        df, "approved", task="classification", champion_predictions=champion_predictions
+    )
+
+    assert result.champion_metrics is not None
+    assert result.champion_metrics["accuracy"] == 1.0
+
+
+def test_champion_predictions_not_given_leaves_champion_metrics_none():
+    df = _labeled_df(seed=25)
+    result = train_auto_challenger(df, "approved", task="classification")
+    assert result.champion_metrics is None
+
+
+def test_to_challenger_upload_defaults_n_samples_from_result():
+    df = _labeled_df_with_nan_target(n=200, n_nan=20, seed=26)
+    result = train_auto_challenger(df, "approved", task="classification")
+
+    payload = to_challenger_upload(result)
+
+    assert payload["n_samples"] == 180
+    assert payload["n_samples_total"] == 200
+    assert payload["n_samples_dropped_unlabeled"] == 20
+    assert payload["population_note"] == result.population_note
+
+
+def test_to_challenger_upload_defaults_champion_metrics_from_result():
+    df = _labeled_df(seed=27)
+    champion_predictions = df["approved"].tolist()
+    result = train_auto_challenger(
+        df, "approved", task="classification", champion_predictions=champion_predictions
+    )
+
+    payload = to_challenger_upload(result)
+
+    assert payload["champion_metrics"] == result.champion_metrics
+
+
+def test_to_challenger_upload_explicit_args_override_result_defaults():
+    df = _labeled_df(seed=28)
+    champion_predictions = df["approved"].tolist()
+    result = train_auto_challenger(
+        df, "approved", task="classification", champion_predictions=champion_predictions
+    )
+
+    override_metrics = {"f1": 0.1, "accuracy": 0.1}
+    payload = to_challenger_upload(result, n_samples=999, champion_metrics=override_metrics)
+
+    assert payload["n_samples"] == 999
+    assert payload["champion_metrics"] == override_metrics
+
+
 def test_train_auto_challenger_passes_immutable_cols_to_get_schema():
     from unittest.mock import patch