spforge 0.8.16__tar.gz → 0.8.17__tar.gz

{spforge-0.8.16/spforge.egg-info → spforge-0.8.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spforge
-Version: 0.8.16
+Version: 0.8.17
 Summary: A flexible framework for generating features, ratings, and building machine learning or other models for training and inference on sports data.
 Author-email: Mathias Holmstrøm <mathiasholmstom@gmail.com>
 License: See LICENSE file

{spforge-0.8.16 → spforge-0.8.17}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spforge"
-version = "0.8.16"
+version = "0.8.17"
 description = "A flexible framework for generating features, ratings, and building machine learning or other models for training and inference on sports data."
 readme = "README.md"
 requires-python = ">=3.11"

{spforge-0.8.16 → spforge-0.8.17}/spforge/scorer/_score.py

@@ -366,18 +366,49 @@ class PWMSE(BaseScorer):
         self.labels = labels
         self.evaluation_labels = evaluation_labels
 
+        self._needs_extension = False
+        self._needs_slicing = False
         self._eval_indices: list[int] | None = None
+        self._extension_mapping: dict[int, int] | None = None
+
         if self.evaluation_labels is not None and self.labels is not None:
-            label_to_idx = {lbl: i for i, lbl in enumerate(self.labels)}
-            self._eval_indices = [label_to_idx[lbl] for lbl in self.evaluation_labels]
+            training_set = set(self.labels)
+            eval_set = set(self.evaluation_labels)
+
+            if eval_set <= training_set:
+                self._needs_slicing = True
+                label_to_idx = {lbl: i for i, lbl in enumerate(self.labels)}
+                self._eval_indices = [label_to_idx[lbl] for lbl in self.evaluation_labels]
+            elif training_set <= eval_set:
+                self._needs_extension = True
+                eval_label_to_idx = {lbl: i for i, lbl in enumerate(self.evaluation_labels)}
+                self._extension_mapping = {
+                    train_idx: eval_label_to_idx[lbl]
+                    for train_idx, lbl in enumerate(self.labels)
+                }
+            else:
+                raise ValueError(
+                    f"evaluation_labels must be a subset or superset of labels. "
+                    f"labels={self.labels}, evaluation_labels={self.evaluation_labels}"
+                )
+
+    def _align_predictions(self, preds: np.ndarray) -> np.ndarray:
+        if self._needs_slicing and self._eval_indices is not None:
+            sliced = preds[:, self._eval_indices]
+            row_sums = sliced.sum(axis=1, keepdims=True)
+            row_sums = np.where(row_sums == 0, 1.0, row_sums)
+            return sliced / row_sums
+
+        if self._needs_extension and self._extension_mapping is not None:
+            n_samples = preds.shape[0]
+            n_eval_labels = len(self.evaluation_labels)
+            extended = np.full((n_samples, n_eval_labels), 1e-5, dtype=np.float64)
+            for train_idx, eval_idx in self._extension_mapping.items():
+                extended[:, eval_idx] = preds[:, train_idx]
+            row_sums = extended.sum(axis=1, keepdims=True)
+            return extended / row_sums
 
-    def _slice_and_renormalize(self, preds: np.ndarray) -> np.ndarray:
-        if self._eval_indices is None:
-            return preds
-        sliced = preds[:, self._eval_indices]
-        row_sums = sliced.sum(axis=1, keepdims=True)
-        row_sums = np.where(row_sums == 0, 1.0, row_sums)
-        return sliced / row_sums
+        return preds
 
     def _get_scoring_labels(self) -> list[int]:
         if self.evaluation_labels is not None:
@@ -446,7 +477,7 @@ class PWMSE(BaseScorer):
 
                 targets = gran_df[self.target].to_numpy().astype(np.float64)
                 preds = np.asarray(gran_df[self.pred_column].to_list(), dtype=np.float64)
-                preds = self._slice_and_renormalize(preds)
+                preds = self._align_predictions(preds)
                 score = self._pwmse_score(targets, preds)
                 if self.compare_to_naive:
                     naive_probs_list = _naive_probability_predictions_for_df(
@@ -464,7 +495,7 @@ class PWMSE(BaseScorer):
 
         targets = df[self.target].to_numpy().astype(np.float64)
         preds = np.asarray(df[self.pred_column].to_list(), dtype=np.float64)
-        preds = self._slice_and_renormalize(preds)
+        preds = self._align_predictions(preds)
         score = self._pwmse_score(targets, preds)
         if self.compare_to_naive:
             naive_probs_list = _naive_probability_predictions_for_df(

{spforge-0.8.16 → spforge-0.8.17/spforge.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spforge
-Version: 0.8.16
+Version: 0.8.17
 Summary: A flexible framework for generating features, ratings, and building machine learning or other models for training and inference on sports data.
 Author-email: Mathias Holmstrøm <mathiasholmstom@gmail.com>
 License: See LICENSE file

{spforge-0.8.16 → spforge-0.8.17}/tests/scorer/test_score.py

@@ -2138,3 +2138,145 @@ def test_scorers_respect_validation_column(scorer_factory, df_factory):
     score_all = scorer_factory().score(df)
     score_valid = scorer_factory().score(df_valid)
     assert score_all == score_valid
+
+
+# ============================================================================
+# PWMSE evaluation_labels Extension Tests
+# ============================================================================
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_extends_predictions(df_type):
+    """PWMSE with evaluation_labels as superset extends predictions with small probs."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.3, 0.5, 0.2],
+                [0.2, 0.6, 0.2],
+            ],
+            "target": [0, 1],
+        },
+    )
+
+    scorer = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+        evaluation_labels=[-1, 0, 1, 2, 3],
+    )
+    score = scorer.score(df)
+
+    n_eval_labels = 5
+    eps = 1e-5
+    preds_original = np.array([[0.3, 0.5, 0.2], [0.2, 0.6, 0.2]])
+    extended = np.full((2, n_eval_labels), eps, dtype=np.float64)
+    extended[:, 1] = preds_original[:, 0]
+    extended[:, 2] = preds_original[:, 1]
+    extended[:, 3] = preds_original[:, 2]
+    row_sums = extended.sum(axis=1, keepdims=True)
+    preds_renorm = extended / row_sums
+
+    eval_labels = np.array([-1, 0, 1, 2, 3], dtype=np.float64)
+    targets = np.array([0, 1], dtype=np.float64)
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    expected = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    assert abs(score - expected) < 1e-10
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_exact_match(df_type):
+    """PWMSE with evaluation_labels identical to labels (no-op)."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.3, 0.5, 0.2],
+                [0.2, 0.6, 0.2],
+            ],
+            "target": [0, 1],
+        },
+    )
+
+    scorer_with_eval = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+        evaluation_labels=[0, 1, 2],
+    )
+    scorer_without_eval = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+    )
+
+    score_with = scorer_with_eval.score(df)
+    score_without = scorer_without_eval.score(df)
+
+    assert abs(score_with - score_without) < 1e-10
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_partial_overlap_raises(df_type):
+    """PWMSE with partial overlap between labels and evaluation_labels raises."""
+    with pytest.raises(ValueError, match="evaluation_labels must be a subset or superset"):
+        PWMSE(
+            pred_column="pred",
+            target="target",
+            labels=[0, 1, 2],
+            evaluation_labels=[1, 2, 3],
+        )
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_extends_with_compare_to_naive(df_type):
+    """PWMSE extension mode works correctly with compare_to_naive."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.8, 0.15, 0.05],
+                [0.1, 0.7, 0.2],
+                [0.05, 0.15, 0.8],
+                [0.3, 0.4, 0.3],
+            ],
+            "target": [0, 1, 2, 1],
+        },
+    )
+
+    scorer = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+        evaluation_labels=[-1, 0, 1, 2, 3],
+        compare_to_naive=True,
+    )
+    score = scorer.score(df)
+
+    n_eval_labels = 5
+    eps = 1e-5
+    preds_original = np.array([
+        [0.8, 0.15, 0.05],
+        [0.1, 0.7, 0.2],
+        [0.05, 0.15, 0.8],
+        [0.3, 0.4, 0.3],
+    ])
+    extended = np.full((4, n_eval_labels), eps, dtype=np.float64)
+    extended[:, 1] = preds_original[:, 0]
+    extended[:, 2] = preds_original[:, 1]
+    extended[:, 3] = preds_original[:, 2]
+    row_sums = extended.sum(axis=1, keepdims=True)
+    preds_renorm = extended / row_sums
+
+    eval_labels = np.array([-1, 0, 1, 2, 3], dtype=np.float64)
+    targets = np.array([0, 1, 2, 1], dtype=np.float64)
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    model_score = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    naive_probs = np.array([0.0, 0.25, 0.5, 0.25, 0.0])
+    naive_preds = np.tile(naive_probs, (4, 1))
+    naive_score = float((diffs_sqd * naive_preds).sum(axis=1).mean())
+
+    expected = naive_score - model_score
+    assert abs(score - expected) < 1e-10