spforge 0.8.16__py3-none-any.whl → 0.8.17__py3-none-any.whl

spforge/scorer/_score.py

@@ -366,18 +366,49 @@ class PWMSE(BaseScorer):
         self.labels = labels
         self.evaluation_labels = evaluation_labels
 
+        self._needs_extension = False
+        self._needs_slicing = False
         self._eval_indices: list[int] | None = None
+        self._extension_mapping: dict[int, int] | None = None
+
         if self.evaluation_labels is not None and self.labels is not None:
-            label_to_idx = {lbl: i for i, lbl in enumerate(self.labels)}
-            self._eval_indices = [label_to_idx[lbl] for lbl in self.evaluation_labels]
+            training_set = set(self.labels)
+            eval_set = set(self.evaluation_labels)
+
+            if eval_set <= training_set:
+                self._needs_slicing = True
+                label_to_idx = {lbl: i for i, lbl in enumerate(self.labels)}
+                self._eval_indices = [label_to_idx[lbl] for lbl in self.evaluation_labels]
+            elif training_set <= eval_set:
+                self._needs_extension = True
+                eval_label_to_idx = {lbl: i for i, lbl in enumerate(self.evaluation_labels)}
+                self._extension_mapping = {
+                    train_idx: eval_label_to_idx[lbl]
+                    for train_idx, lbl in enumerate(self.labels)
+                }
+            else:
+                raise ValueError(
+                    f"evaluation_labels must be a subset or superset of labels. "
+                    f"labels={self.labels}, evaluation_labels={self.evaluation_labels}"
+                )
+
+    def _align_predictions(self, preds: np.ndarray) -> np.ndarray:
+        if self._needs_slicing and self._eval_indices is not None:
+            sliced = preds[:, self._eval_indices]
+            row_sums = sliced.sum(axis=1, keepdims=True)
+            row_sums = np.where(row_sums == 0, 1.0, row_sums)
+            return sliced / row_sums
+
+        if self._needs_extension and self._extension_mapping is not None:
+            n_samples = preds.shape[0]
+            n_eval_labels = len(self.evaluation_labels)
+            extended = np.full((n_samples, n_eval_labels), 1e-5, dtype=np.float64)
+            for train_idx, eval_idx in self._extension_mapping.items():
+                extended[:, eval_idx] = preds[:, train_idx]
+            row_sums = extended.sum(axis=1, keepdims=True)
+            return extended / row_sums
 
-    def _slice_and_renormalize(self, preds: np.ndarray) -> np.ndarray:
-        if self._eval_indices is None:
-            return preds
-        sliced = preds[:, self._eval_indices]
-        row_sums = sliced.sum(axis=1, keepdims=True)
-        row_sums = np.where(row_sums == 0, 1.0, row_sums)
-        return sliced / row_sums
+        return preds
 
     def _get_scoring_labels(self) -> list[int]:
         if self.evaluation_labels is not None:
@@ -446,7 +477,7 @@ class PWMSE(BaseScorer):
 
                 targets = gran_df[self.target].to_numpy().astype(np.float64)
                 preds = np.asarray(gran_df[self.pred_column].to_list(), dtype=np.float64)
-                preds = self._slice_and_renormalize(preds)
+                preds = self._align_predictions(preds)
                 score = self._pwmse_score(targets, preds)
                 if self.compare_to_naive:
                     naive_probs_list = _naive_probability_predictions_for_df(
@@ -464,7 +495,7 @@ class PWMSE(BaseScorer):
 
         targets = df[self.target].to_numpy().astype(np.float64)
         preds = np.asarray(df[self.pred_column].to_list(), dtype=np.float64)
-        preds = self._slice_and_renormalize(preds)
+        preds = self._align_predictions(preds)
         score = self._pwmse_score(targets, preds)
         if self.compare_to_naive:
             naive_probs_list = _naive_probability_predictions_for_df(

{spforge-0.8.16.dist-info → spforge-0.8.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spforge
-Version: 0.8.16
+Version: 0.8.17
 Summary: A flexible framework for generating features, ratings, and building machine learning or other models for training and inference on sports data.
 Author-email: Mathias Holmstrøm <mathiasholmstom@gmail.com>
 License: See LICENSE file

{spforge-0.8.16.dist-info → spforge-0.8.17.dist-info}/RECORD

@@ -62,7 +62,7 @@ spforge/ratings/team_performance_predictor.py,sha256=ThQOmYQUqKBB46ONYHOMM2arXFH
 spforge/ratings/team_start_rating_generator.py,sha256=vK-_m8KwcHopchch_lKNHSGLiiNm5q9Lenm0d1cP_po,5110
 spforge/ratings/utils.py,sha256=_zFemqz2jJkH8rn2EZpDt8N6FELUmYp9qCnPzRtOIGU,4497
 spforge/scorer/__init__.py,sha256=wj8PCvYIl6742Xwmt86c3oy6iqE8Ss-OpwHud6kd9IY,256
-spforge/scorer/_score.py,sha256=TR0T9nJj0aeVgGfOE0fZmXlO66CELulYwxhi7ZAxhvY,56184
+spforge/scorer/_score.py,sha256=kNuqiK3F5mUEAVD7KjWYY7E_AkRrspR362QBm_jyElg,57623
 spforge/transformers/__init__.py,sha256=IPCsMcsgBqG52d0ttATLCY4HvFCQZddExlLt74U-zuI,390
 spforge/transformers/_base.py,sha256=-smr_McQF9bYxM5-Agx6h7Xv_fhZzPfpAdQV-qK18bs,1134
 spforge/transformers/_net_over_predicted.py,sha256=5dC8pvA1DNO0yXPSgJSMGU8zAHi-maUELm7FqFQVo-U,2321
@@ -71,7 +71,7 @@ spforge/transformers/_other_transformer.py,sha256=xLfaFIhkFsigAoitB4x3F8An2j9ymd
 spforge/transformers/_predictor.py,sha256=2sE6gfVrilXzPVcBurSrtqHw33v2ljygQcEYXt9LhZc,3119
 spforge/transformers/_simple_transformer.py,sha256=zGUFNQYMeoDSa2CoQejQNiNmKCBN5amWTvyOchiUHj0,5660
 spforge/transformers/_team_ratio_predictor.py,sha256=g8_bR53Yyv0iNCtol1O9bgJSeZcIco_AfbQuUxQJkeY,6884
-spforge-0.8.16.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+spforge-0.8.17.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
 tests/test_autopipeline.py,sha256=g5SMTTolfRikHZfwIkExuoRjh-ldcr9-F-E1PUNpbpM,22923
 tests/test_autopipeline_context.py,sha256=IuRUY4IA6uMObvbl2pXSaXO2_tl3qX6wEbTZY0dkTMI,1240
 tests/test_feature_generator_pipeline.py,sha256=CK0zVL8PfTncy3RmG9i-YpgwjOIV7yJhV7Q44tbetI8,19020
@@ -99,7 +99,7 @@ tests/ratings/test_player_rating_no_mutation.py,sha256=GzO3Hl__5K68DS3uRLefwnbcT
 tests/ratings/test_ratings_property.py,sha256=ckyfGILXa4tfQvsgyXEzBDNr2DUmHwFRV13N60w66iE,6561
 tests/ratings/test_team_rating_generator.py,sha256=cDnf1zHiYC7pkgydE3MYr8wSTJIq-bPfSqhIRI_4Tic,95357
 tests/ratings/test_utils_scaled_weights.py,sha256=iHxe6ZDUB_I2B6HT0xTGqXBkl7gRlqVV0e_7Lwun5po,4988
-tests/scorer/test_score.py,sha256=_Vd6tKpy_1GeOxU7Omxci4CFf7PvRGMefEI0gv2gV6A,74688
+tests/scorer/test_score.py,sha256=rw3xJs6xqWVpalVMUQz557m2JYGR7PmhrsjfTex0b0c,79121
 tests/scorer/test_score_aggregation_granularity.py,sha256=h-hyFOLzwp-92hYVU7CwvlRJ8jhB4DzXCtqgI-zcoqM,13677
 tests/transformers/test_estimator_transformer_context.py,sha256=5GOHbuWCWBMFwwOTJOuD4oNDsv-qDR0OxNZYGGuMdag,1819
 tests/transformers/test_net_over_predicted.py,sha256=vh7O1iRRPf4vcW9aLhOMAOyatfM5ZnLsQBKNAYsR3SU,3363
@@ -107,7 +107,7 @@ tests/transformers/test_other_transformer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 tests/transformers/test_predictor_transformer.py,sha256=N1aBYLjN3ldpYZLwjih_gTFYSMitrZu-PNK78W6RHaQ,6877
 tests/transformers/test_simple_transformer.py,sha256=wWR0qjLb_uS4HXrJgGdiqugOY1X7kwd1_OPS02IT2b8,4676
 tests/transformers/test_team_ratio_predictor.py,sha256=fOUP_JvNJi-3kom3ZOs1EdG0I6Z8hpLpYKNHu1eWtOw,8562
-spforge-0.8.16.dist-info/METADATA,sha256=POeA0zCWQgYcTHLIvJwlqx8TAM4bL5ec_uTHTR5WoHA,20048
-spforge-0.8.16.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-spforge-0.8.16.dist-info/top_level.txt,sha256=6UW2M5a7WKOeaAi900qQmRKNj5-HZzE8-eUD9Y9LTq0,23
-spforge-0.8.16.dist-info/RECORD,,
+spforge-0.8.17.dist-info/METADATA,sha256=Zc4fLlCtPWuEFvs0DVRZre9OtTvRyVMgdmGV7-s68Ao,20048
+spforge-0.8.17.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+spforge-0.8.17.dist-info/top_level.txt,sha256=6UW2M5a7WKOeaAi900qQmRKNj5-HZzE8-eUD9Y9LTq0,23
+spforge-0.8.17.dist-info/RECORD,,

tests/scorer/test_score.py

@@ -2138,3 +2138,145 @@ def test_scorers_respect_validation_column(scorer_factory, df_factory):
     score_all = scorer_factory().score(df)
     score_valid = scorer_factory().score(df_valid)
     assert score_all == score_valid
+
+
+# ============================================================================
+# PWMSE evaluation_labels Extension Tests
+# ============================================================================
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_extends_predictions(df_type):
+    """PWMSE with evaluation_labels as superset extends predictions with small probs."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.3, 0.5, 0.2],
+                [0.2, 0.6, 0.2],
+            ],
+            "target": [0, 1],
+        },
+    )
+
+    scorer = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+        evaluation_labels=[-1, 0, 1, 2, 3],
+    )
+    score = scorer.score(df)
+
+    n_eval_labels = 5
+    eps = 1e-5
+    preds_original = np.array([[0.3, 0.5, 0.2], [0.2, 0.6, 0.2]])
+    extended = np.full((2, n_eval_labels), eps, dtype=np.float64)
+    extended[:, 1] = preds_original[:, 0]
+    extended[:, 2] = preds_original[:, 1]
+    extended[:, 3] = preds_original[:, 2]
+    row_sums = extended.sum(axis=1, keepdims=True)
+    preds_renorm = extended / row_sums
+
+    eval_labels = np.array([-1, 0, 1, 2, 3], dtype=np.float64)
+    targets = np.array([0, 1], dtype=np.float64)
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    expected = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    assert abs(score - expected) < 1e-10
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_exact_match(df_type):
+    """PWMSE with evaluation_labels identical to labels (no-op)."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.3, 0.5, 0.2],
+                [0.2, 0.6, 0.2],
+            ],
+            "target": [0, 1],
+        },
+    )
+
+    scorer_with_eval = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+        evaluation_labels=[0, 1, 2],
+    )
+    scorer_without_eval = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+    )
+
+    score_with = scorer_with_eval.score(df)
+    score_without = scorer_without_eval.score(df)
+
+    assert abs(score_with - score_without) < 1e-10
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_partial_overlap_raises(df_type):
+    """PWMSE with partial overlap between labels and evaluation_labels raises."""
+    with pytest.raises(ValueError, match="evaluation_labels must be a subset or superset"):
+        PWMSE(
+            pred_column="pred",
+            target="target",
+            labels=[0, 1, 2],
+            evaluation_labels=[1, 2, 3],
+        )
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_extends_with_compare_to_naive(df_type):
+    """PWMSE extension mode works correctly with compare_to_naive."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.8, 0.15, 0.05],
+                [0.1, 0.7, 0.2],
+                [0.05, 0.15, 0.8],
+                [0.3, 0.4, 0.3],
+            ],
+            "target": [0, 1, 2, 1],
+        },
+    )
+
+    scorer = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+        evaluation_labels=[-1, 0, 1, 2, 3],
+        compare_to_naive=True,
+    )
+    score = scorer.score(df)
+
+    n_eval_labels = 5
+    eps = 1e-5
+    preds_original = np.array([
+        [0.8, 0.15, 0.05],
+        [0.1, 0.7, 0.2],
+        [0.05, 0.15, 0.8],
+        [0.3, 0.4, 0.3],
+    ])
+    extended = np.full((4, n_eval_labels), eps, dtype=np.float64)
+    extended[:, 1] = preds_original[:, 0]
+    extended[:, 2] = preds_original[:, 1]
+    extended[:, 3] = preds_original[:, 2]
+    row_sums = extended.sum(axis=1, keepdims=True)
+    preds_renorm = extended / row_sums
+
+    eval_labels = np.array([-1, 0, 1, 2, 3], dtype=np.float64)
+    targets = np.array([0, 1, 2, 1], dtype=np.float64)
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    model_score = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    naive_probs = np.array([0.0, 0.25, 0.5, 0.25, 0.0])
+    naive_preds = np.tile(naive_probs, (4, 1))
+    naive_score = float((diffs_sqd * naive_preds).sum(axis=1).mean())
+
+    expected = naive_score - model_score
+    assert abs(score - expected) < 1e-10