spforge 0.8.2__py3-none-any.whl → 0.8.4__py3-none-any.whl

spforge/scorer/_score.py

@@ -350,6 +350,7 @@ class PWMSE(BaseScorer):
         labels: list[int] | None = None,
         compare_to_naive: bool = False,
         naive_granularity: list[str] | None = None,
+        evaluation_labels: list[int] | None = None,
     ):
         self.pred_column_name = pred_column
         super().__init__(
@@ -363,12 +364,39 @@ class PWMSE(BaseScorer):
             naive_granularity=naive_granularity,
         )
         self.labels = labels
+        self.evaluation_labels = evaluation_labels
+
+        self._eval_indices: list[int] | None = None
+        if self.evaluation_labels is not None and self.labels is not None:
+            label_to_idx = {lbl: i for i, lbl in enumerate(self.labels)}
+            self._eval_indices = [label_to_idx[lbl] for lbl in self.evaluation_labels]
+
+    def _slice_and_renormalize(self, preds: np.ndarray) -> np.ndarray:
+        if self._eval_indices is None:
+            return preds
+        sliced = preds[:, self._eval_indices]
+        row_sums = sliced.sum(axis=1, keepdims=True)
+        row_sums = np.where(row_sums == 0, 1.0, row_sums)
+        return sliced / row_sums
+
+    def _get_scoring_labels(self) -> list[int]:
+        if self.evaluation_labels is not None:
+            return self.evaluation_labels
+        return self.labels
 
     def _pwmse_score(self, targets: np.ndarray, preds: np.ndarray) -> float:
-        labels = np.asarray(self.labels, dtype=np.float64)
+        labels = np.asarray(self._get_scoring_labels(), dtype=np.float64)
         diffs_sqd = (labels[None, :] - targets[:, None]) ** 2
         return float((diffs_sqd * preds).sum(axis=1).mean())
 
+    def _filter_targets_for_evaluation(self, df: IntoFrameT) -> IntoFrameT:
+        if self.evaluation_labels is None:
+            return df
+        eval_set = set(self.evaluation_labels)
+        min_eval, max_eval = min(eval_set), max(eval_set)
+        target_col = nw.col(self.target)
+        return df.filter((target_col >= min_eval) & (target_col <= max_eval))
+
     @narwhals.narwhalify
     def score(self, df: IntoFrameT) -> float | dict[tuple, float]:
         df = apply_filters(df, self.filters)
@@ -386,6 +414,9 @@ class PWMSE(BaseScorer):
                 after,
             )
 
+        # Filter targets outside evaluation_labels range
+        df = self._filter_targets_for_evaluation(df)
+
         if self.aggregation_level:
             first_pred = df[self.pred_column].to_list()[0] if len(df) > 0 else None
             if isinstance(first_pred, (list, np.ndarray)):
@@ -415,12 +446,13 @@ class PWMSE(BaseScorer):
 
                 targets = gran_df[self.target].to_numpy().astype(np.float64)
                 preds = np.asarray(gran_df[self.pred_column].to_list(), dtype=np.float64)
+                preds = self._slice_and_renormalize(preds)
                 score = self._pwmse_score(targets, preds)
                 if self.compare_to_naive:
                     naive_probs_list = _naive_probability_predictions_for_df(
                         gran_df,
                         self.target,
-                        list(self.labels) if self.labels else None,
+                        list(self._get_scoring_labels()) if self._get_scoring_labels() else None,
                         self.naive_granularity,
                     )
                     naive_preds = np.asarray(naive_probs_list, dtype=np.float64)
@@ -432,12 +464,13 @@ class PWMSE(BaseScorer):
 
         targets = df[self.target].to_numpy().astype(np.float64)
         preds = np.asarray(df[self.pred_column].to_list(), dtype=np.float64)
+        preds = self._slice_and_renormalize(preds)
         score = self._pwmse_score(targets, preds)
         if self.compare_to_naive:
             naive_probs_list = _naive_probability_predictions_for_df(
                 df,
                 self.target,
-                list(self.labels) if self.labels else None,
+                list(self._get_scoring_labels()) if self._get_scoring_labels() else None,
                 self.naive_granularity,
             )
             naive_preds = np.asarray(naive_probs_list, dtype=np.float64)
@@ -1358,4 +1391,6 @@ class ThresholdEventScorer(BaseScorer):
             df, self.outcome_column, labels, self.naive_granularity
         )
         naive_score = self._score_with_probabilities(df, naive_list)
+        if isinstance(score, dict) and isinstance(naive_score, dict):
+            return {k: naive_score[k] - score[k] for k in score.keys()}
         return float(naive_score - score)

{spforge-0.8.2.dist-info → spforge-0.8.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spforge
-Version: 0.8.2
+Version: 0.8.4
 Summary: A flexible framework for generating features, ratings, and building machine learning or other models for training and inference on sports data.
 Author-email: Mathias Holmstrøm <mathiasholmstom@gmail.com>
 License: See LICENSE file

{spforge-0.8.2.dist-info → spforge-0.8.4.dist-info}/RECORD

@@ -61,7 +61,7 @@ spforge/ratings/team_performance_predictor.py,sha256=ThQOmYQUqKBB46ONYHOMM2arXFH
 spforge/ratings/team_start_rating_generator.py,sha256=ZJe84sTvE4Yep3d4wKJMMJn2Q4PhcCwkO7Wyd5nsYUA,5110
 spforge/ratings/utils.py,sha256=qms5J5SD-FyXDR2G8giDMbu_AoLgI135pjW4nghxROg,3940
 spforge/scorer/__init__.py,sha256=wj8PCvYIl6742Xwmt86c3oy6iqE8Ss-OpwHud6kd9IY,256
-spforge/scorer/_score.py,sha256=pzI-upJU4bwm33J5CGhV8bY8HquudnS--0Z6bhD4xew,54498
+spforge/scorer/_score.py,sha256=TR0T9nJj0aeVgGfOE0fZmXlO66CELulYwxhi7ZAxhvY,56184
 spforge/transformers/__init__.py,sha256=IPCsMcsgBqG52d0ttATLCY4HvFCQZddExlLt74U-zuI,390
 spforge/transformers/_base.py,sha256=-smr_McQF9bYxM5-Agx6h7Xv_fhZzPfpAdQV-qK18bs,1134
 spforge/transformers/_net_over_predicted.py,sha256=5dC8pvA1DNO0yXPSgJSMGU8zAHi-maUELm7FqFQVo-U,2321
@@ -70,7 +70,7 @@ spforge/transformers/_other_transformer.py,sha256=xLfaFIhkFsigAoitB4x3F8An2j9ymd
 spforge/transformers/_predictor.py,sha256=2sE6gfVrilXzPVcBurSrtqHw33v2ljygQcEYXt9LhZc,3119
 spforge/transformers/_simple_transformer.py,sha256=zGUFNQYMeoDSa2CoQejQNiNmKCBN5amWTvyOchiUHj0,5660
 spforge/transformers/_team_ratio_predictor.py,sha256=g8_bR53Yyv0iNCtol1O9bgJSeZcIco_AfbQuUxQJkeY,6884
-spforge-0.8.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+spforge-0.8.4.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
 tests/test_autopipeline.py,sha256=WXHeqBdjQD6xaXVkzvS8ocz0WVP9R7lN0PiHJ2iD8nA,16911
 tests/test_autopipeline_context.py,sha256=IuRUY4IA6uMObvbl2pXSaXO2_tl3qX6wEbTZY0dkTMI,1240
 tests/test_feature_generator_pipeline.py,sha256=CAgBknWqawqYi5_hxcPmpxrLVa5elMHVv1VrSVRKXEA,17705
@@ -93,7 +93,7 @@ tests/performance_transformers/test_performances_transformers.py,sha256=A-tGiCx7
 tests/ratings/test_player_rating_generator.py,sha256=3mjqlX159QqOlBoY3r_TFkvLwpE4zlLE0fiqpbfk3ps,58547
 tests/ratings/test_ratings_property.py,sha256=ckyfGILXa4tfQvsgyXEzBDNr2DUmHwFRV13N60w66iE,6561
 tests/ratings/test_team_rating_generator.py,sha256=cDnf1zHiYC7pkgydE3MYr8wSTJIq-bPfSqhIRI_4Tic,95357
-tests/scorer/test_score.py,sha256=5uVCZyEYsonrfDL5tY9sYSlyXIk3JJy5VPUP7zHpkqY,63163
+tests/scorer/test_score.py,sha256=KTrGJypQEpU8tmgJ6LU8wK1SRC3PLUXFzZIyiA-UY7U,71749
 tests/scorer/test_score_aggregation_granularity.py,sha256=h-hyFOLzwp-92hYVU7CwvlRJ8jhB4DzXCtqgI-zcoqM,13677
 tests/transformers/test_estimator_transformer_context.py,sha256=5GOHbuWCWBMFwwOTJOuD4oNDsv-qDR0OxNZYGGuMdag,1819
 tests/transformers/test_net_over_predicted.py,sha256=vh7O1iRRPf4vcW9aLhOMAOyatfM5ZnLsQBKNAYsR3SU,3363
@@ -101,7 +101,7 @@ tests/transformers/test_other_transformer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 tests/transformers/test_predictor_transformer.py,sha256=N1aBYLjN3ldpYZLwjih_gTFYSMitrZu-PNK78W6RHaQ,6877
 tests/transformers/test_simple_transformer.py,sha256=wWR0qjLb_uS4HXrJgGdiqugOY1X7kwd1_OPS02IT2b8,4676
 tests/transformers/test_team_ratio_predictor.py,sha256=fOUP_JvNJi-3kom3ZOs1EdG0I6Z8hpLpYKNHu1eWtOw,8562
-spforge-0.8.2.dist-info/METADATA,sha256=xcw8LWeJSYUBQ01Owe9FiI8fNmJVrlRRb2lnBcXSOmo,20219
-spforge-0.8.2.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
-spforge-0.8.2.dist-info/top_level.txt,sha256=6UW2M5a7WKOeaAi900qQmRKNj5-HZzE8-eUD9Y9LTq0,23
-spforge-0.8.2.dist-info/RECORD,,
+spforge-0.8.4.dist-info/METADATA,sha256=XNaD0lL_puuuYmZU59VjenOYpLSRCSx_nswef8yCZ4M,20219
+spforge-0.8.4.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
+spforge-0.8.4.dist-info/top_level.txt,sha256=6UW2M5a7WKOeaAi900qQmRKNj5-HZzE8-eUD9Y9LTq0,23
+spforge-0.8.4.dist-info/RECORD,,

tests/scorer/test_score.py

@@ -372,6 +372,136 @@ def test_pwmse_compare_to_naive_granularity(df_type):
     assert abs(score - expected) < 1e-10
 
 
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_slices_predictions(df_type):
+    """PWMSE with evaluation_labels should only score on specified labels."""
+    # Predictions have 5 labels: [-2, -1, 0, 1, 2]
+    # But we only want to evaluate on inner labels: [-1, 0, 1]
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.1, 0.2, 0.4, 0.2, 0.1],  # Full distribution over 5 labels
+                [0.05, 0.15, 0.5, 0.2, 0.1],
+            ],
+            "target": [0, 1],
+        },
+    )
+
+    # Score with all labels
+    scorer_full = PWMSE(pred_column="pred", target="target", labels=[-2, -1, 0, 1, 2])
+    score_full = scorer_full.score(df)
+
+    # Score with evaluation_labels excluding boundaries
+    scorer_eval = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[-2, -1, 0, 1, 2],
+        evaluation_labels=[-1, 0, 1],
+    )
+    score_eval = scorer_eval.score(df)
+
+    # Scores should be different because evaluation_labels excludes boundary penalties
+    assert score_full != score_eval
+
+    # Manual calculation for evaluation_labels case:
+    # Slice predictions to indices 1, 2, 3 (corresponding to labels -1, 0, 1)
+    # Then renormalize
+    preds_full = np.array([[0.1, 0.2, 0.4, 0.2, 0.1], [0.05, 0.15, 0.5, 0.2, 0.1]])
+    preds_sliced = preds_full[:, 1:4]  # [-1, 0, 1]
+    preds_renorm = preds_sliced / preds_sliced.sum(axis=1, keepdims=True)
+
+    eval_labels = np.array([-1, 0, 1], dtype=np.float64)
+    targets = np.array([0, 1], dtype=np.float64)
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    expected = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    assert abs(score_eval - expected) < 1e-10
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_with_compare_to_naive(df_type):
+    """PWMSE evaluation_labels should also affect naive baseline calculation."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+            ],
+            "target": [-1, 0, 0, 1],  # Targets within evaluation range
+        },
+    )
+
+    scorer = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[-2, -1, 0, 1, 2],
+        evaluation_labels=[-1, 0, 1],
+        compare_to_naive=True,
+    )
+    score = scorer.score(df)
+
+    # Naive should be computed using only evaluation_labels
+    # With targets [-1, 0, 0, 1], naive probs are [1/4, 2/4, 1/4] for labels [-1, 0, 1]
+    eval_labels = np.array([-1, 0, 1], dtype=np.float64)
+    targets = np.array([-1, 0, 0, 1], dtype=np.float64)
+
+    # Model predictions sliced and renormalized
+    preds_full = np.array([[0.1, 0.2, 0.4, 0.2, 0.1]] * 4)
+    preds_sliced = preds_full[:, 1:4]
+    preds_renorm = preds_sliced / preds_sliced.sum(axis=1, keepdims=True)
+
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    model_score = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    # Naive predictions for evaluation_labels only
+    naive_probs = np.array([0.25, 0.5, 0.25])  # Based on target distribution
+    naive_preds = np.tile(naive_probs, (4, 1))
+    naive_score = float((diffs_sqd * naive_preds).sum(axis=1).mean())
+
+    expected = naive_score - model_score
+    assert abs(score - expected) < 1e-10
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_filters_targets_outside_range(df_type):
+    """PWMSE should filter out targets outside evaluation_labels range."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+            ],
+            "target": [-2, 0, 2],  # -2 and 2 are outside evaluation range [-1, 0, 1]
+        },
+    )
+
+    scorer = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[-2, -1, 0, 1, 2],
+        evaluation_labels=[-1, 0, 1],
+    )
+    score = scorer.score(df)
+
+    # Should only use the row with target=0
+    preds_full = np.array([[0.1, 0.2, 0.4, 0.2, 0.1]])
+    preds_sliced = preds_full[:, 1:4]
+    preds_renorm = preds_sliced / preds_sliced.sum(axis=1, keepdims=True)
+
+    eval_labels = np.array([-1, 0, 1], dtype=np.float64)
+    targets = np.array([0], dtype=np.float64)
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    expected = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    assert abs(score - expected) < 1e-10
+
+
 # ============================================================================
 # D. MeanBiasScorer Tests
 # ============================================================================
@@ -1762,6 +1892,129 @@ def test_pwmse__accepts_ndarray_predictions(df_type):
     assert score >= 0
 
 
+# ============================================================================
+# ThresholdEventScorer with granularity and compare_to_naive Tests
+# ============================================================================
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_threshold_event_scorer__granularity_with_compare_to_naive(df_type):
+    """ThresholdEventScorer fails when combining compare_to_naive with granularity.
+
+    Bug: When granularity is set, binary_scorer.score() returns a dict, but
+    the naive comparison tries to do dict - dict which fails with:
+    'unsupported operand type(s) for -: 'dict' and 'dict''
+    """
+    df = create_dataframe(
+        df_type,
+        {
+            "qtr": [1, 1, 1, 2, 2, 2],
+            "dist": [
+                [0.1, 0.2, 0.3, 0.4],
+                [0.2, 0.3, 0.3, 0.2],
+                [0.3, 0.4, 0.2, 0.1],
+                [0.4, 0.3, 0.2, 0.1],
+                [0.1, 0.1, 0.4, 0.4],
+                [0.2, 0.2, 0.3, 0.3],
+            ],
+            "ydstogo": [2.0, 3.0, 1.0, 2.0, 1.0, 3.0],
+            "rush_yards": [3, 2, 0, 1, 2, 4],
+        },
+    )
+
+    scorer = ThresholdEventScorer(
+        dist_column="dist",
+        threshold_column="ydstogo",
+        outcome_column="rush_yards",
+        labels=[0, 1, 2, 3],
+        compare_to_naive=True,
+        granularity=["qtr"],
+    )
+
+    result = scorer.score(df)
+
+    assert isinstance(result, dict)
+    assert len(result) == 2
+    assert (1,) in result
+    assert (2,) in result
+    assert all(isinstance(v, float) for v in result.values())
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_threshold_event_scorer__granularity_with_compare_to_naive_and_naive_granularity(df_type):
+    """ThresholdEventScorer with both granularity and naive_granularity."""
+    df = create_dataframe(
+        df_type,
+        {
+            "qtr": [1, 1, 1, 2, 2, 2],
+            "team": ["A", "A", "B", "A", "B", "B"],
+            "dist": [
+                [0.1, 0.2, 0.3, 0.4],
+                [0.2, 0.3, 0.3, 0.2],
+                [0.3, 0.4, 0.2, 0.1],
+                [0.4, 0.3, 0.2, 0.1],
+                [0.1, 0.1, 0.4, 0.4],
+                [0.2, 0.2, 0.3, 0.3],
+            ],
+            "ydstogo": [2.0, 3.0, 1.0, 2.0, 1.0, 3.0],
+            "rush_yards": [3, 2, 0, 1, 2, 4],
+        },
+    )
+
+    scorer = ThresholdEventScorer(
+        dist_column="dist",
+        threshold_column="ydstogo",
+        outcome_column="rush_yards",
+        labels=[0, 1, 2, 3],
+        compare_to_naive=True,
+        naive_granularity=["team"],
+        granularity=["qtr"],
+    )
+
+    result = scorer.score(df)
+
+    assert isinstance(result, dict)
+    assert len(result) == 2
+    assert (1,) in result
+    assert (2,) in result
+    assert all(isinstance(v, float) for v in result.values())
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_threshold_event_scorer__multi_column_granularity_with_compare_to_naive(df_type):
+    """ThresholdEventScorer with multi-column granularity and compare_to_naive."""
+    df = create_dataframe(
+        df_type,
+        {
+            "qtr": [1, 1, 2, 2],
+            "half": [1, 1, 2, 2],
+            "dist": [
+                [0.1, 0.2, 0.3, 0.4],
+                [0.2, 0.3, 0.3, 0.2],
+                [0.4, 0.3, 0.2, 0.1],
+                [0.1, 0.1, 0.4, 0.4],
+            ],
+            "ydstogo": [2.0, 3.0, 2.0, 1.0],
+            "rush_yards": [3, 2, 1, 2],
+        },
+    )
+
+    scorer = ThresholdEventScorer(
+        dist_column="dist",
+        threshold_column="ydstogo",
+        outcome_column="rush_yards",
+        labels=[0, 1, 2, 3],
+        compare_to_naive=True,
+        granularity=["qtr", "half"],
+    )
+
+    result = scorer.score(df)
+
+    assert isinstance(result, dict)
+    assert len(result) == 2
+    assert all(isinstance(v, float) for v in result.values())
+
+
 @pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
 def test_all_scorers_handle_all_nan_targets(df_type):
     """All scorers handle case where all targets are NaN"""