spforge 0.8.15__py3-none-any.whl → 0.8.17__py3-none-any.whl

spforge/ratings/_player_rating.py

@@ -16,6 +16,7 @@ from spforge.data_structures import (
     MatchPerformance,
     MatchPlayer,
     PlayerRating,
+    PlayerRatingChange,
     PlayerRatingsResult,
     PreMatchPlayerRating,
     PreMatchPlayersCollection,
@@ -78,7 +79,7 @@ class PlayerRatingGenerator(RatingGenerator):
         start_min_count_for_percentiles: int = 50,
         start_team_rating_subtract: float = 80,
         start_team_weight: float = 0,
-        start_max_days_ago_league_entities: int = 120,
+        start_max_days_ago_league_entities: int = 600,
         start_min_match_count_team_rating: int = 2,
         start_harcoded_start_rating: float | None = None,
         column_names: ColumnNames | None = None,
@@ -442,9 +443,9 @@ class PlayerRatingGenerator(RatingGenerator):
             team1_off_rating, team1_def_rating = self._team_off_def_rating_from_collection(c1)
             team2_off_rating, team2_def_rating = self._team_off_def_rating_from_collection(c2)
 
-            player_updates: list[tuple[str, str, float, float, float, float, float, float, int]] = (
-                []
-            )
+            player_updates: list[
+                tuple[str, str, float, float, float, float, float, float, int, str | None]
+            ] = []
 
             for pre_player in c1.pre_match_player_ratings:
                 pid = pre_player.id
@@ -520,6 +521,7 @@ class PlayerRatingGenerator(RatingGenerator):
                         float(off_change),
                         float(def_change),
                         day_number,
+                        pre_player.league,
                     )
                 )
 
@@ -597,6 +599,7 @@ class PlayerRatingGenerator(RatingGenerator):
                         float(off_change),
                         float(def_change),
                         day_number,
+                        pre_player.league,
                     )
                 )
 
@@ -611,6 +614,7 @@ class PlayerRatingGenerator(RatingGenerator):
                 _off_change,
                 _def_change,
                 _dn,
+                _league,
             ) in player_updates:
                 out[cn.player_id].append(pid)
                 out[cn.match_id].append(match_id)
@@ -627,15 +631,18 @@ class PlayerRatingGenerator(RatingGenerator):
             for (
                 pid,
                 team_id,
-                _off_pre,
+                off_pre,
                 _def_pre,
                 _pred_off,
                 _pred_def,
                 off_change,
                 def_change,
                 dn,
+                league,
             ) in player_updates:
-                pending_team_updates.append((pid, team_id, off_change, def_change, dn))
+                pending_team_updates.append(
+                    (pid, team_id, off_pre, off_change, def_change, dn, league)
+                )
 
             if last_update_id is None:
                 last_update_id = update_id
@@ -645,9 +652,11 @@ class PlayerRatingGenerator(RatingGenerator):
 
         return pl.DataFrame(out, strict=False)
 
-    def _apply_player_updates(self, updates: list[tuple[str, str, float, float, int]]) -> None:
+    def _apply_player_updates(
+        self, updates: list[tuple[str, str, float, float, float, int, str | None]]
+    ) -> None:
 
-        for player_id, team_id, off_change, def_change, day_number in updates:
+        for player_id, team_id, pre_rating, off_change, def_change, day_number, league in updates:
             off_state = self._player_off_ratings[player_id]
             off_state.confidence_sum = self._calculate_post_match_confidence_sum(
                 entity_rating=off_state,
@@ -670,6 +679,19 @@ class PlayerRatingGenerator(RatingGenerator):
             def_state.last_match_day_number = int(day_number)
             def_state.most_recent_team_id = team_id
 
+            self.start_rating_generator.update_players_to_leagues(
+                PlayerRatingChange(
+                    id=player_id,
+                    day_number=day_number,
+                    league=league,
+                    participation_weight=1.0,
+                    predicted_performance=0.0,
+                    performance=0.0,
+                    pre_match_rating_value=pre_rating,
+                    rating_change_value=off_change,
+                )
+            )
+
     def _add_rating_features(self, df: pl.DataFrame) -> pl.DataFrame:
         cols_to_add = set((self._features_out or []) + (self.non_predictor_features_out or []))
 

spforge/ratings/start_rating_generator.py

@@ -28,7 +28,7 @@ class StartRatingGenerator:
         min_count_for_percentiles: int = 50,
         team_rating_subtract: float = 80,
         team_weight: float = 0,
-        max_days_ago_league_entities: int = 120,
+        max_days_ago_league_entities: int = 600,
         min_match_count_team_rating: int = 2,
         harcoded_start_rating: float | None = None,
     ):

spforge/ratings/team_start_rating_generator.py

@@ -24,7 +24,7 @@ class TeamStartRatingGenerator:
         league_ratings: dict[str, float] | None = None,
         league_quantile: float = 0.2,
         min_count_for_percentiles: int = 50,
-        max_days_ago_league_entities: int = 120,
+        max_days_ago_league_entities: int = 600,
         min_match_count_team_rating: int = 2,
         harcoded_start_rating: float | None = None,
     ):

spforge/scorer/_score.py

@@ -366,18 +366,49 @@ class PWMSE(BaseScorer):
         self.labels = labels
         self.evaluation_labels = evaluation_labels
 
+        self._needs_extension = False
+        self._needs_slicing = False
         self._eval_indices: list[int] | None = None
+        self._extension_mapping: dict[int, int] | None = None
+
         if self.evaluation_labels is not None and self.labels is not None:
-            label_to_idx = {lbl: i for i, lbl in enumerate(self.labels)}
-            self._eval_indices = [label_to_idx[lbl] for lbl in self.evaluation_labels]
+            training_set = set(self.labels)
+            eval_set = set(self.evaluation_labels)
+
+            if eval_set <= training_set:
+                self._needs_slicing = True
+                label_to_idx = {lbl: i for i, lbl in enumerate(self.labels)}
+                self._eval_indices = [label_to_idx[lbl] for lbl in self.evaluation_labels]
+            elif training_set <= eval_set:
+                self._needs_extension = True
+                eval_label_to_idx = {lbl: i for i, lbl in enumerate(self.evaluation_labels)}
+                self._extension_mapping = {
+                    train_idx: eval_label_to_idx[lbl]
+                    for train_idx, lbl in enumerate(self.labels)
+                }
+            else:
+                raise ValueError(
+                    f"evaluation_labels must be a subset or superset of labels. "
+                    f"labels={self.labels}, evaluation_labels={self.evaluation_labels}"
+                )
+
+    def _align_predictions(self, preds: np.ndarray) -> np.ndarray:
+        if self._needs_slicing and self._eval_indices is not None:
+            sliced = preds[:, self._eval_indices]
+            row_sums = sliced.sum(axis=1, keepdims=True)
+            row_sums = np.where(row_sums == 0, 1.0, row_sums)
+            return sliced / row_sums
+
+        if self._needs_extension and self._extension_mapping is not None:
+            n_samples = preds.shape[0]
+            n_eval_labels = len(self.evaluation_labels)
+            extended = np.full((n_samples, n_eval_labels), 1e-5, dtype=np.float64)
+            for train_idx, eval_idx in self._extension_mapping.items():
+                extended[:, eval_idx] = preds[:, train_idx]
+            row_sums = extended.sum(axis=1, keepdims=True)
+            return extended / row_sums
 
-    def _slice_and_renormalize(self, preds: np.ndarray) -> np.ndarray:
-        if self._eval_indices is None:
-            return preds
-        sliced = preds[:, self._eval_indices]
-        row_sums = sliced.sum(axis=1, keepdims=True)
-        row_sums = np.where(row_sums == 0, 1.0, row_sums)
-        return sliced / row_sums
+        return preds
 
     def _get_scoring_labels(self) -> list[int]:
         if self.evaluation_labels is not None:
@@ -446,7 +477,7 @@ class PWMSE(BaseScorer):
 
                 targets = gran_df[self.target].to_numpy().astype(np.float64)
                 preds = np.asarray(gran_df[self.pred_column].to_list(), dtype=np.float64)
-                preds = self._slice_and_renormalize(preds)
+                preds = self._align_predictions(preds)
                 score = self._pwmse_score(targets, preds)
                 if self.compare_to_naive:
                     naive_probs_list = _naive_probability_predictions_for_df(
@@ -464,7 +495,7 @@ class PWMSE(BaseScorer):
 
         targets = df[self.target].to_numpy().astype(np.float64)
         preds = np.asarray(df[self.pred_column].to_list(), dtype=np.float64)
-        preds = self._slice_and_renormalize(preds)
+        preds = self._align_predictions(preds)
         score = self._pwmse_score(targets, preds)
         if self.compare_to_naive:
             naive_probs_list = _naive_probability_predictions_for_df(

{spforge-0.8.15.dist-info → spforge-0.8.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spforge
-Version: 0.8.15
+Version: 0.8.17
 Summary: A flexible framework for generating features, ratings, and building machine learning or other models for training and inference on sports data.
 Author-email: Mathias Holmstrøm <mathiasholmstom@gmail.com>
 License: See LICENSE file

{spforge-0.8.15.dist-info → spforge-0.8.17.dist-info}/RECORD

@@ -51,18 +51,18 @@ spforge/performance_transformers/_performance_manager.py,sha256=WmjmlMEnq7y75MiI
 spforge/performance_transformers/_performances_transformers.py,sha256=0lxuWjAfWBRXRgQsNJHjw3P-nlTtHBu4_bOVdoy7hq4,15536
 spforge/ratings/__init__.py,sha256=OZVH2Lo6END3n1X8qi4QcyAPlThIwAYwVKCiIuOQSQU,576
 spforge/ratings/_base.py,sha256=dRMkIGj5-2zKddygaEA4g16WCyXon7v8Xa1ymm7IuoM,14335
-spforge/ratings/_player_rating.py,sha256=--HxUGH1iW0r0979FD51V5zjYzH_6mxp5a-Lvc2X7MY,55305
+spforge/ratings/_player_rating.py,sha256=JSTXdaRw_b8ZoZxgmMnZrYG7gPg8GKawqalLd16SK1M,56066
 spforge/ratings/_team_rating.py,sha256=T0kFiv3ykYSrVGGsVRa8ZxLB0WMnagxqdFDzl9yZ_9g,24813
 spforge/ratings/enums.py,sha256=s7z_RcZS6Nlgfa_6tasO8_IABZJwywexe7sep9DJBgo,1739
 spforge/ratings/league_identifier.py,sha256=_KDUKOwoNU6RNFKE5jju4eYFGVNGBdJsv5mhNvMakfc,6019
 spforge/ratings/league_start_rating_optimizer.py,sha256=Q4Vo3QT-r55qP4aD9WftsTB00UOSRvxM1khlyuAGWNM,8582
 spforge/ratings/player_performance_predictor.py,sha256=cMxzQuk0nF1MsT_M32g-3mxVdAEbZ-S7TUjEPYdo3Yg,8361
-spforge/ratings/start_rating_generator.py,sha256=_7hIJ9KRVCwsCoY1GIzY8cuOdHR8RH_BCMeMwQG3E04,6776
+spforge/ratings/start_rating_generator.py,sha256=eSasa5Oe9n4IoTGjFCYyFQAGrJtzrBW-Qor97lmaYuM,6776
 spforge/ratings/team_performance_predictor.py,sha256=ThQOmYQUqKBB46ONYHOMM2arXFH8AkyKpAZzs80SjHA,7217
-spforge/ratings/team_start_rating_generator.py,sha256=ZJe84sTvE4Yep3d4wKJMMJn2Q4PhcCwkO7Wyd5nsYUA,5110
+spforge/ratings/team_start_rating_generator.py,sha256=vK-_m8KwcHopchch_lKNHSGLiiNm5q9Lenm0d1cP_po,5110
 spforge/ratings/utils.py,sha256=_zFemqz2jJkH8rn2EZpDt8N6FELUmYp9qCnPzRtOIGU,4497
 spforge/scorer/__init__.py,sha256=wj8PCvYIl6742Xwmt86c3oy6iqE8Ss-OpwHud6kd9IY,256
-spforge/scorer/_score.py,sha256=TR0T9nJj0aeVgGfOE0fZmXlO66CELulYwxhi7ZAxhvY,56184
+spforge/scorer/_score.py,sha256=kNuqiK3F5mUEAVD7KjWYY7E_AkRrspR362QBm_jyElg,57623
 spforge/transformers/__init__.py,sha256=IPCsMcsgBqG52d0ttATLCY4HvFCQZddExlLt74U-zuI,390
 spforge/transformers/_base.py,sha256=-smr_McQF9bYxM5-Agx6h7Xv_fhZzPfpAdQV-qK18bs,1134
 spforge/transformers/_net_over_predicted.py,sha256=5dC8pvA1DNO0yXPSgJSMGU8zAHi-maUELm7FqFQVo-U,2321
@@ -71,7 +71,7 @@ spforge/transformers/_other_transformer.py,sha256=xLfaFIhkFsigAoitB4x3F8An2j9ymd
 spforge/transformers/_predictor.py,sha256=2sE6gfVrilXzPVcBurSrtqHw33v2ljygQcEYXt9LhZc,3119
 spforge/transformers/_simple_transformer.py,sha256=zGUFNQYMeoDSa2CoQejQNiNmKCBN5amWTvyOchiUHj0,5660
 spforge/transformers/_team_ratio_predictor.py,sha256=g8_bR53Yyv0iNCtol1O9bgJSeZcIco_AfbQuUxQJkeY,6884
-spforge-0.8.15.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+spforge-0.8.17.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
 tests/test_autopipeline.py,sha256=g5SMTTolfRikHZfwIkExuoRjh-ldcr9-F-E1PUNpbpM,22923
 tests/test_autopipeline_context.py,sha256=IuRUY4IA6uMObvbl2pXSaXO2_tl3qX6wEbTZY0dkTMI,1240
 tests/test_feature_generator_pipeline.py,sha256=CK0zVL8PfTncy3RmG9i-YpgwjOIV7yJhV7Q44tbetI8,19020
@@ -94,12 +94,12 @@ tests/hyperparameter_tuning/test_estimator_tuner.py,sha256=iewME41d6LR2aQ0OtohGF
 tests/hyperparameter_tuning/test_rating_tuner.py,sha256=PyCFP3KPc4Iy9E_X9stCVxra14uMgC1tuRwuQ30rO_o,13195
 tests/performance_transformers/test_performance_manager.py,sha256=gjuuV_hb27kCo_kUecPKG3Cbot2Gqis1W3kw2A4ovS4,10690
 tests/performance_transformers/test_performances_transformers.py,sha256=A-tGiCx7kXrj1cVj03Bc7prOeZ1_Ryz8YFx9uj3eK6w,11064
-tests/ratings/test_player_rating_generator.py,sha256=NRD5OtCGBpdWjyup6YCKrt0sIz_GiIxB3l149gr44ts,61543
+tests/ratings/test_player_rating_generator.py,sha256=SKLaBQBsHYslc2Nia2AxZ8A9Cy16MbZAWjLyOjvcMnA,64094
 tests/ratings/test_player_rating_no_mutation.py,sha256=GzO3Hl__5K68DS3uRLefwnbcTJOvBM7cZqww4M21UZM,8493
 tests/ratings/test_ratings_property.py,sha256=ckyfGILXa4tfQvsgyXEzBDNr2DUmHwFRV13N60w66iE,6561
 tests/ratings/test_team_rating_generator.py,sha256=cDnf1zHiYC7pkgydE3MYr8wSTJIq-bPfSqhIRI_4Tic,95357
 tests/ratings/test_utils_scaled_weights.py,sha256=iHxe6ZDUB_I2B6HT0xTGqXBkl7gRlqVV0e_7Lwun5po,4988
-tests/scorer/test_score.py,sha256=_Vd6tKpy_1GeOxU7Omxci4CFf7PvRGMefEI0gv2gV6A,74688
+tests/scorer/test_score.py,sha256=rw3xJs6xqWVpalVMUQz557m2JYGR7PmhrsjfTex0b0c,79121
 tests/scorer/test_score_aggregation_granularity.py,sha256=h-hyFOLzwp-92hYVU7CwvlRJ8jhB4DzXCtqgI-zcoqM,13677
 tests/transformers/test_estimator_transformer_context.py,sha256=5GOHbuWCWBMFwwOTJOuD4oNDsv-qDR0OxNZYGGuMdag,1819
 tests/transformers/test_net_over_predicted.py,sha256=vh7O1iRRPf4vcW9aLhOMAOyatfM5ZnLsQBKNAYsR3SU,3363
@@ -107,7 +107,7 @@ tests/transformers/test_other_transformer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 tests/transformers/test_predictor_transformer.py,sha256=N1aBYLjN3ldpYZLwjih_gTFYSMitrZu-PNK78W6RHaQ,6877
 tests/transformers/test_simple_transformer.py,sha256=wWR0qjLb_uS4HXrJgGdiqugOY1X7kwd1_OPS02IT2b8,4676
 tests/transformers/test_team_ratio_predictor.py,sha256=fOUP_JvNJi-3kom3ZOs1EdG0I6Z8hpLpYKNHu1eWtOw,8562
-spforge-0.8.15.dist-info/METADATA,sha256=0AMne9tkye8kJBUIZLKOzBYI8fwmV03VqhflfuNBl2o,20048
-spforge-0.8.15.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-spforge-0.8.15.dist-info/top_level.txt,sha256=6UW2M5a7WKOeaAi900qQmRKNj5-HZzE8-eUD9Y9LTq0,23
-spforge-0.8.15.dist-info/RECORD,,
+spforge-0.8.17.dist-info/METADATA,sha256=Zc4fLlCtPWuEFvs0DVRZre9OtTvRyVMgdmGV7-s68Ao,20048
+spforge-0.8.17.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+spforge-0.8.17.dist-info/top_level.txt,sha256=6UW2M5a7WKOeaAi900qQmRKNj5-HZzE8-eUD9Y9LTq0,23
+spforge-0.8.17.dist-info/RECORD,,

tests/ratings/test_player_rating_generator.py

@@ -1746,3 +1746,73 @@ def test_fit_transform__player_rating_difference_from_team_projected_feature(bas
     for row in result.iter_rows(named=True):
         expected = row[player_col] - row[team_col]
         assert row[diff_col] == pytest.approx(expected, rel=1e-9)
+
+
+def test_fit_transform__start_league_quantile_uses_existing_player_ratings(base_cn):
+    """
+    Bug reproduction: start_league_quantile should use percentile of existing player
+    ratings for new players, but update_players_to_leagues is never called so
+    _league_player_ratings stays empty and all new players get default rating.
+
+    Expected: New player P_NEW should start at 5th percentile of existing ratings (~920)
+    Actual: New player starts at default 1000 because _league_player_ratings is empty
+    """
+    import numpy as np
+
+    num_existing_players = 60
+    player_ids = [f"P{i}" for i in range(num_existing_players)]
+    team_ids = [f"T{i % 2 + 1}" for i in range(num_existing_players)]
+
+    df1 = pl.DataFrame(
+        {
+            "pid": player_ids,
+            "tid": team_ids,
+            "mid": ["M1"] * num_existing_players,
+            "dt": ["2024-01-01"] * num_existing_players,
+            "perf": [0.3 + (i % 10) * 0.07 for i in range(num_existing_players)],
+            "pw": [1.0] * num_existing_players,
+        }
+    )
+
+    gen = PlayerRatingGenerator(
+        performance_column="perf",
+        column_names=base_cn,
+        auto_scale_performance=True,
+        start_league_quantile=0.05,
+        start_min_count_for_percentiles=50,
+        features_out=[RatingKnownFeatures.PLAYER_OFF_RATING],
+    )
+    gen.fit_transform(df1)
+
+    existing_ratings = [
+        gen._player_off_ratings[pid].rating_value for pid in player_ids
+    ]
+    expected_quantile_rating = np.percentile(existing_ratings, 5)
+
+    srg = gen.start_rating_generator
+    assert len(srg._league_player_ratings.get(None, [])) >= 50, (
+        f"Expected _league_player_ratings to have >=50 entries but got "
+        f"{len(srg._league_player_ratings.get(None, []))}. "
+        "update_players_to_leagues is never called."
+    )
+
+    df2 = pl.DataFrame(
+        {
+            "pid": ["P_NEW", "P0"],
+            "tid": ["T1", "T2"],
+            "mid": ["M2", "M2"],
+            "dt": ["2024-01-02", "2024-01-02"],
+            "pw": [1.0, 1.0],
+        }
+    )
+    result = gen.future_transform(df2)
+
+    new_player_start_rating = result.filter(pl.col("pid") == "P_NEW")[
+        "player_off_rating_perf"
+    ][0]
+
+    assert new_player_start_rating == pytest.approx(expected_quantile_rating, rel=0.1), (
+        f"New player should start at 5th percentile ({expected_quantile_rating:.1f}) "
+        f"but got {new_player_start_rating:.1f}. "
+        "start_league_quantile has no effect because update_players_to_leagues is never called."
+    )

tests/scorer/test_score.py

@@ -2138,3 +2138,145 @@ def test_scorers_respect_validation_column(scorer_factory, df_factory):
     score_all = scorer_factory().score(df)
     score_valid = scorer_factory().score(df_valid)
     assert score_all == score_valid
+
+
+# ============================================================================
+# PWMSE evaluation_labels Extension Tests
+# ============================================================================
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_extends_predictions(df_type):
+    """PWMSE with evaluation_labels as superset extends predictions with small probs."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.3, 0.5, 0.2],
+                [0.2, 0.6, 0.2],
+            ],
+            "target": [0, 1],
+        },
+    )
+
+    scorer = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+        evaluation_labels=[-1, 0, 1, 2, 3],
+    )
+    score = scorer.score(df)
+
+    n_eval_labels = 5
+    eps = 1e-5
+    preds_original = np.array([[0.3, 0.5, 0.2], [0.2, 0.6, 0.2]])
+    extended = np.full((2, n_eval_labels), eps, dtype=np.float64)
+    extended[:, 1] = preds_original[:, 0]
+    extended[:, 2] = preds_original[:, 1]
+    extended[:, 3] = preds_original[:, 2]
+    row_sums = extended.sum(axis=1, keepdims=True)
+    preds_renorm = extended / row_sums
+
+    eval_labels = np.array([-1, 0, 1, 2, 3], dtype=np.float64)
+    targets = np.array([0, 1], dtype=np.float64)
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    expected = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    assert abs(score - expected) < 1e-10
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_exact_match(df_type):
+    """PWMSE with evaluation_labels identical to labels (no-op)."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.3, 0.5, 0.2],
+                [0.2, 0.6, 0.2],
+            ],
+            "target": [0, 1],
+        },
+    )
+
+    scorer_with_eval = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+        evaluation_labels=[0, 1, 2],
+    )
+    scorer_without_eval = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+    )
+
+    score_with = scorer_with_eval.score(df)
+    score_without = scorer_without_eval.score(df)
+
+    assert abs(score_with - score_without) < 1e-10
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_partial_overlap_raises(df_type):
+    """PWMSE with partial overlap between labels and evaluation_labels raises."""
+    with pytest.raises(ValueError, match="evaluation_labels must be a subset or superset"):
+        PWMSE(
+            pred_column="pred",
+            target="target",
+            labels=[0, 1, 2],
+            evaluation_labels=[1, 2, 3],
+        )
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_extends_with_compare_to_naive(df_type):
+    """PWMSE extension mode works correctly with compare_to_naive."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.8, 0.15, 0.05],
+                [0.1, 0.7, 0.2],
+                [0.05, 0.15, 0.8],
+                [0.3, 0.4, 0.3],
+            ],
+            "target": [0, 1, 2, 1],
+        },
+    )
+
+    scorer = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+        evaluation_labels=[-1, 0, 1, 2, 3],
+        compare_to_naive=True,
+    )
+    score = scorer.score(df)
+
+    n_eval_labels = 5
+    eps = 1e-5
+    preds_original = np.array([
+        [0.8, 0.15, 0.05],
+        [0.1, 0.7, 0.2],
+        [0.05, 0.15, 0.8],
+        [0.3, 0.4, 0.3],
+    ])
+    extended = np.full((4, n_eval_labels), eps, dtype=np.float64)
+    extended[:, 1] = preds_original[:, 0]
+    extended[:, 2] = preds_original[:, 1]
+    extended[:, 3] = preds_original[:, 2]
+    row_sums = extended.sum(axis=1, keepdims=True)
+    preds_renorm = extended / row_sums
+
+    eval_labels = np.array([-1, 0, 1, 2, 3], dtype=np.float64)
+    targets = np.array([0, 1, 2, 1], dtype=np.float64)
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    model_score = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    naive_probs = np.array([0.0, 0.25, 0.5, 0.25, 0.0])
+    naive_preds = np.tile(naive_probs, (4, 1))
+    naive_score = float((diffs_sqd * naive_preds).sum(axis=1).mean())
+
+    expected = naive_score - model_score
+    assert abs(score - expected) < 1e-10