spforge 0.8.8__py3-none-any.whl → 0.8.19__py3-none-any.whl

spforge/ratings/_player_rating.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 
 import copy
 import math
+import logging
 from typing import Any, Literal
 
 import narwhals.stable.v2 as nw
@@ -15,6 +16,7 @@ from spforge.data_structures import (
     MatchPerformance,
     MatchPlayer,
     PlayerRating,
+    PlayerRatingChange,
     PlayerRatingsResult,
     PreMatchPlayerRating,
     PreMatchPlayersCollection,
@@ -33,6 +35,8 @@ from spforge.ratings.utils import (
 from spforge.feature_generator._utils import to_polars
 
 PLAYER_STATS = "__PLAYER_STATS"
+_SCALED_PW = "__scaled_participation_weight__"
+_SCALED_PPW = "__scaled_projected_participation_weight__"
 
 
 class PlayerRatingGenerator(RatingGenerator):
@@ -75,12 +79,13 @@ class PlayerRatingGenerator(RatingGenerator):
         start_min_count_for_percentiles: int = 50,
         start_team_rating_subtract: float = 80,
         start_team_weight: float = 0,
-        start_max_days_ago_league_entities: int = 120,
+        start_max_days_ago_league_entities: int = 600,
         start_min_match_count_team_rating: int = 2,
         start_harcoded_start_rating: float | None = None,
         column_names: ColumnNames | None = None,
         output_suffix: str | None = None,
         scale_participation_weights: bool = False,
+        auto_scale_participation_weights: bool = True,
         **kwargs: Any,
     ):
         super().__init__(
@@ -164,6 +169,7 @@ class PlayerRatingGenerator(RatingGenerator):
 
         self.use_off_def_split = bool(use_off_def_split)
         self.scale_participation_weights = bool(scale_participation_weights)
+        self.auto_scale_participation_weights = bool(auto_scale_participation_weights)
         self._participation_weight_max: float | None = None
         self._projected_participation_weight_max: float | None = None
 
@@ -189,9 +195,39 @@ class PlayerRatingGenerator(RatingGenerator):
         column_names: ColumnNames | None = None,
     ) -> DataFrame | IntoFrameT:
         self.column_names = column_names if column_names else self.column_names
+        self._maybe_enable_participation_weight_scaling(df)
         self._set_participation_weight_max(df)
         return super().fit_transform(df, column_names)
 
+    def _maybe_enable_participation_weight_scaling(self, df: DataFrame) -> None:
+        if self.scale_participation_weights or not self.auto_scale_participation_weights:
+            return
+        cn = self.column_names
+        if not cn:
+            return
+
+        pl_df = df.to_native() if df.implementation.is_polars() else df.to_polars().to_native()
+
+        def _out_of_bounds(col_name: str | None) -> bool:
+            if not col_name or col_name not in df.columns:
+                return False
+            col = pl_df[col_name]
+            min_val = col.min()
+            max_val = col.max()
+            if min_val is None or max_val is None:
+                return False
+            eps = 1e-6
+            return min_val < -eps or max_val > (1.0 + eps)
+
+        if _out_of_bounds(cn.participation_weight) or _out_of_bounds(
+            cn.projected_participation_weight
+        ):
+            self.scale_participation_weights = True
+            logging.warning(
+                "Auto-scaling participation weights because values exceed [0, 1]. "
+                "Set scale_participation_weights=True explicitly to silence this warning."
+            )
+
     def _ensure_player_off(self, player_id: str) -> PlayerRating:
         if player_id not in self._player_off_ratings:
             # create with start generator later; initialize to 0 now; overwritten when needed
@@ -240,6 +276,7 @@ class PlayerRatingGenerator(RatingGenerator):
             self._projected_participation_weight_max = self._participation_weight_max
 
     def _scale_participation_weight_columns(self, df: pl.DataFrame) -> pl.DataFrame:
+        """Create internal scaled participation weight columns without mutating originals."""
         if not self.scale_participation_weights:
             return df
         if self._participation_weight_max is None or self._participation_weight_max <= 0:
@@ -254,7 +291,7 @@ class PlayerRatingGenerator(RatingGenerator):
             df = df.with_columns(
                 (pl.col(cn.participation_weight) / denom)
                 .clip(0.0, 1.0)
-                .alias(cn.participation_weight)
+                .alias(_SCALED_PW)
             )
 
         if (
@@ -267,16 +304,38 @@ class PlayerRatingGenerator(RatingGenerator):
             df = df.with_columns(
                 (pl.col(cn.projected_participation_weight) / denom)
                 .clip(0.0, 1.0)
-                .alias(cn.projected_participation_weight)
+                .alias(_SCALED_PPW)
             )
 
         return df
 
+    def _get_participation_weight_col(self) -> str:
+        """Get the column name to use for participation weight (scaled if available)."""
+        cn = self.column_names
+        if self.scale_participation_weights and cn and cn.participation_weight:
+            return _SCALED_PW
+        return cn.participation_weight if cn else ""
+
+    def _get_projected_participation_weight_col(self) -> str:
+        """Get the column name to use for projected participation weight (scaled if available)."""
+        cn = self.column_names
+        if self.scale_participation_weights and cn and cn.projected_participation_weight:
+            return _SCALED_PPW
+        return cn.projected_participation_weight if cn else ""
+
+    def _remove_internal_scaled_columns(self, df: pl.DataFrame) -> pl.DataFrame:
+        """Remove internal scaled columns before returning."""
+        cols_to_drop = [c for c in [_SCALED_PW, _SCALED_PPW] if c in df.columns]
+        if cols_to_drop:
+            df = df.drop(cols_to_drop)
+        return df
+
     def _historical_transform(self, df: pl.DataFrame) -> pl.DataFrame:
         df = self._scale_participation_weight_columns(df)
         match_df = self._create_match_df(df)
         ratings = self._calculate_ratings(match_df)
 
+        # Keep scaled columns for now - they're needed by _add_rating_features
         cols = [
             c
             for c in df.columns
@@ -296,13 +355,15 @@ class PlayerRatingGenerator(RatingGenerator):
             on=[self.column_names.player_id, self.column_names.match_id, self.column_names.team_id],
         )
 
-        return self._add_rating_features(df)
+        result = self._add_rating_features(df)
+        return self._remove_internal_scaled_columns(result)
 
     def _future_transform(self, df: pl.DataFrame) -> pl.DataFrame:
         df = self._scale_participation_weight_columns(df)
         match_df = self._create_match_df(df)
         ratings = self._calculate_future_ratings(match_df)
 
+        # Keep scaled columns for now - they're needed by _add_rating_features
         cols = [
             c
             for c in df.columns
@@ -327,7 +388,8 @@ class PlayerRatingGenerator(RatingGenerator):
             how="left",
         )
 
-        return self._add_rating_features(df_with_ratings)
+        result = self._add_rating_features(df_with_ratings)
+        return self._remove_internal_scaled_columns(result)
 
     def _calculate_ratings(self, match_df: pl.DataFrame) -> pl.DataFrame:
         cn = self.column_names
@@ -381,9 +443,9 @@ class PlayerRatingGenerator(RatingGenerator):
             team1_off_rating, team1_def_rating = self._team_off_def_rating_from_collection(c1)
             team2_off_rating, team2_def_rating = self._team_off_def_rating_from_collection(c2)
 
-            player_updates: list[tuple[str, str, float, float, float, float, float, float, int]] = (
-                []
-            )
+            player_updates: list[
+                tuple[str, str, float, float, float, float, float, float, int, str | None]
+            ] = []
 
             for pre_player in c1.pre_match_player_ratings:
                 pid = pre_player.id
@@ -459,6 +521,7 @@ class PlayerRatingGenerator(RatingGenerator):
                         float(off_change),
                         float(def_change),
                         day_number,
+                        pre_player.league,
                     )
                 )
 
@@ -536,6 +599,7 @@ class PlayerRatingGenerator(RatingGenerator):
                         float(off_change),
                         float(def_change),
                         day_number,
+                        pre_player.league,
                     )
                 )
 
@@ -550,6 +614,7 @@ class PlayerRatingGenerator(RatingGenerator):
                 _off_change,
                 _def_change,
                 _dn,
+                _league,
             ) in player_updates:
                 out[cn.player_id].append(pid)
                 out[cn.match_id].append(match_id)
@@ -566,15 +631,18 @@ class PlayerRatingGenerator(RatingGenerator):
             for (
                 pid,
                 team_id,
-                _off_pre,
+                off_pre,
                 _def_pre,
                 _pred_off,
                 _pred_def,
                 off_change,
                 def_change,
                 dn,
+                league,
             ) in player_updates:
-                pending_team_updates.append((pid, team_id, off_change, def_change, dn))
+                pending_team_updates.append(
+                    (pid, team_id, off_pre, off_change, def_change, dn, league)
+                )
 
             if last_update_id is None:
                 last_update_id = update_id
@@ -584,9 +652,11 @@ class PlayerRatingGenerator(RatingGenerator):
 
         return pl.DataFrame(out, strict=False)
 
-    def _apply_player_updates(self, updates: list[tuple[str, str, float, float, int]]) -> None:
+    def _apply_player_updates(
+        self, updates: list[tuple[str, str, float, float, float, int, str | None]]
+    ) -> None:
 
-        for player_id, team_id, off_change, def_change, day_number in updates:
+        for player_id, team_id, pre_rating, off_change, def_change, day_number, league in updates:
             off_state = self._player_off_ratings[player_id]
             off_state.confidence_sum = self._calculate_post_match_confidence_sum(
                 entity_rating=off_state,
@@ -609,6 +679,19 @@ class PlayerRatingGenerator(RatingGenerator):
             def_state.last_match_day_number = int(day_number)
             def_state.most_recent_team_id = team_id
 
+            self.start_rating_generator.update_players_to_leagues(
+                PlayerRatingChange(
+                    id=player_id,
+                    day_number=day_number,
+                    league=league,
+                    participation_weight=1.0,
+                    predicted_performance=0.0,
+                    performance=0.0,
+                    pre_match_rating_value=pre_rating,
+                    rating_change_value=off_change,
+                )
+            )
+
     def _add_rating_features(self, df: pl.DataFrame) -> pl.DataFrame:
         cols_to_add = set((self._features_out or []) + (self.non_predictor_features_out or []))
 
@@ -763,9 +846,13 @@ class PlayerRatingGenerator(RatingGenerator):
 
         if cn.participation_weight and cn.participation_weight in df.columns:
             player_stat_cols.append(cn.participation_weight)
+        if _SCALED_PW in df.columns:
+            player_stat_cols.append(_SCALED_PW)
 
         if cn.projected_participation_weight and cn.projected_participation_weight in df.columns:
             player_stat_cols.append(cn.projected_participation_weight)
+        if _SCALED_PPW in df.columns:
+            player_stat_cols.append(_SCALED_PPW)
 
         if cn.position and cn.position in df.columns:
             player_stat_cols.append(cn.position)
@@ -821,14 +908,23 @@ class PlayerRatingGenerator(RatingGenerator):
             position = team_player.get(cn.position)
             player_league = team_player.get(cn.league, None)
 
-            participation_weight = (
-                team_player.get(cn.participation_weight, 1.0) if cn.participation_weight else 1.0
-            )
-            projected_participation_weight = (
-                team_player.get(cn.projected_participation_weight, participation_weight)
-                if cn.projected_participation_weight
-                else participation_weight
-            )
+            # Use scaled participation weight if available, otherwise use original
+            if _SCALED_PW in team_player:
+                participation_weight = team_player.get(_SCALED_PW, 1.0)
+            elif cn.participation_weight:
+                participation_weight = team_player.get(cn.participation_weight, 1.0)
+            else:
+                participation_weight = 1.0
+
+            # Use scaled projected participation weight if available, otherwise use original
+            if _SCALED_PPW in team_player:
+                projected_participation_weight = team_player.get(_SCALED_PPW, participation_weight)
+            elif cn.projected_participation_weight:
+                projected_participation_weight = team_player.get(
+                    cn.projected_participation_weight, participation_weight
+                )
+            else:
+                projected_participation_weight = participation_weight
             projected_participation_weights.append(projected_participation_weight)
 
             perf_val = (
@@ -1054,14 +1150,21 @@ class PlayerRatingGenerator(RatingGenerator):
                     position = tp.get(cn.position)
                     league = tp.get(cn.league, None)
 
-                    pw = (
-                        tp.get(cn.participation_weight, 1.0) if cn.participation_weight else 1.0
-                    )
-                    ppw = (
-                        tp.get(cn.projected_participation_weight, pw)
-                        if cn.projected_participation_weight
-                        else pw
-                    )
+                    # Use scaled participation weight if available, otherwise use original
+                    if _SCALED_PW in tp:
+                        pw = tp.get(_SCALED_PW, 1.0)
+                    elif cn.participation_weight:
+                        pw = tp.get(cn.participation_weight, 1.0)
+                    else:
+                        pw = 1.0
+
+                    # Use scaled projected participation weight if available, otherwise use original
+                    if _SCALED_PPW in tp:
+                        ppw = tp.get(_SCALED_PPW, pw)
+                    elif cn.projected_participation_weight:
+                        ppw = tp.get(cn.projected_participation_weight, pw)
+                    else:
+                        ppw = pw
                     proj_w.append(float(ppw))
 
                     mp = MatchPerformance(

spforge/ratings/start_rating_generator.py

@@ -28,7 +28,7 @@ class StartRatingGenerator:
         min_count_for_percentiles: int = 50,
         team_rating_subtract: float = 80,
         team_weight: float = 0,
-        max_days_ago_league_entities: int = 120,
+        max_days_ago_league_entities: int = 600,
         min_match_count_team_rating: int = 2,
         harcoded_start_rating: float | None = None,
     ):

spforge/ratings/team_start_rating_generator.py

@@ -24,7 +24,7 @@ class TeamStartRatingGenerator:
         league_ratings: dict[str, float] | None = None,
         league_quantile: float = 0.2,
         min_count_for_percentiles: int = 50,
-        max_days_ago_league_entities: int = 120,
+        max_days_ago_league_entities: int = 600,
         min_match_count_team_rating: int = 2,
         harcoded_start_rating: float | None = None,
     ):

spforge/ratings/utils.py

@@ -2,6 +2,10 @@ import polars as pl
 
 from spforge.data_structures import ColumnNames
 
+# Internal column names for scaled participation weights
+_SCALED_PW = "__scaled_participation_weight__"
+_SCALED_PPW = "__scaled_projected_participation_weight__"
+
 
 def add_team_rating(
     df: pl.DataFrame,
@@ -46,11 +50,14 @@ def add_team_rating_projected(
     tid = column_names.team_id
     ppw = column_names.projected_participation_weight
 
-    if ppw:
+    # Use scaled column if available (clipped to [0, 1]), otherwise raw column
+    weight_col = _SCALED_PPW if _SCALED_PPW in df.columns else ppw
+
+    if weight_col and weight_col in df.columns:
         return df.with_columns(
             (
-                (pl.col(ppw) * pl.col(player_rating_col)).sum().over([mid, tid])
-                / pl.col(ppw).sum().over([mid, tid])
+                (pl.col(weight_col) * pl.col(player_rating_col)).sum().over([mid, tid])
+                / pl.col(weight_col).sum().over([mid, tid])
             ).alias(team_rating_out)
         )
 
@@ -118,11 +125,14 @@ def add_rating_mean_projected(
     mid = column_names.match_id
     ppw = column_names.projected_participation_weight
 
-    if ppw:
+    # Use scaled column if available (clipped to [0, 1]), otherwise raw column
+    weight_col = _SCALED_PPW if _SCALED_PPW in df.columns else ppw
+
+    if weight_col and weight_col in df.columns:
         return df.with_columns(
             (
-                (pl.col(ppw) * pl.col(player_rating_col)).sum().over(mid)
-                / pl.col(ppw).sum().over(mid)
+                (pl.col(weight_col) * pl.col(player_rating_col)).sum().over(mid)
+                / pl.col(weight_col).sum().over(mid)
             ).alias(rating_mean_out)
         )
 

spforge/scorer/_score.py

@@ -366,18 +366,49 @@ class PWMSE(BaseScorer):
         self.labels = labels
         self.evaluation_labels = evaluation_labels
 
+        self._needs_extension = False
+        self._needs_slicing = False
         self._eval_indices: list[int] | None = None
+        self._extension_mapping: dict[int, int] | None = None
+
         if self.evaluation_labels is not None and self.labels is not None:
-            label_to_idx = {lbl: i for i, lbl in enumerate(self.labels)}
-            self._eval_indices = [label_to_idx[lbl] for lbl in self.evaluation_labels]
+            training_set = set(self.labels)
+            eval_set = set(self.evaluation_labels)
+
+            if eval_set <= training_set:
+                self._needs_slicing = True
+                label_to_idx = {lbl: i for i, lbl in enumerate(self.labels)}
+                self._eval_indices = [label_to_idx[lbl] for lbl in self.evaluation_labels]
+            elif training_set <= eval_set:
+                self._needs_extension = True
+                eval_label_to_idx = {lbl: i for i, lbl in enumerate(self.evaluation_labels)}
+                self._extension_mapping = {
+                    train_idx: eval_label_to_idx[lbl]
+                    for train_idx, lbl in enumerate(self.labels)
+                }
+            else:
+                raise ValueError(
+                    f"evaluation_labels must be a subset or superset of labels. "
+                    f"labels={self.labels}, evaluation_labels={self.evaluation_labels}"
+                )
+
+    def _align_predictions(self, preds: np.ndarray) -> np.ndarray:
+        if self._needs_slicing and self._eval_indices is not None:
+            sliced = preds[:, self._eval_indices]
+            row_sums = sliced.sum(axis=1, keepdims=True)
+            row_sums = np.where(row_sums == 0, 1.0, row_sums)
+            return sliced / row_sums
+
+        if self._needs_extension and self._extension_mapping is not None:
+            n_samples = preds.shape[0]
+            n_eval_labels = len(self.evaluation_labels)
+            extended = np.full((n_samples, n_eval_labels), 1e-5, dtype=np.float64)
+            for train_idx, eval_idx in self._extension_mapping.items():
+                extended[:, eval_idx] = preds[:, train_idx]
+            row_sums = extended.sum(axis=1, keepdims=True)
+            return extended / row_sums
 
-    def _slice_and_renormalize(self, preds: np.ndarray) -> np.ndarray:
-        if self._eval_indices is None:
-            return preds
-        sliced = preds[:, self._eval_indices]
-        row_sums = sliced.sum(axis=1, keepdims=True)
-        row_sums = np.where(row_sums == 0, 1.0, row_sums)
-        return sliced / row_sums
+        return preds
 
     def _get_scoring_labels(self) -> list[int]:
         if self.evaluation_labels is not None:
@@ -446,7 +477,7 @@ class PWMSE(BaseScorer):
 
                 targets = gran_df[self.target].to_numpy().astype(np.float64)
                 preds = np.asarray(gran_df[self.pred_column].to_list(), dtype=np.float64)
-                preds = self._slice_and_renormalize(preds)
+                preds = self._align_predictions(preds)
                 score = self._pwmse_score(targets, preds)
                 if self.compare_to_naive:
                     naive_probs_list = _naive_probability_predictions_for_df(
@@ -464,7 +495,7 @@ class PWMSE(BaseScorer):
 
         targets = df[self.target].to_numpy().astype(np.float64)
         preds = np.asarray(df[self.pred_column].to_list(), dtype=np.float64)
-        preds = self._slice_and_renormalize(preds)
+        preds = self._align_predictions(preds)
         score = self._pwmse_score(targets, preds)
         if self.compare_to_naive:
             naive_probs_list = _naive_probability_predictions_for_df(

spforge/transformers/_other_transformer.py

@@ -8,8 +8,9 @@ from sklearn.base import BaseEstimator, TransformerMixin
 
 
 class GroupByReducer(BaseEstimator, TransformerMixin):
-    def __init__(self, granularity: list[str]):
+    def __init__(self, granularity: list[str], aggregation_weight: str | None = None):
         self.granularity = granularity
+        self.aggregation_weight = aggregation_weight
 
     @nw.narwhalify
     def fit(self, X: IntoFrameT, y: Any = None):
@@ -26,18 +27,47 @@ class GroupByReducer(BaseEstimator, TransformerMixin):
             raise ValueError("Could not find granularity columns in dataframe %s", self.granularity)
 
         non_keys = [c for c in df.columns if c not in keys]
-        num_cols = [c for c in non_keys if pd.api.types.is_numeric_dtype(df[c])]
+        schema = df.schema
+        num_cols = [c for c in non_keys if schema[c].is_numeric()]
         other_cols = [c for c in non_keys if c not in num_cols]
 
         aggs: list[nw.Expr] = []
 
+        # Backwards compatibility: old pickled objects may not have aggregation_weight
+        weight_col = getattr(self, "aggregation_weight", None)
+        has_weight = weight_col and weight_col in df.columns
+
         for c in num_cols:
-            aggs.append(nw.col(c).mean().alias(c))
+            if c == weight_col:
+                aggs.append(nw.col(c).sum().alias(c))
+            elif has_weight:
+                aggs.append((nw.col(c) * nw.col(weight_col)).sum().alias(f"__{c}_weighted_sum"))
+                aggs.append(nw.col(c).mean().alias(f"__{c}_fallback"))
+            else:
+                aggs.append(nw.col(c).mean().alias(c))
 
         for c in other_cols:
             aggs.append(nw.col(c).first().alias(c))
 
+        if has_weight:
+            aggs.append(nw.col(weight_col).sum().alias("__weight_sum"))
+
         out = df.group_by(keys).agg(aggs)
+
+        if has_weight:
+            weighted_cols = [c for c in num_cols if c != weight_col]
+            for c in weighted_cols:
+                out = out.with_columns(
+                    nw.when((~nw.col("__weight_sum").is_null()) & (nw.col("__weight_sum") != 0))
+                    .then(nw.col(f"__{c}_weighted_sum") / nw.col("__weight_sum"))
+                    .otherwise(nw.col(f"__{c}_fallback"))
+                    .alias(c)
+                )
+            drop_cols = [f"__{c}_weighted_sum" for c in weighted_cols]
+            drop_cols += [f"__{c}_fallback" for c in weighted_cols]
+            drop_cols.append("__weight_sum")
+            out = out.drop(drop_cols)
+
         return out
 
     @nw.narwhalify
@@ -59,12 +89,12 @@ class GroupByReducer(BaseEstimator, TransformerMixin):
         if sample_weight is not None:
             df = df.with_columns(nw.lit(sample_weight).alias("__sw"))
 
-        y_is_numeric = df.select(nw.col("__y")).schema["__y"].is_numeric()
+        y_uniques = df.group_by(keys).agg(nw.col("__y").n_unique().alias("__y_nunique"))
+        non_uniform = y_uniques.filter(nw.col("__y_nunique") > 1)
+        if len(non_uniform) > 0:
+            raise ValueError("Target (y) must be uniform within each granularity group")
 
-        if y_is_numeric:
-            agg_exprs = [nw.col("__y").mean().alias("__y")]
-        else:
-            agg_exprs = [nw.col("__y").first().alias("__y")]
+        agg_exprs = [nw.col("__y").first().alias("__y")]
 
         if sample_weight is not None:
             agg_exprs.append(nw.col("__sw").sum().alias("__sw"))

{spforge-0.8.8.dist-info → spforge-0.8.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spforge
-Version: 0.8.8
+Version: 0.8.19
 Summary: A flexible framework for generating features, ratings, and building machine learning or other models for training and inference on sports data.
 Author-email: Mathias Holmstrøm <mathiasholmstom@gmail.com>
 License: See LICENSE file