spforge 0.8.2__py3-none-any.whl → 0.8.8__py3-none-any.whl

tests/hyperparameter_tuning/test_estimator_tuner.py

@@ -0,0 +1,167 @@
+import numpy as np
+import pandas as pd
+import pytest
+from sklearn.base import BaseEstimator
+from sklearn.linear_model import LogisticRegression
+
+from spforge import EstimatorHyperparameterTuner, ParamSpec
+from spforge.cross_validator import MatchKFoldCrossValidator
+from spforge.estimator import SkLearnEnhancerEstimator
+from spforge.scorer import MeanBiasScorer
+
+
+class FakeLGBMClassifier(BaseEstimator):
+    __module__ = "lightgbm.sklearn"
+
+    def __init__(
+        self,
+        n_estimators: int = 100,
+        num_leaves: int = 31,
+        max_depth: int = 5,
+        min_child_samples: int = 20,
+        subsample: float = 1.0,
+        subsample_freq: int = 1,
+        reg_alpha: float = 0.0,
+        reg_lambda: float = 0.0,
+    ):
+        self.n_estimators = n_estimators
+        self.num_leaves = num_leaves
+        self.max_depth = max_depth
+        self.min_child_samples = min_child_samples
+        self.subsample = subsample
+        self.subsample_freq = subsample_freq
+        self.reg_alpha = reg_alpha
+        self.reg_lambda = reg_lambda
+
+    def fit(self, X, y):
+        self.classes_ = np.unique(y)
+        return self
+
+    def predict_proba(self, X):
+        n = len(X)
+        if len(self.classes_) < 2:
+            return np.ones((n, 1))
+        return np.tile([0.4, 0.6], (n, 1))
+
+    def predict(self, X):
+        n = len(X)
+        if len(self.classes_) == 1:
+            return np.full(n, self.classes_[0])
+        proba = self.predict_proba(X)
+        idx = np.argmax(proba, axis=1)
+        return np.array(self.classes_)[idx]
+
+
+@pytest.fixture
+def sample_df():
+    dates = pd.date_range("2024-01-01", periods=12, freq="D")
+    rows = []
+    for i, date in enumerate(dates):
+        rows.append(
+            {
+                "mid": f"M{i // 2}",
+                "date": date,
+                "x1": float(i),
+                "y": 1 if i % 2 == 0 else 0,
+            }
+        )
+    return pd.DataFrame(rows)
+
+
+@pytest.fixture
+def scorer():
+    return MeanBiasScorer(
+        pred_column="y_pred",
+        target="y",
+        validation_column="is_validation",
+    )
+
+
+def test_estimator_tuner_requires_search_space(sample_df, scorer):
+    estimator = LogisticRegression()
+
+    cv = MatchKFoldCrossValidator(
+        match_id_column_name="mid",
+        date_column_name="date",
+        target_column="y",
+        estimator=estimator,
+        prediction_column_name="y_pred",
+        n_splits=2,
+        features=["x1"],
+    )
+
+    tuner = EstimatorHyperparameterTuner(
+        estimator=estimator,
+        cross_validator=cv,
+        scorer=scorer,
+        direction="minimize",
+        n_trials=2,
+        show_progress_bar=False,
+    )
+
+    with pytest.raises(ValueError, match="param_search_space is required"):
+        tuner.optimize(sample_df)
+
+
+def test_estimator_tuner_custom_search_space(sample_df, scorer):
+    estimator = SkLearnEnhancerEstimator(estimator=LogisticRegression())
+
+    cv = MatchKFoldCrossValidator(
+        match_id_column_name="mid",
+        date_column_name="date",
+        target_column="y",
+        estimator=estimator,
+        prediction_column_name="y_pred",
+        n_splits=2,
+        features=["x1"],
+    )
+
+    tuner = EstimatorHyperparameterTuner(
+        estimator=estimator,
+        cross_validator=cv,
+        scorer=scorer,
+        direction="minimize",
+        param_search_space={
+            "C": ParamSpec(
+                param_type="float",
+                low=0.1,
+                high=2.0,
+                log=True,
+            )
+        },
+        n_trials=2,
+        show_progress_bar=False,
+    )
+
+    result = tuner.optimize(sample_df)
+
+    assert "estimator__C" in result.best_params
+    assert isinstance(result.best_value, float)
+
+
+def test_estimator_tuner_lgbm_defaults(sample_df, scorer):
+    estimator = FakeLGBMClassifier()
+
+    cv = MatchKFoldCrossValidator(
+        match_id_column_name="mid",
+        date_column_name="date",
+        target_column="y",
+        estimator=estimator,
+        prediction_column_name="y_pred",
+        n_splits=2,
+        features=["x1"],
+    )
+
+    tuner = EstimatorHyperparameterTuner(
+        estimator=estimator,
+        cross_validator=cv,
+        scorer=scorer,
+        direction="minimize",
+        n_trials=2,
+        show_progress_bar=False,
+    )
+
+    result = tuner.optimize(sample_df)
+
+    assert "n_estimators" in result.best_params
+    assert isinstance(result.best_value, float)

tests/ratings/test_player_rating_generator.py

@@ -1662,3 +1662,30 @@ def test_player_rating_team_with_strong_offense_and_weak_defense_gets_expected_r
 
     assert a_off > start_rating
     assert a_def < start_rating
+
+
+def test_fit_transform__player_rating_difference_from_team_projected_feature(base_cn, sample_df):
+    """PLAYER_RATING_DIFFERENCE_FROM_TEAM_PROJECTED computes player_off_rating - team_off_rating_projected."""
+    gen = PlayerRatingGenerator(
+        performance_column="perf",
+        column_names=base_cn,
+        auto_scale_performance=True,
+        features_out=[
+            RatingKnownFeatures.PLAYER_RATING_DIFFERENCE_FROM_TEAM_PROJECTED,
+            RatingKnownFeatures.PLAYER_OFF_RATING,
+            RatingKnownFeatures.TEAM_OFF_RATING_PROJECTED,
+        ],
+    )
+    result = gen.fit_transform(sample_df)
+
+    diff_col = "player_rating_difference_from_team_projected_perf"
+    player_col = "player_off_rating_perf"
+    team_col = "team_off_rating_projected_perf"
+
+    assert diff_col in result.columns
+    assert player_col in result.columns
+    assert team_col in result.columns
+
+    for row in result.iter_rows(named=True):
+        expected = row[player_col] - row[team_col]
+        assert row[diff_col] == pytest.approx(expected, rel=1e-9)

tests/scorer/test_score.py

@@ -372,6 +372,136 @@ def test_pwmse_compare_to_naive_granularity(df_type):
     assert abs(score - expected) < 1e-10
 
 
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_slices_predictions(df_type):
+    """PWMSE with evaluation_labels should only score on specified labels."""
+    # Predictions have 5 labels: [-2, -1, 0, 1, 2]
+    # But we only want to evaluate on inner labels: [-1, 0, 1]
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.1, 0.2, 0.4, 0.2, 0.1],  # Full distribution over 5 labels
+                [0.05, 0.15, 0.5, 0.2, 0.1],
+            ],
+            "target": [0, 1],
+        },
+    )
+
+    # Score with all labels
+    scorer_full = PWMSE(pred_column="pred", target="target", labels=[-2, -1, 0, 1, 2])
+    score_full = scorer_full.score(df)
+
+    # Score with evaluation_labels excluding boundaries
+    scorer_eval = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[-2, -1, 0, 1, 2],
+        evaluation_labels=[-1, 0, 1],
+    )
+    score_eval = scorer_eval.score(df)
+
+    # Scores should be different because evaluation_labels excludes boundary penalties
+    assert score_full != score_eval
+
+    # Manual calculation for evaluation_labels case:
+    # Slice predictions to indices 1, 2, 3 (corresponding to labels -1, 0, 1)
+    # Then renormalize
+    preds_full = np.array([[0.1, 0.2, 0.4, 0.2, 0.1], [0.05, 0.15, 0.5, 0.2, 0.1]])
+    preds_sliced = preds_full[:, 1:4]  # [-1, 0, 1]
+    preds_renorm = preds_sliced / preds_sliced.sum(axis=1, keepdims=True)
+
+    eval_labels = np.array([-1, 0, 1], dtype=np.float64)
+    targets = np.array([0, 1], dtype=np.float64)
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    expected = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    assert abs(score_eval - expected) < 1e-10
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_with_compare_to_naive(df_type):
+    """PWMSE evaluation_labels should also affect naive baseline calculation."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+            ],
+            "target": [-1, 0, 0, 1],  # Targets within evaluation range
+        },
+    )
+
+    scorer = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[-2, -1, 0, 1, 2],
+        evaluation_labels=[-1, 0, 1],
+        compare_to_naive=True,
+    )
+    score = scorer.score(df)
+
+    # Naive should be computed using only evaluation_labels
+    # With targets [-1, 0, 0, 1], naive probs are [1/4, 2/4, 1/4] for labels [-1, 0, 1]
+    eval_labels = np.array([-1, 0, 1], dtype=np.float64)
+    targets = np.array([-1, 0, 0, 1], dtype=np.float64)
+
+    # Model predictions sliced and renormalized
+    preds_full = np.array([[0.1, 0.2, 0.4, 0.2, 0.1]] * 4)
+    preds_sliced = preds_full[:, 1:4]
+    preds_renorm = preds_sliced / preds_sliced.sum(axis=1, keepdims=True)
+
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    model_score = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    # Naive predictions for evaluation_labels only
+    naive_probs = np.array([0.25, 0.5, 0.25])  # Based on target distribution
+    naive_preds = np.tile(naive_probs, (4, 1))
+    naive_score = float((diffs_sqd * naive_preds).sum(axis=1).mean())
+
+    expected = naive_score - model_score
+    assert abs(score - expected) < 1e-10
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_filters_targets_outside_range(df_type):
+    """PWMSE should filter out targets outside evaluation_labels range."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+                [0.1, 0.2, 0.4, 0.2, 0.1],
+            ],
+            "target": [-2, 0, 2],  # -2 and 2 are outside evaluation range [-1, 0, 1]
+        },
+    )
+
+    scorer = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[-2, -1, 0, 1, 2],
+        evaluation_labels=[-1, 0, 1],
+    )
+    score = scorer.score(df)
+
+    # Should only use the row with target=0
+    preds_full = np.array([[0.1, 0.2, 0.4, 0.2, 0.1]])
+    preds_sliced = preds_full[:, 1:4]
+    preds_renorm = preds_sliced / preds_sliced.sum(axis=1, keepdims=True)
+
+    eval_labels = np.array([-1, 0, 1], dtype=np.float64)
+    targets = np.array([0], dtype=np.float64)
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    expected = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    assert abs(score - expected) < 1e-10
+
+
 # ============================================================================
 # D. MeanBiasScorer Tests
 # ============================================================================
@@ -1762,6 +1892,129 @@ def test_pwmse__accepts_ndarray_predictions(df_type):
     assert score >= 0
 
 
+# ============================================================================
+# ThresholdEventScorer with granularity and compare_to_naive Tests
+# ============================================================================
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_threshold_event_scorer__granularity_with_compare_to_naive(df_type):
+    """ThresholdEventScorer fails when combining compare_to_naive with granularity.
+
+    Bug: When granularity is set, binary_scorer.score() returns a dict, but
+    the naive comparison tries to do dict - dict which fails with:
+    'unsupported operand type(s) for -: 'dict' and 'dict''
+    """
+    df = create_dataframe(
+        df_type,
+        {
+            "qtr": [1, 1, 1, 2, 2, 2],
+            "dist": [
+                [0.1, 0.2, 0.3, 0.4],
+                [0.2, 0.3, 0.3, 0.2],
+                [0.3, 0.4, 0.2, 0.1],
+                [0.4, 0.3, 0.2, 0.1],
+                [0.1, 0.1, 0.4, 0.4],
+                [0.2, 0.2, 0.3, 0.3],
+            ],
+            "ydstogo": [2.0, 3.0, 1.0, 2.0, 1.0, 3.0],
+            "rush_yards": [3, 2, 0, 1, 2, 4],
+        },
+    )
+
+    scorer = ThresholdEventScorer(
+        dist_column="dist",
+        threshold_column="ydstogo",
+        outcome_column="rush_yards",
+        labels=[0, 1, 2, 3],
+        compare_to_naive=True,
+        granularity=["qtr"],
+    )
+
+    result = scorer.score(df)
+
+    assert isinstance(result, dict)
+    assert len(result) == 2
+    assert (1,) in result
+    assert (2,) in result
+    assert all(isinstance(v, float) for v in result.values())
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_threshold_event_scorer__granularity_with_compare_to_naive_and_naive_granularity(df_type):
+    """ThresholdEventScorer with both granularity and naive_granularity."""
+    df = create_dataframe(
+        df_type,
+        {
+            "qtr": [1, 1, 1, 2, 2, 2],
+            "team": ["A", "A", "B", "A", "B", "B"],
+            "dist": [
+                [0.1, 0.2, 0.3, 0.4],
+                [0.2, 0.3, 0.3, 0.2],
+                [0.3, 0.4, 0.2, 0.1],
+                [0.4, 0.3, 0.2, 0.1],
+                [0.1, 0.1, 0.4, 0.4],
+                [0.2, 0.2, 0.3, 0.3],
+            ],
+            "ydstogo": [2.0, 3.0, 1.0, 2.0, 1.0, 3.0],
+            "rush_yards": [3, 2, 0, 1, 2, 4],
+        },
+    )
+
+    scorer = ThresholdEventScorer(
+        dist_column="dist",
+        threshold_column="ydstogo",
+        outcome_column="rush_yards",
+        labels=[0, 1, 2, 3],
+        compare_to_naive=True,
+        naive_granularity=["team"],
+        granularity=["qtr"],
+    )
+
+    result = scorer.score(df)
+
+    assert isinstance(result, dict)
+    assert len(result) == 2
+    assert (1,) in result
+    assert (2,) in result
+    assert all(isinstance(v, float) for v in result.values())
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_threshold_event_scorer__multi_column_granularity_with_compare_to_naive(df_type):
+    """ThresholdEventScorer with multi-column granularity and compare_to_naive."""
+    df = create_dataframe(
+        df_type,
+        {
+            "qtr": [1, 1, 2, 2],
+            "half": [1, 1, 2, 2],
+            "dist": [
+                [0.1, 0.2, 0.3, 0.4],
+                [0.2, 0.3, 0.3, 0.2],
+                [0.4, 0.3, 0.2, 0.1],
+                [0.1, 0.1, 0.4, 0.4],
+            ],
+            "ydstogo": [2.0, 3.0, 2.0, 1.0],
+            "rush_yards": [3, 2, 1, 2],
+        },
+    )
+
+    scorer = ThresholdEventScorer(
+        dist_column="dist",
+        threshold_column="ydstogo",
+        outcome_column="rush_yards",
+        labels=[0, 1, 2, 3],
+        compare_to_naive=True,
+        granularity=["qtr", "half"],
+    )
+
+    result = scorer.score(df)
+
+    assert isinstance(result, dict)
+    assert len(result) == 2
+    assert all(isinstance(v, float) for v in result.values())
+
+
 @pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
 def test_all_scorers_handle_all_nan_targets(df_type):
     """All scorers handle case where all targets are NaN"""
@@ -1795,3 +2048,93 @@ def test_all_scorers_handle_all_nan_targets(df_type):
         assert np.isnan(score) or score == 0.0
     except (ValueError, IndexError):
         pass
+SCORER_VALIDATION_CASES = [
+    pytest.param(
+        lambda: MeanBiasScorer(pred_column="pred", target="target", validation_column="is_validation"),
+        lambda: pd.DataFrame(
+            {
+                "pred": [2.0, 0.0],
+                "target": [1.0, 2.0],
+                "is_validation": [1, 0],
+            }
+        ),
+        id="mean_bias",
+    ),
+    pytest.param(
+        lambda: PWMSE(pred_column="pred", target="target", labels=[0, 1], validation_column="is_validation"),
+        lambda: pd.DataFrame(
+            {
+                "pred": [[0.7, 0.3], [0.4, 0.6]],
+                "target": [0, 1],
+                "is_validation": [1, 0],
+            }
+        ),
+        id="pwmse",
+    ),
+    pytest.param(
+        lambda: SklearnScorer(
+            scorer_function=mean_absolute_error, pred_column="pred", target="target", validation_column="is_validation"
+        ),
+        lambda: pd.DataFrame(
+            {
+                "pred": [1.0, 0.0],
+                "target": [1.0, 0.0],
+                "is_validation": [1, 0],
+            }
+        ),
+        id="sklearn",
+    ),
+    pytest.param(
+        lambda: ProbabilisticMeanBias(
+            pred_column="pred", target="target", class_column_name="classes", validation_column="is_validation"
+        ),
+        lambda: pd.DataFrame(
+            {
+                "pred": [[0.2, 0.8], [0.6, 0.4]],
+                "target": [1, 0],
+                "classes": [[0, 1], [0, 1]],
+                "is_validation": [1, 0],
+            }
+        ),
+        id="probabilistic_mean_bias",
+    ),
+    pytest.param(
+        lambda: OrdinalLossScorer(pred_column="pred", target="target", classes=[0, 1], validation_column="is_validation"),
+        lambda: pd.DataFrame(
+            {
+                "pred": [[0.2, 0.8], [0.6, 0.4]],
+                "target": [1, 0],
+                "is_validation": [1, 0],
+            }
+        ),
+        id="ordinal_loss",
+    ),
+    pytest.param(
+        lambda: ThresholdEventScorer(
+            dist_column="dist",
+            threshold_column="threshold",
+            outcome_column="outcome",
+            comparator=Operator.GREATER_THAN_OR_EQUALS,
+            validation_column="is_validation",
+        ),
+        lambda: pd.DataFrame(
+            {
+                "dist": [[0.2, 0.8], [0.6, 0.4], [0.3, 0.7]],
+                "threshold": [0.5, 0.2, 0.3],
+                "outcome": [1, 0, 1],
+                "is_validation": [1, 1, 0],
+            }
+        ),
+        id="threshold_event",
+    ),
+]
+
+
+@pytest.mark.parametrize("scorer_factory, df_factory", SCORER_VALIDATION_CASES)
+def test_scorers_respect_validation_column(scorer_factory, df_factory):
+    """Scorers should filter on validation_column when specified."""
+    df = df_factory()
+    df_valid = df[df["is_validation"] == 1]
+    score_all = scorer_factory().score(df)
+    score_valid = scorer_factory().score(df_valid)
+    assert score_all == score_valid

tests/test_feature_generator_pipeline.py

@@ -16,6 +16,49 @@ def column_names():
     )
 
 
+class PolarsOnlyGenerator:
+    def __init__(self):
+        self._features_out = ["polars_only_feature"]
+
+    @property
+    def features_out(self):
+        return self._features_out
+
+    def fit_transform(self, df, column_names=None):
+        if not isinstance(df, pl.DataFrame):
+            raise TypeError("Expected polars DataFrame")
+        return df.with_columns((pl.col("points") * 2).alias("polars_only_feature"))
+
+    def transform(self, df):
+        if not isinstance(df, pl.DataFrame):
+            raise TypeError("Expected polars DataFrame")
+        return df.with_columns((pl.col("points") * 2).alias("polars_only_feature"))
+
+    def future_transform(self, df):
+        return self.transform(df)
+
+
+def test_feature_generator_pipeline__passes_native_polars_to_custom_generator(column_names):
+    data = pl.DataFrame(
+        {
+            "game_id": [1, 1],
+            "team_id": ["A", "B"],
+            "player_id": ["p1", "p2"],
+            "date": pd.to_datetime(["2023-01-01", "2023-01-01"]),
+            "points": [10, 15],
+        }
+    )
+
+    pipeline = FeatureGeneratorPipeline(
+        feature_generators=[PolarsOnlyGenerator()],
+        column_names=column_names,
+    )
+
+    result = pipeline.fit_transform(data, column_names=column_names)
+
+    assert "polars_only_feature" in result.columns
+
+
 @pytest.mark.parametrize("df_type", [pd.DataFrame, pl.DataFrame])
 def test_feature_generator_pipeline__fit_transform_preserves_row_count(df_type, column_names):
     """FeatureGeneratorPipeline.fit_transform should preserve row count."""