spforge 0.8.4__py3-none-any.whl → 0.8.18__py3-none-any.whl

tests/scorer/test_score.py

@@ -2048,3 +2048,235 @@ def test_all_scorers_handle_all_nan_targets(df_type):
         assert np.isnan(score) or score == 0.0
     except (ValueError, IndexError):
         pass
+SCORER_VALIDATION_CASES = [
+    pytest.param(
+        lambda: MeanBiasScorer(pred_column="pred", target="target", validation_column="is_validation"),
+        lambda: pd.DataFrame(
+            {
+                "pred": [2.0, 0.0],
+                "target": [1.0, 2.0],
+                "is_validation": [1, 0],
+            }
+        ),
+        id="mean_bias",
+    ),
+    pytest.param(
+        lambda: PWMSE(pred_column="pred", target="target", labels=[0, 1], validation_column="is_validation"),
+        lambda: pd.DataFrame(
+            {
+                "pred": [[0.7, 0.3], [0.4, 0.6]],
+                "target": [0, 1],
+                "is_validation": [1, 0],
+            }
+        ),
+        id="pwmse",
+    ),
+    pytest.param(
+        lambda: SklearnScorer(
+            scorer_function=mean_absolute_error, pred_column="pred", target="target", validation_column="is_validation"
+        ),
+        lambda: pd.DataFrame(
+            {
+                "pred": [1.0, 0.0],
+                "target": [1.0, 0.0],
+                "is_validation": [1, 0],
+            }
+        ),
+        id="sklearn",
+    ),
+    pytest.param(
+        lambda: ProbabilisticMeanBias(
+            pred_column="pred", target="target", class_column_name="classes", validation_column="is_validation"
+        ),
+        lambda: pd.DataFrame(
+            {
+                "pred": [[0.2, 0.8], [0.6, 0.4]],
+                "target": [1, 0],
+                "classes": [[0, 1], [0, 1]],
+                "is_validation": [1, 0],
+            }
+        ),
+        id="probabilistic_mean_bias",
+    ),
+    pytest.param(
+        lambda: OrdinalLossScorer(pred_column="pred", target="target", classes=[0, 1], validation_column="is_validation"),
+        lambda: pd.DataFrame(
+            {
+                "pred": [[0.2, 0.8], [0.6, 0.4]],
+                "target": [1, 0],
+                "is_validation": [1, 0],
+            }
+        ),
+        id="ordinal_loss",
+    ),
+    pytest.param(
+        lambda: ThresholdEventScorer(
+            dist_column="dist",
+            threshold_column="threshold",
+            outcome_column="outcome",
+            comparator=Operator.GREATER_THAN_OR_EQUALS,
+            validation_column="is_validation",
+        ),
+        lambda: pd.DataFrame(
+            {
+                "dist": [[0.2, 0.8], [0.6, 0.4], [0.3, 0.7]],
+                "threshold": [0.5, 0.2, 0.3],
+                "outcome": [1, 0, 1],
+                "is_validation": [1, 1, 0],
+            }
+        ),
+        id="threshold_event",
+    ),
+]
+
+
+@pytest.mark.parametrize("scorer_factory, df_factory", SCORER_VALIDATION_CASES)
+def test_scorers_respect_validation_column(scorer_factory, df_factory):
+    """Scorers should filter on validation_column when specified."""
+    df = df_factory()
+    df_valid = df[df["is_validation"] == 1]
+    score_all = scorer_factory().score(df)
+    score_valid = scorer_factory().score(df_valid)
+    assert score_all == score_valid
+
+
+# ============================================================================
+# PWMSE evaluation_labels Extension Tests
+# ============================================================================
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_extends_predictions(df_type):
+    """PWMSE with evaluation_labels as superset extends predictions with small probs."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.3, 0.5, 0.2],
+                [0.2, 0.6, 0.2],
+            ],
+            "target": [0, 1],
+        },
+    )
+
+    scorer = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+        evaluation_labels=[-1, 0, 1, 2, 3],
+    )
+    score = scorer.score(df)
+
+    n_eval_labels = 5
+    eps = 1e-5
+    preds_original = np.array([[0.3, 0.5, 0.2], [0.2, 0.6, 0.2]])
+    extended = np.full((2, n_eval_labels), eps, dtype=np.float64)
+    extended[:, 1] = preds_original[:, 0]
+    extended[:, 2] = preds_original[:, 1]
+    extended[:, 3] = preds_original[:, 2]
+    row_sums = extended.sum(axis=1, keepdims=True)
+    preds_renorm = extended / row_sums
+
+    eval_labels = np.array([-1, 0, 1, 2, 3], dtype=np.float64)
+    targets = np.array([0, 1], dtype=np.float64)
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    expected = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    assert abs(score - expected) < 1e-10
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_exact_match(df_type):
+    """PWMSE with evaluation_labels identical to labels (no-op)."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.3, 0.5, 0.2],
+                [0.2, 0.6, 0.2],
+            ],
+            "target": [0, 1],
+        },
+    )
+
+    scorer_with_eval = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+        evaluation_labels=[0, 1, 2],
+    )
+    scorer_without_eval = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+    )
+
+    score_with = scorer_with_eval.score(df)
+    score_without = scorer_without_eval.score(df)
+
+    assert abs(score_with - score_without) < 1e-10
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_partial_overlap_raises(df_type):
+    """PWMSE with partial overlap between labels and evaluation_labels raises."""
+    with pytest.raises(ValueError, match="evaluation_labels must be a subset or superset"):
+        PWMSE(
+            pred_column="pred",
+            target="target",
+            labels=[0, 1, 2],
+            evaluation_labels=[1, 2, 3],
+        )
+
+
+@pytest.mark.parametrize("df_type", [pl.DataFrame, pd.DataFrame])
+def test_pwmse__evaluation_labels_extends_with_compare_to_naive(df_type):
+    """PWMSE extension mode works correctly with compare_to_naive."""
+    df = create_dataframe(
+        df_type,
+        {
+            "pred": [
+                [0.8, 0.15, 0.05],
+                [0.1, 0.7, 0.2],
+                [0.05, 0.15, 0.8],
+                [0.3, 0.4, 0.3],
+            ],
+            "target": [0, 1, 2, 1],
+        },
+    )
+
+    scorer = PWMSE(
+        pred_column="pred",
+        target="target",
+        labels=[0, 1, 2],
+        evaluation_labels=[-1, 0, 1, 2, 3],
+        compare_to_naive=True,
+    )
+    score = scorer.score(df)
+
+    n_eval_labels = 5
+    eps = 1e-5
+    preds_original = np.array([
+        [0.8, 0.15, 0.05],
+        [0.1, 0.7, 0.2],
+        [0.05, 0.15, 0.8],
+        [0.3, 0.4, 0.3],
+    ])
+    extended = np.full((4, n_eval_labels), eps, dtype=np.float64)
+    extended[:, 1] = preds_original[:, 0]
+    extended[:, 2] = preds_original[:, 1]
+    extended[:, 3] = preds_original[:, 2]
+    row_sums = extended.sum(axis=1, keepdims=True)
+    preds_renorm = extended / row_sums
+
+    eval_labels = np.array([-1, 0, 1, 2, 3], dtype=np.float64)
+    targets = np.array([0, 1, 2, 1], dtype=np.float64)
+    diffs_sqd = (eval_labels[None, :] - targets[:, None]) ** 2
+    model_score = float((diffs_sqd * preds_renorm).sum(axis=1).mean())
+
+    naive_probs = np.array([0.0, 0.25, 0.5, 0.25, 0.0])
+    naive_preds = np.tile(naive_probs, (4, 1))
+    naive_score = float((diffs_sqd * naive_preds).sum(axis=1).mean())
+
+    expected = naive_score - model_score
+    assert abs(score - expected) < 1e-10

tests/test_autopipeline.py

@@ -12,6 +12,7 @@ from sklearn.linear_model import LinearRegression, LogisticRegression
 
 from spforge import AutoPipeline
 from spforge.estimator import SkLearnEnhancerEstimator
+from spforge.scorer import Filter, Operator
 from spforge.transformers import EstimatorTransformer
 
 
@@ -231,6 +232,27 @@ def test_predict_proba(df_clf):
     assert np.allclose(proba.sum(axis=1), 1.0, atol=1e-6)
 
 
+def test_filter_columns_not_passed_to_estimator(frame):
+    df_pd = pd.DataFrame(
+        {"x": [1.0, 2.0, 3.0, 4.0], "keep": [1, 0, 1, 0], "y": [1.0, 2.0, 3.0, 4.0]}
+    )
+    df = df_pd if frame == "pd" else pl.from_pandas(df_pd)
+
+    model = AutoPipeline(
+        estimator=CaptureEstimator(),
+        estimator_features=["x"],
+        filters=[Filter(column_name="keep", value=1, operator=Operator.EQUALS)],
+    )
+
+    X = _select(df, ["x", "keep"])
+    y = _col(df, "y")
+    model.fit(X, y=y)
+
+    est = _inner_estimator(model)
+    assert "keep" in model.required_features
+    assert "keep" not in est.fit_columns
+
+
 def test_predict_proba_raises_if_not_supported(df_reg):
     model = AutoPipeline(
         estimator=LinearRegression(),
@@ -306,7 +328,18 @@ def test_infer_categorical_from_feature_names_when_only_numeric_features_given(d
     assert any(c.startswith("cat") for c in cap.fit_columns)
 
 
-def test_granularity_groups_rows_before_estimator_fit_and_predict(df_reg):
+def test_granularity_groups_rows_before_estimator_fit_and_predict(frame):
+    df_pd = pd.DataFrame(
+        {
+            "gameid": ["g1", "g1", "g2", "g2", "g3", "g3"],
+            "num1": [1.0, 2.0, np.nan, 4.0, 5.0, 6.0],
+            "num2": [10.0, 20.0, 30.0, 40.0, np.nan, 60.0],
+            "cat1": ["a", "b", "a", None, "b", "c"],
+            "y": [1.0, 1.0, 2.0, 2.0, 3.0, 3.0],
+        }
+    )
+    df = df_pd if frame == "pd" else pl.from_pandas(df_pd)
+
     model = AutoPipeline(
         estimator=CaptureEstimator(),
         estimator_features=["gameid", "num1", "num2", "cat1"],
@@ -317,16 +350,16 @@ def test_granularity_groups_rows_before_estimator_fit_and_predict(df_reg):
         remainder="drop",
     )
 
-    X = _select(df_reg, ["gameid", "num1", "num2", "cat1"])
-    y = _col(df_reg, "y")
+    X = _select(df, ["gameid", "num1", "num2", "cat1"])
+    y = _col(df, "y")
     model.fit(X, y=y)
 
     inner = _inner_estimator(model)
 
-    if isinstance(df_reg, pl.DataFrame):
-        n_groups = df_reg.select(pl.col("gameid").n_unique()).item()
+    if isinstance(df, pl.DataFrame):
+        n_groups = df.select(pl.col("gameid").n_unique()).item()
     else:
-        n_groups = df_reg["gameid"].nunique()
+        n_groups = df["gameid"].nunique()
 
     assert inner.fit_shape[0] == n_groups
 
@@ -551,3 +584,300 @@ def test_autopipeline_is_picklable_after_fit():
     model.fit(df, y)
 
     pickle.dumps(model)
+
+
+# --- Feature Importances Tests ---
+
+
+def test_feature_importances__tree_model():
+    from sklearn.ensemble import RandomForestRegressor
+
+    df = pd.DataFrame(
+        {
+            "num1": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
+            "num2": [10.0, 20.0, 30.0, 40.0, 50.0, 60.0],
+            "cat1": ["a", "b", "a", "b", "a", "b"],
+        }
+    )
+    y = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="y")
+
+    model = AutoPipeline(
+        estimator=RandomForestRegressor(n_estimators=5, random_state=42),
+        estimator_features=["num1", "num2", "cat1"],
+        categorical_handling="ordinal",
+    )
+    model.fit(df, y)
+
+    importances = model.feature_importances_
+
+    assert isinstance(importances, pd.DataFrame)
+    assert list(importances.columns) == ["feature", "importance"]
+    assert len(importances) == 3
+    assert set(importances["feature"].tolist()) == {"num1", "num2", "cat1"}
+    assert all(importances["importance"] >= 0)
+
+
+def test_feature_importances__linear_model():
+    df = pd.DataFrame(
+        {
+            "num1": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0],
+            "num2": [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0],
+        }
+    )
+    y = pd.Series([0, 1, 0, 1, 0, 1, 0, 1], name="y")
+
+    model = AutoPipeline(
+        estimator=LogisticRegression(max_iter=1000),
+        estimator_features=["num1", "num2"],
+        scale_features=True,
+    )
+    model.fit(df, y)
+
+    importances = model.feature_importances_
+
+    assert isinstance(importances, pd.DataFrame)
+    assert list(importances.columns) == ["feature", "importance"]
+    assert len(importances) == 2
+    assert set(importances["feature"].tolist()) == {"num1", "num2"}
+    assert all(importances["importance"] >= 0)
+
+
+def test_feature_importances__not_fitted_raises():
+    model = AutoPipeline(
+        estimator=LinearRegression(),
+        estimator_features=["x"],
+    )
+
+    with pytest.raises(RuntimeError, match="Pipeline not fitted"):
+        _ = model.feature_importances_
+
+
+def test_feature_importances__unsupported_estimator_raises():
+    df = pd.DataFrame({"x": [1.0, 2.0, 3.0, 4.0]})
+    y = pd.Series([1.0, 2.0, 3.0, 4.0], name="y")
+
+    model = AutoPipeline(
+        estimator=DummyRegressor(),
+        estimator_features=["x"],
+    )
+    model.fit(df, y)
+
+    with pytest.raises(RuntimeError, match="does not support feature importances"):
+        _ = model.feature_importances_
+
+
+def test_feature_importances__with_sklearn_enhancer():
+    from sklearn.ensemble import RandomForestRegressor
+
+    df = pd.DataFrame(
+        {
+            "num1": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
+            "num2": [10.0, 20.0, 30.0, 40.0, 50.0, 60.0],
+            "start_date": ["2022-01-01", "2022-01-02", "2022-01-03", "2022-01-04", "2022-01-05", "2022-01-06"],
+        }
+    )
+    y = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="y")
+
+    inner = RandomForestRegressor(n_estimators=5, random_state=42)
+    enhancer = SkLearnEnhancerEstimator(
+        estimator=inner,
+        date_column="start_date",
+        day_weight_epsilon=0.1,
+    )
+
+    model = AutoPipeline(
+        estimator=enhancer,
+        estimator_features=["num1", "num2"],
+    )
+    model.fit(df, y)
+
+    importances = model.feature_importances_
+
+    assert isinstance(importances, pd.DataFrame)
+    assert list(importances.columns) == ["feature", "importance"]
+    assert len(importances) == 2
+    assert set(importances["feature"].tolist()) == {"num1", "num2"}
+
+
+def test_feature_importances__onehot_features():
+    from sklearn.ensemble import RandomForestRegressor
+
+    df = pd.DataFrame(
+        {
+            "num1": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
+            "cat1": ["a", "b", "c", "a", "b", "c"],
+        }
+    )
+    y = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="y")
+
+    model = AutoPipeline(
+        estimator=RandomForestRegressor(n_estimators=5, random_state=42),
+        estimator_features=["num1", "cat1"],
+        categorical_handling="onehot",
+    )
+    model.fit(df, y)
+
+    importances = model.feature_importances_
+
+    assert isinstance(importances, pd.DataFrame)
+    assert list(importances.columns) == ["feature", "importance"]
+    # Should have expanded features: num1 + cat1_a, cat1_b, cat1_c
+    assert len(importances) == 4
+    assert "num1" in importances["feature"].tolist()
+    assert any("cat1_" in f for f in importances["feature"].tolist())
+
+
+def test_feature_importance_names__granularity_uses_deep_feature_names():
+    from sklearn.ensemble import RandomForestRegressor
+
+    df = pd.DataFrame(
+        {
+            "gameid": ["g1", "g1", "g2", "g2"],
+            "num1": [1.0, 2.0, 3.0, 4.0],
+            "num2": [10.0, 20.0, 30.0, 40.0],
+            "y": [1.0, 1.0, 2.0, 2.0],
+        }
+    )
+    y = df["y"]
+
+    model = AutoPipeline(
+        estimator=RandomForestRegressor(n_estimators=5, random_state=42),
+        estimator_features=["gameid", "num1", "num2"],
+        predictor_transformers=[AddConstantPredictionTransformer(col_name="const_pred")],
+        granularity=["gameid"],
+        categorical_features=["gameid"],
+        categorical_handling="ordinal",
+        remainder="drop",
+    )
+    model.fit(df, y)
+
+    names = model.feature_importance_names
+
+    inner = _inner_estimator(model)
+    assert list(names.keys()) == list(inner.feature_names_in_)
+    assert "gameid" not in names
+    assert "const_pred" in names
+
+
+@pytest.mark.parametrize("frame", ["pd", "pl"])
+def test_granularity_with_aggregation_weight__features_weighted(frame):
+    df_pd = pd.DataFrame(
+        {
+            "gameid": ["g1", "g1", "g2", "g2"],
+            "num1": [10.0, 30.0, 20.0, 40.0],
+            "weight": [0.25, 0.75, 0.5, 0.5],
+            "y": [1.0, 1.0, 2.0, 2.0],
+        }
+    )
+    df = df_pd if frame == "pd" else pl.from_pandas(df_pd)
+
+    cap = CaptureEstimator()
+    model = AutoPipeline(
+        estimator=cap,
+        estimator_features=["num1"],
+        granularity=["gameid"],
+        aggregation_weight="weight",
+        remainder="drop",
+    )
+
+    X = _select(df, ["gameid", "num1", "weight"])
+    y = _col(df, "y")
+    model.fit(X, y=y)
+
+    inner = _inner_estimator(model)
+    assert inner.fit_shape[0] == 2
+
+    preds = model.predict(X)
+    assert preds.shape[0] == len(X)
+
+
+@pytest.mark.parametrize("frame", ["pd", "pl"])
+def test_granularity_aggregation_weight__weighted_mean_correct(frame):
+    df_pd = pd.DataFrame(
+        {
+            "gameid": ["g1", "g1"],
+            "num1": [10.0, 30.0],
+            "weight": [0.25, 0.75],
+            "y": [1.0, 1.0],
+        }
+    )
+    df = df_pd if frame == "pd" else pl.from_pandas(df_pd)
+
+    from spforge.transformers._other_transformer import GroupByReducer
+
+    reducer = GroupByReducer(granularity=["gameid"], aggregation_weight="weight")
+    transformed = reducer.fit_transform(df)
+
+    if frame == "pl":
+        num1_val = transformed["num1"].to_list()[0]
+    else:
+        num1_val = transformed["num1"].iloc[0]
+
+    expected = (10.0 * 0.25 + 30.0 * 0.75) / (0.25 + 0.75)
+    assert abs(num1_val - expected) < 1e-6
+
+
+@pytest.mark.parametrize("frame", ["pd", "pl"])
+def test_reduce_y_raises_when_target_not_uniform_per_group(frame):
+    df_pd = pd.DataFrame(
+        {
+            "gameid": ["g1", "g1"],
+            "num1": [10.0, 30.0],
+        }
+    )
+    df = df_pd if frame == "pd" else pl.from_pandas(df_pd)
+
+    from spforge.transformers._other_transformer import GroupByReducer
+
+    reducer = GroupByReducer(granularity=["gameid"])
+
+    y = np.array([1.0, 2.0])
+    with pytest.raises(ValueError, match="Target.*must be uniform"):
+        reducer.reduce_y(df, y)
+
+
+@pytest.mark.parametrize("frame", ["pd", "pl"])
+def test_reduce_y_works_when_target_uniform_per_group(frame):
+    df_pd = pd.DataFrame(
+        {
+            "gameid": ["g1", "g1", "g2", "g2"],
+            "num1": [10.0, 30.0, 20.0, 40.0],
+        }
+    )
+    df = df_pd if frame == "pd" else pl.from_pandas(df_pd)
+
+    from spforge.transformers._other_transformer import GroupByReducer
+
+    reducer = GroupByReducer(granularity=["gameid"])
+
+    y = np.array([1.0, 1.0, 2.0, 2.0])
+    y_out, _ = reducer.reduce_y(df, y)
+
+    assert len(y_out) == 2
+    assert set(y_out) == {1.0, 2.0}
+
+
+@pytest.mark.parametrize("frame", ["pd", "pl"])
+def test_aggregation_weight_sums_weight_column(frame):
+    df_pd = pd.DataFrame(
+        {
+            "gameid": ["g1", "g1"],
+            "num1": [10.0, 30.0],
+            "weight": [0.25, 0.75],
+            "y": [1.0, 1.0],
+        }
+    )
+    df = df_pd if frame == "pd" else pl.from_pandas(df_pd)
+
+    from spforge.transformers._other_transformer import GroupByReducer
+
+    reducer = GroupByReducer(granularity=["gameid"], aggregation_weight="weight")
+    transformed = reducer.fit_transform(df)
+
+    if frame == "pl":
+        weight_val = transformed["weight"].to_list()[0]
+    else:
+        weight_val = transformed["weight"].iloc[0]
+
+    expected = 0.25 + 0.75
+    assert abs(weight_val - expected) < 1e-6

tests/test_feature_generator_pipeline.py

@@ -16,6 +16,49 @@ def column_names():
     )
 
 
+class PolarsOnlyGenerator:
+    def __init__(self):
+        self._features_out = ["polars_only_feature"]
+
+    @property
+    def features_out(self):
+        return self._features_out
+
+    def fit_transform(self, df, column_names=None):
+        if not isinstance(df, pl.DataFrame):
+            raise TypeError("Expected polars DataFrame")
+        return df.with_columns((pl.col("points") * 2).alias("polars_only_feature"))
+
+    def transform(self, df):
+        if not isinstance(df, pl.DataFrame):
+            raise TypeError("Expected polars DataFrame")
+        return df.with_columns((pl.col("points") * 2).alias("polars_only_feature"))
+
+    def future_transform(self, df):
+        return self.transform(df)
+
+
+def test_feature_generator_pipeline__passes_native_polars_to_custom_generator(column_names):
+    data = pl.DataFrame(
+        {
+            "game_id": [1, 1],
+            "team_id": ["A", "B"],
+            "player_id": ["p1", "p2"],
+            "date": pd.to_datetime(["2023-01-01", "2023-01-01"]),
+            "points": [10, 15],
+        }
+    )
+
+    pipeline = FeatureGeneratorPipeline(
+        feature_generators=[PolarsOnlyGenerator()],
+        column_names=column_names,
+    )
+
+    result = pipeline.fit_transform(data, column_names=column_names)
+
+    assert "polars_only_feature" in result.columns
+
+
 @pytest.mark.parametrize("df_type", [pd.DataFrame, pl.DataFrame])
 def test_feature_generator_pipeline__fit_transform_preserves_row_count(df_type, column_names):
     """FeatureGeneratorPipeline.fit_transform should preserve row count."""