dataforge-ml 0.1.0__py3-none-any.whl

tests/integration/test_structural_end_to_end.py

@@ -0,0 +1,219 @@
+import pytest
+from ...profiling.structural import StructuralProfiler
+from ...profiling.config import (
+    ProfileConfig,
+    StructuralProfileResult,
+    SemanticType,
+)
+from ...profiling._numeric_config import NumericStats
+from ...profiling._categorical_config import CategoricalStats
+from ...profiling._datetime_config import DatetimeStats
+from ...profiling._boolean_config import BooleanStats
+from ...profiling._text_config import TextStats
+from ...profiling._target_config import TargetProfileResult
+
+
+def test_happy_path(mixed_df):
+    config = ProfileConfig(compute_correlation=True)
+    result = StructuralProfiler(config).profile(mixed_df)
+
+    assert isinstance(result, StructuralProfileResult)
+    assert set(result.columns.keys()) == set(mixed_df.columns)
+    for col_profile in result.columns.values():
+        assert (
+            col_profile.semantic_type is not None
+        ), f"column '{col_profile.name}' has no semantic_type"
+
+
+    assert result.dataset.row_count == mixed_df.height
+    assert result.dataset.feature_correlation is not None
+
+
+def test_no_correlation(mixed_df):
+    config = ProfileConfig(compute_correlation=False)
+    result = StructuralProfiler(config).profile(mixed_df)
+
+    assert result.dataset.feature_correlation is None
+
+
+def test_boolean_handoff(mixed_df):
+    result = StructuralProfiler(ProfileConfig()).profile(mixed_df)
+
+    cp = result.columns["is_active"]
+    assert cp.semantic_type == SemanticType.Boolean
+    assert cp.stats is not None
+    assert isinstance(cp.stats, BooleanStats)
+    assert cp.stats.mode in (True, False, None)
+
+
+def test_text_handoff(text_df):
+    result = StructuralProfiler(ProfileConfig()).profile(text_df)
+
+    cp = result.columns["review"]
+    assert cp.semantic_type == SemanticType.Text
+    assert cp.stats is not None
+    assert isinstance(cp.stats, TextStats)
+
+    assert cp.stats.vocabulary_size > 0
+    assert cp.stats.char_length_max >= cp.stats.char_length_min
+    assert cp.stats.avg_token_count > 0
+    assert 0.0 <= cp.stats.empty_ratio <= 1.0
+
+
+def test_correlation_consistency(mixed_df):
+    config = ProfileConfig(compute_correlation=True)
+    result = StructuralProfiler(config).profile(mixed_df)
+
+    fc = result.dataset.feature_correlation
+    assert fc is not None
+
+    # age and income are correlated by construction — forward invariant must not be vacuous
+    assert len(fc.near_redundant_pairs) >= 1, (
+        "expected at least one near-redundant pair (age/income are strongly correlated)"
+    )
+
+    # Forward invariant: every near_redundant pair must have both columns co-located
+    # in the same NearRedundancyGroup
+    for pair in fc.pairwise:
+        if not pair.near_redundant:
+            continue
+        assert any(
+            pair.col_a in group.columns and pair.col_b in group.columns
+            for group in fc.near_redundancy_groups
+        ), (
+            f"near_redundant pair ({pair.col_a}, {pair.col_b}) "
+            f"not co-located in any NearRedundancyGroup"
+        )
+
+    # Backward invariant: every column in a redundancy group must have at least
+    # one near_redundant=True pair in pairwise
+    for group in fc.near_redundancy_groups:
+        for col in group.columns:
+            assert any(
+                (p.col_a == col or p.col_b == col) and p.near_redundant
+                for p in fc.pairwise
+            ), (
+                f"column '{col}' is in a NearRedundancyGroup but has no "
+                f"near_redundant=True pair in pairwise"
+            )
+
+    # Matrix symmetry — Pearson
+    for col_a, row in fc.pearson_matrix.items():
+        for col_b, val in row.items():
+            mirror = fc.pearson_matrix.get(col_b, {}).get(col_a)
+            assert mirror is not None and abs(val - mirror) < 1e-10, (
+                f"Pearson matrix asymmetry: [{col_a}][{col_b}]={val} "
+                f"vs [{col_b}][{col_a}]={mirror}"
+            )
+
+    # Matrix symmetry — Spearman
+    for col_a, row in fc.spearman_matrix.items():
+        for col_b, val in row.items():
+            mirror = fc.spearman_matrix.get(col_b, {}).get(col_a)
+            assert mirror is not None and abs(val - mirror) < 1e-10, (
+                f"Spearman matrix asymmetry: [{col_a}][{col_b}]={val} "
+                f"vs [{col_b}][{col_a}]={mirror}"
+            )
+
+    # Suggested drop is a strict subset of its group's columns
+    for group in fc.near_redundancy_groups:
+        group_cols = set(group.columns)
+        drop_cols = set(group.suggested_drop)
+        assert drop_cols < group_cols, (
+            f"suggested_drop {drop_cols} is not a strict subset of "
+            f"group columns {group_cols}"
+        )
+
+
+def test_column_handoffs(mixed_df):
+    result = StructuralProfiler(ProfileConfig()).profile(mixed_df)
+
+    stats_type_for = {
+        SemanticType.Numeric: NumericStats,
+        SemanticType.Categorical: CategoricalStats,
+        SemanticType.Datetime: DatetimeStats,
+        SemanticType.Boolean: BooleanStats,
+    }
+
+    for name, cp in result.columns.items():
+        expected_type = stats_type_for.get(cp.semantic_type)
+        if expected_type is None:
+            continue
+
+        assert cp.stats is not None, (
+            f"column '{name}' has semantic_type={cp.semantic_type} but stats is None"
+        )
+        assert isinstance(cp.stats, expected_type), (
+            f"column '{name}' has semantic_type={cp.semantic_type} "
+            f"but stats type is {type(cp.stats).__name__}, expected {expected_type.__name__}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Override: numeric column forced to Categorical via column_overrides
+# ---------------------------------------------------------------------------
+
+
+def test_column_override_changes_stats_type(override_df):
+    config = ProfileConfig(column_overrides={"score": SemanticType.Categorical})
+    result = StructuralProfiler(config).profile(override_df)
+    cp = result.columns["score"]
+    assert isinstance(cp.stats, CategoricalStats)
+
+
+# ---------------------------------------------------------------------------
+# Target profiling integration
+# ---------------------------------------------------------------------------
+
+
+def test_target_profiling_integration(target_df):
+    config = ProfileConfig(target_columns=["label"])
+    result = StructuralProfiler(config).profile(target_df)
+    assert "label" in result.targets
+    assert isinstance(result.targets["label"], TargetProfileResult)
+
+
+# ---------------------------------------------------------------------------
+# Empty DataFrame does not crash
+# ---------------------------------------------------------------------------
+
+
+def test_empty_dataframe_does_not_crash(empty_df):
+    result = StructuralProfiler(ProfileConfig()).profile(empty_df)
+    assert isinstance(result, StructuralProfileResult)
+
+
+# ---------------------------------------------------------------------------
+# Numeric handoff: float column produces NumericStats on ColumnProfile
+# ---------------------------------------------------------------------------
+
+
+def test_numeric_handoff(mixed_df):
+    result = StructuralProfiler(ProfileConfig()).profile(mixed_df)
+    cp = result.columns["income"]
+    assert cp.stats is not None
+    assert isinstance(cp.stats, NumericStats)
+
+
+# ---------------------------------------------------------------------------
+# Datetime handoff: date column produces DatetimeStats on ColumnProfile
+# ---------------------------------------------------------------------------
+
+
+def test_datetime_handoff(mixed_df):
+    result = StructuralProfiler(ProfileConfig()).profile(mixed_df)
+    cp = result.columns["joined"]
+    assert cp.stats is not None
+    assert isinstance(cp.stats, DatetimeStats)
+
+
+# ---------------------------------------------------------------------------
+# Missingness surfaced at column level for columns with nulls
+# ---------------------------------------------------------------------------
+
+
+def test_missingness_surfaced(mixed_df):
+    result = StructuralProfiler(ProfileConfig()).profile(mixed_df)
+    cp = result.columns["salary"]  # salary has ~10 % nulls by construction
+    assert cp.missingness is not None
+    assert cp.missingness.standard_null_count > 0

tests/unit/profiling/conftest.py

@@ -0,0 +1,81 @@
+from datetime import date, timedelta
+
+import polars as pl
+import pytest
+
+_BASE_DATE = date(2023, 1, 1)
+_N = 60
+
+
+@pytest.fixture(scope="session")
+def empty_df() -> pl.DataFrame:
+    return pl.DataFrame(
+        {
+            "score": pl.Series([], dtype=pl.Float64),
+            "count": pl.Series([], dtype=pl.Int64),
+            "category": pl.Series([], dtype=pl.Utf8),
+            "active": pl.Series([], dtype=pl.Boolean),
+            "event_date": pl.Series([], dtype=pl.Date),
+        }
+    )
+
+
+@pytest.fixture(scope="session")
+def all_null_df() -> pl.DataFrame:
+    nulls = [None] * _N
+    return pl.DataFrame(
+        {
+            "float_col": pl.Series(nulls, dtype=pl.Float64),
+            "int_col": pl.Series(nulls, dtype=pl.Int64),
+            "str_col": pl.Series(nulls, dtype=pl.Utf8),
+            "bool_col": pl.Series(nulls, dtype=pl.Boolean),
+        }
+    )
+
+
+@pytest.fixture(scope="session")
+def single_value_df() -> pl.DataFrame:
+    return pl.DataFrame(
+        {
+            "score": pl.Series([5.0] * _N, dtype=pl.Float64),
+            "count": pl.Series([1] * _N, dtype=pl.Int64),
+            "category": pl.Series(["X"] * _N, dtype=pl.Utf8),
+            "active": pl.Series([True] * _N, dtype=pl.Boolean),
+        }
+    )
+
+
+@pytest.fixture(scope="session")
+def single_row_df() -> pl.DataFrame:
+    return pl.DataFrame(
+        {
+            "score": pl.Series([42.0], dtype=pl.Float64),
+            "count": pl.Series([7], dtype=pl.Int64),
+            "category": pl.Series(["A"], dtype=pl.Utf8),
+            "active": pl.Series([True], dtype=pl.Boolean),
+            "event_date": pl.Series([_BASE_DATE], dtype=pl.Date),
+        }
+    )
+
+
+@pytest.fixture(scope="session")
+def normal_mixed_df() -> pl.DataFrame:
+    _CATEGORIES = ["A", "B", "C", "D", "E"]
+
+    scores = [round(1.5 + i * 1.7 + (i % 7) * 0.3, 2) for i in range(_N)]
+    counts = [i % 20 for i in range(_N)]
+    categories = [_CATEGORIES[i % len(_CATEGORIES)] for i in range(_N)]
+    active = [i % 2 == 0 for i in range(_N)]
+    dates = [_BASE_DATE + timedelta(days=i) for i in range(_N)]
+    salary = [None if i % 10 == 0 else round(30_000.0 + i * 500.0, 2) for i in range(_N)]
+
+    return pl.DataFrame(
+        {
+            "score": pl.Series(scores, dtype=pl.Float64),
+            "count": pl.Series(counts, dtype=pl.Int64),
+            "category": pl.Series(categories, dtype=pl.Utf8),
+            "active": pl.Series(active, dtype=pl.Boolean),
+            "event_date": pl.Series(dates, dtype=pl.Date),
+            "salary": pl.Series(salary, dtype=pl.Float64),
+        }
+    )

tests/unit/profiling/test_boolean_profiler.py

@@ -0,0 +1,91 @@
+import polars as pl
+
+from ....profiling._boolean_profiler import BooleanProfiler
+from ....profiling._boolean_config import BooleanProfileResult, BooleanStats
+
+
+# ---------------------------------------------------------------------------
+# Result type & analysed_columns
+# ---------------------------------------------------------------------------
+
+
+def test_result_type_and_analysed_columns():
+    df = pl.DataFrame(
+        {
+            "flag": pl.Series([True, False, True], dtype=pl.Boolean),
+            "score": pl.Series([1.0, 2.0, 3.0], dtype=pl.Float64),
+        }
+    )
+    result = BooleanProfiler().profile(df, ["flag", "score"])
+    assert isinstance(result, BooleanProfileResult)
+    assert "flag" in result.analysed_columns
+    assert "score" not in result.analysed_columns
+
+
+# ---------------------------------------------------------------------------
+# Counts
+# ---------------------------------------------------------------------------
+
+
+def test_true_false_count_sum_equals_non_null_count():
+    values = [True, False, True, None, True, False, None]
+    df = pl.DataFrame({"flag": pl.Series(values, dtype=pl.Boolean)})
+    stats = BooleanProfiler().profile(df, ["flag"]).columns["flag"]
+    non_null_count = df["flag"].drop_nulls().len()
+    assert stats.true_count + stats.false_count == non_null_count
+
+
+# ---------------------------------------------------------------------------
+# Ratios
+# ---------------------------------------------------------------------------
+
+
+def test_true_ratio_plus_false_ratio_equals_one():
+    values = [True, True, False, True, False, True]
+    df = pl.DataFrame({"flag": pl.Series(values, dtype=pl.Boolean)})
+    stats = BooleanProfiler().profile(df, ["flag"]).columns["flag"]
+    assert abs(stats.true_ratio + stats.false_ratio - 1.0) < 1e-10
+
+
+# ---------------------------------------------------------------------------
+# Mode
+# ---------------------------------------------------------------------------
+
+
+def test_tied_column_mode_is_none():
+    values = [True] * 5 + [False] * 5
+    df = pl.DataFrame({"flag": pl.Series(values, dtype=pl.Boolean)})
+    stats = BooleanProfiler().profile(df, ["flag"]).columns["flag"]
+    assert stats.mode is None
+
+
+# ---------------------------------------------------------------------------
+# Integer {0, 1} columns
+# ---------------------------------------------------------------------------
+
+
+def test_integer_01_eligible_with_correct_counts_and_ratios():
+    values = [1, 0, 1, 1, 0, None]
+    df = pl.DataFrame({"bin": pl.Series(values, dtype=pl.Int64)})
+    result = BooleanProfiler().profile(df, ["bin"])
+    assert "bin" in result.analysed_columns
+    stats = result.columns["bin"]
+    non_null = [v for v in values if v is not None]
+    expected_true = sum(non_null)
+    expected_false = len(non_null) - expected_true
+    assert stats.true_count == expected_true
+    assert stats.false_count == expected_false
+    assert abs(stats.true_ratio + stats.false_ratio - 1.0) < 1e-10
+
+
+# ---------------------------------------------------------------------------
+# All-null boolean column
+# ---------------------------------------------------------------------------
+
+
+def test_all_null_boolean_returns_default_stats_without_crashing():
+    df = pl.DataFrame({"flag": pl.Series([None, None, None], dtype=pl.Boolean)})
+    stats = BooleanProfiler().profile(df, ["flag"]).columns["flag"]
+    assert isinstance(stats, BooleanStats)
+    assert stats.true_count == 0
+    assert stats.false_count == 0

tests/unit/profiling/test_categorical_profiler.py

@@ -0,0 +1,182 @@
+import polars as pl
+import pytest
+
+from ....profiling._categorical import CategoricalProfiler
+from ....profiling._categorical_config import (
+    CategoricalFlag,
+    CategoricalProfileResult,
+    CategoricalStats,
+    ImbalanceMetrics,
+    RareCategoryStats,
+    TopValueEntry,
+)
+
+
+# ---------------------------------------------------------------------------
+# Result type & column eligibility
+# ---------------------------------------------------------------------------
+
+
+def test_result_type(normal_mixed_df):
+    result = CategoricalProfiler().profile(normal_mixed_df, ["category"])
+    assert isinstance(result, CategoricalProfileResult)
+
+
+def test_analysed_columns_only_eligible(normal_mixed_df):
+    result = CategoricalProfiler().profile(normal_mixed_df, ["category", "score"])
+    assert "score" not in result.analysed_columns
+    assert "category" in result.analysed_columns
+
+
+def test_analysed_columns_matches_columns_dict(normal_mixed_df):
+    result = CategoricalProfiler().profile(normal_mixed_df, ["category"])
+    assert set(result.analysed_columns) == set(result.columns.keys())
+
+
+def test_stats_type_per_column(normal_mixed_df):
+    result = CategoricalProfiler().profile(normal_mixed_df, ["category"])
+    assert isinstance(result.columns["category"], CategoricalStats)
+
+
+# ---------------------------------------------------------------------------
+# Cardinality & unique_ratio
+# ---------------------------------------------------------------------------
+
+
+def test_cardinality_equals_distinct_non_null_count():
+    # 5 distinct values, no nulls
+    data = ["A", "B", "C", "D", "E"] * 12  # 60 rows, 5 distinct
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    assert stats.cardinality == 5
+
+
+def test_cardinality_excludes_nulls():
+    # 4 distinct non-null + some nulls
+    data = ["A", "B", "C", "D"] * 10 + [None] * 20
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    assert stats.cardinality == 4
+
+
+def test_unique_ratio_equals_cardinality_over_n_rows():
+    data = ["A", "B", "C", "D", "E"] * 12  # 5 distinct, 60 rows
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    assert abs(stats.unique_ratio - 5 / 60) < 1e-10
+
+
+# ---------------------------------------------------------------------------
+# top_values
+# ---------------------------------------------------------------------------
+
+
+def test_top_values_at_most_ten_entries():
+    # 15 distinct categories → top_values capped at 10
+    data = [str(i) for i in range(15)] * 4  # 60 rows, 15 distinct
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    assert len(stats.top_values) == 10
+
+
+def test_top_values_descending_count_order():
+    # Uneven counts: A=30, B=20, C=10
+    data = ["A"] * 30 + ["B"] * 20 + ["C"] * 10
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    counts = [e.count for e in stats.top_values]
+    assert counts == sorted(counts, reverse=True)
+
+
+def test_top_values_entries_are_top_value_entry_type():
+    data = ["X", "Y", "Z"] * 20
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    for entry in stats.top_values:
+        assert isinstance(entry, TopValueEntry)
+
+
+# ---------------------------------------------------------------------------
+# Imbalance metrics
+# ---------------------------------------------------------------------------
+
+
+def test_imbalance_fields_present_for_multi_category():
+    data = ["A"] * 30 + ["B"] * 20 + ["C"] * 10
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    assert isinstance(stats.imbalance, ImbalanceMetrics)
+    assert stats.imbalance.shannon_entropy > 0.0
+    assert stats.imbalance.gini_impurity > 0.0
+    assert stats.imbalance.class_ratio >= 1.0
+
+
+def test_imbalance_class_ratio_is_one_for_balanced():
+    # Equal counts → max_freq == min_freq → class_ratio = 1.0
+    data = ["A"] * 20 + ["B"] * 20 + ["C"] * 20
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    assert abs(stats.imbalance.class_ratio - 1.0) < 1e-10
+
+
+# ---------------------------------------------------------------------------
+# NearConstant flag
+# ---------------------------------------------------------------------------
+
+
+def test_near_constant_flag_set():
+    # 55/60 = 0.917 > 0.90 → NearConstant
+    data = ["A"] * 55 + ["B"] * 5
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    assert CategoricalFlag.NearConstant in stats.flags
+
+
+def test_near_constant_flag_absent_for_balanced():
+    data = ["A"] * 20 + ["B"] * 20 + ["C"] * 20
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    assert CategoricalFlag.NearConstant not in stats.flags
+
+
+# ---------------------------------------------------------------------------
+# Rare categories
+# ---------------------------------------------------------------------------
+
+
+def test_rare_category_count_correct():
+    # 200 rows: threshold_abs = max(1, floor(0.01*200)) = 2
+    # "C" appears once → count=1 < 2 → rare
+    data = ["A"] * 190 + ["B"] * 9 + ["C"] * 1
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    assert stats.rare_categories.rare_category_count == 1
+
+
+def test_rare_category_count_zero_when_none_rare():
+    # All categories appear frequently
+    data = ["A"] * 100 + ["B"] * 100
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    assert stats.rare_categories.rare_category_count == 0
+
+
+# ---------------------------------------------------------------------------
+# MixedType flag
+# ---------------------------------------------------------------------------
+
+
+def test_mixed_type_flag_set():
+    # 10 numeric-looking strings + 50 non-numeric → minority pct ≈ 16.7%
+    # Wilson lower bound well above 5% threshold → MixedType
+    data = ["apple"] * 50 + ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    assert CategoricalFlag.MixedType in stats.flags
+
+
+def test_mixed_type_flag_absent_for_pure_strings():
+    data = ["apple", "banana", "cherry"] * 20
+    df = pl.DataFrame({"cat": pl.Series(data, dtype=pl.Utf8)})
+    stats = CategoricalProfiler().profile(df, ["cat"]).columns["cat"]
+    assert CategoricalFlag.MixedType not in stats.flags