dataforge-ml 0.1.0__py3-none-any.whl

tests/unit/profiling/test_correlation_profiler.py

@@ -0,0 +1,124 @@
+import polars as pl
+
+from ....profiling._correlation_profiler import CorrelationProfiler
+from ....profiling._correlation_config import CorrelationProfileResult
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_df_three_cols():
+    """Three numeric columns with moderate (non-trivial) correlations."""
+    n = 50
+    a = [float(i % 7) for i in range(n)]
+    b = [float((i * 3) % 11) for i in range(n)]
+    c = [float(i % 13) for i in range(n)]
+    return pl.DataFrame(
+        {
+            "a": pl.Series(a, dtype=pl.Float64),
+            "b": pl.Series(b, dtype=pl.Float64),
+            "c": pl.Series(c, dtype=pl.Float64),
+        }
+    )
+
+
+def _make_df_with_duplicate():
+    """Two identical columns plus one independent column."""
+    n = 40
+    vals = [float(i) for i in range(n)]
+    return pl.DataFrame(
+        {
+            "x": pl.Series(vals, dtype=pl.Float64),
+            "x_copy": pl.Series(vals, dtype=pl.Float64),
+            "y": pl.Series([v * 0.3 + 7.0 for v in vals], dtype=pl.Float64),
+        }
+    )
+
+
+# ---------------------------------------------------------------------------
+# Pearson matrix symmetry  (profile_features)
+# ---------------------------------------------------------------------------
+
+
+def test_pearson_matrix_is_symmetric():
+    df = _make_df_three_cols()
+    cols = ["a", "b", "c"]
+    result = CorrelationProfiler(numeric_columns=cols).profile_features(df, cols)
+    for col_x in cols:
+        for col_y in cols:
+            assert result.pearson_matrix[col_x][col_y] == result.pearson_matrix[col_y][col_x]
+
+
+# ---------------------------------------------------------------------------
+# Spearman matrix symmetry  (profile_features)
+# ---------------------------------------------------------------------------
+
+
+def test_spearman_matrix_is_symmetric():
+    df = _make_df_three_cols()
+    cols = ["a", "b", "c"]
+    result = CorrelationProfiler(numeric_columns=cols).profile_features(df, cols)
+    for col_x in cols:
+        for col_y in cols:
+            assert result.spearman_matrix[col_x][col_y] == result.spearman_matrix[col_y][col_x]
+
+
+# ---------------------------------------------------------------------------
+# Every group column appears in at least one near-redundant pairwise entry
+# (profile_features)
+# ---------------------------------------------------------------------------
+
+
+def test_near_redundancy_group_columns_have_redundant_pairs():
+    df = _make_df_with_duplicate()
+    cols = ["x", "x_copy", "y"]
+    result = CorrelationProfiler(numeric_columns=cols).profile_features(df, cols)
+    for group in result.near_redundancy_groups:
+        for col in group.columns:
+            has_pair = any(
+                (p.col_a == col or p.col_b == col) and p.near_redundant
+                for p in result.pairwise
+            )
+            assert has_pair
+
+
+# ---------------------------------------------------------------------------
+# suggested_drop is a strict subset of the group's columns  (profile_features)
+# ---------------------------------------------------------------------------
+
+
+def test_suggested_drop_is_strict_subset_of_group_columns():
+    df = _make_df_with_duplicate()
+    cols = ["x", "x_copy", "y"]
+    result = CorrelationProfiler(numeric_columns=cols).profile_features(df, cols)
+    for group in result.near_redundancy_groups:
+        drop_set = set(group.suggested_drop)
+        col_set = set(group.columns)
+        assert drop_set < col_set  # non-empty and strictly smaller
+
+
+# ---------------------------------------------------------------------------
+# Identical columns produce a near-redundant pair; profile_target also works
+# ---------------------------------------------------------------------------
+
+
+def test_identical_columns_produce_near_redundant_pair():
+    df = _make_df_with_duplicate()
+    feature_cols = ["x", "x_copy", "y"]
+    profiler = CorrelationProfiler(numeric_columns=feature_cols)
+
+    # profile_features: identical pair must be flagged near-redundant
+    feature_result = profiler.profile_features(df, feature_cols)
+    assert any(
+        {p.col_a, p.col_b} == {"x", "x_copy"}
+        for p in feature_result.near_redundant_pairs
+    )
+
+    # profile_target: second entry point must run without error and attach target info
+    target_result = profiler.profile_target(
+        df, feature_result, feature_cols, [], "y"
+    )
+    assert isinstance(target_result, CorrelationProfileResult)
+    assert target_result.target_column == "y"

tests/unit/profiling/test_datetime_profiler.py

@@ -0,0 +1,133 @@
+from datetime import date, datetime, timedelta, timezone
+
+import polars as pl
+
+from ....profiling._datetime_profiler import DatetimeProfiler
+from ....profiling._datetime_config import (
+    DatetimeFlag,
+    DatetimeProfileResult,
+    InferredGranularity,
+)
+
+
+# ---------------------------------------------------------------------------
+# Result type & analysed_columns
+# ---------------------------------------------------------------------------
+
+
+def test_result_type_and_analysed_columns():
+    df = pl.DataFrame(
+        {
+            "event_date": pl.Series(
+                [date(2024, 1, i + 1) for i in range(5)], dtype=pl.Date
+            ),
+            "score": pl.Series([1.0, 2.0, 3.0, 4.0, 5.0], dtype=pl.Float64),
+        }
+    )
+    result = DatetimeProfiler().profile(df, ["event_date", "score"])
+    assert isinstance(result, DatetimeProfileResult)
+    assert "event_date" in result.analysed_columns
+    assert "score" not in result.analysed_columns
+
+
+# ---------------------------------------------------------------------------
+# Range: min_date <= max_date
+# ---------------------------------------------------------------------------
+
+
+def test_min_date_lte_max_date():
+    dates = [date(2024, 1, 1), date(2024, 6, 15), date(2024, 3, 10)]
+    df = pl.DataFrame({"ts": pl.Series(dates, dtype=pl.Date)})
+    stats = DatetimeProfiler().profile(df, ["ts"]).columns["ts"]
+    assert stats.min_date <= stats.max_date
+
+
+# ---------------------------------------------------------------------------
+# Range: range_days non-negative and consistent with min/max
+# ---------------------------------------------------------------------------
+
+
+def test_range_days_non_negative_and_consistent():
+    dates = [date(2024, 1, 1) + timedelta(days=i) for i in range(30)]
+    df = pl.DataFrame({"ts": pl.Series(dates, dtype=pl.Date)})
+    stats = DatetimeProfiler().profile(df, ["ts"]).columns["ts"]
+    assert stats.date_range_days >= 0
+    expected_days = (stats.max_date - stats.min_date).total_seconds() / 86_400.0
+    assert abs(stats.date_range_days - expected_days) < 1e-6
+
+
+# ---------------------------------------------------------------------------
+# FutureDates flag present and absent
+# ---------------------------------------------------------------------------
+
+
+def test_future_dates_flag_present_and_absent():
+    past_dates = [date(2020, 1, 1), date(2021, 6, 1), date(2022, 12, 31)]
+    future_dates = [date(2099, 1, 1), date(2100, 6, 1)]
+
+    df_future = pl.DataFrame(
+        {"ts": pl.Series(past_dates + future_dates, dtype=pl.Date)}
+    )
+    stats_future = DatetimeProfiler().profile(df_future, ["ts"]).columns["ts"]
+    assert DatetimeFlag.FutureDates in stats_future.flags
+
+    df_past = pl.DataFrame({"ts": pl.Series(past_dates, dtype=pl.Date)})
+    stats_past = DatetimeProfiler().profile(df_past, ["ts"]).columns["ts"]
+    assert DatetimeFlag.FutureDates not in stats_past.flags
+
+
+# ---------------------------------------------------------------------------
+# Inferred granularity bands: daily, hourly, monthly
+# ---------------------------------------------------------------------------
+
+
+def test_inferred_granularity_daily_hourly_monthly():
+    base = datetime(2024, 1, 1, tzinfo=timezone.utc)
+
+    # Daily: 1-day gaps → median gap ≈ 86 400 s → Daily band
+    daily_ts = [base + timedelta(days=i) for i in range(30)]
+    df_daily = pl.DataFrame(
+        {"ts": pl.Series(daily_ts, dtype=pl.Datetime("us", "UTC"))}
+    )
+    stats_daily = DatetimeProfiler().profile(df_daily, ["ts"]).columns["ts"]
+    assert stats_daily.inferred_granularity == InferredGranularity.Daily
+
+    # Hourly: 1-hour gaps → median gap ≈ 3 600 s → Hourly band
+    hourly_ts = [base + timedelta(hours=i) for i in range(48)]
+    df_hourly = pl.DataFrame(
+        {"ts": pl.Series(hourly_ts, dtype=pl.Datetime("us", "UTC"))}
+    )
+    stats_hourly = DatetimeProfiler().profile(df_hourly, ["ts"]).columns["ts"]
+    assert stats_hourly.inferred_granularity == InferredGranularity.Hourly
+
+    # Monthly: first day of each month → median gap ≈ 30 days → Monthly band
+    monthly_ts = [datetime(2024, m, 1, tzinfo=timezone.utc) for m in range(1, 13)]
+    df_monthly = pl.DataFrame(
+        {"ts": pl.Series(monthly_ts, dtype=pl.Datetime("us", "UTC"))}
+    )
+    stats_monthly = DatetimeProfiler().profile(df_monthly, ["ts"]).columns["ts"]
+    assert stats_monthly.inferred_granularity == InferredGranularity.Monthly
+
+
+# ---------------------------------------------------------------------------
+# Temporal signals: month and day-of-week vary, year does not
+# ---------------------------------------------------------------------------
+
+
+def test_temporal_signals_month_and_dow_vary_not_year():
+    # Six dates in 2024 across distinct months and distinct weekdays, same year.
+    # 2024-01-01 = Monday (0), 2024-02-04 = Sunday (6), 2024-03-06 = Wednesday (2)
+    # 2024-04-12 = Friday (4), 2024-05-14 = Tuesday (1), 2024-06-22 = Saturday (5)
+    dates_2024 = [
+        date(2024, 1, 1),
+        date(2024, 2, 4),
+        date(2024, 3, 6),
+        date(2024, 4, 12),
+        date(2024, 5, 14),
+        date(2024, 6, 22),
+    ]
+    df = pl.DataFrame({"ts": pl.Series(dates_2024, dtype=pl.Date)})
+    stats = DatetimeProfiler().profile(df, ["ts"]).columns["ts"]
+    assert stats.signals.has_year is False
+    assert stats.signals.has_month is True
+    assert stats.signals.has_day_of_week is True

tests/unit/profiling/test_missingness_profiler.py

@@ -0,0 +1,51 @@
+import polars as pl
+
+from ....profiling._missingness_profiler import MissingnessProfiler
+
+
+# ---------------------------------------------------------------------------
+# null_count equals actual null count in the series
+# ---------------------------------------------------------------------------
+
+
+def test_null_count_equals_actual_null_count():
+    values = [1, None, 3, None, None, 6, 7, None]
+    df = pl.DataFrame({"x": pl.Series(values, dtype=pl.Int64)})
+    profile = MissingnessProfiler().profile(df, ["x"]).columns["x"]
+    assert profile.standard_null_count == df["x"].null_count()
+
+
+# ---------------------------------------------------------------------------
+# null_ratio equals null_count / n_rows
+# ---------------------------------------------------------------------------
+
+
+def test_null_ratio_equals_null_count_over_n_rows():
+    values = [10, None, 30, None, 50]
+    df = pl.DataFrame({"x": pl.Series(values, dtype=pl.Int64)})
+    profile = MissingnessProfiler().profile(df, ["x"]).columns["x"]
+    expected_ratio = profile.standard_null_count / df.height
+    assert abs(profile.standard_null_ratio - expected_ratio) < 1e-10
+
+
+# ---------------------------------------------------------------------------
+# All-null column produces null_ratio == 1.0 without crashing
+# ---------------------------------------------------------------------------
+
+
+def test_all_null_column_ratio_is_one():
+    df = pl.DataFrame({"x": pl.Series([None, None, None, None], dtype=pl.Int64)})
+    profile = MissingnessProfiler().profile(df, ["x"]).columns["x"]
+    assert profile.effective_null_ratio == 1.0
+
+
+# ---------------------------------------------------------------------------
+# Fully populated column has null_count == 0 and null_ratio == 0.0
+# ---------------------------------------------------------------------------
+
+
+def test_fully_populated_column_has_zero_nulls():
+    df = pl.DataFrame({"x": pl.Series([1, 2, 3, 4, 5], dtype=pl.Int64)})
+    profile = MissingnessProfiler().profile(df, ["x"]).columns["x"]
+    assert profile.standard_null_count == 0
+    assert profile.standard_null_ratio == 0.0

tests/unit/profiling/test_numeric_profiler.py

@@ -0,0 +1,212 @@
+import polars as pl
+import pytest
+
+from ....profiling._numeric_profiler import NumericProfiler
+from ....profiling._numeric_config import (
+    KurtosisTag,
+    NumericFlag,
+    NumericProfileResult,
+    NumericStats,
+    PercentileSnapshot,
+    SkewSeverity,
+)
+
+
+# ---------------------------------------------------------------------------
+# Result type & column eligibility
+# ---------------------------------------------------------------------------
+
+
+def test_result_type(normal_mixed_df):
+    result = NumericProfiler().profile(normal_mixed_df, ["score"])
+    assert isinstance(result, NumericProfileResult)
+
+
+def test_analysed_columns_only_eligible(normal_mixed_df):
+    result = NumericProfiler().profile(normal_mixed_df, ["score", "salary", "category"])
+    assert "category" not in result.analysed_columns
+    assert "score" in result.analysed_columns
+    assert "salary" in result.analysed_columns
+
+
+def test_analysed_columns_matches_columns_dict(normal_mixed_df):
+    result = NumericProfiler().profile(normal_mixed_df, ["score", "salary"])
+    assert set(result.analysed_columns) == set(result.columns.keys())
+
+
+# ---------------------------------------------------------------------------
+# Core stats present for a normal float column
+# ---------------------------------------------------------------------------
+
+
+def test_core_stats_non_null_for_float(normal_mixed_df):
+    stats = NumericProfiler().profile(normal_mixed_df, ["score"]).columns["score"]
+    assert stats.mean is not None
+    assert stats.median is not None
+    assert stats.std is not None
+    assert stats.min is not None
+    assert stats.max is not None
+    assert stats.mean_median_ratio is not None
+
+
+def test_min_lte_max(normal_mixed_df):
+    stats = NumericProfiler().profile(normal_mixed_df, ["score"]).columns["score"]
+    assert stats.min <= stats.max
+
+
+# ---------------------------------------------------------------------------
+# All-null column
+# ---------------------------------------------------------------------------
+
+
+def test_all_null_column_no_crash(all_null_df):
+    result = NumericProfiler().profile(all_null_df, ["float_col"])
+    assert "float_col" in result.analysed_columns
+    stats = result.columns["float_col"]
+    assert isinstance(stats, NumericStats)
+    assert stats.mean is None
+    assert stats.std is None
+    assert stats.min is None
+    assert stats.max is None
+
+
+# ---------------------------------------------------------------------------
+# Single-value column
+# ---------------------------------------------------------------------------
+
+
+def test_single_value_std_and_skewness_zero(single_value_df):
+    stats = NumericProfiler().profile(single_value_df, ["score"]).columns["score"]
+    assert stats.std == 0.0
+    assert stats.skewness == 0.0
+
+
+# ---------------------------------------------------------------------------
+# ScaleAnomaly flag
+# ---------------------------------------------------------------------------
+
+
+def test_scale_anomaly_flag_set():
+    # 0.5 to 5000 → ratio = 10 000 ≥ 10^3 → flag
+    df = pl.DataFrame({"v": pl.Series([0.5, 1.0, 1.5, 2.0, 5000.0] * 12, dtype=pl.Float64)})
+    stats = NumericProfiler().profile(df, ["v"]).columns["v"]
+    assert NumericFlag.ScaleAnomaly in stats.flags
+
+
+def test_scale_anomaly_flag_absent_normal_range():
+    df = pl.DataFrame({"v": pl.Series([10.0, 20.0, 30.0, 40.0, 50.0] * 12, dtype=pl.Float64)})
+    stats = NumericProfiler().profile(df, ["v"]).columns["v"]
+    assert NumericFlag.ScaleAnomaly not in stats.flags
+
+
+# ---------------------------------------------------------------------------
+# NearConstant flag
+# ---------------------------------------------------------------------------
+
+
+def test_near_constant_flag_set():
+    # 55/60 = 0.917 > 0.90 → flag
+    data = [5.0] * 55 + [1.0, 2.0, 3.0, 4.0, 6.0]
+    df = pl.DataFrame({"v": pl.Series(data, dtype=pl.Float64)})
+    stats = NumericProfiler().profile(df, ["v"]).columns["v"]
+    assert NumericFlag.NearConstant in stats.flags
+
+
+def test_near_constant_flag_absent():
+    # 30/60 = 0.50 ≤ 0.90 → no flag
+    data = [5.0] * 30 + [6.0] * 30
+    df = pl.DataFrame({"v": pl.Series(data, dtype=pl.Float64)})
+    stats = NumericProfiler().profile(df, ["v"]).columns["v"]
+    assert NumericFlag.NearConstant not in stats.flags
+
+
+# ---------------------------------------------------------------------------
+# Skewness severity bands
+# ---------------------------------------------------------------------------
+
+
+def test_skewness_severity_normal():
+    # Symmetric uniform 1–60 → |skew| ≈ 0 → Normal
+    data = [float(i) for i in range(1, 61)]
+    df = pl.DataFrame({"v": pl.Series(data, dtype=pl.Float64)})
+    stats = NumericProfiler().profile(df, ["v"]).columns["v"]
+    assert stats.skewness_severity == SkewSeverity.Normal
+
+
+def test_skewness_severity_severe():
+    # 57 near-zero values + 3 extreme values → |skew| >> 2.0 → Severe
+    data = [0.1] * 57 + [100.0, 200.0, 300.0]
+    df = pl.DataFrame({"v": pl.Series(data, dtype=pl.Float64)})
+    stats = NumericProfiler().profile(df, ["v"]).columns["v"]
+    assert stats.skewness_severity == SkewSeverity.Severe
+
+
+# ---------------------------------------------------------------------------
+# Kurtosis tag bands
+# ---------------------------------------------------------------------------
+
+
+def test_kurtosis_tag_leptokurtic():
+    # Mass concentrated at 5.0 with symmetric outliers → excess kurtosis >> 3.0
+    data = [5.0] * 54 + [0.1, 0.1, 0.1, 9.9, 9.9, 9.9]
+    df = pl.DataFrame({"v": pl.Series(data, dtype=pl.Float64)})
+    stats = NumericProfiler().profile(df, ["v"]).columns["v"]
+    assert stats.kurtosis_tag == KurtosisTag.Leptokurtic
+
+
+def test_kurtosis_tag_platykurtic():
+    # Uniform over 4 equally-spaced values → excess kurtosis < -1.0
+    data = [1.0] * 15 + [4.0] * 15 + [7.0] * 15 + [10.0] * 15
+    df = pl.DataFrame({"v": pl.Series(data, dtype=pl.Float64)})
+    stats = NumericProfiler().profile(df, ["v"]).columns["v"]
+    assert stats.kurtosis_tag == KurtosisTag.Platykurtic
+
+
+def test_kurtosis_tag_mesokurtic():
+    # Bell-curve approximation (discrete triangular) → excess kurtosis in (-1, 3)
+    data = [1.0]*3 + [2.0]*7 + [3.0]*12 + [4.0]*16 + [5.0]*12 + [6.0]*7 + [7.0]*3
+    df = pl.DataFrame({"v": pl.Series(data, dtype=pl.Float64)})
+    stats = NumericProfiler().profile(df, ["v"]).columns["v"]
+    assert stats.kurtosis_tag == KurtosisTag.Mesokurtic
+
+
+# ---------------------------------------------------------------------------
+# Percentiles
+# ---------------------------------------------------------------------------
+
+
+def test_percentiles_type_and_all_fields_present(normal_mixed_df):
+    stats = NumericProfiler().profile(normal_mixed_df, ["score"]).columns["score"]
+    p = stats.percentiles
+    assert isinstance(p, PercentileSnapshot)
+    for val in (p.p1, p.p5, p.p25, p.p50, p.p75, p.p95, p.p99):
+        assert val is not None
+
+
+def test_percentiles_monotonically_non_decreasing(normal_mixed_df):
+    p = NumericProfiler().profile(normal_mixed_df, ["score"]).columns["score"].percentiles
+    vals = [p.p1, p.p5, p.p25, p.p50, p.p75, p.p95, p.p99]
+    assert vals == sorted(vals)
+
+
+# ---------------------------------------------------------------------------
+# Discrete vs continuous distribution representation
+# ---------------------------------------------------------------------------
+
+
+def test_integer_column_produces_top_values():
+    # Int64 dtype always triggers the discrete path
+    data = [i % 5 for i in range(60)]
+    df = pl.DataFrame({"v": pl.Series(data, dtype=pl.Int64)})
+    stats = NumericProfiler().profile(df, ["v"]).columns["v"]
+    assert len(stats.top_values) > 0
+    assert len(stats.histogram) == 0
+
+
+def test_continuous_float_produces_histogram():
+    # 60 distinct floats → n_unique > _DISCRETE_MAX_UNIQUE (20) → continuous path
+    data = [round(i * 0.37, 4) for i in range(60)]
+    df = pl.DataFrame({"v": pl.Series(data, dtype=pl.Float64)})
+    stats = NumericProfiler().profile(df, ["v"]).columns["v"]
+    assert len(stats.histogram) > 0
+    assert len(stats.top_values) == 0

tests/unit/profiling/test_target_profiler.py

@@ -0,0 +1,44 @@
+import polars as pl
+import pytest
+
+from ....profiling._target_profiler import TargetProfiler
+from ....profiling._target_config import TargetProblemType
+
+
+# ---------------------------------------------------------------------------
+# Regression: continuous float target
+# ---------------------------------------------------------------------------
+
+
+def test_regression_for_continuous_float_target():
+    df = pl.DataFrame(
+        {"price": pl.Series([float(i) * 1.5 for i in range(50)], dtype=pl.Float64)}
+    )
+    result = TargetProfiler(target_column="price").profile(df)
+    assert result.problem_type == TargetProblemType.Regression
+
+
+# ---------------------------------------------------------------------------
+# Classification: low-cardinality string target
+# ---------------------------------------------------------------------------
+
+
+def test_classification_for_low_cardinality_string_target():
+    vals = ["cat", "dog", "bird"] * 20
+    df = pl.DataFrame({"label": pl.Series(vals, dtype=pl.Utf8)})
+    result = TargetProfiler(target_column="label").profile(df)
+    assert result.problem_type in (
+        TargetProblemType.BinaryClassification,
+        TargetProblemType.MulticlassClassification,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Missing target column raises ValueError
+# ---------------------------------------------------------------------------
+
+
+def test_missing_target_column_raises_value_error():
+    df = pl.DataFrame({"feature": pl.Series([1, 2, 3], dtype=pl.Int64)})
+    with pytest.raises(ValueError):
+        TargetProfiler(target_column="nonexistent").profile(df)

tests/unit/profiling/test_text_profiler.py

@@ -0,0 +1,61 @@
+import polars as pl
+
+from ....profiling._text_profiler import TextProfiler
+
+
+# ---------------------------------------------------------------------------
+# vocabulary_size > 0 for a column with distinct tokens
+# ---------------------------------------------------------------------------
+
+
+def test_vocabulary_size_positive_for_distinct_tokens():
+    df = pl.DataFrame(
+        {"text": pl.Series(["apple banana", "cherry", "date elderberry"], dtype=pl.Utf8)}
+    )
+    stats = TextProfiler().profile(df, ["text"]).columns["text"]
+    assert stats.vocabulary_size > 0
+
+
+# ---------------------------------------------------------------------------
+# char_length_max >= char_length_min for any non-empty text column
+# ---------------------------------------------------------------------------
+
+
+def test_char_length_max_gte_min():
+    df = pl.DataFrame(
+        {"text": pl.Series(["hi", "hello world", "a"], dtype=pl.Utf8)}
+    )
+    stats = TextProfiler().profile(df, ["text"]).columns["text"]
+    assert stats.char_length_max >= stats.char_length_min
+
+
+# ---------------------------------------------------------------------------
+# empty_ratio == 0.0 for no empty strings; == 1.0 for all-empty column
+# ---------------------------------------------------------------------------
+
+
+def test_empty_ratio_absent_and_full():
+    df_no_empty = pl.DataFrame(
+        {"text": pl.Series(["hello", "world", "foo"], dtype=pl.Utf8)}
+    )
+    stats_no_empty = TextProfiler().profile(df_no_empty, ["text"]).columns["text"]
+    assert stats_no_empty.empty_ratio == 0.0
+
+    df_all_empty = pl.DataFrame(
+        {"text": pl.Series(["", "", ""], dtype=pl.Utf8)}
+    )
+    stats_all_empty = TextProfiler().profile(df_all_empty, ["text"]).columns["text"]
+    assert stats_all_empty.empty_ratio == 1.0
+
+
+# ---------------------------------------------------------------------------
+# avg_token_count > 0 for a column with multi-word entries
+# ---------------------------------------------------------------------------
+
+
+def test_avg_token_count_positive_for_multiword():
+    df = pl.DataFrame(
+        {"text": pl.Series(["the quick brown fox", "hello world", "one two three"], dtype=pl.Utf8)}
+    )
+    stats = TextProfiler().profile(df, ["text"]).columns["text"]
+    assert stats.avg_token_count > 0

tests/unit/profiling/test_type_detector.py

@@ -0,0 +1,32 @@
+import polars as pl
+
+from ....profiling._type_detector import TypeDetector
+from ....profiling.config import SemanticType
+
+
+# ---------------------------------------------------------------------------
+# Native pl.Boolean resolves to SemanticType.Boolean
+# ---------------------------------------------------------------------------
+
+
+def test_native_boolean_column_resolves_to_boolean():
+    df = pl.DataFrame(
+        {"flag": pl.Series([True, False, True, True, False], dtype=pl.Boolean)}
+    )
+    info = TypeDetector(columns=["flag"]).detect(df)["flag"]
+    assert info.semantic_type == SemanticType.Boolean
+
+
+# ---------------------------------------------------------------------------
+# High-cardinality string column resolves to Categorical or Text (not Numeric)
+# ---------------------------------------------------------------------------
+
+
+def test_high_cardinality_string_not_numeric():
+    # 80 rows, 40 distinct short strings — high cardinality but below
+    # the 99% identifier threshold so it stays Categorical/Text.
+    vals = ["item_" + str(i % 40) for i in range(80)]
+    df = pl.DataFrame({"name": pl.Series(vals, dtype=pl.Utf8)})
+    info = TypeDetector(columns=["name"]).detect(df)["name"]
+    assert info.semantic_type in (SemanticType.Categorical, SemanticType.Text)
+    assert info.semantic_type != SemanticType.Numeric