dataforge-ml 0.1.0__py3-none-any.whl

profiling/structural.py

@@ -0,0 +1,280 @@
+"""
+StructuralProfiler  –  unified Phase 1 entry point.
+
+Execution order inside profile(df):
+  1. ModalityProfiler      → result.dataset (DatasetStats)
+  2. MissingnessProfiler   → ColumnProfile.missingness + dataset.missingness_matrix
+  3. Row-missingness dist  → dataset.row_distribution
+  4. TypeDetector          → ColumnProfile.semantic_type / type_flags / dtypes
+  5. column_overrides      → replace SemanticType on existing ColumnProfiles
+  6. ColumnTypeProfiler    → route each column to its profiler by SemanticType;
+                             Identifier columns: skip, stats stays None
+  7. target_columns        → TargetProfiler; mark ColumnProfile.is_target=True
+  8. Correlation           → if compute_correlation=True:
+       a. profile_features()  → dataset.feature_correlation  (computed once)
+       b. profile_target()    → dataset.target_correlations[target]
+                                (once per declared target column)
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+import polars as pl
+
+from ._base import ModalityProfiler, ColumnBatchProfiler
+from ._tabular import TabularProfiler
+from ._categorical import CategoricalProfiler
+from ._datetime_profiler import DatetimeProfiler
+from ._numeric_profiler import NumericProfiler
+from ._boolean_profiler import BooleanProfiler
+from ._text_profiler import TextProfiler
+from ._missingness_profiler import MissingnessProfiler
+from ._target_profiler import TargetProfiler
+from ._correlation_profiler import CorrelationProfiler
+from ._type_detector import TypeDetector
+from .config import (
+    ProfileConfig,
+    ColumnProfile,
+    StructuralProfileResult,
+    RowMissingnessDistribution,
+    SemanticType,
+    Modality,
+)
+
+_ROW_DROP_THRESHOLD = 0.50
+
+# ---------------------------------------------------------------------------
+# Registry: SemanticType → ColumnTypeProfiler class
+#
+# Stateless between profile(series, df) calls, so one instance per
+# SemanticType safely handles all columns of that type in one run.
+# Add Boolean / Text profilers here when implemented.
+# ---------------------------------------------------------------------------
+_COLUMN_PROFILER_REGISTRY: dict[SemanticType, type[ColumnBatchProfiler]] = {  # type: ignore[type-arg]
+    SemanticType.Numeric: NumericProfiler,
+    SemanticType.Categorical: CategoricalProfiler,
+    SemanticType.Datetime: DatetimeProfiler,
+    SemanticType.Boolean: BooleanProfiler,
+    SemanticType.Text: TextProfiler,
+}
+
+
+class StructuralProfiler:
+
+    def __init__(self, config: ProfileConfig | None = None) -> None:
+        self.config = config or ProfileConfig()
+
+        if self.config.modality == Modality.Tabular:
+            self.modality_profiler: ModalityProfiler = TabularProfiler(self.config)
+        else:
+            raise NotImplementedError(
+                f"modality {self.config.modality} not supported yet"
+            )
+
+    # ------------------------------------------------------------------
+    # Public entry point
+    # ------------------------------------------------------------------
+
+    def profile(self, data: Any) -> StructuralProfileResult:
+        if not isinstance(data, pl.DataFrame):
+            raise TypeError(
+                f"StructuralProfiler expects a Polars DataFrame, "
+                f"got {type(data).__name__}."
+            )
+
+        result = StructuralProfileResult()
+
+        active_cols = [c for c in data.columns if c not in self.config.exclude_columns]
+
+        # ── 1. Modality profiler ─────────────────────────────────────────
+        # Replaces default DatasetStats with the real one (row_count, memory,
+        # duplicates, etc.).  Must run before anything writes to result.dataset.
+        result.dataset = self.modality_profiler.profile(data)
+
+        # ── 2. Missingness pre-pass ──────────────────────────────────────
+        # setdefault creates ColumnProfile entries; subsequent steps mutate
+        # the same objects via the same setdefault pattern.
+        missingness_result = MissingnessProfiler(config=self.config).profile(
+            data, columns=active_cols
+        )
+        for col_name in missingness_result.analysed_columns:
+            cp = result.columns.setdefault(col_name, ColumnProfile(name=col_name))
+            cp.missingness = missingness_result.columns.get(col_name)
+
+        if missingness_result.correlation_matrix:
+            result.dataset.missingness_matrix = missingness_result.correlation_matrix
+
+        # ── 3. Row-missingness distribution ─────────────────────────────
+        result.dataset.row_distribution = self._compute_row_distribution(
+            df=data,
+            cols=active_cols,
+            n_rows=data.height,
+            overrides=self.config.column_overrides,
+        )
+
+        # ── 4. Type detection ────────────────────────────────────────────
+        # setdefault returns the existing ColumnProfile from step 2, so
+        # missingness and type info land on the same object.
+        type_info = TypeDetector(columns=active_cols).detect(data)
+        for col_name, info in type_info.items():
+            cp = result.columns.setdefault(col_name, ColumnProfile(name=col_name))
+            cp.semantic_type = info.semantic_type
+            cp.type_flags = list(info.flags)
+            cp.original_dtype = info.original_dtype
+            cp.inferred_dtype = info.inferred_dtype
+
+        # ── 5. Apply column_overrides ────────────────────────────────────
+        # All active columns are in result.columns by now (steps 2 + 4).
+        # Overrides for excluded / non-existent columns are silently ignored.
+        for col_name, override_type in self.config.column_overrides.items():
+            if col_name in result.columns:
+                result.columns[col_name].semantic_type = override_type
+
+        # ── 6. Per-column profiling routed by SemanticType ───────────────
+        # Batch all columns of the same SemanticType together and call each
+        # profiler once with (df, column_list) — matching the profiler API.
+        type_to_cols: dict[SemanticType, list[str]] = {}
+        for col_name in active_cols:
+            cp = result.columns.get(col_name)
+            if cp is None or cp.semantic_type is None:
+                continue
+            if cp.semantic_type == SemanticType.Identifier:
+                continue
+            sem_type = cp.semantic_type
+            type_to_cols.setdefault(sem_type, []).append(col_name)
+
+        for sem_type, cols in type_to_cols.items():
+            profiler_cls = _COLUMN_PROFILER_REGISTRY.get(sem_type)  # type: ignore[arg-type]
+            if profiler_cls is None:
+                continue
+            profiler = profiler_cls(config=self.config)
+            try:
+                batch = profiler.profile(data, columns=cols)
+                for col_name in batch.analysed_columns:
+                    if col_name in result.columns:
+                        result.columns[col_name].stats = batch.columns.get(col_name)
+            except Exception:
+                pass
+
+        # ── 7. Target columns ────────────────────────────────────────────
+        # TargetProfiler produces target-specific analysis stored in
+        # result.targets.  cp.stats is NOT overwritten — step 6 already set it.
+        if self.config.target_columns:
+            for target in self.config.target_columns:
+                if target not in data.columns:
+                    continue
+                target_result = TargetProfiler(
+                    target_column=target,
+                    config=self.config,
+                ).profile(data)
+                result.targets[target] = target_result
+
+                # setdefault returns the existing ColumnProfile.
+                cp = result.columns.setdefault(target, ColumnProfile(name=target))
+                cp.is_target = True
+
+        # ── 8. Correlation ───────────────────────────────────────────────
+        if self.config.compute_correlation:
+            # Resolve column lists by detected SemanticType (post-override).
+            numeric_cols = [
+                c
+                for c in active_cols
+                if result.columns.get(c)
+                and result.columns[c].semantic_type == SemanticType.Numeric
+            ]
+            categorical_cols = [
+                c
+                for c in active_cols
+                if result.columns.get(c)
+                and result.columns[c].semantic_type == SemanticType.Categorical
+            ]
+
+            corr_profiler = CorrelationProfiler(
+                numeric_columns=numeric_cols,
+                categorical_columns=categorical_cols,
+                config=self.config,
+            )
+
+            # 8a. Feature-feature matrices — computed ONCE, target-independent.
+            feature_corr = corr_profiler.profile_features(
+                data, numeric_cols
+            )
+            result.dataset.feature_correlation = feature_corr
+
+            # 8b. Per-target analysis — matrices are NOT recomputed; each call
+            #     shallow-copies feature_corr and appends target-specific fields.
+            for target in self.config.target_columns:
+                if target not in data.columns:
+                    continue
+                result.dataset.target_correlations[target] = (
+                    corr_profiler.profile_target(
+                        data, feature_corr, numeric_cols, categorical_cols, target
+                    )
+                )
+
+        return result
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _compute_row_distribution(
+        df: pl.DataFrame,
+        cols: list[str],
+        n_rows: int,
+        overrides: dict[str, SemanticType],
+    ) -> RowMissingnessDistribution:
+        from ._missingness_profiler import (
+            _sentinel_eligible,
+            _inf_eligible,
+            _SENTINEL_STRINGS,
+        )
+
+        dist = RowMissingnessDistribution()
+        if n_rows == 0 or not cols:
+            return dist
+
+        n_cols = len(cols)
+        per_col_exprs = []
+
+        for col_name in cols:
+            dtype = df[col_name].dtype
+            override = overrides.get(col_name)
+            null_e = pl.col(col_name).is_null()
+
+            if _sentinel_eligible(dtype, override):
+                eff = (
+                    null_e
+                    | (pl.col(col_name).str.strip_chars() == "")
+                    | pl.col(col_name).str.to_uppercase().is_in(list(_SENTINEL_STRINGS))
+                )
+            elif _inf_eligible(dtype):
+                eff = (
+                    null_e | pl.col(col_name).is_nan() | pl.col(col_name).is_infinite()
+                )
+            else:
+                eff = null_e
+
+            per_col_exprs.append(eff.cast(pl.Int8).alias(col_name))
+
+        row_missing: pl.Series = df.select(per_col_exprs).select(
+            pl.sum_horizontal(pl.all()).alias("row_missing")
+        )["row_missing"]
+
+        half_threshold = math.ceil(n_cols * _ROW_DROP_THRESHOLD)
+
+        dist.pct_zero_missing = float((row_missing == 0).sum()) / n_rows
+        dist.pct_one_to_two = (
+            float(((row_missing >= 1) & (row_missing <= 2)).sum()) / n_rows
+        )
+        dist.pct_three_to_five = (
+            float(((row_missing >= 3) & (row_missing <= 5)).sum()) / n_rows
+        )
+        dist.pct_over_five = float((row_missing > 5).sum()) / n_rows
+        dist.drop_candidate_row_count = int((row_missing >= half_threshold).sum())
+        dist.pct_over_half_missing = dist.drop_candidate_row_count / n_rows
+
+        return dist

splitting/__init__.py

@@ -0,0 +1,4 @@
+from ._config import FoldResult, SplitResult
+from ._splitter import DataSplitter
+
+__all__ = ["DataSplitter", "SplitResult", "FoldResult"]

splitting/_config.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import polars as pl
+
+
+@dataclass
+class SplitResult:
+    """
+    Attributes
+    ----------
+    train : pl.DataFrame
+        Training partition.
+    test : pl.DataFrame
+        Test/hold-out partition.
+    train_size : int
+        Number of rows in the training partition.
+    test_size : int
+        Number of rows in the test partition.
+    train_ratio : float
+        Fraction of total rows assigned to training (0.0–1.0).
+    test_ratio : float
+        Fraction of total rows assigned to testing (0.0–1.0).
+    """
+
+    train: pl.DataFrame
+    test: pl.DataFrame
+    train_size: int
+    test_size: int
+    train_ratio: float
+    test_ratio: float
+
+
+@dataclass
+class FoldResult:
+    """
+    Attributes
+    ----------
+    train : pl.DataFrame
+        Training partition for this fold.
+    val : pl.DataFrame
+        Validation partition for this fold.
+    fold_index : int
+        Zero-based index of this fold within the CV run.
+    train_size : int
+        Number of rows in the training partition.
+    val_size : int
+        Number of rows in the validation partition.
+    """
+
+    train: pl.DataFrame
+    val: pl.DataFrame
+    fold_index: int
+    train_size: int
+    val_size: int

splitting/_splitter.py

@@ -0,0 +1,202 @@
+"""
+DataSplitter: constructor, random_split, time_split, and kfold implementation.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any, List, Optional
+
+import polars as pl
+from sklearn.model_selection import KFold, ShuffleSplit, StratifiedKFold, StratifiedShuffleSplit
+
+from ._config import FoldResult, SplitResult
+
+_UNSET = object()
+
+
+class DataSplitter:
+    """
+    Splits a Polars DataFrame into train/test or cross-validation folds.
+
+    Parameters
+    ----------
+    df : pl.DataFrame
+        Source data. Must be non-empty.
+    target : str, optional
+        Name of the target column. Required for stratified splits.
+    random_seed : int, optional
+        Seed forwarded to sklearn splitters for reproducibility.
+    """
+
+    def __init__(
+        self,
+        df: pl.DataFrame,
+        target: Optional[str] = None,
+        random_seed: Optional[int] = None,
+    ) -> None:
+        if not isinstance(df, pl.DataFrame):
+            raise TypeError(f"df must be a polars DataFrame, got {type(df).__name__}")
+        if df.is_empty():
+            raise ValueError("df must not be empty")
+        if target is not None and target not in df.columns:
+            raise ValueError(f"target column '{target}' not found in df")
+
+        self._df = df
+        self._target = target
+        self._random_seed = random_seed
+
+    def random_split(self, test_size: float, stratify=_UNSET) -> SplitResult:
+        """
+        Return a single randomised train/test split.
+
+        Parameters
+        ----------
+        test_size : float
+            Fraction of rows to reserve for the test set (0 < test_size < 1).
+        stratify : bool, optional
+            Whether to stratify on the target column.
+            Defaults to True when a target was provided, False otherwise.
+
+        Returns
+        -------
+        SplitResult
+        """
+        if stratify is _UNSET:
+            stratify = self._target is not None
+        if stratify and self._target is None:
+            raise ValueError(
+                "stratify=True requires a target column; "
+                "pass target= when constructing DataSplitter"
+            )
+
+        if stratify:
+            splitter = StratifiedShuffleSplit(
+                n_splits=1, test_size=test_size, random_state=self._random_seed
+            )
+            y = self._df[self._target].to_numpy()
+            train_idx, test_idx = next(splitter.split(self._df, y))
+        else:
+            splitter = ShuffleSplit(
+                n_splits=1, test_size=test_size, random_state=self._random_seed
+            )
+            train_idx, test_idx = next(splitter.split(self._df))
+
+        train_df = self._df[train_idx]
+        test_df = self._df[test_idx]
+        total = len(self._df)
+
+        return SplitResult(
+            train=train_df,
+            test=test_df,
+            train_size=len(train_df),
+            test_size=len(test_df),
+            train_ratio=len(train_df) / total,
+            test_ratio=len(test_df) / total,
+        )
+
+    def time_split(
+        self,
+        time_column: str,
+        test_size: Optional[float] = None,
+        cutoff: Optional[Any] = None,
+    ) -> SplitResult:
+        """
+        Return a chronological train/test split with no temporal leakage.
+
+        The DataFrame is sorted ascending by ``time_column`` before splitting.
+        ``cutoff`` takes priority over ``test_size`` when both are supplied.
+
+        Parameters
+        ----------
+        time_column : str
+            Column to sort by. Must exist in the DataFrame.
+        test_size : float, optional
+            Fraction of rows (from the end of the sorted series) to use as
+            the test set.  ``floor(len(df) * test_size)`` rows are taken.
+        cutoff : scalar, optional
+            Threshold value.  Rows where ``time_column >= cutoff`` go to
+            test; all earlier rows go to train.
+
+        Returns
+        -------
+        SplitResult
+        """
+        if time_column not in self._df.columns:
+            raise ValueError(f"time_column '{time_column}' not found in df")
+        if test_size is None and cutoff is None:
+            raise ValueError("Either test_size or cutoff must be provided")
+
+        sorted_df = self._df.sort(time_column)
+        total = len(sorted_df)
+
+        if cutoff is not None:
+            train_df = sorted_df.filter(pl.col(time_column) < cutoff)
+            test_df = sorted_df.filter(pl.col(time_column) >= cutoff)
+        else:
+            n_test = math.floor(total * test_size)
+            n_train = total - n_test
+            train_df = sorted_df[:n_train]
+            test_df = sorted_df[n_train:]
+
+        return SplitResult(
+            train=train_df,
+            test=test_df,
+            train_size=len(train_df),
+            test_size=len(test_df),
+            train_ratio=len(train_df) / total,
+            test_ratio=len(test_df) / total,
+        )
+
+    def kfold(self, k: int, stratify=_UNSET) -> List[FoldResult]:
+        """
+        Return a list of ``k`` cross-validation folds.
+
+        Parameters
+        ----------
+        k : int
+            Number of folds.
+        stratify : bool, optional
+            Whether to stratify on the target column.
+            Defaults to True when a target was provided, False otherwise.
+
+        Returns
+        -------
+        list[FoldResult]
+            Exactly ``k`` folds with zero-based ``fold_index``.
+        """
+        if stratify is _UNSET:
+            stratify = self._target is not None
+        if stratify and self._target is None:
+            raise ValueError(
+                "stratify=True requires a target column; "
+                "pass target= when constructing DataSplitter"
+            )
+
+        if stratify:
+            folder = StratifiedKFold(
+                n_splits=k, shuffle=True, random_state=self._random_seed
+            )
+            y = self._df[self._target].to_numpy()
+            splits = folder.split(self._df, y)
+        else:
+            folder = KFold(
+                n_splits=k, shuffle=True, random_state=self._random_seed
+            )
+            splits = folder.split(self._df)
+
+        folds: List[FoldResult] = []
+        for fold_index, (train_idx, val_idx) in enumerate(splits):
+            train_df = self._df[train_idx]
+            val_df = self._df[val_idx]
+            folds.append(
+                FoldResult(
+                    train=train_df,
+                    val=val_df,
+                    fold_index=fold_index,
+                    train_size=len(train_df),
+                    val_size=len(val_df),
+                )
+            )
+
+        return folds

tests/conftest.py

@@ -0,0 +1,7 @@
+import numpy as np
+import pytest
+
+
+@pytest.fixture(scope="session")
+def rng():
+    return np.random.default_rng(42)

tests/integration/conftest.py

@@ -0,0 +1,82 @@
+
+import polars as pl
+import pytest
+
+
+@pytest.fixture(scope="session")
+def override_df():
+    n = 60
+    return pl.DataFrame(
+        {
+            "score": pl.Series([float(i) for i in range(n)], dtype=pl.Float64),
+            "category": pl.Series(["A", "B", "C"] * (n // 3), dtype=pl.Utf8),
+        }
+    )
+
+
+@pytest.fixture(scope="session")
+def target_df(rng):
+    n = 100
+    features = rng.normal(0, 1, size=n).tolist()
+    labels = ["pos", "neg"] * (n // 2)
+    return pl.DataFrame(
+        {
+            "feature": pl.Series(features, dtype=pl.Float64),
+            "label": pl.Series(labels, dtype=pl.Utf8),
+        }
+    )
+
+
+@pytest.fixture(scope="session")
+def empty_df():
+    return pl.DataFrame(
+        {
+            "x": pl.Series([], dtype=pl.Float64),
+            "y": pl.Series([], dtype=pl.Utf8),
+        }
+    )
+
+
+@pytest.fixture(scope="session")
+def text_df():
+    n = 200
+    topics = ["science", "art", "history", "technology", "nature", "music"]
+    texts = [
+        f"A detailed description covering the topic of {topics[i % len(topics)]} "
+        f"with multiple words that comfortably exceed the free-text threshold in row {i}"
+        for i in range(n)
+    ]
+    return pl.DataFrame({"review": pl.Series(texts, dtype=pl.Utf8)})
+
+
+@pytest.fixture(scope="session")
+def mixed_df(rng):
+    n = 300
+
+    age = rng.integers(18, 75, size=n)
+    income = age * 1200 + rng.normal(0, 5000, size=n)
+
+    salary = rng.normal(50_000, 15_000, size=n).tolist()
+    null_mask = rng.random(n) < 0.10
+    salary = [None if null_mask[i] else salary[i] for i in range(n)]
+
+    country_choices = ["US", "UK", "CA", "AU", "DE"]
+    country = [country_choices[i % len(country_choices)] for i in range(n)]
+
+    names = [f"person_{i}" for i in range(n)]
+
+    is_active = [bool(v) for v in rng.integers(0, 2, size=n)]
+
+    from datetime import date, timedelta
+    base = date(2020, 1, 1)
+    joined = [base + timedelta(days=int(d)) for d in rng.integers(0, 1460, size=n)]
+
+    return pl.DataFrame({
+        "age": pl.Series(age.tolist(), dtype=pl.Int64),
+        "income": pl.Series(income.tolist(), dtype=pl.Float64),
+        "salary": pl.Series(salary, dtype=pl.Float64),
+        "country": pl.Series(country, dtype=pl.Utf8),
+        "name": pl.Series(names, dtype=pl.Utf8),
+        "is_active": pl.Series(is_active, dtype=pl.Boolean),
+        "joined": pl.Series(joined, dtype=pl.Date),
+    })