dataforge-ml 0.1.0__py3-none-any.whl

profiling/_target_config.py

@@ -0,0 +1,74 @@
+"""
+Configuration and result dataclasses for Target Variable profiling.
+
+Determines the nature of the predictive task (Regression vs Classification)
+and flags critical issues like missing labels or severe imbalances.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Optional
+
+from ._categorical_config import CategoricalColumnProfile
+from ._numeric_config import ColumnNumericProfile
+
+class TargetProblemType(StrEnum):
+    Regression = "regression"
+    BinaryClassification = "binary_classification"
+    MulticlassClassification = "multiclass_classification"
+    Unknown = "unknown"
+
+class TargetFlag(StrEnum):
+    ContainsMissing = "contains_missing"      # Target has >0 missing values; must drop or reframe
+    HighImbalance = "high_imbalance"          # Class ratio > 5 (requires handling in Phase 5)
+    SevereImbalance = "severe_imbalance"      # Class ratio > 20 (accuracy metric is meaningless)
+    HighlySkewed = "highly_skewed"            # Numeric target is severely skewed (consider log transform)
+    IsIdentifier = "is_identifier"            # Target looks like an ID column (useless for modeling)
+
+@dataclass
+class TargetProfileResult:
+    """
+    Profile specific to the designated target variable.
+    """
+    column: str
+    problem_type: TargetProblemType
+
+    # Missingness (Critical for targets)
+    missing_count: int = 0
+    missing_ratio: float = 0.0
+
+    # Underlying profile data depending on the problem type
+    numeric_profile: Optional[ColumnNumericProfile] = None
+    categorical_profile: Optional[CategoricalColumnProfile] = None
+
+    flags: list[TargetFlag] = field(default_factory=list)
+
+    def has_flag(self, flag: TargetFlag) -> bool:
+        return flag in self.flags
+
+    def __str__(self) -> str:
+        lines = [
+            "=== Target Variable Profile ===",
+            f"  Column        : {self.column}",
+            f"  Problem Type  : {self.problem_type}",
+            f"  Missingness   : {self.missing_count:,} rows ({self.missing_ratio:.2%})",
+        ]
+        
+        if self.has_flag(TargetFlag.ContainsMissing):
+            lines.append("    [!] WARNING: Target contains missing values. Imputation is not recommended.")
+
+        if self.categorical_profile and self.problem_type in (TargetProblemType.BinaryClassification, TargetProblemType.MulticlassClassification):
+            im = self.categorical_profile.imbalance
+            lines.append(f"  Classes       : {self.categorical_profile.cardinality:,}")
+            lines.append(f"  Class Ratio   : {im.class_ratio:.2f}")
+            lines.append(f"  Gini Impurity : {im.gini_impurity:.4f}")
+            
+        if self.numeric_profile and self.problem_type == TargetProblemType.Regression:
+            lines.append(f"  Mean / Median : {self.numeric_profile.mean:.4f} / {self.numeric_profile.median:.4f}")
+            lines.append(f"  Skewness      : {self.numeric_profile.skewness:.4f} [{self.numeric_profile.skew_severity}]")
+
+        if self.flags:
+            lines.append(f"  Flags         : {', '.join(self.flags)}")
+            
+        return "\n".join(lines)

profiling/_target_profiler.py

@@ -0,0 +1,156 @@
+"""
+TargetProfiler  –  Phase 1 extension: Target Variable Profiling.
+
+Performs robust dtype detection to determine the problem framework
+(Regression vs Classification) and assesses critical target health metrics:
+  1. Target Missingness (Any missingness flags the dataset for row-dropping)
+  2. Class Imbalance (For Classification tasks)
+  3. Skewness / Normalcy (For Regression tasks)
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import polars as pl
+
+from ._base import DatasetLevelProfiler
+from .config import ProfileConfig
+from ._target_config import (
+    TargetFlag,
+    TargetProblemType,
+    TargetProfileResult,
+)
+
+# Reuse your internal profilers to prevent duplication
+from ._type_detector import TypeDetector, TypeFlag, NumericKind
+from ._missingness_profiler import MissingnessProfiler
+from ._categorical import CategoricalProfiler
+from ._numeric_profiler import NumericProfiler
+from ._numeric_config import SkewSeverity
+
+
+class TargetProfiler(DatasetLevelProfiler[TargetProfileResult]):
+    """
+    Analyzes the target variable to set up downstream ML behavior.
+    """
+
+    def __init__(self, target_column: str, config: ProfileConfig | None = None) -> None:
+        super().__init__(config)
+        self.target_column = target_column
+
+    def profile(self, data: pl.DataFrame, **kwargs) -> TargetProfileResult:
+        if self.target_column not in data.columns:
+            raise ValueError(
+                f"Target column '{self.target_column}' not found in the DataFrame."
+            )
+
+        return self._run(data)
+
+    def _run(self, df: pl.DataFrame) -> TargetProfileResult:
+        series = df[self.target_column]
+        n_rows = df.height
+
+        # 1. Type Detection -> Problem Type Mapping
+        detector = TypeDetector(columns=[self.target_column])
+        type_info = detector.detect(df)[self.target_column]
+        problem_type = self._determine_problem_type(series, type_info, n_rows)
+
+        result = TargetProfileResult(
+            column=self.target_column, problem_type=problem_type
+        )
+
+        if type_info.has_flag(TypeFlag.IdentifierColumn):
+            result.flags.append(TargetFlag.IsIdentifier)
+
+        # 2. Target Missingness Check
+        # We reuse MissingnessProfiler's static method to get standard + effective nulls
+        col_miss_profile, _ = MissingnessProfiler._profile_column(
+            series, self.target_column, n_rows
+        )
+        result.missing_count = col_miss_profile.effective_null_count
+        result.missing_ratio = col_miss_profile.effective_null_ratio
+
+        if result.missing_count > 0:
+            result.flags.append(TargetFlag.ContainsMissing)
+
+        # 3. Problem-Specific Profiling
+        if problem_type in (
+            TargetProblemType.BinaryClassification,
+            TargetProblemType.MulticlassClassification,
+        ):
+            self._profile_classification(series, n_rows, result)
+        elif problem_type == TargetProblemType.Regression:
+            self._profile_regression(series, n_rows, result)
+
+        return result
+
+    def _determine_problem_type(
+        self, series: pl.Series, type_info: Any, n_rows: int
+    ) -> TargetProblemType:
+        """Map TypeDetector results to an ML Problem Type with cardinality safety."""
+
+        # 1. Reject Identifiers completely
+        if type_info.has_flag(TypeFlag.IdentifierColumn):
+            return TargetProblemType.Unknown
+
+        # 2. Obvious Booleans -> Binary Classification
+        if type_info.has_flag(TypeFlag.BooleanCandidate):
+            return TargetProblemType.BinaryClassification
+
+        # 3. Categorical (Strings OR Integers acting as categories)
+        is_string = series.dtype in (pl.Utf8, pl.String)
+        is_encoded_int = type_info.has_flag(TypeFlag.EncodedCategory)
+
+        if is_string or is_encoded_int:
+            n_unique = series.drop_nulls().n_unique()
+
+            # SAFEGUARD: If a string has too many unique values, it's not a classification target.
+            # E.g., free text, high-cardinality IDs, or raw JSON strings.
+            # Threshold: > 100 classes is usually beyond standard ML classification scope.
+            if n_unique > 100 or (n_unique / max(n_rows, 1) > 0.05 and n_rows > 1000):
+                return TargetProblemType.Unknown
+
+            if n_unique == 2:
+                return TargetProblemType.BinaryClassification
+            elif n_unique > 2:
+                return TargetProblemType.MulticlassClassification
+
+        # 4. Confirmed Numerics -> Regression
+        # Note: TypeDetector strips the 'NumericKind' if it was flagged as an EncodedCategory.
+        # So we won't accidentally treat [0, 1, 2] classes as a regression target here.
+        if type_info.numeric_kind in (NumericKind.Continuous, NumericKind.Discrete):
+            return TargetProblemType.Regression
+
+        # If it's a string with too many unique values, or an unparsed datetime, etc.
+        return TargetProblemType.Unknown
+
+    def _profile_classification(
+        self, series: pl.Series, n_rows: int, result: TargetProfileResult
+    ) -> None:
+        """Generates categorical metrics and checks for class imbalance."""
+        cat_profiler = CategoricalProfiler(config=self.config)
+
+        # Internally compute cardinality, top values, and imbalance metrics
+        cat_profile = cat_profiler._profile_column(series, self.target_column, n_rows)
+        result.categorical_profile = cat_profile
+
+        # Flag Imbalances
+        ratio = cat_profile.imbalance.class_ratio
+        if ratio > 20.0:
+            result.flags.append(TargetFlag.SevereImbalance)
+        elif ratio > 5.0:
+            result.flags.append(TargetFlag.HighImbalance)
+
+    def _profile_regression(
+        self, series: pl.Series, n_rows: int, result: TargetProfileResult
+    ) -> None:
+        """Generates numeric metrics and checks for target skewness."""
+        num_profiler = NumericProfiler(config=self.config)
+
+        num_profile = num_profiler._profile_column(series, n_rows)
+        result.numeric_profile = num_profile
+
+        # Flag Skewness (Highly skewed targets often require Log/Yeo-Johnson transforms)
+        if num_profile.skewness_severity in (SkewSeverity.High, SkewSeverity.Severe):
+            result.flags.append(TargetFlag.HighlySkewed)

profiling/_text_config.py

@@ -0,0 +1,40 @@
+"""
+Result dataclass for free-text column profiling.
+
+Populated by TextProfiler.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class TextStats:
+    avg_token_count: float = 0.0
+    median_token_count: float = 0.0
+    vocabulary_size: int = 0
+    char_length_min: int = 0
+    char_length_max: int = 0
+    char_length_mean: float = 0.0
+    char_length_median: float = 0.0
+    empty_ratio: float = 0.0
+    whitespace_ratio: float = 0.0
+
+
+@dataclass
+class TextProfileResult:
+    """
+    Text profile for all eligible columns.
+
+    Attributes
+    ----------
+    columns : dict[str, TextStats]
+        Per-column text profiles, keyed by column name.
+    analysed_columns : list[str]
+        Columns that were actually profiled (after schema intersection
+        and eligibility check).
+    """
+
+    columns: dict[str, TextStats] = field(default_factory=dict)
+    analysed_columns: list[str] = field(default_factory=list)

profiling/_text_profiler.py

@@ -0,0 +1,194 @@
+"""
+TextProfiler  –  Phase 1 extension: Free-Text Column Profiling.
+
+Handles columns classified as SemanticType.Text (free-text string columns).
+All computation is Polars-native — no external NLP libraries, no language
+detection.
+
+Per-column metrics
+------------------
+1.  avg_token_count    – mean whitespace-split token count across non-null rows
+2.  median_token_count – median whitespace-split token count across non-null rows
+3.  vocabulary_size    – count of distinct tokens across all non-null values
+4.  char_length_min    – shortest non-null string (characters)
+5.  char_length_max    – longest non-null string (characters)
+6.  char_length_mean   – mean character length across non-null strings
+7.  char_length_median – median character length across non-null strings
+8.  empty_ratio        – fraction of total rows that are empty strings ("")
+9.  whitespace_ratio   – fraction of total rows that are whitespace-only
+                         (includes empty strings, since strip → "")
+
+Definitions
+-----------
+- "token"         : any run of non-whitespace characters produced by
+                    str.split_whitespace() semantics, i.e.
+                    ``pl.col(c).str.split(" ")`` with empty-string elements
+                    filtered out.  We use Polars ``str.count_matches`` on
+                    ``r"\\S+"`` which counts exactly these tokens in a single
+                    vectorised pass.
+- "empty string"  : len == 0 after no stripping.
+- "whitespace-only": len == 0 after str.strip_chars().
+- Null values are excluded from all per-row metrics and from ratio
+  denominators **except** empty_ratio / whitespace_ratio, which are
+  computed over total row count (nulls contribute 0, not counted as empty).
+
+Eligibility
+-----------
+A column is eligible when:
+  - It has a SemanticType.Text override in ProfileConfig.column_overrides, OR
+  - Its Polars dtype is pl.Utf8 (alias pl.String) and no other override is set.
+
+Integration
+-----------
+Drop ``TextProfiler`` into the profiler loop in ``structural.py`` alongside
+``NumericProfiler``, ``CategoricalProfiler``, ``DatetimeProfiler``, and
+``BooleanProfiler``::
+
+    sub_result = TextProfiler(config=self.config).profile(data, columns=active_cols)
+    for col_name, col_stats in sub_result.columns.items():
+        result.columns.setdefault(col_name, ColumnProfile(name=col_name)).stats = col_stats
+"""
+
+from __future__ import annotations
+
+import polars as pl
+
+from ._base import ColumnBatchProfiler
+from .config import (
+    ProfileConfig,
+    TextStats,
+    SemanticType,
+)
+from ._text_config import TextProfileResult
+
+# Regex that counts non-whitespace token runs — used with str.count_matches.
+_TOKEN_PATTERN: str = r"\S+"
+
+
+class TextProfiler(ColumnBatchProfiler[TextProfileResult]):
+    """
+    Free-text column profiler for Polars DataFrames.
+
+    A column is eligible when:
+      - It has a ``SemanticType.Text`` override in
+        ``ProfileConfig.column_overrides``, OR
+      - Its Polars dtype is ``pl.Utf8`` / ``pl.String`` and no override is set.
+
+    Non-eligible columns are silently skipped.
+
+    Parameters
+    ----------
+    config : ProfileConfig | None
+        Shared profiling configuration.
+    """
+
+    def __init__(self, config: ProfileConfig | None = None) -> None:
+        super().__init__(config)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def profile(
+        self,
+        data: pl.DataFrame,
+        columns: list[str],
+    ) -> TextProfileResult:
+        return self._run(data, columns)
+
+    # ------------------------------------------------------------------
+    # Eligibility
+    # ------------------------------------------------------------------
+
+    def _eligible(self, series: pl.Series) -> bool:
+        override = self.config.column_overrides.get(series.name)
+
+        if override == SemanticType.Text:
+            return True
+
+        # Any other explicit override takes precedence
+        if override is not None:
+            return False
+
+        # Native string dtype (pl.Utf8 is the canonical name; pl.String is
+        # an alias in newer Polars — check both for cross-version safety)
+        return series.dtype in (pl.Utf8, pl.String)
+
+    # ------------------------------------------------------------------
+    # Orchestration
+    # ------------------------------------------------------------------
+
+    def _run(
+        self,
+        df: pl.DataFrame,
+        columns: list[str],
+    ) -> TextProfileResult:
+        result = TextProfileResult()
+
+        available = [
+            c
+            for c in self._resolve_columns(df.columns, columns)
+            if self._eligible(df[c])
+        ]
+        result.analysed_columns = available
+
+        for col_name in available:
+            result.columns[col_name] = self._profile_column(
+                df[col_name], df.height
+            )
+
+        return result
+
+    # ------------------------------------------------------------------
+    # Per-column driver
+    # ------------------------------------------------------------------
+
+    def _profile_column(
+        self,
+        series: pl.Series,
+        n_rows: int,
+    ) -> TextStats:
+        profile = TextStats()
+
+        if n_rows == 0:
+            return profile
+
+        # ── 1. Empty / whitespace ratios (computed over ALL rows, nulls → 0) ──
+        # null rows do not count as empty or whitespace-only.
+        non_null_mask = series.is_not_null()
+        empty_mask = non_null_mask & (series.str.len_chars() == 0)
+        stripped = series.str.strip_chars()
+        whitespace_mask = non_null_mask & (stripped.str.len_chars() == 0)
+
+        profile.empty_ratio = float(empty_mask.sum()) / n_rows
+        profile.whitespace_ratio = float(whitespace_mask.sum()) / n_rows
+
+        # ── 2. Work on non-null values only from here on ─────────────────────
+        non_null = series.drop_nulls()
+        n_non_null = non_null.len()
+
+        if n_non_null == 0:
+            return profile
+
+        # ── 3. Token counts (whitespace-split, Polars regex count) ────────────
+        # str.count_matches counts non-overlapping matches of r"\S+",
+        # which is exactly the set of whitespace-delimited tokens.
+        token_counts: pl.Series = non_null.str.count_matches(_TOKEN_PATTERN)
+
+        profile.avg_token_count = float(token_counts.mean())  # type: ignore[arg-type]
+        profile.median_token_count = float(token_counts.median())  # type: ignore[arg-type]
+
+        # Re-derive cleanly to avoid the chained reference issue above:
+        exploded = non_null.str.split(" ").explode().drop_nulls()
+        non_empty_tokens = exploded.filter(exploded != "")
+        profile.vocabulary_size = non_empty_tokens.n_unique()
+
+        # ── 5. Character-length distribution ─────────────────────────────────
+        char_lengths: pl.Series = non_null.str.len_chars().cast(pl.Float64)
+
+        profile.char_length_min = int(char_lengths.min())  # type: ignore[arg-type]
+        profile.char_length_max = int(char_lengths.max())  # type: ignore[arg-type]
+        profile.char_length_mean = float(char_lengths.mean())  # type: ignore[arg-type]
+        profile.char_length_median = float(char_lengths.median())  # type: ignore[arg-type]
+
+        return profile