featuresmith-core 0.1.0__py3-none-any.whl

featuresmith/profiling/profiler.py

@@ -0,0 +1,209 @@
+"""Main orchestration module for profiling datasets."""
+
+from __future__ import annotations
+
+import time
+from datetime import UTC, datetime
+
+import polars as pl
+
+from featuresmith.core.dataset import Dataset
+from featuresmith.core.profile_result import (
+    ColumnProfile,
+    DatasetSummary,
+    ExecutionMetadata,
+    ProfileResult,
+)
+from featuresmith.profiling.categorical import profile_categorical_column
+from featuresmith.profiling.correlation import compute_correlations
+from featuresmith.profiling.datetime import profile_datetime_column
+from featuresmith.profiling.duplicates import analyze_duplicates
+from featuresmith.profiling.missing import analyze_missing_values
+from featuresmith.profiling.numeric import profile_numeric_column
+from featuresmith.profiling.quality import (
+    find_constant_columns,
+    find_fully_empty_columns,
+)
+from featuresmith.profiling.summary import build_dataset_metadata, classify_logical_type
+from featuresmith.profiling.text import profile_text_column
+
+
+def _get_non_null_unique_count(dataset: Dataset, col_name: str) -> int:
+    """Compute the number of unique non-null values in a column."""
+    df = dataset.dataframe
+    if dataset.backend == "pandas":
+        return int(df[col_name].dropna().nunique())
+    else:
+        # Polars
+        return int(df.select(pl.col(col_name).drop_nulls().n_unique())[0, 0])
+
+
+def _get_missing_count(dataset: Dataset, col_name: str) -> int:
+    """Compute the number of missing values in a column."""
+    df = dataset.dataframe
+    if dataset.backend == "pandas":
+        return int(df[col_name].isna().sum())
+    else:
+        # Polars
+        return int(df.select(pl.col(col_name).null_count())[0, 0])
+
+
+def profile_dataset(
+    dataset: Dataset,
+    max_correlation_columns: int = 100,
+    max_frequency_table_size: int = 1000,
+) -> ProfileResult:
+    """Orchestrate deterministic profiling of a dataset.
+
+    Args:
+        dataset: The normalized Dataset object to profile.
+        max_correlation_columns: Limit correlation computations to prevent
+            combinatorial blowup (default 100).
+
+    Returns:
+        ProfileResult: A strongly-typed statistical profile containing dataset-wide
+            summaries, column-level profiles, missingness summaries, duplicate
+            statistics, and a Pearson correlation matrix.
+
+    Notes:
+        Logical type classification uses a heuristic mapping columns into
+        "numeric", "categorical", "datetime", or "text". Pearson correlation
+        matrix is computed for numeric columns only and is capped to the first
+        `max_correlation_columns` numeric columns to prevent performance degradation.
+
+    Examples:
+        >>> import pandas as pd
+        >>> from featuresmith.core.dataset import Dataset
+        >>> from featuresmith.profiling.profiler import profile_dataset
+        >>> df = pd.DataFrame({"a": [1, 2, 3]})
+        >>> ds = Dataset.from_dataframe(df, backend="pandas")
+        >>> prof = profile_dataset(ds)
+        >>> prof.dataset_summary.row_count
+        3
+    """
+    start_time_iso = datetime.now(UTC).isoformat()
+    start_perf = time.perf_counter()
+
+    row_count = dataset.row_count
+    column_count = dataset.column_count
+
+    numeric_profiles = {}
+    categorical_profiles = {}
+    datetime_profiles = {}
+    text_profiles = {}
+    column_profiles = {}
+
+    # 1. Profile each column individually based on classified logical type
+    for col_name in dataset.schema.names:
+        logical_type = classify_logical_type(dataset, col_name)
+        dtype_str = dataset.dtypes[col_name]
+
+        if logical_type == "numeric":
+            num_prof = profile_numeric_column(dataset, col_name)
+            numeric_profiles[col_name] = num_prof
+
+            missing_count = num_prof.missing_count
+            missing_percentage = num_prof.missing_percentage
+            is_constant = num_prof.unique_count <= 1
+            is_fully_empty = num_prof.count == 0
+
+        elif logical_type == "categorical":
+            cat_prof = profile_categorical_column(
+                dataset, col_name, max_frequency_table_size=max_frequency_table_size
+            )
+            categorical_profiles[col_name] = cat_prof
+
+            missing_count = cat_prof.missing_count
+            missing_percentage = (
+                float((missing_count / row_count) * 100.0) if row_count > 0 else 0.0
+            )
+            is_constant = cat_prof.unique_count <= 1
+            is_fully_empty = cat_prof.cardinality == 0 and missing_count == row_count
+
+        elif logical_type == "datetime":
+            dt_prof = profile_datetime_column(dataset, col_name)
+            datetime_profiles[col_name] = dt_prof
+
+            missing_count = dt_prof.missing_count
+            missing_percentage = (
+                float((missing_count / row_count) * 100.0) if row_count > 0 else 0.0
+            )
+            is_constant = dt_prof.minimum == dt_prof.maximum or (
+                dt_prof.minimum is None and dt_prof.maximum is None
+            )
+            is_fully_empty = dt_prof.minimum is None and dt_prof.maximum is None
+
+        else:  # logical_type == "text"
+            txt_prof = profile_text_column(dataset, col_name)
+            text_profiles[col_name] = txt_prof
+
+            missing_count = _get_missing_count(dataset, col_name)
+            missing_percentage = (
+                float((missing_count / row_count) * 100.0) if row_count > 0 else 0.0
+            )
+            unique_cnt = _get_non_null_unique_count(dataset, col_name)
+            is_constant = unique_cnt <= 1
+            is_fully_empty = txt_prof.avg_length is None and missing_count == row_count
+
+        column_profiles[col_name] = ColumnProfile(
+            name=col_name,
+            dtype=dtype_str,
+            logical_type=logical_type,
+            missing_count=missing_count,
+            missing_percentage=missing_percentage,
+            is_constant=is_constant,
+            is_fully_empty=is_fully_empty,
+        )
+
+    # 2. Identify constant and fully empty columns across the dataset
+    constant_columns = find_constant_columns(column_profiles)
+    fully_empty_columns = find_fully_empty_columns(column_profiles)
+
+    # 3. Compute summaries
+    missing_value_summary = analyze_missing_values(dataset)
+    duplicate_summary = analyze_duplicates(
+        dataset, constant_columns, fully_empty_columns
+    )
+    correlation_summary = compute_correlations(
+        dataset, list(numeric_profiles.keys()), max_correlation_columns
+    )
+    dataset_metadata = build_dataset_metadata(dataset)
+
+    # 4. Global counts
+    dataset_summary = DatasetSummary(
+        row_count=row_count,
+        column_count=column_count,
+        size_in_bytes=dataset.file_size,
+        missing_percentage=missing_value_summary.dataset_missing_percentage,
+        duplicate_percentage=duplicate_summary.duplicate_percentage,
+        num_numeric_columns=len(numeric_profiles),
+        num_categorical_columns=len(categorical_profiles),
+        num_datetime_columns=len(datetime_profiles),
+        num_text_columns=len(text_profiles),
+        num_constant_columns=len(constant_columns),
+        num_fully_empty_columns=len(fully_empty_columns),
+    )
+
+    elapsed_time = time.perf_counter() - start_perf
+
+    from featuresmith import __version__ as pkg_version
+
+    execution_metadata = ExecutionMetadata(
+        start_time=start_time_iso,
+        duration_seconds=elapsed_time,
+        featuresmith_version=pkg_version,
+    )
+
+    return ProfileResult(
+        dataset_summary=dataset_summary,
+        column_profiles=column_profiles,
+        numeric_profiles=numeric_profiles,
+        categorical_profiles=categorical_profiles,
+        datetime_profiles=datetime_profiles,
+        text_profiles=text_profiles,
+        missing_value_summary=missing_value_summary,
+        duplicate_summary=duplicate_summary,
+        correlation_summary=correlation_summary,
+        dataset_metadata=dataset_metadata,
+        execution_metadata=execution_metadata,
+    )

featuresmith/profiling/quality.py

@@ -0,0 +1,29 @@
+"""Quality analysis utilities."""
+
+from __future__ import annotations
+
+from featuresmith.core.profile_result import ColumnProfile
+
+
+def find_constant_columns(column_profiles: dict[str, ColumnProfile]) -> list[str]:
+    """Identify columns that contain constant values.
+
+    Args:
+        column_profiles: Mapping of column name to ColumnProfile.
+
+    Returns:
+        List of constant column names.
+    """
+    return [name for name, profile in column_profiles.items() if profile.is_constant]
+
+
+def find_fully_empty_columns(column_profiles: dict[str, ColumnProfile]) -> list[str]:
+    """Identify columns that are completely empty (all missing).
+
+    Args:
+        column_profiles: Mapping of column name to ColumnProfile.
+
+    Returns:
+        List of fully empty column names.
+    """
+    return [name for name, profile in column_profiles.items() if profile.is_fully_empty]

featuresmith/profiling/summary.py

@@ -0,0 +1,117 @@
+"""Helper functions for logical type classification and dataset summaries."""
+
+from __future__ import annotations
+
+import pandas as pd
+import polars as pl
+
+from featuresmith.core.dataset import Dataset
+from featuresmith.core.profile_result import DatasetMetadata
+
+
+def classify_logical_type(dataset: Dataset, col_name: str) -> str:
+    """Classify the logical type of a column.
+
+    Args:
+        dataset: The normalized dataset.
+        col_name: Name of the column to classify.
+
+    Returns:
+        One of "numeric", "categorical", "datetime", or "text".
+    """
+    df = dataset.dataframe
+
+    # 1. Datetime detection
+    if dataset.backend == "pandas":
+        series = df[col_name]
+        if pd.api.types.is_datetime64_any_dtype(series):
+            return "datetime"
+    elif dataset.backend == "polars":
+        col_type = df.schema[col_name]
+        if col_type in (pl.Date, pl.Datetime, pl.Time, pl.Duration):
+            return "datetime"
+
+    # 2. Numeric detection
+    if dataset.backend == "pandas":
+        series = df[col_name]
+        if pd.api.types.is_numeric_dtype(series) and not pd.api.types.is_bool_dtype(
+            series
+        ):
+            return "numeric"
+    elif dataset.backend == "polars":
+        col_type = df.schema[col_name]
+        if col_type.is_numeric():
+            return "numeric"
+
+    # 3. Boolean/Categorical/String detection
+    if dataset.backend == "pandas":
+        series = df[col_name]
+        if pd.api.types.is_bool_dtype(series):
+            return "categorical"
+    elif dataset.backend == "polars":
+        col_type = df.schema[col_name]
+        if col_type == pl.Boolean:
+            return "categorical"
+
+    # If it is category, object, string or anything else, we analyze string properties
+    # Let's count unique values and average string length.
+    if dataset.backend == "pandas":
+        series = df[col_name].dropna()
+        if len(series) == 0:
+            return "categorical"
+
+        # Convert to string to check length
+        str_series = series.astype(str)
+        total_len = str_series.str.len().sum()
+        avg_len = total_len / len(series)
+        unique_count = str_series.nunique()
+        non_null_count = len(series)
+    else:
+        # Polars
+        series = df.select(pl.col(col_name).drop_nulls())
+        non_null_count = len(series)
+        if non_null_count == 0:
+            return "categorical"
+
+        # Compute stats in Polars
+        stats = series.select(
+            [
+                pl.col(col_name)
+                .cast(pl.String)
+                .str.len_chars()
+                .mean()
+                .alias("avg_len"),
+                pl.col(col_name).cast(pl.String).n_unique().alias("unique_count"),
+            ]
+        )
+        avg_len = stats.get_column("avg_len")[0] or 0.0
+        unique_count = stats.get_column("unique_count")[0] or 0
+
+    # Heuristic for Text vs Categorical
+    if avg_len >= 20:
+        return "text"
+    if (
+        non_null_count > 0
+        and (unique_count / non_null_count) > 0.5
+        and unique_count > 10
+    ):
+        return "text"
+
+    return "categorical"
+
+
+def build_dataset_metadata(dataset: Dataset) -> DatasetMetadata:
+    """Build the DatasetMetadata structure.
+
+    Args:
+        dataset: The normalized dataset.
+
+    Returns:
+        A DatasetMetadata object.
+    """
+    return DatasetMetadata(
+        source=dataset.source,
+        file_size=dataset.file_size,
+        backend=dataset.backend,
+        custom_metadata=dict(dataset.metadata),
+    )

featuresmith/profiling/text.py

@@ -0,0 +1,109 @@
+"""Text column profiling module."""
+
+from __future__ import annotations
+
+import polars as pl
+
+from featuresmith.core.dataset import Dataset
+from featuresmith.core.profile_result import TextProfile
+
+
+def profile_text_column(dataset: Dataset, col_name: str) -> TextProfile:
+    """Profile a single text column.
+
+    Args:
+        dataset: The normalized dataset.
+        col_name: Name of the column.
+
+    Returns:
+        A TextProfile object.
+    """
+    df = dataset.dataframe
+    row_count = dataset.row_count
+
+    if dataset.backend == "pandas":
+        series = df[col_name]
+        non_null_series = series.dropna().astype(str)
+        count = len(non_null_series)
+
+        if count == 0:
+            return TextProfile(
+                column_name=col_name,
+                avg_length=None,
+                min_length=None,
+                max_length=None,
+                empty_strings=int((series == "").sum()) if row_count > 0 else 0,
+                whitespace_only=0,
+                char_count=0,
+                word_count=0,
+            )
+
+        lengths = non_null_series.str.len()
+        avg_length = float(lengths.mean())
+        min_length = int(lengths.min())
+        max_length = int(lengths.max())
+
+        # Count empty strings in the entire series (including any null/nan that might be empty? No, exact matches to "")
+        empty_strings = int((series == "").sum())
+
+        # Whitespace-only matches: one or more whitespace characters
+        whitespace_only = int(non_null_series.str.match(r"^\s+$").sum())
+        char_count = int(lengths.sum())
+
+        # Word counts: find all non-whitespace chunks and sum their counts
+        word_count = int(non_null_series.str.findall(r"\S+").str.len().sum())
+
+    else:
+        # Polars
+        col = pl.col(col_name)
+        non_null_col = col.drop_nulls().cast(pl.String)
+
+        # Select all stats in one pass
+        res = df.select(
+            [
+                non_null_col.len().alias("count"),
+                non_null_col.str.len_chars().mean().alias("avg_len"),
+                non_null_col.str.len_chars().min().alias("min_len"),
+                non_null_col.str.len_chars().max().alias("max_len"),
+                (col == "").sum().alias("empty_strings"),
+                col.str.contains(r"^\s+$").sum().alias("whitespace_only"),
+                non_null_col.str.len_chars().sum().alias("char_count"),
+                non_null_col.str.extract_all(r"\S+")
+                .list.len()
+                .sum()
+                .alias("word_count"),
+            ]
+        )
+
+        count = int(res.get_column("count")[0])
+        empty_strings = int(res.get_column("empty_strings")[0] or 0)
+        whitespace_only = int(res.get_column("whitespace_only")[0] or 0)
+
+        if count == 0:
+            return TextProfile(
+                column_name=col_name,
+                avg_length=None,
+                min_length=None,
+                max_length=None,
+                empty_strings=empty_strings,
+                whitespace_only=whitespace_only,
+                char_count=0,
+                word_count=0,
+            )
+
+        avg_length = float(res.get_column("avg_len")[0])
+        min_length = int(res.get_column("min_len")[0])
+        max_length = int(res.get_column("max_len")[0])
+        char_count = int(res.get_column("char_count")[0])
+        word_count = int(res.get_column("word_count")[0] or 0)
+
+    return TextProfile(
+        column_name=col_name,
+        avg_length=avg_length,
+        min_length=min_length,
+        max_length=max_length,
+        empty_strings=empty_strings,
+        whitespace_only=whitespace_only,
+        char_count=char_count,
+        word_count=word_count,
+    )

featuresmith/py.typed

@@ -0,0 +1 @@
+# Marker file for PEP 561.

featuresmith/rules/__init__.py

@@ -0,0 +1,27 @@
+"""Rule engine and deterministic rule definitions for Featuresmith."""
+
+from featuresmith.rules.base import BaseRule
+from featuresmith.rules.cardinality import HighCardinalityRule
+from featuresmith.rules.constants import ConstantColumnsRule, FullyEmptyColumnsRule
+from featuresmith.rules.correlation import HighCorrelationRule
+from featuresmith.rules.duplicates import DuplicateRowsRule
+from featuresmith.rules.engine import RuleEngine
+from featuresmith.rules.leakage import LeakageRuleTargetCorrelation
+from featuresmith.rules.missing import MissingValueThresholdRule
+from featuresmith.rules.outliers import OutlierDetectionRule
+from featuresmith.rules.registry import RuleRegistry, default_registry
+
+__all__ = [
+    "BaseRule",
+    "RuleEngine",
+    "RuleRegistry",
+    "default_registry",
+    "MissingValueThresholdRule",
+    "DuplicateRowsRule",
+    "ConstantColumnsRule",
+    "FullyEmptyColumnsRule",
+    "HighCardinalityRule",
+    "OutlierDetectionRule",
+    "HighCorrelationRule",
+    "LeakageRuleTargetCorrelation",
+]

featuresmith/rules/base.py

@@ -0,0 +1,69 @@
+"""Abstract base class for all Featuresmith rules."""
+
+from __future__ import annotations
+
+import abc
+
+from featuresmith.core.profile_result import ProfileResult
+from featuresmith.core.rule_finding import RuleFinding
+
+
+class BaseRule(abc.ABC):
+    """Abstract base class for deterministic rule evaluation.
+
+    Attributes:
+        id: Unique stable identifier for the rule (e.g. "quality.missing_value_threshold").
+        name: Short human-readable name for the rule.
+        description: High-level description of what the rule flags.
+        category: Category of the rule ("quality", "statistical", "leakage").
+        severity: Default severity level ("info", "warning", "critical").
+        enabled_by_default: True if the rule runs by default.
+    """
+
+    @property
+    @abc.abstractmethod
+    def id(self) -> str:
+        """The stable namespaced identifier of the rule."""
+        pass
+
+    @property
+    @abc.abstractmethod
+    def name(self) -> str:
+        """A human-readable name of the rule."""
+        pass
+
+    @property
+    @abc.abstractmethod
+    def description(self) -> str:
+        """A detailed description of the rule's purpose."""
+        pass
+
+    @property
+    @abc.abstractmethod
+    def category(self) -> str:
+        """The category of the rule: 'quality', 'statistical', or 'leakage'."""
+        pass
+
+    @property
+    @abc.abstractmethod
+    def severity(self) -> str:
+        """The severity level: 'info', 'warning', or 'critical'."""
+        pass
+
+    @property
+    @abc.abstractmethod
+    def enabled_by_default(self) -> bool:
+        """Whether this rule is active by default in the engine."""
+        pass
+
+    @abc.abstractmethod
+    def evaluate(self, profile: ProfileResult) -> list[RuleFinding]:
+        """Evaluate this rule against a computed ProfileResult.
+
+        Args:
+            profile: The precomputed ProfileResult structure.
+
+        Returns:
+            A list of RuleFinding instances.
+        """
+        pass

featuresmith/rules/cardinality.py

@@ -0,0 +1,95 @@
+"""Rule for detecting categorical columns with high cardinality."""
+
+from __future__ import annotations
+
+from featuresmith.core.profile_result import ProfileResult
+from featuresmith.core.rule_finding import RuleFinding
+from featuresmith.rules.base import BaseRule
+
+
+class HighCardinalityRule(BaseRule):
+    """Flags categorical columns that have an unusually high unique value ratio."""
+
+    def __init__(self, threshold: float = 0.50, min_cardinality: int = 20) -> None:
+        """Initialize the high cardinality rule.
+
+        Args:
+            threshold: Unique ratio threshold (0.0 to 1.0), defined as
+                unique count / non-missing count.
+            min_cardinality: Minimum unique value count required to flag high cardinality.
+
+        Raises:
+            ValueError: If threshold is not between 0.0 and 1.0.
+        """
+        if not (0.0 <= threshold <= 1.0):
+            raise ValueError("threshold must be a ratio between 0.0 and 1.0.")
+        self.threshold = threshold
+        self.min_cardinality = min_cardinality
+
+    @property
+    def id(self) -> str:
+        return "statistical.high_cardinality"
+
+    @property
+    def name(self) -> str:
+        return "High Cardinality"
+
+    @property
+    def description(self) -> str:
+        return "Detects categorical columns with unusually high unique value counts relative to size."
+
+    @property
+    def category(self) -> str:
+        return "statistical"
+
+    @property
+    def severity(self) -> str:
+        return "warning"
+
+    @property
+    def enabled_by_default(self) -> bool:
+        return True
+
+    def evaluate(self, profile: ProfileResult) -> list[RuleFinding]:
+        findings: list[RuleFinding] = []
+
+        for col_name, col_prof in profile.column_profiles.items():
+            if col_prof.logical_type == "categorical":
+                cat_prof = profile.categorical_profiles.get(col_name)
+                if cat_prof is not None:
+                    cardinality = cat_prof.cardinality
+                    non_missing = (
+                        profile.dataset_summary.row_count - col_prof.missing_count
+                    )
+
+                    if non_missing > 0:
+                        ratio = cardinality / non_missing
+                        if (
+                            ratio > self.threshold
+                            and cardinality >= self.min_cardinality
+                        ):
+                            findings.append(
+                                RuleFinding(
+                                    rule_id=self.id,
+                                    rule_name=self.name,
+                                    category=self.category,
+                                    severity=self.severity,
+                                    column_name=col_name,
+                                    title=f"High cardinality in column '{col_name}'",
+                                    description=(
+                                        f"Column '{col_name}' is categorical but has a high ratio "
+                                        f"of unique values: {cardinality} unique values out of "
+                                        f"{non_missing} non-null rows ({ratio * 100:.2f}% ratio)."
+                                    ),
+                                    evidence={
+                                        "cardinality": cardinality,
+                                        "non_missing_count": non_missing,
+                                        "unique_ratio": ratio,
+                                        "threshold": self.threshold,
+                                        "min_cardinality": self.min_cardinality,
+                                    },
+                                    confidence=1.0,
+                                )
+                            )
+
+        return findings