featuresmith-core 0.1.0__py3-none-any.whl

featuresmith/rules/constants.py

@@ -0,0 +1,115 @@
+"""Rules for constant and fully empty columns."""
+
+from __future__ import annotations
+
+from featuresmith.core.profile_result import ProfileResult
+from featuresmith.core.rule_finding import RuleFinding
+from featuresmith.rules.base import BaseRule
+
+
+class ConstantColumnsRule(BaseRule):
+    """Flags columns that contain only one unique value (excluding fully empty)."""
+
+    @property
+    def id(self) -> str:
+        return "quality.constant_columns"
+
+    @property
+    def name(self) -> str:
+        return "Constant Columns"
+
+    @property
+    def description(self) -> str:
+        return "Detects columns containing only one unique value, offering no predictive power."
+
+    @property
+    def category(self) -> str:
+        return "quality"
+
+    @property
+    def severity(self) -> str:
+        return "warning"
+
+    @property
+    def enabled_by_default(self) -> bool:
+        return True
+
+    def evaluate(self, profile: ProfileResult) -> list[RuleFinding]:
+        findings: list[RuleFinding] = []
+
+        for col_name, col_prof in profile.column_profiles.items():
+            # A column is constant if it has only 1 unique value and is not fully empty
+            if col_prof.is_constant and not col_prof.is_fully_empty:
+                findings.append(
+                    RuleFinding(
+                        rule_id=self.id,
+                        rule_name=self.name,
+                        category=self.category,
+                        severity=self.severity,
+                        column_name=col_name,
+                        title=f"Constant column '{col_name}'",
+                        description=(
+                            f"Column '{col_name}' contains only one unique value "
+                            f"(excluding nulls) and provides no variance."
+                        ),
+                        evidence={
+                            "is_constant": True,
+                            "is_fully_empty": False,
+                        },
+                        confidence=1.0,
+                    )
+                )
+
+        return findings
+
+
+class FullyEmptyColumnsRule(BaseRule):
+    """Flags columns that contain only missing (null) values."""
+
+    @property
+    def id(self) -> str:
+        return "quality.fully_empty_columns"
+
+    @property
+    def name(self) -> str:
+        return "Fully Empty Columns"
+
+    @property
+    def description(self) -> str:
+        return "Detects columns containing only null values."
+
+    @property
+    def category(self) -> str:
+        return "quality"
+
+    @property
+    def severity(self) -> str:
+        return "critical"
+
+    @property
+    def enabled_by_default(self) -> bool:
+        return True
+
+    def evaluate(self, profile: ProfileResult) -> list[RuleFinding]:
+        findings: list[RuleFinding] = []
+
+        for col_name, col_prof in profile.column_profiles.items():
+            if col_prof.is_fully_empty:
+                findings.append(
+                    RuleFinding(
+                        rule_id=self.id,
+                        rule_name=self.name,
+                        category=self.category,
+                        severity=self.severity,
+                        column_name=col_name,
+                        title=f"Fully empty column '{col_name}'",
+                        description=f"Column '{col_name}' contains only null/missing values.",
+                        evidence={
+                            "is_fully_empty": True,
+                            "missing_percentage": 100.0,
+                        },
+                        confidence=1.0,
+                    )
+                )
+
+        return findings

featuresmith/rules/correlation.py

@@ -0,0 +1,77 @@
+"""Rule for detecting highly correlated numeric feature pairs."""
+
+from __future__ import annotations
+
+from featuresmith.core.profile_result import ProfileResult
+from featuresmith.core.rule_finding import RuleFinding
+from featuresmith.rules.base import BaseRule
+
+
+class HighCorrelationRule(BaseRule):
+    """Flags pairs of numeric columns with Pearson correlation exceeding a threshold."""
+
+    def __init__(self, threshold: float = 0.90) -> None:
+        """Initialize the high correlation rule.
+
+        Args:
+            threshold: Pearson correlation coefficient threshold (0.0 to 1.0).
+        """
+        self.threshold = threshold
+
+    @property
+    def id(self) -> str:
+        return "statistical.high_correlation"
+
+    @property
+    def name(self) -> str:
+        return "High Correlation"
+
+    @property
+    def description(self) -> str:
+        return "Detects numeric feature pairs with high linear correlation (Pearson)."
+
+    @property
+    def category(self) -> str:
+        return "statistical"
+
+    @property
+    def severity(self) -> str:
+        return "warning"
+
+    @property
+    def enabled_by_default(self) -> bool:
+        return True
+
+    def evaluate(self, profile: ProfileResult) -> list[RuleFinding]:
+        findings: list[RuleFinding] = []
+        pearson = profile.correlation_summary.pearson
+
+        columns = list(pearson.keys())
+        for i, col1 in enumerate(columns):
+            for col2 in columns[i + 1 :]:
+                corr = pearson[col1].get(col2)
+                if corr is not None and abs(corr) >= self.threshold:
+                    findings.append(
+                        RuleFinding(
+                            rule_id=self.id,
+                            rule_name=self.name,
+                            category=self.category,
+                            severity=self.severity,
+                            column_name=col1,  # Primary column is col1
+                            title=f"High correlation between '{col1}' and '{col2}'",
+                            description=(
+                                f"Columns '{col1}' and '{col2}' are highly correlated "
+                                f"with a Pearson correlation coefficient of {corr:.3f} "
+                                f"(exceeds threshold of {self.threshold:.3f})."
+                            ),
+                            evidence={
+                                "column_a": col1,
+                                "column_b": col2,
+                                "correlation": corr,
+                                "threshold": self.threshold,
+                            },
+                            confidence=1.0,
+                        )
+                    )
+
+        return findings

featuresmith/rules/duplicates.py

@@ -0,0 +1,76 @@
+"""Rule for detecting excessive duplicate rows."""
+
+from __future__ import annotations
+
+from featuresmith.core.profile_result import ProfileResult
+from featuresmith.core.rule_finding import RuleFinding
+from featuresmith.rules.base import BaseRule
+
+
+class DuplicateRowsRule(BaseRule):
+    """Flags datasets that contain an excessive percentage of duplicate rows."""
+
+    def __init__(self, threshold: float = 10.0) -> None:
+        """Initialize the duplicate rows rule.
+
+        Args:
+            threshold: Duplicate rows percentage threshold (0.0 to 100.0).
+
+        Raises:
+            ValueError: If threshold is not between 0.0 and 100.0.
+        """
+        if not (0.0 <= threshold <= 100.0):
+            raise ValueError("threshold must be a percentage between 0.0 and 100.0.")
+        self.threshold = threshold
+
+    @property
+    def id(self) -> str:
+        return "quality.duplicate_rows"
+
+    @property
+    def name(self) -> str:
+        return "Duplicate Rows"
+
+    @property
+    def description(self) -> str:
+        return "Detects if the percentage of duplicate rows exceeds a threshold."
+
+    @property
+    def category(self) -> str:
+        return "quality"
+
+    @property
+    def severity(self) -> str:
+        return "warning"
+
+    @property
+    def enabled_by_default(self) -> bool:
+        return True
+
+    def evaluate(self, profile: ProfileResult) -> list[RuleFinding]:
+        findings: list[RuleFinding] = []
+
+        summary = profile.duplicate_summary
+        if summary.duplicate_percentage > self.threshold:
+            findings.append(
+                RuleFinding(
+                    rule_id=self.id,
+                    rule_name=self.name,
+                    category=self.category,
+                    severity=self.severity,
+                    column_name=None,
+                    title="Excessive duplicate rows detected",
+                    description=(
+                        f"The dataset contains {summary.duplicate_percentage:.2f}% duplicate rows "
+                        f"({summary.duplicate_rows_count} rows), exceeding the threshold of {self.threshold:.2f}%."
+                    ),
+                    evidence={
+                        "duplicate_rows_count": summary.duplicate_rows_count,
+                        "duplicate_percentage": summary.duplicate_percentage,
+                        "threshold": self.threshold,
+                    },
+                    confidence=1.0,
+                )
+            )
+
+        return findings

featuresmith/rules/engine.py

@@ -0,0 +1,156 @@
+"""Orchestrator for executing deterministic rules."""
+
+from __future__ import annotations
+
+import inspect
+import time
+import traceback
+from typing import Any
+
+from featuresmith.core.profile_result import ProfileResult
+from featuresmith.core.rule_finding import RuleFinding
+from featuresmith.core.rule_result import RuleResult
+from featuresmith.rules.registry import RuleRegistry, default_registry
+
+
+class RuleEngine:
+    """Orchestrates rule execution and generates a unified RuleResult."""
+
+    def __init__(self, registry: RuleRegistry | None = None) -> None:
+        """Initialize RuleEngine with a registry, defaulting to built-ins."""
+        self.registry = registry or default_registry()
+
+    def run(
+        self,
+        profile: ProfileResult,
+        *,
+        target_column: str | None = None,
+        enabled_rules: list[str] | None = None,
+        rule_config: dict[str, dict[str, Any]] | None = None,
+    ) -> RuleResult:
+        """Execute rules against the given ProfileResult.
+
+        Args:
+            profile: The computed ProfileResult.
+            target_column: Optional name of the target column for leakage detection.
+            enabled_rules: Optional list of rule IDs to execute. If None, runs
+                all rules where enabled_by_default is True.
+            rule_config: Optional dictionary of rule configurations, keyed by rule ID.
+
+        Returns:
+            A RuleResult containing execution stats, findings, and any failures.
+        """
+        # Validate rule configuration before execution
+        if rule_config:
+            for rule_id, config in rule_config.items():
+                rule = self.registry.get(rule_id)
+                if not rule:
+                    raise ValueError(f"Unknown rule ID in config: '{rule_id}'")
+
+                sig = inspect.signature(rule.__class__)
+                try:
+                    check_config = dict(config)
+                    if (
+                        rule_id == "leakage.potential_leakage"
+                        and "target_column" not in check_config
+                        and target_column is not None
+                    ):
+                        check_config["target_column"] = target_column
+
+                    # 1. Bind parameters to check names
+                    bound = sig.bind(**check_config)
+
+                    # 2. Check parameter types against annotations
+                    from typing import get_type_hints
+
+                    type_hints = get_type_hints(rule.__class__.__init__)
+
+                    for param_name, param_value in bound.arguments.items():
+                        if param_name in type_hints:
+                            annotation = type_hints[param_name]
+                            import types
+                            from typing import Union, get_args, get_origin
+
+                            def is_instance_of_type(val: Any, t: Any) -> bool:
+                                origin = get_origin(t)
+                                if origin is Union or origin is types.UnionType:
+                                    return any(
+                                        is_instance_of_type(val, arg)
+                                        for arg in get_args(t)
+                                    )
+                                if t is Any:
+                                    return True
+                                if t is None or t is type(None):
+                                    return val is None
+                                if t is float and isinstance(val, int):
+                                    return True
+                                if isinstance(t, type):
+                                    return isinstance(val, t)
+                                return True
+
+                            if not is_instance_of_type(param_value, annotation):
+                                raise TypeError(
+                                    f"Incorrect type for parameter '{param_name}'. "
+                                    f"Expected {annotation}, got {type(param_value).__name__}."
+                                )
+
+                    # 3. Instantiate to check range/value constraints
+                    _ = rule.__class__(**check_config)
+                except (TypeError, ValueError) as e:
+                    raise type(e)(
+                        f"Invalid configuration for rule '{rule_id}': {e}"
+                    ) from e
+
+        start_time = time.perf_counter()
+
+        findings: list[RuleFinding] = []
+        executed_rules: list[str] = []
+        failed_rules: dict[str, str] = {}
+
+        # Determine which rules to run
+        all_rules = self.registry.list_rules()
+        rules_to_run = []
+        for rule in all_rules:
+            if enabled_rules is not None:
+                if rule.id in enabled_rules:
+                    rules_to_run.append(rule)
+            else:
+                if rule.enabled_by_default:
+                    rules_to_run.append(rule)
+
+        for rule in rules_to_run:
+            try:
+                # Resolve rule configuration overrides
+                config = {}
+                if rule_config and rule.id in rule_config:
+                    config.update(rule_config[rule.id])
+
+                # Inject target column for leakage detection rule
+                if rule.id == "leakage.potential_leakage" and target_column is not None:
+                    config["target_column"] = target_column
+
+                # If we have configuration, instantiate a new configured instance of the rule
+                if config:
+                    rule_instance = rule.__class__(**config)
+                else:
+                    rule_instance = rule
+
+                # Evaluate the rule
+                rule_findings = rule_instance.evaluate(profile)
+                findings.extend(rule_findings)
+                executed_rules.append(rule.id)
+            except Exception as e:
+                # Isolate rule failure to prevent crashing the whole pipeline
+                failed_rules[rule.id] = "".join(
+                    traceback.format_exception(type(e), e, e.__traceback__)
+                )
+
+        execution_time_ms = (time.perf_counter() - start_time) * 1000.0
+
+        return RuleResult(
+            profile=profile,
+            findings=findings,
+            executed_rules=executed_rules,
+            execution_time_ms=execution_time_ms,
+            failed_rules=failed_rules,
+        )

featuresmith/rules/leakage.py

@@ -0,0 +1,93 @@
+"""Rule for detecting potential target leakage using high correlation."""
+
+from __future__ import annotations
+
+from featuresmith.core.profile_result import ProfileResult
+from featuresmith.core.rule_finding import RuleFinding
+from featuresmith.rules.base import BaseRule
+
+
+class LeakageRuleTargetCorrelation(BaseRule):
+    """Flags columns that have an extremely high correlation with the target column."""
+
+    def __init__(
+        self, target_column: str | None = None, threshold: float = 0.99
+    ) -> None:
+        """Initialize the target correlation leakage rule.
+
+        Args:
+            target_column: The name of the target column. If None, the rule will skip execution.
+            threshold: Pearson correlation coefficient threshold (default 0.99).
+        """
+        self.target_column = target_column
+        self.threshold = threshold
+
+    @property
+    def id(self) -> str:
+        return "leakage.potential_leakage"
+
+    @property
+    def name(self) -> str:
+        return "Potential Target Leakage"
+
+    @property
+    def description(self) -> str:
+        return (
+            "Detects columns that are extremely highly correlated with the target column, "
+            "which often indicates they contain information from the future (leakage)."
+        )
+
+    @property
+    def category(self) -> str:
+        return "leakage"
+
+    @property
+    def severity(self) -> str:
+        return "critical"
+
+    @property
+    def enabled_by_default(self) -> bool:
+        return True
+
+    def evaluate(self, profile: ProfileResult) -> list[RuleFinding]:
+        findings: list[RuleFinding] = []
+
+        if self.target_column is None:
+            # Without a target column, we do not infer one, per "No target inference" requirement.
+            return findings
+
+        pearson = profile.correlation_summary.pearson
+
+        # Verify target column is numeric/has computed correlations
+        if self.target_column not in pearson:
+            return findings
+
+        for col_name in pearson:
+            if col_name == self.target_column:
+                continue
+
+            corr = pearson[col_name].get(self.target_column)
+            if corr is not None and abs(corr) >= self.threshold:
+                findings.append(
+                    RuleFinding(
+                        rule_id=self.id,
+                        rule_name=self.name,
+                        category=self.category,
+                        severity=self.severity,
+                        column_name=col_name,
+                        title=f"Potential target leakage in '{col_name}'",
+                        description=(
+                            f"Column '{col_name}' is extremely highly correlated with "
+                            f"target '{self.target_column}' (Pearson correlation = {corr:.3f}, "
+                            f"threshold {self.threshold:.3f}), which suggests potential target leakage."
+                        ),
+                        evidence={
+                            "target_column": self.target_column,
+                            "correlation": corr,
+                            "threshold": self.threshold,
+                        },
+                        confidence=1.0,
+                    )
+                )
+
+        return findings

featuresmith/rules/missing.py

@@ -0,0 +1,83 @@
+"""Rule for detecting columns with missing values exceeding a threshold."""
+
+from __future__ import annotations
+
+from featuresmith.core.profile_result import ProfileResult
+from featuresmith.core.rule_finding import RuleFinding
+from featuresmith.rules.base import BaseRule
+
+
+class MissingValueThresholdRule(BaseRule):
+    """Flags columns that exceed a configurable missing-value threshold."""
+
+    def __init__(self, threshold: float = 20.0) -> None:
+        """Initialize the missing value rule.
+
+        Args:
+            threshold: Missing value percentage threshold (0.0 to 100.0).
+
+        Raises:
+            ValueError: If threshold is not between 0.0 and 100.0.
+        """
+        if not (0.0 <= threshold <= 100.0):
+            raise ValueError("threshold must be a percentage between 0.0 and 100.0.")
+        self.threshold = threshold
+
+    @property
+    def id(self) -> str:
+        return "quality.missing_value_threshold"
+
+    @property
+    def name(self) -> str:
+        return "Missing Value Threshold"
+
+    @property
+    def description(self) -> str:
+        return (
+            "Detects columns where the percentage of missing values "
+            "exceeds a configurable threshold."
+        )
+
+    @property
+    def category(self) -> str:
+        return "quality"
+
+    @property
+    def severity(self) -> str:
+        return "warning"
+
+    @property
+    def enabled_by_default(self) -> bool:
+        return True
+
+    def evaluate(self, profile: ProfileResult) -> list[RuleFinding]:
+        findings: list[RuleFinding] = []
+
+        for col_name, col_prof in profile.column_profiles.items():
+            if col_prof.missing_percentage > self.threshold:
+                # Severity can escalate to critical if missingness is extremely high (e.g. > 50%)
+                severity = (
+                    "critical" if col_prof.missing_percentage > 50.0 else self.severity
+                )
+                findings.append(
+                    RuleFinding(
+                        rule_id=self.id,
+                        rule_name=self.name,
+                        category=self.category,
+                        severity=severity,
+                        column_name=col_name,
+                        title=f"High missing values in column '{col_name}'",
+                        description=(
+                            f"Column '{col_name}' has {col_prof.missing_percentage:.2f}% "
+                            f"missing values, exceeding the threshold of {self.threshold:.2f}%."
+                        ),
+                        evidence={
+                            "missing_count": col_prof.missing_count,
+                            "missing_percentage": col_prof.missing_percentage,
+                            "threshold": self.threshold,
+                        },
+                        confidence=1.0,
+                    )
+                )
+
+        return findings