ml4t-diagnostic 0.1.0a1__py3-none-any.whl

ml4t/diagnostic/validation/dataframe.py

@@ -0,0 +1,274 @@
+"""DataFrame validation utilities."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import polars as pl
+
+
+class ValidationError(ValueError):
+    """Validation error with helpful context."""
+
+    def __init__(self, message: str, context: dict[str, Any] | None = None):
+        """Initialize validation error.
+
+        Args:
+            message: Error message
+            context: Additional context (columns, types, etc.)
+        """
+        self.context = context or {}
+        super().__init__(self._format_message(message))
+
+    def _format_message(self, message: str) -> str:
+        """Format error message with context."""
+        if not self.context:
+            return message
+
+        context_str = "\n".join(f"  {k}: {v}" for k, v in self.context.items())
+        return f"{message}\nContext:\n{context_str}"
+
+
+class DataFrameValidator:
+    """Validator for Polars DataFrames.
+
+    Examples:
+        >>> validator = DataFrameValidator(df)
+        >>> validator.require_columns(["close", "volume"])
+        >>> validator.require_numeric(["close", "volume"])
+        >>> validator.check_nulls(allow_nulls=False)
+    """
+
+    def __init__(self, df: pl.DataFrame):
+        """Initialize validator.
+
+        Args:
+            df: DataFrame to validate
+        """
+        self.df = df
+
+    def require_columns(self, columns: list[str]) -> DataFrameValidator:
+        """Require specific columns to exist.
+
+        Args:
+            columns: Required column names
+
+        Returns:
+            Self for chaining
+
+        Raises:
+            ValidationError: If required columns missing
+        """
+        missing = [col for col in columns if col not in self.df.columns]
+
+        if missing:
+            raise ValidationError(
+                f"Missing required columns: {missing}",
+                context={
+                    "required": columns,
+                    "available": self.df.columns,
+                    "missing": missing,
+                },
+            )
+
+        return self
+
+    def require_numeric(self, columns: list[str]) -> DataFrameValidator:
+        """Require columns to be numeric types.
+
+        Args:
+            columns: Column names that must be numeric
+
+        Returns:
+            Self for chaining
+
+        Raises:
+            ValidationError: If columns not numeric
+        """
+        non_numeric = []
+
+        for col in columns:
+            if col not in self.df.columns:
+                continue
+
+            dtype = self.df[col].dtype
+            if not dtype.is_numeric():
+                non_numeric.append((col, str(dtype)))
+
+        if non_numeric:
+            raise ValidationError(
+                f"Non-numeric columns: {[col for col, _ in non_numeric]}",
+                context={
+                    "expected": "numeric types (Int*, Float*, Decimal)",
+                    "actual": dict(non_numeric),
+                },
+            )
+
+        return self
+
+    def check_nulls(
+        self, columns: list[str] | None = None, allow_nulls: bool = False
+    ) -> DataFrameValidator:
+        """Check for null values in columns.
+
+        Args:
+            columns: Columns to check (None = all columns)
+            allow_nulls: Whether nulls are allowed
+
+        Returns:
+            Self for chaining
+
+        Raises:
+            ValidationError: If nulls found when not allowed
+        """
+        check_columns = columns or self.df.columns
+
+        if not allow_nulls:
+            null_counts = {}
+
+            for col in check_columns:
+                if col not in self.df.columns:
+                    continue
+
+                null_count = self.df[col].null_count()
+                if null_count > 0:
+                    null_counts[col] = null_count
+
+            if null_counts:
+                total_nulls = sum(null_counts.values())
+                raise ValidationError(
+                    f"Found {total_nulls} null values",
+                    context={
+                        "null_columns": list(null_counts.keys()),
+                        "null_counts": null_counts,
+                    },
+                )
+
+        return self
+
+    def check_empty(self) -> DataFrameValidator:
+        """Check if DataFrame is empty.
+
+        Returns:
+            Self for chaining
+
+        Raises:
+            ValidationError: If DataFrame is empty
+        """
+        if len(self.df) == 0:
+            raise ValidationError(
+                "DataFrame is empty",
+                context={"shape": self.df.shape, "columns": self.df.columns},
+            )
+
+        return self
+
+    def check_min_rows(self, min_rows: int) -> DataFrameValidator:
+        """Check minimum number of rows.
+
+        Args:
+            min_rows: Minimum required rows
+
+        Returns:
+            Self for chaining
+
+        Raises:
+            ValidationError: If too few rows
+        """
+        if len(self.df) < min_rows:
+            raise ValidationError(
+                f"Insufficient rows: {len(self.df)} < {min_rows}",
+                context={"required": min_rows, "actual": len(self.df)},
+            )
+
+        return self
+
+
+def validate_dataframe(
+    df: pl.DataFrame,
+    required_columns: list[str] | None = None,
+    numeric_columns: list[str] | None = None,
+    allow_nulls: bool = True,
+    min_rows: int = 1,
+) -> None:
+    """Validate DataFrame structure and content.
+
+    Args:
+        df: DataFrame to validate
+        required_columns: Columns that must exist
+        numeric_columns: Columns that must be numeric
+        allow_nulls: Whether null values are allowed
+        min_rows: Minimum number of rows required
+
+    Raises:
+        ValidationError: If validation fails
+
+    Examples:
+        >>> validate_dataframe(
+        ...     df,
+        ...     required_columns=["close", "volume"],
+        ...     numeric_columns=["close", "volume"],
+        ...     allow_nulls=False,
+        ...     min_rows=100
+        ... )
+    """
+    validator = DataFrameValidator(df)
+
+    validator.check_empty().check_min_rows(min_rows)
+
+    if required_columns:
+        validator.require_columns(required_columns)
+
+    if numeric_columns:
+        validator.require_numeric(numeric_columns)
+
+    if not allow_nulls:
+        validator.check_nulls(allow_nulls=False)
+
+
+def validate_schema(df: pl.DataFrame, expected_schema: dict[str, str | type]) -> None:
+    """Validate DataFrame schema matches expected types.
+
+    Args:
+        df: DataFrame to validate
+        expected_schema: Map of column name to expected type
+
+    Raises:
+        ValidationError: If schema doesn't match
+
+    Examples:
+        >>> validate_schema(df, {
+        ...     "close": "Float64",
+        ...     "volume": "Int64",
+        ...     "date": pl.Date
+        ... })
+    """
+    mismatches = {}
+
+    for col, expected_type in expected_schema.items():
+        if col not in df.columns:
+            mismatches[col] = ("missing", expected_type)
+            continue
+
+        actual_type = df[col].dtype
+
+        # Handle string type names
+        if isinstance(expected_type, str):
+            expected_type_str = expected_type
+            actual_type_str = str(actual_type)
+        else:
+            expected_type_str = str(expected_type)
+            actual_type_str = str(actual_type)
+
+        if expected_type_str not in actual_type_str:
+            mismatches[col] = (actual_type_str, expected_type_str)
+
+    if mismatches:
+        raise ValidationError(
+            "Schema mismatch",
+            context={
+                "mismatches": {
+                    col: f"expected {exp}, got {act}" for col, (act, exp) in mismatches.items()
+                }
+            },
+        )

ml4t/diagnostic/validation/returns.py

@@ -0,0 +1,280 @@
+"""Returns validation utilities."""
+
+from __future__ import annotations
+
+from typing import SupportsFloat, cast
+
+import polars as pl
+
+from ml4t.diagnostic.validation.dataframe import ValidationError
+
+
+class ReturnsValidator:
+    """Validator for returns series.
+
+    Examples:
+        >>> validator = ReturnsValidator(returns)
+        >>> validator.check_numeric()
+        >>> validator.check_bounds(-0.5, 0.5)
+        >>> validator.check_distribution()
+    """
+
+    def __init__(self, returns: pl.Series | pl.DataFrame, column: str | None = None):
+        """Initialize validator.
+
+        Args:
+            returns: Returns series or DataFrame
+            column: Column name if DataFrame provided
+        """
+        if isinstance(returns, pl.DataFrame):
+            if column is None:
+                raise ValueError("column required when passing DataFrame")
+            self.returns = returns[column]
+        else:
+            self.returns = returns
+
+    def check_numeric(self) -> ReturnsValidator:
+        """Check that returns are numeric.
+
+        Returns:
+            Self for chaining
+
+        Raises:
+            ValidationError: If not numeric
+        """
+        if not self.returns.dtype.is_numeric():
+            raise ValidationError(
+                "Returns must be numeric",
+                context={"dtype": str(self.returns.dtype)},
+            )
+
+        return self
+
+    def check_bounds(
+        self, lower: float | None = None, upper: float | None = None
+    ) -> ReturnsValidator:
+        """Check returns fall within bounds.
+
+        Args:
+            lower: Lower bound (inclusive)
+            upper: Upper bound (inclusive)
+
+        Returns:
+            Self for chaining
+
+        Raises:
+            ValidationError: If returns out of bounds
+        """
+        self.check_numeric()
+
+        # Drop nulls for bounds checking
+        clean_returns = self.returns.drop_nulls()
+
+        if len(clean_returns) == 0:
+            return self
+
+        if lower is not None:
+            min_val = float(cast(SupportsFloat, clean_returns.min()))
+            if min_val < lower:
+                out_of_bounds = (clean_returns < lower).sum()
+                raise ValidationError(
+                    f"Returns below lower bound: {min_val:.4f} < {lower}",
+                    context={
+                        "lower_bound": lower,
+                        "min_value": min_val,
+                        "count_out_of_bounds": out_of_bounds,
+                    },
+                )
+
+        if upper is not None:
+            max_val = float(cast(SupportsFloat, clean_returns.max()))
+            if max_val > upper:
+                out_of_bounds = (clean_returns > upper).sum()
+                raise ValidationError(
+                    f"Returns above upper bound: {max_val:.4f} > {upper}",
+                    context={
+                        "upper_bound": upper,
+                        "max_value": max_val,
+                        "count_out_of_bounds": out_of_bounds,
+                    },
+                )
+
+        return self
+
+    def check_finite(self) -> ReturnsValidator:
+        """Check for infinite values.
+
+        Returns:
+            Self for chaining
+
+        Raises:
+            ValidationError: If infinite values found
+        """
+        self.check_numeric()
+
+        # Check for inf/-inf
+        is_inf = self.returns.is_infinite()
+        inf_count = is_inf.sum()
+
+        if inf_count > 0:
+            raise ValidationError(
+                f"Found {inf_count} infinite values",
+                context={"infinite_count": inf_count},
+            )
+
+        return self
+
+    def check_nulls(self, allow_nulls: bool = False) -> ReturnsValidator:
+        """Check for null values.
+
+        Args:
+            allow_nulls: Whether nulls are allowed
+
+        Returns:
+            Self for chaining
+
+        Raises:
+            ValidationError: If nulls found when not allowed
+        """
+        if not allow_nulls:
+            null_count = self.returns.null_count()
+
+            if null_count > 0:
+                raise ValidationError(
+                    f"Found {null_count} null values",
+                    context={
+                        "null_count": null_count,
+                        "total_count": len(self.returns),
+                    },
+                )
+
+        return self
+
+    def check_distribution(
+        self,
+        max_abs_skew: float | None = None,
+        max_abs_kurtosis: float | None = None,
+    ) -> ReturnsValidator:
+        """Check distribution characteristics.
+
+        Args:
+            max_abs_skew: Maximum absolute skewness (None = no check)
+            max_abs_kurtosis: Maximum absolute excess kurtosis (None = no check)
+
+        Returns:
+            Self for chaining
+
+        Raises:
+            ValidationError: If distribution extreme
+        """
+        self.check_numeric()
+
+        clean_returns = self.returns.drop_nulls()
+
+        if len(clean_returns) < 30:
+            # Need sufficient data for distribution checks
+            return self
+
+        if max_abs_skew is not None:
+            # Calculate skewness (simplified)
+            mean = float(cast(SupportsFloat, clean_returns.mean()))
+            std = float(cast(SupportsFloat, clean_returns.std()))
+
+            if std > 0:
+                skew = float(cast(SupportsFloat, ((clean_returns - mean) ** 3).mean())) / (std**3)
+
+                if abs(skew) > max_abs_skew:
+                    raise ValidationError(
+                        f"Extreme skewness detected: {skew:.2f}",
+                        context={
+                            "skewness": skew,
+                            "max_allowed": max_abs_skew,
+                        },
+                    )
+
+        if max_abs_kurtosis is not None:
+            # Calculate excess kurtosis (simplified)
+            mean = float(cast(SupportsFloat, clean_returns.mean()))
+            std = float(cast(SupportsFloat, clean_returns.std()))
+
+            if std > 0:
+                kurtosis = (
+                    float(cast(SupportsFloat, ((clean_returns - mean) ** 4).mean())) / (std**4) - 3
+                )
+
+                if abs(kurtosis) > max_abs_kurtosis:
+                    raise ValidationError(
+                        f"Extreme kurtosis detected: {kurtosis:.2f}",
+                        context={
+                            "kurtosis": kurtosis,
+                            "max_allowed": max_abs_kurtosis,
+                        },
+                    )
+
+        return self
+
+
+def validate_returns(
+    returns: pl.Series | pl.DataFrame,
+    column: str | None = None,
+    bounds: tuple[float, float] | None = None,
+    allow_nulls: bool = False,
+    check_finite: bool = True,
+) -> None:
+    """Validate returns series.
+
+    Args:
+        returns: Returns series or DataFrame
+        column: Column name if DataFrame
+        bounds: (lower, upper) bounds for returns
+        allow_nulls: Whether null values allowed
+        check_finite: Whether to check for infinite values
+
+    Raises:
+        ValidationError: If validation fails
+
+    Examples:
+        >>> validate_returns(
+        ...     returns,
+        ...     bounds=(-0.5, 0.5),
+        ...     allow_nulls=False,
+        ...     check_finite=True
+        ... )
+    """
+    validator = ReturnsValidator(returns, column)
+
+    validator.check_numeric()
+
+    if not allow_nulls:
+        validator.check_nulls(allow_nulls=False)
+
+    if check_finite:
+        validator.check_finite()
+
+    if bounds is not None:
+        lower, upper = bounds
+        validator.check_bounds(lower, upper)
+
+
+def validate_bounds(
+    returns: pl.Series | pl.DataFrame,
+    column: str | None = None,
+    lower: float | None = None,
+    upper: float | None = None,
+) -> None:
+    """Validate returns fall within bounds.
+
+    Args:
+        returns: Returns series or DataFrame
+        column: Column name if DataFrame
+        lower: Lower bound (inclusive)
+        upper: Upper bound (inclusive)
+
+    Raises:
+        ValidationError: If returns out of bounds
+
+    Examples:
+        >>> validate_bounds(returns, lower=-1.0, upper=1.0)
+    """
+    validator = ReturnsValidator(returns, column)
+    validator.check_numeric().check_bounds(lower, upper)