ml4t-diagnostic 0.1.0a1__py3-none-any.whl

ml4t/diagnostic/utils/dependencies.py

@@ -0,0 +1,318 @@
+"""Optional dependency checking and validation utilities.
+
+This module provides centralized dependency checking for optional ML libraries.
+It ensures clear error messages and graceful degradation when dependencies
+are unavailable.
+
+Example:
+    >>> from ml4t.diagnostic.utils.dependencies import check_dependency, DEPS
+    >>>
+    >>> # Check if LightGBM is available
+    >>> if check_dependency("lightgbm"):
+    ...     import lightgbm as lgb
+    ...     # Use LightGBM
+    ... else:
+    ...     print("LightGBM not available, using fallback")
+    >>>
+    >>> # Get dependency information
+    >>> print(DEPS.lightgbm.install_cmd)  # pip install lightgbm
+    >>> print(DEPS.lightgbm.purpose)      # Feature importance, boosting models
+"""
+
+from __future__ import annotations
+
+import importlib
+import warnings
+from dataclasses import dataclass
+
+
+@dataclass
+class DependencyInfo:
+    """Information about an optional dependency.
+
+    Attributes:
+        name: Package name (e.g., "lightgbm")
+        import_name: Import name (e.g., "lightgbm" or "lgb")
+        install_cmd: pip install command
+        purpose: What this dependency is used for
+        features: List of features requiring this dependency
+        alternatives: Alternative packages that can be used instead
+    """
+
+    name: str
+    import_name: str
+    install_cmd: str
+    purpose: str
+    features: list[str]
+    alternatives: list[str] | None = None
+
+    def __post_init__(self) -> None:
+        if self.alternatives is None:
+            self.alternatives = []
+
+    @property
+    def is_available(self) -> bool:
+        """Check if this dependency is installed."""
+        try:
+            importlib.import_module(self.import_name)
+            return True
+        except ImportError:
+            return False
+
+    def require(self, feature: str | None = None) -> None:
+        """Raise ImportError with helpful message if dependency not available.
+
+        Args:
+            feature: Specific feature name requesting this dependency
+
+        Raises:
+            ImportError: If dependency is not available
+        """
+        if not self.is_available:
+            msg = f"{self.name} is required"
+            if feature:
+                msg += f" for {feature}"
+            msg += f". Install with: {self.install_cmd}"
+
+            if self.alternatives:
+                msg += f"\n  Alternatives: {', '.join(self.alternatives)}"
+
+            raise ImportError(msg)
+
+    def warn_if_missing(self, feature: str | None = None, action: str = "skipping") -> bool:
+        """Warn if dependency is missing, return availability status.
+
+        Args:
+            feature: Specific feature name requesting this dependency
+            action: What will happen without this dependency (e.g., "skipping", "using fallback")
+
+        Returns:
+            bool: True if available, False if missing
+        """
+        if not self.is_available:
+            msg = f"{self.name} not available - {action}"
+            if feature:
+                msg += f" {feature}"
+            msg += f". Install with: {self.install_cmd}"
+
+            if self.alternatives:
+                msg += f" (or use: {', '.join(self.alternatives)})"
+
+            warnings.warn(msg, UserWarning, stacklevel=2)
+            return False
+        return True
+
+
+class OptionalDependencies:
+    """Registry of all optional dependencies with their metadata."""
+
+    def __init__(self):
+        self._deps: dict[str, DependencyInfo] = {}
+        self._register_dependencies()
+
+    def _register_dependencies(self):
+        """Register all known optional dependencies."""
+
+        # ML Libraries
+        self._deps["lightgbm"] = DependencyInfo(
+            name="LightGBM",
+            import_name="lightgbm",
+            install_cmd="pip install lightgbm",
+            purpose="Feature importance (MDI, permutation), boosting models",
+            features=[
+                "FeatureOutcome.run_analysis (ML importance)",
+                "MDI feature importance",
+                "Permutation importance",
+            ],
+            alternatives=["xgboost", "scikit-learn RandomForest"],
+        )
+
+        self._deps["xgboost"] = DependencyInfo(
+            name="XGBoost",
+            import_name="xgboost",
+            install_cmd="pip install xgboost",
+            purpose="Domain classifier drift detection, boosting models",
+            features=[
+                "compute_domain_classifier_drift (XGBoost backend)",
+                "Drift detection with XGBoost",
+            ],
+            alternatives=["lightgbm", "scikit-learn RandomForest"],
+        )
+
+        self._deps["shap"] = DependencyInfo(
+            name="SHAP",
+            import_name="shap",
+            install_cmd="pip install shap",
+            purpose="Shapley value feature importance and interactions",
+            features=[
+                "SHAP-based feature importance",
+                "Feature interactions analysis",
+                "Model interpretation",
+            ],
+            alternatives=["Permutation importance", "MDI importance"],
+        )
+
+        # Other optional dependencies
+        self._deps["plotly"] = DependencyInfo(
+            name="Plotly",
+            import_name="plotly",
+            install_cmd="pip install plotly",
+            purpose="Interactive visualizations and dashboards",
+            features=[
+                "create_evaluation_dashboard",
+                "Interactive plots",
+                "HTML reports",
+            ],
+            alternatives=["matplotlib", "seaborn"],
+        )
+
+    def __getattr__(self, name: str) -> DependencyInfo:
+        """Access dependencies as attributes (e.g., DEPS.lightgbm)."""
+        if name in self._deps:
+            return self._deps[name]
+        raise AttributeError(f"Unknown dependency: {name}")
+
+    def __getitem__(self, name: str) -> DependencyInfo:
+        """Access dependencies as items (e.g., DEPS["lightgbm"])."""
+        return self._deps[name]
+
+    def get(self, name: str, default=None) -> DependencyInfo | None:
+        """Get dependency info, return default if not found."""
+        return self._deps.get(name, default)
+
+    def check(self, name: str) -> bool:
+        """Check if a dependency is available."""
+        if name in self._deps:
+            return self._deps[name].is_available
+        return False
+
+    def check_multiple(self, names: list[str]) -> dict[str, bool]:
+        """Check availability of multiple dependencies.
+
+        Args:
+            names: List of dependency names to check
+
+        Returns:
+            Dict mapping dependency name to availability status
+        """
+        return {name: self.check(name) for name in names}
+
+    def get_missing(self, names: list[str]) -> list[str]:
+        """Get list of missing dependencies from a list.
+
+        Args:
+            names: List of dependency names to check
+
+        Returns:
+            List of missing dependency names
+        """
+        return [name for name in names if not self.check(name)]
+
+    def warn_missing(self, names: list[str], feature: str | None = None) -> list[str]:
+        """Warn about missing dependencies, return list of missing ones.
+
+        Args:
+            names: List of dependency names to check
+            feature: Feature name using these dependencies
+
+        Returns:
+            List of missing dependency names
+        """
+        missing = []
+        for name in names:
+            if name in self._deps and not self._deps[name].is_available:
+                self._deps[name].warn_if_missing(feature)
+                missing.append(name)
+        return missing
+
+    def summary(self) -> str:
+        """Generate summary of all dependencies and their status."""
+        lines = ["Optional Dependencies Status:"]
+        lines.append("=" * 60)
+
+        for _name, info in sorted(self._deps.items()):
+            status = "✓ Installed" if info.is_available else "✗ Missing"
+            lines.append(f"{info.name:15} {status:15} {info.purpose}")
+            if not info.is_available:
+                lines.append(f"  → Install: {info.install_cmd}")
+
+        return "\n".join(lines)
+
+
+# Global instance
+DEPS = OptionalDependencies()
+
+
+def check_dependency(name: str) -> bool:
+    """Quick check if a dependency is available.
+
+    Args:
+        name: Dependency name (e.g., "lightgbm", "shap")
+
+    Returns:
+        bool: True if available, False otherwise
+
+    Example:
+        >>> if check_dependency("lightgbm"):
+        ...     import lightgbm as lgb
+        ...     # Use LightGBM
+    """
+    return DEPS.check(name)
+
+
+def require_dependency(name: str, feature: str | None = None) -> None:
+    """Require a dependency, raise ImportError if missing.
+
+    Args:
+        name: Dependency name
+        feature: Feature name requiring this dependency
+
+    Raises:
+        ImportError: If dependency is not available
+
+    Example:
+        >>> require_dependency("shap", "SHAP analysis")
+        >>> import shap  # Safe to import now
+    """
+    if name in DEPS._deps:
+        DEPS[name].require(feature)
+    else:
+        raise ImportError(f"Unknown dependency: {name}")
+
+
+def warn_if_missing(name: str, feature: str | None = None, action: str = "skipping") -> bool:
+    """Warn if dependency is missing, return availability status.
+
+    Args:
+        name: Dependency name
+        feature: Feature name requesting this dependency
+        action: What will happen without this dependency
+
+    Returns:
+        bool: True if available, False if missing
+
+    Example:
+        >>> if warn_if_missing("lightgbm", "feature importance", "using fallback"):
+        ...     import lightgbm as lgb
+        ...     # Use LightGBM
+        ... else:
+        ...     # Use fallback method
+    """
+    if name in DEPS._deps:
+        return DEPS[name].warn_if_missing(feature, action)
+    warnings.warn(f"Unknown dependency: {name}", stacklevel=2)
+    return False
+
+
+def get_dependency_summary() -> str:
+    """Get summary of all optional dependencies and their status.
+
+    Returns:
+        str: Formatted summary of dependencies
+
+    Example:
+        >>> print(get_dependency_summary())
+        Optional Dependencies Status:
+        ...
+    """
+    return DEPS.summary()

ml4t/diagnostic/utils/sessions.py

@@ -0,0 +1,127 @@
+"""Session assignment utilities for financial time-series data.
+
+This module provides utilities to assign session dates to intraday data,
+enabling session-aware cross-validation where sessions are the atomic unit.
+"""
+
+import pandas as pd
+
+try:
+    import pandas_market_calendars as mcal  # noqa: F401 (availability check)
+
+    HAS_MARKET_CALENDARS = True
+except ImportError:
+    HAS_MARKET_CALENDARS = False
+
+
+def assign_session_dates(
+    df: pd.DataFrame,
+    calendar: str = "CME_Equity",
+    timezone: str = "UTC",
+    session_column: str = "session_date",
+) -> pd.DataFrame:
+    """Assign trading session dates to intraday data.
+
+    This function adds a session_date column to the DataFrame, where each
+    timestamp is assigned to its trading session. Sessions are atomic units
+    for cross-validation - we don't split within a session.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        DataFrame with DatetimeIndex (may be tz-naive or tz-aware)
+    calendar : str, default='CME_Equity'
+        Name of pandas_market_calendars calendar
+        Examples: 'CME_Equity', 'NYSE', 'LSE', 'TSX'
+    timezone : str, default='UTC'
+        Timezone for calendar operations
+    session_column : str, default='session_date'
+        Name of the column to add with session dates
+
+    Returns
+    -------
+    pd.DataFrame
+        DataFrame with added session_date column
+
+    Notes
+    -----
+    - For CME futures: Sunday 5pm CT - Friday 4pm CT is one session
+    - For US equities: Standard trading day 9:30am - 4pm ET
+    - If df already has the session_column, it will be overwritten
+
+    Examples
+    --------
+    >>> df = pd.read_parquet('nq_data.parquet')  # Has DatetimeIndex
+    >>> df = assign_session_dates(df, calendar='CME_Equity', timezone='America/Chicago')
+    >>> df.groupby('session_date').size()  # Samples per session
+
+    For data that already has session_date:
+    >>> if 'session_date' not in df.columns:
+    ...     df = assign_session_dates(df)
+    """
+    if not HAS_MARKET_CALENDARS:
+        raise ImportError(
+            "pandas_market_calendars is required for session assignment. "
+            "Install with: pip install pandas_market_calendars"
+        )
+
+    if not isinstance(df.index, pd.DatetimeIndex):
+        raise ValueError(f"DataFrame must have a DatetimeIndex. Got index type: {type(df.index)}")
+
+    # Import here to avoid circular dependency
+    from ml4t.diagnostic.splitters.calendar import TradingCalendar
+    from ml4t.diagnostic.splitters.calendar_config import CalendarConfig
+
+    # Create calendar configuration
+    config = CalendarConfig(exchange=calendar, timezone=timezone, localize_naive=True)
+
+    # Get trading calendar
+    trading_calendar = TradingCalendar(config)
+
+    # Assign sessions (vectorized, fast)
+    sessions = trading_calendar.get_sessions(df.index)
+
+    # Add as column (copy to avoid modifying original)
+    result = df.copy()
+    result[session_column] = sessions
+
+    return result
+
+
+def get_complete_sessions(
+    df: pd.DataFrame, session_column: str = "session_date", min_samples: int = 100
+) -> pd.Series:
+    """Get list of complete sessions with sufficient data.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        DataFrame with session_date column
+    session_column : str, default='session_date'
+        Name of the column containing session dates
+    min_samples : int, default=100
+        Minimum samples per session to consider complete
+
+    Returns
+    -------
+    pd.Series
+        Session dates that are complete (have >= min_samples)
+
+    Examples
+    --------
+    >>> df = assign_session_dates(df)
+    >>> complete = get_complete_sessions(df, min_samples=500)
+    >>> df_clean = df[df['session_date'].isin(complete)]
+    """
+    if session_column not in df.columns:
+        raise ValueError(
+            f"DataFrame does not have '{session_column}' column. Run assign_session_dates() first."
+        )
+
+    # Count samples per session
+    session_counts = df.groupby(session_column).size()
+
+    # Filter to complete sessions
+    complete_sessions = session_counts[session_counts >= min_samples].index
+
+    return complete_sessions.to_series(name=session_column)

ml4t/diagnostic/validation/__init__.py

@@ -0,0 +1,54 @@
+"""Data validation utilities for ML4T Diagnostic inputs.
+
+Provides comprehensive validation for DataFrames, time series, returns,
+and other common financial data inputs.
+
+Examples:
+    >>> from ml4t.diagnostic.validation import validate_dataframe, validate_returns
+    >>>
+    >>> # Validate DataFrame structure
+    >>> validate_dataframe(
+    ...     df,
+    ...     required_columns=["close", "volume"],
+    ...     numeric_columns=["close", "volume"]
+    ... )
+    >>>
+    >>> # Validate returns series
+    >>> validate_returns(returns, allow_nulls=False, bounds=(-0.5, 0.5))
+"""
+
+from ml4t.diagnostic.validation.dataframe import (
+    DataFrameValidator,
+    ValidationError,
+    validate_dataframe,
+    validate_schema,
+)
+from ml4t.diagnostic.validation.returns import (
+    ReturnsValidator,
+    validate_bounds,
+    validate_returns,
+)
+from ml4t.diagnostic.validation.timeseries import (
+    TimeSeriesValidator,
+    validate_frequency,
+    validate_index,
+    validate_timeseries,
+)
+
+__all__ = [
+    # Core
+    "ValidationError",
+    # DataFrame validation
+    "DataFrameValidator",
+    "validate_dataframe",
+    "validate_schema",
+    # Time series validation
+    "TimeSeriesValidator",
+    "validate_timeseries",
+    "validate_index",
+    "validate_frequency",
+    # Returns validation
+    "ReturnsValidator",
+    "validate_returns",
+    "validate_bounds",
+]