ml4t-diagnostic 0.1.0a1__py3-none-any.whl

ml4t/diagnostic/logging/wandb.py

@@ -0,0 +1,412 @@
+"""Weights & Biases integration for experiment tracking.
+
+This module provides hooks for logging ml4t-diagnostic experiments to W&B,
+enabling tracking of evaluation metrics, hyperparameters, and
+visualizations across experiments.
+"""
+
+import numbers
+import warnings
+from typing import Any, SupportsFloat, cast
+
+import numpy as np
+import pandas as pd
+
+try:
+    import wandb  # type: ignore[import-not-found,unused-ignore]
+
+    HAS_WANDB = True
+except ImportError:
+    HAS_WANDB = False
+
+
+class WandbLogger:
+    """Logger for Weights & Biases experiment tracking.
+
+    This class provides a unified interface for logging ml4t-diagnostic
+    experiments to W&B, handling initialization, metric logging,
+    and artifact management.
+    """
+
+    def __init__(
+        self,
+        project: str | None = None,
+        entity: str | None = None,
+        name: str | None = None,
+        config: dict[str, Any] | None = None,
+        tags: list[str] | None = None,
+        notes: str | None = None,
+        disabled: bool = False,
+    ):
+        """Initialize W&B logger.
+
+        Parameters
+        ----------
+        project : str, optional
+            W&B project name
+        entity : str, optional
+            W&B entity (team or username)
+        name : str, optional
+            Run name
+        config : dict, optional
+            Configuration dictionary to log
+        tags : list[str], optional
+            Tags for the run
+        notes : str, optional
+            Notes about the run
+        disabled : bool
+            If True, disables W&B logging
+        """
+        self.disabled = disabled or not HAS_WANDB
+        self.run = None
+
+        if self.disabled:
+            if not HAS_WANDB and not disabled:
+                warnings.warn(
+                    "wandb not installed. Install with: pip install wandb",
+                    stacklevel=2,
+                )
+            return
+
+        # Initialize W&B run
+        self.run = wandb.init(
+            project=project or "ml4t-diagnostic",
+            entity=entity,
+            name=name,
+            config=config,
+            tags=tags or [],
+            notes=notes,
+            reinit=True,
+        )
+
+    def log_config(self, config: dict[str, Any]) -> None:
+        """Log configuration parameters.
+
+        Parameters
+        ----------
+        config : dict
+            Configuration dictionary
+        """
+        if self.disabled or self.run is None:
+            return
+
+        # Flatten nested config for W&B
+        flat_config = self._flatten_dict(config)
+        wandb.config.update(flat_config)
+
+    def log_metrics(
+        self,
+        metrics: dict[str, Any],
+        step: int | None = None,
+        prefix: str = "",
+    ) -> None:
+        """Log evaluation metrics.
+
+        Parameters
+        ----------
+        metrics : dict
+            Metrics to log
+        step : int, optional
+            Step number (e.g., CV fold)
+        prefix : str
+            Prefix for metric names
+        """
+        if self.disabled or self.run is None:
+            return
+
+        # Prepare metrics for logging
+        log_dict = {}
+
+        for name, value in metrics.items():
+            key = f"{prefix}{name}" if prefix else name
+
+            if isinstance(value, dict):
+                # Handle nested metrics (e.g., with confidence intervals)
+                for sub_key, sub_value in value.items():
+                    if isinstance(sub_value, numbers.Number):
+                        log_dict[f"{key}/{sub_key}"] = float(cast(SupportsFloat, sub_value))
+            elif isinstance(value, numbers.Number):
+                log_dict[key] = float(cast(SupportsFloat, value))
+            elif isinstance(value, list | np.ndarray):
+                # Log array statistics
+                if len(value) > 0:
+                    log_dict[f"{key}/mean"] = float(np.mean(value))
+                    log_dict[f"{key}/std"] = float(np.std(value))
+                    log_dict[f"{key}/min"] = float(np.min(value))
+                    log_dict[f"{key}/max"] = float(np.max(value))
+
+        if step is not None:
+            log_dict["step"] = step
+
+        wandb.log(log_dict)
+
+    def log_fold_results(
+        self,
+        fold_idx: int,
+        train_size: int,
+        test_size: int,
+        metrics: dict[str, Any],
+    ) -> None:
+        """Log results from a single CV fold.
+
+        Parameters
+        ----------
+        fold_idx : int
+            Fold index
+        train_size : int
+            Training set size
+        test_size : int
+            Test set size
+        metrics : dict
+            Fold metrics
+        """
+        if self.disabled or self.run is None:
+            return
+
+        # Add metrics with fold prefix
+        self.log_metrics(metrics, step=fold_idx, prefix="fold/")
+
+        # Log fold metadata
+        wandb.log(
+            {
+                "fold/train_size": train_size,
+                "fold/test_size": test_size,
+                "fold/train_test_ratio": train_size / test_size if test_size > 0 else 0,
+            },
+            step=fold_idx,
+        )
+
+    def log_statistical_tests(self, tests: dict[str, Any]) -> None:
+        """Log statistical test results.
+
+        Parameters
+        ----------
+        tests : dict
+            Statistical test results
+        """
+        if self.disabled or self.run is None:
+            return
+
+        log_dict = {}
+
+        for test_name, result in tests.items():
+            if isinstance(result, dict):
+                for key, value in result.items():
+                    if isinstance(value, numbers.Number):
+                        log_dict[f"stats/{test_name}/{key}"] = float(cast(SupportsFloat, value))
+                    elif key == "significant" and isinstance(value, bool):
+                        log_dict[f"stats/{test_name}/{key}"] = int(value)
+
+        wandb.log(log_dict)
+
+    def log_figure(
+        self,
+        figure: Any,
+        name: str,
+        step: int | None = None,
+    ) -> None:
+        """Log a Plotly figure.
+
+        Parameters
+        ----------
+        figure : plotly.graph_objects.Figure
+            Figure to log
+        name : str
+            Figure name
+        step : int, optional
+            Step number
+        """
+        if self.disabled or self.run is None:
+            return
+
+        # Convert Plotly figure to W&B
+        wandb.log({f"plots/{name}": figure}, step=step)
+
+    def log_evaluation_summary(
+        self,
+        result: Any,  # EvaluationResult
+        _predictions: Any | None = None,
+        _returns: Any | None = None,
+    ) -> None:
+        """Log complete evaluation summary.
+
+        Parameters
+        ----------
+        result : EvaluationResult
+            Evaluation result object
+        predictions : array-like, optional
+            Predictions for additional logging
+        returns : array-like, optional
+            Returns for additional logging
+        """
+        if self.disabled or self.run is None:
+            return
+
+        # Log summary metrics
+        summary = result.summary()
+
+        # Log aggregate metrics
+        self.log_metrics(summary["metrics"], prefix="summary/")
+
+        # Log statistical tests
+        if summary.get("statistical_tests"):
+            self.log_statistical_tests(summary["statistical_tests"])
+
+        # Log metadata
+        wandb.log(
+            {
+                "summary/tier": result.tier,
+                "summary/n_folds": summary["n_folds"],
+                "summary/splitter": result.splitter_name,
+            },
+        )
+
+        # Create summary table
+        if result.fold_results:
+            fold_data = []
+            for fold in result.fold_results:
+                fold_row = {"fold": fold.get("fold", 0)}
+                fold_row.update(
+                    {k: v for k, v in fold.items() if isinstance(v, numbers.Number)},
+                )
+                fold_data.append(fold_row)
+
+            fold_table = wandb.Table(dataframe=pd.DataFrame(fold_data))
+            wandb.log({"tables/fold_results": fold_table})
+
+    def log_artifact(
+        self,
+        artifact_path: str,
+        name: str,
+        artifact_type: str = "evaluation",
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """Log an artifact (model, dataset, etc.).
+
+        Parameters
+        ----------
+        artifact_path : str
+            Path to artifact file
+        name : str
+            Artifact name
+        artifact_type : str
+            Type of artifact
+        metadata : dict, optional
+            Additional metadata
+        """
+        if self.disabled or self.run is None:
+            return
+
+        artifact = wandb.Artifact(
+            name=name,
+            type=artifact_type,
+            metadata=metadata or {},
+        )
+        artifact.add_file(artifact_path)
+        wandb.log_artifact(artifact)
+
+    def finish(self) -> None:
+        """Finish the W&B run."""
+        if self.disabled or self.run is None:
+            return
+
+        wandb.finish()
+
+    def __enter__(self):
+        """Context manager entry."""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit."""
+        self.finish()
+
+    @staticmethod
+    def _flatten_dict(
+        d: dict[str, Any],
+        parent_key: str = "",
+        sep: str = "/",
+    ) -> dict[str, Any]:
+        """Flatten nested dictionary."""
+        items: list[tuple[str, Any]] = []
+
+        for k, v in d.items():
+            new_key = f"{parent_key}{sep}{k}" if parent_key else k
+
+            if isinstance(v, dict):
+                items.extend(WandbLogger._flatten_dict(v, new_key, sep=sep).items())
+            else:
+                items.append((new_key, v))
+
+        return dict(items)
+
+
+def log_experiment(
+    evaluator: Any,
+    X: Any,
+    y: Any,
+    model: Any,
+    project: str | None = None,
+    config: dict[str, Any] | None = None,
+    tags: list[str] | None = None,
+    **kwargs: Any,
+) -> Any:
+    """Convenience function to run and log an experiment.
+
+    Parameters
+    ----------
+    evaluator : ml4t-diagnostic.Evaluator
+        Configured evaluator
+    X : array-like
+        Features
+    y : array-like
+        Labels
+    model : estimator
+        Model to evaluate
+    project : str, optional
+        W&B project name
+    config : dict, optional
+        Additional config to log
+    tags : list[str], optional
+        Experiment tags
+    **kwargs : Any
+        Additional arguments passed to evaluate()
+
+    Returns:
+    -------
+    EvaluationResult
+        Result with W&B logging
+    """
+    if not HAS_WANDB:
+        warnings.warn(
+            "wandb not installed. Running without logging. Install with: pip install wandb",
+            stacklevel=2,
+        )
+        return evaluator.evaluate(X, y, model, **kwargs)
+
+    # Initialize logger
+    with WandbLogger(project=project, config=config, tags=tags) as logger:
+        # Log evaluator configuration
+        logger.log_config(
+            {
+                "evaluator": {
+                    "tier": evaluator.tier,
+                    "splitter": evaluator.splitter.__class__.__name__,
+                    "metrics": evaluator.metrics,
+                    "statistical_tests": evaluator.statistical_tests,
+                    "confidence_level": evaluator.confidence_level,
+                    "bootstrap_samples": evaluator.bootstrap_samples,
+                },
+            },
+        )
+
+        # Log model info if available
+        if hasattr(model, "get_params"):
+            logger.log_config({"model": model.get_params()})
+
+        # Run evaluation
+        result = evaluator.evaluate(X, y, model, **kwargs)
+
+        # Log results
+        logger.log_evaluation_summary(result)
+
+        return result

ml4t/diagnostic/metrics/__init__.py

@@ -0,0 +1,9 @@
+"""
+Metrics module for ML4T Diagnostic.
+
+Provides statistical metrics and percentile computation utilities for model evaluation.
+"""
+
+from ml4t.diagnostic.metrics.percentiles import compute_fold_percentiles
+
+__all__ = ["compute_fold_percentiles"]

ml4t/diagnostic/metrics/percentiles.py

@@ -0,0 +1,128 @@
+"""
+Percentile computation utilities for threshold-based signal generation.
+
+Provides fast percentile computation from fold-specific predictions using Polars,
+designed to prevent data leakage by computing thresholds from training data only.
+"""
+
+from collections.abc import Sequence
+
+import pandas as pd
+import polars as pl
+
+
+def compute_fold_percentiles(
+    predictions: pd.DataFrame | pl.DataFrame,
+    percentiles: Sequence[float],
+    fold_col: str = "fold_id",
+    iteration_col: str = "iteration",
+    prediction_col: str = "prediction",
+    verbose: bool = True,
+) -> pd.DataFrame:
+    """
+    Compute percentiles from predictions grouped by fold and iteration.
+
+    Uses efficient Polars group_by operation to compute percentiles 10-50x faster
+    than nested loops. Designed for threshold-based signal generation where
+    thresholds must be computed from TRAINING predictions only to prevent data leakage.
+
+    Performance: ~50-100ms for 89M predictions with 26 percentiles (vs 5-10s with loops)
+
+    Args:
+        predictions: DataFrame with predictions to compute percentiles from
+            Must contain: fold_col, iteration_col, prediction_col
+        percentiles: List of percentiles to compute (e.g., [0.1, 0.5, 1, ..., 99, 99.5, 99.9])
+            Values should be in range [0, 100]
+        fold_col: Name of fold identifier column (default: "fold_id")
+        iteration_col: Name of iteration/checkpoint column (default: "iteration")
+        prediction_col: Name of prediction values column (default: "prediction")
+        verbose: Print progress information (default: True)
+
+    Returns:
+        DataFrame with columns: [fold_col, iteration_col, p{percentile}, ...]
+        - One row per (fold, iteration) combination
+        - Percentile columns named like "p0.1", "p99.9", etc.
+
+    Example:
+        >>> # Training predictions: 13 folds × 10 iterations × 687k samples
+        >>> import pandas as pd
+        >>> predictions = pd.DataFrame({
+        ...     'fold_id': [0] * 1000 + [1] * 1000,
+        ...     'iteration': [50] * 500 + [100] * 500 + [50] * 500 + [100] * 500,
+        ...     'prediction': np.random.rand(2000)
+        ... })
+        >>>
+        >>> # Compute percentiles for LONG and SHORT strategies
+        >>> percentiles = [0.1, 0.5, 1, 5, 10, 90, 95, 99, 99.5, 99.9]
+        >>> thresholds = compute_fold_percentiles(predictions, percentiles)
+        >>>
+        >>> # Result: 2 rows (2 folds) × 2 iterations = 4 rows
+        >>> thresholds.shape
+        (4, 12)  # 2 meta columns + 10 percentile columns
+        >>>
+        >>> # Use for signal generation
+        >>> fold_0_iter_100 = thresholds[
+        ...     (thresholds['fold_id'] == 0) & (thresholds['iteration'] == 100)
+        ... ]
+        >>> long_threshold = fold_0_iter_100['p95'].values[0]
+        >>> short_threshold = fold_0_iter_100['p5'].values[0]
+
+    Methodology:
+        1. Convert predictions to Polars (if pandas)
+        2. Group by (fold_id, iteration)
+        3. Compute all percentiles in single aggregation
+        4. Return as pandas DataFrame
+
+    Data Leakage Prevention:
+        CRITICAL: This function should ONLY be called on TRAINING predictions.
+        - Training: compute_fold_percentiles(train_predictions) → save thresholds
+        - Validation: Apply saved thresholds to OOS predictions
+        - NEVER: compute_fold_percentiles(val_predictions) → data leakage!
+
+    Performance Notes:
+        - Polars group_by is 10-50x faster than nested loops
+        - Memory usage: O(n_predictions) for single pass
+        - Time complexity: O(n * log(n)) for sorting within groups
+        - Recommended for predictions > 1M rows
+    """
+    if verbose:
+        print("\nComputing fold-specific percentiles (Fast Polars Method)...")
+
+    # Convert to Polars if pandas
+    preds_pl = pl.from_pandas(predictions) if isinstance(predictions, pd.DataFrame) else predictions
+
+    # Validate required columns
+    required_cols = {fold_col, iteration_col, prediction_col}
+    available_cols = set(preds_pl.columns)
+    missing = required_cols - available_cols
+    if missing:
+        raise ValueError(f"Missing required columns: {missing}. Available: {available_cols}")
+
+    # Convert percentiles to quantiles (0-1 range)
+    quantiles = [p / 100 for p in percentiles]
+
+    # Compute percentiles with single group_by operation
+    percentiles_df = (
+        preds_pl.group_by([fold_col, iteration_col])
+        .agg(
+            [
+                pl.col(prediction_col).quantile(q, interpolation="linear").alias(f"p{p}")
+                for q, p in zip(quantiles, percentiles, strict=False)
+            ]
+        )
+        .sort([fold_col, iteration_col])
+    )
+
+    # Convert back to pandas for compatibility
+    result = percentiles_df.to_pandas()
+
+    if verbose:
+        n_folds = result[fold_col].nunique()
+        n_iterations = result[iteration_col].nunique()
+        print(f"✓ Computed {len(result)} percentile arrays")
+        print(
+            f"✓ Structure: {n_folds} folds × {n_iterations} iterations × {len(percentiles)} percentiles"
+        )
+        print(f"✓ Percentile columns: {sorted([c for c in result.columns if c.startswith('p')])}")
+
+    return result

ml4t/diagnostic/py.typed

@@ -0,0 +1 @@
+# PEP 561 marker file - this package supports type checking

ml4t/diagnostic/reporting/__init__.py

@@ -0,0 +1,43 @@
+"""Report generation module for ML4T Diagnostic results.
+
+Provides flexible report generation in multiple formats:
+- HTML: Rich, styled reports with tables and charts
+- JSON: Machine-readable structured output
+- Markdown: Human-readable documentation
+
+Examples:
+    >>> from ml4t.diagnostic.reporting import ReportFactory, ReportFormat
+    >>> from ml4t.diagnostic.results import FeatureDiagnosticsResult
+    >>>
+    >>> # Generate HTML report
+    >>> html_report = ReportFactory.render(result, ReportFormat.HTML)
+    >>>
+    >>> # Generate JSON report
+    >>> json_report = ReportFactory.render(result, ReportFormat.JSON, indent=4)
+    >>>
+    >>> # Generate Markdown report
+    >>> md_report = ReportFactory.render(result, ReportFormat.MARKDOWN)
+    >>>
+    >>> # Save to file
+    >>> generator = ReportFactory.create(ReportFormat.HTML)
+    >>> html = generator.render(result)
+    >>> generator.save(html, "report.html")
+"""
+
+from ml4t.diagnostic.reporting.base import ReportFactory, ReportFormat, ReportGenerator
+
+# Import renderers to trigger registration
+from ml4t.diagnostic.reporting.html_renderer import HTMLReportGenerator
+from ml4t.diagnostic.reporting.json_renderer import JSONReportGenerator
+from ml4t.diagnostic.reporting.markdown_renderer import MarkdownReportGenerator
+
+__all__ = [
+    # Factory and base
+    "ReportFactory",
+    "ReportFormat",
+    "ReportGenerator",
+    # Renderers
+    "HTMLReportGenerator",
+    "JSONReportGenerator",
+    "MarkdownReportGenerator",
+]