aponyx 0.1.18__py3-none-any.whl

aponyx/evaluation/performance/report.py

@@ -0,0 +1,541 @@
+"""
+Markdown report generation for performance evaluation results.
+
+Generates human-readable reports with performance metrics, attribution,
+stability analysis, and interpretation guidance.
+"""
+
+import logging
+from datetime import datetime
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import pandas as pd
+import quantstats as qs
+
+from .config import PerformanceResult
+
+logger = logging.getLogger(__name__)
+
+
+def generate_quantstats_tearsheet(
+    returns: pd.Series,
+    benchmark: pd.Series | None,
+    output_path: Path,
+    title: str,
+    periods_per_year: int = 252,
+) -> Path | None:
+    """
+    Generate quantstats HTML tearsheet from returns series.
+
+    Parameters
+    ----------
+    returns : pd.Series
+        Daily percentage returns with DatetimeIndex.
+    benchmark : pd.Series | None
+        Benchmark daily percentage returns with DatetimeIndex.
+        If None, generates tearsheet without benchmark comparison.
+    output_path : Path
+        Full path (including filename) for HTML tearsheet.
+    title : str
+        Title for the tearsheet report.
+    periods_per_year : int, default=252
+        Number of trading periods per year (252 for daily).
+
+    Returns
+    -------
+    Path | None
+        Path to saved tearsheet HTML file.
+        Returns None if quantstats unavailable or generation fails.
+
+    Notes
+    -----
+    - Requires quantstats library (install with viz extras: pip install -e ".[viz]")
+    - Fallback behavior: logs warning at DEBUG level and returns None
+    - Uses quantstats default tearsheet template with all sections
+    - Output directory created automatically if missing
+    - Existing files overwritten without warning
+
+    Examples
+    --------
+    >>> tearsheet_path = generate_quantstats_tearsheet(
+    ...     returns=strategy_returns,
+    ...     benchmark=None,
+    ...     output_path=Path("reports/performance/strategy_tearsheet.html"),
+    ...     title="CDX ETF Basis - Simple Threshold",
+    ... )
+    """
+    try:
+        # Ensure output directory exists
+        output_path = Path(output_path)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Ensure series have names (quantstats requires this)
+        if returns.name is None:
+            returns = returns.copy()
+            returns.name = "Strategy"
+        if benchmark is not None and benchmark.name is None:
+            benchmark = benchmark.copy()
+            benchmark.name = "Benchmark"
+
+        # Generate tearsheet
+        logger.info(
+            "Generating quantstats tearsheet: %s (returns=%d, benchmark=%s)",
+            output_path.name,
+            len(returns),
+            "provided" if benchmark is not None else "none",
+        )
+
+        qs.reports.html(
+            returns,
+            benchmark=benchmark,
+            output=str(output_path),
+            title=title,
+            periods_per_year=periods_per_year,
+        )
+
+        # Clean up matplotlib figures to avoid memory issues
+        plt.close("all")
+
+        logger.info("Saved quantstats tearsheet to %s", output_path)
+        return output_path
+
+    except Exception as e:
+        logger.warning(
+            "Failed to generate quantstats tearsheet: %s: %s",
+            type(e).__name__,
+            e,
+        )
+        return None
+
+
+def generate_performance_report(
+    result: PerformanceResult,
+    signal_id: str,
+    strategy_id: str,
+    generate_tearsheet: bool = True,
+    output_dir: Path | None = None,
+) -> str:
+    """
+    Generate Markdown report from performance evaluation result.
+
+    Parameters
+    ----------
+    result : PerformanceResult
+        Performance evaluation result to document.
+    signal_id : str
+        Signal identifier (for header).
+    strategy_id : str
+        Strategy identifier (for header).
+    generate_tearsheet : bool, default=True
+        If True and quantstats available, generate HTML tearsheet alongside report.
+    output_dir : Path | None, default=None
+        Output directory for tearsheet. Required if generate_tearsheet=True.
+        Ignored if generate_tearsheet=False.
+
+    Returns
+    -------
+    str
+        Formatted Markdown report.
+
+    Notes
+    -----
+    Report includes:
+    - Header with identifiers and stability score
+    - Executive summary with key insights
+    - Extended metrics section
+    - Subperiod stability analysis
+    - Return attribution breakdown
+    - Recommendations
+    - Footer with metadata
+
+    Tearsheet generation:
+    - Only occurs if generate_tearsheet=True AND quantstats is installed
+    - Uses returns from result.metadata['returns'] if available
+    - Uses benchmark from result.metadata['benchmark'] if available
+    - Silently skips if dependencies unavailable (DEBUG-level logging)
+    - Filename format: {signal_id}_{strategy_id}_{timestamp}_tearsheet.html
+
+    Examples
+    --------
+    >>> report = generate_performance_report(result, "cdx_etf_basis", "simple_threshold")
+    >>> print(report[:100])
+    """
+    # Stability indicator
+    if result.stability_score >= 0.7:
+        stability_indicator = "[STRONG]"
+    elif result.stability_score >= 0.4:
+        stability_indicator = "[MODERATE]"
+    else:
+        stability_indicator = "[WEAK]"
+
+    # Extract key metrics (use dataclass field access)
+    metrics = result.metrics
+    subperiod = result.subperiod_analysis
+    attribution = result.attribution
+
+    # Build report
+    report = f"""# Backtest Performance Evaluation Report
+
+**Signal:** `{signal_id}`  
+**Strategy:** `{strategy_id}`  
+**Evaluation Date:** {result.timestamp}  
+**Evaluator Version:** {result.metadata.get("evaluator_version", "unknown")}
+
+---
+
+## Executive Summary
+
+### Stability Assessment: {stability_indicator}
+
+**Overall Stability Score:** {result.stability_score:.3f}
+
+{result.summary}
+
+---
+
+## Basic Backtest Metrics
+
+| Metric | Value |
+|--------|-------|
+| Total Return | ${metrics.total_return:,.2f} |
+| Annualized Return | ${metrics.annualized_return:,.2f} |
+| Sharpe Ratio | {metrics.sharpe_ratio:.3f} |
+| Sortino Ratio | {metrics.sortino_ratio:.3f} |
+| Max Drawdown | ${metrics.max_drawdown:,.2f} |
+| Calmar Ratio | {metrics.calmar_ratio:.3f} |
+| Annualized Volatility | ${metrics.annualized_volatility:,.2f} |
+
+### Trade Statistics
+
+| Metric | Value |
+|--------|-------|
+| Total Trades | {metrics.n_trades} |
+| Hit Rate | {metrics.hit_rate:.1%} |
+| Average Win | ${metrics.avg_win:,.2f} |
+| Average Loss | ${metrics.avg_loss:,.2f} |
+| Win/Loss Ratio | {metrics.win_loss_ratio:.3f} |
+| Avg Holding Days | {metrics.avg_holding_days:.1f} |
+
+---
+
+## Extended Performance Metrics
+
+### Risk-Adjusted Returns
+
+| Metric | Value |
+|--------|-------|
+| Rolling Sharpe (Mean) | {metrics.rolling_sharpe_mean:.3f} |
+| Rolling Sharpe (Std Dev) | {metrics.rolling_sharpe_std:.3f} |
+| Profit Factor | {metrics.profit_factor:.3f} |
+| Tail Ratio (95th pct) | {metrics.tail_ratio:.3f} |
+| Consistency Score (21d) | {metrics.consistency_score:.1%} |
+
+**Interpretation:**
+
+"""
+
+    # Add metric interpretations
+    if metrics.profit_factor > 1.5:
+        report += "- Strong profitability with gross wins substantially exceeding gross losses\n"
+    elif metrics.profit_factor > 1.0:
+        report += "- Positive profitability with gross wins exceeding gross losses\n"
+    else:
+        report += "- Weak profitability with gross losses approaching or exceeding gross wins\n"
+
+    if metrics.tail_ratio > 1.2:
+        report += (
+            "- Favorable tail asymmetry with larger upside than downside extremes\n"
+        )
+    elif metrics.tail_ratio > 0.8:
+        report += (
+            "- Balanced tail distribution with similar upside and downside extremes\n"
+        )
+    else:
+        report += (
+            "- Negative tail asymmetry with larger downside than upside extremes\n"
+        )
+
+    if metrics.consistency_score > 0.6:
+        report += "- High consistency with majority of rolling windows profitable\n"
+    elif metrics.consistency_score > 0.4:
+        report += "- Moderate consistency with mixed profitable/unprofitable periods\n"
+    else:
+        report += "- Low consistency with frequent unprofitable rolling windows\n"
+
+    # Drawdown recovery
+    max_dd_recovery = metrics.max_dd_recovery_days
+    if max_dd_recovery == float("inf"):
+        recovery_text = "Not recovered"
+    else:
+        recovery_text = f"{max_dd_recovery:.0f} days"
+
+    report += f"""
+### Drawdown Recovery
+
+| Metric | Value |
+|--------|-------|
+| Max Drawdown Recovery | {recovery_text} |
+| Average Recovery Time | {metrics.avg_recovery_days:.1f} days |
+| Number of Drawdowns | {metrics.n_drawdowns} |
+
+"""
+
+    if max_dd_recovery == float("inf"):
+        report += "**Warning:** Maximum drawdown has not been recovered as of backtest end date.\n"
+
+    # Subperiod stability
+    report += f"""
+---
+
+## Subperiod Stability Analysis
+
+**Number of Subperiods:** {len(subperiod["subperiod_returns"])}  
+**Profitable Periods:** {subperiod["positive_periods"]}/{len(subperiod["subperiod_returns"])}  
+**Consistency Rate:** {subperiod["consistency_rate"]:.1%}
+
+| Period | Return | Sharpe |
+|--------|--------|--------|
+"""
+
+    for i, (ret, sharpe) in enumerate(
+        zip(subperiod["subperiod_returns"], subperiod["subperiod_sharpes"]), 1
+    ):
+        report += f"| {i} | {ret:,.2f} | {sharpe:.3f} |\n"
+
+    report += "\n**Interpretation:**\n\n"
+
+    if subperiod["consistency_rate"] >= 0.75:
+        report += (
+            "Excellent temporal consistency with strong performance across most subperiods. "
+            "Strategy appears robust to different market conditions.\n"
+        )
+    elif subperiod["consistency_rate"] >= 0.5:
+        report += (
+            "Moderate temporal consistency with mixed performance across subperiods. "
+            "Performance may be regime-dependent.\n"
+        )
+    else:
+        report += (
+            "Weak temporal consistency with performance concentrated in few subperiods. "
+            "Strategy may be vulnerable to regime changes or overfitted to specific conditions.\n"
+        )
+
+    # Return attribution
+    direction = attribution["direction"]
+    signal_strength = attribution["signal_strength"]
+    win_loss = attribution["win_loss"]
+
+    report += f"""
+---
+
+## Return Attribution
+
+### Directional Attribution
+
+| Direction | P&L | Contribution |
+|-----------|-----|--------------|
+| Long | {direction["long_pnl"]:,.2f} | {direction["long_pct"]:.1%} |
+| Short | {direction["short_pnl"]:,.2f} | {direction["short_pct"]:.1%} |
+
+"""
+
+    if abs(direction["long_pct"]) > 0.7:
+        bias = "long" if direction["long_pct"] > 0 else "short"
+        report += f"**Strong {bias} bias** - Returns highly concentrated in {bias} positions.\n"
+    else:
+        report += "**Balanced exposure** - Returns distributed across both long and short positions.\n"
+
+    report += "\n### Signal Strength Attribution\n\n"
+    report += "| Quantile | P&L | Contribution |\n"
+    report += "|----------|-----|--------------|\n"
+
+    n_quantiles = result.config.attribution_quantiles
+    for i in range(1, n_quantiles + 1):
+        pnl = signal_strength[f"q{i}_pnl"]
+        pct = signal_strength[f"q{i}_pct"]
+        report += f"| Q{i} | {pnl:,.2f} | {pct:.1%} |\n"
+
+    report += "\n"
+
+    # Check if highest quantile contributed most
+    highest_q_pct = signal_strength[f"q{n_quantiles}_pct"]
+    if highest_q_pct > 0.4:
+        report += (
+            "**Strong signal strength relationship** - Highest conviction signals contributed "
+            f"most to returns ({highest_q_pct:.1%}).\n"
+        )
+    elif highest_q_pct < 0.2:
+        report += (
+            "**Weak signal strength relationship** - Returns not concentrated in highest "
+            "conviction signals. Signal strength may not add value.\n"
+        )
+    else:
+        report += "**Moderate signal strength relationship** - Mixed contribution across signal strengths.\n"
+
+    report += f"""
+### Win/Loss Decomposition
+
+| Category | Amount | Contribution |
+|----------|--------|--------------|
+| Gross Wins | {win_loss["gross_wins"]:,.2f} | {win_loss["win_contribution"]:.1%} |
+| Gross Losses | {win_loss["gross_losses"]:,.2f} | {win_loss["loss_contribution"]:.1%} |
+| Net P&L | {win_loss["net_pnl"]:,.2f} | — |
+
+---
+
+## Recommendations
+
+"""
+
+    # Generate recommendations based on metrics
+    recommendations = []
+
+    if result.stability_score < 0.5:
+        recommendations.append(
+            "[WARNING] **Low stability score** - Review strategy robustness and consider regime filters"
+        )
+
+    if metrics.profit_factor < 1.0:
+        recommendations.append(
+            "[FAIL] **Negative profit factor** - Strategy is unprofitable; do not deploy"
+        )
+
+    if subperiod["consistency_rate"] < 0.5:
+        recommendations.append(
+            "[WARNING] **Low consistency** - Performance concentrated in few periods; assess regime dependency"
+        )
+
+    if max_dd_recovery == float("inf"):
+        recommendations.append(
+            "[WARNING] **Unrecovered drawdown** - Current strategy underwater; reassess viability"
+        )
+
+    if metrics.tail_ratio < 0.8:
+        recommendations.append(
+            "[WARNING] **Negative skew** - Downside risk exceeds upside potential; review risk controls"
+        )
+
+    if not recommendations:
+        recommendations.append(
+            "[PASS] **Performance acceptable** - Consider proceeding with further validation and stress testing"
+        )
+        recommendations.append(
+            "Next steps: comparative analysis against alternative signals/strategies"
+        )
+        recommendations.append(
+            "Recommended: transaction cost sensitivity analysis and regime-conditional performance review"
+        )
+
+    for rec in recommendations:
+        report += f"{rec}\n\n"
+
+    report += f"""
+---
+
+## Report Metadata
+
+**Generated:** {datetime.now().isoformat()}  
+**Evaluator:** aponyx.evaluation.performance  
+**Configuration:**
+- Minimum Observations: {result.config.min_obs}
+- Subperiods: {result.config.n_subperiods}
+- Rolling Window: {result.config.rolling_window} days
+- Attribution Quantiles: {result.config.attribution_quantiles}
+
+**Reproducibility:** All metrics computed from backtest P&L with deterministic methods.
+
+---
+
+*This report was auto-generated from performance evaluation results.*
+"""
+
+    logger.debug(
+        "Generated performance report for %s/%s: %d characters",
+        signal_id,
+        strategy_id,
+        len(report),
+    )
+
+    # Generate quantstats tearsheet if requested
+    if generate_tearsheet and output_dir is not None:
+        returns = result.metadata.get("returns")
+        benchmark = result.metadata.get("benchmark")
+
+        if returns is not None:
+            timestamp_str = datetime.now().strftime("%Y%m%d_%H%M%S")
+            tearsheet_filename = (
+                f"{signal_id}_{strategy_id}_{timestamp_str}_tearsheet.html"
+            )
+            tearsheet_path = Path(output_dir) / tearsheet_filename
+            title = f"{signal_id} - {strategy_id}"
+
+            generate_quantstats_tearsheet(
+                returns=returns,
+                benchmark=benchmark,
+                output_path=tearsheet_path,
+                title=title,
+            )
+        else:
+            logger.debug(
+                "Skipping tearsheet generation - returns not found in result.metadata"
+            )
+
+    return report
+
+
+def save_report(
+    report: str,
+    signal_id: str,
+    strategy_id: str,
+    output_dir: Path,
+    timestamp: str | None = None,
+) -> Path:
+    """
+    Save report to Markdown file.
+
+    Parameters
+    ----------
+    report : str
+        Markdown report text.
+    signal_id : str
+        Signal identifier (for filename).
+    strategy_id : str
+        Strategy identifier (for filename).
+    output_dir : Path
+        Directory to save report.
+    timestamp : str or None, optional
+        Timestamp string (YYYYMMDD_HHMMSS). If None, generates new timestamp.
+
+    Returns
+    -------
+    Path
+        Path to saved report file.
+
+    Notes
+    -----
+    Filename format: performance_analysis_{YYYYMMDD_HHMMSS}.md
+    Creates output directory if it doesn't exist.
+
+    Examples
+    --------
+    >>> from aponyx.config import PERFORMANCE_REPORTS_DIR
+    >>> path = save_report(report, "cdx_etf_basis", "simple_threshold", PERFORMANCE_REPORTS_DIR)
+    >>> print(path)
+    """
+    # Create output directory
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Generate or use provided timestamp
+    if timestamp is None:
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = f"performance_analysis_{timestamp}.md"
+    output_path = output_dir / filename
+
+    # Write report
+    output_path.write_text(report, encoding="utf-8")
+
+    logger.info("Saved performance report to %s", output_path)
+
+    return output_path

aponyx/evaluation/suitability/__init__.py

@@ -0,0 +1,67 @@
+"""
+Signal-product suitability evaluation module.
+
+This module provides tools to evaluate whether signals contain economically
+and statistically meaningful information for traded products. Acts as a
+research screening gate before proceeding to strategy design and backtesting.
+
+Public API
+----------
+- evaluate_signal_suitability: Main evaluation function
+- SuitabilityConfig: Configuration dataclass
+- SuitabilityResult: Result dataclass
+- SuitabilityRegistry: Evaluation tracking with CRUD operations
+- EvaluationEntry: Registry entry dataclass
+- generate_suitability_report: Markdown report generator
+- save_report: Report file persistence
+- compute_forward_returns: Target construction utility
+
+Examples
+--------
+>>> from aponyx.evaluation.suitability import (
+...     evaluate_signal_suitability,
+...     SuitabilityConfig,
+...     generate_suitability_report,
+...     save_report,
+...     SuitabilityRegistry,
+... )
+>>> from aponyx.config import EVALUATION_DIR, SUITABILITY_REGISTRY_PATH
+>>>
+>>> # Evaluate signal
+>>> result = evaluate_signal_suitability(signal, target_change)
+>>> print(result.decision, result.composite_score)
+>>>
+>>> # Generate report
+>>> report = generate_suitability_report(result, "cdx_etf_basis", "cdx_ig_5y")
+>>> report_path = save_report(report, "cdx_etf_basis", "cdx_ig_5y", EVALUATION_DIR)
+>>>
+>>> # Register evaluation
+>>> registry = SuitabilityRegistry(SUITABILITY_REGISTRY_PATH)
+>>> eval_id = registry.register_evaluation(result, "cdx_etf_basis", "cdx_ig_5y")
+"""
+
+from aponyx.evaluation.suitability.config import SuitabilityConfig
+from aponyx.evaluation.suitability.evaluator import (
+    SuitabilityResult,
+    evaluate_signal_suitability,
+    compute_forward_returns,
+)
+from aponyx.evaluation.suitability.registry import (
+    SuitabilityRegistry,
+    EvaluationEntry,
+)
+from aponyx.evaluation.suitability.report import (
+    generate_suitability_report,
+    save_report,
+)
+
+__all__ = [
+    "SuitabilityConfig",
+    "SuitabilityResult",
+    "evaluate_signal_suitability",
+    "compute_forward_returns",
+    "SuitabilityRegistry",
+    "EvaluationEntry",
+    "generate_suitability_report",
+    "save_report",
+]