aponyx 0.1.18__py3-none-any.whl

aponyx/workflows/config.py

@@ -0,0 +1,122 @@
+"""
+Workflow configuration management.
+
+Defines immutable configuration for workflow execution including
+signal/strategy selection, data sources, and execution options.
+"""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+
+from aponyx.config import DATA_WORKFLOWS_DIR
+
+StepName = Literal[
+    "data",
+    "signal",
+    "suitability",
+    "backtest",
+    "performance",
+    "visualization",
+]
+
+# DataSource now accepts any string to support dynamic source discovery
+DataSource = str
+
+
+@dataclass(frozen=True)
+class WorkflowConfig:
+    """
+    Immutable workflow execution configuration.
+
+    Attributes
+    ----------
+    label : str
+        Workflow label (lowercase, underscores only, pattern: ^[a-z][a-z0-9_]*$).
+        Used for workflow identification and directory naming.
+    signal_name : str
+        Signal name from signal catalog.
+    strategy_name : str
+        Strategy name from strategy catalog.
+    product : str
+        Product identifier for backtesting (e.g., "cdx_ig_5y", "cdx_hy_5y").
+    data_source : str
+        Data source type (e.g., "synthetic", "file", "bloomberg", or custom sources).
+    security_mapping : dict[str, str] | None
+        Maps generic instrument types to specific securities.
+        Example: {"cdx": "cdx_ig_5y", "etf": "hyg", "vix": "vix"}
+        If None, uses defaults from indicator catalog.
+    indicator_transformation_override : str | None
+        Override indicator transformation from catalog (must exist in indicator_transformation.json).
+        If None, uses indicator_transformation from signal catalog.
+        Example: "spread_momentum_5d" to swap indicator while keeping score/signal transformations.
+    score_transformation_override : str | None
+        Override score transformation from catalog (must exist in score_transformation.json).
+        If None, uses score_transformation from signal catalog.
+        Example: "z_score_60d" to swap normalization window while keeping indicator/signal transformations.
+    signal_transformation_override : str | None
+        Override signal transformation from catalog (must exist in signal_transformation.json).
+        If None, uses signal_transformation from signal catalog.
+        Example: "bounded_2_0" to swap trading rules while keeping indicator/score transformations.
+    steps : list[StepName] | None
+        Specific steps to execute (None = all steps in order).
+    force_rerun : bool
+        Force re-execution even if cached outputs exist.
+    output_dir : Path
+        Base directory for workflow outputs.
+
+    Notes
+    -----
+    Configuration is frozen to prevent accidental mutation during execution.
+    Use dataclasses.replace() to create modified copies if needed.
+
+    Four-Stage Transformation Pipeline
+    -----------------------------------
+    Security → Indicator → Score → Signal → Position
+
+    Each signal references exactly one transformation from each stage (1:1:1 relationship).
+
+    Runtime overrides allow swapping components at any stage without editing catalogs:
+    - security_mapping: Override which securities to load for each instrument type
+    - indicator_transformation_override: Swap indicator while keeping score/signal transformations
+    - score_transformation_override: Swap normalization while keeping indicator/signal transformations
+    - signal_transformation_override: Swap trading rules while keeping indicator/score transformations
+    """
+
+    label: str
+    signal_name: str
+    strategy_name: str
+    product: str
+    data_source: DataSource = "synthetic"
+    security_mapping: dict[str, str] | None = None
+    indicator_transformation_override: str | None = None
+    score_transformation_override: str | None = None
+    signal_transformation_override: str | None = None
+    steps: list[StepName] | None = None
+    force_rerun: bool = False
+    output_dir: Path = field(default_factory=lambda: DATA_WORKFLOWS_DIR)
+
+    def __post_init__(self) -> None:
+        """Validate configuration on initialization."""
+        import re
+
+        # Validate label format
+        if not re.match(r"^[a-z][a-z0-9_]*$", self.label):
+            raise ValueError(
+                f"Label '{self.label}' is invalid. "
+                "Must start with lowercase letter and contain only lowercase letters, numbers, and underscores."
+            )
+
+        # Validate steps
+        if self.steps is not None:
+            valid_steps = {
+                "data",
+                "signal",
+                "suitability",
+                "backtest",
+                "performance",
+                "visualization",
+            }
+            invalid = set(self.steps) - valid_steps
+            if invalid:
+                raise ValueError(f"Invalid steps: {invalid}")

aponyx/workflows/engine.py

@@ -0,0 +1,279 @@
+"""
+Workflow orchestration engine.
+
+Coordinates sequential execution of workflow steps with dependency tracking,
+caching, error handling, and progress logging.
+"""
+
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from .config import WorkflowConfig
+from .steps import WorkflowStep
+from .registry import StepRegistry
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowEngine:
+    """
+    Workflow execution orchestrator.
+
+    Manages sequential pipeline execution with:
+    - Dependency resolution (data → signal → backtest → ...)
+    - Smart caching (skip completed steps)
+    - Error handling (save partial results)
+    - Progress tracking (structured logging)
+
+    Parameters
+    ----------
+    config : WorkflowConfig
+        Workflow execution configuration.
+
+    Examples
+    --------
+    Execute full workflow:
+        >>> config = WorkflowConfig(
+        ...     signal_name="spread_momentum",
+        ...     strategy_name="balanced",
+        ... )
+        >>> engine = WorkflowEngine(config)
+        >>> results = engine.execute()
+
+    Execute specific steps:
+        >>> config = WorkflowConfig(
+        ...     signal_name="spread_momentum",
+        ...     strategy_name="balanced",
+        ...     steps=["data", "signal", "backtest"],
+        ... )
+        >>> engine = WorkflowEngine(config)
+        >>> results = engine.execute()
+    """
+
+    def __init__(self, config: WorkflowConfig) -> None:
+        self.config = config
+        self._registry = StepRegistry()
+        self._steps = self._resolve_steps()
+        self._context: dict[str, Any] = {}
+        self._start_time: datetime | None = None
+
+    def execute(self) -> dict[str, Any]:
+        """
+        Execute workflow pipeline.
+
+        Returns
+        -------
+        dict[str, Any]
+            Workflow results with keys:
+            - steps_completed: int (number of steps executed)
+            - steps_skipped: int (number cached steps skipped)
+            - output_dir: Path (workflow output directory)
+            - duration_seconds: float (total execution time)
+            - errors: list[dict] (errors if any step failed)
+
+        Notes
+        -----
+        Steps execute in dependency order. If step N fails, steps N+1...
+        are skipped but results from steps 1...N-1 are preserved.
+        """
+        self._start_time = datetime.now()
+
+        logger.info(
+            "Starting workflow: signal=%s, strategy=%s, source=%s, steps=%d",
+            self.config.signal_name,
+            self.config.strategy_name,
+            self.config.data_source,
+            len(self._steps),
+        )
+
+        # Create workflow output directory upfront
+        output_dir = self._create_output_directory()
+
+        # Add output_dir to context for steps to use
+        self._context["output_dir"] = output_dir
+
+        completed = 0
+        skipped = 0
+        errors = []
+
+        for idx, step in enumerate(self._steps, start=1):
+            step_num = f"{idx}/{len(self._steps)}"
+
+            # Check cache
+            if self._should_skip_step(step):
+                logger.info("Step %s: %s (cached)", step_num, step.name)
+                # Load cached output into context for downstream steps
+                try:
+                    cached_output = step.load_cached_output()
+                    self._context[step.name] = cached_output
+                except Exception as e:
+                    logger.warning(
+                        "Failed to load cached output for %s: %s. Re-running step.",
+                        step.name,
+                        str(e),
+                    )
+                    # Fall through to execute step instead
+                else:
+                    skipped += 1
+                    continue
+
+            # Execute step
+            try:
+                logger.info("Step %s: %s", step_num, step.name)
+                output = step.execute(self._context)
+                self._context[step.name] = output
+                completed += 1
+                logger.info("Step %s: %s complete", step_num, step.name)
+
+            except Exception as e:
+                logger.error("Step %s: %s failed - %s", step_num, step.name, str(e))
+                errors.append(
+                    {
+                        "step": step.name,
+                        "error": str(e),
+                        "type": type(e).__name__,
+                    }
+                )
+                break  # Stop execution on first error
+
+        duration = (datetime.now() - self._start_time).total_seconds()
+
+        result = {
+            "steps_completed": completed,
+            "steps_skipped": skipped,
+            "output_dir": output_dir,
+            "duration_seconds": duration,
+            "errors": errors,
+        }
+
+        # Save workflow metadata
+        self._save_metadata(output_dir, completed, skipped, errors, duration)
+
+        if errors:
+            logger.error(
+                "Workflow failed: completed=%d, skipped=%d, failed=%d (%.1fs)",
+                completed,
+                skipped,
+                len(errors),
+                duration,
+            )
+        else:
+            logger.info(
+                "Workflow complete: completed=%d, skipped=%d (%.1fs)",
+                completed,
+                skipped,
+                duration,
+            )
+
+        return result
+
+    def _resolve_steps(self) -> list[WorkflowStep]:
+        """
+        Resolve workflow steps from configuration.
+
+        Returns
+        -------
+        list[WorkflowStep]
+            Ordered list of step instances to execute.
+
+        Notes
+        -----
+        If config.steps is None, returns all steps in dependency order.
+        If config.steps is specified, returns subset in correct order.
+        """
+        all_steps = self._registry.get_all_steps(self.config)
+
+        if self.config.steps is None:
+            return all_steps
+
+        # Filter to requested steps (maintain order)
+        requested = set(self.config.steps)
+        return [s for s in all_steps if s.name in requested]
+
+    def _should_skip_step(self, step: WorkflowStep) -> bool:
+        """
+        Determine if step should be skipped (cached).
+
+        Parameters
+        ----------
+        step : WorkflowStep
+            Step to check.
+
+        Returns
+        -------
+        bool
+            True if step output exists and force_rerun is False.
+        """
+        if self.config.force_rerun:
+            return False
+        return step.output_exists()
+
+    def _create_output_directory(self) -> Path:
+        """
+        Create timestamped output directory for workflow.
+
+        Returns
+        -------
+        Path
+            Created output directory path.
+
+        Notes
+        -----
+        Format: workflows/{label}_{timestamp}/
+        """
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        dirname = f"{self.config.label}_{timestamp}"
+        output_dir = self.config.output_dir / dirname
+        output_dir.mkdir(parents=True, exist_ok=True)
+        return output_dir
+
+    def _save_metadata(
+        self,
+        output_dir: Path,
+        completed: int,
+        skipped: int,
+        errors: list[dict[str, Any]],
+        duration: float,
+    ) -> None:
+        """
+        Save workflow metadata to metadata.json.
+
+        Parameters
+        ----------
+        output_dir : Path
+            Workflow output directory.
+        completed : int
+            Number of completed steps.
+        skipped : int
+            Number of skipped steps.
+        errors : list of dict
+            Error details if any.
+        duration : float
+            Execution duration in seconds.
+        """
+        from ..persistence import save_json
+
+        # Extract securities_used from signal step if available
+        securities_used = self._context.get("signal", {}).get("securities_used", {})
+
+        metadata = {
+            "label": self.config.label,
+            "signal": self.config.signal_name,
+            "strategy": self.config.strategy_name,
+            "product": self.config.product,
+            "data_source": self.config.data_source,
+            "securities_used": securities_used,
+            "timestamp": self._start_time.isoformat() if self._start_time else None,
+            "duration_seconds": duration,
+            "steps_completed": completed,
+            "steps_skipped": skipped,
+            "steps_total": len(self._steps),
+            "status": "failed" if errors else "completed",
+            "errors": errors if errors else None,
+        }
+
+        metadata_path = output_dir / "metadata.json"
+        save_json(metadata, metadata_path)
+        logger.debug("Saved workflow metadata: %s", metadata_path)

aponyx/workflows/registry.py

@@ -0,0 +1,116 @@
+"""
+Workflow step registry.
+
+Central factory for creating workflow step instances.
+Decouples engine from concrete step implementations.
+"""
+
+import logging
+from typing import TYPE_CHECKING
+
+from .config import WorkflowConfig
+from .concrete_steps import (
+    DataStep,
+    SignalStep,
+    SuitabilityStep,
+    BacktestStep,
+    PerformanceStep,
+    VisualizationStep,
+)
+
+if TYPE_CHECKING:
+    from .steps import WorkflowStep
+
+logger = logging.getLogger(__name__)
+
+
+class StepRegistry:
+    """
+    Factory for workflow step instances.
+
+    Centralizes step creation and ensures consistent dependency order.
+
+    Examples
+    --------
+    Get all steps for workflow:
+        >>> registry = StepRegistry()
+        >>> config = WorkflowConfig(signal_name="spread_momentum", strategy_name="balanced")
+        >>> steps = registry.get_all_steps(config)
+
+    Get specific step:
+        >>> step = registry.get_step("data", config)
+    """
+
+    def __init__(self) -> None:
+        self._step_order = [
+            "data",
+            "signal",
+            "suitability",
+            "backtest",
+            "performance",
+            "visualization",
+        ]
+
+    def get_canonical_order(self) -> list[str]:
+        """
+        Get canonical workflow step order.
+
+        Returns
+        -------
+        list[str]
+            Ordered list of step names.
+        """
+        return self._step_order.copy()
+
+    def get_all_steps(self, config: WorkflowConfig) -> list["WorkflowStep"]:
+        """
+        Create all workflow steps in dependency order.
+
+        Parameters
+        ----------
+        config : WorkflowConfig
+            Workflow configuration.
+
+        Returns
+        -------
+        list[WorkflowStep]
+            Ordered list of step instances.
+        """
+        return [self._create_step(name, config) for name in self._step_order]
+
+    def get_step(self, name: str, config: WorkflowConfig) -> "WorkflowStep":
+        """
+        Create single workflow step by name.
+
+        Parameters
+        ----------
+        name : str
+            Step name (data, signal, suitability, backtest, performance, visualization).
+        config : WorkflowConfig
+            Workflow configuration.
+
+        Returns
+        -------
+        WorkflowStep
+            Step instance.
+
+        Raises
+        ------
+        ValueError
+            If step name is invalid.
+        """
+        if name not in self._step_order:
+            raise ValueError(f"Unknown step: {name}")
+        return self._create_step(name, config)
+
+    def _create_step(self, name: str, config: WorkflowConfig) -> "WorkflowStep":
+        """Create step instance by name."""
+        step_classes = {
+            "data": DataStep,
+            "signal": SignalStep,
+            "suitability": SuitabilityStep,
+            "backtest": BacktestStep,
+            "performance": PerformanceStep,
+            "visualization": VisualizationStep,
+        }
+        return step_classes[name](config)

aponyx/workflows/steps.py

@@ -0,0 +1,180 @@
+"""
+Workflow step abstractions.
+
+Defines protocol for executable workflow steps with dependency tracking,
+caching, and standardized I/O.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Protocol
+
+from .config import WorkflowConfig
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowStep(Protocol):
+    """
+    Protocol for executable workflow steps.
+
+    All workflow steps must implement this interface for orchestration.
+
+    Attributes
+    ----------
+    name : str
+        Step identifier (used for caching and logging).
+    config : WorkflowConfig
+        Workflow configuration.
+
+    Methods
+    -------
+    execute(context)
+        Execute step logic and return output data.
+    output_exists()
+        Check if step output already exists (for caching).
+    get_output_path()
+        Return path to expected output files.
+    """
+
+    name: str
+    config: WorkflowConfig
+
+    def execute(self, context: dict[str, Any]) -> dict[str, Any]:
+        """
+        Execute workflow step.
+
+        Parameters
+        ----------
+        context : dict[str, Any]
+            Outputs from previous steps (keyed by step name).
+
+        Returns
+        -------
+        dict[str, Any]
+            Step output data to pass to subsequent steps.
+
+        Notes
+        -----
+        Steps should be idempotent: running twice produces same results.
+        Use context["data"] to access data from DataStep, etc.
+        """
+        ...
+
+    def output_exists(self) -> bool:
+        """
+        Check if step output files exist.
+
+        Returns
+        -------
+        bool
+            True if all required outputs exist, False otherwise.
+
+        Notes
+        -----
+        Used by caching logic to skip completed steps.
+        Should check file existence and basic validation.
+        """
+        ...
+
+    def get_output_path(self) -> Path:
+        """
+        Get expected output directory path.
+
+        Returns
+        -------
+        Path
+            Directory where step outputs are saved.
+        """
+        ...
+
+    def load_cached_output(self) -> dict[str, Any]:
+        """
+        Load cached output from previous execution.
+
+        Returns
+        -------
+        dict[str, Any]
+            Cached step output data.
+
+        Raises
+        ------
+        FileNotFoundError
+            If cached output files don't exist.
+        ValueError
+            If cached output is invalid or corrupted.
+
+        Notes
+        -----
+        Called when step is skipped due to caching.
+        Must restore same output structure as execute() would return.
+        """
+        ...
+
+
+class BaseWorkflowStep(ABC):
+    """
+    Abstract base class for workflow steps.
+
+    Provides common functionality for concrete step implementations.
+
+    Parameters
+    ----------
+    config : WorkflowConfig
+        Workflow configuration.
+    """
+
+    def __init__(self, config: WorkflowConfig) -> None:
+        self.config = config
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Step identifier."""
+        ...
+
+    @abstractmethod
+    def execute(self, context: dict[str, Any]) -> dict[str, Any]:
+        """Execute step logic."""
+        ...
+
+    @abstractmethod
+    def output_exists(self) -> bool:
+        """Check if output exists."""
+        ...
+
+    @abstractmethod
+    def get_output_path(self) -> Path:
+        """Get output directory."""
+        ...
+
+    def load_cached_output(self) -> dict[str, Any]:
+        """
+        Load cached output from previous execution.
+
+        Default implementation raises NotImplementedError.
+        Steps that support caching must override this method.
+
+        Returns
+        -------
+        dict[str, Any]
+            Cached step output data.
+
+        Raises
+        ------
+        NotImplementedError
+            If step doesn't support loading cached outputs.
+        """
+        raise NotImplementedError(
+            f"Step {self.name} doesn't support loading cached outputs. "
+            "Override load_cached_output() method."
+        )
+
+    def _log_start(self) -> None:
+        """Log step start."""
+        logger.info("Starting step: %s", self.name)
+
+    def _log_complete(self, output: dict[str, Any]) -> None:
+        """Log step completion."""
+        logger.info("Completed step: %s", self.name)