oscura 0.7.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

oscura/utils/pipeline/parallel.py

@@ -1,449 +0,0 @@
-"""Parallel pipeline execution with automatic dependency analysis.
-
-This module provides a parallel-capable Pipeline that analyzes stage dependencies
-and executes independent stages concurrently using thread or process pools.
-
-The ParallelPipeline maintains full API compatibility with the standard Pipeline
-while providing linear speedup for independent transformations.
-
-Example:
-    >>> from oscura.utils.pipeline import ParallelPipeline
-    >>> # Create pipeline with independent branches
-    >>> pipeline = ParallelPipeline([
-    ...     ('filter1', LowPassFilter(cutoff=1e6)),
-    ...     ('filter2', HighPassFilter(cutoff=1e5)),
-    ...     ('merge', MergeTransformer())
-    ... ])
-    >>> # Independent filters run in parallel
-    >>> result = pipeline.transform(trace)
-
-Performance:
-    For N independent stages, ParallelPipeline provides up to Nx speedup
-    compared to sequential execution. Overhead is ~10ms for thread pool,
-    ~50ms for process pool.
-
-References:
-    IEEE 1057-2017: Parallel processing for digitizer characterization
-    sklearn.pipeline.Pipeline: Sequential pipeline pattern
-"""
-
-from __future__ import annotations
-
-import os
-from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
-from typing import TYPE_CHECKING, Literal
-
-from .pipeline import Pipeline
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-    from oscura.core.types import WaveformTrace
-
-    from .base import TraceTransformer
-
-
-class ParallelPipeline(Pipeline):
-    """Pipeline with parallel execution of independent stages.
-
-    Analyzes dependencies between pipeline stages and executes independent
-    stages concurrently. Automatically determines dependency graph from
-    stage inputs/outputs.
-
-    Stages are independent if they:
-    1. Don't modify shared state
-    2. Only depend on the input trace (not other stage outputs)
-    3. Can be executed in any order
-
-    Attributes:
-        steps: List of (name, transformer) tuples defining the pipeline stages.
-        named_steps: Dictionary mapping step names to transformers.
-        executor_type: Type of executor ('thread' or 'process').
-        max_workers: Maximum number of parallel workers (None = auto).
-
-    Example - Independent Filters:
-        >>> # These filters are independent - they both take the input trace
-        >>> pipeline = ParallelPipeline([
-        ...     ('lowpass', LowPassFilter(cutoff=1e6)),
-        ...     ('highpass', HighPassFilter(cutoff=1e5)),
-        ...     ('bandpass', BandPassFilter(low=1e5, high=1e6)),
-        ... ], executor_type='thread', max_workers=3)
-        >>> result = pipeline.transform(trace)  # All 3 run in parallel
-
-    Example - Mixed Sequential and Parallel:
-        >>> # First stage sequential, then parallel analysis
-        >>> pipeline = ParallelPipeline([
-        ...     ('preprocess', Normalize()),      # Sequential
-        ...     ('fft', FFT()),                   # Parallel (from preprocessed)
-        ...     ('wavelet', WaveletTransform()),  # Parallel (from preprocessed)
-        ...     ('merge', CombineResults())       # Sequential (waits for fft+wavelet)
-        ... ])
-
-    Example - Process Pool for CPU-Intensive Tasks:
-        >>> # Use process pool for heavy computation
-        >>> pipeline = ParallelPipeline([
-        ...     ('fft1', FFT(nfft=65536)),
-        ...     ('fft2', FFT(nfft=32768)),
-        ...     ('fft3', FFT(nfft=16384)),
-        ... ], executor_type='process', max_workers=None)  # Auto worker count
-
-    Performance Characteristics:
-        Thread pool:
-            - Best for I/O-bound tasks (file loading, network)
-            - Low overhead (~10ms startup)
-            - Shared memory (no serialization)
-            - Limited by GIL for CPU-bound tasks
-
-        Process pool:
-            - Best for CPU-bound tasks (FFT, filtering, analysis)
-            - Higher overhead (~50ms startup)
-            - Requires picklable transformers
-            - True parallelism (bypasses GIL)
-
-    References:
-        concurrent.futures.ThreadPoolExecutor
-        concurrent.futures.ProcessPoolExecutor
-    """
-
-    def __init__(
-        self,
-        steps: Sequence[tuple[str, TraceTransformer]],
-        executor_type: Literal["thread", "process"] = "thread",
-        max_workers: int | None = None,
-    ) -> None:
-        """Initialize parallel pipeline with executor configuration.
-
-        Args:
-            steps: Sequence of (name, transformer) tuples. Each transformer
-                must be a TraceTransformer instance.
-            executor_type: Type of executor to use:
-                - 'thread': ThreadPoolExecutor (default, low overhead)
-                - 'process': ProcessPoolExecutor (true parallelism, higher overhead)
-            max_workers: Maximum number of parallel workers. If None, uses
-                automatic selection:
-                - Thread pool: min(32, num_cpu + 4)
-                - Process pool: num_cpu
-
-        Raises:
-            ValueError: If step names are not unique, empty, or executor_type invalid.
-
-        Example:
-            >>> # Auto worker count (recommended)
-            >>> pipeline = ParallelPipeline([
-            ...     ('stage1', Transformer1()),
-            ...     ('stage2', Transformer2())
-            ... ])
-            >>> # Explicit worker count
-            >>> pipeline = ParallelPipeline([
-            ...     ('stage1', Transformer1()),
-            ...     ('stage2', Transformer2())
-            ... ], max_workers=4)
-        """
-        # Initialize parent Pipeline
-        super().__init__(steps)
-
-        # Validate executor type
-        if executor_type not in ("thread", "process"):
-            raise ValueError(f"executor_type must be 'thread' or 'process', got '{executor_type}'")
-
-        self.executor_type = executor_type
-        self.max_workers = max_workers
-        self._dependency_graph: dict[str, list[str]] = {}
-        self._execution_order: list[list[str]] = []
-
-        # Analyze dependencies on initialization
-        self._analyze_dependencies()
-
-    def _analyze_dependencies(self) -> None:
-        """Analyze dependencies between pipeline stages.
-
-        Builds a dependency graph by examining which stages depend on outputs
-        from other stages. Currently uses a conservative approach:
-
-        - First stage has no dependencies
-        - All other stages depend on the previous stage (sequential)
-        - Future enhancement: analyze transformer inputs to detect true dependencies
-
-        This conservative approach ensures correctness while still allowing
-        parallel execution when stages are explicitly independent.
-
-        The dependency analysis produces an execution order as a list of
-        "generations" (lists of stage names). Stages in the same generation
-        can be executed in parallel.
-
-        Example:
-            For pipeline: [filter1, filter2, merge]
-            If all stages are sequential:
-                execution_order = [['filter1'], ['filter2'], ['merge']]
-            If filter1 and filter2 are independent:
-                execution_order = [['filter1', 'filter2'], ['merge']]
-
-        NOTE: Current implementation is conservative and assumes sequential
-        dependencies. Future versions will support dependency hints via
-        transformer metadata to enable automatic parallelization.
-
-        References:
-            FUTURE-002: Advanced dependency analysis
-        """
-        # Build dependency graph (conservative: each stage depends on previous)
-        self._dependency_graph = {}
-
-        for i, (name, _transformer) in enumerate(self.steps):
-            if i == 0:
-                # First stage has no dependencies
-                self._dependency_graph[name] = []
-            else:
-                # Each stage depends on the previous stage
-                prev_name = self.steps[i - 1][0]
-                self._dependency_graph[name] = [prev_name]
-
-        # Build execution order (topological sort by generations)
-        self._execution_order = self._compute_execution_order()
-
-    def _compute_execution_order(self) -> list[list[str]]:
-        """Compute execution order as list of parallel generations.
-
-        Uses topological sort to group stages into generations where each
-        generation contains stages that can execute in parallel.
-
-        Returns:
-            List of generations, where each generation is a list of stage names
-            that can execute in parallel.
-
-        Raises:
-            ValueError: If circular dependencies detected.
-
-        Example:
-            For graph: {A: [], B: [A], C: [A], D: [B, C]}
-            Returns: [[A], [B, C], [D]]
-        """
-        # Copy dependency graph (we'll modify it)
-        deps = {name: set(deps) for name, deps in self._dependency_graph.items()}
-        generations: list[list[str]] = []
-
-        # All stage names
-        all_stages = set(self._dependency_graph.keys())
-        completed: set[str] = set()
-
-        # Build generations
-        while completed != all_stages:
-            # Find all stages with no remaining dependencies
-            ready = [name for name in all_stages - completed if not deps[name]]
-
-            if not ready:
-                # Cycle detected (shouldn't happen with valid pipeline)
-                raise ValueError("Circular dependency detected in pipeline")
-
-            generations.append(ready)
-            completed.update(ready)
-
-            # Remove completed stages from dependencies
-            for name in all_stages - completed:
-                deps[name] -= set(ready)
-
-        return generations
-
-    def _get_max_workers(self) -> int:
-        """Get the maximum number of workers to use.
-
-        Returns:
-            Number of workers, using automatic selection if max_workers is None.
-
-        Example:
-            >>> pipeline._get_max_workers()
-            8  # On 8-core machine with thread executor
-        """
-        if self.max_workers is not None:
-            return self.max_workers
-
-        # Automatic worker count selection
-        cpu_count = os.cpu_count() or 4
-
-        if self.executor_type == "thread":
-            # Thread pool: min(32, cpu_count + 4) - default from ThreadPoolExecutor
-            return min(32, cpu_count + 4)
-        else:
-            # Process pool: cpu_count - default from ProcessPoolExecutor
-            return cpu_count
-
-    def transform(self, trace: WaveformTrace) -> WaveformTrace:
-        """Transform trace through pipeline with parallel execution.
-
-        Executes stages in parallel when possible, according to the dependency
-        graph. Stages in the same generation run concurrently.
-
-        Args:
-            trace: Input WaveformTrace to transform.
-
-        Returns:
-            Transformed WaveformTrace after passing through all stages.
-
-        Example:
-            >>> result = pipeline.transform(trace)
-
-        Performance:
-            For N independent stages with T execution time each:
-            - Sequential: N * T
-            - Parallel: T + overhead (~10-50ms)
-            - Speedup: ~Nx (minus overhead)
-        """
-        current = trace
-        self._intermediate_results.clear()
-
-        # Choose executor based on configuration
-        executor_class = (
-            ThreadPoolExecutor if self.executor_type == "thread" else ProcessPoolExecutor
-        )
-
-        with executor_class(max_workers=self._get_max_workers()) as executor:
-            # Execute each generation in parallel
-            for generation in self._execution_order:
-                if len(generation) == 1:
-                    # Single stage - execute directly (no overhead)
-                    name = generation[0]
-                    transformer = self.named_steps[name]
-                    current = transformer.transform(current)
-                    self._intermediate_results[name] = current
-                else:
-                    # Multiple stages - execute in parallel
-                    futures = {}
-                    for name in generation:
-                        transformer = self.named_steps[name]
-                        future = executor.submit(transformer.transform, current)
-                        futures[name] = future
-
-                    # Collect results
-                    results = {}
-                    for name, future in futures.items():
-                        result = future.result()
-                        results[name] = result
-                        self._intermediate_results[name] = result
-
-                    # For conservative sequential execution, current is the last result
-                    # For true parallel execution with merge, this would be handled differently
-                    current = results[generation[-1]]
-
-        return current
-
-    def fit(self, trace: WaveformTrace) -> ParallelPipeline:
-        """Fit all transformers in the pipeline.
-
-        Fits each transformer sequentially on the output of the previous stage.
-        Fitting is always sequential because it may modify transformer state.
-
-        Args:
-            trace: Reference WaveformTrace to fit to.
-
-        Returns:
-            Self for method chaining.
-
-        Example:
-            >>> pipeline = ParallelPipeline([
-            ...     ('normalize', AdaptiveNormalizer()),
-            ...     ('filter', AdaptiveFilter())
-            ... ])
-            >>> pipeline.fit(reference_trace)
-
-        NOTE: fit() is always sequential because transformers may have
-        interdependent learned parameters. Use transform() for parallel execution.
-        """
-        # Fitting is always sequential - reuse parent implementation
-        super().fit(trace)
-        return self
-
-    def clone(self) -> ParallelPipeline:
-        """Create a copy of this parallel pipeline.
-
-        Returns:
-            New ParallelPipeline instance with cloned transformers and same configuration.
-
-        Example:
-            >>> pipeline_copy = pipeline.clone()
-        """
-        cloned_steps = [(name, transformer.clone()) for name, transformer in self.steps]
-        return ParallelPipeline(
-            cloned_steps, executor_type=self.executor_type, max_workers=self.max_workers
-        )
-
-    def get_dependency_graph(self) -> dict[str, list[str]]:
-        """Get the dependency graph for pipeline stages.
-
-        Returns:
-            Dictionary mapping stage names to list of dependency stage names.
-
-        Example:
-            >>> graph = pipeline.get_dependency_graph()
-            >>> print(graph)
-            {'filter1': [], 'filter2': ['filter1'], 'merge': ['filter2']}
-
-        References:
-            API-006: Pipeline Dependency Analysis
-        """
-        return self._dependency_graph.copy()
-
-    def get_execution_order(self) -> list[list[str]]:
-        """Get the execution order as parallel generations.
-
-        Returns:
-            List of generations, where each generation is a list of stage names
-            that will execute in parallel.
-
-        Example:
-            >>> order = pipeline.get_execution_order()
-            >>> print(order)
-            [['filter1', 'filter2'], ['merge']]
-            >>> # filter1 and filter2 run in parallel, then merge
-
-        References:
-            API-006: Pipeline Dependency Analysis
-        """
-        return [gen.copy() for gen in self._execution_order]
-
-    def set_parallel_config(
-        self,
-        executor_type: Literal["thread", "process"] | None = None,
-        max_workers: int | None = None,
-    ) -> ParallelPipeline:
-        """Update parallel execution configuration.
-
-        Args:
-            executor_type: New executor type ('thread' or 'process'). If None, keeps current.
-            max_workers: New max worker count. If None, keeps current (may be auto).
-
-        Returns:
-            Self for method chaining.
-
-        Raises:
-            ValueError: If executor_type is invalid.
-
-        Example:
-            >>> # Switch to process pool with 4 workers
-            >>> pipeline.set_parallel_config(executor_type='process', max_workers=4)
-            >>> # Switch to auto worker count
-            >>> pipeline.set_parallel_config(max_workers=None)
-
-        References:
-            API-006: Pipeline Parallel Configuration
-        """
-        if executor_type is not None:
-            if executor_type not in ("thread", "process"):
-                raise ValueError(
-                    f"executor_type must be 'thread' or 'process', got '{executor_type}'"
-                )
-            self.executor_type = executor_type
-
-        if max_workers is not None:
-            self.max_workers = max_workers
-
-        return self
-
-    def __repr__(self) -> str:
-        """String representation of the parallel pipeline."""
-        step_strs = [
-            f"('{name}', {transformer.__class__.__name__})" for name, transformer in self.steps
-        ]
-        config_str = f"executor={self.executor_type}, workers={self.max_workers or 'auto'}"
-        return "ParallelPipeline([\n  " + ",\n  ".join(step_strs) + f"\n], {config_str})"
-
-
-__all__ = ["ParallelPipeline"]