oscura 0.5.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

oscura/{batch → workflows/batch}/advanced.py

@@ -16,13 +16,20 @@ from dataclasses import asdict, dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal
 
-import pandas as pd
+# Lazy import for optional dataframe support
+try:
+    import pandas as pd
+
+    _HAS_PANDAS = True
+except ImportError:
+    pd = None  # type: ignore[assignment]
+    _HAS_PANDAS = False
 
 if TYPE_CHECKING:
     from collections.abc import Callable
 
 try:
-    from tqdm import tqdm  # type: ignore[import-untyped]
+    from tqdm import tqdm
 
     HAS_TQDM = True
 except ImportError:
@@ -269,7 +276,7 @@ def _run_with_timeout(
     if result_container["error"] is not None:
         raise result_container["error"]
 
-    return result_container["result"], False  # type: ignore[return-value]
+    return result_container["result"], False
 
 
 class AdvancedBatchProcessor:
@@ -287,7 +294,7 @@ class AdvancedBatchProcessor:
         - Processing continues to the next file
 
     Example:
-        >>> from oscura.batch.advanced import AdvancedBatchProcessor, BatchConfig
+        >>> from oscura.workflows.batch.advanced import AdvancedBatchProcessor, BatchConfig
         >>> config = BatchConfig(
         ...     on_error='skip',
         ...     checkpoint_dir='./checkpoints',
@@ -335,9 +342,21 @@ class AdvancedBatchProcessor:
             ...     analysis_fn=analyze_trace
             ... )
 
+        Raises:
+            ImportError: If pandas is not installed.
+
         References:
             API-012: Advanced Batch Control
         """
+        if not _HAS_PANDAS:
+            raise ImportError(
+                "Batch processing requires pandas.\n\n"
+                "Install with:\n"
+                "  pip install oscura[dataframes]    # DataFrame support\n"
+                "  pip install oscura[standard]      # Recommended\n"
+                "  pip install oscura[all]           # Everything\n"
+            )
+
         # Try to resume from checkpoint
         remaining_files = self._resume_or_start(files, checkpoint_name)
 
@@ -392,39 +411,38 @@ class AdvancedBatchProcessor:
 
         return files
 
-    def _process_files(
+    def _create_progress_bar(self) -> Any:
+        """Create progress bar if requested.
+
+        Returns:
+            tqdm progress bar or None.
+        """
+        if not (self.config.progress_bar and HAS_TQDM):
+            return None
+
+        total = self.checkpoint.total_files if self.checkpoint else 0
+        initial = (
+            len(self.checkpoint.completed_files) + len(self.checkpoint.failed_files)
+            if self.checkpoint
+            else 0
+        )
+        return tqdm(total=total, initial=initial, desc="Processing files")
+
+    def _create_file_processor(
         self,
-        files: list[str | Path],
         analysis_fn: Callable[[str | Path], dict[str, Any]],
-        checkpoint_name: str,
-        **kwargs: Any,
-    ) -> None:
-        """Process files with parallel execution and checkpointing.
+        kwargs: dict[str, Any],
+    ) -> Callable[[str | Path], FileResult]:
+        """Create wrapper function for processing a single file.
 
         Args:
-            files: Files to process.
-            analysis_fn: Analysis function.
-            checkpoint_name: Checkpoint file name.
-            **kwargs: Additional arguments for analysis_fn.
+            analysis_fn: Analysis function to apply.
+            kwargs: Keyword arguments for analysis_fn.
 
-        Raises:
-            RuntimeError: If processing is stopped due to error and on_error is "stop".
+        Returns:
+            Wrapper function that returns FileResult.
         """
-        if not files:
-            return
 
-        # Create progress bar if requested
-        pbar = None
-        if self.config.progress_bar and HAS_TQDM:
-            total = self.checkpoint.total_files if self.checkpoint else len(files)
-            initial = (
-                len(self.checkpoint.completed_files) + len(self.checkpoint.failed_files)
-                if self.checkpoint
-                else 0
-            )
-            pbar = tqdm(total=total, initial=initial, desc="Processing files")
-
-        # Wrapper for file processing with error handling and timeout
         def _process_one(file_path: str | Path) -> FileResult:
             import time
 
@@ -469,6 +487,105 @@ class AdvancedBatchProcessor:
                     duration=duration,
                 )
 
+        return _process_one
+
+    def _retrieve_future_result(
+        self, future: concurrent.futures.Future[FileResult], file_path: str | Path
+    ) -> FileResult:
+        """Retrieve result from future with timeout handling.
+
+        Args:
+            future: Future to retrieve result from.
+            file_path: Path being processed (for error messages).
+
+        Returns:
+            FileResult from future or error result if timeout/exception.
+        """
+        try:
+            # Apply timeout on result retrieval as backup enforcement
+            retrieval_timeout = (
+                self.config.timeout_per_file * 1.1 if self.config.timeout_per_file else None
+            )
+            return future.result(timeout=retrieval_timeout)
+        except concurrent.futures.TimeoutError:
+            # Backup timeout triggered
+            return FileResult(
+                file=str(file_path),
+                success=False,
+                error=f"Processing timed out (backup enforcement) "
+                f"after {self.config.timeout_per_file}s",
+                timed_out=True,
+            )
+        except Exception as e:
+            # Unexpected error during result retrieval
+            return FileResult(
+                file=str(file_path),
+                success=False,
+                error=f"Error retrieving result: {e}",
+                traceback=traceback.format_exc(),
+            )
+
+    def _update_checkpoint_with_result(self, file_result: FileResult) -> None:
+        """Update checkpoint with processing result.
+
+        Args:
+            file_result: Result to add to checkpoint.
+        """
+        if not self.checkpoint:
+            return
+
+        self.checkpoint.results.append(file_result)
+        if file_result.success:
+            self.checkpoint.completed_files.append(file_result.file)
+        else:
+            self.checkpoint.failed_files.append(file_result.file)
+
+    def _handle_file_error(self, file_result: FileResult, pbar: Any) -> None:
+        """Handle error from file processing.
+
+        Args:
+            file_result: Result with error.
+            pbar: Progress bar to close if stopping.
+
+        Raises:
+            RuntimeError: If on_error is "stop".
+        """
+        if self.config.on_error == "stop":
+            if pbar:
+                pbar.close()
+            raise RuntimeError(
+                f"Processing stopped due to error in {file_result.file}: {file_result.error}"
+            )
+        elif self.config.on_error == "warn":
+            timeout_note = " (TIMEOUT)" if file_result.timed_out else ""
+            print(
+                f"Warning: Error processing {file_result.file}{timeout_note}: {file_result.error}"
+            )
+
+    def _process_files(
+        self,
+        files: list[str | Path],
+        analysis_fn: Callable[[str | Path], dict[str, Any]],
+        checkpoint_name: str,
+        **kwargs: Any,
+    ) -> None:
+        """Process files with parallel execution and checkpointing.
+
+        Args:
+            files: Files to process.
+            analysis_fn: Analysis function.
+            checkpoint_name: Checkpoint file name.
+            **kwargs: Additional arguments for analysis_fn.
+
+        Raises:
+            RuntimeError: If processing is stopped due to error and on_error is "stop".
+        """
+        if not files:
+            return
+
+        pbar = self._create_progress_bar()
+        _process_one = self._create_file_processor(analysis_fn, kwargs)
+
         # Process files
         processed_count = 0
         executor_class = ThreadPoolExecutor if self.config.use_threads else ProcessPoolExecutor
@@ -478,58 +595,16 @@ class AdvancedBatchProcessor:
             futures = {executor.submit(_process_one, f): f for f in files}
 
             # Process results as they complete
-            # Use future.result() with timeout for additional enforcement layer
             for future in as_completed(futures):
                 file_path = futures[future]
-                try:
-                    # Apply timeout on result retrieval as backup enforcement
-                    # This catches cases where the thread-based timeout didn't work
-                    # (e.g., CPU-bound operations that don't release the GIL)
-                    retrieval_timeout = (
-                        self.config.timeout_per_file * 1.1 if self.config.timeout_per_file else None
-                    )
-                    file_result = future.result(timeout=retrieval_timeout)
-                except concurrent.futures.TimeoutError:
-                    # Backup timeout triggered - future.result() timed out
-                    file_result = FileResult(
-                        file=str(file_path),
-                        success=False,
-                        error=f"Processing timed out (backup enforcement) "
-                        f"after {self.config.timeout_per_file}s",
-                        timed_out=True,
-                    )
-                except Exception as e:
-                    # Unexpected error during result retrieval
-                    file_result = FileResult(
-                        file=str(file_path),
-                        success=False,
-                        error=f"Error retrieving result: {e}",
-                        traceback=traceback.format_exc(),
-                    )
+                file_result = self._retrieve_future_result(future, file_path)
 
                 # Update checkpoint
-                if self.checkpoint:
-                    self.checkpoint.results.append(file_result)
-                    if file_result.success:
-                        self.checkpoint.completed_files.append(file_result.file)
-                    else:
-                        self.checkpoint.failed_files.append(file_result.file)
+                self._update_checkpoint_with_result(file_result)
 
                 # Handle errors
                 if not file_result.success:
-                    if self.config.on_error == "stop":
-                        if pbar:
-                            pbar.close()
-                        raise RuntimeError(
-                            f"Processing stopped due to error in {file_result.file}: "
-                            f"{file_result.error}"
-                        )
-                    elif self.config.on_error == "warn":
-                        timeout_note = " (TIMEOUT)" if file_result.timed_out else ""
-                        print(
-                            f"Warning: Error processing {file_result.file}{timeout_note}: "
-                            f"{file_result.error}"
-                        )
+                    self._handle_file_error(file_result, pbar)
 
                 # Update progress
                 processed_count += 1