speedy-utils 1.1.42__py3-none-any.whl → 1.1.44__py3-none-any.whl

speedy_utils/multi_worker/process.py

@@ -9,11 +9,491 @@ os.environ["RAY_ACCEL_ENV_VAR_OVERRI" \
 "DE_ON_ZERO"] = "0"
 os.environ["RAY_DEDUP_LOGS"] = "1"
 os.environ["RAY_LOG_TO_STDERR"] = "0"
+os.environ["RAY_IGNORE_UNHANDLED_ERRORS"] = "1"
 
 from ..__imports import *
 import tempfile
+import inspect
+import linecache
+import traceback as tb_module
 from .progress import create_progress_tracker, ProgressPoller, get_ray_progress_actor
 
+# Import thread tracking functions if available
+try:
+    from .thread import _prune_dead_threads, _track_executor_threads
+except ImportError:
+    _prune_dead_threads = None  # type: ignore[assignment]
+    _track_executor_threads = None  # type: ignore[assignment]
+
+
+# ─── error handler types ────────────────────────────────────────
+ErrorHandlerType = Literal['raise', 'ignore', 'log']
+
+
+class ErrorStats:
+    """Thread-safe error statistics tracker."""
+
+    def __init__(
+        self,
+        func_name: str,
+        max_error_files: int = 100,
+        write_logs: bool = True
+    ):
+        self._lock = threading.Lock()
+        self._success_count = 0
+        self._error_count = 0
+        self._first_error_shown = False
+        self._max_error_files = max_error_files
+        self._write_logs = write_logs
+        self._error_log_dir = self._get_error_log_dir(func_name)
+        if write_logs:
+            self._error_log_dir.mkdir(parents=True, exist_ok=True)
+
+    @staticmethod
+    def _get_error_log_dir(func_name: str) -> Path:
+        """Generate unique error log directory with run counter."""
+        base_dir = Path('.cache/speedy_utils/error_logs')
+        base_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Find the next run counter
+        counter = 1
+        existing = list(base_dir.glob(f'{func_name}_run_*'))
+        if existing:
+            counters = []
+            for p in existing:
+                try:
+                    parts = p.name.split('_run_')
+                    if len(parts) == 2:
+                        counters.append(int(parts[1]))
+                except (ValueError, IndexError):
+                    pass
+            if counters:
+                counter = max(counters) + 1
+        
+        return base_dir / f'{func_name}_run_{counter}'
+
+    def record_success(self) -> None:
+        with self._lock:
+            self._success_count += 1
+
+    def record_error(
+        self,
+        idx: int,
+        error: Exception,
+        input_value: Any,
+        func_name: str,
+    ) -> str | None:
+        """
+        Record an error and write to log file.
+        Returns the log file path if written, None otherwise.
+        """
+        with self._lock:
+            self._error_count += 1
+            should_show_first = not self._first_error_shown
+            if should_show_first:
+                self._first_error_shown = True
+            should_write = (
+                self._write_logs and self._error_count <= self._max_error_files
+            )
+
+        log_path = None
+        if should_write:
+            log_path = self._write_error_log(
+                idx, error, input_value, func_name
+            )
+
+        if should_show_first:
+            self._print_first_error(error, input_value, func_name, log_path)
+
+        return log_path
+
+    def _write_error_log(
+        self,
+        idx: int,
+        error: Exception,
+        input_value: Any,
+        func_name: str,
+    ) -> str:
+        """Write error details to a log file."""
+        from io import StringIO
+        from rich.console import Console
+        
+        log_path = self._error_log_dir / f'{idx}.log'
+        
+        output = StringIO()
+        console = Console(file=output, width=120, no_color=False)
+        
+        # Format traceback
+        tb_lines = self._format_traceback(error)
+        
+        console.print(f'{"=" * 60}')
+        console.print(f'Error at index: {idx}')
+        console.print(f'Function: {func_name}')
+        console.print(f'Error Type: {type(error).__name__}')
+        console.print(f'Error Message: {error}')
+        console.print(f'{"=" * 60}')
+        console.print('')
+        console.print('Input:')
+        console.print('-' * 40)
+        try:
+            import json
+            console.print(json.dumps(input_value, indent=2))
+        except Exception:
+            console.print(repr(input_value))
+        console.print('')
+        console.print('Traceback:')
+        console.print('-' * 40)
+        for line in tb_lines:
+            console.print(line)
+        
+        with open(log_path, 'w') as f:
+            f.write(output.getvalue())
+        
+        return str(log_path)
+
+    def _format_traceback(self, error: Exception) -> list[str]:
+        """Format traceback with context lines like Rich panel."""
+        lines = []
+        frames = _extract_frames_from_traceback(error)
+        
+        for filepath, lineno, funcname, frame_locals in frames:
+            lines.append(f'│ {filepath}:{lineno} in {funcname} │')
+            lines.append('│' + ' ' * 70 + '│')
+            
+            # Get context lines
+            context_size = 3
+            start_line = max(1, lineno - context_size)
+            end_line = lineno + context_size + 1
+            
+            for line_num in range(start_line, end_line):
+                line_text = linecache.getline(filepath, line_num).rstrip()
+                if line_text:
+                    num_str = str(line_num).rjust(4)
+                    if line_num == lineno:
+                        lines.append(f'│   {num_str} ❱     {line_text}')
+                    else:
+                        lines.append(f'│   {num_str} │     {line_text}')
+            lines.append('')
+        
+        return lines
+
+    def _print_first_error(
+        self,
+        error: Exception,
+        input_value: Any,
+        func_name: str,
+        log_path: str | None,
+    ) -> None:
+        """Print the first error to screen with Rich formatting."""
+        try:
+            from rich.console import Console
+            from rich.panel import Panel
+            console = Console(stderr=True)
+            
+            tb_lines = self._format_traceback(error)
+            
+            console.print()
+            console.print(
+                Panel(
+                    '\n'.join(tb_lines),
+                    title='[bold red]First Error (continuing with remaining items)[/bold red]',
+                    border_style='yellow',
+                    expand=False,
+                )
+            )
+            console.print(
+                f'[bold red]{type(error).__name__}[/bold red]: {error}'
+            )
+            if log_path:
+                console.print(f'[dim]Error log: {log_path}[/dim]')
+            console.print()
+        except ImportError:
+            # Fallback to plain print
+            print(f'\n--- First Error ---', file=sys.stderr)
+            print(f'{type(error).__name__}: {error}', file=sys.stderr)
+            if log_path:
+                print(f'Error log: {log_path}', file=sys.stderr)
+            print('', file=sys.stderr)
+
+    @property
+    def success_count(self) -> int:
+        with self._lock:
+            return self._success_count
+
+    @property
+    def error_count(self) -> int:
+        with self._lock:
+            return self._error_count
+
+    def get_postfix_dict(self) -> dict[str, int]:
+        """Get dict for pbar postfix."""
+        with self._lock:
+            return {'ok': self._success_count, 'err': self._error_count}
+
+
+def _should_skip_frame(filepath: str) -> bool:
+    """Check if a frame should be filtered from traceback display."""
+    skip_patterns = [
+        'ray/_private',
+        'ray/worker',
+        'site-packages/ray',
+        'speedy_utils/multi_worker',
+        'concurrent/futures',
+        'multiprocessing/',
+        'fastcore/parallel',
+        'fastcore/foundation',
+        'fastcore/basics',
+        'site-packages/fastcore',
+        '/threading.py',
+        '/concurrent/',
+    ]
+    return any(skip in filepath for skip in skip_patterns)
+
+
+def _should_show_local(name: str, value: object) -> bool:
+    """Check if a local variable should be displayed in traceback."""
+    import types
+    
+    # Skip dunder variables
+    if name.startswith('__') and name.endswith('__'):
+        return False
+    
+    # Skip modules
+    if isinstance(value, types.ModuleType):
+        return False
+    
+    # Skip type objects and classes
+    if isinstance(value, type):
+        return False
+    
+    # Skip functions and methods
+    if isinstance(value, (types.FunctionType, types.MethodType, types.BuiltinFunctionType)):
+        return False
+    
+    # Skip common typing aliases
+    value_str = str(value)
+    if value_str.startswith('typing.'):
+        return False
+    
+    # Skip large objects that would clutter output
+    if value_str.startswith('<') and any(x in value_str for x in ['module', 'function', 'method', 'built-in']):
+        return False
+    
+    return True
+
+
+def _format_locals(frame_locals: dict) -> list[str]:
+    """Format local variables for display, filtering out noisy imports."""
+    from rich.pretty import Pretty
+    from rich.console import Console
+    from io import StringIO
+    
+    # Filter locals
+    clean_locals = {k: v for k, v in frame_locals.items() if _should_show_local(k, v)}
+    
+    if not clean_locals:
+        return []
+    
+    lines = []
+    lines.append('[dim]╭─ locals ─╮[/dim]')
+    
+    # Format each local variable
+    for name, value in clean_locals.items():
+        # Use Rich's Pretty for nice formatting
+        try:
+            console = Console(file=StringIO(), width=60)
+            console.print(Pretty(value), end='')
+            value_str = console.file.getvalue().strip()
+            # Limit length
+            if len(value_str) > 100:
+                value_str = value_str[:97] + '...'
+        except Exception:
+            value_str = repr(value)
+            if len(value_str) > 100:
+                value_str = value_str[:97] + '...'
+        
+        lines.append(f'[dim]│[/dim] {name} = {value_str}')
+    
+    lines.append('[dim]╰──────────╯[/dim]')
+    return lines
+
+
+def _format_frame_with_context(filepath: str, lineno: int, funcname: str, frame_locals: dict | None = None) -> list[str]:
+    """Format a single frame with context lines and optional locals."""
+    lines = []
+    # Frame header
+    lines.append(
+        f'[cyan]{filepath}[/cyan]:[yellow]{lineno}[/yellow] '
+        f'in [green]{funcname}[/green]'
+    )
+    lines.append('')
+    
+    # Get context lines
+    context_size = 3
+    start_line = max(1, lineno - context_size)
+    end_line = lineno + context_size + 1
+    
+    for line_num in range(start_line, end_line):
+        import linecache
+        line_text = linecache.getline(filepath, line_num).rstrip()
+        if line_text:
+            num_str = str(line_num).rjust(4)
+            if line_num == lineno:
+                lines.append(f'[dim]{num_str}[/dim] [red]❱[/red] {line_text}')
+            else:
+                lines.append(f'[dim]{num_str} │[/dim] {line_text}')
+    
+    # Add locals if available
+    if frame_locals:
+        locals_lines = _format_locals(frame_locals)
+        if locals_lines:
+            lines.append('')
+            lines.extend(locals_lines)
+    
+    lines.append('')
+    return lines
+
+
+def _display_formatted_error(
+    exc_type_name: str,
+    exc_msg: str,
+    frames: list[tuple[str, int, str, dict]],
+    caller_info: dict | None,
+    backend: str,
+    pbar=None,
+) -> None:
+    
+    # Suppress additional error logs
+    os.environ['RAY_IGNORE_UNHANDLED_ERRORS'] = '1'
+
+    # Close progress bar cleanly if provided
+    if pbar is not None:
+        pbar.close()
+
+    from rich.console import Console
+    from rich.panel import Panel
+    console = Console(stderr=True)
+
+    if frames or caller_info:
+        display_lines = []
+        
+        # Add caller frame first if available (no locals for caller)
+        if caller_info:
+            display_lines.extend(_format_frame_with_context(
+                caller_info['filename'],
+                caller_info['lineno'],
+                caller_info['function'],
+                None  # Don't show locals for caller frame
+            ))
+        
+        # Add error frames with locals
+        for filepath, lineno, funcname, frame_locals in frames:
+            display_lines.extend(_format_frame_with_context(
+                filepath, lineno, funcname, frame_locals
+            ))
+
+        # Display the traceback
+        console.print()
+        console.print(
+            Panel(
+                '\n'.join(display_lines),
+                title=f'[bold red]Traceback (most recent call last) [{backend}][/bold red]',
+                border_style='red',
+                expand=False,
+            )
+        )
+        console.print(f'[bold red]{exc_type_name}[/bold red]: {exc_msg}')
+        console.print()
+    else:
+        # No frames found, minimal output
+        console.print()
+        console.print(f'[bold red]{exc_type_name}[/bold red]: {exc_msg}')
+        console.print()
+
+    # Ensure output is flushed
+    sys.stderr.flush()
+    sys.stdout.flush()
+    sys.exit(1)
+
+
+def _extract_frames_from_traceback(error: Exception) -> list[tuple[str, int, str, dict]]:
+    """Extract user frames from exception traceback object with locals."""
+    frames = []
+    if hasattr(error, '__traceback__') and error.__traceback__ is not None:
+        tb = error.__traceback__
+        while tb is not None:
+            frame = tb.tb_frame
+            filename = frame.f_code.co_filename
+            lineno = tb.tb_lineno
+            funcname = frame.f_code.co_name
+            
+            if not _should_skip_frame(filename):
+                # Get local variables from the frame
+                frame_locals = dict(frame.f_locals)
+                frames.append((filename, lineno, funcname, frame_locals))
+            
+            tb = tb.tb_next
+    return frames
+
+
+def _extract_frames_from_ray_error(ray_task_error: Exception) -> list[tuple[str, int, str, dict]]:
+    """Extract user frames from Ray's string traceback representation."""
+    frames = []
+    error_str = str(ray_task_error)
+    lines = error_str.split('\n')
+    
+    import re
+    for i, line in enumerate(lines):
+        # Match: File "path", line N, in func
+        file_match = re.match(r'\s*File "([^"]+)", line (\d+), in (.+)', line)
+        if file_match:
+            filepath, lineno, funcname = file_match.groups()
+            if not _should_skip_frame(filepath):
+                # Ray doesn't preserve locals, so use empty dict
+                frames.append((filepath, int(lineno), funcname, {}))
+    
+    return frames
+
+
+def _reraise_worker_error(error: Exception, pbar=None, caller_info=None, backend: str = 'unknown') -> None:
+    """
+    Re-raise the original exception from a worker error with clean traceback.
+    Works for multiprocessing, threadpool, and other backends with real tracebacks.
+    """
+    frames = _extract_frames_from_traceback(error)
+    _display_formatted_error(
+        exc_type_name=type(error).__name__,
+        exc_msg=str(error),
+        frames=frames,
+        caller_info=caller_info,
+        backend=backend,
+        pbar=pbar,
+    )
+
+
+def _reraise_ray_error(ray_task_error: Exception, pbar=None, caller_info=None) -> None:
+    """
+    Re-raise the original exception from a RayTaskError with clean traceback.
+    Parses Ray's string traceback and displays with full context.
+    """
+    # Get the exception info
+    cause = ray_task_error.cause if hasattr(ray_task_error, 'cause') else None
+    if cause is None:
+        cause = ray_task_error.__cause__
+
+    exc_type_name = type(cause).__name__ if cause else 'Error'
+    exc_msg = str(cause) if cause else str(ray_task_error)
+    
+    frames = _extract_frames_from_ray_error(ray_task_error)
+    _display_formatted_error(
+        exc_type_name=exc_type_name,
+        exc_msg=exc_msg,
+        frames=frames,
+        caller_info=caller_info,
+        backend='ray',
+        pbar=pbar,
+    )
+
 
 SPEEDY_RUNNING_PROCESSES: list[psutil.Process] = []
 _SPEEDY_PROCESSES_LOCK = threading.Lock()
@@ -96,7 +576,7 @@ def _build_cache_dir(func: Callable, items: list[Any]) -> Path:
     path = Path('.cache') / run_id
     path.mkdir(parents=True, exist_ok=True)
     return path
-_DUMP_THREADS = []
+_DUMP_INTERMEDIATE_THREADS = []
 def wrap_dump(func: Callable, cache_dir: Path | None, dump_in_thread: bool = True):
     """Wrap a function so results are dumped to .pkl when cache_dir is set."""
     if cache_dir is None:
@@ -115,7 +595,7 @@ def wrap_dump(func: Callable, cache_dir: Path | None, dump_in_thread: bool = Tru
 
         if dump_in_thread:
             thread = threading.Thread(target=save)
-            _DUMP_THREADS.append(thread)
+            _DUMP_INTERMEDIATE_THREADS.append(thread)
             # count thread
             # print(f'Thread count: {threading.active_count()}')
             while threading.active_count() > 16:
@@ -343,8 +823,7 @@ def multi_process(
     workers: int | None = None,
     lazy_output: bool = False,
     progress: bool = True,
-    # backend: str = "ray",   # "seq", "ray", or "fastcore"
-    backend: Literal['seq', 'ray', 'mp', 'threadpool', 'safe'] = 'mp',
+    backend: Literal['seq', 'ray', 'mp', 'safe'] = 'mp',
     desc: str | None = None,
     shared_kwargs: list[str] | None = None,
     dump_in_thread: bool = True,
@@ -352,6 +831,8 @@ def multi_process(
     log_worker: Literal['zero', 'first', 'all'] = 'first',
     total_items: int | None = None,
     poll_interval: float = 0.3,
+    error_handler: ErrorHandlerType = 'log',
+    max_error_files: int = 100,
     **func_kwargs: Any,
 ) -> list[Any]:
     """
@@ -360,36 +841,44 @@ def multi_process(
     backend:
         - "seq": run sequentially
         - "ray": run in parallel with Ray
-        - "mp": run in parallel with multiprocessing (uses threadpool to avoid fork warnings)
-        - "threadpool": run in parallel with thread pool
+        - "mp": run in parallel with thread pool (uses ThreadPoolExecutor)
         - "safe": run in parallel with thread pool (explicitly safe for tests)
 
     shared_kwargs:
-        - Optional list of kwarg names that should be shared via Ray's zero-copy object store
+        - Optional list of kwarg names that should be shared via Ray's
+          zero-copy object store
         - Only works with Ray backend
-        - Useful for large objects (e.g., models, datasets) that should be shared across workers
-        - Example: shared_kwargs=['model', 'tokenizer'] for sharing large ML models
+        - Useful for large objects (e.g., models, datasets)
+        - Example: shared_kwargs=['model', 'tokenizer']
 
     dump_in_thread:
         - Whether to dump results to disk in a separate thread (default: True)
-        - If False, dumping is done synchronously, which may block but ensures data is saved before returning
+        - If False, dumping is done synchronously
 
     ray_metrics_port:
         - Optional port for Ray metrics export (Ray backend only)
-        - Set to 0 to disable Ray metrics
-        - If None, uses Ray's default behavior
 
     log_worker:
         - Control worker stdout/stderr noise
-        - 'first': only first worker emits logs, others are silenced (default)
-        - 'all': allow worker prints (may overlap tqdm)
-        - 'zero': silence worker stdout/stderr to keep progress bar clean
+        - 'first': only first worker emits logs (default)
+        - 'all': allow worker prints
+        - 'zero': silence all worker output
 
     total_items:
         - Optional item-level total for progress tracking (Ray backend only)
 
     poll_interval:
-        - Poll interval in seconds for progress actor updates (Ray backend only)
+        - Poll interval in seconds for progress actor updates (Ray only)
+
+    error_handler:
+        - 'raise': raise exception on first error (default)
+        - 'ignore': continue processing, return None for failed items
+        - 'log': same as ignore, but logs errors to files
+
+    max_error_files:
+        - Maximum number of error log files to write (default: 100)
+        - Error logs are written to .cache/speedy_utils/error_logs/{idx}.log
+        - First error is always printed to screen with the log file path
 
     If lazy_output=True, every result is saved to .pkl and
     the returned list contains file paths.
@@ -426,7 +915,7 @@ def multi_process(
                 )
 
     # Prefer Ray backend when shared kwargs are requested
-    if backend != 'ray':
+    if shared_kwargs and backend != 'ray':
         warnings.warn(
             "shared_kwargs only supported with 'ray' backend, switching backend to 'ray'",
             UserWarning,
@@ -462,21 +951,62 @@ def multi_process(
     else:
         desc = f'Multi-process [{backend}]'
 
+    # Initialize error stats for error handling
+    func_name = getattr(func, '__name__', repr(func))
+    error_stats = ErrorStats(
+        func_name=func_name,
+        max_error_files=max_error_files,
+        write_logs=error_handler == 'log'
+    )
+
+    def _update_pbar_postfix(pbar: tqdm) -> None:
+        """Update pbar with success/error counts."""
+        postfix = error_stats.get_postfix_dict()
+        pbar.set_postfix(postfix)
+
+    def _wrap_with_error_handler(
+        f: Callable,
+        idx: int,
+        input_value: Any,
+        error_stats_ref: ErrorStats,
+        handler: ErrorHandlerType,
+    ) -> Callable:
+        """Wrap function to handle errors based on error_handler setting."""
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            try:
+                result = f(*args, **kwargs)
+                error_stats_ref.record_success()
+                return result
+            except Exception as e:
+                if handler == 'raise':
+                    raise
+                error_stats_ref.record_error(idx, e, input_value, func_name)
+                return None
+        return wrapper
+
     # ---- sequential backend ----
     if backend == 'seq':
         results: list[Any] = []
         with tqdm(total=total, desc=desc, disable=not progress, file=sys.stdout) as pbar:
-            for x in items:
-                results.append(
-                    _call_with_log_control(
+            for idx, x in enumerate(items):
+                try:
+                    result = _call_with_log_control(
                         f_wrapped,
                         x,
                         func_kwargs,
                         log_worker,
                         log_gate_path,
                     )
-                )
+                    error_stats.record_success()
+                    results.append(result)
+                except Exception as e:
+                    if error_handler == 'raise':
+                        raise
+                    error_stats.record_error(idx, e, x, func_name)
+                    results.append(None)
                 pbar.update(1)
+                _update_pbar_postfix(pbar)
         _cleanup_log_gate(log_gate_path)
         return results
 
@@ -484,6 +1014,16 @@ def multi_process(
     if backend == 'ray':
         import ray as _ray_module
 
+        # Capture caller frame for better error reporting
+        caller_frame = inspect.currentframe()
+        caller_info = None
+        if caller_frame and caller_frame.f_back:
+            caller_info = {
+                'filename': caller_frame.f_back.f_code.co_filename,
+                'lineno': caller_frame.f_back.f_lineno,
+                'function': caller_frame.f_back.f_code.co_name,
+            }
+
         results = []
         gate_path_str = str(log_gate_path) if log_gate_path else None
         with tqdm(total=total, desc=desc, disable=not progress, file=sys.stdout) as pbar:
@@ -496,7 +1036,7 @@ def multi_process(
             progress_poller = None
             if total_items is not None:
                 progress_actor = create_progress_tracker(total_items, desc or "Items")
-                shared_refs['progress_actor'] = progress_actor  # Pass actor handle directly (not via put)
+                shared_refs['progress_actor'] = progress_actor
 
             if shared_kwargs:
                 for kw in shared_kwargs:
@@ -520,11 +1060,9 @@ def multi_process(
                 dereferenced = {}
                 for k, v in shared_refs_dict.items():
                     if k == 'progress_actor':
-                        # Pass actor handle directly (don't dereference)
                         dereferenced[k] = v
                     else:
                         dereferenced[k] = _ray_in_task.get(v)
-                # Merge with regular kwargs
                 all_kwargs = {**dereferenced, **regular_kwargs_dict}
                 return _call_with_log_control(
                     f_wrapped,
@@ -540,21 +1078,32 @@ def multi_process(
 
             t_start = time.time()
             
-            # Start progress poller if using item-level progress
             if progress_actor is not None:
-                # Update pbar total to show items instead of tasks
                 pbar.total = total_items
                 pbar.refresh()
                 progress_poller = ProgressPoller(progress_actor, pbar, poll_interval)
                 progress_poller.start()
             
-            for r in refs:
-                results.append(_ray_module.get(r))
+            for idx, r in enumerate(refs):
+                try:
+                    result = _ray_module.get(r)
+                    error_stats.record_success()
+                    results.append(result)
+                except _ray_module.exceptions.RayTaskError as e:
+                    if error_handler == 'raise':
+                        if progress_poller is not None:
+                            progress_poller.stop()
+                        _reraise_ray_error(e, pbar, caller_info)
+                    # Extract original error from RayTaskError
+                    cause = e.cause if hasattr(e, 'cause') else e.__cause__
+                    original_error = cause if cause else e
+                    error_stats.record_error(idx, original_error, items[idx], func_name)
+                    results.append(None)
+                
                 if progress_actor is None:
-                    # Only update task-level progress if not using item-level
                     pbar.update(1)
+                _update_pbar_postfix(pbar)
             
-            # Stop progress poller
             if progress_poller is not None:
                 progress_poller.stop()
                 
@@ -564,73 +1113,132 @@ def multi_process(
         _cleanup_log_gate(log_gate_path)
         return results
 
-    # ---- fastcore backend ----
+    # ---- fastcore/thread backend (mp) ----
     if backend == 'mp':
-        worker_func = functools.partial(
-            _call_with_log_control,
-            f_wrapped,
-            func_kwargs=func_kwargs,
-            log_mode=log_worker,
-            gate_path=log_gate_path,
-        )
-        with _patch_fastcore_progress_bar(leave=True):
-            results = list(parallel(
-                worker_func, items, n_workers=workers, progress=progress, threadpool=False
-            ))
-        _track_multiprocessing_processes()  # Track multiprocessing workers
-        _prune_dead_processes()  # Clean up dead processes
-        _cleanup_log_gate(log_gate_path)
-        return results
-
-    if backend == 'threadpool':
-        worker_func = functools.partial(
-            _call_with_log_control,
-            f_wrapped,
-            func_kwargs=func_kwargs,
-            log_mode=log_worker,
-            gate_path=log_gate_path,
-        )
-        with _patch_fastcore_progress_bar(leave=True):
-            results = list(parallel(
-                worker_func, items, n_workers=workers, progress=progress, threadpool=True
-            ))
-        _cleanup_log_gate(log_gate_path)
-        return results
-
-    if backend == 'safe':
-        # Completely safe backend for tests - no multiprocessing, no external progress bars
         import concurrent.futures
-
-        # Import thread tracking from thread module
-        try:
-            from .thread import _prune_dead_threads, _track_executor_threads
-
-            worker_func = functools.partial(
-                _call_with_log_control,
+        
+        # Capture caller frame for better error reporting
+        caller_frame = inspect.currentframe()
+        caller_info = None
+        if caller_frame and caller_frame.f_back:
+            caller_info = {
+                'filename': caller_frame.f_back.f_code.co_filename,
+                'lineno': caller_frame.f_back.f_lineno,
+                'function': caller_frame.f_back.f_code.co_name,
+            }
+        
+        def worker_func(x):
+            return _call_with_log_control(
                 f_wrapped,
-                func_kwargs=func_kwargs,
-                log_mode=log_worker,
-                gate_path=log_gate_path,
+                x,
+                func_kwargs,
+                log_worker,
+                log_gate_path,
             )
+        
+        results: list[Any] = [None] * total
+        with tqdm(total=total, desc=desc, disable=not progress, file=sys.stdout) as pbar:
+            try:
+                from .thread import _prune_dead_threads, _track_executor_threads
+                has_thread_tracking = True
+            except ImportError:
+                has_thread_tracking = False
+            
             with concurrent.futures.ThreadPoolExecutor(
                 max_workers=workers
             ) as executor:
-                _track_executor_threads(executor)  # Track threads
-                results = list(executor.map(worker_func, items))
-            _prune_dead_threads()  # Clean up dead threads
-        except ImportError:
-            # Fallback if thread module not available
-            worker_func = functools.partial(
-                _call_with_log_control,
+                if has_thread_tracking:
+                    _track_executor_threads(executor)
+                
+                # Submit all tasks
+                future_to_idx = {
+                    executor.submit(worker_func, x): idx
+                    for idx, x in enumerate(items)
+                }
+                
+                # Process results as they complete
+                for future in concurrent.futures.as_completed(future_to_idx):
+                    idx = future_to_idx[future]
+                    try:
+                        result = future.result()
+                        error_stats.record_success()
+                        results[idx] = result
+                    except Exception as e:
+                        if error_handler == 'raise':
+                            # Cancel remaining futures
+                            for f in future_to_idx:
+                                f.cancel()
+                            _reraise_worker_error(e, pbar, caller_info, backend='mp')
+                        error_stats.record_error(idx, e, items[idx], func_name)
+                        results[idx] = None
+                    pbar.update(1)
+                    _update_pbar_postfix(pbar)
+            
+            if _prune_dead_threads is not None:
+                _prune_dead_threads()
+            
+        _track_multiprocessing_processes()
+        _prune_dead_processes()
+        _cleanup_log_gate(log_gate_path)
+        return results
+
+    if backend == 'safe':
+        # Completely safe backend for tests - no multiprocessing
+        import concurrent.futures
+        
+        # Capture caller frame for better error reporting
+        caller_frame = inspect.currentframe()
+        caller_info = None
+        if caller_frame and caller_frame.f_back:
+            caller_info = {
+                'filename': caller_frame.f_back.f_code.co_filename,
+                'lineno': caller_frame.f_back.f_lineno,
+                'function': caller_frame.f_back.f_code.co_name,
+            }
+
+        def worker_func(x):
+            return _call_with_log_control(
                 f_wrapped,
-                func_kwargs=func_kwargs,
-                log_mode=log_worker,
-                gate_path=log_gate_path,
+                x,
+                func_kwargs,
+                log_worker,
+                log_gate_path,
             )
+        
+        results: list[Any] = [None] * total
+        with tqdm(total=total, desc=desc, disable=not progress, file=sys.stdout) as pbar:
             with concurrent.futures.ThreadPoolExecutor(
                 max_workers=workers
             ) as executor:
-                results = list(executor.map(worker_func, items))
+                if _track_executor_threads is not None:
+                    _track_executor_threads(executor)
+                
+                # Submit all tasks
+                future_to_idx = {
+                    executor.submit(worker_func, x): idx
+                    for idx, x in enumerate(items)
+                }
+                
+                # Process results as they complete
+                for future in concurrent.futures.as_completed(future_to_idx):
+                    idx = future_to_idx[future]
+                    try:
+                        result = future.result()
+                        error_stats.record_success()
+                        results[idx] = result
+                    except Exception as e:
+                        if error_handler == 'raise':
+                            for f in future_to_idx:
+                                f.cancel()
+                            _reraise_worker_error(e, pbar, caller_info, backend='safe')
+                        error_stats.record_error(idx, e, items[idx], func_name)
+                        results[idx] = None
+                    pbar.update(1)
+                    _update_pbar_postfix(pbar)
+            
+            if _prune_dead_threads is not None:
+                _prune_dead_threads()
+        
         _cleanup_log_gate(log_gate_path)
         return results
 
@@ -692,6 +1300,8 @@ def cleanup_phantom_workers():
 
 __all__ = [
     'SPEEDY_RUNNING_PROCESSES',
+    'ErrorStats',
+    'ErrorHandlerType',
     'multi_process',
     'cleanup_phantom_workers',
     'create_progress_tracker',