lattice-sub 1.3.0__py3-none-any.whl → 1.5.3__py3-none-any.whl

{lattice_sub-1.3.0.dist-info → lattice_sub-1.5.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lattice-sub
-Version: 1.3.0
+Version: 1.5.3
 Summary: Lattice subtraction for cryo-EM micrographs - removes periodic crystal signals to reveal non-periodic features
 Author-email: George Stephenson <george.stephenson@colorado.edu>, Vignesh Kasinath <vignesh.kasinath@colorado.edu>
 License: MIT
@@ -29,6 +29,7 @@ Requires-Dist: scikit-image>=0.21
 Requires-Dist: torch>=2.0
 Requires-Dist: matplotlib>=3.7
 Requires-Dist: kornia>=0.7
+Requires-Dist: watchdog>=3.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.4; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"

lattice_sub-1.5.3.dist-info/RECORD

@@ -0,0 +1,18 @@
+lattice_sub-1.5.3.dist-info/licenses/LICENSE,sha256=2kPoH0cbEp0cVEGqMpyF2IQX1npxdtQmWJB__HIRSb0,1101
+lattice_subtraction/__init__.py,sha256=ampJE0j9aT_ENZ1Sddx041CQL2bhaDyWbkkd3WQeofg,1828
+lattice_subtraction/batch.py,sha256=G_-sxtNT46gOr22miDcG69uOnud3x6a9hEqNv4HpfwQ,20984
+lattice_subtraction/cli.py,sha256=7SZTIA2RTGFVTIMRE9ymOmKx01y1OfEkw_SNgkhykjA,27294
+lattice_subtraction/config.py,sha256=uzwKb5Zi3phHUk2ZgoiLsQdwFdN-rTiY8n02U91SObc,8426
+lattice_subtraction/core.py,sha256=VzcecSZHRuBuHUc2jHGv8LalINL75RH0aTpABI708y8,16265
+lattice_subtraction/io.py,sha256=uHku6rJ0jeCph7w-gOIDJx-xpNoF6PZcLfb5TBTOiw0,4594
+lattice_subtraction/masks.py,sha256=HIamrACmbQDkaCV4kXhnjMDSwIig4OtQFLig9A8PMO8,11741
+lattice_subtraction/processing.py,sha256=tmnj5K4Z9HCQhRpJ-iMd9Bj_uTRuvDEWyUenh8MCWEM,8341
+lattice_subtraction/threshold_optimizer.py,sha256=yEsGM_zt6YjgEulEZqtRy113xOFB69aHJIETm2xSS6k,15398
+lattice_subtraction/ui.py,sha256=-PdBSE6yF78ze85aHuGPNCziYsrQRyXQDUjYRdiTJFM,11436
+lattice_subtraction/visualization.py,sha256=noRhBXi_Xd1b5deBfVo0Bk0f3d2kqlf3_SQwAPJC0E0,12032
+lattice_subtraction/watch.py,sha256=wIXjdw0ofB71J8OvU1E5piq5BHEQG73R1VVgdzxVIow,17313
+lattice_sub-1.5.3.dist-info/METADATA,sha256=XppWofRhaYp6OMqzP862YyIWA5zek9FluCDEOFJjVOU,14930
+lattice_sub-1.5.3.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+lattice_sub-1.5.3.dist-info/entry_points.txt,sha256=o8PzJR8kFnXlKZufoYGBIHpiosM-P4PZeKZXJjtPS6Y,61
+lattice_sub-1.5.3.dist-info/top_level.txt,sha256=BOuW-sm4G-fQtsWPRdeLzWn0WS8sDYVNKIMj5I3JXew,20
+lattice_sub-1.5.3.dist-info/RECORD,,

lattice_subtraction/__init__.py

@@ -19,7 +19,7 @@ Example:
     >>> result.save("output.mrc")
 """
 
-__version__ = "1.3.0"
+__version__ = "1.5.3"
 __author__ = "George Stephenson & Vignesh Kasinath"
 
 from .config import Config
@@ -38,10 +38,12 @@ from .visualization import (
 )
 from .processing import subtract_background_gpu
 from .ui import TerminalUI, get_ui, is_interactive
+from .watch import LiveBatchProcessor, LiveStats
 
 __all__ = [
     "LatticeSubtractor",
-    "BatchProcessor", 
+    "BatchProcessor",
+    "LiveBatchProcessor",
     "Config",
     "read_mrc",
     "write_mrc",
@@ -55,5 +57,6 @@ __all__ = [
     "OptimizationResult",
     "find_optimal_threshold",
     "subtract_background_gpu",
+    "LiveStats",
     "__version__",
 ]

lattice_subtraction/batch.py

@@ -272,6 +272,19 @@ class BatchProcessor:
             available_gpus = _get_available_gpus()
             
             if len(available_gpus) > 1 and total > 1:
+                # Print GPU list
+                try:
+                    import torch
+                    from .ui import get_ui, Colors
+                    ui = get_ui(quiet=self.config._quiet)
+                    print()
+                    for gpu_id in available_gpus:
+                        gpu_name = torch.cuda.get_device_name(gpu_id)
+                        print(f"  {ui._colorize('✓', Colors.GREEN)} GPU {gpu_id}: {gpu_name}")
+                    print()
+                except Exception:
+                    pass
+                
                 # Multi-GPU processing
                 successful, failed_files = self._process_multi_gpu(
                     file_pairs, available_gpus, show_progress
@@ -407,17 +420,6 @@ class BatchProcessor:
         total = len(file_pairs)
         num_gpus = len(gpu_ids)
         
-        # Print multi-GPU info with GPU names
-        try:
-            import torch
-            gpu_names = [torch.cuda.get_device_name(i) for i in gpu_ids]
-            print(f"✓ Using {num_gpus} GPUs: {', '.join(f'GPU {i}' for i in gpu_ids)}")
-            print("")
-            for i, name in zip(gpu_ids, gpu_names):
-                print(f"  ✓ GPU {i}: {name}")
-        except Exception:
-            print(f"✓ Using {num_gpus} GPUs")
-        
         # Check GPU memory on first GPU (assume similar for all)
         if file_pairs:
             try:

lattice_subtraction/cli.py

@@ -24,6 +24,7 @@ from .batch import BatchProcessor
 from .visualization import generate_visualizations, save_comparison_visualization
 from .ui import get_ui, get_gpu_name
 from .io import read_mrc
+from .watch import LiveBatchProcessor
 
 
 # CUDA version to PyTorch index URL mapping
@@ -526,6 +527,11 @@ def process(
     is_flag=True,
     help="Force CPU processing (disable GPU auto-detection)",
 )
+@click.option(
+    "--live",
+    is_flag=True,
+    help="Watch mode: continuously monitor input directory for new files (Press Ctrl+C to stop)",
+)
 def batch(
     input_dir: str,
     output_dir: str,
@@ -541,6 +547,7 @@ def batch(
     verbose: bool,
     quiet: bool,
     cpu: bool,
+    live: bool,
 ):
     """
     Batch process a directory of micrographs.
@@ -549,6 +556,11 @@ def batch(
     INPUT_DIR: Directory containing input MRC files
     OUTPUT_DIR: Directory for processed output files
     """
+    # Validate options
+    if live and recursive:
+        click.echo("Error: --live and --recursive cannot be used together", err=True)
+        sys.exit(1)
+    
     # Initialize UI
     ui = get_ui(quiet=quiet)
     ui.print_banner()
@@ -568,8 +580,87 @@ def batch(
             backend="numpy" if cpu else "auto",
         )
     
-    # Count files first
+    # Print configuration
+    gpu_name = get_gpu_name() if not cpu else None
+    ui.print_config(cfg.pixel_ang, cfg.threshold, cfg.backend, gpu_name)
+    
     input_path = Path(input_dir)
+    output_path = Path(output_dir)
+    
+    # LIVE WATCH MODE
+    if live:
+        logger.info(f"Starting live watch mode: {input_dir} -> {output_dir}")
+        
+        # Determine number of workers
+        # For live mode, default to 1 worker to avoid GPU memory issues
+        # Files typically arrive one at a time, so parallel processing isn't needed
+        if jobs is not None:
+            num_workers = jobs
+        else:
+            num_workers = 1  # Single worker is optimal for live mode
+        
+        ui.show_watch_startup(str(input_path))
+        ui.start_timer()
+        
+        # Create live processor
+        live_processor = LiveBatchProcessor(
+            config=cfg,
+            output_prefix=prefix,
+            debounce_seconds=2.0,
+        )
+        
+        # Start watching and processing
+        stats = live_processor.watch_and_process(
+            input_dir=input_path,
+            output_dir=output_path,
+            pattern=pattern,
+            ui=ui,
+            num_workers=num_workers,
+        )
+        
+        # Print summary
+        print()  # Extra newline after counter
+        ui.print_summary(processed=stats.total_processed, failed=stats.total_failed)
+        
+        if stats.total_failed > 0:
+            ui.print_warning(f"{stats.total_failed} file(s) failed to process")
+            for file_path, error in stats.failed_files[:5]:
+                ui.print_error(f"{file_path.name}: {error}")
+            if len(stats.failed_files) > 5:
+                ui.print_error(f"... and {len(stats.failed_files) - 5} more failures")
+        
+        logger.info(f"Live mode complete: {stats.total_processed} processed, {stats.total_failed} failed")
+        
+        # Generate visualizations if requested
+        if vis and stats.total_processed > 0:
+            ui.print_info(f"Generating visualizations in: {vis}")
+            limit_msg = f" (first {num_vis})" if num_vis else ""
+            logger.info(f"Generating visualizations{limit_msg}")
+            
+            viz_success, viz_total = generate_visualizations(
+                input_dir=input_dir,
+                output_dir=output_dir,
+                viz_dir=vis,
+                prefix=prefix,
+                pattern=pattern,
+                show_progress=True,
+                limit=num_vis,
+                config=cfg,
+            )
+            logger.info(f"Visualizations: {viz_success}/{viz_total} created")
+        
+        # Exit with error code if any files failed
+        if stats.total_failed > 0:
+            sys.exit(1)
+        
+        return
+    
+    # NORMAL BATCH MODE
+    # Count files first
+    if recursive:
+        files = list(input_path.rglob(pattern))
+    else:
+        files = list(input_path.glob(pattern))
     if recursive:
         files = list(input_path.rglob(pattern))
     else:

lattice_subtraction/ui.py

@@ -11,7 +11,7 @@ suppressed to avoid polluting downstream processing.
 
 import sys
 import time
-from typing import Optional
+from typing import Optional, List
 
 
 class Colors:
@@ -230,6 +230,67 @@ class TerminalUI:
         if not self.interactive:
             return
         print(f"    {self._colorize('|-', Colors.DIM)} Saved: {path}")
+    
+    def show_watch_startup(self, input_dir: str) -> None:
+        """Show live watch mode startup message."""
+        if not self.interactive:
+            return
+        
+        print(self._colorize("  Live Watch Mode", Colors.BOLD))
+        print(self._colorize("  ---------------", Colors.DIM))
+        print(f"    Watching: {input_dir}")
+        print(f"    {self._colorize('Press Ctrl+C to stop watching and finalize', Colors.YELLOW)}")
+        print()
+    
+    def show_watch_stopped(self) -> None:
+        """Show message when watching is stopped."""
+        if not self.interactive:
+            return
+        print()  # New line after counter
+        print(f"  {self._colorize('[STOPPED]', Colors.YELLOW)} Watch mode stopped")
+        print()
+    
+    def show_live_counter_header(self) -> None:
+        """Show header for live counter display."""
+        if not self.interactive:
+            return
+        # No header needed - counter updates in place
+        pass
+    
+    def update_live_counter(self, count: int, total: int, avg_time: float, latest: str) -> None:
+        """
+        Update the live processing counter in place.
+        
+        Args:
+            count: Number of files processed
+            total: Total number of files in input directory
+            avg_time: Average processing time per file
+            latest: Name of most recently processed file
+        """
+        if not self.interactive:
+            return
+        
+        # Format time display
+        if avg_time > 0:
+            time_str = f"{avg_time:.1f}s/file"
+        else:
+            time_str = "--s/file"
+        
+        # Truncate filename if too long
+        max_filename_len = 40
+        if len(latest) > max_filename_len:
+            latest = "..." + latest[-(max_filename_len-3):]
+        
+        # Build counter line with X/Y format
+        count_str = f"{self._colorize(str(count), Colors.GREEN)}/{total}"
+        counter = f"  Processed: {count_str} files"
+        avg = f"Avg: {self._colorize(time_str, Colors.CYAN)}"
+        file_info = f"Latest: {latest}"
+        
+        line = f"{counter} | {avg} | {file_info}"
+        
+        # Print with carriage return to overwrite previous line
+        print(f"\r{line}", end="", flush=True)
 
 
 def get_ui(quiet: bool = False) -> TerminalUI:

lattice_subtraction/visualization.py

@@ -139,13 +139,15 @@ def create_comparison_figure_with_threshold(
     optimal_threshold: float,
     optimal_quality: float,
     title: str = "Lattice Subtraction Comparison",
-    figsize: Tuple[int, int] = (24, 6),
+    figsize: Tuple[int, int] = (14, 12),
     dpi: int = 150,
 ):
     """
     Create a 4-panel comparison figure with threshold optimization curve.
     
-    Layout: [Original] [Subtracted] [Difference] [Threshold vs Quality]
+    Layout (2x2 grid):
+        [Original]        [Subtracted]
+        [Difference]      [Threshold Curve]
     
     Args:
         original: Original image array
@@ -166,44 +168,44 @@ def create_comparison_figure_with_threshold(
     # Compute difference
     difference = original - processed
     
-    # Create figure with 4 panels (1 row, 4 columns)
-    fig, axes = plt.subplots(1, 4, figsize=figsize)
+    # Create figure with 4 panels (2 rows, 2 columns)
+    fig, axes = plt.subplots(2, 2, figsize=figsize)
     
     # Contrast limits from original
     vmin, vmax = np.percentile(original, [1, 99])
     
-    # Panel 1: Original
-    axes[0].imshow(original, cmap='gray', vmin=vmin, vmax=vmax)
-    axes[0].set_title(f'Original\n{original.shape}')
-    axes[0].axis('off')
+    # Panel 1 (top-left): Original
+    axes[0, 0].imshow(original, cmap='gray', vmin=vmin, vmax=vmax)
+    axes[0, 0].set_title(f'Original\n{original.shape}')
+    axes[0, 0].axis('off')
     
-    # Panel 2: Lattice Subtracted
-    axes[1].imshow(processed, cmap='gray', vmin=vmin, vmax=vmax)
-    axes[1].set_title(f'Lattice Subtracted\n{processed.shape}')
-    axes[1].axis('off')
+    # Panel 2 (top-right): Lattice Subtracted
+    axes[0, 1].imshow(processed, cmap='gray', vmin=vmin, vmax=vmax)
+    axes[0, 1].set_title(f'Lattice Subtracted\n{processed.shape}')
+    axes[0, 1].axis('off')
     
-    # Panel 3: Difference (removed lattice)
+    # Panel 3 (bottom-left): Difference (removed lattice)
     diff_std = np.std(difference)
-    axes[2].imshow(
+    axes[1, 0].imshow(
         difference, 
         cmap='RdBu_r',
         vmin=-diff_std * 3, 
         vmax=diff_std * 3
     )
-    axes[2].set_title('Difference (Removed Lattice)')
-    axes[2].axis('off')
+    axes[1, 0].set_title('Difference (Removed Lattice)')
+    axes[1, 0].axis('off')
     
-    # Panel 4: Threshold vs Quality Score curve
-    axes[3].plot(thresholds, quality_scores, 'b-', linewidth=2, label='Quality Score')
-    axes[3].axvline(x=optimal_threshold, color='r', linestyle='--', linewidth=2, 
+    # Panel 4 (bottom-right): Threshold vs Quality Score curve
+    axes[1, 1].plot(thresholds, quality_scores, 'b-', linewidth=2, label='Quality Score')
+    axes[1, 1].axvline(x=optimal_threshold, color='r', linestyle='--', linewidth=2, 
                     label=f'Optimal: {optimal_threshold:.3f}')
-    axes[3].scatter([optimal_threshold], [optimal_quality], color='r', s=100, zorder=5)
-    axes[3].set_xlabel('Threshold', fontsize=11)
-    axes[3].set_ylabel('Lattice Removal Efficacy', fontsize=11)
-    axes[3].set_title(f'Threshold Optimization\nOptimal = {optimal_threshold:.3f}')
-    axes[3].legend(loc='best', fontsize=9)
-    axes[3].grid(True, alpha=0.3)
-    axes[3].set_xlim(thresholds.min(), thresholds.max())
+    axes[1, 1].scatter([optimal_threshold], [optimal_quality], color='r', s=100, zorder=5)
+    axes[1, 1].set_xlabel('Threshold', fontsize=11)
+    axes[1, 1].set_ylabel('Lattice Removal Efficacy', fontsize=11)
+    axes[1, 1].set_title(f'Threshold Optimization\nOptimal = {optimal_threshold:.3f}')
+    axes[1, 1].legend(loc='best', fontsize=9)
+    axes[1, 1].grid(True, alpha=0.3)
+    axes[1, 1].set_xlim(thresholds.min(), thresholds.max())
     
     # Title
     plt.suptitle(title, fontsize=14)

lattice_subtraction/watch.py

@@ -0,0 +1,478 @@
+"""
+Live watch mode for processing files as they arrive.
+
+This module provides functionality for monitoring a directory and
+processing MRC files as they are created/modified, enabling real-time
+processing pipelines (e.g., from motion correction output).
+"""
+
+import logging
+import time
+import threading
+from pathlib import Path
+from typing import Set, Dict, Optional, List, Tuple
+from queue import Queue, Empty
+from dataclasses import dataclass
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler, FileSystemEvent
+
+from .config import Config
+from .core import LatticeSubtractor
+from .io import write_mrc
+from .ui import TerminalUI
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class LiveStats:
+    """Statistics for live processing."""
+    
+    total_processed: int = 0
+    total_failed: int = 0
+    total_time: float = 0.0
+    failed_files: List[Tuple[Path, str]] = None
+    
+    def __post_init__(self):
+        if self.failed_files is None:
+            self.failed_files = []
+    
+    @property
+    def average_time(self) -> float:
+        """Get average processing time per file."""
+        if self.total_processed == 0:
+            return 0.0
+        return self.total_time / self.total_processed
+    
+    def add_success(self, processing_time: float):
+        """Record a successful processing."""
+        self.total_processed += 1
+        self.total_time += processing_time
+    
+    def add_failure(self, file_path: Path, error: str):
+        """Record a failed processing."""
+        self.total_failed += 1
+        self.failed_files.append((file_path, error))
+
+
+class MRCFileHandler(FileSystemEventHandler):
+    """
+    Handles file system events for MRC files.
+    
+    Implements debouncing to ensure files are completely written before
+    processing. Files are added to a processing queue after being stable
+    for a specified duration.
+    """
+    
+    def __init__(
+        self,
+        pattern: str,
+        file_queue: Queue,
+        processed_files: Set[Path],
+        processor: 'LiveBatchProcessor',
+        debounce_seconds: float = 2.0,
+    ):
+        """
+        Initialize file handler.
+        
+        Args:
+            pattern: Glob pattern for matching files (e.g., "*.mrc")
+            file_queue: Queue to add detected files to
+            processed_files: Set of already processed file paths
+            processor: Parent LiveBatchProcessor for updating totals
+            debounce_seconds: Time to wait after last modification before processing
+        """
+        super().__init__()
+        self.pattern = pattern
+        self.file_queue = file_queue
+        self.processed_files = processed_files
+        self.processor = processor
+        self.debounce_seconds = debounce_seconds
+        
+        # Track file modification times for debouncing
+        self._pending_files: Dict[Path, float] = {}
+        self._lock = threading.Lock()
+        
+        # Start debounce checker thread
+        self._running = True
+        self._checker_thread = threading.Thread(target=self._check_pending_files, daemon=True)
+        self._checker_thread.start()
+    
+    def stop(self):
+        """Stop the debounce checker thread."""
+        self._running = False
+        if self._checker_thread.is_alive():
+            self._checker_thread.join(timeout=5.0)
+    
+    def _matches_pattern(self, path: Path) -> bool:
+        """Check if file matches the pattern."""
+        import fnmatch
+        return fnmatch.fnmatch(path.name, self.pattern)
+    
+    def _check_pending_files(self):
+        """Background thread to check for stable files ready to process."""
+        while self._running:
+            time.sleep(0.5)  # Check every 0.5 seconds
+            
+            current_time = time.time()
+            files_to_queue = []
+            
+            with self._lock:
+                # Find files that are stable (no modifications for debounce_seconds)
+                for file_path, last_mod in list(self._pending_files.items()):
+                    if current_time - last_mod >= self.debounce_seconds:
+                        files_to_queue.append(file_path)
+                        del self._pending_files[file_path]
+            
+            # Queue stable files
+            for file_path in files_to_queue:
+                if file_path not in self.processed_files and file_path.exists():
+                    logger.debug(f"Queueing stable file: {file_path}")
+                    self.file_queue.put(file_path)
+                    # Increment total count when new file is queued
+                    with self.processor._lock:
+                        self.processor.total_files += 1
+    
+    def on_created(self, event: FileSystemEvent):
+        """Handle file creation events."""
+        if event.is_directory:
+            return
+        
+        file_path = Path(event.src_path)
+        
+        if self._matches_pattern(file_path) and file_path not in self.processed_files:
+            logger.debug(f"File created: {file_path}")
+            with self._lock:
+                self._pending_files[file_path] = time.time()
+    
+    def on_modified(self, event: FileSystemEvent):
+        """Handle file modification events."""
+        if event.is_directory:
+            return
+        
+        file_path = Path(event.src_path)
+        
+        if self._matches_pattern(file_path) and file_path not in self.processed_files:
+            logger.debug(f"File modified: {file_path}")
+            with self._lock:
+                self._pending_files[file_path] = time.time()
+
+
+class LiveBatchProcessor:
+    """
+    Live batch processor that watches a directory and processes files as they arrive.
+    
+    This processor monitors an input directory for new MRC files and processes them
+    in real-time as they are created (e.g., from motion correction output).
+    
+    Features:
+    - File debouncing to ensure complete writes
+    - Real-time progress counter (instead of progress bar)
+    - Resilient error handling (continues on failures)
+    - Support for multi-GPU, single-GPU, and CPU processing
+    - Deferred visualization generation (after watching stops)
+    """
+    
+    def __init__(
+        self,
+        config: Config,
+        output_prefix: str = "sub_",
+        debounce_seconds: float = 2.0,
+    ):
+        """
+        Initialize live batch processor.
+        
+        Args:
+            config: Processing configuration
+            output_prefix: Prefix for output filenames
+            debounce_seconds: Time to wait after file modification before processing
+        """
+        self.config = config
+        self.output_prefix = output_prefix
+        self.debounce_seconds = debounce_seconds
+        
+        # Processing state
+        self.file_queue: Queue = Queue()
+        self.processed_files: Set[Path] = set()
+        self.stats = LiveStats()
+        self.total_files: int = 0  # Total files in input directory
+        
+        # Worker threads
+        self._workers: List[threading.Thread] = []
+        self._running = False
+        self._lock = threading.Lock()
+        
+        # File system watcher
+        self.observer: Optional[Observer] = None
+        self.handler: Optional[MRCFileHandler] = None
+    
+    def _create_subtractor(self, device_id: Optional[int] = None) -> LatticeSubtractor:
+        """Create a LatticeSubtractor instance with optional device override."""
+        if device_id is not None:
+            # Create config copy with specific device
+            from dataclasses import replace
+            config = replace(self.config, device_id=device_id)
+        else:
+            config = self.config
+        
+        # Enable quiet mode to suppress GPU messages on each file
+        config._quiet = True
+        
+        return LatticeSubtractor(config)
+    
+    def _process_worker(
+        self,
+        output_dir: Path,
+        ui: TerminalUI,
+        device_id: Optional[int] = None,
+    ):
+        """
+        Worker thread that processes files from the queue.
+        
+        Args:
+            output_dir: Output directory for processed files
+            ui: Terminal UI for displaying progress
+            device_id: Optional GPU device ID (for multi-GPU)
+        """
+        # Set CUDA device for this thread if GPU is being used
+        if device_id is not None:
+            import torch
+            torch.cuda.set_device(device_id)
+            logger.debug(f"Worker initialized on GPU {device_id}")
+        
+        while self._running:
+            try:
+                # Get file from queue with timeout
+                file_path = self.file_queue.get(timeout=0.5)
+            except Empty:
+                continue
+            
+            # Create subtractor on-demand for each file to avoid memory buildup
+            subtractor = self._create_subtractor(device_id)
+            
+            # Process the file
+            output_name = f"{self.output_prefix}{file_path.name}"
+            output_path = output_dir / output_name
+            
+            start_time = time.time()
+            
+            try:
+                result = subtractor.process(file_path)
+                result.save(output_path, pixel_size=self.config.pixel_ang)
+                
+                processing_time = time.time() - start_time
+                
+                with self._lock:
+                    self.stats.add_success(processing_time)
+                    self.processed_files.add(file_path)
+                
+                # Update UI counter
+                ui.update_live_counter(
+                    count=self.stats.total_processed,
+                    total=self.total_files,
+                    avg_time=self.stats.average_time,
+                    latest=file_path.name,
+                )
+                
+                # Don't log to console in live mode - it breaks the in-place counter update
+            
+            except Exception as e:
+                with self._lock:
+                    self.stats.add_failure(file_path, str(e))
+                    self.processed_files.add(file_path)  # Don't retry
+                
+                logger.error(f"Failed to process {file_path.name}: {e}")
+            
+            finally:
+                # Clean up memory after each file in live mode
+                del subtractor
+                if device_id is not None:
+                    import torch
+                    torch.cuda.empty_cache()
+                
+                self.file_queue.task_done()
+    
+    def watch_and_process(
+        self,
+        input_dir: Path,
+        output_dir: Path,
+        pattern: str,
+        ui: TerminalUI,
+        num_workers: int = 1,
+    ) -> LiveStats:
+        """
+        Start watching directory and processing files as they arrive.
+        
+        This method blocks until KeyboardInterrupt (Ctrl+C) is received.
+        
+        Args:
+            input_dir: Directory to watch for new files
+            output_dir: Output directory for processed files
+            pattern: Glob pattern for matching files (e.g., "*.mrc")
+            ui: Terminal UI for displaying progress
+            num_workers: Number of processing workers (for multi-GPU or CPU)
+            
+        Returns:
+            LiveStats with processing statistics
+        """
+        # Create output directory
+        output_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Check for existing files in directory
+        all_files = list(input_dir.glob(pattern))
+        
+        # If files already exist, process them with batch mode first (multi-GPU)
+        if all_files:
+            from .batch import BatchProcessor
+            
+            ui.print_info(f"Found {len(all_files)} existing files - processing with batch mode first")
+            
+            # Create file pairs for batch processing
+            file_pairs = []
+            for file_path in all_files:
+                output_name = f"{self.output_prefix}{file_path.name}"
+                output_path = output_dir / output_name
+                file_pairs.append((file_path, output_path))
+                self.processed_files.add(file_path)  # Mark as processed
+            
+            # Process with BatchProcessor (will use multi-GPU if available)
+            batch_processor = BatchProcessor(
+                config=self.config,
+                num_workers=num_workers,
+                output_prefix="",  # Already included in output paths
+            )
+            
+            result = batch_processor.process_directory(
+                input_dir=input_dir,
+                output_dir=output_dir,
+                pattern=pattern,
+                recursive=False,
+                show_progress=True,
+            )
+            
+            # Update stats with batch results
+            self.stats.total_processed = result.successful
+            self.stats.total_failed = result.failed
+            self.total_files = len(all_files)  # Set initial total
+            
+            ui.print_info(f"Batch processing complete: {result.successful}/{result.total} files")
+            print()
+            
+            # Check if any new files arrived during batch processing
+            current_files = set(input_dir.glob(pattern))
+            new_during_batch = current_files - set(all_files)
+            if new_during_batch:
+                ui.print_info(f"Found {len(new_during_batch)} files added during batch processing - queueing now")
+                for file_path in new_during_batch:
+                    if file_path not in self.processed_files:
+                        self.file_queue.put(file_path)
+                        self.total_files += 1  # Increment total for each new file
+        else:
+            # No existing files, start fresh
+            self.total_files = 0
+        
+        # Setup file system watcher
+        self.handler = MRCFileHandler(
+            pattern=pattern,
+            file_queue=self.file_queue,
+            processed_files=self.processed_files,
+            processor=self,
+            debounce_seconds=self.debounce_seconds,
+        )
+        
+        self.observer = Observer()
+        self.observer.schedule(self.handler, str(input_dir), recursive=False)
+        self.observer.start()
+        
+        # Determine GPU setup for workers
+        device_ids = self._get_worker_devices(num_workers)
+        
+        # Print GPU list at startup (non-dynamic, just info)
+        if device_ids and device_ids[0] is not None:
+            try:
+                import torch
+                from .ui import Colors
+                unique_gpus = sorted(set(d for d in device_ids if d is not None))
+                print()
+                for gpu_id in unique_gpus:
+                    gpu_name = torch.cuda.get_device_name(gpu_id)
+                    print(f"  {ui._colorize('✓', Colors.GREEN)} GPU {gpu_id}: {gpu_name}")
+                print()
+            except Exception as e:
+                pass  # Silently skip if GPU info unavailable
+        
+        # Start processing workers
+        self._running = True
+        for i, device_id in enumerate(device_ids):
+            worker = threading.Thread(
+                target=self._process_worker,
+                args=(output_dir, ui, device_id),
+                daemon=True,
+            )
+            worker.start()
+            self._workers.append(worker)
+        
+        # Show initial counter
+        ui.show_live_counter_header()
+        ui.update_live_counter(count=0, total=self.total_files, avg_time=0.0, latest="waiting...")
+        
+        # Wait for interrupt
+        try:
+            while True:
+                time.sleep(1.0)
+        except KeyboardInterrupt:
+            ui.show_watch_stopped()
+        
+        # Cleanup
+        self._shutdown(ui)
+        
+        return self.stats
+    
+    def _get_worker_devices(self, num_workers: int) -> List[Optional[int]]:
+        """
+        Determine device IDs for workers based on available GPUs.
+        
+        Returns:
+            List of device IDs (None for CPU workers)
+        """
+        # Check if CPU-only mode is forced
+        if self.config.backend == "numpy":
+            return [None] * num_workers
+        
+        # Check for GPU availability
+        try:
+            import torch
+            if torch.cuda.is_available():
+                gpu_count = torch.cuda.device_count()
+                
+                if gpu_count > 1 and num_workers > 1:
+                    # Multi-GPU: assign workers to GPUs in round-robin
+                    return [i % gpu_count for i in range(num_workers)]
+                else:
+                    # Single GPU: all workers use GPU 0
+                    return [0] * num_workers
+        except ImportError:
+            pass
+        
+        # CPU mode: all workers use None (CPU)
+        return [None] * num_workers
+    
+    def _shutdown(self, ui: TerminalUI):
+        """Shutdown workers and observer cleanly."""
+        # Stop accepting new files
+        if self.observer:
+            self.observer.stop()
+            self.observer.join(timeout=5.0)
+        
+        if self.handler:
+            self.handler.stop()
+        
+        # Wait for queue to be processed
+        ui.print_info("Processing remaining queued files...")
+        self.file_queue.join()
+        
+        # Stop workers
+        self._running = False
+        for worker in self._workers:
+            worker.join(timeout=5.0)

lattice_sub-1.3.0.dist-info/RECORD

@@ -1,17 +0,0 @@
-lattice_sub-1.3.0.dist-info/licenses/LICENSE,sha256=2kPoH0cbEp0cVEGqMpyF2IQX1npxdtQmWJB__HIRSb0,1101
-lattice_subtraction/__init__.py,sha256=TNaJXvSgCQdvYUYJfS5scn92YjORiGfLot9WadZ8u28,1737
-lattice_subtraction/batch.py,sha256=zJzvUnr8dznvxE8jaPKDLJ7AcJg8Cbfv5nVo0FzZz1I,20891
-lattice_subtraction/cli.py,sha256=W99XQClUMKaaFQxle0W-ILQ6UuYRFXZVJWD4qXpcIj4,24063
-lattice_subtraction/config.py,sha256=uzwKb5Zi3phHUk2ZgoiLsQdwFdN-rTiY8n02U91SObc,8426
-lattice_subtraction/core.py,sha256=VzcecSZHRuBuHUc2jHGv8LalINL75RH0aTpABI708y8,16265
-lattice_subtraction/io.py,sha256=uHku6rJ0jeCph7w-gOIDJx-xpNoF6PZcLfb5TBTOiw0,4594
-lattice_subtraction/masks.py,sha256=HIamrACmbQDkaCV4kXhnjMDSwIig4OtQFLig9A8PMO8,11741
-lattice_subtraction/processing.py,sha256=tmnj5K4Z9HCQhRpJ-iMd9Bj_uTRuvDEWyUenh8MCWEM,8341
-lattice_subtraction/threshold_optimizer.py,sha256=yEsGM_zt6YjgEulEZqtRy113xOFB69aHJIETm2xSS6k,15398
-lattice_subtraction/ui.py,sha256=Sp_a-yNmBRZJxll8h9T_H5-_KsI13zGYmHcbcpVpbR8,9176
-lattice_subtraction/visualization.py,sha256=hWFz49NBBrS7d6ofO0VyJ6-v8Q6hPG1dijbDtecMOQs,11890
-lattice_sub-1.3.0.dist-info/METADATA,sha256=pKwt8TcftbZGm1gvWZGO1n3iQiI4JB3E_ix3InB-4D0,14901
-lattice_sub-1.3.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-lattice_sub-1.3.0.dist-info/entry_points.txt,sha256=o8PzJR8kFnXlKZufoYGBIHpiosM-P4PZeKZXJjtPS6Y,61
-lattice_sub-1.3.0.dist-info/top_level.txt,sha256=BOuW-sm4G-fQtsWPRdeLzWn0WS8sDYVNKIMj5I3JXew,20
-lattice_sub-1.3.0.dist-info/RECORD,,