cdo-toolkit 0.1.0__py3-none-any.whl

cdo_toolkit/pipeline.py

@@ -0,0 +1,2230 @@
+"""CDO regridding pipeline core."""
+
+import hashlib
+import logging
+import multiprocessing as mp
+import os
+import shutil
+import signal
+import tempfile
+import threading
+import time
+import traceback
+import uuid
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import psutil
+import xarray as xa
+from cdo import Cdo
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+try:
+    import fcntl
+except ImportError:
+    fcntl = None
+
+from cdo_toolkit.constants import NC4_ENCODING_KEYS, REGRID_ERROR_LOGGER_NAME
+from cdo_toolkit.errors import init_regrid_error_log, log_regrid_error, weight_file_lock
+from cdo_toolkit.memory import MemoryMonitor
+from cdo_toolkit.paths import is_weights_or_cache_file, weight_cache_dir_for_input
+from cdo_toolkit.resolution import calc_resolution
+from cdo_toolkit.timing import format_processing_time, get_processing_time, print_timestamp
+from cdo_toolkit.ui import BatchRegridUI, RegridProgressUI
+from cdo_toolkit.cmip import representative_files_by_directory
+from cdo_toolkit.workers import poll_batch_progress, process_single_file_standalone
+
+class CDORegridPipeline:
+    """
+    CDO regridding pipeline with all advanced features.
+    """
+    
+    def __init__(
+        self,
+        target_resolution: tuple[float, float] = (1.0, 1.0),    # TODO: automate this based on native resolution of input file
+        target_grid: str = "lonlat",
+        weight_cache_dir: Optional[Path] = None,
+        extract_surface: bool = False,
+        extract_seafloor: bool = False,
+        use_regrid_cache: bool = True,
+        use_seafloor_cache: bool = True,
+        verbose: bool = True,
+        verbose_diagnostics: bool = False,
+        max_memory_gb: float = 8.0,
+        max_workers: Optional[int] = 16,
+        chunk_size_gb: float = 2.0,
+        enable_parallel: bool = True,
+        enable_chunking: bool = True,
+        memory_monitoring: bool = True,
+        cleanup_weights: bool = False,
+    ):
+        """
+        Initialize the CDO regridding pipeline.
+
+        Pipeline behaviour:
+        - If neither extract_surface nor extract_seafloor: regrid the whole file (use weight cache if use_regrid_cache).
+        - If extract_seafloor: identify seafloor indices (from cache if use_seafloor_cache), extract seafloor, regrid only that.
+        - If extract_surface: extract top level and regrid only that (use weight cache if use_regrid_cache).
+        - If both extract_surface and extract_seafloor: perform seafloor then surface sequentially (use regrid_single_file twice or regrid_single_file_extreme_levels).
+
+        Parameters
+        ----------
+        target_resolution (tuple): Target resolution as (lon_res, lat_res) in degrees.
+        target_grid (str): Target grid type ('lonlat', 'gaussian', etc.).
+        weight_cache_dir (Path, optional): Directory to cache regrid weights.
+        extract_surface (bool): If True, extract top level only and regrid that.
+        extract_seafloor (bool): If True, extract seafloor values (deepest non-NaN) and regrid only that.
+        use_regrid_cache (bool): If True, reuse existing regrid weight files when present.
+        use_seafloor_cache (bool): If True, reuse in-memory seafloor depth indices for files in the same directory.
+        verbose (bool): Enable verbose output (progress UI, etc.).
+        verbose_diagnostics (bool): If True, print Grid type, File size, Levels, Large file messages (max verbosity).
+        max_memory_gb (float): Maximum memory usage in GB.
+        max_workers (int, optional): Maximum number of parallel workers.
+        chunk_size_gb (float): Maximum chunk size in GB for large files.
+        enable_parallel (bool): Enable parallel processing.
+        enable_chunking (bool): Enable chunked processing for large files.
+        memory_monitoring (bool): Enable memory usage monitoring.
+        """
+        self.target_resolution = target_resolution
+        self.target_grid = target_grid
+        self.extract_surface = extract_surface
+        self.extract_seafloor = extract_seafloor
+        self.use_regrid_cache = use_regrid_cache
+        self.use_seafloor_cache = use_seafloor_cache
+        self.verbose = verbose
+        self.verbose_diagnostics = verbose_diagnostics
+        self.max_memory_gb = max_memory_gb
+        self.chunk_size_gb = chunk_size_gb
+        self.enable_parallel = enable_parallel
+        self.enable_chunking = enable_chunking
+        self.memory_monitoring = memory_monitoring
+        self.cleanup_weights = cleanup_weights
+        self.prune_regridded = True
+        
+        # Cache for seafloor depth indices per directory (for optimization)
+        self._seafloor_depth_cache: dict[str, dict[str, int]] = {}
+                
+        # ensure not requesting more workers than available
+        if max_workers is None:
+            self.max_workers = min(self.max_workers, mp.cpu_count())
+        else:
+            self.max_workers = max_workers
+        
+        # set up CDO with optimized settings
+        self.cdo = self._setup_cdo()
+        
+        # set up console and logger for output
+        self.console = Console()
+        self.logger = self._setup_logger()
+        self._error_log_path: Optional[Path] = None
+
+        # weight cache management
+        self.weight_cache_dir = Path(weight_cache_dir) if weight_cache_dir else None
+        if self.weight_cache_dir:
+            self.weight_cache_dir.mkdir(parents=True, exist_ok=True)
+        self.weight_cache: dict[str, Path] = {}
+        if self.weight_cache_dir and self.cleanup_weights:
+            self.cleanup_weight_files()
+        
+        # monitor memory to prevent antisocial behaviour on shared machines andout of memory errors
+        self.memory_monitor = MemoryMonitor() if memory_monitoring else None
+        
+        # prepare to store meta-data and processing statistics
+        self.stats = {
+            'files_processed': 0,
+            'weights_reused': 0,
+            'weights_generated': 0,
+            'errors': 0,
+            'total_size_gb': 0.0,
+            'chunks_processed': 0,
+            'memory_peak_gb': 0.0,
+            'grid_types': {
+                'structured': 0,
+                'curvilinear': 0,
+                'tripolar_ocean': 0,
+                'unstructured_ncells': 0,
+                'unknown': 0,
+            }
+        }
+        
+        # cache for file info to avoid repeated expensive operations
+        self._file_info_cache: dict[Path, dict] = {}
+        
+        # track created files for cleanup on interrupt
+        self._created_files: list[Path] = []
+        self._progress_state = None
+        self._progress_key: Optional[str] = None
+        self._setup_signal_handlers()
+        
+        # timing tracking
+        self._start_time: Optional[time.struct_time] = None
+        self.end_time: Optional[time.struct_time] = None
+        
+    def _get_target_variable(self, file_path: Path) -> str:
+        """Get the target variable for the file from the file_path. N.B. this is specific to CMIP data"""
+        return file_path.stem.split("_")[0]
+        
+    def _prune_regridded(self, input_files: list[Path], overwrite: bool = False) -> list[Path]:
+        """Prune files with 'regridded' in the name to avoid processing them twice."""
+        # ignore any files which have 'cdo_weights' as any parent directory
+        input_files = [file for file in input_files if 'cdo_weights' not in file.parents]
+        # TODO: if ever getting levels from a regridded file will need to rethink this
+        if self.prune_regridded:
+            if overwrite:   # delete existing regridded files
+                for file in input_files:
+                    if 'regridded' in file.name:
+                        if self.verbose:
+                            self.console.print(f"[yellow]Removing existing regridded file: {file.name}[/yellow]")
+                        file.unlink()
+            else:   # skip existing regridded files (don't regenerate)
+                if self.verbose:
+                    regridded_files = [file for file in input_files if 'regridded' in file.name]
+                    if regridded_files:
+                        self.console.print(f"[blue]Skipping {len(regridded_files)} existing regridded files (overwrite=False)[/blue]")
+        return [file for file in input_files if 'regridded' in file.name and not "_chunk_" in file.name]
+    
+    def _cleanup_files_by_pattern(
+        self, 
+        input_files: list[Path], 
+        pattern: str,
+        file_type: str,
+        exclude_regridded: bool = False
+    ) -> list[Path]:
+        """Generic helper to clean up files matching a pattern.
+        
+        Args:
+            input_files: List of file paths to check
+            pattern: Pattern to match in filename (e.g., '_top_level', '_chunk_')
+            file_type: Description for logging (e.g., '_top_level', '_chunk_')
+            exclude_regridded: If True, exclude files with 'regridded' in name
+            
+        Returns:
+            List of cleaned files (with problematic files removed)
+        """
+        cleaned_files = []
+        removed_count = 0
+        
+        for file_path in input_files:
+            should_remove = pattern in file_path.name
+            if exclude_regridded and should_remove:
+                should_remove = "regridded" not in file_path.name
+            
+            if should_remove:
+                if self.verbose:
+                    self.console.print(
+                        f"[yellow]Removing problematic {file_type} file: {file_path.name}[/yellow]"
+                    )
+                try:
+                    file_path.unlink()
+                    removed_count += 1
+                except Exception as e:
+                    self.logger.warning(f"Could not remove {file_path}: {e}")
+            else:
+                cleaned_files.append(file_path)
+        
+        if removed_count > 0 and self.verbose:
+            self.console.print(
+                f"[blue]Cleaned up {removed_count} problematic {file_type} files[/blue]"
+            )
+        
+        return cleaned_files
+    
+    def _cleanup_top_level_files(self, input_files: list[Path]) -> list[Path]:
+        """Clean up existing _top_level files that may cause HDF errors."""
+        # TODO: why do I need to get rid of top_level files?
+        return self._cleanup_files_by_pattern(
+            input_files, 
+            pattern='_top_level',
+            file_type='_top_level',
+            exclude_regridded=True
+        )
+    
+    def _cleanup_chunk_files(self, input_files: list[Path]) -> list[Path]:
+        """Clean up existing _chunk_ files that may cause HDF errors."""
+        return self._cleanup_files_by_pattern(
+            input_files,
+            pattern='_chunk_',
+            file_type='_chunk_',
+            exclude_regridded=False
+        )
+    
+    def _cleanup_problematic_files(self, input_files: list[Path]) -> list[Path]:
+        """Clean up all problematic files (_top_level and _chunk_) that may cause HDF errors.
+        Returns the list of input files that remain for processing (non-problematic).
+        """
+        remaining = self._cleanup_files_by_pattern(
+            input_files,
+            pattern='_top_level',
+            file_type='_top_level',
+            exclude_regridded=True  # keep regridded top_level files (final form)
+        )
+        return self._cleanup_files_by_pattern(
+            remaining,
+            pattern='_chunk_',
+            file_type='_chunk_',
+            exclude_regridded=False  # get rid of everything that's still a chunk
+        )
+    
+    def _setup_signal_handlers(self):
+        """Set up signal handlers for graceful cleanup on keyboard interrupt."""
+        import signal
+        
+        def signal_handler(signum, frame):
+            """Handle keyboard interrupt and cleanup created files."""
+            if self.verbose:
+                self.console.print(f"\n[yellow]Received interrupt signal ({signum}). Cleaning up...[/yellow]")
+            
+            self._cleanup_created_files()
+            
+            if self.verbose:
+                self.console.print(f"[red]Interrupted. Cleaned up {len(self._created_files)} created files.[/red]")
+            
+            # exit gracefully
+            import sys
+            sys.exit(1)
+        
+        # register signal handlers
+        signal.signal(signal.SIGINT, signal_handler)  # Ctrl+C
+        signal.signal(signal.SIGTERM, signal_handler)  # termination signal
+    
+    def _cleanup_created_files(self):
+        """Clean up all files created during processing."""
+        for file_path in self._created_files:
+            try:
+                if file_path.exists():
+                    file_path.unlink()
+                    if self.verbose_diagnostics:
+                        self.logger.info(f"Cleaned up created file: {file_path.name}")
+            except Exception as e:
+                self.logger.warning(f"Could not clean up {file_path}: {e}")
+        
+        self._created_files.clear()
+    
+    def _track_created_file(self, file_path: Path):
+        """Track a file that was created during processing for cleanup."""
+        self._created_files.append(file_path)
+    
+    def _setup_cdo(self) -> Cdo:
+        """Set up CDO with optimized performance settings."""
+        import os
+        
+        # set CDO environment variables for optimal performance
+        os.environ["CDO_NETCDF_COMPRESSION"] = "0"  # disable compression for speed TODO: compress if space becomes an issue
+        os.environ["CDO_NETCDF_64BIT_OFFSET"] = "1"  # use 64-bit offsets for large files
+        os.environ["CDO_NETCDF_USE_PARALLEL"] = "1" if self.enable_parallel else "0" # enable parallel I/O 
+        os.environ["CDO_NUM_THREADS"] = str(self.max_workers)
+        
+        return Cdo()
+    
+    def _setup_logger(self) -> logging.Logger:
+        """Pipeline logger; errors go to the shared session log via log_regrid_error."""
+        return logging.getLogger(REGRID_ERROR_LOGGER_NAME)
+
+    def set_error_log_path(self, path: Path) -> None:
+        """Ensure the shared regrid error log is initialized for this run."""
+        self._error_log_path = init_regrid_error_log(path)
+
+    def _log_error(self, message: str, exc: Optional[BaseException] = None) -> None:
+        log_regrid_error(message, exc=exc)
+    
+    def _detect_grid_type(self, ds: xa.Dataset, dims: dict) -> str:
+        """
+        Detect the type of grid structure with comprehensive support.
+        
+        Returns:
+        - 'unstructured_ncells': Grid with ncells dimension (true unstructured)
+        - 'tripolar_ocean': Tri-polar ocean grid with (y,x) dimensions, 2D lat/lon coords, and cell boundaries
+        - 'curvilinear': Grid with (j,i) dimensions and 2D lat/lon coordinates
+        - 'structured': Regular lon/lat grid with 1D coordinates
+        - 'unknown': Cannot determine grid type
+        """
+        # check for true unstructured grid (ncells dimension)
+        if 'ncells' in dims:
+            return 'unstructured_ncells'
+        
+        # check for tri-polar ocean grid (y,x dimensions with 2D lat/lon coords and cell boundaries)
+        if 'y' in dims and 'x' in dims:
+            lat_coords = ['lat', 'latitude']
+            lon_coords = ['lon', 'longitude']
+            
+            has_2d_lat = any(coord in ds.coords and ds[coord].ndim == 2 for coord in lat_coords)
+            has_2d_lon = any(coord in ds.coords and ds[coord].ndim == 2 for coord in lon_coords)
+            
+            if has_2d_lat and has_2d_lon:
+                # Check for cell boundaries (indicative of tri-polar ocean grids)
+                has_bounds_lat = ('bounds_lat' in list(ds.coords) or 'bounds_lat' in list(ds.data_vars))
+                has_bounds_lon = ('bounds_lon' in list(ds.coords) or 'bounds_lon' in list(ds.data_vars))
+                has_nvertex = 'nvertex' in dims
+                
+                # Check for ocean-specific variables
+                ocean_vars = ['so', 'thetao', 'uo', 'vo', 'wo', 'zos', 'mlotst', 'sithick']
+                has_ocean_vars = any(var in ds.data_vars for var in ocean_vars)
+                
+                if (has_bounds_lat and has_bounds_lon and has_nvertex) or has_ocean_vars:
+                    return 'tripolar_ocean'
+                else:
+                    return 'curvilinear'
+        
+        # check for curvilinear grid (j,i dimensions with 2D lat/lon coords)
+        if 'j' in dims and 'i' in dims:
+            # check if lat/lon are 2D coordinates on (j,i) grid
+            lat_coords = ['lat', 'latitude']
+            lon_coords = ['lon', 'longitude']
+            
+            has_2d_lat = any(coord in ds.coords and ds[coord].ndim == 2 for coord in lat_coords)
+            has_2d_lon = any(coord in ds.coords and ds[coord].ndim == 2 for coord in lon_coords)
+            
+            if has_2d_lat and has_2d_lon:
+                return 'curvilinear'
+        
+        # check for structured grid (1D lat/lon coordinates)
+        lat_coords = ['lat', 'latitude']
+        lon_coords = ['lon', 'longitude']
+        
+        has_1d_lat = any(coord in ds.coords and ds[coord].ndim == 1 for coord in lat_coords)
+        has_1d_lon = any(coord in ds.coords and ds[coord].ndim == 1 for coord in lon_coords)
+        
+        if has_1d_lat and has_1d_lon:
+            return 'structured'
+        
+        raise ValueError(f"Could not determine grid type for file TODO")     # TODO: get access to file_path to raise error
+
+    def _has_grid_corners(self, ds: xa.Dataset) -> bool:
+        """Return True when CDO can use conservative remapping (cell corners present)."""
+        names = set(ds.coords) | set(ds.data_vars)
+        lat_bounds = {"bounds_lat", "lat_bnds", "vertices_latitude"} & names
+        lon_bounds = {"bounds_lon", "lon_bnds", "vertices_longitude"} & names
+        if lat_bounds and lon_bounds:
+            return True
+
+        for lat_name, lon_name in (("lat", "lon"), ("latitude", "longitude")):
+            if lat_name not in ds.coords or lon_name not in ds.coords:
+                continue
+            lat_bnds = ds[lat_name].attrs.get("bounds")
+            lon_bnds = ds[lon_name].attrs.get("bounds")
+            if lat_bnds in names and lon_bnds in names:
+                return True
+        return False
+
+    def _regrid_operators(self, grid_type: str, has_grid_corners: bool) -> tuple[str, str]:
+        """Return CDO (remap, gen_weights) operators for this source grid."""
+        if grid_type == "structured" or has_grid_corners:
+            return "remapcon", "gencon"
+        return "remapbil", "genbil"
+
+    def _whether_multi_level(self, dims: dict) -> tuple[bool, list[str], int]:
+        """Whether the file has a multi-level dimension.
+        
+        Returns:
+        - has_level (bool): Whether the file has level dimensions
+        - level_dims (list[str]): List of level dimension names found
+        - level_count (int): Total number of level dimensions
+        """
+        level_dims = ['lev', 'level', 'depth', 'z']
+        has_level = any(dim in dims for dim in level_dims)
+        level_count = sum(dims.get(dim, 0) for dim in level_dims)
+        return has_level, level_dims, level_count
+    
+    def _has_level_lightweight(self, file_path: Path) -> bool:
+        """Lightweight check for level dimensions without opening the full dataset.
+        
+        This method only reads the dimensions from the NetCDF file header,
+        avoiding the expensive operation of loading the full dataset.
+        
+        Args:
+            file_path (Path): Path to the NetCDF file
+            
+        Returns:
+            bool: True if the file has level dimensions, False otherwise
+        """
+        try:
+            # Use xarray's minimal loading to get just dimensions
+            with xa.open_dataset(file_path, decode_times=False) as ds:
+                dims = dict(ds.sizes)
+                has_level, _, _ = self._whether_multi_level(dims)
+                return has_level
+        except Exception as e:
+            self.logger.warning(f"Could not check levels in {file_path}: {e}")
+            return False
+    
+    def _get_file_info(self, file_path: Path) -> dict:
+        """Get comprehensive information about a file with caching.
+        
+        Returns:
+        - 'file_size_gb': File size in GB
+        - 'dims': Dimensions of the file
+        - 'has_level': Whether the file has a level dimension
+        - 'level_count': Number of level dimensions
+        - 'coords': Coordinate information
+        - 'grid_type': Grid type
+        - 'estimated_memory_gb': Estimated memory usage in GB
+        - 'time_steps': Number of time steps
+        """
+        # check cache first
+        if file_path in self._file_info_cache:
+            return self._file_info_cache[file_path]
+        
+        try:
+            ds = xa.open_dataset(file_path, decode_times=False)
+            
+            # get metadata
+            file_size_gb = file_path.stat().st_size / (1024**3) # faster than ds.nbytes
+            dims = dict(ds.sizes)    
+            has_level, _, level_count = self._whether_multi_level(dims)
+
+            # get coordinate information
+            coords_info = {}
+            try:
+                for coord in ['lon', 'longitude', 'lat', 'latitude']:
+                    if coord in ds.coords:
+                        coord_data = ds[coord]
+                        coords_info[coord] = {
+                            'shape': coord_data.shape,
+                            'ndim': coord_data.ndim,
+                            'size': coord_data.size,
+                        }
+            except Exception as coord_error:
+                self.logger.warning(f"Could not get coordinate info for {file_path}: {coord_error}")
+                coords_info = {}
+            
+            # determine grid type
+            try:
+                grid_type = self._detect_grid_type(ds, dims)
+            except Exception as grid_error:
+                self.logger.warning(f"Could not detect grid type for {file_path}: {grid_error}")
+                grid_type = 'unknown'
+
+            has_grid_corners = (
+                grid_type == "structured" or self._has_grid_corners(ds)
+            )
+            
+            # estimate memory usage required for regridding
+            estimated_memory_gb = file_size_gb * 3  # rough but conservative estimate: when regridding will have original, multiple, and maybe an intermediate array in memory
+            
+            file_info = {
+                'file_size_gb': file_size_gb,
+                'dims': dims,
+                'has_level': has_level,
+                'level_count': level_count,
+                'coords': coords_info,
+                'grid_type': grid_type,
+                'has_grid_corners': has_grid_corners,
+                'estimated_memory_gb': estimated_memory_gb,
+                'time_steps': dims.get('time', 1),
+            }
+            
+            # cache the result
+            self._file_info_cache[file_path] = file_info
+            return file_info
+            
+        except Exception as e:
+            self.logger.warning(f"Could not analyze file {file_path}: {e}")
+            error_info = {
+                'file_size_gb': 0.0,
+                'dims': {},
+                'has_level': False,
+                'level_count': 0,
+                'coords': {},
+                'grid_type': 'unknown',
+                'has_grid_corners': False,
+                'estimated_memory_gb': 0.0,
+                'time_steps': None,
+            }
+            # cache the error result too to avoid repeated failures
+            self._file_info_cache[file_path] = error_info
+            return error_info
+    
+    def _get_grid_signature(self, file_info: dict) -> str:
+        """Generate a unique signature for the grid based on file info to avoid duplicating weights."""
+        remap_op, gen_op = self._regrid_operators(
+            file_info['grid_type'],
+            file_info.get('has_grid_corners', False),
+        )
+        signature_data = {
+            'coords': file_info['coords'],
+            'dims': file_info['dims'],
+            'grid_type': file_info['grid_type'],
+            'remap_op': remap_op,
+            'gen_op': gen_op,
+        }
+        
+        signature_str = str(sorted(signature_data.items()))
+        return hashlib.md5(signature_str.encode()).hexdigest()[:12]
+    
+    def _get_weight_path(self, grid_signature: str) -> Path:
+        """Get the path for regrid weights based on grid signature."""
+        if self.weight_cache_dir is None:
+            raise RuntimeError("weight_cache_dir is not set")
+        return self.weight_cache_dir / f"weights_{grid_signature}.nc"
+
+    def _cleanup_stale_chunks_for_file(self, file_path: Path) -> None:
+        """Remove leftover chunk files from prior failed runs of the same source file."""
+        stem = file_path.stem.replace("_top_level", "").replace("_seafloor", "")
+        for chunk in file_path.parent.glob(f"{stem}_chunk_*.nc"):
+            try:
+                chunk.unlink()
+            except OSError as e:
+                self._log_error(f"Could not remove stale chunk {chunk}: {e}")
+
+    def _is_valid_weight_file(self, weight_path: Path) -> bool:
+        """Return False for missing, empty, or corrupt cached weight files."""
+        if not weight_path.exists() or weight_path.stat().st_size == 0:
+            return False
+        try:
+            with xa.open_dataset(weight_path, decode_times=False) as ds:
+                return len(ds.dims) > 0
+        except Exception:
+            return False
+
+    def _invalidate_weight_file(self, weight_path: Path) -> None:
+        if weight_path.exists():
+            try:
+                weight_path.unlink()
+            except OSError:
+                pass
+    
+    def _generate_output_filename(
+        self,
+        input_path: Path,
+        has_level: bool,
+        extract_surface: bool = False,
+        extract_seafloor: bool = False,
+    ) -> str:
+        """Generate output filename by modifying the input filename.
+
+        Args:
+            input_path (Path): Path to the input file
+            has_level (bool): Whether the source has a level/depth dimension
+            extract_surface (bool): Whether we extracted top level only
+            extract_seafloor (bool): Whether we extracted seafloor only
+
+        Returns:
+            str: Generated output filename (e.g. name_regridded.nc, name_top_level_regridded.nc, name_seafloor_regridded.nc)
+        """
+        name = input_path.name
+        if extract_seafloor and "_seafloor" not in name:
+            name = name.replace(input_path.suffix, "_seafloor" + input_path.suffix)
+        if extract_surface and "_top_level" not in name and has_level:
+            name = name.replace(input_path.suffix, "_top_level" + input_path.suffix)
+        return name.replace(input_path.suffix, "_regridded" + input_path.suffix)
+    
+    def _get_representative_file(self, input_files: list[Path]) -> Optional[Path]:
+        """Get a representative file from a list of files for resolution calculation."""
+        return pick_representative_file(input_files)
+    
+    def _calculate_target_resolution(self, input_file: Path) -> tuple[float, float]:
+        """Calculate target resolution based on dataset's nominal_resolution attribute.
+        
+        Args:
+            input_file (Path): Path to the input file to analyze
+            
+        Returns:
+            tuple[float, float]: (lon_res, lat_res) in degrees, or (9999.0, 9999.0) if calculation fails
+        """
+        try:
+            with xa.open_dataset(input_file, decode_times=False) as ds:
+                # check if nominal_resolution attribute exists
+                if 'nominal_resolution' not in ds.attrs:
+                    self.logger.warning(f"No 'nominal_resolution' attribute found in {input_file.name}")
+                    return (9999.0, 9999.0)
+                
+                # get native resolution
+                native_res = ds.attrs['nominal_resolution']
+                if self.verbose_diagnostics:
+                    self.logger.info(f"Found nominal_resolution: {native_res}")
+                
+                # calculate target resolution
+                target_res = calc_resolution(native_res)
+                
+                if target_res == 9999.0:
+                    self.logger.warning(f"Could not parse nominal_resolution '{native_res}' from {input_file.name}")
+                    return (9999.0, 9999.0)
+                
+                # return same resolution for both lon and lat (regular grid)
+                return (target_res, target_res)
+                
+        except Exception as e:
+            self.logger.warning(f"Error calculating target resolution from {input_file.name}: {e}")
+            return (9999.0, 9999.0)
+    
+    def _generate_target_grid_description(self, representative_file: Optional[Path] = None) -> str:
+        """Generate CDO target grid description.
+        
+        Args:
+            representative_file (Path, optional): File to use for resolution calculation.
+                                                 If None, uses the pipeline's target_resolution.
+        """
+        # try to calculate target resolution from a representative file
+        if representative_file and representative_file.exists():
+            lon_res, lat_res = self._calculate_target_resolution(representative_file)
+            if lon_res == 9999.0 or lat_res == 9999.0:
+                if (
+                    hasattr(self, "_representative_file")
+                    and self._representative_file
+                    and self._representative_file.exists()
+                    and self._representative_file != representative_file
+                ):
+                    lon_res, lat_res = self._calculate_target_resolution(self._representative_file)
+                if lon_res == 9999.0 or lat_res == 9999.0:
+                    lon_res, lat_res = self.target_resolution
+                    if self.verbose_diagnostics:
+                        self.console.print(f"[yellow]Using pipeline target resolution: {lon_res}° x {lat_res}°[/yellow]")
+            else:
+                if self.verbose_diagnostics:
+                    self.console.print(f"[green]Calculated target resolution from {representative_file.name}: {lon_res:.3f}° x {lat_res:.3f}° (to 3 decimal places)[/green]")
+        elif hasattr(self, '_representative_file') and self._representative_file and self._representative_file.exists():
+            # use the pipeline's representative file
+            lon_res, lat_res = self._calculate_target_resolution(self._representative_file)
+            if lon_res == 9999.0 or lat_res == 9999.0:
+                # fall back to pipeline's target resolution
+                lon_res, lat_res = self.target_resolution
+                if self.verbose_diagnostics:
+                    self.console.print(f"[yellow]Using pipeline target resolution: {lon_res}° x {lat_res}°[/yellow]")
+            else:
+                if self.verbose_diagnostics:
+                    self.console.print(f"[green]Calculated target resolution from {self._representative_file.name}: {lon_res}° x {lat_res}°[/green]")
+        else:
+            # use pipeline's target resolution
+            lon_res, lat_res = self.target_resolution
+            if self.verbose_diagnostics:
+                self.console.print(f"[blue]Using pipeline target resolution: {lon_res}° x {lat_res}°[/blue]")
+        
+        if self.target_grid == "lonlat":
+            # regular lon/lat grid
+            xsize = int(360 / lon_res)
+            ysize = int(180 / lat_res)
+            xfirst = -180 + lon_res / 2 # TODO: check this logic
+            yfirst = -90 + lat_res / 2 # TODO: the extent of the grid is not always -180 to 180 and -90 to 90: does this matter? Hard to say, since trying to get everything onto the same grid. Does CDO account for this?
+            
+            return f"""gridtype = lonlat
+xsize = {xsize}
+ysize = {ysize}
+xfirst = {xfirst}
+xinc = {lon_res}
+yfirst = {yfirst}
+yinc = {lat_res}"""
+        
+        elif self.target_grid == "gaussian":    # equally spaced along latitude, unequally along longitude
+            # Gaussian grid
+            ysize = int(180 / lat_res)
+            return f"""gridtype = gaussian
+ysize = {ysize}"""
+        
+        else:
+            raise ValueError(f"Unsupported target grid type: {self.target_grid}")
+    def _should_chunk_file(self, file_info: dict) -> bool:
+        """Determine if a file should be processed in chunks based on file size, memory usage, and time steps.
+        
+        Returns (bool): True if the file should be processed in chunks, False otherwise
+        """
+        if not self.enable_chunking:
+            return False
+        
+        return (
+            file_info['file_size_gb'] > self.chunk_size_gb or
+            file_info['estimated_memory_gb'] > self.max_memory_gb or
+            file_info['time_steps'] > 100
+        )
+    
+    def _notify_progress(
+        self,
+        phase: str,
+        *,
+        chunks_done: int = 0,
+        chunks_total: int = 0,
+        pct: Optional[int] = None,
+        ui: Optional["RegridProgressUI"] = None,
+        ui_input_path: Optional[Path] = None,
+        regrid_mode: str = "complete",
+    ) -> None:
+        """Report per-file progress to the local UI and/or shared parallel-worker state."""
+        ui_path = ui_input_path
+        if chunks_total > 0:
+            if ui is not None and ui_path is not None:
+                ui.update_chunk_progress(
+                    ui_path, chunks_done, chunks_total, phase=phase, regrid_mode=regrid_mode
+                )
+            if self._progress_state is not None and self._progress_key:
+                self._progress_state[self._progress_key] = {
+                    "phase": phase,
+                    "chunks_done": chunks_done,
+                    "chunks_total": chunks_total,
+                }
+        elif pct is not None:
+            if ui is not None and ui_path is not None:
+                ui.update_file_progress(ui_path, pct, phase, regrid_mode=regrid_mode)
+            if self._progress_state is not None and self._progress_key:
+                self._progress_state[self._progress_key] = {
+                    "phase": phase,
+                    "chunks_done": pct,
+                    "chunks_total": 0,
+                }
+
+    def _chunk_file_by_time(
+        self,
+        file_path: Path,
+        chunk_size: int = 10,
+        name_suffix: Optional[str] = None,
+        ui: Optional["RegridProgressUI"] = None,
+        ui_input_path: Optional[Path] = None,
+        regrid_mode: str = "complete",
+    ) -> list[Path]:
+        """Split a file into time chunks; chunk files are written to a writable directory.
+
+        When name_suffix is set (e.g. '_top_level', '_seafloor'), chunk stems use it so
+        names stay consistent with extract_surface/extract_seafloor even when preparation
+        returned the original file (no level dim or preparation failed).
+
+        Returns (list[Path]): List of paths to the chunked files
+        """
+        ds = None
+        chunk_files = []
+        try:
+            ds = xa.open_dataset(file_path, decode_times=False)
+            
+            # Check if file has time dimension and enough time steps to chunk
+            if 'time' not in ds.dims:
+                self.logger.warning(f"File {file_path.name} has no 'time' dimension, skipping chunking")
+                return [file_path]
+            
+            time_length = len(ds.time)
+            if time_length <= chunk_size:
+                if self.verbose:
+                    self.logger.debug(f"File {file_path.name} has {time_length} time steps (<= {chunk_size}), skipping chunking")
+                return [file_path]
+            
+            time_chunks = list(range(0, time_length, chunk_size))
+            total_chunks = len(time_chunks)
+            _ui_path = ui_input_path if ui_input_path is not None else file_path
+            self._notify_progress(
+                "creating",
+                chunks_done=0,
+                chunks_total=total_chunks,
+                ui=ui,
+                ui_input_path=_ui_path,
+                regrid_mode=regrid_mode,
+            )
+
+            # Chunk stem: use name_suffix when in surface/seafloor mode so names are e.g.
+            # *_chunk_000_top_level even if prepared_path was the original file.
+            if name_suffix:
+                base_stem = (
+                    file_path.stem.replace("_top_level", "").replace("_seafloor", "").rstrip("_")
+                    or file_path.stem
+                )
+                chunk_stem_template = f"{base_stem}_chunk_{{i:03d}}{name_suffix}"
+            elif "_top_level" in file_path.stem:
+                base_stem = file_path.stem.replace("_top_level", "")
+                chunk_stem_template = f"{base_stem}_chunk_{{i:03d}}_top_level"
+            elif "_seafloor" in file_path.stem:
+                base_stem = file_path.stem.replace("_seafloor", "")
+                chunk_stem_template = f"{base_stem}_chunk_{{i:03d}}_seafloor"
+            else:
+                chunk_stem_template = f"{file_path.stem}_chunk_{{i:03d}}"
+
+            # Encoding for chunk writes: preserve source encoding (netCDF4-compatible keys only) and avoid _FillValue on coords/bounds.
+            # Omit chunksizes so chunk time dimension (smaller than source) does not trigger "chunksize cannot exceed dimension size".
+            chunk_encoding = {}
+            for v in ds.variables:
+                raw = getattr(ds[v], "encoding", None) or {}
+                enc = {k: raw[k] for k in (raw if isinstance(raw, dict) else {}) if k in NC4_ENCODING_KEYS and k != "chunksizes"}
+                chunk_encoding[v] = enc
+                if v in ds.coords or v in ("lat_bnds", "lon_bnds", "lat", "lon", "time", "time_bnds"):
+                    chunk_encoding[v]["_FillValue"] = None
+            
+            # Write chunks to a writable directory (input dir may be read-only, e.g. shared data)
+            out_dir = file_path.parent if os.access(file_path.parent, os.W_OK) else Path(tempfile.gettempdir())
+            if out_dir != file_path.parent:
+                # Unique suffix so concurrent runs don't collide when using system temp
+                unique_suffix = f"_{os.getpid()}_{abs(hash(file_path)) % 1000000:06d}"
+            else:
+                unique_suffix = ""
+
+            # Create chunks
+            for i, start_idx in enumerate(time_chunks):
+                end_idx = min(start_idx + chunk_size, time_length)
+                ds_chunk = ds.isel(time=slice(start_idx, end_idx))
+                
+                # Verify chunk has data before writing
+                if ds_chunk.sizes.get('time', 0) == 0:
+                    self.logger.warning(f"Skipping empty chunk {i} for {file_path.name}")
+                    continue
+                
+                chunk_path = out_dir / f"{chunk_stem_template.format(i=i)}{unique_suffix}{file_path.suffix}"
+                
+                # Write chunk file (same encoding as source so CDO accepts seafloor/top_level chunks)
+                try:
+                    ds_chunk.to_netcdf(chunk_path, encoding=chunk_encoding)
+                    
+                    # Verify chunk file was created and is not empty
+                    if not chunk_path.exists() or chunk_path.stat().st_size == 0:
+                        self._log_error(f"Chunk file {chunk_path.name} is empty or was not created")
+                        if chunk_path.exists():
+                            chunk_path.unlink()
+                        continue
+                    
+                    # Track the created chunk file for cleanup
+                    self._track_created_file(chunk_path)
+                    chunk_files.append(chunk_path)
+                    self._notify_progress(
+                        "creating",
+                        chunks_done=len(chunk_files),
+                        chunks_total=total_chunks,
+                        ui=ui,
+                        ui_input_path=_ui_path,
+                        regrid_mode=regrid_mode,
+                    )
+
+                except Exception as chunk_error:
+                    self._log_error(
+                        f"Failed to write chunk {i} for {file_path.name}: {chunk_error}",
+                        exc=chunk_error if isinstance(chunk_error, BaseException) else None,
+                    )
+                    # Clean up failed chunk file if it was partially created
+                    if chunk_path.exists():
+                        try:
+                            chunk_path.unlink()
+                        except Exception:
+                            pass
+                    raise  # Re-raise to trigger cleanup
+            
+            # If no chunks were created successfully, return original file
+            if not chunk_files:
+                self.logger.warning(f"No chunks created for {file_path.name}, returning original file")
+                return [file_path]
+            
+            return chunk_files
+            
+        except Exception as e:
+            self._log_error(f"Failed to chunk file {file_path}: {e}", exc=e)
+            
+            # Clean up any chunk files that were created before the error
+            for chunk_file in chunk_files:
+                try:
+                    if chunk_file.exists() and chunk_file != file_path:
+                        self.logger.warning(f"Cleaning up incomplete chunk file: {chunk_file.name}")
+                        chunk_file.unlink()
+                except Exception as cleanup_error:
+                    self._log_error(
+                        f"Failed to clean up chunk file {chunk_file.name}: {cleanup_error}",
+                        exc=cleanup_error,
+                    )
+            
+            return [file_path]
+        finally:
+            # Always close the dataset if it was opened
+            if ds is not None:
+                try:
+                    ds.close()
+                except Exception:
+                    pass
+    
+    def _extract_seafloor_values(
+        self,
+        file_path: Path,
+        ui: Optional["RegridProgressUI"] = None,
+        ui_input_path: Optional[Path] = None,
+    ) -> Path:
+        """
+        Extract seafloor values (deepest non-NaN values along depth dimension) from a file.
+        Uses cached depth indices for files in the same directory for optimization.
+        For large files, uses chunked reading and reports progress when ui is provided.
+
+        Args:
+            file_path (Path): Path to the input file
+            ui: Optional progress UI (updates with phases: depth indices, extracting vars, writing)
+            ui_input_path: Path to show in UI (usually same as file_path)
+
+        Returns (Path): Path to the seafloor-extracted file
+        """
+        _ui_path = ui_input_path if ui_input_path is not None else file_path
+
+        def _progress(pct: int, msg: str) -> None:
+            if ui is not None:
+                ui.update_file_progress(_ui_path, pct, msg, regrid_mode="seafloor")
+            self.logger.info(f"Seafloor extraction: {msg}")
+
+        try:
+            # Write to a writable dir (temp if input dir is read-only)
+            out_dir = file_path.parent if os.access(file_path.parent, os.W_OK) else Path(tempfile.gettempdir())
+            seafloor_path = out_dir / f"{file_path.stem}_seafloor{file_path.suffix}"
+            if seafloor_path.exists():
+                # Validate existing file: re-extract if empty or no data vars (e.g. corrupt or from old code)
+                try:
+                    with xa.open_dataset(seafloor_path, decode_times=False) as existing:
+                        if existing.data_vars and all(existing.sizes.get(d, 0) > 0 for d in next(iter(existing.data_vars.values())).dims):
+                            if self.verbose:
+                                self.console.print(f"[cyan]Using existing seafloor file: {seafloor_path.name}[/cyan]")
+                            return seafloor_path
+                except Exception:
+                    pass
+                seafloor_path.unlink(missing_ok=True)
+                if self.verbose:
+                    self.console.print(f"[yellow]Re-extracting seafloor (existing file invalid or empty)[/yellow]")
+
+            target_variable = self._get_target_variable(file_path)
+            # Chunked read for large files: avoids loading the full 3D/4D array at once
+            ds = xa.open_dataset(file_path, decode_times=False, chunks="auto")
+            has_level, level_dims, level_count = self._whether_multi_level(dict(ds.sizes))
+            
+            if not has_level or level_count == 0:
+                # No depth dimension, return original file
+                return file_path
+            
+            # Find the level dimension that exists in the dataset
+            level_dim = None
+            for dim in level_dims:
+                if dim in ds.dims:
+                    level_dim = dim
+                    break
+            
+            if level_dim is None:
+                return file_path
+
+
+            # Get or compute depth indices for this directory (use cache only if use_seafloor_cache)
+            dir_key = str(file_path.parent)
+            cache_key = f"{level_dim}_{ds.sizes[level_dim]}"
+            use_cache = self.use_seafloor_cache
+            if dir_key not in self._seafloor_depth_cache:
+                self._seafloor_depth_cache[dir_key] = {}
+            cache_hit = use_cache and (cache_key in self._seafloor_depth_cache[dir_key])
+
+            if not cache_hit:
+                # Compute seafloor depth indices: find deepest non-NaN value for each spatial location
+                _progress(11, "Seafloor: computing depth indices (may take several minutes for large files)")
+                if self.verbose:
+                    self.console.print(f"[blue]Computing seafloor depth indices for {file_path.name}...[/blue]")
+
+                # Get a representative data variable that has the level dimension
+                # (exclude mesh/coord vars that may have only ncells, vertices)
+                data_vars_with_level = [
+                    v for v in ds.data_vars
+                    if level_dim in ds[v].dims
+                ]
+                if not data_vars_with_level:
+                    self.logger.warning(
+                        f"No data variables with level dimension '{level_dim}' in {file_path.name}"
+                    )
+                    ds.close()
+                    return file_path
+                # var_name = data_vars_with_level[0]  # TODO: surely this should be the target variable (ie uo)?
+                var_data = ds[target_variable]
+                # Take first time step only if the variable has a time dimension
+                if "time" in var_data.dims:
+                    var_data = var_data.isel(time=0)
+                
+                # Find deepest non-NaN index along level dimension
+                level_size = ds.sizes[level_dim]
+                
+                # Get spatial dimensions (all dims except level, time, bnds)
+                spatial_dims = [d for d in var_data.dims if d not in [level_dim, 'time', 'bnds']]   # with bnds was returning with a bnds coordinate. May be other rogue variables out there
+                
+                if spatial_dims:
+                    # For each spatial location, find deepest non-NaN value
+                    # Create a mask of valid (non-NaN) values using DataArray's isnull method
+                    valid_mask = ~var_data.isnull()
+                    
+                    # Check which locations have any valid values
+                    has_valid = valid_mask.any(dim=level_dim)
+                    
+                    # Find the deepest valid index for each spatial location
+                    # Reverse along level dimension and find first valid (which is deepest)
+                    # argmax returns index of first True (deepest valid)
+                    reversed_valid = valid_mask.isel({level_dim: slice(None, None, -1)})
+                    reversed_argmax = reversed_valid.argmax(dim=level_dim)
+                    
+                    # Convert back to original indexing (deepest = level_size - 1 - reversed_index)
+                    deepest_idx = level_size - 1 - reversed_argmax
+                    
+                    # Handle cases where all values are NaN (set to last index)
+                    # Also check if argmax returned 0 (which could mean all False or first is True)
+                    # If no valid values exist, argmax will return 0, but we need to check if that's valid
+                    all_nan_mask = ~has_valid
+                    # For locations with no valid values, use last index
+                    deepest_idx = deepest_idx.where(~all_nan_mask, level_size - 1)
+                    
+                    # Convert to numpy array for indexing (triggers chunked compute if dask)
+                    depth_indices_array = deepest_idx.values
+                else:
+                    # No spatial dimensions (unlikely but handle it)
+                    # Just find deepest non-NaN across all data
+                    valid_mask = ~var_data.isnull()
+                    if valid_mask.any():
+                        # Find deepest valid index
+                        reversed_valid = valid_mask.isel({level_dim: slice(None, None, -1)})
+                        reversed_argmax = reversed_valid.argmax(dim=level_dim)
+                        deepest_idx = level_size - 1 - reversed_argmax
+                        depth_indices_array = int(deepest_idx.values) if hasattr(deepest_idx, 'values') else int(deepest_idx)
+                    else:
+                        # All NaN, use last index
+                        depth_indices_array = level_size - 1
+                
+                # Cache the depth indices
+                if use_cache:
+                    self._seafloor_depth_cache[dir_key][cache_key] = depth_indices_array
+                _progress(12, "Seafloor: depth indices done" + (" (cached)" if use_cache else ""))
+                if self.verbose:
+                    self.console.print(f"[green]Computed seafloor depth indices[/green]" + (" (cached)" if use_cache else ""))
+            else:
+                # Use cached depth indices
+                depth_indices_array = self._seafloor_depth_cache[dir_key][cache_key]
+                _progress(12, "Seafloor: using cached depth indices")
+                if self.verbose:
+                    self.console.print(f"[cyan]Using cached seafloor depth indices[/cyan]")
+                # spatial_dims needed when depth_indices_array is an array; derive from first data var with level_dim
+                if not (isinstance(depth_indices_array, (int, np.integer)) or np.isscalar(depth_indices_array)):
+                    data_vars_with_level = [v for v in ds.data_vars if level_dim in ds[v].dims]
+                    if data_vars_with_level:
+                        spatial_dims = [d for d in ds[data_vars_with_level[0]].dims if d not in [level_dim, "time"]]
+                    else:
+                        spatial_dims = []
+
+            # Extract seafloor values: same structural approach as top_level (isel one level, drop level dim).
+            # Build seafloor dataset from the source dataset so the written file has the same layout CDO
+            # expects (identical to top_level: data at one level, written with plain to_netcdf).
+            if self.verbose:
+                self.console.print(f"[blue]Extracting seafloor values from {file_path.name}...[/blue]")
+
+            _progress(15, "Seafloor: writing file")
+
+            # Copy dataset and apply seafloor selection to each variable that has level_dim
+            seafloor_ds = ds.copy(deep=False)
+            if isinstance(depth_indices_array, (int, np.integer)) or np.isscalar(depth_indices_array):
+                seafloor_ds = seafloor_ds.isel({level_dim: int(depth_indices_array)})
+            else:
+                depth_idx_da = xa.DataArray(
+                    depth_indices_array,
+                    dims=spatial_dims,
+                    coords={d: seafloor_ds[target_variable].coords[d] for d in spatial_dims if d in seafloor_ds[target_variable].coords},
+                )
+                for var_name in list(seafloor_ds.data_vars):
+                    if level_dim in seafloor_ds[var_name].dims:
+                        seafloor_ds[var_name] = seafloor_ds[var_name].isel({level_dim: depth_idx_da})
+                seafloor_ds = seafloor_ds.drop_vars(level_dim, errors="ignore")
+            seafloor_ds = seafloor_ds.load()
+
+            # Write next to original file, same way as top_level: plain to_netcdf (no encoding/engine/format)
+            write_dir = seafloor_path.parent if os.access(seafloor_path.parent, os.W_OK) else Path(tempfile.gettempdir())
+            write_path = write_dir / (seafloor_path.name + f".tmp.{os.getpid()}.{uuid.uuid4().hex[:8]}")
+            try:
+                seafloor_ds.to_netcdf(write_path)
+                write_path.rename(seafloor_path)
+            finally:
+                if write_path.exists():
+                    write_path.unlink(missing_ok=True)
+            ds.close()
+            seafloor_ds.close()
+            
+            # Track the created file for cleanup
+            self._track_created_file(seafloor_path)
+            
+            if self.verbose:
+                self.console.print(f"[green]Extracted seafloor values: {seafloor_path.name}[/green]")
+            
+            return seafloor_path
+            
+        except Exception as e:
+            self._log_error(f"Failed to extract seafloor values from {file_path}: {e}", exc=e)
+            # Return None or raise exception to indicate failure
+            # Returning original file_path would be misleading
+            raise RuntimeError(f"Seafloor extraction failed for {file_path.name}: {e}") from e
+    
+    def _prepare_file_for_regridding(
+        self,
+        file_path: Path,
+        ui: Optional["RegridProgressUI"] = None,
+        ui_input_path: Optional[Path] = None,
+    ) -> Path:
+        """
+        Prepare a file for regridding according to pipeline mode.
+
+        - If extract_seafloor: extract seafloor values (use seafloor cache if use_seafloor_cache), return path to seafloor file.
+        - If extract_surface: extract top level only, return path to top-level file.
+        - If neither: return file_path (regrid whole file).
+
+        ui / ui_input_path: optional progress UI; if set, seafloor extraction will update progress (e.g. "Seafloor: computing depth indices").
+        Returns (Path): Path to prepared file (possibly a temporary/prepared NetCDF).
+        """
+        if self.extract_seafloor:
+            if "_seafloor" in file_path.stem:
+                return file_path
+            try:
+                prepared_path = self._extract_seafloor_values(
+                    file_path, ui=ui, ui_input_path=ui_input_path
+                )
+                return prepared_path
+            except Exception as e:
+                self._log_error(f"Seafloor extraction failed, cannot proceed with regridding: {e}", exc=e)
+                raise
+
+        if not self.extract_surface:
+            return file_path
+
+        try:
+            if "_top_level" in file_path.stem:  # TODO: is top level always correct for the sea surface value?
+                return file_path
+            ds = xa.open_dataset(file_path, decode_times=False)
+            has_level, level_dims, level_count = self._whether_multi_level(dict(ds.sizes))
+            if not isinstance(level_dims, list):
+                self._log_error(f"level_dims is not a list: {type(level_dims)} = {level_dims}")
+                return file_path
+            if not has_level or level_count == 0:
+                return file_path
+            for dim in level_dims:
+                if dim in ds.dims:
+                    ds = ds.isel({dim: 0})
+                    break
+            # Write to a writable dir (temp if input dir is read-only). Use a unique
+            # path per process so concurrent workers never write the same file (avoids
+            # PermissionError and races when the same source is processed in parallel).
+            prep_dir = file_path.parent if os.access(file_path.parent, os.W_OK) else Path(tempfile.gettempdir())
+            unique_suffix = f"_{os.getpid()}_{uuid.uuid4().hex[:8]}"
+            prepared_path = prep_dir / f"{file_path.stem}_top_level{unique_suffix}{file_path.suffix}"
+            ds.to_netcdf(prepared_path)
+            self._track_created_file(prepared_path)
+            if self.verbose_diagnostics:
+                self.console.print(f"[cyan]Prepared file (top level): {prepared_path.name}[/cyan]")
+                self.logger.info(f"Prepared file (top level): {prepared_path.name}")
+            return prepared_path
+        except Exception as e:
+            self.logger.warning(f"Could not prepare file {file_path}: {e}")
+            self._log_error(f"Could not prepare file {file_path}: {e}", exc=e)
+            return file_path
+
+    def _ensure_top_level_file_for_chunking(self, file_path: Path) -> Path:
+        """When chunking in extract_surface mode, ensure we chunk a top-level-only file.
+
+        If preparation was skipped (prepared_path == input_path) but the file has multiple
+        levels, chunk files would be named *_top_level but contain all levels. This helper
+        extracts top level to a temp file when needed so chunk content matches the name.
+        """
+        try:
+            with xa.open_dataset(file_path, decode_times=False) as ds:
+                has_level, level_dims, level_count = self._whether_multi_level(dict(ds.sizes))
+                if not has_level or level_count <= 1:
+                    return file_path
+                for dim in level_dims:
+                    if dim in ds.dims:
+                        ds = ds.isel({dim: 0})
+                        break
+                prep_dir = file_path.parent if os.access(file_path.parent, os.W_OK) else Path(tempfile.gettempdir())
+                unique_suffix = f"_{os.getpid()}_{uuid.uuid4().hex[:8]}"
+                top_path = prep_dir / f"{file_path.stem}_top_level{unique_suffix}{file_path.suffix}"
+                ds.to_netcdf(top_path)
+                self._track_created_file(top_path)
+                self.logger.info(
+                    f"Created top-level temp file for chunking (was multi-level): {top_path.name}"
+                )
+                return top_path
+        except Exception as e:
+            self.logger.warning(f"Could not create top-level file for chunking: {e}")
+            return file_path
+
+    def _is_valid_prepared_file(self, prepared_path: Path) -> bool:
+        """Return False if prepared file is missing, empty, or has no data variables (CDO would fail)."""
+        if not prepared_path.exists():
+            self.logger.warning(f"Prepared file does not exist: {prepared_path}")
+            return False
+        if prepared_path.stat().st_size == 0:
+            self.logger.warning(
+                f"Prepared file is empty (0 bytes), skipping regridding: {prepared_path.name}"
+            )
+            return False
+        try:
+            with xa.open_dataset(prepared_path, decode_times=False) as ds:
+                if not ds.data_vars:
+                    self.logger.warning(
+                        f"Prepared file has no data variables, skipping regridding: {prepared_path.name}"
+                    )
+                    return False
+        except Exception as e:
+            self.logger.warning(
+                f"Prepared file could not be read or has unsupported structure: {prepared_path.name} ({e})"
+            )
+            return False
+        return True
+
+    def _regrid_chunked_file(
+        self,
+        input_path: Path,
+        output_path: Path,
+        grid_signature: str,
+        grid_type: str,
+        chunk_files: list[Path],
+        weight_cache_dir: Optional[Path] = None,
+        ui: Optional["RegridProgressUI"] = None,
+        ui_input_path: Optional[Path] = None,
+        regrid_mode: str = "complete",
+    ) -> bool:
+        """Regrid a file that has been split into chunks, with optional parallel processing.
+        
+        Args:
+        - input_path (Path): Path to the input file
+        - output_path (Path): Path to the output file
+        - grid_signature (str): Unique signature for the grid
+        - grid_type (str): Type of grid
+        - chunk_files (list[Path]): List of paths to the chunked files
+        
+        Returns (bool): True if successful, False otherwise
+        """
+        try:
+            if weight_cache_dir is not None:
+                self.weight_cache_dir = Path(weight_cache_dir)
+                self.weight_cache_dir.mkdir(parents=True, exist_ok=True)
+
+            weight_path = self._get_weight_path(grid_signature)
+            grid_desc = self._generate_target_grid_description(input_path)
+
+            with tempfile.TemporaryDirectory() as tmpdir:
+                tmpdir = Path(tmpdir)
+                grid_file = tmpdir / "target_grid.txt"
+
+                with open(grid_file, 'w') as f:
+                    f.write(grid_desc)
+
+                weights_exist = (
+                    self.use_regrid_cache and self._is_valid_weight_file(weight_path)
+                )
+
+                if not weights_exist and len(chunk_files) > 0:
+                    if self.verbose_diagnostics:
+                        self.console.print("[blue]Generating weights from first chunk...[/blue]")
+                    first_chunk_output = tmpdir / "chunk_000.nc"
+                    if not self._regrid_without_weights(
+                        chunk_files[0], first_chunk_output, grid_file, grid_type
+                    ):
+                        self._log_error(
+                            f"Failed to regrid first chunk for weights: {chunk_files[0]}"
+                        )
+                        return False
+                    if not self._save_weights(chunk_files[0], weight_path, grid_file):
+                        self._log_error(f"Failed to save weights to {weight_path}")
+                        return False
+                    weights_exist = self._is_valid_weight_file(weight_path)
+                    if not weights_exist:
+                        self._log_error(f"Weight file invalid after save: {weight_path}")
+                        return False
+                    if self.verbose_diagnostics:
+                        self.console.print("[green]Weights generated and saved[/green]")
+
+                chunk_outputs = []
+                total_chunks = len(chunk_files)
+                _ui_path = ui_input_path if ui_input_path is not None else input_path
+                self._notify_progress(
+                    "regridding",
+                    chunks_done=0,
+                    chunks_total=total_chunks,
+                    ui=ui,
+                    ui_input_path=_ui_path,
+                    regrid_mode=regrid_mode,
+                )
+
+                for i, chunk_file in enumerate(chunk_files):
+                    chunk_output = tmpdir / f"chunk_{i:03d}.nc"
+
+                    if weights_exist:
+                        if not self._regrid_with_weights(
+                            chunk_file, chunk_output, grid_file, weight_path, grid_type
+                        ):
+                            self._log_error(
+                                f"Failed to regrid chunk {i} with weights: {chunk_file}"
+                            )
+                            return False
+                    elif not self._regrid_without_weights(
+                        chunk_file, chunk_output, grid_file, grid_type
+                    ):
+                        self._log_error(f"Failed to regrid chunk {i} without weights: {chunk_file}")
+                        return False
+
+                    chunk_outputs.append(chunk_output)
+                    self.stats['chunks_processed'] += 1
+                    self._notify_progress(
+                        "regridding",
+                        chunks_done=i + 1,
+                        chunks_total=total_chunks,
+                        ui=ui,
+                        ui_input_path=_ui_path,
+                        regrid_mode=regrid_mode,
+                    )
+
+                # combine chunks into single file
+                if len(chunk_outputs) > 1:
+                    self._notify_progress(
+                        "combining",
+                        chunks_done=total_chunks,
+                        chunks_total=total_chunks,
+                        ui=ui,
+                        ui_input_path=_ui_path,
+                        regrid_mode=regrid_mode,
+                    )
+                    if self.verbose_diagnostics:
+                        self.console.print(f"[blue]Combining {len(chunk_outputs)} chunks...[/blue]")
+                    self._combine_chunks(chunk_outputs, output_path)
+                else:
+                    shutil.copy2(chunk_outputs[0], output_path)
+                
+                # clean up chunk files
+                for chunk_file in chunk_files:
+                    if chunk_file != input_path and chunk_file.exists():
+                        try:
+                            chunk_file.unlink()
+                            self.console.print(f"[green]Cleaned up chunk file: {chunk_file.name}[/green]") if self.verbose_diagnostics else None
+                        except Exception as cleanup_error:
+                            self.console.print(f"[red]Failed to clean up chunk file {chunk_file.name}: {cleanup_error}[/red]") if self.verbose_diagnostics else None
+                
+                return True
+                
+        except Exception as e:
+            self._log_error(f"Failed to regrid chunked file {input_path}: {e}", exc=e)
+            
+            # Clean up chunk files even if regridding failed
+            for chunk_file in chunk_files:
+                if chunk_file != input_path and chunk_file.exists():
+                    try:
+                        self.logger.warning(f"Cleaning up chunk file after error: {chunk_file.name}")
+                        chunk_file.unlink()
+                    except Exception as cleanup_error:
+                        self._log_error(
+                        f"Failed to clean up chunk file {chunk_file.name}: {cleanup_error}",
+                        exc=cleanup_error,
+                    )
+            
+            return False
+
+    def _combine_chunks(self, chunk_outputs: list[Path], output_path: Path):
+        """Combine regridded chunks into a single file.
+        
+        Args:
+        - chunk_outputs (list[Path]): List of paths to the regridded chunks
+        - output_path (Path): Path to the output file
+        
+        Returns (bool): True if successful, False otherwise
+        """
+        try:
+            # load all chunks   # TODO: xa.open_mfdataset complains about dask not being installed
+            datasets = [xa.open_dataset(chunk) for chunk in chunk_outputs]
+            combined = xa.concat(datasets, dim='time')  # combine along time dimension
+            combined.to_netcdf(output_path)
+            # close datasets to free memory
+            for ds in datasets:
+                ds.close()
+                
+        except Exception as e:
+            self._log_error(f"Failed to combine chunks: {e}", exc=e)
+            return False
+    
+    def _regrid_with_weights(
+        self,
+        input_path: Path,
+        output_path: Path,
+        grid_file: Path,
+        weight_path: Path,
+        grid_type: str = "structured",
+        _allow_regenerate: bool = True,
+    ) -> bool:
+        """Regrid using existing weights; regenerate once if cache is missing or corrupt."""
+        try:
+            with weight_file_lock(weight_path):
+                if not self._is_valid_weight_file(weight_path):
+                    if not _allow_regenerate:
+                        return False
+                    self._invalidate_weight_file(weight_path)
+                    if not self._regrid_without_weights(
+                        input_path, output_path, grid_file, grid_type
+                    ):
+                        return False
+                    self._save_weights(input_path, weight_path, grid_file)
+                    return output_path.exists() and output_path.stat().st_size > 0
+
+                self.cdo.remap(
+                    str(grid_file),
+                    str(weight_path),
+                    input=str(input_path),
+                    output=str(output_path),
+                )
+            return True
+        except Exception as e:
+            err = str(e).lower()
+            corrupt = (
+                "hdf error" in err
+                or "nc_open failed" in err
+                or "no such file" in err
+            )
+            if corrupt:
+                self._invalidate_weight_file(weight_path)
+            self._log_error(f"Failed to regrid {input_path} with weights: {e}", exc=e)
+            if corrupt and _allow_regenerate:
+                return self._regrid_with_weights(
+                    input_path,
+                    output_path,
+                    grid_file,
+                    weight_path,
+                    grid_type=grid_type,
+                    _allow_regenerate=False,
+                )
+            return False
+    
+    def _regrid_without_weights(self, input_path: Path, output_path: Path, grid_file: Path, grid_type: str):
+        """Regrid without existing weights; conservative if corners exist, else bilinear."""
+        file_info = self._get_file_info(input_path)
+        has_corners = file_info.get(
+            "has_grid_corners", grid_type == "structured"
+        )
+        remap_op, _ = self._regrid_operators(grid_type, has_corners)
+        try:
+            getattr(self.cdo, remap_op)(
+                str(grid_file),
+                input=str(input_path),
+                output=str(output_path),
+            )
+            return True
+        except Exception as e:
+            err = str(e).lower()
+            if remap_op == "remapcon" and "corner coordinates missing" in err:
+                try:
+                    self.cdo.remapbil(
+                        str(grid_file),
+                        input=str(input_path),
+                        output=str(output_path),
+                    )
+                    return True
+                except Exception as fallback_error:
+                    self._log_error(
+                        f"Failed to regrid {input_path} of type {grid_type} "
+                        f"without weights (remapbil fallback): {fallback_error}",
+                        exc=fallback_error,
+                    )
+                    return False
+            self._log_error(
+                f"Failed to regrid {input_path} of type {grid_type} without weights: {e}",
+                exc=e,
+            )
+            return False
+    
+    def _regrid_single_file(
+        self,
+        input_path: Path,
+        output_path: Path,
+        grid_signature: str,
+        grid_type: str,
+        force_regenerate_weights: bool = False,
+    ) -> bool:
+        """
+        Regrid a single file using CDO with grid-type-specific handling.
+        
+        Args:
+        - input_path (Path): Path to the input file
+        - output_path (Path): Path to the output file
+        - grid_signature (str): Unique signature for the grid
+        - grid_type (str): Type of grid
+        - force_regenerate_weights (bool): Whether to force regeneration of weights
+        
+        Returns (bool): True if successful, False otherwise
+        """
+        try:
+            weight_path = self._get_weight_path(grid_signature)
+            grid_desc = self._generate_target_grid_description(input_path)
+
+            with tempfile.TemporaryDirectory() as tmpdir:
+                tmpdir = Path(tmpdir)
+                grid_file = tmpdir / "target_grid.txt"
+                with open(grid_file, 'w') as f:
+                    f.write(grid_desc)
+
+                used_cached_weights = False
+                if self.use_regrid_cache and not force_regenerate_weights:
+                    if self._is_valid_weight_file(weight_path):
+                        used_cached_weights = True
+                        if self.verbose_diagnostics:
+                            self.console.print(f"[green]Reusing weights: {weight_path.name}[/green]")
+                        self.stats['weights_reused'] += 1
+                        if not self._regrid_with_weights(
+                            input_path, output_path, grid_file, weight_path, grid_type
+                        ):
+                            used_cached_weights = False
+                    else:
+                        self._invalidate_weight_file(weight_path)
+
+                if not used_cached_weights:
+                    if self.verbose_diagnostics:
+                        self.console.print(f"[yellow]Generating new weights: {weight_path.name}[/yellow]")
+                    if not self._regrid_without_weights(input_path, output_path, grid_file, grid_type):
+                        return False
+                    self._save_weights(input_path, weight_path, grid_file)
+                    self.stats['weights_generated'] += 1
+
+                if output_path.exists() and output_path.stat().st_size > 0:
+                    return True
+                self._log_error(f"Output file is empty or missing: {output_path}")
+                return False
+
+        except Exception as e:
+            self._log_error(f"Failed to regrid {input_path}: {e}", exc=e)
+            return False
+    
+    def _save_weights(self, input_path: Path, weight_path: Path, grid_file: Path) -> None:
+        """Save regrid weights for future reuse with grid-type-specific method.
+        
+        Args:
+        - input_path (Path): Path to the input file
+        - weight_path (Path): Path to the weight file
+        - grid_file (Path): Path to the grid file
+        
+        Returns (None): None
+        """
+        try:
+            file_info = self._get_file_info(input_path)
+            _, gen_op = self._regrid_operators(
+                file_info.get("grid_type", "unknown"),
+                file_info.get("has_grid_corners", False),
+            )
+            weight_path.parent.mkdir(parents=True, exist_ok=True)
+            with weight_file_lock(weight_path):
+                with tempfile.TemporaryDirectory() as tmpdir:
+                    tmpdir = Path(tmpdir)
+                    temp_weights = tmpdir / "temp_weights.nc"
+                    try:
+                        getattr(self.cdo, gen_op)(
+                            str(grid_file),
+                            input=str(input_path),
+                            output=str(temp_weights),
+                        )
+                    except Exception as e:
+                        if gen_op == "gencon" and "corner coordinates missing" in str(e).lower():
+                            self.cdo.genbil(
+                                str(grid_file),
+                                input=str(input_path),
+                                output=str(temp_weights),
+                            )
+                        else:
+                            raise
+                    staging = weight_path.with_suffix(weight_path.suffix + ".part")
+                    shutil.copy2(temp_weights, staging)
+                    os.replace(staging, weight_path)
+            return True
+        except Exception as e:
+            self.logger.warning(f"Could not save weights: {e}")
+            return False
+    
+    def regrid_file(
+        self,
+        input_path: Path,
+        output_path: Optional[Path] = None,
+        force_regenerate_weights: bool = False,
+        overwrite: bool = False,
+        ui: Optional[RegridProgressUI] = None,
+    ) -> bool:
+        """
+        Regrid a single file with comprehensive grid type support and memory optimization.
+        
+        Args:
+        - input_path (Path): Path to the input file
+        - output_path (Path): Path to the output file
+        - force_regenerate_weights (bool): Whether to force regeneration of weights
+        - overwrite (bool): If True, overwrite existing output files
+        
+        Returns (bool): True if successful, False otherwise
+        """
+        if not input_path.exists():
+            self._log_error(f"Input file does not exist: {input_path}")
+            return False
+
+        self.weight_cache_dir = weight_cache_dir_for_input(input_path)
+        self.weight_cache_dir.mkdir(parents=True, exist_ok=True)
+        init_regrid_error_log(self._error_log_path)
+
+        file_info = self._get_file_info(input_path)
+        regrid_mode = "seafloor" if self.extract_seafloor else ("surface" if self.extract_surface else "complete")
+        if ui:
+            ui.start_file_processing(input_path, file_info, regrid_mode=regrid_mode)
+        
+        if output_path is None:
+            output_filename = self._generate_output_filename(
+                input_path, file_info["has_level"], self.extract_surface, self.extract_seafloor
+            )
+            output_path = input_path.parent / output_filename
+        
+        if output_path.exists():
+            if overwrite:   # handle overwrite logic
+                if self.verbose:
+                    self.console.print(f"[yellow]Overwriting existing file: {output_path.name}[/yellow]")
+                try:
+                    output_path.unlink()
+                except Exception as e:
+                    self._log_error(f"Failed to remove existing file {output_path}: {e}", exc=e)
+                    if ui:
+                        ui.complete_file(input_path, success=False, message=f"Failed to remove existing file: {e}")
+                    return False
+            else:
+                if self.verbose:
+                    self.console.print(f"[yellow]Skipping existing file: {output_path.name}[/yellow]")
+                if ui:
+                    ui.skip_file(input_path, "File already exists")
+                return True
+        
+        grid_type = file_info["grid_type"]
+
+        # Step description for progress UI
+        if self.extract_seafloor:
+            step_msg = "Extracting seafloor"
+        elif self.extract_surface:
+            step_msg = "Extracting surface (top level)"
+        else:
+            step_msg = "Regridding whole file"
+        if self.verbose and self.verbose_diagnostics:
+            self.console.print(Panel(f"[bold cyan]{step_msg}[/bold cyan]", style="dim"))
+        self._notify_progress(
+            step_msg, pct=10, ui=ui, ui_input_path=input_path, regrid_mode=regrid_mode
+        )
+        prepared_path = self._prepare_file_for_regridding(input_path, ui=ui, ui_input_path=input_path)
+        if self.extract_surface and prepared_path == input_path:
+            self.logger.warning(
+                f"Surface extraction skipped for {input_path.name} (no level dim or preparation failed); "
+                "regridding whole file. Chunk names will still use _top_level for consistency."
+            )
+        if self.extract_seafloor and prepared_path == input_path and "_seafloor" not in input_path.stem:
+            self.logger.warning(
+                f"Seafloor extraction skipped for {input_path.name}; regridding whole file."
+            )
+
+        if not self._is_valid_prepared_file(prepared_path):
+            self._log_error(
+                f"Skipping regridding: prepared file is empty or invalid (CDO would report 'No arrays found'): {prepared_path.name}"
+            )
+            if ui:
+                ui.complete_file(input_path, success=False, message="Prepared file empty or invalid")
+            return False
+
+        # Use prepared file's metadata for chunking/display (e.g. seafloor extract is single-level and smaller)
+        if prepared_path != input_path:
+            file_info = self._get_file_info(prepared_path)
+            grid_type = file_info['grid_type']
+
+        self.stats["grid_types"][grid_type] += 1
+
+        if self.verbose_diagnostics:
+            self.console.print(f"[blue]Grid type: {grid_type}[/blue]")
+            self.console.print(f"[blue]File size: {file_info['file_size_gb']:.2f} GB[/blue]")
+            if file_info.get("has_level"):
+                self.console.print(f"[blue]Levels: {file_info['level_count']}[/blue]")
+
+        should_chunk = self._should_chunk_file(file_info)
+        if should_chunk and self.verbose_diagnostics:
+            self.console.print(
+                f"[yellow]Large file detected ({file_info['file_size_gb']:.2f} GB), using chunked processing[/yellow]"
+            )
+        if should_chunk:
+            self._notify_progress(
+                "Chunking large file",
+                pct=20,
+                ui=ui,
+                ui_input_path=input_path,
+                regrid_mode=regrid_mode,
+            )
+        try:
+            grid_signature = self._get_grid_signature(file_info)
+            if should_chunk:
+                # When extract_surface but preparation was skipped, prepared_path may be the
+                # full multi-level file; chunking it would produce *_chunk_*_top_level.nc with
+                # all levels. Ensure we chunk a top-level-only file.
+                if self.extract_surface and prepared_path == input_path:
+                    prepared_path = self._ensure_top_level_file_for_chunking(input_path)
+                self._cleanup_stale_chunks_for_file(prepared_path)
+                name_suffix = "_top_level" if self.extract_surface else ("_seafloor" if self.extract_seafloor else None)
+                chunk_files = self._chunk_file_by_time(
+                    prepared_path,
+                    name_suffix=name_suffix,
+                    ui=ui,
+                    ui_input_path=input_path,
+                    regrid_mode=regrid_mode,
+                )
+                success = self._regrid_chunked_file(
+                    prepared_path,
+                    output_path,
+                    grid_signature,
+                    grid_type,
+                    chunk_files,
+                    weight_cache_dir=self.weight_cache_dir,
+                    ui=ui,
+                    ui_input_path=input_path,
+                    regrid_mode=regrid_mode,
+                )
+            else:
+                self._notify_progress(
+                    "Regridding",
+                    pct=30,
+                    ui=ui,
+                    ui_input_path=input_path,
+                    regrid_mode=regrid_mode,
+                )
+                force_weights = force_regenerate_weights or not self.use_regrid_cache
+                success = self._regrid_single_file(
+                    prepared_path,
+                    output_path,
+                    grid_signature,
+                    grid_type,
+                    force_regenerate_weights=force_weights,
+                )
+            
+            if success:
+                self.stats['files_processed'] += 1
+                self.stats['total_size_gb'] += file_info['file_size_gb']
+                
+                # update memory monitoring
+                if self.memory_monitor:
+                    self.memory_monitor.update_peak()
+                    self.stats['memory_peak_gb'] = self.memory_monitor.get_peak_memory_gb()
+                
+                if ui:   # complete UI progress
+                    ui.complete_file(input_path, success=True)
+            else:
+                if ui:
+                    ui.complete_file(input_path, success=False, message="Regridding failed")
+            
+            if prepared_path != input_path and prepared_path.exists():
+                prepared_path.unlink()  # clean up prepared file if it was created
+            
+            return success
+            
+        except Exception as e:
+            self._log_error(f"Error processing {input_path}: {e}", exc=e)
+            self.stats['errors'] += 1
+            if ui:
+                ui.complete_file(input_path, success=False, message=f"Error: {str(e)}")
+            return False
+    
+    def regrid_batch(
+        self,
+        input_files: list[Path],
+        output_dir: Optional[Path] = None,
+        group_by_directory: bool = True,
+        overwrite: bool = False,
+        use_ui: bool = True,
+    ) -> dict[str, list[Path]]:
+        """Regrid a batch of files efficiently with optional parallel processing.
+        
+        Args:
+        - input_files (list[Path]): List of input files to regrid
+        - output_dir (Path): Output directory. If None, outputs go to same directory as input
+        - group_by_directory (bool): Group files by directory to maximize weight reuse
+        - overwrite (bool): If True, overwrite existing output files
+        
+        Returns (dict[str, list[Path]]): Dictionary mapping status to list of file paths
+        
+        Note: has two child functions, _regrid_batch_sequential and _regrid_batch_parallel depending on number of files, and whether parallel processing is enabled and is successful.
+        """
+        results = {
+            'successful': [],
+            'failed': [],
+            'skipped': [],
+        }
+        
+        if isinstance(input_files, Path):
+            input_files = [input_files]
+            self.console.print(f"[yellow]Input is a single file, will be processed sequentially[/yellow]") if self.verbose_diagnostics else None
+        # Exclude weight/cache files from the batch
+        input_files = [f for f in input_files if not is_weights_or_cache_file(f)]
+        representative_by_dir = representative_files_by_directory(input_files)
+        if representative_by_dir and self.verbose:
+            self.console.print(
+                f"[blue]Resolution representatives: {len(representative_by_dir)} "
+                f"leaf director{'y' if len(representative_by_dir) == 1 else 'ies'}[/blue]"
+            )
+
+        # clean up problematic files (_top_level and _chunk_) first
+        input_files = self._cleanup_problematic_files(input_files)
+
+        # Deduplicate by resolved path so the same original file is never processed by more than one worker
+        seen: set[Path] = set()
+        deduped: list[Path] = []
+        for f in input_files:
+            try:
+                r = f.resolve()
+            except OSError:
+                r = f
+            if r not in seen:
+                seen.add(r)
+                deduped.append(f)
+        if len(deduped) < len(input_files) and self.verbose:
+            self.console.print(
+                f"[blue]Deduplicated input: {len(input_files)} -> {len(deduped)} files (same path only processed once)[/blue]"
+            )
+        input_files = deduped
+        
+        if overwrite:
+            # unlink any files with 'regridded' in the name in order to reprocess them
+            for file in input_files:
+                if 'regridded' in file.name:
+                    if self.verbose:
+                        self.console.print(f"[yellow]Removing existing regridded file: {file.name}[/yellow]")
+                    file.unlink()
+            # prune any files with 'regridded' in the name to avoid processing them twice
+            input_files = [
+                file for file in input_files
+                if 'regridded' not in file.name and not file.suffix.endswith('.part')
+            ]   # TODO: use _prune_regridded function instead (or remove it)
+            
+            # also check for and remove existing output files that would be created from input files
+            if self.verbose:
+                self.console.print(f"[blue]Checking for existing output files to remove (overwrite=True)[/blue]")
+            for file in input_files[:]:  # use slice copy to avoid modifying list while iterating
+                # generate the expected output filename using lightweight check
+                has_level = self._has_level_lightweight(file)
+                output_filename = self._generate_output_filename(
+                    file, has_level, self.extract_surface, self.extract_seafloor
+                )
+                if output_dir:
+                    expected_output = output_dir / output_filename
+                else:
+                    expected_output = file.parent / output_filename
+                
+                if expected_output.exists():
+                    if self.verbose:
+                        self.console.print(f"[yellow]Removing existing output file: {expected_output.name}[/yellow]")
+                    expected_output.unlink()
+        else:
+            # when overwrite=False, just prune file names with 'regridded' in the name from processing list but don't delete them
+            if self.verbose:
+                regridded_files = [file for file in input_files if 'regridded' in file.name]
+                if regridded_files:
+                    self.console.print(f"[blue]Skipping {len(regridded_files)} existing regridded files (overwrite=False)[/blue]")
+            input_files = [
+                file for file in input_files
+                if 'regridded' not in file.name and not file.suffix.endswith('.part')
+            ]
+        
+        # Always redirect pipeline errors (seafloor, regrid, etc.) to a log file so they are
+        # captured even when running quiet or without UI
+        if input_files:
+            self._error_log_path = init_regrid_error_log()
+            self.set_error_log_path(self._error_log_path)
+            if use_ui and self.verbose:
+                self.console.print(f"[dim]Errors logged to: {self._error_log_path}[/dim]")
+
+        self.start_time = print_timestamp(self.console, "START")
+        
+        if not self.enable_parallel or len(input_files) < 2 or self.max_workers in [None, 1]:
+            # process sequentially
+            results = self._regrid_batch_sequential(
+                input_files, output_dir, group_by_directory, overwrite, use_ui, representative_by_dir
+            )
+            self.end_time = print_timestamp(self.console, "END")
+            return results
+        
+        ui = None
+        poll_thread = None
+        stop_poll = threading.Event()
+        try:
+            # initialize compact UI for parallel processing
+            if use_ui and self.verbose:
+                batch_mode = "seafloor" if self.extract_seafloor else ("surface" if self.extract_surface else "complete")
+                ui = BatchRegridUI(
+                    input_files,
+                    max_workers=self.max_workers,
+                    verbose=self.verbose,
+                    regrid_mode=batch_mode,
+                )
+                ui.__enter__()
+            
+            # process files in parallel (individual file processing)
+            if self.verbose:
+                self.console.print(f"[green]Processing {len(input_files)} files in parallel with {self.max_workers} workers[/green]")
+            
+            progress_manager = mp.Manager()
+            progress_state = progress_manager.dict()
+            if ui:
+                poll_thread = threading.Thread(
+                    target=poll_batch_progress,
+                    args=(ui, progress_state, stop_poll),
+                    daemon=True,
+                )
+                poll_thread.start()
+
+            with ProcessPoolExecutor(max_workers=self.max_workers) as executor:
+                futures = {
+                    executor.submit(
+                        process_single_file_standalone,
+                        file_path,
+                        output_dir,
+                        self.target_resolution,
+                        self.target_grid,
+                        weight_cache_dir_for_input(file_path),
+                        self.extract_surface,
+                        self.extract_seafloor,
+                        self.use_regrid_cache,
+                        self.use_seafloor_cache,
+                        self.max_memory_gb,
+                        self.chunk_size_gb,
+                        self.enable_chunking,
+                        overwrite,
+                        representative_by_dir.get(file_path.parent),
+                        self.verbose,
+                        self._error_log_path,
+                        progress_state,
+                    ): file_path
+                    for file_path in input_files
+                }
+
+                combined_stats = {
+                    'files_processed': 0,
+                    'weights_reused': 0,
+                    'weights_generated': 0,
+                    'chunks_processed': 0,
+                    'errors': 0,
+                    'total_size_gb': 0.0,
+                    'memory_peak_gb': 0.0,
+                    'grid_types': {}
+                }
+                
+                for future in as_completed(futures):
+                    file_result = future.result()
+
+                    if ui:
+                        ui.update_file_result(file_result["file_path"], file_result)
+                    
+                    # combine statistics from worker
+                    if 'stats' in file_result:
+                        worker_stats = file_result['stats']
+                        combined_stats['files_processed'] += worker_stats.get('files_processed', 0)
+                        combined_stats['weights_reused'] += worker_stats.get('weights_reused', 0)
+                        combined_stats['weights_generated'] += worker_stats.get('weights_generated', 0)
+                        combined_stats['chunks_processed'] += worker_stats.get('chunks_processed', 0)
+                        combined_stats['errors'] += worker_stats.get('errors', 0)
+                        combined_stats['total_size_gb'] += worker_stats.get('total_size_gb', 0.0)
+                        combined_stats['memory_peak_gb'] = max(combined_stats['memory_peak_gb'], worker_stats.get('memory_peak_gb', 0.0))
+                        
+                        # combine grid types
+                        for grid_type, count in worker_stats.get('grid_types', {}).items():
+                            if grid_type not in combined_stats['grid_types']:
+                                combined_stats['grid_types'][grid_type] = 0
+                            combined_stats['grid_types'][grid_type] += count
+                    
+                    if file_result['success']:
+                        if file_result.get('skipped', False):
+                            results['skipped'].append(file_result['file_path'])
+                        else:
+                            results['successful'].append(file_result['file_path'])
+                    else:
+                        results['failed'].append(file_result['file_path'])
+                
+                self.stats.update(combined_stats)
+        except Exception as e:
+            self._log_error(f"Error processing batch in parallel: {e}", exc=e)
+            self.logger.warning(f"Falling back to sequential processing")
+            results = self._regrid_batch_sequential(
+                input_files, output_dir, group_by_directory, overwrite, use_ui, representative_by_dir
+            )
+            self.end_time = print_timestamp(self.console, "END")
+            return results
+        finally:
+            if poll_thread is not None:
+                stop_poll.set()
+                poll_thread.join(timeout=2)
+            if ui:
+                self.end_time = print_timestamp(self.console, "END")
+                self.stats["processing_time"] = format_processing_time(
+                    get_processing_time(self.start_time, self.end_time)
+                )
+                ui._update_stats(self.stats)
+                ui.print_summary()
+                ui.__exit__(None, None, None)
+            else:
+                self.end_time = print_timestamp(self.console, "END")
+        
+        return results
+    
+    def _regrid_batch_sequential(
+        self,
+        input_files: list[Path],
+        output_dir: Optional[Path] = None,
+        group_by_directory: bool = True,
+        overwrite: bool = False,
+        use_ui: bool = True,
+        representative_by_dir: Optional[dict[Path, Path]] = None,
+    ) -> dict[str, list[Path]]:
+        """Sequential batch processing fallback. Used if parallel processing fails or is not enabled, if there is only one file, or if the number of workers is not specified or is set to 1.
+        
+        Args:
+        - input_files (list[Path]): List of input files to regrid
+        - output_dir (Path): Output directory. If None, outputs go to same directory as input
+        - group_by_directory (bool): Group files by directory to maximize weight reuse
+        
+        Returns (dict[str, list[Path]]): Dictionary mapping status to list of file paths
+        
+        Note: has one regridding child function, regrid_file, for processing a single file.
+        """
+        results = {
+            'successful': [],
+            'failed': [],
+            'skipped': [],
+        }
+        
+        # clean up problematic files (_top_level and _chunk_) first
+        input_files = self._cleanup_problematic_files(input_files)
+        # remove completed files from input_files (any containing 'regridded' in the name)
+        chunk_files = [file for file in input_files if 'chunk' in file.name]
+        regridded_files = [file for file in input_files if 'regridded' in file.name]
+        input_files = [file for file in input_files if file not in regridded_files]
+        # remove residual files (any containing 'chunk' in the name): TODO: these shouldn't exist at this point
+        input_files = [file for file in input_files if file not in chunk_files]
+        print(f"Removed {len(chunk_files)} and {len(regridded_files)} files from input_files before regridding")
+        
+        # initialize UI if requested
+        ui = None
+        if use_ui and self.verbose:
+            ui = RegridProgressUI(
+                input_files,
+                verbose=self.verbose,
+                verbose_diagnostics=self.verbose_diagnostics,
+                log_file=self._error_log_path,
+            )
+            ui.__enter__()
+        # group files by directory if requested
+        if group_by_directory:
+            file_groups = self._group_files_by_directory(input_files)
+        else:
+            file_groups = {'all': input_files}
+        
+        # process each group
+        for group_name, files in file_groups.items():
+            if self.verbose:
+                self.console.print(f"\n[blue]Processing group: {group_name}[/blue]")
+                self.console.print(f"[blue]Files in group: {len(files)}[/blue]")
+
+            # process files in group
+            for file_path in files:
+                try:
+                    dir_representative = (representative_by_dir or {}).get(file_path.parent)
+                    if dir_representative:
+                        self._representative_file = dir_representative
+
+                    # Use lightweight check for has_level to avoid expensive full file analysis
+                    has_level = self._has_level_lightweight(file_path)
+                    
+                    # determine output path
+                    if output_dir:
+                        output_filename = self._generate_output_filename(
+                            file_path, has_level, self.extract_surface, self.extract_seafloor
+                        )
+                        output_path = output_dir / output_filename
+                    else:
+                        output_filename = self._generate_output_filename(
+                            file_path, has_level, self.extract_surface, self.extract_seafloor
+                        )
+                        output_path = file_path.parent / output_filename
+
+                    # check if output already exists
+                    if output_path.exists():
+                        results["skipped"].append(file_path)
+                        if self.verbose:
+                            self.console.print(f"[yellow]Skipping (exists): {file_path.name}[/yellow]")
+                        continue
+                    
+                    # regrid file
+                    success = self.regrid_file(file_path, output_path, overwrite=overwrite, ui=ui)
+                    
+                    if success:
+                        results['successful'].append(file_path)
+                        if self.verbose:
+                            self.console.print(f"[green]Success: {file_path.name}[/green]")
+                    else:
+                        results['failed'].append(file_path)
+                        if self.verbose:
+                            self.console.print(f"[red]Failed: {file_path.name}[/red]")
+                        
+                except Exception as e:
+                    self._log_error(f"Error processing {file_path}: {e}", exc=e)
+                    results['failed'].append(file_path)
+                    if ui:
+                        ui.complete_file(file_path, success=False, message=f"Error: {str(e)}")
+        
+        if ui:
+            self.stats['processing_time'] = format_processing_time(get_processing_time(self.start_time, self.end_time))
+            ui._update_stats(self.stats)
+            ui.print_summary()
+            ui.__exit__(None, None, None)
+        
+        return results
+    
+    
+    def _group_files_by_directory(self, files: list[Path]) -> dict[str, list[Path]]:
+        """Group files by their parent directory.
+        
+        Args:
+        - files (list[Path]): List of files to group
+        
+        Returns (dict[str, list[Path]]): Dictionary mapping directory to list of file paths
+        """
+        groups = {}
+        for file_path in files:
+            parent_dir = str(file_path.parent)
+            if parent_dir not in groups:
+                groups[parent_dir] = []
+            groups[parent_dir].append(file_path)
+        return groups
+    
+    def print_statistics(self):
+        """Print comprehensive processing statistics."""
+        table = Table(show_header=True, header_style="bold cyan")
+        table.add_column("Metric", style="bold")
+        table.add_column("Value", style="bold")
+        
+        table.add_row("Files Processed", str(self.stats['files_processed']))
+        table.add_row("Weights Reused", str(self.stats['weights_reused']))
+        table.add_row("Weights Generated", str(self.stats['weights_generated']))
+        table.add_row("Chunks Processed", str(self.stats['chunks_processed']))
+        table.add_row("Errors", str(self.stats['errors']))
+        table.add_row("Total Size (GB)", f"{self.stats['total_size_gb']:.2f}")
+        table.add_row("Memory Peak (GB)", f"{self.stats['memory_peak_gb']:.2f}")
+        
+        # add grid type statistics
+        table.add_row("", "")  # Empty row
+        table.add_row("Grid Types", "")
+        for grid_type, count in self.stats['grid_types'].items():
+            if count > 0:
+                table.add_row(f"  {grid_type}", str(count))
+        
+        self.console.print(Panel(table, title="[cyan]CDO Regridding Statistics[/cyan]", border_style="cyan"))
+    
+    def cleanup_weight_files(self, confirm: bool = True):
+        """Delete all cached weight files on demand.
+        
+        Args:
+        - confirm (bool): Require user confirmation before deleting files (default: True)
+        # TODO: add confirmation option to main function?
+        Returns (None): None
+        """
+        try:
+            weight_files = list(self.weight_cache_dir.glob("weights_*.nc"))
+            if not weight_files:
+                if self.verbose:
+                    self.console.print("[yellow]No weight files to clean up.[/yellow]")
+                return
+
+            if confirm:
+                from rich.prompt import Confirm
+                proceed = Confirm.ask(
+                    f"[red]Are you sure you want to delete all {len(weight_files)} cached weight files in {self.weight_cache_dir}?[/red]"
+                )
+                if not proceed:
+                    if self.verbose:
+                        self.console.print("[yellow]Cleanup cancelled by user.[/yellow]")
+                    return
+
+            for weight_file in weight_files:
+                try:
+                    weight_file.unlink()
+                    if self.verbose:
+                        self.console.print(f"[cyan]Cleaned up {weight_file.name}[/cyan]")
+                except Exception as e:
+                    self._log_error(f"Error deleting weight file {weight_file}: {e}", exc=e)
+            if self.verbose:
+                self.console.print(f"[green]Cleaned up {len(weight_files)} weight file(s).[/green]")
+        except Exception as e:
+            self._log_error(f"Error cleaning up weights: {e}", exc=e)
+