masster 0.4.18__py3-none-any.whl → 0.4.19__py3-none-any.whl

masster/_version.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 
-__version__ = "0.4.18"
+__version__ = "0.4.19"
 
 
 def get_version():

masster/study/defaults/merge_def.py

@@ -1,7 +1,7 @@
 """Parameter class for Study merge method."""
 
 from dataclasses import dataclass, field
-from typing import Any
+from typing import Any, Optional
 
 
 @dataclass
@@ -36,6 +36,9 @@ class merge_defaults:
     max_nr_conflicts: int = 0
     link_ms2: bool = True
     
+    # Parallel processing parameters
+    threads: Optional[int] = None
+    
     # KD-Strict specific parameters
     optimize_rt_tol: bool = False
     rt_tol_range: tuple = (0.5, 4.0)  
@@ -115,6 +118,14 @@ class merge_defaults:
                 "description": "Whether to link MS2 spectra to consensus features",
                 "default": True,
             },
+            # Parallel processing parameters
+            "threads": {
+                "dtype": [int, type(None)],
+                "description": "Number of parallel threads/processes for chunked methods (None=original sequential)",
+                "default": None,
+                "min_value": 1,
+                "max_value": 32,
+            },
             # KD-Strict specific parameters
             "optimize_rt_tol": {
                 "dtype": bool,
@@ -217,7 +228,37 @@ class merge_defaults:
         metadata = self._param_metadata[param_name]
         expected_dtype = metadata["dtype"]
 
-        # Type checking
+        # Handle Optional types (list of types including None)
+        if isinstance(expected_dtype, list):
+            # Check if value matches any of the allowed types
+            valid_type = False
+            for dtype in expected_dtype:
+                if dtype is type(None) and value is None:
+                    return True  # None is explicitly allowed
+                elif dtype is int and isinstance(value, int):
+                    valid_type = True
+                    break
+                elif dtype is float and isinstance(value, (int, float)):
+                    valid_type = True
+                    break
+                elif dtype is bool and isinstance(value, bool):
+                    valid_type = True
+                    break
+                elif dtype is str and isinstance(value, str):
+                    valid_type = True
+                    break
+            
+            if not valid_type:
+                return False
+            
+            # For None values, skip further validation
+            if value is None:
+                return True
+            
+            # Use the first non-None type for range validation
+            expected_dtype = next((dt for dt in expected_dtype if dt is not type(None)), expected_dtype[0])
+
+        # Type checking for non-Optional types
         if expected_dtype is int:
             if not isinstance(value, int):
                 try:

masster/study/merge.py

@@ -10,9 +10,269 @@ from datetime import datetime
 from tqdm import tqdm
 import pyopenms as oms
 import polars as pl
+from concurrent.futures import ProcessPoolExecutor, as_completed
 from masster.study.defaults import merge_defaults
 
 
+def _process_kd_chunk_parallel(chunk_data):
+    """
+    Process a single KD chunk in parallel by reconstructing FeatureMaps from features_df slice.
+    
+    Args:
+        chunk_data: Dictionary containing chunk processing parameters
+        
+    Returns:
+        Tuple of (chunk_start_idx, serialized_consensus_features)
+    """
+    import pyopenms as oms
+    
+    chunk_start_idx = chunk_data['chunk_start_idx']
+    chunk_features_data = chunk_data['chunk_features_data']  # List of feature dicts
+    chunk_samples_data = chunk_data['chunk_samples_data']    # List of sample dicts
+    params_dict = chunk_data['params']
+    
+    # Reconstruct FeatureMaps from features data for each sample in the chunk
+    chunk_maps = []
+    
+    for sample_data in chunk_samples_data:
+        sample_uid = sample_data['sample_uid']
+        
+        # Filter features for this specific sample
+        sample_features = [f for f in chunk_features_data if f['sample_uid'] == sample_uid]
+        
+        # Create FeatureMap for this sample
+        feature_map = oms.FeatureMap()
+        
+        # Add each feature to the map
+        for feature_dict in sample_features:
+            feature = oms.Feature()
+            feature.setRT(float(feature_dict['rt']))
+            feature.setMZ(float(feature_dict['mz']))
+            feature.setIntensity(float(feature_dict['inty']))
+            feature.setCharge(int(feature_dict.get('charge', 0)))
+            
+            # Set unique ID using feature_id for mapping back
+            feature.setUniqueId(int(feature_dict['feature_id']))
+            
+            feature_map.push_back(feature)
+        
+        chunk_maps.append(feature_map)
+    
+    # Create the chunk consensus map
+    chunk_consensus_map = oms.ConsensusMap()
+    
+    # Set up file descriptions for chunk
+    file_descriptions = chunk_consensus_map.getColumnHeaders()
+    for j, (feature_map, sample_data) in enumerate(zip(chunk_maps, chunk_samples_data)):
+        file_description = file_descriptions.get(j, oms.ColumnHeader())
+        file_description.filename = sample_data['sample_name']
+        file_description.size = feature_map.size()
+        file_description.unique_id = feature_map.getUniqueId()
+        file_descriptions[j] = file_description
+    
+    chunk_consensus_map.setColumnHeaders(file_descriptions)
+    
+    # Use KD algorithm for chunk
+    grouper = oms.FeatureGroupingAlgorithmKD()
+    chunk_params = grouper.getParameters()
+    chunk_params.setValue("mz_unit", "Da")
+    chunk_params.setValue("nr_partitions", params_dict['nr_partitions'])
+    chunk_params.setValue("warp:enabled", "true")
+    chunk_params.setValue("warp:rt_tol", params_dict['rt_tol'])
+    chunk_params.setValue("warp:mz_tol", params_dict['mz_tol'])
+    chunk_params.setValue("link:rt_tol", params_dict['rt_tol'])
+    chunk_params.setValue("link:mz_tol", params_dict['mz_tol'])
+    chunk_params.setValue("link:min_rel_cc_size", params_dict['min_rel_cc_size'])
+    chunk_params.setValue("link:max_pairwise_log_fc", params_dict['max_pairwise_log_fc'])
+    chunk_params.setValue("link:max_nr_conflicts", params_dict['max_nr_conflicts'])
+    
+    grouper.setParameters(chunk_params)
+    grouper.group(chunk_maps, chunk_consensus_map)
+    
+    # Serialize the consensus map result for cross-process communication
+    consensus_features = []
+    for consensus_feature in chunk_consensus_map:
+        feature_data = {
+            'rt': consensus_feature.getRT(),
+            'mz': consensus_feature.getMZ(),
+            'intensity': consensus_feature.getIntensity(),
+            'quality': consensus_feature.getQuality(),
+            'unique_id': str(consensus_feature.getUniqueId()),
+            'features': []
+        }
+        
+        # Get constituent features
+        for feature_handle in consensus_feature.getFeatureList():
+            feature_handle_data = {
+                'unique_id': str(feature_handle.getUniqueId()),
+                'map_index': feature_handle.getMapIndex()
+            }
+            feature_data['features'].append(feature_handle_data)
+        
+        consensus_features.append(feature_data)
+    
+    return chunk_start_idx, consensus_features
+
+
+def _deserialize_consensus_features(consensus_features):
+    """
+    Deserialize consensus features back into an OpenMS ConsensusMap.
+    
+    Args:
+        consensus_features: List of serialized consensus feature dictionaries
+        
+    Returns:
+        OpenMS ConsensusMap object
+    """
+    import pyopenms as oms
+    
+    consensus_map = oms.ConsensusMap()
+    
+    for feature_data in consensus_features:
+        consensus_feature = oms.ConsensusFeature()
+        consensus_feature.setRT(float(feature_data['rt']))
+        consensus_feature.setMZ(float(feature_data['mz']))
+        consensus_feature.setIntensity(float(feature_data['intensity']))
+        consensus_feature.setQuality(float(feature_data['quality']))
+        consensus_feature.setUniqueId(int(feature_data['unique_id']))
+        
+        # Reconstruct feature handles (simplified approach)
+        feature_handles = []
+        for handle_data in feature_data['features']:
+            feature_handle = oms.FeatureHandle()
+            feature_handle.setUniqueId(int(handle_data['unique_id']))
+            feature_handle.setMapIndex(int(handle_data['map_index']))
+            feature_handles.append(feature_handle)
+        
+        # Set the feature list - properly add feature handles back to consensus feature
+        if feature_handles:
+            # Add each feature handle to the consensus feature using the correct OpenMS API
+            for feature_handle in feature_handles:
+                consensus_feature.getFeatureList().append(feature_handle)
+        
+        consensus_map.push_back(consensus_feature)
+    
+    return consensus_map
+
+
+def _process_qt_chunk_parallel(chunk_data):
+    """
+    Process a single QT chunk in parallel by reconstructing FeatureMaps from features_df slice.
+    
+    Args:
+        chunk_data: Dictionary containing chunk processing parameters
+        
+    Returns:
+        Tuple of (chunk_start_idx, serialized_consensus_features)
+    """
+    import pyopenms as oms
+    
+    chunk_start_idx = chunk_data['chunk_start_idx']
+    chunk_features_data = chunk_data['chunk_features_data']  # List of feature dicts
+    chunk_samples_data = chunk_data['chunk_samples_data']    # List of sample dicts
+    params_dict = chunk_data['params']
+    
+    # Reconstruct FeatureMaps from features data for each sample in the chunk
+    chunk_maps = []
+    
+    for sample_data in chunk_samples_data:
+        sample_uid = sample_data['sample_uid']
+        
+        # Filter features for this specific sample
+        sample_features = [f for f in chunk_features_data if f['sample_uid'] == sample_uid]
+        
+        # Create FeatureMap for this sample
+        feature_map = oms.FeatureMap()
+        
+        # Add each feature to the map
+        for feature_dict in sample_features:
+            feature = oms.Feature()
+            feature.setRT(float(feature_dict['rt']))
+            feature.setMZ(float(feature_dict['mz']))
+            feature.setIntensity(float(feature_dict['inty']))
+            feature.setCharge(int(feature_dict.get('charge', 0)))
+            
+            # Set unique ID using feature_id for mapping back
+            feature.setUniqueId(int(feature_dict['feature_id']))
+            
+            feature_map.push_back(feature)
+        
+        chunk_maps.append(feature_map)
+    
+    # Create the chunk consensus map
+    chunk_consensus_map = oms.ConsensusMap()
+    
+    # Set up file descriptions for chunk
+    file_descriptions = chunk_consensus_map.getColumnHeaders()
+    for j, (feature_map, sample_data) in enumerate(zip(chunk_maps, chunk_samples_data)):
+        file_description = file_descriptions.get(j, oms.ColumnHeader())
+        file_description.filename = sample_data['sample_name']
+        file_description.size = feature_map.size()
+        file_description.unique_id = feature_map.getUniqueId()
+        file_descriptions[j] = file_description
+    
+    chunk_consensus_map.setColumnHeaders(file_descriptions)
+    
+    # Use QT algorithm for chunk
+    grouper = oms.FeatureGroupingAlgorithmQT()
+    chunk_params = grouper.getParameters()
+    chunk_params.setValue("distance_RT:max_difference", params_dict['rt_tol'])
+    chunk_params.setValue("distance_MZ:max_difference", params_dict['mz_tol'])
+    chunk_params.setValue("distance_MZ:unit", "Da")
+    chunk_params.setValue("ignore_charge", "true")
+    chunk_params.setValue("nr_partitions", params_dict['nr_partitions'])
+    
+    grouper.setParameters(chunk_params)
+    grouper.group(chunk_maps, chunk_consensus_map)
+    
+    # Serialize the consensus map result for cross-process communication
+    consensus_features = []
+    for consensus_feature in chunk_consensus_map:
+        feature_data = {
+            'rt': consensus_feature.getRT(),
+            'mz': consensus_feature.getMZ(),
+            'intensity': consensus_feature.getIntensity(),
+            'quality': consensus_feature.getQuality(),
+            'unique_id': str(consensus_feature.getUniqueId()),
+            'features': []
+        }
+        
+        # Get constituent features
+        for feature_handle in consensus_feature.getFeatureList():
+            feature_handle_data = {
+                'unique_id': str(feature_handle.getUniqueId()),
+                'map_index': feature_handle.getMapIndex()
+            }
+            feature_data['features'].append(feature_handle_data)
+        
+        consensus_features.append(feature_data)
+    
+    return chunk_start_idx, consensus_features
+
+
+def _serialize_feature_map(feature_map):
+    """
+    Serialize a FeatureMap to a list of dictionaries for multiprocessing.
+    
+    Args:
+        feature_map: OpenMS FeatureMap object
+        
+    Returns:
+        List of feature dictionaries
+    """
+    features_data = []
+    for feature in feature_map:
+        feature_data = {
+            'rt': feature.getRT(),
+            'mz': feature.getMZ(), 
+            'intensity': feature.getIntensity(),
+            'charge': feature.getCharge(),
+            'unique_id': feature.getUniqueId()
+        }
+        features_data.append(feature_data)
+    return features_data
+
+
 def merge(self, **kwargs) -> None:
     """
     Group features across samples into consensus features using various algorithms.
@@ -34,6 +294,8 @@ def merge(self, **kwargs) -> None:
           m/z tolerance in Da (Daltons) for all methods
         - chunk_size : int, default 500
           Chunk size for 'chunked' method
+        - threads : int, default 1
+          Number of parallel processes for chunked methods (kd_chunked, qt_chunked)
         - nr_partitions : int, default 500
           Number of partitions in m/z dimension for KD algorithms
         - min_rel_cc_size : float, default 0.3
@@ -54,9 +316,19 @@ def merge(self, **kwargs) -> None:
     - NoWarp: Memory efficient KD without RT warping for large datasets
     - KD-Chunked: Memory-optimized KD algorithm for very large datasets (>5000 samples)
       Uses optimized partitioning for better memory management while maintaining
-      full cross-sample consensus feature detection.
+      full cross-sample consensus feature detection. Supports parallel processing.
     - QT-Chunked: Memory-optimized QT algorithm for very large datasets (>5000 samples)
       Uses QT clustering in first stage with optimized cross-chunk consensus building.
+      Supports parallel processing.
+
+    Parallel Processing
+    ------------------
+    For kd_chunked and qt_chunked methods, use threads > 1 to enable parallel processing
+    of chunk alignments. This can significantly reduce processing time for large datasets
+    by processing multiple chunks simultaneously in separate processes.
+    
+    Example:
+        study.merge(method='kd_chunked', threads=4, chunk_size=200)
     """
     start_time = time.time()
     
@@ -774,7 +1046,7 @@ def _merge_kd_nowarp(self, params: merge_defaults) -> oms.ConsensusMap:
 
 
 def _merge_kd_chunked(self, params: merge_defaults, cached_adducts_df=None, cached_valid_adducts=None) -> oms.ConsensusMap:
-    """KD-based chunked merge with proper cross-chunk consensus building"""
+    """KD-based chunked merge with proper cross-chunk consensus building and optional parallel processing"""
     
     n_samples = len(self.features_maps)
     if n_samples <= params.chunk_size:
@@ -790,54 +1062,130 @@ def _merge_kd_chunked(self, params: merge_defaults, cached_adducts_df=None, cach
         chunk_end = min(i + params.chunk_size, n_samples)
         chunks.append((i, self.features_maps[i:chunk_end]))
     
-    self.logger.debug(f"Processing {len(chunks)} chunks of max {params.chunk_size} samples")
+    self.logger.debug(f"Processing {len(chunks)} chunks of max {params.chunk_size} samples using {params.threads or 'sequential'} thread(s)")
     
     # Process each chunk to create chunk consensus maps
     chunk_consensus_maps = []
     
-    for chunk_idx, (chunk_start_idx, chunk_maps) in enumerate(tqdm(chunks, desc="Chunk", disable=self.log_level not in ["TRACE", "DEBUG", "INFO"])):
-        chunk_consensus_map = oms.ConsensusMap()
-        
-        # Set up file descriptions for chunk
-        file_descriptions = chunk_consensus_map.getColumnHeaders()
-        for j, feature_map in enumerate(chunk_maps):
-            file_description = file_descriptions.get(j, oms.ColumnHeader())
-            file_description.filename = self.samples_df.row(chunk_start_idx + j, named=True)["sample_name"]
-            file_description.size = feature_map.size()
-            file_description.unique_id = feature_map.getUniqueId()
-            file_descriptions[j] = file_description
-        
-        chunk_consensus_map.setColumnHeaders(file_descriptions)
+    if params.threads is None:
+        # Sequential processing (original behavior)
+        for chunk_idx, (chunk_start_idx, chunk_maps) in enumerate(tqdm(chunks, desc=f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | INFO     | {self.log_label}KD Chunk", disable=self.log_level not in ["TRACE", "DEBUG", "INFO"])):
+            chunk_consensus_map = oms.ConsensusMap()
+            
+            # Set up file descriptions for chunk
+            file_descriptions = chunk_consensus_map.getColumnHeaders()
+            for j, feature_map in enumerate(chunk_maps):
+                file_description = file_descriptions.get(j, oms.ColumnHeader())
+                file_description.filename = self.samples_df.row(chunk_start_idx + j, named=True)["sample_name"]
+                file_description.size = feature_map.size()
+                file_description.unique_id = feature_map.getUniqueId()
+                file_descriptions[j] = file_description
+            
+            chunk_consensus_map.setColumnHeaders(file_descriptions)
+            
+            # Use KD algorithm for chunk
+            grouper = oms.FeatureGroupingAlgorithmKD()
+            chunk_params = grouper.getParameters()
+            chunk_params.setValue("mz_unit", "Da")
+            chunk_params.setValue("nr_partitions", params.nr_partitions)
+            chunk_params.setValue("warp:enabled", "true")
+            chunk_params.setValue("warp:rt_tol", params.rt_tol)
+            chunk_params.setValue("warp:mz_tol", params.mz_tol)
+            chunk_params.setValue("link:rt_tol", params.rt_tol)
+            chunk_params.setValue("link:mz_tol", params.mz_tol)
+            chunk_params.setValue("link:min_rel_cc_size", params.min_rel_cc_size)
+            chunk_params.setValue("link:max_pairwise_log_fc", params.max_pairwise_log_fc)
+            chunk_params.setValue("link:max_nr_conflicts", params.max_nr_conflicts)
+            
+            grouper.setParameters(chunk_params)
+            grouper.group(chunk_maps, chunk_consensus_map)
+            
+            chunk_consensus_maps.append((chunk_start_idx, chunk_consensus_map))
+    
+    else:
+        # Parallel processing
+        self.logger.info(f"Processing chunks in parallel using {params.threads} processes")
         
-        # Use KD algorithm for chunk
-        grouper = oms.FeatureGroupingAlgorithmKD()
-        chunk_params = grouper.getParameters()
-        chunk_params.setValue("mz_unit", "Da")
-        chunk_params.setValue("nr_partitions", params.nr_partitions)
-        chunk_params.setValue("warp:enabled", "true")
-        chunk_params.setValue("warp:rt_tol", params.rt_tol)
-        chunk_params.setValue("warp:mz_tol", params.mz_tol)
-        chunk_params.setValue("link:rt_tol", params.rt_tol)
-        chunk_params.setValue("link:mz_tol", params.mz_tol)
-        chunk_params.setValue("link:min_rel_cc_size", params.min_rel_cc_size)
-        chunk_params.setValue("link:max_pairwise_log_fc", params.max_pairwise_log_fc)
-        chunk_params.setValue("link:max_nr_conflicts", params.max_nr_conflicts)
+        # Prepare chunk data for parallel processing using features_df slices
+        chunk_data_list = []
+        for chunk_idx, (chunk_start_idx, chunk_maps) in enumerate(chunks):
+            # Get the sample UIDs for this chunk
+            chunk_sample_uids = []
+            chunk_samples_df_rows = []
+            for j in range(len(chunk_maps)):
+                sample_row = self.samples_df.row(chunk_start_idx + j, named=True)
+                chunk_sample_uids.append(sample_row['sample_uid'])
+                chunk_samples_df_rows.append(sample_row)
+            
+            # Create a DataFrame for this chunk's samples
+            chunk_samples_df = pl.DataFrame(chunk_samples_df_rows)
+            
+            # Filter features_df for this chunk's samples and select only necessary columns
+            chunk_features_df = self.features_df.filter(
+                pl.col('sample_uid').is_in(chunk_sample_uids)
+            ).select([
+                'sample_uid', 'rt', 'mz', 'inty', 'charge', 'feature_id'
+            ])
+            
+            # Convert DataFrames to serializable format (lists of dicts)
+            chunk_features_data = chunk_features_df.to_dicts()
+            chunk_samples_data = chunk_samples_df.to_dicts()
+            
+            chunk_data = {
+                'chunk_start_idx': chunk_start_idx,
+                'chunk_features_data': chunk_features_data,  # List of dicts instead of DataFrame
+                'chunk_samples_data': chunk_samples_data,    # List of dicts instead of DataFrame
+                'params': {
+                    'nr_partitions': params.nr_partitions,
+                    'rt_tol': params.rt_tol,
+                    'mz_tol': params.mz_tol,
+                    'min_rel_cc_size': params.min_rel_cc_size,
+                    'max_pairwise_log_fc': params.max_pairwise_log_fc,
+                    'max_nr_conflicts': params.max_nr_conflicts
+                }
+            }
+            chunk_data_list.append(chunk_data)
         
-        grouper.setParameters(chunk_params)
-        grouper.group(chunk_maps, chunk_consensus_map)
+        # Process chunks in parallel
+        with ProcessPoolExecutor(max_workers=params.threads) as executor:
+            # Submit all chunk processing tasks
+            future_to_chunk = {executor.submit(_process_kd_chunk_parallel, chunk_data): i 
+                             for i, chunk_data in enumerate(chunk_data_list)}
+            
+            # Collect results with progress tracking
+            completed_chunks = 0
+            total_chunks = len(chunk_data_list)
+            serialized_chunk_results = []
+            
+            for future in as_completed(future_to_chunk):
+                chunk_idx = future_to_chunk[future]
+                try:
+                    chunk_start_idx, consensus_features = future.result()
+                    serialized_chunk_results.append((chunk_start_idx, consensus_features))
+                    completed_chunks += 1
+                    n_samples_in_chunk = len(chunk_data_list[chunk_idx]['chunk_samples_data'])
+                    self.logger.info(f"Completed chunk {completed_chunks}/{total_chunks} (samples {chunk_start_idx + 1}-{chunk_start_idx + n_samples_in_chunk})")
+                except Exception as exc:
+                    self.logger.error(f"Chunk {chunk_idx} generated an exception: {exc}")
+                    raise exc
         
-        chunk_consensus_maps.append((chunk_start_idx, chunk_consensus_map))
-    
-    # Merge chunk results with proper cross-chunk consensus building
+        # Store serialized results for _merge_chunk_results to handle directly  
+        chunk_consensus_maps = []
+        for chunk_start_idx, consensus_features in sorted(serialized_chunk_results):
+            # Store serialized data directly for _merge_chunk_results to handle
+            chunk_consensus_maps.append((chunk_start_idx, consensus_features))
+    
+    # Merge chunk results with proper cross-chunk consensus building  
+    # _merge_chunk_results now handles both ConsensusMap objects (sequential) and serialized data (parallel)
     _merge_chunk_results(self, chunk_consensus_maps, params, cached_adducts_df, cached_valid_adducts)
     
-    # Create a dummy consensus map for compatibility (since other functions expect it)
+    # Return a dummy consensus map for compatibility (consensus features are stored in self.consensus_df)
     consensus_map = oms.ConsensusMap()
     return consensus_map
 
 
 def _merge_qt_chunked(self, params: merge_defaults, cached_adducts_df=None, cached_valid_adducts=None) -> oms.ConsensusMap:
-    """QT-based chunked merge with proper cross-chunk consensus building"""
+    """QT-based chunked merge with proper cross-chunk consensus building and optional parallel processing"""
     
     n_samples = len(self.features_maps)
     if n_samples <= params.chunk_size:
@@ -853,43 +1201,116 @@ def _merge_qt_chunked(self, params: merge_defaults, cached_adducts_df=None, cach
         chunk_end = min(i + params.chunk_size, n_samples)
         chunks.append((i, self.features_maps[i:chunk_end]))
     
-    self.logger.debug(f"Processing {len(chunks)} chunks of max {params.chunk_size} samples")
+    self.logger.debug(f"Processing {len(chunks)} chunks of max {params.chunk_size} samples using {params.threads or 'sequential'} thread(s)")
     
     # Process each chunk to create chunk consensus maps
     chunk_consensus_maps = []
     
-    for chunk_idx, (chunk_start_idx, chunk_maps) in enumerate(tqdm(chunks, desc="Chunk", disable=self.log_level not in ["TRACE", "DEBUG", "INFO"])):
-        chunk_consensus_map = oms.ConsensusMap()
-        
-        # Set up file descriptions for chunk
-        file_descriptions = chunk_consensus_map.getColumnHeaders()
-        for j, feature_map in enumerate(chunk_maps):
-            file_description = file_descriptions.get(j, oms.ColumnHeader())
-            file_description.filename = self.samples_df.row(chunk_start_idx + j, named=True)["sample_name"]
-            file_description.size = feature_map.size()
-            file_description.unique_id = feature_map.getUniqueId()
-            file_descriptions[j] = file_description
-        
-        chunk_consensus_map.setColumnHeaders(file_descriptions)
+    if params.threads is None:
+        # Sequential processing (original behavior)
+        for chunk_idx, (chunk_start_idx, chunk_maps) in enumerate(tqdm(chunks, desc=f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | INFO     | {self.log_label}QT Chunk", disable=self.log_level not in ["TRACE", "DEBUG", "INFO"])):
+            chunk_consensus_map = oms.ConsensusMap()
+            
+            # Set up file descriptions for chunk
+            file_descriptions = chunk_consensus_map.getColumnHeaders()
+            for j, feature_map in enumerate(chunk_maps):
+                file_description = file_descriptions.get(j, oms.ColumnHeader())
+                file_description.filename = self.samples_df.row(chunk_start_idx + j, named=True)["sample_name"]
+                file_description.size = feature_map.size()
+                file_description.unique_id = feature_map.getUniqueId()
+                file_descriptions[j] = file_description
+            
+            chunk_consensus_map.setColumnHeaders(file_descriptions)
+            
+            # Use QT algorithm for chunk (main difference from KD chunked)
+            grouper = oms.FeatureGroupingAlgorithmQT()
+            chunk_params = grouper.getParameters()
+            chunk_params.setValue("distance_RT:max_difference", params.rt_tol)
+            chunk_params.setValue("distance_MZ:max_difference", params.mz_tol)
+            chunk_params.setValue("distance_MZ:unit", "Da")
+            chunk_params.setValue("ignore_charge", "true")
+            chunk_params.setValue("nr_partitions", params.nr_partitions)
+            
+            grouper.setParameters(chunk_params)
+            grouper.group(chunk_maps, chunk_consensus_map)
+            
+            chunk_consensus_maps.append((chunk_start_idx, chunk_consensus_map))
+    
+    else:
+        # Parallel processing
+        self.logger.info(f"Processing chunks in parallel using {params.threads} processes")
         
-        # Use QT algorithm for chunk (main difference from KD chunked)
-        grouper = oms.FeatureGroupingAlgorithmQT()
-        chunk_params = grouper.getParameters()
-        chunk_params.setValue("distance_RT:max_difference", params.rt_tol)
-        chunk_params.setValue("distance_MZ:max_difference", params.mz_tol)
-        chunk_params.setValue("distance_MZ:unit", "Da")
-        chunk_params.setValue("ignore_charge", "true")
-        chunk_params.setValue("nr_partitions", params.nr_partitions)
+        # Prepare chunk data for parallel processing using features_df slices
+        chunk_data_list = []
+        for chunk_idx, (chunk_start_idx, chunk_maps) in enumerate(chunks):
+            # Get the sample UIDs for this chunk
+            chunk_sample_uids = []
+            chunk_samples_df_rows = []
+            for j in range(len(chunk_maps)):
+                sample_row = self.samples_df.row(chunk_start_idx + j, named=True)
+                chunk_sample_uids.append(sample_row['sample_uid'])
+                chunk_samples_df_rows.append(sample_row)
+            
+            # Create a DataFrame for this chunk's samples
+            chunk_samples_df = pl.DataFrame(chunk_samples_df_rows)
+            
+            # Filter features_df for this chunk's samples and select only necessary columns
+            chunk_features_df = self.features_df.filter(
+                pl.col('sample_uid').is_in(chunk_sample_uids)
+            ).select([
+                'sample_uid', 'rt', 'mz', 'inty', 'charge', 'feature_id'
+            ])
+            
+            # Convert DataFrames to serializable format (lists of dicts)
+            chunk_features_data = chunk_features_df.to_dicts()
+            chunk_samples_data = chunk_samples_df.to_dicts()
+            
+            chunk_data = {
+                'chunk_start_idx': chunk_start_idx,
+                'chunk_features_data': chunk_features_data,  # List of dicts instead of DataFrame
+                'chunk_samples_data': chunk_samples_data,    # List of dicts instead of DataFrame
+                'params': {
+                    'nr_partitions': params.nr_partitions,
+                    'rt_tol': params.rt_tol,
+                    'mz_tol': params.mz_tol,
+                }
+            }
+            chunk_data_list.append(chunk_data)
         
-        grouper.setParameters(chunk_params)
-        grouper.group(chunk_maps, chunk_consensus_map)
+        # Process chunks in parallel
+        with ProcessPoolExecutor(max_workers=params.threads) as executor:
+            # Submit all chunk processing tasks
+            future_to_chunk = {executor.submit(_process_qt_chunk_parallel, chunk_data): i 
+                             for i, chunk_data in enumerate(chunk_data_list)}
+            
+            # Collect results with progress tracking
+            completed_chunks = 0
+            total_chunks = len(chunk_data_list)
+            serialized_chunk_results = []
+            
+            for future in as_completed(future_to_chunk):
+                chunk_idx = future_to_chunk[future]
+                try:
+                    chunk_start_idx, consensus_features = future.result()
+                    serialized_chunk_results.append((chunk_start_idx, consensus_features))
+                    completed_chunks += 1
+                    n_samples_in_chunk = len(chunk_data_list[chunk_idx]['chunk_samples_data'])
+                    self.logger.info(f"Completed chunk {completed_chunks}/{total_chunks} (samples {chunk_start_idx + 1}-{chunk_start_idx + n_samples_in_chunk})")
+                except Exception as exc:
+                    self.logger.error(f"Chunk {chunk_idx} generated an exception: {exc}")
+                    raise exc
         
-        chunk_consensus_maps.append((chunk_start_idx, chunk_consensus_map))
-    
-    # Merge chunk results with proper cross-chunk consensus building
+        # Store serialized results for _merge_chunk_results to handle directly  
+        chunk_consensus_maps = []
+        for chunk_start_idx, consensus_features in sorted(serialized_chunk_results):
+            # Store serialized data directly for _merge_chunk_results to handle
+            chunk_consensus_maps.append((chunk_start_idx, consensus_features))
+    
+    # Merge chunk results with proper cross-chunk consensus building  
+    # _merge_chunk_results now handles both ConsensusMap objects (sequential) and serialized data (parallel)
     _merge_chunk_results(self, chunk_consensus_maps, params, cached_adducts_df, cached_valid_adducts)
     
-    # Create a dummy consensus map for compatibility (since other functions expect it)
+    # Return a dummy consensus map for compatibility (consensus features are stored in self.consensus_df)
     consensus_map = oms.ConsensusMap()
     return consensus_map
 
@@ -927,61 +1348,128 @@ def _merge_chunk_results(self, chunk_consensus_maps: list, params: merge_default
     all_chunk_consensus = []
     consensus_id_counter = 0
     
-    for chunk_idx, (chunk_start_idx, chunk_consensus_map) in enumerate(chunk_consensus_maps):
-        for consensus_feature in chunk_consensus_map:
+    for chunk_idx, (chunk_start_idx, chunk_data) in enumerate(chunk_consensus_maps):
+        # Handle both ConsensusMap objects (sequential) and serialized data (parallel)
+        if isinstance(chunk_data, list):
+            # Parallel processing: chunk_data is a list of serialized consensus feature dictionaries
+            consensus_features_data = chunk_data
+        else:
+            # Sequential processing: chunk_data is a ConsensusMap object
+            chunk_consensus_map = chunk_data
+            consensus_features_data = []
+            
+            # Extract data from ConsensusMap and convert to serialized format
+            for consensus_feature in chunk_consensus_map:
+                # Extract feature_uids from this consensus feature
+                feature_uids = []
+                feature_data_list = []
+                sample_uids = []
+                
+                for feature_handle in consensus_feature.getFeatureList():
+                    fuid = str(feature_handle.getUniqueId())
+                    if fuid not in feature_uid_map:
+                        continue
+                        
+                    feature_uid = feature_uid_map[fuid]
+                    feature_data = features_lookup.get(feature_uid)
+                    if feature_data:
+                        feature_uids.append(feature_uid)
+                        feature_data_list.append(feature_data)
+                        sample_uids.append(chunk_start_idx + feature_handle.getMapIndex() + 1)
+
+                if not feature_data_list:
+                    # No retrievable feature metadata (possible stale map reference) -> skip
+                    continue
+                    
+                # Convert ConsensusFeature to serialized format
+                consensus_feature_data = {
+                    'rt': consensus_feature.getRT(),
+                    'mz': consensus_feature.getMZ(),
+                    'intensity': consensus_feature.getIntensity(),
+                    'quality': consensus_feature.getQuality(),
+                    'feature_uids': feature_uids,
+                    'feature_data_list': feature_data_list,
+                    'sample_uids': sample_uids
+                }
+                consensus_features_data.append(consensus_feature_data)
+        
+        # Process the consensus features (now all in serialized format)
+        for consensus_feature_data in consensus_features_data:
             # ACCEPT ALL consensus features (size >=1) here.
             # Reason: A feature that is globally present in many samples can still
             # appear only once inside a given sample chunk. Early filtering at
             # size>=2 causes irreversible loss and underestimates the final
             # consensus count (observed ~296 vs 950 for KD). We defer filtering
             # strictly to the final global min_samples.
-                
-            # Extract feature_uids from this consensus feature
-            feature_uids = []
-            feature_data_list = []
-            sample_uids = []
             
-            for feature_handle in consensus_feature.getFeatureList():
-                fuid = str(feature_handle.getUniqueId())
-                if fuid not in feature_uid_map:
+            # For parallel processing, feature data is already extracted
+            if isinstance(chunk_data, list):
+                # Extract feature_uids and data from serialized format for parallel processing
+                feature_uids = []
+                feature_data_list = []
+                sample_uids = []
+                
+                for handle_data in consensus_feature_data['features']:
+                    fuid = str(handle_data['unique_id'])
+                    if fuid not in feature_uid_map:
+                        continue
+                        
+                    feature_uid = feature_uid_map[fuid]
+                    feature_data = features_lookup.get(feature_uid)
+                    if feature_data:
+                        feature_uids.append(feature_uid)
+                        feature_data_list.append(feature_data)
+                        sample_uids.append(chunk_start_idx + handle_data['map_index'] + 1)
+                
+                if not feature_data_list:
                     continue
                     
-                feature_uid = feature_uid_map[fuid]
-                feature_data = features_lookup.get(feature_uid)
-                if feature_data:
-                    feature_uids.append(feature_uid)
-                    feature_data_list.append(feature_data)
-                    sample_uids.append(chunk_start_idx + feature_handle.getMapIndex() + 1)
+                # Get RT/MZ from consensus feature data
+                consensus_rt = consensus_feature_data['rt']
+                consensus_mz = consensus_feature_data['mz']
+                consensus_intensity = consensus_feature_data['intensity']
+                consensus_quality = consensus_feature_data['quality']
+            else:
+                # Sequential processing: data is already extracted above
+                feature_uids = consensus_feature_data['feature_uids']
+                feature_data_list = consensus_feature_data['feature_data_list'] 
+                sample_uids = consensus_feature_data['sample_uids']
+                consensus_rt = consensus_feature_data['rt']
+                consensus_mz = consensus_feature_data['mz']
+                consensus_intensity = consensus_feature_data['intensity']
+                consensus_quality = consensus_feature_data['quality']
 
             if not feature_data_list:
                 # No retrievable feature metadata (possible stale map reference) -> skip
-                continue            # Derive RT / m/z ranges from underlying features (used for robust cross-chunk stitching)
+                continue
+                
+            # Derive RT / m/z ranges from underlying features (used for robust cross-chunk stitching)
             rt_vals_local = [fd.get("rt") for fd in feature_data_list if fd.get("rt") is not None]
             mz_vals_local = [fd.get("mz") for fd in feature_data_list if fd.get("mz") is not None]
             if rt_vals_local:
                 rt_min_local = min(rt_vals_local)
                 rt_max_local = max(rt_vals_local)
             else:
-                rt_min_local = rt_max_local = consensus_feature.getRT()
+                rt_min_local = rt_max_local = consensus_rt
             if mz_vals_local:
                 mz_min_local = min(mz_vals_local)
                 mz_max_local = max(mz_vals_local)
             else:
-                mz_min_local = mz_max_local = consensus_feature.getMZ()
+                mz_min_local = mz_max_local = consensus_mz
                 
             # Store chunk consensus with feature tracking
             chunk_consensus_data = {
                 'consensus_id': consensus_id_counter,
                 'chunk_idx': chunk_idx,
                 'chunk_start_idx': chunk_start_idx,
-                'mz': consensus_feature.getMZ(),
-                'rt': consensus_feature.getRT(),
+                'mz': consensus_mz,
+                'rt': consensus_rt,
                 'mz_min': mz_min_local,
                 'mz_max': mz_max_local,
                 'rt_min': rt_min_local,
                 'rt_max': rt_max_local,
-                'intensity': consensus_feature.getIntensity(),
-                'quality': consensus_feature.getQuality(),
+                'intensity': consensus_intensity,
+                'quality': consensus_quality,
                 'feature_uids': feature_uids,
                 'feature_data_list': feature_data_list,
                 'sample_uids': sample_uids,
@@ -1479,9 +1967,6 @@ def _cluster_consensus_features(features: list, rt_tol: float, mz_tol: float) ->
     return list(groups_by_root.values())
 
 
-# Note: Restored proper chunked implementation with cross-chunk consensus clustering
-
-
 def _reset_consensus_data(self):
     """Reset consensus-related DataFrames at the start of merge."""
     self.consensus_df = pl.DataFrame()

{masster-0.4.18.dist-info → masster-0.4.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: masster
-Version: 0.4.18
+Version: 0.4.19
 Summary: Mass spectrometry data analysis package
 Project-URL: homepage, https://github.com/zamboni-lab/masster
 Project-URL: repository, https://github.com/zamboni-lab/masster

{masster-0.4.18.dist-info → masster-0.4.19.dist-info}/RECORD

@@ -1,5 +1,5 @@
 masster/__init__.py,sha256=HHjKhCjkAc98LhoQfu4C6L-W2vfTEc1iXaPTxxcl_4A,800
-masster/_version.py,sha256=OUcHIwT4wa5AqV46S88edNYE4u4sKsoESNk3lFdxs_c,257
+masster/_version.py,sha256=Kro6JvBTMqNf6tOgI2r5d4TbaZIIR85ax7tdT3uQKL8,257
 masster/chromatogram.py,sha256=iYpdv8C17zVnlWvOFgAn9ns2uFGiF-GgoYf5QVVAbHs,19319
 masster/logger.py,sha256=W50V_uh8RSYwGxDrDFhOuj5jpu2tKJyt_16lMw9kQwA,14755
 masster/spectrum.py,sha256=_upC_g2N9gwTaflXAugs9pSXpKUmzbIehofDordk7WI,47718
@@ -43,7 +43,7 @@ masster/study/h5.py,sha256=LiVGUAtULyPpZIUmKVJSaV38huJb8FsKOUWBOqiv0QU,82363
 masster/study/helpers.py,sha256=M5_q8O5tuFchKPW04PTuj3X335lDA2VZqcs4D8ZQJEk,158604
 masster/study/id.py,sha256=6NUBBKZCFOU1wlDKM0eXQeOIStSZCRNJ_3x7ZaIHzmM,55263
 masster/study/load.py,sha256=CQQY_7BzagE3oQTdDlqNyfuMdVWIAft-M4a2WCFnxp0,70695
-masster/study/merge.py,sha256=2Vqj0OaTZxwtjYu1l5PmRpMmT8_cHh-R761FUvBE_Sk,95741
+masster/study/merge.py,sha256=Xk7Zt6x0p_myjWQXuzXbXSlwXPSujWjMPowaqnEEmWQ,118778
 masster/study/parameters.py,sha256=0elaF7YspTsB7qyajWAbRNL2VfKlGz5GJLifmO8IGkk,3276
 masster/study/plot.py,sha256=SimX-IlqISEItAnTBsx4xsdYHRAevfN41cCENVns1lw,88236
 masster/study/processing.py,sha256=u1MSRKTzcqHNz_dClSUSfgTxkNRdBLXtVyO5LXuW_uk,41031
@@ -60,7 +60,7 @@ masster/study/defaults/find_ms2_def.py,sha256=RL0DFG41wQ05U8UQKUGr3vzSl3mU0m0knQ
 masster/study/defaults/identify_def.py,sha256=96rxoCAPQj_yX-3mRoD2LTkTLJgG27eJQqwarLv5jL0,10580
 masster/study/defaults/integrate_chrom_def.py,sha256=0MNIWGTjty-Zu-NTQsIweuj3UVqEY3x1x8pK0mPwYak,7264
 masster/study/defaults/integrate_def.py,sha256=Vf4SAzdBfnsSZ3IRaF0qZvWu3gMDPHdgPfMYoPKeWv8,7246
-masster/study/defaults/merge_def.py,sha256=X7mTCgtQhglOTjwg06oSMFSbLBJSKsHmJeVVfYE2qHE,13272
+masster/study/defaults/merge_def.py,sha256=K7sfwEGfgcWU85zorbWNFaxDhqRH52pxQoKv9Jn2qhY,15030
 masster/study/defaults/study_def.py,sha256=h8dYbi9xv0sesCSQik49Z53IkskMmNtW6ixl7it5pL0,16033
 masster/wizard/README.md,sha256=mL1A3YWJZOefpJ6D0-HqGLkVRmUlOpwyVFdvJBeeoZM,14149
 masster/wizard/__init__.py,sha256=A9GHQvkq4lSRIA8V6AKB-TJy8s_npH8i1baUGdkw_is,364
@@ -68,8 +68,8 @@ masster/wizard/example.py,sha256=xEZFTH9UZ8HKOm6s3JL8Js0Uw5ChnISWBHSZCL32vsM,798
 masster/wizard/test_structure.py,sha256=h88gsYYCG6iDRjqPZC_r1H1T8y79j0E-K6OrwuHaSCU,1586
 masster/wizard/test_wizard.py,sha256=CMp1cpjH3iYYC5Fy6puF_K0kfwwk3bgOsSbUGW-t7Xk,8986
 masster/wizard/wizard.py,sha256=jMLHy4cXgNEE_-vshFmA7BNEByhfA6tV7O91jhiMYuw,48054
-masster-0.4.18.dist-info/METADATA,sha256=pn-XNHgHqlY1KgiYkQ2Dyke9E1nnCP3mn-ja5W5QPyM,44207
-masster-0.4.18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-masster-0.4.18.dist-info/entry_points.txt,sha256=ZHguQ_vPmdbpqq2uGtmEOLJfgP-DQ1T0c07Lxh30wc8,58
-masster-0.4.18.dist-info/licenses/LICENSE,sha256=bx5iLIKjgAdYQ7sISn7DsfHRKkoCUm1154sJJKhgqnU,35184
-masster-0.4.18.dist-info/RECORD,,
+masster-0.4.19.dist-info/METADATA,sha256=fcnG14G4Fbp7mOCQ3aKL0qvkuexeUUjm79P1dDpT_Kg,44207
+masster-0.4.19.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+masster-0.4.19.dist-info/entry_points.txt,sha256=ZHguQ_vPmdbpqq2uGtmEOLJfgP-DQ1T0c07Lxh30wc8,58
+masster-0.4.19.dist-info/licenses/LICENSE,sha256=bx5iLIKjgAdYQ7sISn7DsfHRKkoCUm1154sJJKhgqnU,35184
+masster-0.4.19.dist-info/RECORD,,