masster 0.4.17__tar.gz → 0.4.18__tar.gz

{masster-0.4.17 → masster-0.4.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: masster
-Version: 0.4.17
+Version: 0.4.18
 Summary: Mass spectrometry data analysis package
 Project-URL: homepage, https://github.com/zamboni-lab/masster
 Project-URL: repository, https://github.com/zamboni-lab/masster

{masster-0.4.17 → masster-0.4.18}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [project]
 name = "masster"
-version = "0.4.17"
+version = "0.4.18"
 description = "Mass spectrometry data analysis package"
 authors = [
     { name = "Zamboni Lab" }

{masster-0.4.17 → masster-0.4.18}/src/masster/_version.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 
-__version__ = "0.4.17"
+__version__ = "0.4.18"
 
 
 def get_version():

{masster-0.4.17 → masster-0.4.18}/src/masster/study/defaults/merge_def.py

@@ -25,25 +25,25 @@ class merge_defaults:
         link_ms2 (bool): Whether to link MS2 spectra to consensus features. Default is True.
     """
 
-    method: str = "quality"
-    min_samples: int = 10
+    method: str = "qt"
+    min_samples: int = 2
     rt_tol: float = 5.0
     mz_tol: float = 0.01
-    chunk_size: int = 300
+    chunk_size: int = 500
     nr_partitions: int = 1000
-    min_rel_cc_size: float = 0.2
+    min_rel_cc_size: float = 0.1
     max_pairwise_log_fc: float = -1.0
     max_nr_conflicts: int = 0
     link_ms2: bool = True
     
     # KD-Strict specific parameters
     optimize_rt_tol: bool = False
-    rt_tol_range: tuple = (0.8, 2.0)  
-    rt_tol_steps: int = 5
-    secondary_merge_rt_tol: float = 0.5
+    rt_tol_range: tuple = (0.5, 4.0)  
+    rt_tol_steps: int = 7
+    secondary_merge_rt_tol: float = 1.0
     secondary_merge_mz_tol: float = 0.005
     min_sample_overlap: float = 0.8
-    max_rt_spread: float = None  # Will default to 2x rt_tol
+    max_rt_spread: float = 2.0  # Will default to 2x rt_tol
     min_coherence: float = 0.0
 
     _param_metadata: dict[str, dict[str, Any]] = field(
@@ -53,7 +53,8 @@ class merge_defaults:
                 "description": "Merge method (algorithm) to use",
                 "default": "quality",
                 "allowed_values": ["sensitivity", "qt", "nowarp", "chunked", "quality", 
-                                 "kd", "kd-nowarp", "kd_nowarp", "kd-strict", "kd_strict"],
+                                 "kd", "kd-nowarp", "kd_nowarp", "kd-strict", "kd_strict",
+                                 "kd_chunked", "kd-chunked", "qt_chunked", "qt-chunked"],
             },
             "min_samples": {
                 "dtype": int,

{masster-0.4.17 → masster-0.4.18}/src/masster/study/merge.py

@@ -1,6 +1,6 @@
 """
 Unified merge module for the Study class.
-Supports multiple merge methods: 'kd', 'qt', 'kd-nowarp', 'chunked'
+Supports multiple merge methods: 'kd', 'qt', 'kd-nowarp', 'kd_chunked', 'qt_chunked'
 """
 
 import time
@@ -25,7 +25,7 @@ def merge(self, **kwargs) -> None:
     **kwargs : dict
         Parameters from merge_defaults class:
         - method : str, default 'quality'
-          Merge algorithm: 'sensitivity', 'qt', 'nowarp', 'chunked', 'quality'
+          Merge algorithm: 'sensitivity', 'qt', 'nowarp', 'kd_chunked', 'qt_chunked', 'quality'
         - min_samples : int, default 10  
           Minimum number of samples for consensus feature
         - rt_tol : float, default 2.0
@@ -52,9 +52,11 @@ def merge(self, **kwargs) -> None:
     - Sensitivity: Best raw sensitivity, O(n log n), maximum feature detection
     - QT: Thorough but slow O(n²), good for <1000 samples  
     - NoWarp: Memory efficient KD without RT warping for large datasets
-    - Chunked: Memory-optimized KD algorithm for very large datasets (>5000 samples)
+    - KD-Chunked: Memory-optimized KD algorithm for very large datasets (>5000 samples)
       Uses optimized partitioning for better memory management while maintaining
       full cross-sample consensus feature detection.
+    - QT-Chunked: Memory-optimized QT algorithm for very large datasets (>5000 samples)
+      Uses QT clustering in first stage with optimized cross-chunk consensus building.
     """
     start_time = time.time()
     
@@ -76,7 +78,12 @@ def merge(self, **kwargs) -> None:
         'kd_nowarp': 'nowarp',
         'kd-strict': 'quality',
         'kd_strict': 'quality',
-        'kdstrict': 'quality'
+        'kdstrict': 'quality',
+        'chunked': 'kd_chunked',  # Map old 'chunked' to 'kd_chunked'
+        'qtchunked': 'qt_chunked',  # QT chunked variants
+        'qt-chunked': 'qt_chunked',
+        'kdchunked': 'kd_chunked',  # KD chunked variants 
+        'kd-chunked': 'kd_chunked'
     }
     
     if params.method in method_mapping:
@@ -85,8 +92,8 @@ def merge(self, **kwargs) -> None:
         self.logger.info(f"Method '{old_method}' is deprecated. Using '{params.method}' instead.")
     
     # Validate method
-    if params.method not in ['sensitivity', 'qt', 'nowarp', 'chunked', 'quality']:
-        raise ValueError(f"Invalid method '{params.method}'. Must be one of: ['sensitivity', 'qt', 'nowarp', 'chunked', 'quality']")
+    if params.method not in ['sensitivity', 'qt', 'nowarp', 'kd_chunked', 'qt_chunked', 'quality']:
+        raise ValueError(f"Invalid method '{params.method}'. Must be one of: ['sensitivity', 'qt', 'nowarp', 'kd_chunked', 'qt_chunked', 'quality']")
     
     # Persist last used params for diagnostics
     try:
@@ -147,9 +154,12 @@ def merge(self, **kwargs) -> None:
     elif params.method == 'quality':
         consensus_map = _merge_kd_strict(self, params)
         # Note: _merge_kd_strict handles both consensus_df and consensus_mapping_df directly
-    elif params.method == 'chunked':
-        consensus_map = _merge_chunked(self, params, cached_adducts_df, cached_valid_adducts)
-        # Note: _merge_chunked populates consensus_df directly, no need to extract
+    elif params.method == 'kd_chunked':
+        consensus_map = _merge_kd_chunked(self, params, cached_adducts_df, cached_valid_adducts)
+        # Note: _merge_kd_chunked populates consensus_df directly, no need to extract
+    elif params.method == 'qt_chunked':
+        consensus_map = _merge_qt_chunked(self, params, cached_adducts_df, cached_valid_adducts)
+        # Note: _merge_qt_chunked populates consensus_df directly, no need to extract
     
     # Perform adduct grouping
     self._perform_adduct_grouping(params.rt_tol, params.mz_tol)
@@ -189,9 +199,9 @@ def _merge_kd(self, params: merge_defaults) -> oms.ConsensusMap:
     params_oms.setValue("warp:mz_tol", params.mz_tol)
     params_oms.setValue("link:rt_tol", params.rt_tol)
     params_oms.setValue("link:mz_tol", params.mz_tol)
-    params_oms.setValue("link:min_rel_cc_size", params.min_rel_cc_size)
-    params_oms.setValue("link:max_pairwise_log_fc", params.max_pairwise_log_fc)
-    params_oms.setValue("link:max_nr_conflicts", params.max_nr_conflicts)
+    #params_oms.setValue("link:min_rel_cc_size", params.min_rel_cc_size)
+    #params_oms.setValue("link:max_pairwise_log_fc", params.max_pairwise_log_fc)
+    #params_oms.setValue("link:max_nr_conflicts", params.max_nr_conflicts)
     #params_oms.setValue("link:charge_merging", "With_charge_zero") THIS LEADS TO A CRASH
     
     grouper.setParameters(params_oms)
@@ -227,9 +237,9 @@ def _merge_qt(self, params: merge_defaults) -> oms.ConsensusMap:
     params_oms.setValue("distance_MZ:max_difference", params.mz_tol)
     params_oms.setValue("distance_MZ:unit", "Da")  # QT now uses Da like all other methods
     params_oms.setValue("ignore_charge", "true")
-    params_oms.setValue("min_rel_cc_size", params.min_rel_cc_size)
-    params_oms.setValue("max_pairwise_log_fc", params.max_pairwise_log_fc)
-    params_oms.setValue("max_nr_conflicts", params.max_nr_conflicts)
+    #params_oms.setValue("min_rel_cc_size", params.min_rel_cc_size)
+    #params_oms.setValue("max_pairwise_log_fc", params.max_pairwise_log_fc)
+    #params_oms.setValue("max_nr_conflicts", params.max_nr_conflicts)
     params_oms.setValue("nr_partitions", params.nr_partitions)
 
     grouper.setParameters(params_oms)
@@ -763,8 +773,8 @@ def _merge_kd_nowarp(self, params: merge_defaults) -> oms.ConsensusMap:
     return consensus_map
 
 
-def _merge_chunked(self, params: merge_defaults, cached_adducts_df=None, cached_valid_adducts=None) -> oms.ConsensusMap:
-    """Chunked merge with proper cross-chunk consensus building"""
+def _merge_kd_chunked(self, params: merge_defaults, cached_adducts_df=None, cached_valid_adducts=None) -> oms.ConsensusMap:
+    """KD-based chunked merge with proper cross-chunk consensus building"""
     
     n_samples = len(self.features_maps)
     if n_samples <= params.chunk_size:
@@ -826,6 +836,64 @@ def _merge_chunked(self, params: merge_defaults, cached_adducts_df=None, cached_
     return consensus_map
 
 
+def _merge_qt_chunked(self, params: merge_defaults, cached_adducts_df=None, cached_valid_adducts=None) -> oms.ConsensusMap:
+    """QT-based chunked merge with proper cross-chunk consensus building"""
+    
+    n_samples = len(self.features_maps)
+    if n_samples <= params.chunk_size:
+        self.logger.info(f"Dataset size ({n_samples}) ≤ chunk_size, using QT merge")
+        consensus_map = _merge_qt(self, params)
+        # Extract consensus features to populate consensus_df for chunked method consistency
+        self._extract_consensus_features(consensus_map, params.min_samples, cached_adducts_df, cached_valid_adducts)
+        return consensus_map
+    
+    # Process in chunks
+    chunks = []
+    for i in range(0, n_samples, params.chunk_size):
+        chunk_end = min(i + params.chunk_size, n_samples)
+        chunks.append((i, self.features_maps[i:chunk_end]))
+    
+    self.logger.debug(f"Processing {len(chunks)} chunks of max {params.chunk_size} samples")
+    
+    # Process each chunk to create chunk consensus maps
+    chunk_consensus_maps = []
+    
+    for chunk_idx, (chunk_start_idx, chunk_maps) in enumerate(tqdm(chunks, desc="Chunk", disable=self.log_level not in ["TRACE", "DEBUG", "INFO"])):
+        chunk_consensus_map = oms.ConsensusMap()
+        
+        # Set up file descriptions for chunk
+        file_descriptions = chunk_consensus_map.getColumnHeaders()
+        for j, feature_map in enumerate(chunk_maps):
+            file_description = file_descriptions.get(j, oms.ColumnHeader())
+            file_description.filename = self.samples_df.row(chunk_start_idx + j, named=True)["sample_name"]
+            file_description.size = feature_map.size()
+            file_description.unique_id = feature_map.getUniqueId()
+            file_descriptions[j] = file_description
+        
+        chunk_consensus_map.setColumnHeaders(file_descriptions)
+        
+        # Use QT algorithm for chunk (main difference from KD chunked)
+        grouper = oms.FeatureGroupingAlgorithmQT()
+        chunk_params = grouper.getParameters()
+        chunk_params.setValue("distance_RT:max_difference", params.rt_tol)
+        chunk_params.setValue("distance_MZ:max_difference", params.mz_tol)
+        chunk_params.setValue("distance_MZ:unit", "Da")
+        chunk_params.setValue("ignore_charge", "true")
+        chunk_params.setValue("nr_partitions", params.nr_partitions)
+        
+        grouper.setParameters(chunk_params)
+        grouper.group(chunk_maps, chunk_consensus_map)
+        
+        chunk_consensus_maps.append((chunk_start_idx, chunk_consensus_map))
+    
+    # Merge chunk results with proper cross-chunk consensus building
+    _merge_chunk_results(self, chunk_consensus_maps, params, cached_adducts_df, cached_valid_adducts)
+    
+    # Create a dummy consensus map for compatibility (since other functions expect it)
+    consensus_map = oms.ConsensusMap()
+    return consensus_map
+
+
 def _merge_chunk_results(self, chunk_consensus_maps: list, params: merge_defaults, cached_adducts_df=None, cached_valid_adducts=None) -> None:
     """
     Scalable aggregation of chunk consensus maps into final consensus_df.

{masster-0.4.17 → masster-0.4.18}/src/masster/study/processing.py

@@ -97,7 +97,6 @@ def align(self, **kwargs):
         _align_kd_algorithm(self, fmaps, params)
     else:
         self.logger.error(f"Unknown alignment algorithm '{algorithm}'")
-        self.logger.error(f"Unknown alignment algorithm '{algorithm}'")
 
     # check if rt_original exists in features_df, if not, add it after rt
     if "rt_original" not in self.features_df.columns:

{masster-0.4.17 → masster-0.4.18}/uv.lock

@@ -1374,7 +1374,7 @@ wheels = [
 
 [[package]]
 name = "masster"
-version = "0.4.17"
+version = "0.4.18"
 source = { editable = "." }
 dependencies = [
     { name = "alpharaw" },