masster 0.4.17__py3-none-any.whl → 0.4.18__py3-none-any.whl

masster/_version.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 
-__version__ = "0.4.17"
+__version__ = "0.4.18"
 
 
 def get_version():

masster/study/defaults/merge_def.py

@@ -25,25 +25,25 @@ class merge_defaults:
         link_ms2 (bool): Whether to link MS2 spectra to consensus features. Default is True.
     """
 
-    method: str = "quality"
-    min_samples: int = 10
+    method: str = "qt"
+    min_samples: int = 2
     rt_tol: float = 5.0
     mz_tol: float = 0.01
-    chunk_size: int = 300
+    chunk_size: int = 500
     nr_partitions: int = 1000
-    min_rel_cc_size: float = 0.2
+    min_rel_cc_size: float = 0.1
     max_pairwise_log_fc: float = -1.0
     max_nr_conflicts: int = 0
     link_ms2: bool = True
     
     # KD-Strict specific parameters
     optimize_rt_tol: bool = False
-    rt_tol_range: tuple = (0.8, 2.0)  
-    rt_tol_steps: int = 5
-    secondary_merge_rt_tol: float = 0.5
+    rt_tol_range: tuple = (0.5, 4.0)  
+    rt_tol_steps: int = 7
+    secondary_merge_rt_tol: float = 1.0
     secondary_merge_mz_tol: float = 0.005
     min_sample_overlap: float = 0.8
-    max_rt_spread: float = None  # Will default to 2x rt_tol
+    max_rt_spread: float = 2.0  # Will default to 2x rt_tol
     min_coherence: float = 0.0
 
     _param_metadata: dict[str, dict[str, Any]] = field(
@@ -53,7 +53,8 @@ class merge_defaults:
                 "description": "Merge method (algorithm) to use",
                 "default": "quality",
                 "allowed_values": ["sensitivity", "qt", "nowarp", "chunked", "quality", 
-                                 "kd", "kd-nowarp", "kd_nowarp", "kd-strict", "kd_strict"],
+                                 "kd", "kd-nowarp", "kd_nowarp", "kd-strict", "kd_strict",
+                                 "kd_chunked", "kd-chunked", "qt_chunked", "qt-chunked"],
             },
             "min_samples": {
                 "dtype": int,

masster/study/merge.py

@@ -1,6 +1,6 @@
 """
 Unified merge module for the Study class.
-Supports multiple merge methods: 'kd', 'qt', 'kd-nowarp', 'chunked'
+Supports multiple merge methods: 'kd', 'qt', 'kd-nowarp', 'kd_chunked', 'qt_chunked'
 """
 
 import time
@@ -25,7 +25,7 @@ def merge(self, **kwargs) -> None:
     **kwargs : dict
         Parameters from merge_defaults class:
         - method : str, default 'quality'
-          Merge algorithm: 'sensitivity', 'qt', 'nowarp', 'chunked', 'quality'
+          Merge algorithm: 'sensitivity', 'qt', 'nowarp', 'kd_chunked', 'qt_chunked', 'quality'
         - min_samples : int, default 10  
           Minimum number of samples for consensus feature
         - rt_tol : float, default 2.0
@@ -52,9 +52,11 @@ def merge(self, **kwargs) -> None:
     - Sensitivity: Best raw sensitivity, O(n log n), maximum feature detection
     - QT: Thorough but slow O(n²), good for <1000 samples  
     - NoWarp: Memory efficient KD without RT warping for large datasets
-    - Chunked: Memory-optimized KD algorithm for very large datasets (>5000 samples)
+    - KD-Chunked: Memory-optimized KD algorithm for very large datasets (>5000 samples)
       Uses optimized partitioning for better memory management while maintaining
       full cross-sample consensus feature detection.
+    - QT-Chunked: Memory-optimized QT algorithm for very large datasets (>5000 samples)
+      Uses QT clustering in first stage with optimized cross-chunk consensus building.
     """
     start_time = time.time()
     
@@ -76,7 +78,12 @@ def merge(self, **kwargs) -> None:
         'kd_nowarp': 'nowarp',
         'kd-strict': 'quality',
         'kd_strict': 'quality',
-        'kdstrict': 'quality'
+        'kdstrict': 'quality',
+        'chunked': 'kd_chunked',  # Map old 'chunked' to 'kd_chunked'
+        'qtchunked': 'qt_chunked',  # QT chunked variants
+        'qt-chunked': 'qt_chunked',
+        'kdchunked': 'kd_chunked',  # KD chunked variants 
+        'kd-chunked': 'kd_chunked'
     }
     
     if params.method in method_mapping:
@@ -85,8 +92,8 @@ def merge(self, **kwargs) -> None:
         self.logger.info(f"Method '{old_method}' is deprecated. Using '{params.method}' instead.")
     
     # Validate method
-    if params.method not in ['sensitivity', 'qt', 'nowarp', 'chunked', 'quality']:
-        raise ValueError(f"Invalid method '{params.method}'. Must be one of: ['sensitivity', 'qt', 'nowarp', 'chunked', 'quality']")
+    if params.method not in ['sensitivity', 'qt', 'nowarp', 'kd_chunked', 'qt_chunked', 'quality']:
+        raise ValueError(f"Invalid method '{params.method}'. Must be one of: ['sensitivity', 'qt', 'nowarp', 'kd_chunked', 'qt_chunked', 'quality']")
     
     # Persist last used params for diagnostics
     try:
@@ -147,9 +154,12 @@ def merge(self, **kwargs) -> None:
     elif params.method == 'quality':
         consensus_map = _merge_kd_strict(self, params)
         # Note: _merge_kd_strict handles both consensus_df and consensus_mapping_df directly
-    elif params.method == 'chunked':
-        consensus_map = _merge_chunked(self, params, cached_adducts_df, cached_valid_adducts)
-        # Note: _merge_chunked populates consensus_df directly, no need to extract
+    elif params.method == 'kd_chunked':
+        consensus_map = _merge_kd_chunked(self, params, cached_adducts_df, cached_valid_adducts)
+        # Note: _merge_kd_chunked populates consensus_df directly, no need to extract
+    elif params.method == 'qt_chunked':
+        consensus_map = _merge_qt_chunked(self, params, cached_adducts_df, cached_valid_adducts)
+        # Note: _merge_qt_chunked populates consensus_df directly, no need to extract
     
     # Perform adduct grouping
     self._perform_adduct_grouping(params.rt_tol, params.mz_tol)
@@ -189,9 +199,9 @@ def _merge_kd(self, params: merge_defaults) -> oms.ConsensusMap:
     params_oms.setValue("warp:mz_tol", params.mz_tol)
     params_oms.setValue("link:rt_tol", params.rt_tol)
     params_oms.setValue("link:mz_tol", params.mz_tol)
-    params_oms.setValue("link:min_rel_cc_size", params.min_rel_cc_size)
-    params_oms.setValue("link:max_pairwise_log_fc", params.max_pairwise_log_fc)
-    params_oms.setValue("link:max_nr_conflicts", params.max_nr_conflicts)
+    #params_oms.setValue("link:min_rel_cc_size", params.min_rel_cc_size)
+    #params_oms.setValue("link:max_pairwise_log_fc", params.max_pairwise_log_fc)
+    #params_oms.setValue("link:max_nr_conflicts", params.max_nr_conflicts)
     #params_oms.setValue("link:charge_merging", "With_charge_zero") THIS LEADS TO A CRASH
     
     grouper.setParameters(params_oms)
@@ -227,9 +237,9 @@ def _merge_qt(self, params: merge_defaults) -> oms.ConsensusMap:
     params_oms.setValue("distance_MZ:max_difference", params.mz_tol)
     params_oms.setValue("distance_MZ:unit", "Da")  # QT now uses Da like all other methods
     params_oms.setValue("ignore_charge", "true")
-    params_oms.setValue("min_rel_cc_size", params.min_rel_cc_size)
-    params_oms.setValue("max_pairwise_log_fc", params.max_pairwise_log_fc)
-    params_oms.setValue("max_nr_conflicts", params.max_nr_conflicts)
+    #params_oms.setValue("min_rel_cc_size", params.min_rel_cc_size)
+    #params_oms.setValue("max_pairwise_log_fc", params.max_pairwise_log_fc)
+    #params_oms.setValue("max_nr_conflicts", params.max_nr_conflicts)
     params_oms.setValue("nr_partitions", params.nr_partitions)
 
     grouper.setParameters(params_oms)
@@ -763,8 +773,8 @@ def _merge_kd_nowarp(self, params: merge_defaults) -> oms.ConsensusMap:
     return consensus_map
 
 
-def _merge_chunked(self, params: merge_defaults, cached_adducts_df=None, cached_valid_adducts=None) -> oms.ConsensusMap:
-    """Chunked merge with proper cross-chunk consensus building"""
+def _merge_kd_chunked(self, params: merge_defaults, cached_adducts_df=None, cached_valid_adducts=None) -> oms.ConsensusMap:
+    """KD-based chunked merge with proper cross-chunk consensus building"""
     
     n_samples = len(self.features_maps)
     if n_samples <= params.chunk_size:
@@ -826,6 +836,64 @@ def _merge_chunked(self, params: merge_defaults, cached_adducts_df=None, cached_
     return consensus_map
 
 
+def _merge_qt_chunked(self, params: merge_defaults, cached_adducts_df=None, cached_valid_adducts=None) -> oms.ConsensusMap:
+    """QT-based chunked merge with proper cross-chunk consensus building"""
+    
+    n_samples = len(self.features_maps)
+    if n_samples <= params.chunk_size:
+        self.logger.info(f"Dataset size ({n_samples}) ≤ chunk_size, using QT merge")
+        consensus_map = _merge_qt(self, params)
+        # Extract consensus features to populate consensus_df for chunked method consistency
+        self._extract_consensus_features(consensus_map, params.min_samples, cached_adducts_df, cached_valid_adducts)
+        return consensus_map
+    
+    # Process in chunks
+    chunks = []
+    for i in range(0, n_samples, params.chunk_size):
+        chunk_end = min(i + params.chunk_size, n_samples)
+        chunks.append((i, self.features_maps[i:chunk_end]))
+    
+    self.logger.debug(f"Processing {len(chunks)} chunks of max {params.chunk_size} samples")
+    
+    # Process each chunk to create chunk consensus maps
+    chunk_consensus_maps = []
+    
+    for chunk_idx, (chunk_start_idx, chunk_maps) in enumerate(tqdm(chunks, desc="Chunk", disable=self.log_level not in ["TRACE", "DEBUG", "INFO"])):
+        chunk_consensus_map = oms.ConsensusMap()
+        
+        # Set up file descriptions for chunk
+        file_descriptions = chunk_consensus_map.getColumnHeaders()
+        for j, feature_map in enumerate(chunk_maps):
+            file_description = file_descriptions.get(j, oms.ColumnHeader())
+            file_description.filename = self.samples_df.row(chunk_start_idx + j, named=True)["sample_name"]
+            file_description.size = feature_map.size()
+            file_description.unique_id = feature_map.getUniqueId()
+            file_descriptions[j] = file_description
+        
+        chunk_consensus_map.setColumnHeaders(file_descriptions)
+        
+        # Use QT algorithm for chunk (main difference from KD chunked)
+        grouper = oms.FeatureGroupingAlgorithmQT()
+        chunk_params = grouper.getParameters()
+        chunk_params.setValue("distance_RT:max_difference", params.rt_tol)
+        chunk_params.setValue("distance_MZ:max_difference", params.mz_tol)
+        chunk_params.setValue("distance_MZ:unit", "Da")
+        chunk_params.setValue("ignore_charge", "true")
+        chunk_params.setValue("nr_partitions", params.nr_partitions)
+        
+        grouper.setParameters(chunk_params)
+        grouper.group(chunk_maps, chunk_consensus_map)
+        
+        chunk_consensus_maps.append((chunk_start_idx, chunk_consensus_map))
+    
+    # Merge chunk results with proper cross-chunk consensus building
+    _merge_chunk_results(self, chunk_consensus_maps, params, cached_adducts_df, cached_valid_adducts)
+    
+    # Create a dummy consensus map for compatibility (since other functions expect it)
+    consensus_map = oms.ConsensusMap()
+    return consensus_map
+
+
 def _merge_chunk_results(self, chunk_consensus_maps: list, params: merge_defaults, cached_adducts_df=None, cached_valid_adducts=None) -> None:
     """
     Scalable aggregation of chunk consensus maps into final consensus_df.

masster/study/processing.py

@@ -97,7 +97,6 @@ def align(self, **kwargs):
         _align_kd_algorithm(self, fmaps, params)
     else:
         self.logger.error(f"Unknown alignment algorithm '{algorithm}'")
-        self.logger.error(f"Unknown alignment algorithm '{algorithm}'")
 
     # check if rt_original exists in features_df, if not, add it after rt
     if "rt_original" not in self.features_df.columns:

{masster-0.4.17.dist-info → masster-0.4.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: masster
-Version: 0.4.17
+Version: 0.4.18
 Summary: Mass spectrometry data analysis package
 Project-URL: homepage, https://github.com/zamboni-lab/masster
 Project-URL: repository, https://github.com/zamboni-lab/masster

{masster-0.4.17.dist-info → masster-0.4.18.dist-info}/RECORD

@@ -1,5 +1,5 @@
 masster/__init__.py,sha256=HHjKhCjkAc98LhoQfu4C6L-W2vfTEc1iXaPTxxcl_4A,800
-masster/_version.py,sha256=A-Vx5wjFdgUfquBN1kWTW90q7wTOwZx-uonA2Xl-IWc,257
+masster/_version.py,sha256=OUcHIwT4wa5AqV46S88edNYE4u4sKsoESNk3lFdxs_c,257
 masster/chromatogram.py,sha256=iYpdv8C17zVnlWvOFgAn9ns2uFGiF-GgoYf5QVVAbHs,19319
 masster/logger.py,sha256=W50V_uh8RSYwGxDrDFhOuj5jpu2tKJyt_16lMw9kQwA,14755
 masster/spectrum.py,sha256=_upC_g2N9gwTaflXAugs9pSXpKUmzbIehofDordk7WI,47718
@@ -43,10 +43,10 @@ masster/study/h5.py,sha256=LiVGUAtULyPpZIUmKVJSaV38huJb8FsKOUWBOqiv0QU,82363
 masster/study/helpers.py,sha256=M5_q8O5tuFchKPW04PTuj3X335lDA2VZqcs4D8ZQJEk,158604
 masster/study/id.py,sha256=6NUBBKZCFOU1wlDKM0eXQeOIStSZCRNJ_3x7ZaIHzmM,55263
 masster/study/load.py,sha256=CQQY_7BzagE3oQTdDlqNyfuMdVWIAft-M4a2WCFnxp0,70695
-masster/study/merge.py,sha256=-gc-255NTKxkJZcIRl1wqQsMMi0m8zoZ10BkGsINFDc,92012
+masster/study/merge.py,sha256=2Vqj0OaTZxwtjYu1l5PmRpMmT8_cHh-R761FUvBE_Sk,95741
 masster/study/parameters.py,sha256=0elaF7YspTsB7qyajWAbRNL2VfKlGz5GJLifmO8IGkk,3276
 masster/study/plot.py,sha256=SimX-IlqISEItAnTBsx4xsdYHRAevfN41cCENVns1lw,88236
-masster/study/processing.py,sha256=pm98FrQHoM3ov6qmjKuVN9h2KBhGgCLEZCRS7zpmJFM,41104
+masster/study/processing.py,sha256=u1MSRKTzcqHNz_dClSUSfgTxkNRdBLXtVyO5LXuW_uk,41031
 masster/study/save.py,sha256=YCvp4xhnG16sNXaT2mFDBoCrIMub0Es61B97qLo0maw,6705
 masster/study/study.py,sha256=LO_hbJOOCZzeA3uterPKImFgPG6fCNQKMSVMtEwW3DU,38815
 masster/study/study5_schema.json,sha256=c0w24QdHak01m04I1VPu97KvF2468FcaqROhf6pmLk4,7507
@@ -60,7 +60,7 @@ masster/study/defaults/find_ms2_def.py,sha256=RL0DFG41wQ05U8UQKUGr3vzSl3mU0m0knQ
 masster/study/defaults/identify_def.py,sha256=96rxoCAPQj_yX-3mRoD2LTkTLJgG27eJQqwarLv5jL0,10580
 masster/study/defaults/integrate_chrom_def.py,sha256=0MNIWGTjty-Zu-NTQsIweuj3UVqEY3x1x8pK0mPwYak,7264
 masster/study/defaults/integrate_def.py,sha256=Vf4SAzdBfnsSZ3IRaF0qZvWu3gMDPHdgPfMYoPKeWv8,7246
-masster/study/defaults/merge_def.py,sha256=Q31JwAaVGgVPEVIsiyeiOsF97c48IKe48HXuqh-sA_k,13189
+masster/study/defaults/merge_def.py,sha256=X7mTCgtQhglOTjwg06oSMFSbLBJSKsHmJeVVfYE2qHE,13272
 masster/study/defaults/study_def.py,sha256=h8dYbi9xv0sesCSQik49Z53IkskMmNtW6ixl7it5pL0,16033
 masster/wizard/README.md,sha256=mL1A3YWJZOefpJ6D0-HqGLkVRmUlOpwyVFdvJBeeoZM,14149
 masster/wizard/__init__.py,sha256=A9GHQvkq4lSRIA8V6AKB-TJy8s_npH8i1baUGdkw_is,364
@@ -68,8 +68,8 @@ masster/wizard/example.py,sha256=xEZFTH9UZ8HKOm6s3JL8Js0Uw5ChnISWBHSZCL32vsM,798
 masster/wizard/test_structure.py,sha256=h88gsYYCG6iDRjqPZC_r1H1T8y79j0E-K6OrwuHaSCU,1586
 masster/wizard/test_wizard.py,sha256=CMp1cpjH3iYYC5Fy6puF_K0kfwwk3bgOsSbUGW-t7Xk,8986
 masster/wizard/wizard.py,sha256=jMLHy4cXgNEE_-vshFmA7BNEByhfA6tV7O91jhiMYuw,48054
-masster-0.4.17.dist-info/METADATA,sha256=uIdQNkAXQQzMkcVM53y_pUBZPzwqOx0lxGW8nmB1lz8,44207
-masster-0.4.17.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-masster-0.4.17.dist-info/entry_points.txt,sha256=ZHguQ_vPmdbpqq2uGtmEOLJfgP-DQ1T0c07Lxh30wc8,58
-masster-0.4.17.dist-info/licenses/LICENSE,sha256=bx5iLIKjgAdYQ7sISn7DsfHRKkoCUm1154sJJKhgqnU,35184
-masster-0.4.17.dist-info/RECORD,,
+masster-0.4.18.dist-info/METADATA,sha256=pn-XNHgHqlY1KgiYkQ2Dyke9E1nnCP3mn-ja5W5QPyM,44207
+masster-0.4.18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+masster-0.4.18.dist-info/entry_points.txt,sha256=ZHguQ_vPmdbpqq2uGtmEOLJfgP-DQ1T0c07Lxh30wc8,58
+masster-0.4.18.dist-info/licenses/LICENSE,sha256=bx5iLIKjgAdYQ7sISn7DsfHRKkoCUm1154sJJKhgqnU,35184
+masster-0.4.18.dist-info/RECORD,,