msfiddle 0.1.0__py3-none-any.whl

msfiddle/config/fiddle_tcn_qtof.yml

@@ -0,0 +1,196 @@
+# ------------------------
+# QTOF train & validation 
+# ------------------------
+agilent_qtof: 
+  instrument_type: ESI-QTOF
+  instrument: ['Unknown']
+  ms_level: '2'
+  atom_type: ['C', 'O', 'N', 'H', 'P', 'S', 'F', 'Cl', 'B', 'Br', 'I', 'Na', 'K']
+  precursor_type: ['[M+H]+', '[M-H]-']
+  min_mz: 50
+  max_mz: 1500
+  min_peak_num: 5
+  max_atom_num: 300
+  min_atom_num: 10
+  ppm_tolerance: 10
+
+nist20_qtof: 
+  instrument_type: Q-TOF
+  instrument: ['Agilent QTOF 6530']
+  ms_level: 'MS2'
+  atom_type: ['C', 'O', 'N', 'H', 'P', 'S', 'F', 'Cl', 'B', 'Br', 'I', 'Na', 'K']
+  precursor_type: ['[M+H]+', '[M+Na]+', '[2M+H]+', # +1
+                    '[M+H-H2O]+', '[M-H2O+H]+', '[M+H-2H2O]+', '[M+H-NH3]+', '[M+H+NH3]+', '[M+NH4]+', '[M+H-CH2O2]+', '[M+H-CH4O2]+', # +1+-neutrue
+                    '[M-H]-', '[2M-H]-', # -1
+                    '[M-H-CO2]-', '[M-CHO2]-', '[M-H-H2O]-', '[M-H2O-H]-', # -1+-neutrue
+                    '[M+2H]2+', ] # +2
+  min_mz: 50
+  max_mz: 1500
+  min_peak_num: 5
+  max_atom_num: 300
+  min_atom_num: 10
+  ppm_tolerance: 10
+
+nist23_qtof: 
+  instrument_type: Q-TOF
+  instrument: ['Agilent QTOF 6530']
+  ms_level: 'MS2'
+  atom_type: ['C', 'O', 'N', 'H', 'P', 'S', 'F', 'Cl', 'B', 'Br', 'I', 'Na', 'K']
+  precursor_type: ['[M+H]+', '[M+Na]+', '[2M+H]+', # +1
+                    '[M+H-H2O]+', '[M-H2O+H]+', '[M+H-2H2O]+', '[M+H-NH3]+', '[M+H+NH3]+', '[M+NH4]+', '[M+H-CH2O2]+', '[M+H-CH4O2]+', # +1+-neutrue
+                    '[M-H]-', '[2M-H]-', # -1
+                    '[M-H-CO2]-', '[M-CHO2]-', '[M-H-H2O]-', '[M-H2O-H]-', # -1+-neutrue
+                    '[M+2H]2+', ] # +2
+  min_mz: 50
+  max_mz: 1500
+  min_peak_num: 5
+  max_atom_num: 300
+  min_atom_num: 10
+  ppm_tolerance: 10
+
+mona_qtof:
+  instrument_type: ['LC-ESI-QTOF', 
+                    'ESI-QFT'] 
+  instrument: ['Agilent 6530 Q-TOF', 'Waters Xevo G2 Q-Tof', 'Agilent qTOF 6545', 'Agilent 6550 iFunnel']
+  ms_level: 'MS2'
+  atom_type: ['C', 'O', 'N', 'H', 'P', 'S', 'F', 'Cl', 'B', 'Br', 'I', 'Na', 'K']
+  precursor_type: ['[M+H]+', '[M+Na]+', '[2M+H]+', # +1
+                    '[M+H-H2O]+', '[M-H2O+H]+', '[M+H-2H2O]+', '[M+H-NH3]+', '[M+H+NH3]+', '[M+NH4]+', '[M+H-CH2O2]+', '[M+H-CH4O2]+', # +1+-neutrue
+                    '[M-H]-', '[2M-H]-', # -1
+                    '[M-H-CO2]-', '[M-CHO2]-', '[M-H-H2O]-', '[M-H2O-H]-', # -1+-neutrue
+                    '[M+2H]2+', ] # +2
+  min_mz: 50
+  max_mz: 1500
+  min_peak_num: 5
+  max_atom_num: 300
+  min_atom_num: 10
+  ppm_tolerance: 10
+
+waters_qtof:
+  instrument_type: ['QTOF']
+  instrument: ['Waters QTOF']
+  atom_type: ['C', 'O', 'N', 'H', 'P', 'S', 'F', 'Cl', 'B', 'Br', 'I', 'Na', 'K']
+  precursor_type: ['[M+H]+', '[M-H]-']
+  min_mz: 50
+  max_mz: 1500
+  min_peak_num: 5
+  max_atom_num: 300
+  min_atom_num: 10
+  ppm_tolerance: 10
+
+gnps_qtof: 
+  instrument_type: ['hcd']
+  instrument: ['qtof', 'quadrupole']
+  atom_type: ['C', 'O', 'N', 'H', 'P', 'S', 'F', 'Cl', 'B', 'Br', 'I', 'Na', 'K']
+  precursor_type: ['[M+H]+', '[M+Na]+', '[2M+H]+', # +1
+                    '[M+H-H2O]+', '[M-H2O+H]+', '[M+H-2H2O]+', '[M+H-NH3]+', '[M+H+NH3]+', '[M+NH4]+', '[M+H-CH2O2]+', '[M+H-CH4O2]+', # +1+-neutrue
+                    '[M-H]-', '[2M-H]-', # -1
+                    '[M-H-CO2]-', '[M-CHO2]-', '[M-H-H2O]-', '[M-H2O-H]-', # -1+-neutrue
+                    '[M+2H]2+', ] # +2
+  min_mz: 50
+  max_mz: 1500
+  min_peak_num: 5
+  max_atom_num: 300
+  min_atom_num: 10
+  ppm_tolerance: 10
+
+# ------------------------
+# model settings
+# ------------------------
+encoding:
+  resolution: 0.2
+  max_mz: 1500
+  max_atom_num: 300
+  use_simulated_precursor_mz: False
+
+  precursor_type:
+    '[M+H]+': 1
+    '[M-H]-': 2
+    '[M+Na]+': 3
+    '[M+2H]2+': 4
+    '[2M+H]+': 5
+    '[2M-H]-': 6
+    
+  type2charge: 
+    '[M+H]+': '+1'
+    '[M-H]-': '-1'
+    '[M+Na]+': '+1'
+    '[M+2H]2+': '+2'
+    '[2M+H]+': '+1'
+    '[2M-H]-': '-1'
+    
+    '[M+H-H2O]+': '+1'
+    '[M-H2O+H]+': '+1'
+    '[M+H-2H2O]+': '+1'
+    '[M+H-NH3]+': '+1'
+    '[M+H+NH3]+': '+1'
+    '[M+NH4]+': '+1'
+    '[M+H-CH2O2]+': '+1'
+    '[M+H-CH4O2]+': '+1'
+
+    '[M-H-CO2]-': '-1' 
+    '[M-CHO2]-': '-1' 
+    '[M-H-H2O]-': '-1'
+    '[M-H2O-H]+': '-1'
+
+model: 
+  resolution: 0.2
+  max_mz: 1500
+  padding_dim: 20 # 1500 / 0.2 = 7500; 7500 / 2**5 (# pooling layer) = 234.375; 235 * (2**5) = 7520 
+  num_add: 6
+  input_channels: 1
+
+  tcn_channels: [32, 32, 64, 128, 256, 512]
+  tcn_dilations: [1, 2, 4, 8, 8, 8]
+  tcn_kernel_sizes: [45, 43, 41, 39, 37, 35]
+  tcn_dropout: 0.2
+
+  # metadata_dim: 16 
+  add_embedding_dim: 4
+  ce_embedding_dim: 4
+  mass_embedding_dim: 4
+  embedding_dim: 512
+  
+  formula_decoder_layers: [416, 208, 104, 52, 26, 13]
+  mass_decoder_layers: [416, 208, 104, 26, 13, 1]
+  atomnum_decoder_layers: [416, 208, 104, 26, 13, 1]
+  hcnum_decoder_layers: [416, 208, 104, 26, 13, 1]
+  output_dim: 13
+
+  fdr_decoder_layers: [416, 208, 104, 26, 13]
+
+# ------------------------
+# training settings
+# ------------------------
+train: 
+  epochs: 200
+  batch_size: 512 # 128
+  num_workers: 8
+  lr: 0.002 # 0.001
+  weight_decay: 0.01
+  warmup_ratio: 0.1
+  early_stop_step: 10
+  patience: 5
+  noised_times: 3
+  
+train_fdr:
+  epochs: 100
+  batch_size: 128
+  num_workers: 8
+  lr: 0.0001
+  early_stop_step: 10
+
+# ------------------------
+# postprocessing settings
+# ------------------------
+post_processing: 
+  refine_atom_type: ['C', 'O', 'N', 'H']
+  refine_atom_num: [5, 5, 5, -1]
+
+  mass_tolerance: 10.0 # target space
+  ppm_mode: True
+  time_out: -1 # do not limit the time
+  maxium_miss_atom_num: 50 # search space
+  top_k: 5
+  buddy_fdr_thr: 0.6
+  sirius_score_thr: -5

msfiddle/dataset.py

@@ -0,0 +1,152 @@
+import re
+import copy
+import pickle
+from pyteomics import mgf
+from tqdm import tqdm
+import numpy as np
+from decimal import *
+import pandas as pd
+import math
+from copy import deepcopy
+import torch
+from torch.utils.data import Dataset
+
+from .utils import ATOMS_INDEX, generate_ms, parse_collision_energy, unify_precursor_type, formula_to_dict, formula_to_vector
+
+
+
+# Used for testing
+class MGFDataset(Dataset):
+	def __init__(self, path, encoder): 
+		self.data = []
+		self.general_filter_config = {
+			'min_mz': 50, 
+			'max_mz': 1500, 
+			'min_peak_num': 5, 
+		}
+		self.use_simulated_precursor_mz = encoder['use_simulated_precursor_mz']
+		if self.use_simulated_precursor_mz: 
+			self.precursor_mz_key = 'simulated_precursor_mz'
+		else:
+			self.precursor_mz_key = 'precursor_mz'
+		
+		# read data from mgf file
+		spectra = mgf.read(path)
+		print(len(spectra), 'spectra loaded from', path)
+		
+		# filter out invalid data (add the other rules if needed)
+		spectra, _ = self.filter_spec(spectra, self.general_filter_config, type2charge=encoder['type2charge'])
+		print(len(spectra), 'spectra left after filtering')
+
+		# convert mgf to pkl
+		self.load_mgf_spectra(spectra, encoder)
+		print(len(self.data), 'spectra loaded into the dataset')
+
+	def __len__(self): 
+		return len(self.data)
+
+	def __getitem__(self, idx): 
+		return self.data[idx]['title'], self.data[idx]['precursor_type'], self.data[idx]['spec'][:, 0], self.data[idx]['env'], self.data[idx]['neutral_add']
+		
+	def filter_spec(self, spectra, general_filter_config, type2charge): 
+		clean_spectra = []
+		invalid_spectra = []
+		for spectrum in spectra: 
+			if not self.has_all_keys(spectrum): 
+				print('MGFError: lacking necessary keys in mgf file, skip this spectrum')
+				print('expected keys in mgf: (\'title\', \'{}\', \'precursor_type\', \'collision_energy\')'.format(self.precursor_mz_key))
+				continue
+
+			# filter out invalid data
+			maxium_mz = np.max(spectrum['m/z array'])
+			minium_mz = np.min(spectrum['m/z array'])
+			if maxium_mz < general_filter_config['min_mz'] or maxium_mz > general_filter_config['max_mz'] or \
+				len(spectrum['m/z array']) < general_filter_config['min_peak_num']: 
+				invalid_spectra.append(spectrum)
+			else: 
+				clean_spectra.append(spectrum)
+		return clean_spectra, invalid_spectra
+
+	def has_all_keys(self, spec): 
+		if self.use_simulated_precursor_mz:
+			keys = ['title', 'simulated_precursor_mz', 'precursor_type', 'collision_energy']
+		else:
+			keys = ['title', 'precursor_mz', 'precursor_type', 'collision_energy']
+		for k in keys:
+			if k not in spec['params'].keys(): 
+				return False
+		return True
+	
+	def load_mgf_spectra(self, spectra, encoder):  
+		for spectrum in spectra: 
+			good_spec, _, _, spec_arr = generate_ms(x=spectrum['m/z array'], 
+													y=spectrum['intensity array'], 
+													precursor_mz=float(spectrum['params'][self.precursor_mz_key]), 
+													resolution=encoder['resolution'], 
+													max_mz=encoder['max_mz'], 
+													charge=int(encoder['type2charge'][spectrum['params']['precursor_type']]))
+			if not good_spec: 
+				continue
+
+			adjust_neutral_add_vec, adjust_precursor_type = self.melt_neutral_precursor(spectrum['params']['precursor_type'])
+			
+			ce, nce = parse_collision_energy(ce_str=spectrum['params']['collision_energy'], 
+						precursor_mz=float(spectrum['params'][self.precursor_mz_key]), 
+						charge=abs(int(encoder['type2charge'][spectrum['params']['precursor_type']])))
+			if ce == None and nce == None: # can not process '30-50 eV'
+				continue
+
+			env_arr = np.array([float(spectrum['params'][self.precursor_mz_key]), 
+								nce, 
+								encoder['precursor_type'][adjust_precursor_type]]) 
+			
+			na_arr = np.array(adjust_neutral_add_vec)
+
+			self.data.append({'title': spectrum['params']['title'], 'precursor_type': spectrum['params']['precursor_type'], 
+								'spec': spec_arr, 'env': env_arr, 'neutral_add': na_arr})
+
+	def melt_neutral_precursor(self, precursor_type): # Used for testing only 
+		precursor_type = unify_precursor_type(precursor_type)
+
+		neutrue_list = ['CH4O2', 'CH2O2', 'H2O', 'NH3', 'CO2']
+		neutrue_counts = {}
+
+		# Step 1: Get the count of neutral losses/adducts
+		for neutrue in neutrue_list:
+			if neutrue in precursor_type:
+				pattern = r'([-+]?\d*)' + neutrue
+				count = re.findall(pattern, precursor_type)
+				if count: 
+					count = count[0]
+					if count == '+': count = 1
+					elif count == '-': count = -1
+					else: count = int(count)
+					neutrue_counts[neutrue] = count
+					break # only one neutral adduct is allowed
+
+		# Step 2: Remove neutral losses/adducts from the precursor type string
+		pattern = r'([-+]?\d*)(?:' + '|'.join(neutrue_list) + ')'
+		adjust_precursor_type = re.sub(pattern, '', precursor_type)
+
+		# Step 3: Convert the neutral losses/adducts into vector
+		adjust_neutral_add = {}
+		for neutrue, count in neutrue_counts.items(): 
+			n_dict = formula_to_dict(neutrue)
+			for k, v in n_dict.items(): 
+				if k in adjust_neutral_add:
+					adjust_neutral_add[k] += v * count
+				else:
+					adjust_neutral_add[k] = v * count
+		adjust_neutral_add_vec = self.formula_dict_to_vector(adjust_neutral_add)
+		
+		return adjust_neutral_add_vec, adjust_precursor_type
+
+	def formula_dict_to_vector(self, formula_dict): 
+		vector = [0] * len(ATOMS_INDEX)
+
+		for atom, count in formula_dict.items(): 
+			index = ATOMS_INDEX.get(atom, None)
+			if index is not None:
+				vector[index] = int(count) if count else 1
+
+		return vector