CoreMS 4.0.0__py3-none-any.whl

examples/archive/scripts/GC-MS NetCDF.py

@@ -0,0 +1,217 @@
+import warnings
+warnings.filterwarnings("ignore")
+
+import sys
+sys.path.append("./")
+
+from pathlib import Path
+from multiprocessing import Pool
+import cProfile
+
+
+from corems.molecular_id.input.nistMSI import ReadNistMSI
+from corems.mass_spectra.input.andiNetCDF import ReadAndiNetCDF
+from corems.molecular_id.search.compoundSearch import LowResMassSpectralMatch
+from corems.mass_spectra.calc.GC_RI_Calibration import get_rt_ri_pairs
+from support_code.nmdc.filefinder import get_dirname, get_filename
+
+import glob
+
+def start_sql_from_file():
+
+    ref_lib_path = Path.cwd() / "tests/tests_data/gcms/" / "PNNLMetV20191015.MSL"
+    sql_obj = ReadNistMSI(ref_lib_path).get_sqlLite_obj()
+    return sql_obj
+
+def sql_database(file_location):
+
+    sqlLite_obj = ReadNistMSI(file_location).get_sqlLite_obj()
+
+    min_max_rt = (18.037, 18.037)
+    min_max_ri = (1637.30, 1737.30)
+
+    sqlLite_obj.query_min_max_ri((1637.30, 1638.30))
+    sqlLite_obj.query_min_max_rt((17.111, 18.111))
+    sqlLite_obj.query_min_max_ri_and_rt((1637.30, 1638.30), (17.111, 18.111))
+
+def stand_alone():
+
+    file_path = get_filename()
+
+    reader_gcms = ReadAndiNetCDF(file_path)
+
+    reader_gcms.run()
+
+    gcms = reader_gcms.get_gcms_obj()
+
+    gcms.process_chromatogram()
+
+def get_gcms(file_path):
+
+    reader_gcms = ReadAndiNetCDF(file_path)
+
+    reader_gcms.run()
+
+    gcms = reader_gcms.get_gcms_obj()
+
+    # gcms.process_chromatogram()
+
+    return gcms
+
+def get_reference_dict(calibration_file_path=False):
+
+    from PySide2.QtWidgets import QFileDialog, QApplication
+    from PySide2.QtCore import Qt
+
+    if not calibration_file_path:
+        app = QApplication(sys.argv)
+        file_dialog = QFileDialog()
+        file_dialog.setWindowFlags(Qt.WindowStaysOnTopHint)
+        file_path = file_dialog.getOpenFileName(None, "FAMES REF FILE", filter="*.cdf")[0]
+        file_dialog.close()
+        app.exit()
+    else:
+        file_path = calibration_file_path
+
+    if not file_path:
+        return None
+
+    else:
+
+        gcms_ref_obj = get_gcms(file_path)
+        # sql_obj = start_sql_from_file()
+        rt_ri_pairs = get_rt_ri_pairs(gcms_ref_obj)  # sql_obj=sql_obj)
+        # !!!!!! READ !!!!! use the previous two lines if db/pnnl_lowres_gcms_compounds.sqlite does not exist
+        # and comment the next line
+        # rt_ri_pairs = get_rt_ri_pairs(gcms_ref_obj)
+
+        return rt_ri_pairs, file_path
+
+def run(args):
+
+    file_path, ref_dict, cal_file_path = args
+
+    gcms = get_gcms(file_path)
+
+    gcms.process_chromatogram()
+
+    gcms.calibrate_ri(ref_dict, cal_file_path)
+
+    # sql_obj = start_sql_from_file()
+
+    lowResSearch = LowResMassSpectralMatch(gcms)  # sql_obj=sql_obj)
+    # !!!!!! READ !!!!! use the previous two lines if db/pnnl_lowres_gcms_compounds.sqlite does not exist
+    # and comment the next line
+    # lowResSearch = LowResMassSpectralMatch(gcms)
+    lowResSearch.run()
+
+    return gcms
+
+def auto_calibrate_and_search(file_locations, output_file_name, jobs, calibration_file_path):
+
+    ref_dict, cal_file_path = get_reference_dict(calibration_file_path=calibration_file_path)
+
+    if ref_dict:
+
+        # run in multiprocessing mode
+        pool = Pool(jobs)
+        args = [(file_path, ref_dict, cal_file_path) for file_path in file_locations]
+        gcmss = pool.map(run, args)
+        pool.close()
+        pool.join()
+        for gcms in gcmss:
+
+            gcms.to_hdf()
+            gcms.to_csv(output_file_name)
+            # print(output_file_name)
+
+
+def calibrate_and_search(out_put_file_name, jobs):
+
+    from PySide2.QtWidgets import QFileDialog
+    from PySide2.QtCore import Qt
+
+    ref_dict, cal_file_path = get_reference_dict()
+    if ref_dict:
+
+        file_dialog = QFileDialog()
+        file_dialog.setWindowFlags(Qt.WindowStaysOnTopHint)
+
+        if file_dialog:
+
+            file_locations = file_dialog.getOpenFileNames(None, "Standard Compounds Files", filter="*.cdf")
+            file_dialog.close()
+
+            # run in multiprocessing mode
+            pool = Pool(jobs)
+            args = [(file_path, ref_dict, cal_file_path) for file_path in file_locations[0]]
+            gcmss = pool.map(run, args)
+            pool.close()
+            pool.join()
+
+            for file_index, gcms in enumerate(gcmss):
+
+                file_path = Path(file_locations[0][file_index])
+                # print(out_put_file_name)
+
+                gcms.to_csv(out_put_file_name, write_metadata=True, id_label="emsl:")
+
+                # gcms.to_excel(out_put_file_name)
+                # gcms.to_pandas(out_put_file_name)
+                gcms.to_hdf()
+
+                # df = gcms.get_dataframe()
+                # json_data = gcms.to_json()
+
+                # print(json_data)
+
+                # gcms.plot_processed_chromatogram()
+
+                # gcms.plot_gc_peaks()
+
+                # gcms.plot_chromatogram()
+
+                # gcms.plot_smoothed_chromatogram()
+
+                # gcms.plot_baseline_subtraction()
+
+                # gcms.plot_detected_baseline()
+
+                # matplotlib.pyplot.show()
+
+def worker(args):
+
+    cProfile.runctx('run(args)', globals(), locals(), 'gc-ms.prof')
+
+def auto_process(jobs):
+
+    import os
+    rootdir = get_dirname()
+
+    out_put_file_names = list(os.walk(rootdir))[0][1]
+
+    # print(out_put_file_names[0])
+    for out_put_file_name in out_put_file_names:
+        # print(out_put_file_name)
+
+        file_locations = glob.glob(str((rootdir / out_put_file_name)) + "/*.cdf")
+        calibration_file_path = ''
+        for file_path in file_locations:
+            if "FAME" in file_path:
+                calibration_file_path = file_path
+        if calibration_file_path:
+            auto_calibrate_and_search(file_locations, out_put_file_name, jobs, calibration_file_path)
+        else:
+            print("Could not find a calibration experimental file for {}".format(out_put_file_name))
+
+
+if __name__ == '__main__':
+    # import matplotlib
+    # matplotlib.use('TkAgg')
+    # %%
+    cores = 4
+    out_put_file_group_name = 'sql_test'
+    calibrate_and_search(out_put_file_group_name, cores)
+    # start_sql_from_file()
+    # auto_process(cores)
+    # stand_alone()

examples/archive/scripts/HR-MS Thermo Raw 21T.py

@@ -0,0 +1,136 @@
+__author__ = "Yuri E. Corilo"
+__date__ = "Jun 09, 2021"
+
+import warnings
+warnings.filterwarnings("ignore")
+
+import sys
+sys.path.append("./")
+
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+# from PySide2.QtWidgets import QFileDialog, QApplication
+# from PySide2.QtCore import Qt
+
+from corems.mass_spectra.input import rawFileReader
+from corems.molecular_id.factory.classification import HeteroatomsClassification, Labels
+from corems.molecular_id.search.priorityAssignment import OxygenPriorityAssignment
+from corems.molecular_id.search.molecularFormulaSearch import SearchMolecularFormulas
+from corems.encapsulation.factory.parameters import MSParameters
+
+def run_thermo(file_location):
+
+    MSParameters.mass_spectrum.noise_threshold_method = 'relative_abundance'
+    MSParameters.mass_spectrum.noise_threshold_min_relative_abundance = 0.1
+
+    # MSParameters.mass_spectrum.noise_threshold_method = 'log'
+    # MSParameters.mass_spectrum.noise_threshold_min_s2n = 6
+
+    parser = rawFileReader.ImportMassSpectraThermoMSFileReader(file_location)
+
+    parser.chromatogram_settings.scans = (-1, -1)
+    
+    tic_data, ax = parser.get_tic(ms_type='MS', plot=True)
+
+    plt.show()
+
+    print(parser.get_all_filters())
+
+    transient_time_list = parser.get_icr_transient_times()
+
+    print(transient_time_list)
+
+    # sums all the mass spectra
+    mass_spectrum = parser.get_average_mass_spectrum()
+
+    # sums scans in selected range
+    parser.chromatogram_settings.scans = (1, 10)
+    
+    mass_spectrum = parser.get_average_mass_spectrum()
+    
+    parser.chromatogram_settings.scans = [1]
+    # sums scans in selected range
+    mass_spectrum = parser.get_average_mass_spectrum()
+
+    return mass_spectrum
+
+def run_assignment(file_location):
+
+    # mass_spectrum = run_bruker(file_location)
+    # mass_spectrum = get_masslist(file_location)
+    mass_spectrum = run_thermo(file_location)
+
+    mass_spectrum.molecular_search_settings.error_method = 'None'
+    mass_spectrum.molecular_search_settings.min_ppm_error = -5
+    mass_spectrum.molecular_search_settings.max_ppm_error = 5
+
+    mass_spectrum.molecular_search_settings.url_database = None
+    mass_spectrum.molecular_search_settings.min_dbe = 0
+    mass_spectrum.molecular_search_settings.max_dbe = 50
+
+    mass_spectrum.molecular_search_settings.usedAtoms['C'] = (1, 100)
+    mass_spectrum.molecular_search_settings.usedAtoms['H'] = (4, 200)
+    mass_spectrum.molecular_search_settings.usedAtoms['O'] = (1, 30)
+    mass_spectrum.molecular_search_settings.usedAtoms['N'] = (0, 0)
+    mass_spectrum.molecular_search_settings.usedAtoms['S'] = (0, 0)
+    mass_spectrum.molecular_search_settings.usedAtoms['Cl'] = (0, 0)
+    mass_spectrum.molecular_search_settings.usedAtoms['Br'] = (0, 0)
+    mass_spectrum.molecular_search_settings.usedAtoms['P'] = (0, 0)
+    mass_spectrum.molecular_search_settings.usedAtoms['Na'] = (0, 0)
+
+    mass_spectrum.molecular_search_settings.isProtonated = True
+    mass_spectrum.molecular_search_settings.isRadical = False
+    mass_spectrum.molecular_search_settings.isAdduct = False
+
+    # mass_spectrum.filter_by_max_resolving_power(15, 2)
+    SearchMolecularFormulas(mass_spectrum, first_hit=False).run_worker_mass_spectrum()
+
+    mass_spectrum.percentile_assigned(report_error=True)
+    mass_spectrum.molecular_search_settings.score_method = "prob_score"
+    mass_spectrum.molecular_search_settings.output_score_method = "prob_score"
+
+    # export_calc_isotopologues(mass_spectrum, "15T_Neg_ESI_SRFA_Calc_Isotopologues")
+
+    mass_spectrum_by_classes = HeteroatomsClassification(mass_spectrum, choose_molecular_formula=True)
+
+    mass_spectrum_by_classes.plot_ms_assigned_unassigned()
+    plt.show()
+    mass_spectrum_by_classes.plot_mz_error()
+    plt.show()
+    mass_spectrum_by_classes.plot_ms_class("O2")
+    plt.show()
+    # dataframe = mass_spectrum_by_classes.to_dataframe()
+    return mass_spectrum
+
+    # class_plot(dataframe)
+
+if __name__ == "__main__":
+
+    # app = QApplication(sys.argv)
+    # file_dialog = QFileDialog()
+    # file_dialog.setWindowFlags(Qt.WindowStaysOnTopHint)
+    # file_location = file_dialog.getOpenFileName()[0]
+    # app.quit()
+
+    file_location = "tests/tests_data/ftms/Exploris_SRFA_Example.raw"
+    # change parameters here
+
+    mass_spectrum = run_thermo(file_location)
+
+    print("acquisition_time:",  mass_spectrum.acquisition_time )
+    ax = mass_spectrum.plot_mz_domain_profile()
+
+    #for mspeak in mass_spectrum:
+    #    mspeak.plot(ax=ax)
+    #plt.show()
+    #plt.savefig("test.png")
+
+    #mass_spectrum.plot_profile_and_noise_threshold()
+    #plt.show()
+    #plt.savefig("test.png")
+
+    mass_spectrum = run_assignment(file_location)
+
+    mass_spectrum.to_csv(mass_spectrum.sample_name)
+    # print("polarity", mass_spectrum.polarity)

examples/archive/scripts/LC-ICPMS_metal_peaks.py

@@ -0,0 +1,297 @@
+##### CWD 2021-10-27
+#### (1) import .csv with LC-ICPMS data; (2) for each metal, pick peaks; (3) return dictionary with RT as key, metal as value
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Tuple
+import warnings
+import sys
+
+sys.path.append("./")
+warnings.filterwarnings("ignore")
+
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+
+from corems.encapsulation.factory.parameters import LCMSParameters
+from corems.mass_spectra.input import rawFileReader
+from corems.mass_spectra.calc import SignalProcessing as sp
+from corems.molecular_id.search.molecularFormulaSearch import SearchMolecularFormulas
+
+@dataclass
+class TIC_Data:
+     '''
+    Scans: [int]
+        original thermo scan numbers
+    Time: [floats]
+        list of retention times
+    TIC: [floats]
+        total ion chromatogram
+    Apexes: [int]    
+        original thermo apex scan number after peak picking 
+     '''
+     
+     scans : List[int] = field(default_factory=list)
+     time : List[float] = field(default_factory=list)
+     tic : List[float] = field(default_factory=list)
+     apexes : List[int] = field(default_factory=list)
+
+@dataclass
+class EIC_Data:
+     '''
+    Scans: [int]
+        original thermo scan numbers
+    Time: [floats]
+        list of retention times
+    EIC: [floats]
+        extracted ion chromatogram
+    Apexes: [int]    
+        original thermo apex scan number after peak picking 
+    
+     '''
+     metal: str
+     scans : List[int] = field(default_factory=list)
+     time : List[float] = field(default_factory=list)
+     eic : List[float] = field(default_factory=list)
+     apexes : List[int] = field(default_factory=list)
+
+def eic_centroid_detector(max_tic, eic_data:EIC_Data, parameters:LCMSParameters, smooth=True):
+    # Do peak picking and store results inside EIC_Data class    
+	
+	max_prominence = parameters.lc_ms.peak_max_prominence_percent
+
+	max_height = parameters.lc_ms.peak_height_max_percent
+
+	signal_threshold = parameters.lc_ms.eic_signal_threshold
+
+	min_peak_datapoints = parameters.lc_ms.min_peak_datapoints
+
+	correct_baseline = parameters.lc_ms.correct_eic_baseline
+	
+	peak_derivative_threshold = parameters.lc_ms.peak_derivative_threshold
+
+	if smooth:
+		
+		eic_signal = smooth_signal(eic_data.eic, parameters)
+	
+	else:
+	
+		eic_signal = eic_data.eic
+
+	peak_indexes_generator = sp.peak_picking_first_derivative(eic_data.time, eic_signal, max_height, max_prominence, 
+															  max(eic_signal), min_peak_datapoints,
+															  peak_derivative_threshold, 
+															  signal_threshold=signal_threshold, 
+															  correct_baseline=correct_baseline,
+															  check_abundance=True,
+															  plot_res=False,)
+	eic_data.apexes = [i for i in peak_indexes_generator]
+	
+	plt.plot(eic_data.time/60, eic_signal, label=eic_data.metal)
+	
+	for peak_index in eic_data.apexes:
+		plt.plot(eic_data.time[peak_index[1]]/60, eic_signal[peak_index[1]], marker='x')
+	
+	plt.xlabel('Retention Time (min)')
+	plt.ylabel('Intensity (cps)')
+	plt.legend()
+	plt.show()
+
+def smooth_signal(signal, parameters:LCMSParameters):
+            
+	implemented_smooth_method = parameters.lc_ms.implemented_smooth_method 
+	
+	pol_order = parameters.lc_ms.savgol_pol_order
+
+	smooth_method = parameters.lc_ms.smooth_method
+
+	window_len = parameters.lc_ms.smooth_window
+
+	return sp.smooth_signal(signal, window_len, smooth_method, pol_order, implemented_smooth_method)
+
+def get_data(icpdata: pd.DataFrame, parameters:LCMSParameters) -> Tuple[TIC_Data, Dict[str, EIC_Data]]:
+
+	eic_metal_dict = {}
+
+	tic_data = TIC_Data(time= [], scans= [], tic= [], apexes= [])
+	
+	tic = np.zeros(len(icpdata))
+
+	scans = icpdata.Number
+
+	for columns_label in icpdata.columns[1:]:
+		
+		if columns_label[0:4] == 'Time':
+			
+			rts = icpdata[columns_label].to_numpy()
+			
+			metal_label = columns_label.replace('Time ', '')
+
+			eic_signal = icpdata[metal_label] 
+			
+			tic = tic + eic_signal
+
+			eic_data = EIC_Data(metal= metal_label, time= rts, 
+								scans= scans, eic= eic_signal, apexes= [])
+
+			
+			eic_metal_dict[metal_label] = eic_data
+
+	tic_data.scans = scans
+	tic_data.tic = list(tic)
+
+	#fig, ax = plt.subplots()
+	#ax.plot(rts, smooth_signal(tic_data.tic, parameters))
+	plt.xlabel('Retention Time')
+	plt.ylabel('Intensity (cps)')
+	plt.show()
+
+	return tic_data, eic_metal_dict
+
+def get_metal_data(icpfile, parameters:LCMSParameters):
+	
+	icpdata = pd.read_csv(icpfile, sep=',')
+	
+	tic_data, dict_metal_eicdata = get_data(icpdata, parameters)
+
+	max_tic = max(tic_data.tic)
+	
+	for metal, eic_data in dict_metal_eicdata.items():
+
+		eic_centroid_detector(max_tic, eic_data, parameters)
+
+		#print(metal, eic_data.apexes)
+	
+	return dict_metal_eicdata
+
+def search_ms1_data(icrfile:str, dict_metal_eicdata:  Dict[str, EIC_Data], parameters:LCMSParameters):
+	
+	'''place holder for parsing and search LC FT-MS data'''
+	
+	lcms_obj, parser = run_thermo(icrfile, parameters)	
+	
+	tic_data, ax_tic = lcms_obj.get_tic(ms_type='MS !d', peak_detection=True, 
+                                      smooth=False, plot=True)
+	
+	plt.show()
+
+	for metal, eic_data in dict_metal_eicdata.items():
+		
+		print(metal, eic_data.apexes)
+		
+		for peak_indexex in eic_data.apexes:
+			
+			ftms_scans_index = ([find_nearest_scan(eic_data.time[i], tic_data) for i in peak_indexex])
+			ftms_scans = [tic_data.scans[i] for i in ftms_scans_index]
+			ftms_times = [tic_data.time[i] for i in ftms_scans_index]
+			
+			retention_time = tic_data.time[ftms_scans_index[1]]
+
+			print(ftms_scans)
+			print(ftms_times)
+
+			parser.chromatogram_settings.scans = (ftms_scans[0], ftms_scans[-1])
+			
+			mass_spec = parser.get_average_mass_spectrum(auto_process=False)
+			mass_spec.retention_time = retention_time
+
+			mass_spec.settings = parameters.mass_spectrum
+			mass_spec.molecular_search_settings = parameters.ms1_molecular_search
+			mass_spec.mspeaks_settings = parameters.ms_peak
+			mass_spec.process_mass_spec()
+
+			metal_atom = ''.join(i for i in metal if not i.isdigit())
+			mass_spec.molecular_search_settings.usedAtoms[metal_atom] = (1,1)
+			
+			ax = mass_spec.plot_profile_and_noise_threshold()
+
+			SearchMolecularFormulas(mass_spec, first_hit=False).run_worker_mass_spectrum()
+			mass_spec.molecular_search_settings.usedAtoms[metal_atom] = (0,0)
+
+			mass_spec.percentile_assigned(report_error=True)
+			print(metal)
+			filename = '{}_rt{}_{}'.format(metal, retention_time, mass_spec.sample_name).replace(".", "_")
+			print(filename)
+			mass_spec.to_csv(filename, write_metadata=False)
+			
+			for peak in mass_spec:
+            
+				for mf in peak:
+					is_assigned = True
+					
+					annotation = "Mol. Form = {}\nm\z = {:.4f}\nerror = {:.4f}\nconfidence score = {:.2f}\nisotopologue score = {:.2f}".format(mf.string_formated, peak.mz_exp, mf.mz_error, mf.confidence_score, mf.isotopologue_similarity)
+					
+					ax.annotate(annotation , xy=(peak.mz_exp, peak.abundance),
+												xytext=(+3, np.sign(peak.abundance)*-40), textcoords="offset points",
+												horizontalalignment="left",
+												verticalalignment="bottom" if peak.abundance > 0 else "top")
+			plt.show()
+			#time_range.append([eic_data.time[i]/60 for i in peak_indexex])
+
+
+
+def run_thermo(file_location, parameters:LCMSParameters) -> Tuple[rawFileReader.DataDependentLCMS, rawFileReader.ImportDataDependentThermoMSFileReader]:
+    
+	#LCMSParameters.lc_ms.smooth_window = 3
+	
+	#LCMSParameters.lc_ms.min_peak_datapoints = 5
+	#LCMSParameters.lc_ms.peak_height_min_percent = 1
+	#LCMSParameters.lc_ms.peak_derivative_threshold = 1
+
+	#LCMSParameters.lc_ms.peak_max_prominence_percent = 1
+	#LCMSParameters.lc_ms.peak_height_max_percent = 1
+	
+	parser = rawFileReader.ImportDataDependentThermoMSFileReader(file_location)
+	parser.parameters = parameters
+	
+	parser.find_nearest_scan()
+	
+	lcms_obj = parser.get_lcms_obj()
+	
+	return lcms_obj, parser
+
+
+def find_nearest_scan(rt, ftms_data):
+
+	lst = np.asarray(ftms_data.time)
+    
+	idx = (np.abs(lst - (rt/60))).argmin()
+	
+	return idx
+
+if __name__ == '__main__':
+	
+	#icpfile = "tests/tests_data/icpms/cwd_211018_day7_8_c18_1uMcobalamin_10uL.csv"
+	icpfile = 'tests/tests_data/icpms/161220_soils_hypercarb_3_kansas_qH2O.csv'
+	icrfile = "tests/tests_data/icpms/rmb_161221_kansas_h2o_2.raw"
+	
+	parameters = LCMSParameters()
+
+	parameters.lc_ms.smooth_window = 301
+	parameters.lc_ms.eic_signal_threshold = 1 #0-1
+	parameters.lc_ms.min_peak_datapoints = 5
+	parameters.lc_ms.correct_eic_baseline = False
+	parameters.lc_ms.peak_max_prominence_percent = 0.1
+	parameters.lc_ms.peak_height_max_percent = 1
+
+	parameters.mass_spectrum.noise_threshold_method = 'log'
+	parameters.mass_spectrum.noise_threshold_min_std = 1
+
+	parameters.ms1_molecular_search.error_method = 'None'
+	parameters.ms1_molecular_search.min_ppm_error = -1
+	parameters.ms1_molecular_search.max_ppm_error = 1
+	
+	parameters.ms1_molecular_search.usedAtoms['C'] = (10, 100)
+	parameters.ms1_molecular_search.usedAtoms['H'] = (4, 200)
+	parameters.ms1_molecular_search.usedAtoms['O'] = (1, 30)
+	parameters.ms1_molecular_search.usedAtoms['N'] = (1, 12)
+	parameters.ms1_molecular_search.usedAtoms['S'] = (0, 0)
+
+	parameters.ms1_molecular_search.isProtonated = True
+	parameters.ms1_molecular_search.isRadical = False
+	parameters.ms1_molecular_search.isAdduct = False
+
+	dict_metal_eicdata = get_metal_data(icpfile, parameters)
+	
+	icr_data = search_ms1_data(icrfile, dict_metal_eicdata, parameters)
+