CoreMS 4.0.0__py3-none-any.whl

support_code/nmdc/nom/nom_workflow.py

@@ -0,0 +1,312 @@
+import warnings
+
+
+warnings.filterwarnings("ignore")
+
+import sys
+sys.path.append("./")
+
+
+from pathlib import Path
+import cProfile
+import json
+import pstats
+from multiprocessing import Pool, Process
+
+from pandas import DataFrame
+from matplotlib import pyplot as plt
+
+from corems.mass_spectrum.calc.Calibration import MzDomainCalibration
+
+from corems.mass_spectrum.input.massList import ReadMassList
+from corems.molecular_id.factory.classification import HeteroatomsClassification, Labels
+from corems.molecular_id.search.priorityAssignment import OxygenPriorityAssignment
+from corems.molecular_id.search.molecularFormulaSearch import SearchMolecularFormulas
+from corems import SuppressPrints
+from corems.transient.input.brukerSolarix import ReadBrukerSolarix
+from corems.mass_spectra.input import rawFileReader
+from corems.encapsulation.factory.parameters import MSParameters
+from support_code.nmdc.metadata.nmdc_registration import DMS_Mapping, NMDC_Metadata
+from support_code.nmdc.filefinder import get_dirnames, get_filename
+
+def run_bruker(file_location):
+
+    with ReadBrukerSolarix(file_location) as transient:
+
+        MSParameters.mass_spectrum.noise_threshold_methodmethod = 'log'
+        MSParameters.mass_spectrum.noise_threshold_min_s2n = 6
+
+        mass_spectrum = transient.get_mass_spectrum(plot_result=False, auto_process=True)
+        # mass_spectrum.plot_profile_and_noise_threshold()
+        # plt.show()
+
+        return mass_spectrum, transient.transient_time
+
+def run_thermo(file_location):
+
+    MSParameters.mass_spectrum.noise_threshold_method = 'log'
+    MSParameters.mass_spectrum.noise_threshold_min_std = 3
+    MSParameters.ms_peak.peak_min_prominence_percent = 0.2
+
+    parser = rawFileReader.ImportMassSpectraThermoMSFileReader(file_location)
+
+    # mass_spectrum = transient.get_mass_spectrum(plot_result=False, auto_process=True)
+
+    mass_spectrum = parser.get_average_mass_spectrum()
+
+    return mass_spectrum, 3
+
+
+def calspec(msobj, refmasslist, order=2):
+
+    calfn = MzDomainCalibration(msobj, refmasslist)
+    ref_mass_list_fmt = calfn.load_ref_mass_list()
+
+    imzmeas, mzrefs = calfn.find_calibration_points(ref_mass_list_fmt,
+                                                    calib_ppm_error_threshold=(-1.0, 1.0),
+                                                    calib_snr_threshold=4)
+
+    if len(mzrefs) < 5:
+        imzmeas, mzrefs = calfn.find_calibration_points(ref_mass_list_fmt,
+                                                        calib_ppm_error_threshold=(-1.5, 1.5),
+                                                        calib_snr_threshold=4)
+
+    if len(mzrefs) < 5:
+        imzmeas, mzrefs = calfn.find_calibration_points(ref_mass_list_fmt,
+                                                        calib_ppm_error_threshold=(-3, 3),
+                                                        calib_snr_threshold=4)
+
+    if len(mzrefs) < 5:
+        imzmeas, mzrefs = calfn.find_calibration_points(ref_mass_list_fmt,
+                                                        calib_ppm_error_threshold=(-5, 5),
+                                                        calib_snr_threshold=4)    
+
+    if len(mzrefs) < 5:
+
+        imzmeas, mzrefs = calfn.find_calibration_points(ref_mass_list_fmt,
+                                                        calib_ppm_error_threshold=(-7, 7),
+                                                        calib_snr_threshold=4) 
+
+    if len(mzrefs) < 5:
+
+        imzmeas, mzrefs = calfn.find_calibration_points(ref_mass_list_fmt,
+                                                        calib_ppm_error_threshold=(-10, 10),
+                                                        calib_snr_threshold=4)
+
+    calfn.recalibrate_mass_spectrum(imzmeas, mzrefs, order=order)
+
+def set_parameters(mass_spectrum, field_strength=12, pos=False):
+
+    if field_strength == 12:
+
+        mass_spectrum.settings.max_calib_ppm_error = 10
+        mass_spectrum.settings.min_calib_ppm_error = -10
+
+        mass_spectrum.molecular_search_settings.error_method = 'None'
+        mass_spectrum.molecular_search_settings.min_ppm_error = -1
+        mass_spectrum.molecular_search_settings.max_ppm_error = 1
+
+        mass_spectrum.settings.calib_sn_threshold = 4
+
+    elif field_strength == 15:
+
+        mass_spectrum.settings.max_calib_ppm_error = 3
+        mass_spectrum.settings.min_calib_ppm_error = -3
+
+        mass_spectrum.molecular_search_settings.error_method = 'None'
+        mass_spectrum.molecular_search_settings.min_ppm_error = -1
+        mass_spectrum.molecular_search_settings.max_ppm_error = 1
+
+        mass_spectrum.settings.calib_sn_threshold = 4
+
+    elif field_strength == 7:
+
+        mass_spectrum.settings.max_calib_ppm_error = 3
+        mass_spectrum.settings.min_calib_ppm_error = -3
+
+        mass_spectrum.molecular_search_settings.error_method = 'None'
+        mass_spectrum.molecular_search_settings.min_ppm_error = -1
+        mass_spectrum.molecular_search_settings.max_ppm_error = 1
+
+        mass_spectrum.settings.calib_sn_threshold = 4
+    # 21T Data
+    else:
+
+        mass_spectrum.settings.max_calib_ppm_error = 1
+        mass_spectrum.settings.min_calib_ppm_error = -1
+
+        mass_spectrum.molecular_search_settings.error_method = 'None'
+        mass_spectrum.molecular_search_settings.min_ppm_error = -0.5
+        mass_spectrum.molecular_search_settings.max_ppm_error = 0.5
+
+    mass_spectrum.molecular_search_settings.url_database = "postgresql+psycopg2://coremsappdb:coremsapppnnl@localhost:5432/coremsapp"
+    mass_spectrum.molecular_search_settings.min_dbe = 0
+    mass_spectrum.molecular_search_settings.max_dbe = 40
+
+    if pos:
+
+        mass_spectrum.molecular_search_settings.usedAtoms['C'] = (1, 100)
+        mass_spectrum.molecular_search_settings.usedAtoms['H'] = (4, 200)
+        mass_spectrum.molecular_search_settings.usedAtoms['O'] = (1, 12)
+        mass_spectrum.molecular_search_settings.usedAtoms['N'] = (0, 3)
+        mass_spectrum.molecular_search_settings.usedAtoms['S'] = (0, 1)
+
+    else:
+
+        mass_spectrum.molecular_search_settings.usedAtoms['C'] = (1, 100)
+        mass_spectrum.molecular_search_settings.usedAtoms['H'] = (4, 200)
+        mass_spectrum.molecular_search_settings.usedAtoms['O'] = (0, 22)
+        mass_spectrum.molecular_search_settings.usedAtoms['N'] = (0, 1)
+        mass_spectrum.molecular_search_settings.usedAtoms['S'] = (0, 1)
+
+    mass_spectrum.molecular_search_settings.usedAtoms['Cl'] = (0, 0)
+    mass_spectrum.molecular_search_settings.usedAtoms['Br'] = (0, 0)
+    mass_spectrum.molecular_search_settings.usedAtoms['P'] = (0, 0)
+    mass_spectrum.molecular_search_settings.usedAtoms['Na'] = (0, 0)
+
+    mass_spectrum.molecular_search_settings.isProtonated = True
+    mass_spectrum.molecular_search_settings.isRadical = False
+    mass_spectrum.molecular_search_settings.isAdduct = False
+
+def run_nmdc_workflow(args):
+    # mass_spectrum = get_masslist(file_location)
+    try:
+        file_location, ref_calibration_file, field_strength = args
+
+        if field_strength == 21:
+
+            # return "21T", None
+            # print("{}   {}".format("21T", file_location))
+            print("{} {}  {}".format("processing", field_strength, file_location))
+            mass_spectrum, transient_time = run_thermo(file_location)
+
+        else:
+
+            print("{} {}  {}".format("processing", field_strength, file_location))
+            mass_spectrum, transient_time = run_bruker(file_location)
+            # return "not 21T", None
+
+        is_pos = True if mass_spectrum.polarity > 0 else False
+            
+        if is_pos:
+            
+            return "positive", None
+
+        if len(mass_spectrum) < 30:
+
+            print("{}   {}".format("too few peaks", file_location))
+            return "too few peaks", None
+
+        set_parameters(mass_spectrum, field_strength=field_strength, pos=is_pos)
+
+        if ref_calibration_file:
+
+            calspec(mass_spectrum, ref_calibration_file)
+            # MzDomainCalibration(mass_spectrum, ref_calibration_file).run()
+
+        # mass_spectrum.filter_by_max_resolving_power(field_strength, transient_time)
+
+        SearchMolecularFormulas(mass_spectrum, first_hit=False).run_worker_mass_spectrum()
+        mass_spectrum.percentile_assigned(report_error=True)
+        mass_spectrum.molecular_search_settings.score_method = "prob_score"
+        mass_spectrum.molecular_search_settings.output_score_method = "prob_score"
+
+        return "all_good", mass_spectrum
+    
+    except Exception as inst:
+
+        print(type(inst))    # the exception instance
+        print(inst.args)     # arguments stored in .args
+        print(inst)
+        return 'error', None
+
+def monitor(target):
+
+    import psutil
+    import time
+
+    worker_process = Process(target=target)
+    worker_process.start()
+    p = psutil.Process(worker_process.pid)
+
+    # log cpu usage of `worker_process` every 10 ms
+    cpu_percents = []
+    while worker_process.is_alive():
+        cpu_percents.append(p.cpu_percent())
+        time.sleep(0.01)
+
+    worker_process.join()
+    return cpu_percents
+
+def worker(file_location):
+
+    cProfile.runctx('run_assignment(file_location)', globals(), locals(), 'di-fticr-di.prof')
+    # stats = pstats.Stats("topics.prof")
+    # stats.strip_dirs().sort_stats("time").print_stats()
+
+def run_nom_nmdc_data_processing():
+
+    file_ext = '.raw'  # '.d' 
+    data_dir = Path("/Users/eber373/OneDrive - PNNL/Documents/Data/FT_ICR_MS/Spruce_Data/")
+    dms_file_path = Path("/Users/eber373/OneDrive - PNNL/Documents/Data/FT_ICR_MS/Spruce_Data/SPRUCE_FTICR_Peat.xlsx")
+
+    results_dir = Path("results/")
+    registration_path = results_dir / "spruce_ftms_nom_data_products.json"
+    failed_files = results_dir / "nom_failed_files.json"
+    pos_files = results_dir / "pos_files.json"
+
+    field_strength = 21
+    cores = 4
+    ref_calibration_path = False
+
+    # file_paths = get_dirnames()
+    ref_calibration_path = Path("db/Hawkes_neg.ref")
+
+    dms_mapping = DMS_Mapping(dms_file_path)
+    selected_files = dms_mapping.get_selected_sample_list()
+
+    failed_list = []
+    pos_list = []
+    
+    for file_name, field_strength in selected_files:
+
+        in_file_path = data_dir / file_name / file_name.with_suffix(file_ext)    
+
+        try:
+
+            issue, ms = run_nmdc_workflow((in_file_path, ref_calibration_path, field_strength))
+
+            if ms:
+
+                file_name = Path(in_file_path.name)
+
+                output_file_path = results_dir / file_name.with_suffix('.csv')
+                ms.to_csv(output_file_path, write_metadata=False)
+
+                nmdc = NMDC_Metadata(in_file_path, ref_calibration_path, output_file_path, dms_file_path)
+                nmdc.create_nmdc_ftms_metadata(ms, registration_path)
+
+            else:
+                print(issue)
+
+        except Exception as inst:
+
+            print(type(inst))    # the exception instance
+            print(inst.args)     # arguments stored in .args
+            print(inst)
+            failed_list.append(str(in_file_path))
+
+    with failed_files.open('w') as json_file:
+
+        json_file.write(json.dumps(failed_list, indent=1))
+
+
+if __name__ == "__main__":
+
+    # run_multiprocess()
+    # cpu_percents = monitor(target=run_multiprocess)
+    # print(cpu_percents)
+    run_nom_nmdc_data_processing()
+    # file_location = get_filename()
+    # if file_location:
+    #    run_assignment(file_location)

support_code/windows_only_importers/BrukerCompassXtract.py

@@ -0,0 +1,180 @@
+from ctypes import c_long
+from threading import Thread
+
+from comtypes.automation import BSTR
+from comtypes.client import CreateObject
+from numpy import array
+
+
+from corems.encapsulation.factory.parameters import default_parameters
+from corems.encapsulation.constant import Labels
+from corems.mass_spectra.factory.LC_Class import LCMSBase
+from corems.mass_spectrum.factory.MassSpectrumClasses import MassSpecProfile
+
+__author__ = "Yuri E. Corilo"
+__date__ = "July 10, 2019"
+
+
+class ImportLCMSBrukerCompassXtract(Thread):
+    """Class for importing LC-MS data from Bruker Compass Xtract files.
+
+    Parameters:
+    -----------
+    file_location : str
+        The path to the Bruker Compass Xtract file.
+    auto_process : bool, optional
+        Flag indicating whether to automatically process the imported data.
+        Default is True.
+
+    Attributes:
+    -----------
+    initial_scan_number : int
+        The initial scan number to import.
+    final_scan_number : int
+        The final scan number to import.
+
+    Methods:
+    --------
+    check_scan(scan: int) -> bool:
+        Check if the given scan number is valid.
+    get_scans_numbers() -> int:
+        Get the total number of scans in the file.
+    get_polarity_mode(spectrum) -> int:
+        Get the polarity mode of a given spectrum.
+    check_load_success() -> None:
+        Check if the file was loaded successfully.
+    get_bruker_tics() -> array:
+        Get the total ion current (TIC) array from the file.
+    get_bruker_retention_time() -> array:
+        Get the retention time array from the file.
+    get_data(spectra, scan: int) -> dict:
+        Get the mass spectrum data for a given scan number.
+    run() -> None:
+        Run the import process.
+    _import_mass_spectra(d_params: dict) -> None:
+        Import the mass spectra from the file.
+    get_lcms_obj() -> LCMSBase:
+        Get the LCMSBase object.
+
+    """
+
+    def __init__(self, file_location: str, auto_process: bool = True):
+        Thread.__init__(self)
+        self.lcms: LCMSBase = LCMSBase(file_location)
+        self.Bruker_Library = CreateObject("EDAL.MSAnalysis")
+        self.res = self.Bruker_Library.Open(file_location)
+        self.check_load_success()
+        self._initial_scan_number: int = 1
+        self._final_scan_number: int = self.get_scans_numbers()
+        self.file_location: str = file_location
+        self.auto_process: bool = auto_process
+
+    @property
+    def initial_scan_number(self) -> int:
+        return self._initial_scan_number
+
+    @initial_scan_number.setter
+    def initial_scan_number(self, initial_scan_number: int) -> None:
+        if self.check_scan(initial_scan_number):
+            self._initial_scan_number = initial_scan_number
+        else:
+            raise Exception(
+                f"startscan and finalscan should be less than {self.get_scans_numbers()}"
+            )
+
+    @property
+    def final_scan_number(self) -> int:
+        return self._final_scan_number
+
+    @final_scan_number.setter
+    def final_scan_number(self, final_scan_number: int) -> None:
+        if self.check_scan(final_scan_number):
+            self._final_scan_number = final_scan_number
+        else:
+            raise Exception(
+                f"startscan and finalscan should be less than {self.get_scans_numbers()}"
+            )
+
+    def check_scan(self, scan: int) -> bool:
+        scan_numbers = self.get_scans_numbers()
+        return scan <= scan_numbers
+
+    def get_scans_numbers(self) -> int:
+        scan_numbers = self.Bruker_Library.MSSpectrumCollection.Count
+        return scan_numbers
+
+    def get_polarity_mode(self, spectrum) -> int:
+        polarity_symbol = spectrum.Polarity
+        if polarity_symbol == 0:
+            return 1
+        elif polarity_symbol == 1:
+            return -1
+        else:
+            raise IOError("Could not read mass spectrum polarity mode")
+
+    def check_load_success(self) -> None:
+        if self.res == 0:
+            self.break_it = False
+        else:
+            raise ImportError(str(self.res))
+
+    def get_bruker_tics(self) -> array:
+        strAnalysisData = "SumIntensity"
+        if self.Bruker_Library.HasAnalysisData(strAnalysisData):
+            tics_array = self.Bruker_Library.GetAnalysisData(strAnalysisData)
+            tics_array = array(tics_array)
+        return tics_array
+
+    def get_bruker_retention_time(self) -> array:
+        strAnalysisData = "RetentionTime"
+        if self.Bruker_Library.HasAnalysisData(strAnalysisData):
+            tics_array = self.Bruker_Library.GetAnalysisData(strAnalysisData)
+            tics_array = array(tics_array)
+        else:
+            tics_array = [0]
+        return tics_array
+
+    @staticmethod
+    def get_data(spectra, scan: int) -> dict:
+        spectrum = spectra[scan]
+        is_profile = c_long(1)
+        masslist = spectrum.GetMassIntensityValues(is_profile)
+        data_dict = {
+            Labels.mz: array(masslist[0]),
+            Labels.abundance: array(masslist[1]),
+            Labels.rp: None,
+            Labels.s2n: None,
+        }
+        return data_dict
+
+    def run(self) -> None:
+        d_parameters = default_parameters(self.file_location)
+        self._import_mass_spectra(d_parameters)
+
+    def _import_mass_spectra(self, d_params: dict) -> None:
+        spectra = self.Bruker_Library.MSSpectrumCollection
+        list_rt = self.get_bruker_retention_time()
+        list_Tics = self.get_bruker_tics()
+        list_scans = list()
+        for scan_number in range(self.initial_scan_number, self.final_scan_number + 1):
+            if spectra[scan_number].MSMSStage == 1:
+                d_params["label"] = Labels.bruker_profile
+                d_params["polarity"] = self.get_polarity_mode(spectra[scan_number])
+                d_params["rt"] = list_rt[scan_number - 1]
+                d_params["scan_number"] = scan_number
+                list_scans.append(scan_number)
+                data_dict = self.get_data(spectra, scan_number)
+                mass_spec = MassSpecProfile(
+                    data_dict, d_params, auto_process=self.auto_process
+                )
+                mass_spec.process_mass_spec()
+                self.lcms.add_mass_spectrum(mass_spec)
+        self.lcms.retention_time = list_rt
+        self.lcms.tic = list_Tics
+        self.lcms.scans_number = list_scans
+
+    def get_lcms_obj(self) -> LCMSBase:
+        if self.lcms.get(self._initial_scan_number):
+            return self.lcms
+        else:
+            raise Exception("Returning an empty LCMSBase object.")