CoreMS 4.0.0__py3-none-any.whl

support_code/nmdc/lipidomics/manifest_examples.py

@@ -0,0 +1,49 @@
+"""
+Simple examples demonstrating manifest creation utility usage.
+"""
+
+from pathlib import Path
+from corems.mass_spectra.input.corems_hdf5 import create_manifest_from_folder
+
+# Example 1: Basic usage - auto-select middle sample as center
+print("Example 1: Basic usage (middle sample as center)")
+print("-" * 60)
+manifest_path = create_manifest_from_folder(
+    folder_path=Path("/Volumes/LaCie/nmdc_data/collection_testing/blanchard_lipid/mini_collection_test_out2"),
+    overwrite=True
+)
+print()
+
+# Example 2: Custom batch threshold
+print("Example 2: Custom batch threshold (24 hours)")
+print("-" * 60)
+manifest_path = create_manifest_from_folder(
+    folder_path=Path("/Volumes/LaCie/nmdc_data/collection_testing/blanchard_lipid/mini_collection_test_out2"),
+    output_path=Path("/Volumes/LaCie/nmdc_data/collection_testing/blanchard_lipid/mini_collection_test_out2/manifest_24hr.csv"),
+    batch_time_threshold_hours=24.0,
+    overwrite=True
+)
+print()
+
+# Example 3: Specify a sample as center by name
+print("Example 3: Specify center sample by name")
+print("-" * 60)
+manifest_path = create_manifest_from_folder(
+    folder_path=Path("/Volumes/LaCie/nmdc_data/collection_testing/blanchard_lipid/mini_collection_test_out2"),
+    output_path=Path("/Volumes/LaCie/nmdc_data/collection_testing/blanchard_lipid/mini_collection_test_out2/manifest_custom_center.csv"),
+    batch_time_threshold_hours=12.0,
+    center_name="Blanch_Nat_Lip_C_7_AB_M_07_POS_23Jan18_Brandi-WCSH5801",
+    overwrite=True
+)
+print()
+
+# Example 4: Strict batch threshold
+print("Example 4: Strict batch threshold (1 hour)")
+print("-" * 60)
+manifest_path = create_manifest_from_folder(
+    folder_path=Path("/Volumes/LaCie/nmdc_data/collection_testing/blanchard_lipid/mini_collection_test_out2"),
+    output_path=Path("/Volumes/LaCie/nmdc_data/collection_testing/blanchard_lipid/mini_collection_test_out2/manifest_1hr.csv"),
+    batch_time_threshold_hours=1.0,
+    overwrite=True
+)
+print()

support_code/nmdc/metabolomics/gcms_workflow.py

@@ -0,0 +1,158 @@
+import warnings
+warnings.filterwarnings("ignore")
+
+import sys
+sys.path.append("./")
+
+from pathlib import Path
+from multiprocessing import Pool
+import cProfile
+
+
+from corems.molecular_id.input.nistMSI import ReadNistMSI
+from corems.mass_spectra.input.andiNetCDF import ReadAndiNetCDF
+from corems.molecular_id.search.compoundSearch import LowResMassSpectralMatch
+from corems.mass_spectra.calc.GC_RI_Calibration import get_rt_ri_pairs
+from support_code.nmdc.filefinder import get_dirname, get_filename
+
+import glob
+from nmdc.metadata.nmdc_registration import NMDC_Metadata
+
+def start_sql_from_file():
+
+    ref_lib_path = Path.cwd() / "tests/tests_data/gcms/" / "PNNLMetV20191015.MSL"
+    sql_obj = ReadNistMSI(ref_lib_path).get_sqlLite_obj()
+    return sql_obj
+
+
+def get_gcms(file_path):
+
+    reader_gcms = ReadAndiNetCDF(file_path)
+
+    reader_gcms.run()
+
+    gcms = reader_gcms.get_gcms_obj()
+
+    # gcms.process_chromatogram()
+
+    return gcms
+
+def get_reference_dict(calibration_file_path=False):
+
+    from PySide2.QtWidgets import QFileDialog, QApplication
+    from PySide2.QtCore import Qt
+
+    if not calibration_file_path:
+        app = QApplication(sys.argv)
+        file_dialog = QFileDialog()
+        file_dialog.setWindowFlags(Qt.WindowStaysOnTopHint)
+        file_path = file_dialog.getOpenFileName(None, "FAMES REF FILE", filter="*.cdf")[0]
+        file_dialog.close()
+        app.exit()
+    else:
+        file_path = calibration_file_path
+
+    if not file_path:
+        return None
+
+    else:
+
+        gcms_ref_obj = get_gcms(file_path)
+        # sql_obj = start_sql_from_file()
+        rt_ri_pairs = get_rt_ri_pairs(gcms_ref_obj)  # sql_obj=sql_obj)
+        # !!!!!! READ !!!!! use the previous two lines if db/pnnl_lowres_gcms_compounds.sqlite does not exist
+        # and comment the next line
+        # rt_ri_pairs = get_rt_ri_pairs(gcms_ref_obj)
+
+        return rt_ri_pairs, file_path
+
+def run(args):
+
+    file_path, ref_dict, cal_file_path = args
+
+    gcms = get_gcms(file_path)
+
+    gcms.process_chromatogram()
+
+    gcms.calibrate_ri(ref_dict, cal_file_path)
+
+    # sql_obj = start_sql_from_file()
+
+    lowResSearch = LowResMassSpectralMatch(gcms)  # sql_obj=sql_obj)
+    # !!!!!! READ !!!!! use the previous two lines if db/pnnl_lowres_gcms_compounds.sqlite does not exist
+    # and comment the next line
+    # lowResSearch = LowResMassSpectralMatch(gcms)
+    lowResSearch.run()
+
+    return gcms
+
+def calibrate_and_search(out_put_file_name, jobs, dms_file_path):
+
+    from PySide2.QtWidgets import QFileDialog
+    from PySide2.QtCore import Qt
+
+    ref_dict, cal_file_path = get_reference_dict()
+    if ref_dict:
+
+        file_dialog = QFileDialog()
+        file_dialog.setWindowFlags(Qt.WindowStaysOnTopHint)
+
+        if file_dialog:
+
+            file_locations = file_dialog.getOpenFileNames(None, "Standard Compounds Files", filter="*.cdf")
+            file_dialog.close()
+
+            # run in multiprocessing mode
+            pool = Pool(jobs)
+            args = [(file_path, ref_dict, cal_file_path) for file_path in file_locations[0]]
+            gcmss = pool.map(run, args)
+            pool.close()
+            pool.join()
+
+            for file_index, gcms in enumerate(gcmss):
+
+                file_path = Path(file_locations[0][file_index])
+                # print(out_put_file_name)
+
+                gcms.to_csv(file_path, write_metadata=False, id_label="emsl:")
+                nmdc = NMDC_Metadata(file_path, cal_file_path, file_path.with_suffix(".cvs"), dms_file_path)
+                nmdc.create_nmdc_gcms_metadata(gcms)
+
+                # gcms.to_excel(out_put_file_name)
+                # gcms.to_pandas(out_put_file_name)
+                # gcms.to_hdf()
+
+                # df = gcms.get_dataframe()
+                # json_data = gcms.to_json()
+
+                # print(json_data)
+
+                # gcms.plot_processed_chromatogram()
+
+                # gcms.plot_gc_peaks()
+
+                # gcms.plot_chromatogram()
+
+                # gcms.plot_smoothed_chromatogram()
+
+                # gcms.plot_baseline_subtraction()
+
+                # gcms.plot_detected_baseline()
+
+                # matplotlib.pyplot.show()
+
+def worker(args):
+
+    cProfile.runctx('run(args)', globals(), locals(), 'gc-ms.prof')
+
+
+
+if __name__ == '__main__':
+    # import matplotlib
+    # matplotlib.use('TkAgg')
+    # %%
+    cores = 4
+    out_put_file_group_name = 'sql_test'
+    dms_file_path="db/GC-MS Metabolomics Experiments to Process Final.xlsx"
+    calibrate_and_search(out_put_file_group_name, cores, dms_file_path)
+   

support_code/nmdc/metabolomics/lcms_metabolomics_targeted_search.py

@@ -0,0 +1,59 @@
+from pathlib import Path
+
+from corems.mass_spectra.input.rawFileReader import ImportMassSpectraThermoMSFileReader
+
+if __name__ == "__main__":
+    # =============================================================================
+    # Configuration
+    # =============================================================================
+    # Paths
+    base_path = Path("/Volumes/LaCie/nmdc_data/collection_testing/dev_test/")
+    raw_data_path = base_path / "raw"
+
+    # Instantiate LCMS object from the first raw data file
+    first_raw_file = next(raw_data_path.glob("*.raw"))
+    parser = ImportMassSpectraThermoMSFileReader(first_raw_file)
+    lcms_obj = parser.get_lcms_obj(spectra="ms1")
+    assert lcms_obj is not None, "Failed to instantiate LCMS object."
+
+    # Set parameters for the LCMS object, only ones we NEED to change from defaults to get things running
+    # Ideally, we'd load carefully chosen parameters from a file here for general processing
+    lcms_obj.parameters.mass_spectrum['ms2'].mass_spectrum.noise_threshold_method = "relative_abundance"
+
+    # Prepare a search dictionary for a targeted search
+    # Here are examples of target m/z and RT values
+    # Could derive this from an msp file.
+    target_mz_list = [254.2837, 521.325988, 503.3134460449219, 317.2839050292969]
+    target_rt_list = [5.83, 6.017, 7.977, 6.8658]
+    target_search_dict = {
+        "target_mz_list": target_mz_list,
+        "target_rt_list": target_rt_list,
+        "mz_tolerance_ppm": 5, #QUESTION: per target or bulk?
+        "rt_tolerance": 0.5, #QUESTION: per target or bulk?
+        "type": "internal standard"
+    }
+    
+    # Look for mass features in targeted search mode
+    lcms_obj.find_mass_features(
+        targeted_search = True, 
+        target_search_dict=target_search_dict
+        )
+    # Alternatively, or additionally, we could do normal (untargeted) mass feature finding here
+    # lcms_obj.find_mass_features(targeted_search=False)
+
+    # Let's integrate, add ms1 and ms2
+    lcms_obj.integrate_mass_features()
+    lcms_obj.add_associated_ms1(use_parser=False, spectrum_mode="profile")
+    lcms_obj.add_associated_ms2_dda(use_parser=True, spectrum_mode="centroid")
+
+    # Here we could do formula searching and MS2 search against a MSP database with MS2s of the search targets (similar to test_lcms_metabolomics.py)
+    # BUT, we'd need a compatible msp file, and that's not ultra simple
+    
+    # Check out the mass feature dataframe
+    mf_df = lcms_obj.mass_features_to_df(drop_na_cols=True)
+
+    # Can visualize as well
+    lcms_obj.mass_features[0].plot(return_fig = False)
+
+    # Here could do some post-processing to report the "best hit" per target and include confidence metrics
+    print("here")

support_code/nmdc/metabolomics/lcms_metabolomics_workflow.py

@@ -0,0 +1,248 @@
+import os
+from pathlib import Path
+from multiprocessing import Pool
+import warnings
+
+from corems.molecular_id.search.database_interfaces import MSPInterface
+from corems.mass_spectra.input.corems_hdf5 import ReadCoreMSHDFMassSpectra
+from corems.mass_spectra.output.export import LCMSMetabolomicsExport
+
+from support_code.nmdc.lipidomics.lipidomics_workflow import (
+    instantiate_lcms_obj,
+    set_params_on_lcms_obj,
+    check_scan_translator,
+    add_mass_features,
+    molecular_formula_search,
+    process_ms2,
+)
+
+
+def prepare_metadata(msp_file_path):
+    print("Parsing MSP file...")
+    my_msp = MSPInterface(file_path=msp_file_path)
+    print("Parsing MSP file complete.")
+    metadata = {
+        "fe": {"positive": None, "negative": None},
+        "molecular_metadata": {},
+    }
+    msp_positive, metabolite_metadata_positive = (
+        my_msp.get_metabolomics_spectra_library(
+            polarity="positive",
+            format="flashentropy",
+            normalize=True,
+            fe_kwargs={
+                "normalize_intensity": True,
+                "min_ms2_difference_in_da": 0.02,  # for cleaning spectra
+                "max_ms2_tolerance_in_da": 0.01,  # for setting search space
+                "max_indexed_mz": 3000,
+                "precursor_ions_removal_da": None,
+                "noise_threshold": 0,
+            },
+        )
+    )
+    metadata["fe"]["positive"] = msp_positive
+    metadata["molecular_metadata"] = metabolite_metadata_positive
+
+    msp_negative, metabolite_metadata_negative = (
+        my_msp.get_metabolomics_spectra_library(
+            polarity="negative",
+            format="flashentropy",
+            normalize=True,
+            fe_kwargs={
+                "normalize_intensity": True,
+                "min_ms2_difference_in_da": 0.02,  # for cleaning spectra
+                "max_ms2_tolerance_in_da": 0.01,  # for setting search space
+                "max_indexed_mz": 3000,
+                "precursor_ions_removal_da": None,
+                "noise_threshold": 0,
+            },
+        )
+    )
+    metadata["fe"]["negative"] = msp_negative
+    metadata["molecular_metadata"].update(metabolite_metadata_negative)
+
+    return metadata
+
+
+def export_results(myLCMSobj, out_path, molecular_metadata=None, final=False):
+    """Export results to hdf5 and csv as a lipid report
+
+    Parameters
+    ----------
+    myLCMSobj : corems LCMS object
+        LCMS object to process
+    out_path : str or Path
+        Path to output file
+    molecular_metadata : dict
+        Dict with molecular metadata
+    final : bool
+        Whether to export final results
+
+    Returns
+    -------
+    None, exports results to hdf5 and csv as a lipid report
+    """
+    exporter = LCMSMetabolomicsExport(out_path, myLCMSobj)
+    exporter.to_hdf(overwrite=True)
+    if final:
+        # Do not show warnings, these are expected
+        exporter.report_to_csv(molecular_metadata=molecular_metadata)
+    else:
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            exporter.report_to_csv()
+
+
+def run_ms2_search(out_path, metadata, scan_translator=None):
+    """Run ms2 spectral search and export final results
+
+    Parameters
+    ----------
+    out_path : str or Path
+        Path to output file
+    metadata : dict
+        Dict with keys "mzs", "fe", and "molecular_metadata" with values of dicts of precursor mzs (negative and positive), flash entropy search databases (negative and positive), and molecular metadata, respectively
+
+    Returns
+    -------
+    None, runs ms2 spectral search and exports final results
+    """
+    # Read in the intermediate results
+    out_path = Path(out_path)
+    out_path_hdf5 = str(out_path) + ".corems/" + out_path.stem + ".hdf5"
+    parser = ReadCoreMSHDFMassSpectra(out_path_hdf5)
+    myLCMSobj = parser.get_lcms_obj()
+    process_ms2(myLCMSobj, metadata, scan_translator=scan_translator)
+    export_results(
+        myLCMSobj,
+        out_path=str(out_path),
+        molecular_metadata=metadata["molecular_metadata"],
+        final=True,
+    )
+
+
+def run_lcms_metabolomics_workflow(
+    file_in,
+    out_path,
+    params_toml,
+    scan_translator,
+    metadata,
+):
+    myLCMSobj = instantiate_lcms_obj(file_in)
+    set_params_on_lcms_obj(myLCMSobj, params_toml, verbose)
+    
+    # If the ms1 data are centroided, switch the peak picking method to centroided persistent homology
+    # and set the noise threshold method to relative abundance
+    ms1_scan_df = myLCMSobj.scan_df[myLCMSobj.scan_df.ms_level == 1]
+    if all(x == "centroid" for x in ms1_scan_df.ms_format.to_list()):
+        # Switch peak picking method to centroided persistent homology
+        myLCMSobj.parameters.lc_ms.peak_picking_method = "centroided_persistent_homology"
+        myLCMSobj.parameters.mass_spectrum[
+            "ms1"
+        ].mass_spectrum.noise_threshold_method = "relative_abundance"
+
+    myLCMSobj.parameters.mass_spectrum[
+        "ms1"
+    ].mass_spectrum.noise_threshold_min_relative_abundance = 0.1
+
+    check_scan_translator(myLCMSobj, scan_translator)
+    add_mass_features(myLCMSobj, scan_translator)
+    myLCMSobj.remove_unprocessed_data()
+    molecular_formula_search(myLCMSobj)
+    # Just for testing, not needed for workflow
+    export_results(
+        myLCMSobj,
+        out_path=str(out_path),
+        molecular_metadata=metadata["molecular_metadata"],
+        final=False,
+    )
+    process_ms2(myLCMSobj, metadata, scan_translator=scan_translator)
+    export_results(
+        myLCMSobj,
+        out_path=str(out_path),
+        molecular_metadata=metadata["molecular_metadata"],
+        final=True,
+    )
+
+
+def run_lcms_metabolomics_workflow_batch(
+    file_dir,
+    out_dir,
+    params_toml,
+    msp_file_path,
+    scan_translator=None,
+    cores=1,
+):
+    """Run the LCMS metabolomics workflow on a batch of files
+
+    Parameters
+    ----------
+    file_dir : str or Path
+        Path to directory with raw files
+    out_dir : str or Path
+        Path to output directory
+    params_toml : str or Path
+        Path to toml file with parameters for all samples
+    msp_file_path : str or Path
+        Path to msp file with spectral library for MS2 search
+    scan_translator : str or Path
+        Path to toml file with scan translator for MS2 search
+    cores : int
+        Number of cores to use for processing
+    """
+    # Make output dir and get list of files to process
+    out_dir.mkdir(parents=True, exist_ok=True)
+    files_list = [
+        f for f in file_dir.iterdir() if f.suffix.lower() in {".raw", ".mzml"}
+    ]
+    out_paths_list = [out_dir / f.stem for f in files_list]
+
+    # Prepare search databases for ms2 search
+    my_msp_FE = prepare_metadata(msp_file_path)
+
+    # Run signal processing, get associated ms1, add associated ms2, do ms1 molecular search, and export temp results
+    # Note that this is exactly the same as the lipidomics workflow
+    if cores == 1 or len(files_list) == 1:
+        for file_in, file_out in list(zip(files_list, out_paths_list)):
+            print(f"Processing {file_in}")
+            run_lcms_metabolomics_workflow(
+                file_in=file_in,
+                out_path=file_out,
+                params_toml=params_toml,
+                scan_translator=scan_translator,
+                metadata=my_msp_FE,
+            )
+    elif cores > 1:
+        raise ValueError(
+            "Parallel processing is not yet supported for LCMS metabolomics workflow."
+        )
+
+
+if __name__ == "__main__":
+    # Set input variables to run
+    msp_file_path = "/Users/heal742/LOCAL/05_NMDC/02_MetaMS/metams/test_data/test_lcms_metab_data/20250407_database.msp"
+    # msp_file_path = "/Users/heal742/LOCAL/corems_dev/corems/tests/tests_data/lcms/test_db.msp"
+    file_dir = Path(
+        "/Users/heal742/Library/CloudStorage/OneDrive-PNNL/Documents/_DMS_data/_NMDC/_massive/_bioscales_lcms/problem"
+    )
+    params_toml = Path(
+        "/Users/heal742/LOCAL/05_NMDC/02_MetaMS/data_processing/_bioscales_lcms_metabolomics/metadata/bioscales_hilic_corems.toml"
+    )
+    scan_translator = Path(
+        "/Users/heal742/LOCAL/05_NMDC/02_MetaMS/data_processing/_bioscales_lcms_metabolomics/metadata/bioscales_scan_translator.toml"
+    )
+    out_dir = Path("tmp_data/__lcms_metab_test_data_250509")
+    verbose = True
+    cores = 1
+
+    # Set up output directory
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    run_lcms_metabolomics_workflow_batch(
+        file_dir=file_dir,
+        out_dir=out_dir,
+        params_toml=params_toml,
+        msp_file_path=msp_file_path,
+        scan_translator=scan_translator,
+        cores=cores,
+    )