CoreMS 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- corems/__init__.py +63 -0
- corems/chroma_peak/__init__.py +0 -0
- corems/chroma_peak/calc/ChromaPeakCalc.py +480 -0
- corems/chroma_peak/calc/__init__.py +0 -0
- corems/chroma_peak/calc/subset.py +196 -0
- corems/chroma_peak/factory/__init__.py +0 -0
- corems/chroma_peak/factory/chroma_peak_classes.py +1178 -0
- corems/encapsulation/__init__.py +0 -0
- corems/encapsulation/constant.py +1283 -0
- corems/encapsulation/factory/__init__.py +0 -0
- corems/encapsulation/factory/parameters.py +392 -0
- corems/encapsulation/factory/processingSetting.py +1244 -0
- corems/encapsulation/input/__init__.py +0 -0
- corems/encapsulation/input/parameter_from_json.py +579 -0
- corems/encapsulation/output/__init__.py +0 -0
- corems/encapsulation/output/parameter_to_dict.py +142 -0
- corems/encapsulation/output/parameter_to_json.py +332 -0
- corems/mass_spectra/__init__.py +0 -0
- corems/mass_spectra/calc/GC_Calc.py +84 -0
- corems/mass_spectra/calc/GC_Deconvolution.py +558 -0
- corems/mass_spectra/calc/GC_RI_Calibration.py +44 -0
- corems/mass_spectra/calc/MZSearch.py +249 -0
- corems/mass_spectra/calc/SignalProcessing.py +656 -0
- corems/mass_spectra/calc/__init__.py +0 -0
- corems/mass_spectra/calc/lc_calc.py +5790 -0
- corems/mass_spectra/calc/lc_calc_operations.py +1127 -0
- corems/mass_spectra/factory/GC_Class.py +791 -0
- corems/mass_spectra/factory/__init__.py +0 -0
- corems/mass_spectra/factory/chromat_data.py +51 -0
- corems/mass_spectra/factory/lc_class.py +2721 -0
- corems/mass_spectra/input/__init__.py +0 -0
- corems/mass_spectra/input/andiNetCDF.py +200 -0
- corems/mass_spectra/input/boosterHDF5.py +216 -0
- corems/mass_spectra/input/brukerSolarix.py +153 -0
- corems/mass_spectra/input/brukerSolarix_utils.py +73 -0
- corems/mass_spectra/input/corems_hdf5.py +1710 -0
- corems/mass_spectra/input/massList.py +133 -0
- corems/mass_spectra/input/mzml.py +668 -0
- corems/mass_spectra/input/parserbase.py +239 -0
- corems/mass_spectra/input/rawFileReader.py +1839 -0
- corems/mass_spectra/output/__init__.py +0 -0
- corems/mass_spectra/output/export.py +2800 -0
- corems/mass_spectrum/__init__.py +0 -0
- corems/mass_spectrum/calc/AutoRecalibration.py +237 -0
- corems/mass_spectrum/calc/Calibration.py +602 -0
- corems/mass_spectrum/calc/CalibrationCalc.py +253 -0
- corems/mass_spectrum/calc/KendrickGroup.py +239 -0
- corems/mass_spectrum/calc/MassErrorPrediction.py +436 -0
- corems/mass_spectrum/calc/MassSpectrumCalc.py +303 -0
- corems/mass_spectrum/calc/MeanResolvingPowerFilter.py +212 -0
- corems/mass_spectrum/calc/NoiseCalc.py +371 -0
- corems/mass_spectrum/calc/NoiseCalc_Bayes.py +93 -0
- corems/mass_spectrum/calc/PeakPicking.py +994 -0
- corems/mass_spectrum/calc/__init__.py +0 -0
- corems/mass_spectrum/factory/MassSpectrumClasses.py +1753 -0
- corems/mass_spectrum/factory/__init__.py +0 -0
- corems/mass_spectrum/input/__init__.py +0 -0
- corems/mass_spectrum/input/baseClass.py +531 -0
- corems/mass_spectrum/input/boosterHDF5.py +161 -0
- corems/mass_spectrum/input/coremsHDF5.py +475 -0
- corems/mass_spectrum/input/massList.py +402 -0
- corems/mass_spectrum/input/numpyArray.py +133 -0
- corems/mass_spectrum/output/__init__.py +0 -0
- corems/mass_spectrum/output/export.py +841 -0
- corems/molecular_formula/__init__.py +0 -0
- corems/molecular_formula/calc/MolecularFormulaCalc.py +885 -0
- corems/molecular_formula/calc/__init__.py +0 -0
- corems/molecular_formula/factory/MolecularFormulaFactory.py +895 -0
- corems/molecular_formula/factory/__init__.py +0 -0
- corems/molecular_formula/input/__init__.py +0 -0
- corems/molecular_formula/input/masslist_ref.py +355 -0
- corems/molecular_id/__init__.py +0 -0
- corems/molecular_id/calc/ClusterFilter.py +251 -0
- corems/molecular_id/calc/MolecularFilter.py +122 -0
- corems/molecular_id/calc/SpectralSimilarity.py +632 -0
- corems/molecular_id/calc/__init__.py +0 -0
- corems/molecular_id/calc/math_distance.py +1637 -0
- corems/molecular_id/data/FAMES_REF.msp +980 -0
- corems/molecular_id/data/PNNLMetV20191015.msp +157267 -0
- corems/molecular_id/factory/EI_SQL.py +650 -0
- corems/molecular_id/factory/MolecularLookupTable.py +914 -0
- corems/molecular_id/factory/__init__.py +0 -0
- corems/molecular_id/factory/classification.py +884 -0
- corems/molecular_id/factory/lipid_molecular_metadata.py +50 -0
- corems/molecular_id/factory/molecularSQL.py +827 -0
- corems/molecular_id/factory/spectrum_search_results.py +119 -0
- corems/molecular_id/input/__init__.py +0 -0
- corems/molecular_id/input/nistMSI.py +148 -0
- corems/molecular_id/search/__init__.py +0 -0
- corems/molecular_id/search/compoundSearch.py +214 -0
- corems/molecular_id/search/database_interfaces.py +1527 -0
- corems/molecular_id/search/findOxygenPeaks.py +330 -0
- corems/molecular_id/search/lcms_spectral_search.py +348 -0
- corems/molecular_id/search/molecularFormulaSearch.py +1117 -0
- corems/molecular_id/search/priorityAssignment.py +723 -0
- corems/ms_peak/__init__.py +0 -0
- corems/ms_peak/calc/MSPeakCalc.py +1010 -0
- corems/ms_peak/calc/__init__.py +0 -0
- corems/ms_peak/factory/MSPeakClasses.py +542 -0
- corems/ms_peak/factory/__init__.py +0 -0
- corems/transient/__init__.py +0 -0
- corems/transient/calc/TransientCalc.py +362 -0
- corems/transient/calc/__init__.py +0 -0
- corems/transient/factory/TransientClasses.py +457 -0
- corems/transient/factory/__init__.py +0 -0
- corems/transient/input/__init__.py +0 -0
- corems/transient/input/brukerSolarix.py +461 -0
- corems/transient/input/midasDatFile.py +172 -0
- corems-4.0.0.dist-info/METADATA +475 -0
- corems-4.0.0.dist-info/RECORD +171 -0
- corems-4.0.0.dist-info/WHEEL +5 -0
- corems-4.0.0.dist-info/licenses/LICENSE +22 -0
- corems-4.0.0.dist-info/top_level.txt +4 -0
- examples/archive/scripts/CoreMS_tutorial.py +94 -0
- examples/archive/scripts/DI HR-MS Halogens Bruker.py +196 -0
- examples/archive/scripts/DI HR-MS MassList.py +385 -0
- examples/archive/scripts/GC-MS MetabRef.py +213 -0
- examples/archive/scripts/GC-MS NetCDF.py +217 -0
- examples/archive/scripts/HR-MS Thermo Raw 21T.py +136 -0
- examples/archive/scripts/LC-ICPMS_metal_peaks.py +297 -0
- examples/archive/scripts/LCMS-Thermo.py +460 -0
- examples/archive/scripts/LCMS_isotopes.py +283 -0
- examples/archive/scripts/MSParams_example.py +21 -0
- examples/archive/scripts/Molecular Formula Data Aggreation.py +84 -0
- examples/archive/scripts/Single Mz Search.py +69 -0
- examples/test_notebooks.py +145 -0
- ext_lib/ChemstationMSFileReader.dll +0 -0
- ext_lib/ChemstationMSFileReader.xml +126 -0
- ext_lib/RawFileReaderLicense.doc +0 -0
- ext_lib/ThermoFisher.CommonCore.BackgroundSubtraction.dll +0 -0
- ext_lib/ThermoFisher.CommonCore.BackgroundSubtraction.xml +2307 -0
- ext_lib/ThermoFisher.CommonCore.Data.dll +0 -0
- ext_lib/ThermoFisher.CommonCore.Data.xml +28974 -0
- ext_lib/ThermoFisher.CommonCore.MassPrecisionEstimator.dll +0 -0
- ext_lib/ThermoFisher.CommonCore.MassPrecisionEstimator.xml +241 -0
- ext_lib/ThermoFisher.CommonCore.RawFileReader.dll +0 -0
- ext_lib/ThermoFisher.CommonCore.RawFileReader.xml +31174 -0
- ext_lib/__init__.py +0 -0
- ext_lib/dotnet/OpenMcdf.Extensions.dll +0 -0
- ext_lib/dotnet/OpenMcdf.dll +0 -0
- ext_lib/dotnet/OpenMcdf.xml +1154 -0
- ext_lib/dotnet/System.IO.FileSystem.AccessControl.dll +0 -0
- ext_lib/dotnet/System.IO.FileSystem.AccessControl.xml +506 -0
- ext_lib/dotnet/System.Security.AccessControl.dll +0 -0
- ext_lib/dotnet/System.Security.AccessControl.xml +2043 -0
- ext_lib/dotnet/System.Security.Principal.Windows.dll +0 -0
- ext_lib/dotnet/System.Security.Principal.Windows.xml +1011 -0
- ext_lib/dotnet/ThermoFisher.CommonCore.BackgroundSubtraction.dll +0 -0
- ext_lib/dotnet/ThermoFisher.CommonCore.BackgroundSubtraction.xml +2307 -0
- ext_lib/dotnet/ThermoFisher.CommonCore.Data.dll +0 -0
- ext_lib/dotnet/ThermoFisher.CommonCore.Data.xml +29148 -0
- ext_lib/dotnet/ThermoFisher.CommonCore.MassPrecisionEstimator.dll +0 -0
- ext_lib/dotnet/ThermoFisher.CommonCore.MassPrecisionEstimator.xml +241 -0
- ext_lib/dotnet/ThermoFisher.CommonCore.RawFileReader.dll +0 -0
- ext_lib/dotnet/ThermoFisher.CommonCore.RawFileReader.xml +31492 -0
- ext_lib/version +18 -0
- support_code/atom_parsers/AtomsDescription.py +359 -0
- support_code/atom_parsers/CreateAtomsDescription.py +56 -0
- support_code/nmdc/filefinder.py +90 -0
- support_code/nmdc/lipidomics/lipidomics_workflow.py +748 -0
- support_code/nmdc/lipidomics/manifest_examples.py +49 -0
- support_code/nmdc/metabolomics/gcms_workflow.py +158 -0
- support_code/nmdc/metabolomics/lcms_metabolomics_targeted_search.py +59 -0
- support_code/nmdc/metabolomics/lcms_metabolomics_workflow.py +248 -0
- support_code/nmdc/metabolomics/metabolomics_collection.py +628 -0
- support_code/nmdc/metadata/dms_api.py +42 -0
- support_code/nmdc/nom/archived_scripts/nmdc_metadata_gen.py +288 -0
- support_code/nmdc/nom/archived_scripts/nom_grow_workflow.py +209 -0
- support_code/nmdc/nom/nom_workflow.py +312 -0
- support_code/windows_only_importers/BrukerCompassXtract.py +180 -0
- support_code/windows_only_importers/ThermoMSFileReader.py +405 -0
|
@@ -0,0 +1,2721 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pandas as pd
|
|
5
|
+
import warnings
|
|
6
|
+
import multiprocessing
|
|
7
|
+
|
|
8
|
+
import matplotlib.pyplot as plt
|
|
9
|
+
|
|
10
|
+
from corems.encapsulation.factory.parameters import LCMSParameters, LCMSCollectionParameters
|
|
11
|
+
from corems.mass_spectra.calc.lc_calc import LCCalculations, PHCalculations, LCMSCollectionCalculations
|
|
12
|
+
from corems.molecular_id.search.lcms_spectral_search import LCMSSpectralSearch
|
|
13
|
+
from corems.mass_spectrum.input.numpyArray import ms_from_array_profile, ms_from_array_centroid
|
|
14
|
+
from corems.mass_spectra.calc.lc_calc import find_closest
|
|
15
|
+
from corems.chroma_peak.factory.chroma_peak_classes import LCMSMassFeature
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class MassSpectraBase:
|
|
19
|
+
"""Base class for mass spectra objects.
|
|
20
|
+
|
|
21
|
+
Parameters
|
|
22
|
+
-----------
|
|
23
|
+
file_location : str or Path
|
|
24
|
+
The location of the file containing the mass spectra data.
|
|
25
|
+
analyzer : str, optional
|
|
26
|
+
The type of analyzer used to generate the mass spectra data. Defaults to 'Unknown'.
|
|
27
|
+
instrument_label : str, optional
|
|
28
|
+
The type of instrument used to generate the mass spectra data. Defaults to 'Unknown'.
|
|
29
|
+
sample_name : str, optional
|
|
30
|
+
The name of the sample; defaults to the file name if not provided to the parser. Defaults to None.
|
|
31
|
+
spectra_parser : object, optional
|
|
32
|
+
The spectra parser object used to create the mass spectra object. Defaults to None.
|
|
33
|
+
|
|
34
|
+
Attributes
|
|
35
|
+
-----------
|
|
36
|
+
spectra_parser_class : class
|
|
37
|
+
The class of the spectra parser used to create the mass spectra object.
|
|
38
|
+
file_location : str or Path
|
|
39
|
+
The location of the file containing the mass spectra data.
|
|
40
|
+
sample_name : str
|
|
41
|
+
The name of the sample; defaults to the file name if not provided to the parser.
|
|
42
|
+
analyzer : str
|
|
43
|
+
The type of analyzer used to generate the mass spectra data. Derived from the spectra parser.
|
|
44
|
+
instrument_label : str
|
|
45
|
+
The type of instrument used to generate the mass spectra data. Derived from the spectra parser.
|
|
46
|
+
_scan_info : dict
|
|
47
|
+
A dictionary containing the scan data with columns for scan number, scan time, ms level, precursor m/z,
|
|
48
|
+
scan text, and scan window (lower and upper).
|
|
49
|
+
Associated with the property scan_df, which returns a pandas DataFrame or can set this attribute from a pandas DataFrame.
|
|
50
|
+
_ms : dict
|
|
51
|
+
A dictionary containing mass spectra for the dataset, keys of dictionary are scan numbers. Initialized as an empty dictionary.
|
|
52
|
+
_ms_unprocessed: dictionary of pandas.DataFrames or None
|
|
53
|
+
A dictionary of unprocssed mass spectra data, as an (optional) intermediate data product for peak picking.
|
|
54
|
+
Key is ms_level, and value is dataframe with columns for scan number, m/z, and intensity. Default is None.
|
|
55
|
+
|
|
56
|
+
Methods
|
|
57
|
+
--------
|
|
58
|
+
* add_mass_spectra(scan_list, spectrum_mode: str = 'profile', use_parser = True, auto_process=True).
|
|
59
|
+
Add mass spectra (or singlel mass spectrum) to _ms slot, from a list of scans
|
|
60
|
+
* get_time_of_scan_id(scan).
|
|
61
|
+
Returns the scan time for the specified scan number.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
file_location,
|
|
67
|
+
analyzer="Unknown",
|
|
68
|
+
instrument_label="Unknown",
|
|
69
|
+
sample_name=None,
|
|
70
|
+
spectra_parser=None,
|
|
71
|
+
):
|
|
72
|
+
if isinstance(file_location, str):
|
|
73
|
+
file_location = Path(file_location)
|
|
74
|
+
else:
|
|
75
|
+
file_location = file_location
|
|
76
|
+
if not file_location.exists():
|
|
77
|
+
raise FileExistsError("File does not exist: " + str(file_location))
|
|
78
|
+
|
|
79
|
+
if sample_name:
|
|
80
|
+
self.sample_name = sample_name
|
|
81
|
+
else:
|
|
82
|
+
self.sample_name = file_location.stem
|
|
83
|
+
|
|
84
|
+
self.file_location = file_location
|
|
85
|
+
self.analyzer = analyzer
|
|
86
|
+
self.instrument_label = instrument_label
|
|
87
|
+
self._raw_file_location = None
|
|
88
|
+
|
|
89
|
+
# Add the spectra parser class to the object if it is not None
|
|
90
|
+
if spectra_parser is not None:
|
|
91
|
+
self.spectra_parser_class = spectra_parser.__class__
|
|
92
|
+
if self.spectra_parser_class.__name__ == "ReadCoreMSHDFMassSpectra":
|
|
93
|
+
self.raw_file_location = spectra_parser.get_raw_file_location()
|
|
94
|
+
|
|
95
|
+
# Check that spectra_parser.sample_name is same as sample_name etc, raise warning if not
|
|
96
|
+
if (
|
|
97
|
+
self.sample_name is not None
|
|
98
|
+
and self.sample_name != self.spectra_parser.sample_name
|
|
99
|
+
and self.spectra_parser_class.__name__ != "ReadCoreMSHDFMassSpectra"
|
|
100
|
+
):
|
|
101
|
+
warnings.warn(
|
|
102
|
+
"sample_name provided to MassSpectraBase object does not match sample_name provided to spectra parser object",
|
|
103
|
+
UserWarning,
|
|
104
|
+
)
|
|
105
|
+
if self.analyzer != self.spectra_parser.analyzer:
|
|
106
|
+
warnings.warn(
|
|
107
|
+
"analyzer provided to MassSpectraBase object does not match analyzer provided to spectra parser object",
|
|
108
|
+
UserWarning,
|
|
109
|
+
)
|
|
110
|
+
if self.instrument_label != self.spectra_parser.instrument_label:
|
|
111
|
+
warnings.warn(
|
|
112
|
+
"instrument provided to MassSpectraBase object does not match instrument provided to spectra parser object",
|
|
113
|
+
UserWarning,
|
|
114
|
+
)
|
|
115
|
+
if self.file_location != self.spectra_parser.file_location:
|
|
116
|
+
warnings.warn(
|
|
117
|
+
"file_location provided to MassSpectraBase object does not match file_location provided to spectra parser object",
|
|
118
|
+
UserWarning,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
# Instantiate empty dictionaries for scan information and mass spectra
|
|
122
|
+
self._scan_info = {}
|
|
123
|
+
self._ms = {}
|
|
124
|
+
self._ms_unprocessed = {}
|
|
125
|
+
|
|
126
|
+
@property
|
|
127
|
+
def spectra_parser(self):
|
|
128
|
+
"""Returns an instance of the spectra parser class."""
|
|
129
|
+
# Check if a file exists at the raw_file_location
|
|
130
|
+
if not Path(self.raw_file_location).exists():
|
|
131
|
+
raise FileNotFoundError(
|
|
132
|
+
f"Raw file not found at location: {self.raw_file_location}, update raw_file_location property to point to correct location."
|
|
133
|
+
)
|
|
134
|
+
return self.spectra_parser_class(self.raw_file_location)
|
|
135
|
+
|
|
136
|
+
@property
|
|
137
|
+
def raw_file_location(self):
|
|
138
|
+
"""Returns the file_location unless the _raw_file_location is not None."""
|
|
139
|
+
return self._raw_file_location if self._raw_file_location is not None else self.file_location
|
|
140
|
+
|
|
141
|
+
@raw_file_location.setter
|
|
142
|
+
def raw_file_location(self, value):
|
|
143
|
+
self._raw_file_location = value
|
|
144
|
+
|
|
145
|
+
def add_mass_spectrum(self, mass_spec):
|
|
146
|
+
"""Adds a mass spectrum to the dataset.
|
|
147
|
+
|
|
148
|
+
Parameters
|
|
149
|
+
-----------
|
|
150
|
+
mass_spec : MassSpectrum
|
|
151
|
+
The corems MassSpectrum object to be added to the dataset.
|
|
152
|
+
|
|
153
|
+
Notes
|
|
154
|
+
-----
|
|
155
|
+
This is a helper function for the add_mass_spectra() method, and is not intended to be called directly.
|
|
156
|
+
"""
|
|
157
|
+
# check if mass_spec has a scan_number attribute
|
|
158
|
+
if not hasattr(mass_spec, "scan_number"):
|
|
159
|
+
raise ValueError(
|
|
160
|
+
"Mass spectrum must have a scan_number attribute to be added to the dataset correctly"
|
|
161
|
+
)
|
|
162
|
+
self._ms[mass_spec.scan_number] = mass_spec
|
|
163
|
+
|
|
164
|
+
def add_mass_spectra(
|
|
165
|
+
self,
|
|
166
|
+
scan_list,
|
|
167
|
+
spectrum_mode=None,
|
|
168
|
+
ms_level=1,
|
|
169
|
+
use_parser=True,
|
|
170
|
+
auto_process=True,
|
|
171
|
+
ms_params=None,
|
|
172
|
+
):
|
|
173
|
+
"""Add mass spectra to _ms dictionary, from a list of scans or single scan
|
|
174
|
+
|
|
175
|
+
Notes
|
|
176
|
+
-----
|
|
177
|
+
The mass spectra will inherit the mass_spectrum, ms_peak, and molecular_search parameters from the LCMSBase object.
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
Parameters
|
|
181
|
+
-----------
|
|
182
|
+
scan_list : list of ints
|
|
183
|
+
List of scans to use to populate _ms slot
|
|
184
|
+
spectrum_mode : str or None
|
|
185
|
+
The spectrum mode to use for the mass spectra.
|
|
186
|
+
If None, method will use the spectrum mode from the spectra parser to ascertain the spectrum mode (this allows for mixed types).
|
|
187
|
+
Defaults to None.
|
|
188
|
+
ms_level : int, optional
|
|
189
|
+
The MS level to use for the mass spectra.
|
|
190
|
+
This is used to pass the molecular_search parameters from the LCMS object to the individual MassSpectrum objects.
|
|
191
|
+
Defaults to 1.
|
|
192
|
+
using_parser : bool
|
|
193
|
+
Whether to use the mass spectra parser to get the mass spectra. Defaults to True.
|
|
194
|
+
auto_process : bool
|
|
195
|
+
Whether to auto-process the mass spectra. Defaults to True.
|
|
196
|
+
ms_params : MSParameters or None
|
|
197
|
+
The mass spectrum parameters to use for the mass spectra. If None, uses the globally set MSParameters.
|
|
198
|
+
|
|
199
|
+
Raises
|
|
200
|
+
------
|
|
201
|
+
TypeError
|
|
202
|
+
If scan_list is not a list of ints
|
|
203
|
+
ValueError
|
|
204
|
+
If polarity is not 'positive' or 'negative'
|
|
205
|
+
If ms_level is not 1 or 2
|
|
206
|
+
"""
|
|
207
|
+
|
|
208
|
+
# check if scan_list is a list or a single int; if single int, convert to list
|
|
209
|
+
if isinstance(scan_list, int):
|
|
210
|
+
scan_list = [scan_list]
|
|
211
|
+
if not isinstance(scan_list, list):
|
|
212
|
+
raise TypeError("scan_list must be a list of integers")
|
|
213
|
+
for scan in scan_list:
|
|
214
|
+
if not isinstance(scan, int):
|
|
215
|
+
raise TypeError("scan_list must be a list of integers")
|
|
216
|
+
|
|
217
|
+
# set polarity to -1 if negative mode, 1 if positive mode (for mass spectrum creation)
|
|
218
|
+
if self.polarity == "negative":
|
|
219
|
+
polarity = -1
|
|
220
|
+
elif self.polarity == "positive":
|
|
221
|
+
polarity = 1
|
|
222
|
+
else:
|
|
223
|
+
raise ValueError(
|
|
224
|
+
"Polarity not set for dataset, must be a either 'positive' or 'negative'"
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
# is not using_parser, check that ms1 and ms2 are not None
|
|
228
|
+
if not use_parser:
|
|
229
|
+
if ms_level not in self._ms_unprocessed.keys():
|
|
230
|
+
raise ValueError(
|
|
231
|
+
"ms_level {} not found in _ms_unprocessed dictionary".format(
|
|
232
|
+
ms_level
|
|
233
|
+
)
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
scan_list = list(set(scan_list))
|
|
237
|
+
scan_list.sort()
|
|
238
|
+
|
|
239
|
+
# Skip scans that have already been added to _ms to avoid redundant reprocessing
|
|
240
|
+
already_added = [s for s in scan_list if s in self._ms]
|
|
241
|
+
if already_added:
|
|
242
|
+
warnings.warn(
|
|
243
|
+
"Skipping {} scan(s) already present in _ms: {}".format(
|
|
244
|
+
len(already_added), already_added
|
|
245
|
+
),
|
|
246
|
+
UserWarning,
|
|
247
|
+
)
|
|
248
|
+
scan_list = [s for s in scan_list if s not in self._ms]
|
|
249
|
+
if not scan_list:
|
|
250
|
+
return
|
|
251
|
+
|
|
252
|
+
if not use_parser:
|
|
253
|
+
if self._ms_unprocessed[ms_level] is None:
|
|
254
|
+
raise ValueError(
|
|
255
|
+
"No unprocessed data found for ms_level {}".format(ms_level)
|
|
256
|
+
)
|
|
257
|
+
if (
|
|
258
|
+
len(
|
|
259
|
+
np.setdiff1d(
|
|
260
|
+
scan_list, self._ms_unprocessed[ms_level].scan.tolist()
|
|
261
|
+
)
|
|
262
|
+
)
|
|
263
|
+
> 0
|
|
264
|
+
):
|
|
265
|
+
raise ValueError(
|
|
266
|
+
"Not all scans in scan_list are present in the unprocessed data"
|
|
267
|
+
)
|
|
268
|
+
# Prepare the ms_df for parsing
|
|
269
|
+
ms_df = self._ms_unprocessed[ms_level].copy().set_index("scan", drop=False)
|
|
270
|
+
|
|
271
|
+
if use_parser:
|
|
272
|
+
# Use batch function to get all mass spectra at once
|
|
273
|
+
if spectrum_mode is None:
|
|
274
|
+
# get spectrum mode from _scan_info for each scan
|
|
275
|
+
spectrum_modes = [self.scan_df.loc[scan, "ms_format"] for scan in scan_list]
|
|
276
|
+
spectrum_mode_batch = spectrum_modes[0] if len(set(spectrum_modes)) == 1 else None
|
|
277
|
+
else:
|
|
278
|
+
spectrum_mode_batch = spectrum_mode
|
|
279
|
+
|
|
280
|
+
ms_list = self.spectra_parser.get_mass_spectra_from_scan_list(
|
|
281
|
+
scan_list=scan_list,
|
|
282
|
+
spectrum_mode=spectrum_mode_batch,
|
|
283
|
+
auto_process=False,
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
# Process each mass spectrum
|
|
287
|
+
for i, scan in enumerate(scan_list):
|
|
288
|
+
ms = ms_list[i] if i < len(ms_list) else None
|
|
289
|
+
if ms is not None:
|
|
290
|
+
if ms_params is not None:
|
|
291
|
+
ms.parameters = ms_params
|
|
292
|
+
ms.scan_number = scan
|
|
293
|
+
if auto_process:
|
|
294
|
+
ms.process_mass_spec()
|
|
295
|
+
self.add_mass_spectrum(ms)
|
|
296
|
+
else:
|
|
297
|
+
# Original non-parser logic remains unchanged
|
|
298
|
+
for scan in scan_list:
|
|
299
|
+
ms = None
|
|
300
|
+
if spectrum_mode is None:
|
|
301
|
+
# get spectrum mode from _scan_info
|
|
302
|
+
spectrum_mode_scan = self.scan_df.loc[scan, "ms_format"]
|
|
303
|
+
else:
|
|
304
|
+
spectrum_mode_scan = spectrum_mode
|
|
305
|
+
|
|
306
|
+
my_ms_df = ms_df.loc[scan]
|
|
307
|
+
if spectrum_mode_scan == "profile":
|
|
308
|
+
# Check this - it might be better to use the MassSpectrumProfile class to instantiate the mass spectrum
|
|
309
|
+
ms = ms_from_array_profile(
|
|
310
|
+
my_ms_df.mz,
|
|
311
|
+
my_ms_df.intensity,
|
|
312
|
+
self.file_location,
|
|
313
|
+
polarity=polarity,
|
|
314
|
+
auto_process=False,
|
|
315
|
+
)
|
|
316
|
+
else:
|
|
317
|
+
ms = ms_from_array_centroid(
|
|
318
|
+
mz = my_ms_df.mz,
|
|
319
|
+
abundance = my_ms_df.intensity,
|
|
320
|
+
rp = [np.nan] * len(my_ms_df.mz),
|
|
321
|
+
s2n = [np.nan] * len(my_ms_df.mz),
|
|
322
|
+
dataname = self.file_location,
|
|
323
|
+
polarity=polarity,
|
|
324
|
+
auto_process=False,
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
# Set the mass spectrum parameters, auto-process if auto_process is True, and add to the dataset
|
|
328
|
+
if ms is not None:
|
|
329
|
+
if ms_params is not None:
|
|
330
|
+
ms.parameters = ms_params
|
|
331
|
+
ms.scan_number = scan
|
|
332
|
+
if auto_process:
|
|
333
|
+
ms.process_mass_spec()
|
|
334
|
+
self.add_mass_spectrum(ms)
|
|
335
|
+
|
|
336
|
+
def get_time_of_scan_id(self, scan):
|
|
337
|
+
"""Returns the scan time for the specified scan number.
|
|
338
|
+
|
|
339
|
+
Parameters
|
|
340
|
+
-----------
|
|
341
|
+
scan : int
|
|
342
|
+
The scan number of the desired scan time.
|
|
343
|
+
|
|
344
|
+
Returns
|
|
345
|
+
--------
|
|
346
|
+
float
|
|
347
|
+
The scan time for the specified scan number (in minutes).
|
|
348
|
+
|
|
349
|
+
Raises
|
|
350
|
+
------
|
|
351
|
+
ValueError
|
|
352
|
+
If no scan time is found for the specified scan number.
|
|
353
|
+
"""
|
|
354
|
+
# Check if _retenion_time_list is empty and raise error if so
|
|
355
|
+
if len(self._retention_time_list) == 0:
|
|
356
|
+
raise ValueError("No retention times found in dataset")
|
|
357
|
+
rt = self._retention_time_list[self._scans_number_list.index(scan)]
|
|
358
|
+
return rt
|
|
359
|
+
|
|
360
|
+
@property
|
|
361
|
+
def scan_df(self):
|
|
362
|
+
"""
|
|
363
|
+
pandas.DataFrame : A pandas DataFrame containing the scan info data with columns for scan number, scan time, ms level, precursor m/z, scan text, and scan window (lower and upper).
|
|
364
|
+
"""
|
|
365
|
+
scan_df = pd.DataFrame.from_dict(self._scan_info)
|
|
366
|
+
return scan_df
|
|
367
|
+
|
|
368
|
+
@property
|
|
369
|
+
def ms(self):
|
|
370
|
+
"""
|
|
371
|
+
dictionary : contains the key associated with mass spectra and values are the associated MassSpecProfiles
|
|
372
|
+
"""
|
|
373
|
+
return self._ms
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
@scan_df.setter
|
|
377
|
+
def scan_df(self, df):
|
|
378
|
+
"""
|
|
379
|
+
Sets the scan data for the dataset.
|
|
380
|
+
|
|
381
|
+
Parameters
|
|
382
|
+
-----------
|
|
383
|
+
df : pandas.DataFrame
|
|
384
|
+
A pandas DataFrame containing the scan data with columns for scan number, scan time, ms level,
|
|
385
|
+
precursor m/z, scan text, and scan window (lower and upper).
|
|
386
|
+
"""
|
|
387
|
+
self._scan_info = df.to_dict()
|
|
388
|
+
|
|
389
|
+
def __getitem__(self, scan_number):
|
|
390
|
+
return self._ms.get(scan_number)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
class LCMSBase(MassSpectraBase, LCCalculations, PHCalculations, LCMSSpectralSearch):
|
|
394
|
+
"""A class representing a liquid chromatography-mass spectrometry (LC-MS) data object.
|
|
395
|
+
|
|
396
|
+
This class is not intended to be instantiated directly, but rather to be instantiated by an appropriate mass spectra parser using the get_lcms_obj() method.
|
|
397
|
+
|
|
398
|
+
Parameters
|
|
399
|
+
-----------
|
|
400
|
+
file_location : str or Path
|
|
401
|
+
The location of the file containing the mass spectra data.
|
|
402
|
+
analyzer : str, optional
|
|
403
|
+
The type of analyzer used to generate the mass spectra data. Defaults to 'Unknown'.
|
|
404
|
+
instrument_label : str, optional
|
|
405
|
+
The type of instrument used to generate the mass spectra data. Defaults to 'Unknown'.
|
|
406
|
+
sample_name : str, optional
|
|
407
|
+
The name of the sample; defaults to the file name if not provided to the parser. Defaults to None.
|
|
408
|
+
spectra_parser : object, optional
|
|
409
|
+
The spectra parser object used to create the mass spectra object. Defaults to None.
|
|
410
|
+
|
|
411
|
+
Attributes
|
|
412
|
+
-----------
|
|
413
|
+
polarity : str
|
|
414
|
+
The polarity of the ionization mode used for the dataset.
|
|
415
|
+
_parameters : LCMSParameters
|
|
416
|
+
The parameters used for all methods called on the LCMSBase object. Set upon instantiation from LCMSParameters.
|
|
417
|
+
_retention_time_list : numpy.ndarray
|
|
418
|
+
An array of retention times for the dataset.
|
|
419
|
+
_scans_number_list : list
|
|
420
|
+
A list of scan numbers for the dataset.
|
|
421
|
+
_tic_list : numpy.ndarray
|
|
422
|
+
An array of total ion current (TIC) values for the dataset.
|
|
423
|
+
eics : dict
|
|
424
|
+
A dictionary containing extracted ion chromatograms (EICs) for the dataset.
|
|
425
|
+
Key is the mz of the EIC. Initialized as an empty dictionary.
|
|
426
|
+
mass_features : dictionary of LCMSMassFeature objects
|
|
427
|
+
A dictionary containing mass features for the dataset.
|
|
428
|
+
Key is mass feature ID. Initialized as an empty dictionary.
|
|
429
|
+
induced_mass_features: dictionary of LCMSMassFeature objects
|
|
430
|
+
A dictionary containing mass features from a collection that don't
|
|
431
|
+
satisfy criteria for initial mass features. Key is mass feature ID.
|
|
432
|
+
Initialized as an empty dictionary.
|
|
433
|
+
missing_mass_features: pandas.DataFrame
|
|
434
|
+
A table of clusters in a given sample for which a mass feature was
|
|
435
|
+
sought and not found
|
|
436
|
+
spectral_search_results : dictionary of MS2SearchResults objects
|
|
437
|
+
A dictionary containing spectral search results for the dataset.
|
|
438
|
+
Key is scan number : precursor mz. Initialized as an empty dictionary.
|
|
439
|
+
|
|
440
|
+
Methods
|
|
441
|
+
--------
|
|
442
|
+
* get_parameters_json().
|
|
443
|
+
Returns the parameters used for the LC-MS analysis in JSON format.
|
|
444
|
+
* add_associated_ms2_dda(add_to_lcmsobj=True, auto_process=True, use_parser=True)
|
|
445
|
+
Adds which MS2 scans are associated with each mass feature to the
|
|
446
|
+
mass_features dictionary and optionally adds the MS2 spectra to the _ms dictionary.
|
|
447
|
+
* add_associated_ms1(add_to_lcmsobj=True, auto_process=True, use_parser=True)
|
|
448
|
+
Adds the MS1 spectra associated with each mass feature to the
|
|
449
|
+
mass_features dictionary and adds the MS1 spectra to the _ms dictionary.
|
|
450
|
+
* mass_features_to_df()
|
|
451
|
+
Returns a pandas dataframe summarizing the mass features in the dataset.
|
|
452
|
+
* set_tic_list_from_data(overwrite=False)
|
|
453
|
+
Sets the TIC list from the mass spectrum objects within the _ms dictionary.
|
|
454
|
+
* set_retention_time_from_data(overwrite=False)
|
|
455
|
+
Sets the retention time list from the data in the _ms dictionary.
|
|
456
|
+
* set_scans_number_from_data(overwrite=False)
|
|
457
|
+
Sets the scan number list from the data in the _ms dictionary.
|
|
458
|
+
* plot_composite_mz_features(binsize = 1e-4, ph_int_min_thresh = 0.001, mf_plot = True, ms2_plot = True, return_fig = False)
|
|
459
|
+
Generates plot of M/Z features comparing scan time vs M/Z value
|
|
460
|
+
* search_for_targeted_mass_feature(ms1df: pd.DataFrame, sample: pd.Series, tol_flag = 0)
|
|
461
|
+
Searches for mass features in specific M/Z and scan time windows that
|
|
462
|
+
were missed by the persistent homology search
|
|
463
|
+
"""
|
|
464
|
+
|
|
465
|
+
def __init__(
|
|
466
|
+
self,
|
|
467
|
+
file_location,
|
|
468
|
+
analyzer="Unknown",
|
|
469
|
+
instrument_label="Unknown",
|
|
470
|
+
sample_name=None,
|
|
471
|
+
spectra_parser=None,
|
|
472
|
+
):
|
|
473
|
+
super().__init__(
|
|
474
|
+
file_location, analyzer, instrument_label, sample_name, spectra_parser
|
|
475
|
+
)
|
|
476
|
+
self.polarity = ""
|
|
477
|
+
self._parameters = LCMSParameters()
|
|
478
|
+
self._retention_time_list = []
|
|
479
|
+
self._scans_number_list = []
|
|
480
|
+
self._tic_list = []
|
|
481
|
+
self.eics = {}
|
|
482
|
+
self.mass_features = {}
|
|
483
|
+
self.induced_mass_features = {}
|
|
484
|
+
self.spectral_search_results = {}
|
|
485
|
+
|
|
486
|
+
def get_eic_mz_for_mass_feature(self, mf_mz, tolerance=0.0001):
|
|
487
|
+
"""Get the EIC dictionary key (m/z) that best matches a mass feature's m/z.
|
|
488
|
+
|
|
489
|
+
Finds the closest EIC m/z key within the specified tolerance.
|
|
490
|
+
|
|
491
|
+
Parameters
|
|
492
|
+
----------
|
|
493
|
+
mf_mz : float
|
|
494
|
+
The m/z value of the mass feature to match.
|
|
495
|
+
tolerance : float, optional
|
|
496
|
+
Maximum m/z difference for matching. Default is 0.0001 Da.
|
|
497
|
+
|
|
498
|
+
Returns
|
|
499
|
+
-------
|
|
500
|
+
float or None
|
|
501
|
+
The EIC dictionary key (m/z) of the closest matching EIC,
|
|
502
|
+
or None if no EIC is within tolerance.
|
|
503
|
+
"""
|
|
504
|
+
if not hasattr(self, 'eics') or not self.eics:
|
|
505
|
+
return None
|
|
506
|
+
|
|
507
|
+
best_eic_mz = None
|
|
508
|
+
best_diff = tolerance
|
|
509
|
+
for eic_mz in self.eics.keys():
|
|
510
|
+
diff = abs(mf_mz - eic_mz)
|
|
511
|
+
if diff < best_diff:
|
|
512
|
+
best_diff = diff
|
|
513
|
+
best_eic_mz = eic_mz
|
|
514
|
+
return best_eic_mz
|
|
515
|
+
|
|
516
|
+
def associate_eics_with_mass_features(self, tolerance=0.0001, induced=False):
|
|
517
|
+
"""Associate EICs with mass features using tolerance-based m/z matching.
|
|
518
|
+
|
|
519
|
+
Associates EIC_Data objects from self.eics with mass features by finding
|
|
520
|
+
the closest EIC within the specified m/z tolerance. This is more robust
|
|
521
|
+
than exact matching which can fail due to floating point precision issues.
|
|
522
|
+
|
|
523
|
+
Parameters
|
|
524
|
+
----------
|
|
525
|
+
tolerance : float, optional
|
|
526
|
+
Maximum m/z difference for matching EICs to mass features. Default is 0.0001 Da.
|
|
527
|
+
induced : bool, optional
|
|
528
|
+
If True, associates EICs with induced_mass_features instead of mass_features.
|
|
529
|
+
Default is False.
|
|
530
|
+
|
|
531
|
+
Notes
|
|
532
|
+
-----
|
|
533
|
+
For each mass feature, this method finds the EIC with the closest m/z value
|
|
534
|
+
within the tolerance window and assigns it to the mass feature's _eic_data attribute.
|
|
535
|
+
If multiple EICs are within tolerance, the one with the smallest m/z difference is chosen.
|
|
536
|
+
"""
|
|
537
|
+
# Select which mass features dictionary to use
|
|
538
|
+
mf_dict = self.induced_mass_features if induced else self.mass_features
|
|
539
|
+
|
|
540
|
+
# Use the _eic_mz attribute on each mass_feature to find the closest matching EIC
|
|
541
|
+
for idx in mf_dict.keys():
|
|
542
|
+
mf_mz = mf_dict[idx]._eic_mz
|
|
543
|
+
# Find closest EIC within tolerance
|
|
544
|
+
best_match = None
|
|
545
|
+
best_diff = tolerance
|
|
546
|
+
for eic_mz, eic_data in self.eics.items():
|
|
547
|
+
diff = abs(mf_mz - eic_mz)
|
|
548
|
+
if diff < best_diff:
|
|
549
|
+
best_diff = diff
|
|
550
|
+
best_match = eic_data
|
|
551
|
+
if best_match is not None:
|
|
552
|
+
mf_dict[idx]._eic_data = best_match
|
|
553
|
+
|
|
554
|
+
def get_parameters_json(self):
|
|
555
|
+
"""Returns the parameters stored for the LC-MS object in JSON format.
|
|
556
|
+
|
|
557
|
+
Returns
|
|
558
|
+
--------
|
|
559
|
+
str
|
|
560
|
+
The parameters used for the LC-MS analysis in JSON format.
|
|
561
|
+
"""
|
|
562
|
+
return self.parameters.to_json()
|
|
563
|
+
|
|
564
|
+
def remove_unprocessed_data(self, ms_level=None):
|
|
565
|
+
"""Removes the unprocessed data from the LCMSBase object.
|
|
566
|
+
|
|
567
|
+
Parameters
|
|
568
|
+
-----------
|
|
569
|
+
ms_level : int, optional
|
|
570
|
+
The MS level to remove the unprocessed data for. If None, removes unprocessed data for all MS levels.
|
|
571
|
+
|
|
572
|
+
Raises
|
|
573
|
+
------
|
|
574
|
+
ValueError
|
|
575
|
+
If ms_level is not 1 or 2.
|
|
576
|
+
|
|
577
|
+
Notes
|
|
578
|
+
-----
|
|
579
|
+
This method is useful for freeing up memory after the data has been processed.
|
|
580
|
+
"""
|
|
581
|
+
if ms_level is None:
|
|
582
|
+
for ms_level in self._ms_unprocessed.keys():
|
|
583
|
+
self._ms_unprocessed[ms_level] = None
|
|
584
|
+
if ms_level not in [1, 2]:
|
|
585
|
+
raise ValueError("ms_level must be 1 or 2")
|
|
586
|
+
self._ms_unprocessed[ms_level] = None
|
|
587
|
+
|
|
588
|
+
def _filter_ms2_scans_by_integration_bounds(self, mf_dict=None):
|
|
589
|
+
"""Filter MS2 scans to only include those within integration bounds.
|
|
590
|
+
|
|
591
|
+
Removes MS2 scan numbers that fall outside the start_scan to final_scan range
|
|
592
|
+
for each mass feature. This should be called after integration sets the bounds.
|
|
593
|
+
|
|
594
|
+
Parameters
|
|
595
|
+
----------
|
|
596
|
+
mf_dict : dict, optional
|
|
597
|
+
Dictionary of mass features to filter. If None, uses self.mass_features.
|
|
598
|
+
|
|
599
|
+
Returns
|
|
600
|
+
-------
|
|
601
|
+
int
|
|
602
|
+
Number of MS2 scans removed across all mass features.
|
|
603
|
+
"""
|
|
604
|
+
if mf_dict is None:
|
|
605
|
+
mf_dict = self.mass_features
|
|
606
|
+
|
|
607
|
+
total_removed = 0
|
|
608
|
+
|
|
609
|
+
for mf_id, mf in mf_dict.items():
|
|
610
|
+
# Only filter if integration bounds are set and MS2 scans exist
|
|
611
|
+
if (hasattr(mf, 'start_scan') and hasattr(mf, 'final_scan') and
|
|
612
|
+
mf.start_scan is not None and mf.final_scan is not None and
|
|
613
|
+
mf.ms2_scan_numbers is not None and len(mf.ms2_scan_numbers) > 0):
|
|
614
|
+
|
|
615
|
+
# Filter scan numbers to only those within bounds
|
|
616
|
+
original_count = len(mf.ms2_scan_numbers)
|
|
617
|
+
mf.ms2_scan_numbers = [
|
|
618
|
+
scan for scan in mf.ms2_scan_numbers
|
|
619
|
+
if mf.start_scan <= scan <= mf.final_scan
|
|
620
|
+
]
|
|
621
|
+
removed = original_count - len(mf.ms2_scan_numbers)
|
|
622
|
+
total_removed += removed
|
|
623
|
+
|
|
624
|
+
return total_removed
|
|
625
|
+
|
|
626
|
+
def _find_ms2_scans_for_mass_features(self, mf_ids=None, scan_filter=None):
|
|
627
|
+
"""Find MS2 scans associated with mass features.
|
|
628
|
+
|
|
629
|
+
This helper method finds MS2 scans that match mass features based on RT and m/z tolerances.
|
|
630
|
+
It updates the ms2_scan_numbers attribute on each mass feature.
|
|
631
|
+
|
|
632
|
+
Parameters
|
|
633
|
+
----------
|
|
634
|
+
mf_ids : list of int, optional
|
|
635
|
+
List of mass feature IDs to find MS2 for. If None, finds for all mass features.
|
|
636
|
+
scan_filter : str, optional
|
|
637
|
+
Filter string for MS2 scans (e.g., 'hcd'). Default is None.
|
|
638
|
+
|
|
639
|
+
Returns
|
|
640
|
+
-------
|
|
641
|
+
list
|
|
642
|
+
List of unique MS2 scan numbers found across all mass features.
|
|
643
|
+
|
|
644
|
+
Raises
|
|
645
|
+
------
|
|
646
|
+
ValueError
|
|
647
|
+
If no MS2 scans are found in the dataset.
|
|
648
|
+
"""
|
|
649
|
+
# Get mass features to process
|
|
650
|
+
if mf_ids is None:
|
|
651
|
+
mf_ids = list(self.mass_features.keys())
|
|
652
|
+
|
|
653
|
+
# Get mass features dataframe
|
|
654
|
+
mf_df = self.mass_features_to_df()
|
|
655
|
+
mf_df = mf_df.loc[mf_ids].copy()
|
|
656
|
+
|
|
657
|
+
# Find ms2 scans that have a precursor m/z value
|
|
658
|
+
ms2_scans = self.scan_df[self.scan_df.ms_level == 2]
|
|
659
|
+
ms2_scans = ms2_scans[~ms2_scans.precursor_mz.isna()]
|
|
660
|
+
ms2_scans = ms2_scans[ms2_scans.tic > 0]
|
|
661
|
+
|
|
662
|
+
if len(ms2_scans) == 0:
|
|
663
|
+
raise ValueError("No DDA scans found in dataset")
|
|
664
|
+
|
|
665
|
+
if scan_filter is not None:
|
|
666
|
+
ms2_scans = ms2_scans[ms2_scans.scan_text.str.contains(scan_filter)]
|
|
667
|
+
|
|
668
|
+
# Get tolerances from parameters
|
|
669
|
+
time_tol = self.parameters.lc_ms.ms2_dda_rt_tolerance
|
|
670
|
+
mz_tol = self.parameters.lc_ms.ms2_dda_mz_tolerance
|
|
671
|
+
|
|
672
|
+
# For each mass feature, find the ms2 scans that are within the roi scan time and mz range
|
|
673
|
+
dda_scans = []
|
|
674
|
+
for i, row in mf_df.iterrows():
|
|
675
|
+
ms2_scans_filtered = ms2_scans[
|
|
676
|
+
ms2_scans.scan_time.between(
|
|
677
|
+
row.scan_time - time_tol, row.scan_time + time_tol
|
|
678
|
+
)
|
|
679
|
+
]
|
|
680
|
+
ms2_scans_filtered = ms2_scans_filtered[
|
|
681
|
+
ms2_scans_filtered.precursor_mz.between(
|
|
682
|
+
row.mz - mz_tol, row.mz + mz_tol
|
|
683
|
+
)
|
|
684
|
+
]
|
|
685
|
+
scan_list = ms2_scans_filtered.scan.tolist()
|
|
686
|
+
if scan_list:
|
|
687
|
+
# Filter scans by integration bounds if they exist
|
|
688
|
+
mf = self.mass_features[i]
|
|
689
|
+
if (hasattr(mf, 'start_scan') and hasattr(mf, 'final_scan') and
|
|
690
|
+
mf.start_scan is not None and mf.final_scan is not None):
|
|
691
|
+
# Only keep scans within integration bounds
|
|
692
|
+
scan_list = [s for s in scan_list if mf.start_scan <= s <= mf.final_scan]
|
|
693
|
+
|
|
694
|
+
if scan_list: # Only add if there are still scans after filtering
|
|
695
|
+
self.mass_features[i].ms2_scan_numbers = (
|
|
696
|
+
scan_list + list(self.mass_features[i].ms2_scan_numbers)
|
|
697
|
+
)
|
|
698
|
+
dda_scans.extend(scan_list)
|
|
699
|
+
|
|
700
|
+
return list(set(dda_scans))
|
|
701
|
+
|
|
702
|
+
def add_associated_ms2_dda(
|
|
703
|
+
self,
|
|
704
|
+
auto_process=True,
|
|
705
|
+
use_parser=True,
|
|
706
|
+
spectrum_mode=None,
|
|
707
|
+
ms_params_key="ms2",
|
|
708
|
+
scan_filter=None,
|
|
709
|
+
):
|
|
710
|
+
"""Add MS2 spectra associated with mass features to the dataset.
|
|
711
|
+
|
|
712
|
+
Populates the mass_features ms2_scan_numbers attribute (on mass_features dictionary on LCMSObject)
|
|
713
|
+
|
|
714
|
+
Parameters
|
|
715
|
+
-----------
|
|
716
|
+
auto_process : bool, optional
|
|
717
|
+
If True, auto-processes the MS2 spectra before adding it to the object's _ms dictionary. Default is True.
|
|
718
|
+
use_parser : bool, optional
|
|
719
|
+
If True, envoke the spectra parser to get the MS2 spectra. Default is True.
|
|
720
|
+
spectrum_mode : str or None, optional
|
|
721
|
+
The spectrum mode to use for the mass spectra. If None, method will use the spectrum mode
|
|
722
|
+
from the spectra parser to ascertain the spectrum mode (this allows for mixed types).
|
|
723
|
+
Defaults to None. (faster if defined, otherwise will check each scan)
|
|
724
|
+
ms_params_key : string, optional
|
|
725
|
+
The key of the mass spectrum parameters to use for the mass spectra, accessed from the LCMSObject.parameters.mass_spectrum attribute.
|
|
726
|
+
Defaults to 'ms2'.
|
|
727
|
+
scan_filter : str
|
|
728
|
+
A string to filter the scans to add to the _ms dictionary. If None, all scans are added. Defaults to None.
|
|
729
|
+
"hcd" will pull out only HCD scans.
|
|
730
|
+
|
|
731
|
+
Raises
|
|
732
|
+
------
|
|
733
|
+
ValueError
|
|
734
|
+
If mass_features is not set, must run find_mass_features() first.
|
|
735
|
+
If no MS2 scans are found in the dataset.
|
|
736
|
+
If no precursor m/z values are found in MS2 scans, not a DDA dataset.
|
|
737
|
+
"""
|
|
738
|
+
# Check if mass_features is set, raise error if not
|
|
739
|
+
if self.mass_features is None:
|
|
740
|
+
raise ValueError(
|
|
741
|
+
"mass_features not set, must run find_mass_features() first"
|
|
742
|
+
)
|
|
743
|
+
|
|
744
|
+
# reconfigure ms_params to get the correct mass spectrum parameters from the key
|
|
745
|
+
ms_params = self.parameters.mass_spectrum[ms_params_key]
|
|
746
|
+
|
|
747
|
+
# Find MS2 scans for all mass features
|
|
748
|
+
dda_scans = self._find_ms2_scans_for_mass_features(scan_filter=scan_filter)
|
|
749
|
+
|
|
750
|
+
# Load MS2 spectra
|
|
751
|
+
self.add_mass_spectra(
|
|
752
|
+
scan_list=dda_scans,
|
|
753
|
+
auto_process=auto_process,
|
|
754
|
+
spectrum_mode=spectrum_mode,
|
|
755
|
+
use_parser=use_parser,
|
|
756
|
+
ms_params=ms_params,
|
|
757
|
+
)
|
|
758
|
+
|
|
759
|
+
# Associate appropriate _ms attribute to appropriate mass feature's ms2_mass_spectra attribute
|
|
760
|
+
for mf_id in self.mass_features:
|
|
761
|
+
if self.mass_features[mf_id].ms2_scan_numbers is not None:
|
|
762
|
+
for dda_scan in self.mass_features[mf_id].ms2_scan_numbers:
|
|
763
|
+
if dda_scan in self._ms.keys():
|
|
764
|
+
self.mass_features[mf_id].ms2_mass_spectra[dda_scan] = self._ms[
|
|
765
|
+
dda_scan
|
|
766
|
+
]
|
|
767
|
+
|
|
768
|
+
def add_associated_ms1(
|
|
769
|
+
self, auto_process=True, use_parser=True, spectrum_mode=None, induced_features=False
|
|
770
|
+
):
|
|
771
|
+
"""Add MS1 spectra associated with mass features to the dataset.
|
|
772
|
+
|
|
773
|
+
Parameters
|
|
774
|
+
-----------
|
|
775
|
+
auto_process : bool, optional
|
|
776
|
+
If True, auto-processes the MS1 spectra before adding it to the object's _ms dictionary. Default is True.
|
|
777
|
+
use_parser : bool, optional
|
|
778
|
+
If True, envoke the spectra parser to get the MS1 spectra. Default is True.
|
|
779
|
+
spectrum_mode : str or None, optional
|
|
780
|
+
The spectrum mode to use for the mass spectra. If None, method will use the spectrum mode
|
|
781
|
+
from the spectra parser to ascertain the spectrum mode (this allows for mixed types).
|
|
782
|
+
Defaults to None. (faster if defined, otherwise will check each scan)
|
|
783
|
+
induced_features : bool, optional
|
|
784
|
+
If True, add associated MS1 of the induced mass features instead of the primary mass features
|
|
785
|
+
|
|
786
|
+
Raises
|
|
787
|
+
------
|
|
788
|
+
ValueError
|
|
789
|
+
If mass_features is not set, must run find_mass_features() first.
|
|
790
|
+
If apex scans are not profile mode, all apex scans must be profile mode for averaging.
|
|
791
|
+
If number of scans to average is not 1 or an integer with an integer median (i.e. 3, 5, 7, 9).
|
|
792
|
+
If deconvolute is True and no EICs are found, did you run integrate_mass_features() first?
|
|
793
|
+
"""
|
|
794
|
+
# Check if mass_features is set, raise error if not
|
|
795
|
+
if self.mass_features is None:
|
|
796
|
+
raise ValueError(
|
|
797
|
+
"mass_features not set, must run find_mass_features() first"
|
|
798
|
+
)
|
|
799
|
+
|
|
800
|
+
if induced_features:
|
|
801
|
+
mf_dict = self.induced_mass_features
|
|
802
|
+
else:
|
|
803
|
+
mf_dict = self.mass_features
|
|
804
|
+
|
|
805
|
+
scans_to_average = self.parameters.lc_ms.ms1_scans_to_average
|
|
806
|
+
|
|
807
|
+
## sketchy work around for induced mass features
|
|
808
|
+
scan_list = [
|
|
809
|
+
int(mf_dict[x].apex_scan) for x in mf_dict if int(mf_dict[x].apex_scan) != -99
|
|
810
|
+
]
|
|
811
|
+
|
|
812
|
+
if scans_to_average == 1:
|
|
813
|
+
# Add to LCMSobj
|
|
814
|
+
self.add_mass_spectra(
|
|
815
|
+
scan_list = scan_list,
|
|
816
|
+
auto_process=auto_process,
|
|
817
|
+
use_parser=use_parser,
|
|
818
|
+
spectrum_mode=spectrum_mode,
|
|
819
|
+
ms_params=self.parameters.mass_spectrum["ms1"],
|
|
820
|
+
)
|
|
821
|
+
|
|
822
|
+
elif (
|
|
823
|
+
(scans_to_average - 1) % 2
|
|
824
|
+
) == 0: # scans_to_average = 3, 5, 7 etc, mirror l/r around apex
|
|
825
|
+
apex_scans = list(set(scan_list))
|
|
826
|
+
# Check if all apex scans are profile mode, raise error if not
|
|
827
|
+
if not all(self.scan_df.loc[apex_scans, "ms_format"] == "profile"):
|
|
828
|
+
raise ValueError("All apex scans must be profile mode for averaging")
|
|
829
|
+
|
|
830
|
+
# First get sets of scans to average
|
|
831
|
+
def get_scans_from_apex(ms1_scans, apex_scan, scans_to_average):
|
|
832
|
+
ms1_idx_start = ms1_scans.index(apex_scan) - int(
|
|
833
|
+
(scans_to_average - 1) / 2
|
|
834
|
+
)
|
|
835
|
+
if ms1_idx_start < 0:
|
|
836
|
+
ms1_idx_start = 0
|
|
837
|
+
ms1_idx_end = (
|
|
838
|
+
ms1_scans.index(apex_scan) + int((scans_to_average - 1) / 2) + 1
|
|
839
|
+
)
|
|
840
|
+
if ms1_idx_end > (len(ms1_scans) - 1):
|
|
841
|
+
ms1_idx_end = len(ms1_scans) - 1
|
|
842
|
+
scan_list = ms1_scans[ms1_idx_start:ms1_idx_end]
|
|
843
|
+
return scan_list
|
|
844
|
+
|
|
845
|
+
ms1_scans = self.ms1_scans
|
|
846
|
+
scans_lists = [
|
|
847
|
+
get_scans_from_apex(ms1_scans, apex_scan, scans_to_average)
|
|
848
|
+
for apex_scan in apex_scans
|
|
849
|
+
]
|
|
850
|
+
|
|
851
|
+
# set polarity to -1 if negative mode, 1 if positive mode (for mass spectrum creation)
|
|
852
|
+
if self.polarity == "negative":
|
|
853
|
+
polarity = -1
|
|
854
|
+
elif self.polarity == "positive":
|
|
855
|
+
polarity = 1
|
|
856
|
+
|
|
857
|
+
if not use_parser:
|
|
858
|
+
# Perform checks and prepare _ms_unprocessed dictionary if use_parser is False (saves time to do this once)
|
|
859
|
+
ms1_unprocessed = self._ms_unprocessed[1].copy()
|
|
860
|
+
# Set the index on _ms_unprocessed[1] to scan number
|
|
861
|
+
ms1_unprocessed = ms1_unprocessed.set_index("scan", drop=False)
|
|
862
|
+
self._ms_unprocessed[1] = ms1_unprocessed
|
|
863
|
+
|
|
864
|
+
# Check that all the scans in scan_lists are indexs in self._ms_unprocessed[1]
|
|
865
|
+
scans_lists_flat = list(
|
|
866
|
+
set([scan for sublist in scans_lists for scan in sublist])
|
|
867
|
+
)
|
|
868
|
+
if (
|
|
869
|
+
len(
|
|
870
|
+
np.setdiff1d(
|
|
871
|
+
np.sort(scans_lists_flat),
|
|
872
|
+
np.sort(ms1_unprocessed.index.values),
|
|
873
|
+
)
|
|
874
|
+
)
|
|
875
|
+
> 0
|
|
876
|
+
):
|
|
877
|
+
raise ValueError(
|
|
878
|
+
"Not all scans to average are present in the unprocessed data"
|
|
879
|
+
)
|
|
880
|
+
|
|
881
|
+
for scan_list_average, apex_scan in zip(scans_lists, apex_scans):
|
|
882
|
+
# Get unprocessed mass spectrum from scans
|
|
883
|
+
ms = self.get_average_mass_spectrum(
|
|
884
|
+
scan_list=scan_list_average,
|
|
885
|
+
apex_scan=apex_scan,
|
|
886
|
+
spectrum_mode="profile",
|
|
887
|
+
ms_level=1,
|
|
888
|
+
auto_process=auto_process,
|
|
889
|
+
use_parser=use_parser,
|
|
890
|
+
perform_checks=False,
|
|
891
|
+
polarity=polarity,
|
|
892
|
+
ms_params=self.parameters.mass_spectrum["ms1"],
|
|
893
|
+
)
|
|
894
|
+
# Add mass spectrum to LCMS object and associated with mass feature
|
|
895
|
+
self.add_mass_spectrum(ms)
|
|
896
|
+
|
|
897
|
+
if not use_parser:
|
|
898
|
+
# Reset the index on _ms_unprocessed[1] to not be scan number
|
|
899
|
+
ms1_unprocessed = ms1_unprocessed.reset_index(drop=True)
|
|
900
|
+
self._ms_unprocessed[1] = ms1_unprocessed
|
|
901
|
+
else:
|
|
902
|
+
raise ValueError(
|
|
903
|
+
"Number of scans to average must be 1 or an integer with an integer median (i.e. 3, 5, 7, 9)"
|
|
904
|
+
)
|
|
905
|
+
|
|
906
|
+
# Associate the ms1 spectra with the mass features
|
|
907
|
+
for k in mf_dict.keys():
|
|
908
|
+
## another induced feature work around
|
|
909
|
+
if mf_dict[k].apex_scan != -99:
|
|
910
|
+
mf_dict[k].mass_spectrum = self._ms[
|
|
911
|
+
mf_dict[k].apex_scan
|
|
912
|
+
]
|
|
913
|
+
mf_dict[k].update_mz()
|
|
914
|
+
|
|
915
|
+
def mass_features_to_df(self, induced_features=False, drop_na_cols=False, include_cols=None):
|
|
916
|
+
"""Returns a pandas dataframe summarizing the mass features.
|
|
917
|
+
|
|
918
|
+
The dataframe contains the following columns: mf_id, mz, apex_scan, scan_time, intensity,
|
|
919
|
+
persistence, area, monoisotopic_mf_id, and isotopologue_type. The index is set to mf_id (mass feature ID).
|
|
920
|
+
Parameters
|
|
921
|
+
-----------
|
|
922
|
+
induced_features : bool, optional
|
|
923
|
+
If True, calls the induced_mass_features dictionary. Defaults to False.
|
|
924
|
+
drop_na_cols : bool, optional
|
|
925
|
+
If True, drops columns that are entirely NA. Defaults to False.
|
|
926
|
+
include_cols : list of str, optional
|
|
927
|
+
If provided, only includes the specified columns in the output (in addition to 'mf_id' which is always included as the index).
|
|
928
|
+
If None, includes all available columns. Defaults to None.
|
|
929
|
+
|
|
930
|
+
Raises
|
|
931
|
+
--------
|
|
932
|
+
ValueError
|
|
933
|
+
If the sample provided doesn't contain the mass feature data.
|
|
934
|
+
|
|
935
|
+
Returns
|
|
936
|
+
--------
|
|
937
|
+
pandas.DataFrame
|
|
938
|
+
A pandas dataframe of mass features with the following columns:
|
|
939
|
+
mf_id, mz, apex_scan, scan_time, intensity, persistence, area.
|
|
940
|
+
"""
|
|
941
|
+
import pandas as pd
|
|
942
|
+
|
|
943
|
+
def mass_spectrum_to_string(
|
|
944
|
+
mass_spec, normalize=True, min_normalized_abun=0.01
|
|
945
|
+
):
|
|
946
|
+
"""Converts a mass spectrum to a string of m/z:abundance pairs.
|
|
947
|
+
|
|
948
|
+
Parameters
|
|
949
|
+
-----------
|
|
950
|
+
mass_spec : MassSpectrum
|
|
951
|
+
A MassSpectrum object to be converted to a string.
|
|
952
|
+
normalize : bool, optional
|
|
953
|
+
If True, normalizes the abundance values to a maximum of 1. Defaults to True.
|
|
954
|
+
min_normalized_abun : float, optional
|
|
955
|
+
The minimum normalized abundance value to include in the string, only used if normalize is True. Defaults to 0.01.
|
|
956
|
+
|
|
957
|
+
Returns
|
|
958
|
+
--------
|
|
959
|
+
str
|
|
960
|
+
A string of m/z:abundance pairs from the mass spectrum, separated by a semicolon.
|
|
961
|
+
"""
|
|
962
|
+
mz_np = mass_spec.to_dataframe()["m/z"].values
|
|
963
|
+
abun_np = mass_spec.to_dataframe()["Peak Height"].values
|
|
964
|
+
if normalize:
|
|
965
|
+
abun_np = abun_np / abun_np.max()
|
|
966
|
+
mz_abun = np.column_stack((mz_np, abun_np))
|
|
967
|
+
if normalize:
|
|
968
|
+
mz_abun = mz_abun[mz_abun[:, 1] > min_normalized_abun]
|
|
969
|
+
mz_abun_str = [
|
|
970
|
+
str(round(mz, ndigits=4)) + ":" + str(round(abun, ndigits=2))
|
|
971
|
+
for mz, abun in mz_abun
|
|
972
|
+
]
|
|
973
|
+
return "; ".join(mz_abun_str)
|
|
974
|
+
|
|
975
|
+
if induced_features:
|
|
976
|
+
mf_dict = self.induced_mass_features
|
|
977
|
+
else:
|
|
978
|
+
mf_dict = self.mass_features
|
|
979
|
+
|
|
980
|
+
if len(mf_dict) == 0:
|
|
981
|
+
# Return an empty dataframe with the expected structure
|
|
982
|
+
# This allows collection processing to continue even if some samples have no features
|
|
983
|
+
return pd.DataFrame()
|
|
984
|
+
|
|
985
|
+
cols_in_df = [
|
|
986
|
+
"id",
|
|
987
|
+
"apex_scan",
|
|
988
|
+
"start_scan",
|
|
989
|
+
"final_scan",
|
|
990
|
+
"retention_time",
|
|
991
|
+
"intensity",
|
|
992
|
+
"persistence",
|
|
993
|
+
"area",
|
|
994
|
+
"dispersity_index",
|
|
995
|
+
"normalized_dispersity_index",
|
|
996
|
+
"tailing_factor",
|
|
997
|
+
"gaussian_similarity",
|
|
998
|
+
"noise_score",
|
|
999
|
+
"noise_score_min",
|
|
1000
|
+
"noise_score_max",
|
|
1001
|
+
"monoisotopic_mf_id",
|
|
1002
|
+
"isotopologue_type",
|
|
1003
|
+
"mass_spectrum_deconvoluted_parent",
|
|
1004
|
+
"ms2_scan_numbers",
|
|
1005
|
+
"type"
|
|
1006
|
+
]
|
|
1007
|
+
|
|
1008
|
+
df_mf_list = []
|
|
1009
|
+
for mf_id in mf_dict.keys():
|
|
1010
|
+
# Find cols_in_df that are in single_mf
|
|
1011
|
+
df_keys = list(
|
|
1012
|
+
set(cols_in_df).intersection(mf_dict[mf_id].__dir__())
|
|
1013
|
+
)
|
|
1014
|
+
dict_mf = {}
|
|
1015
|
+
# Get the values for each key in df_keys from the mass feature object
|
|
1016
|
+
for key in df_keys:
|
|
1017
|
+
value = getattr(mf_dict[mf_id], key)
|
|
1018
|
+
# Wrap list/array values in a list so pandas treats them as single cell values
|
|
1019
|
+
if key == 'ms2_scan_numbers' and isinstance(value, (list, np.ndarray)):
|
|
1020
|
+
dict_mf[key] = [value]
|
|
1021
|
+
else:
|
|
1022
|
+
dict_mf[key] = value
|
|
1023
|
+
if len(mf_dict[mf_id].ms2_scan_numbers) > 0:
|
|
1024
|
+
# Add MS2 spectra info
|
|
1025
|
+
best_ms2_spectrum = mf_dict[mf_id].best_ms2
|
|
1026
|
+
if best_ms2_spectrum is not None:
|
|
1027
|
+
dict_mf["ms2_spectrum"] = mass_spectrum_to_string(best_ms2_spectrum)
|
|
1028
|
+
if len(mf_dict[mf_id].associated_mass_features_deconvoluted) > 0:
|
|
1029
|
+
dict_mf["associated_mass_features"] = ", ".join(
|
|
1030
|
+
map(
|
|
1031
|
+
str,
|
|
1032
|
+
mf_dict[mf_id].associated_mass_features_deconvoluted,
|
|
1033
|
+
)
|
|
1034
|
+
)
|
|
1035
|
+
if mf_dict[mf_id]._half_height_width is not None:
|
|
1036
|
+
dict_mf["half_height_width"] = mf_dict[
|
|
1037
|
+
mf_id
|
|
1038
|
+
].half_height_width
|
|
1039
|
+
# Check if EIC for mass feature is set
|
|
1040
|
+
df_mf_single = pd.DataFrame(dict_mf, index=[mf_id])
|
|
1041
|
+
df_mf_single["mz"] = mf_dict[mf_id].mz
|
|
1042
|
+
df_mf_list.append(df_mf_single)
|
|
1043
|
+
df_mf = pd.concat(df_mf_list)
|
|
1044
|
+
|
|
1045
|
+
# rename _area to area and id to mf_id
|
|
1046
|
+
df_mf = df_mf.rename(
|
|
1047
|
+
columns={
|
|
1048
|
+
"id": "mf_id",
|
|
1049
|
+
"retention_time": "scan_time",
|
|
1050
|
+
}
|
|
1051
|
+
)
|
|
1052
|
+
|
|
1053
|
+
# reorder columns
|
|
1054
|
+
col_order = [
|
|
1055
|
+
"mf_id",
|
|
1056
|
+
"type",
|
|
1057
|
+
"scan_time",
|
|
1058
|
+
"mz",
|
|
1059
|
+
"apex_scan",
|
|
1060
|
+
"start_scan",
|
|
1061
|
+
"final_scan",
|
|
1062
|
+
"intensity",
|
|
1063
|
+
"persistence",
|
|
1064
|
+
"area",
|
|
1065
|
+
"half_height_width",
|
|
1066
|
+
"tailing_factor",
|
|
1067
|
+
"dispersity_index",
|
|
1068
|
+
"normalized_dispersity_index",
|
|
1069
|
+
"gaussian_similarity",
|
|
1070
|
+
"noise_score",
|
|
1071
|
+
"noise_score_min",
|
|
1072
|
+
"noise_score_max",
|
|
1073
|
+
"monoisotopic_mf_id",
|
|
1074
|
+
"isotopologue_type",
|
|
1075
|
+
"mass_spectrum_deconvoluted_parent",
|
|
1076
|
+
"associated_mass_features",
|
|
1077
|
+
"ms2_scan_numbers",
|
|
1078
|
+
"ms2_spectrum",
|
|
1079
|
+
]
|
|
1080
|
+
# drop columns that are not in col_order
|
|
1081
|
+
cols_to_order = [col for col in col_order if col in df_mf.columns]
|
|
1082
|
+
df_mf = df_mf[cols_to_order]
|
|
1083
|
+
|
|
1084
|
+
# reset index to mf_id
|
|
1085
|
+
df_mf = df_mf.set_index("mf_id")
|
|
1086
|
+
df_mf.index.name = "mf_id"
|
|
1087
|
+
|
|
1088
|
+
if 'half_height_width' in df_mf.columns:
|
|
1089
|
+
df_mf["half_height_width"] = df_mf["half_height_width"].astype('float64')
|
|
1090
|
+
if 'tailing_factor' in df_mf.columns:
|
|
1091
|
+
df_mf["tailing_factor"] = df_mf["tailing_factor"].astype('float64')
|
|
1092
|
+
if 'dispersity_index' in df_mf.columns:
|
|
1093
|
+
df_mf["dispersity_index"] = df_mf["dispersity_index"].astype('float64')
|
|
1094
|
+
if 'normalized_dispersity_index' in df_mf.columns:
|
|
1095
|
+
df_mf["normalized_dispersity_index"] = df_mf["normalized_dispersity_index"].astype('float64')
|
|
1096
|
+
|
|
1097
|
+
# Filter columns if include_cols is specified
|
|
1098
|
+
if include_cols is not None:
|
|
1099
|
+
# Ensure include_cols is a list
|
|
1100
|
+
if not isinstance(include_cols, list):
|
|
1101
|
+
raise ValueError("include_cols must be a list of column names")
|
|
1102
|
+
# Keep only requested columns that exist in the dataframe
|
|
1103
|
+
available_cols = [col for col in include_cols if col in df_mf.columns]
|
|
1104
|
+
df_mf = df_mf[available_cols]
|
|
1105
|
+
|
|
1106
|
+
# Drop columns that are entirely NA if requested
|
|
1107
|
+
if drop_na_cols:
|
|
1108
|
+
df_mf = df_mf.dropna(axis=1, how='all')
|
|
1109
|
+
|
|
1110
|
+
return df_mf
|
|
1111
|
+
|
|
1112
|
+
def mass_features_ms1_annot_to_df(self, suppress_warnings=False):
|
|
1113
|
+
"""Returns a pandas dataframe summarizing the MS1 annotations for the mass features in the dataset.
|
|
1114
|
+
|
|
1115
|
+
Parameters
|
|
1116
|
+
-----------
|
|
1117
|
+
suppress_warnings : bool, optional
|
|
1118
|
+
If True, suppresses the warning when no MS1 annotations are found.
|
|
1119
|
+
Useful when calling from collection-level methods. Default is False.
|
|
1120
|
+
|
|
1121
|
+
Returns
|
|
1122
|
+
--------
|
|
1123
|
+
pandas.DataFrame
|
|
1124
|
+
A pandas dataframe of MS1 annotations for the mass features in the dataset.
|
|
1125
|
+
The index is set to mf_id (mass feature ID)
|
|
1126
|
+
|
|
1127
|
+
Raises
|
|
1128
|
+
------
|
|
1129
|
+
Warning
|
|
1130
|
+
If no MS1 annotations were found for the mass features in the dataset
|
|
1131
|
+
(unless suppress_warnings=True).
|
|
1132
|
+
"""
|
|
1133
|
+
annot_df_list_ms1 = []
|
|
1134
|
+
for mf_id in self.mass_features.keys():
|
|
1135
|
+
if self.mass_features[mf_id].mass_spectrum is None:
|
|
1136
|
+
pass
|
|
1137
|
+
else:
|
|
1138
|
+
# Add ms1 annotations to ms1 annotation list
|
|
1139
|
+
if (
|
|
1140
|
+
np.abs(
|
|
1141
|
+
(
|
|
1142
|
+
self.mass_features[mf_id].ms1_peak.mz_exp
|
|
1143
|
+
- self.mass_features[mf_id].mz
|
|
1144
|
+
)
|
|
1145
|
+
)
|
|
1146
|
+
< 0.01
|
|
1147
|
+
):
|
|
1148
|
+
# Get the molecular formula from the mass spectrum
|
|
1149
|
+
annot_df = self.mass_features[mf_id].mass_spectrum.to_dataframe()
|
|
1150
|
+
# Subset to pull out only the peak associated with the mass feature
|
|
1151
|
+
annot_df = annot_df[
|
|
1152
|
+
annot_df["Index"] == self.mass_features[mf_id].ms1_peak.index
|
|
1153
|
+
].copy()
|
|
1154
|
+
|
|
1155
|
+
# If there are more than 1 row, remove any rows without a molecular formula
|
|
1156
|
+
if len(annot_df) > 1:
|
|
1157
|
+
annot_df = annot_df[~annot_df["Molecular Formula"].isna()]
|
|
1158
|
+
|
|
1159
|
+
# Remove the index column and add column for mf_id
|
|
1160
|
+
annot_df = annot_df.drop(columns=["Index"])
|
|
1161
|
+
annot_df["mf_id"] = mf_id
|
|
1162
|
+
annot_df_list_ms1.append(annot_df)
|
|
1163
|
+
|
|
1164
|
+
if len(annot_df_list_ms1) > 0:
|
|
1165
|
+
annot_ms1_df_full = pd.concat(annot_df_list_ms1)
|
|
1166
|
+
annot_ms1_df_full = annot_ms1_df_full.set_index("mf_id")
|
|
1167
|
+
annot_ms1_df_full.index.name = "mf_id"
|
|
1168
|
+
|
|
1169
|
+
else:
|
|
1170
|
+
annot_ms1_df_full = None
|
|
1171
|
+
# Warn that no ms1 annotations were found (unless suppressed)
|
|
1172
|
+
if not suppress_warnings:
|
|
1173
|
+
warnings.warn(
|
|
1174
|
+
"No MS1 annotations found for mass features in dataset, were MS1 spectra added and processed within the dataset?",
|
|
1175
|
+
UserWarning,
|
|
1176
|
+
)
|
|
1177
|
+
|
|
1178
|
+
return annot_ms1_df_full
|
|
1179
|
+
|
|
1180
|
+
def mass_features_ms2_annot_to_df(self, molecular_metadata=None, suppress_warnings=False):
|
|
1181
|
+
"""Returns a pandas dataframe summarizing the MS2 annotations for the mass features in the dataset.
|
|
1182
|
+
|
|
1183
|
+
Parameters
|
|
1184
|
+
-----------
|
|
1185
|
+
molecular_metadata : dict of MolecularMetadata objects
|
|
1186
|
+
A dictionary of MolecularMetadata objects, keyed by ref_mol_id. Defaults to None.
|
|
1187
|
+
suppress_warnings : bool, optional
|
|
1188
|
+
If True, suppresses the warning when no MS2 annotations are found.
|
|
1189
|
+
Useful when calling from collection-level methods. Default is False.
|
|
1190
|
+
|
|
1191
|
+
Returns
|
|
1192
|
+
--------
|
|
1193
|
+
pandas.DataFrame
|
|
1194
|
+
A pandas dataframe of MS2 annotations for the mass features in the dataset,
|
|
1195
|
+
and optionally molecular metadata. The index is set to mf_id (mass feature ID)
|
|
1196
|
+
|
|
1197
|
+
Raises
|
|
1198
|
+
------
|
|
1199
|
+
Warning
|
|
1200
|
+
If no MS2 annotations were found for the mass features in the dataset
|
|
1201
|
+
(unless suppress_warnings=True).
|
|
1202
|
+
"""
|
|
1203
|
+
annot_df_list_ms2 = []
|
|
1204
|
+
for mf_id in self.mass_features.keys():
|
|
1205
|
+
if len(self.mass_features[mf_id].ms2_similarity_results) > 0:
|
|
1206
|
+
# Add ms2 annotations to ms2 annotation list
|
|
1207
|
+
for result in self.mass_features[mf_id].ms2_similarity_results:
|
|
1208
|
+
annot_df_ms2 = result.to_dataframe()
|
|
1209
|
+
annot_df_ms2["mf_id"] = mf_id
|
|
1210
|
+
annot_df_list_ms2.append(annot_df_ms2)
|
|
1211
|
+
|
|
1212
|
+
if len(annot_df_list_ms2) > 0:
|
|
1213
|
+
annot_ms2_df_full = pd.concat(annot_df_list_ms2)
|
|
1214
|
+
if molecular_metadata is not None:
|
|
1215
|
+
molecular_metadata_df = pd.concat(
|
|
1216
|
+
[
|
|
1217
|
+
pd.DataFrame.from_dict(v.__dict__, orient="index").transpose()
|
|
1218
|
+
for k, v in molecular_metadata.items()
|
|
1219
|
+
],
|
|
1220
|
+
ignore_index=True,
|
|
1221
|
+
)
|
|
1222
|
+
molecular_metadata_df = molecular_metadata_df.rename(
|
|
1223
|
+
columns={"id": "ref_mol_id"}
|
|
1224
|
+
)
|
|
1225
|
+
annot_ms2_df_full = annot_ms2_df_full.merge(
|
|
1226
|
+
molecular_metadata_df, on="ref_mol_id", how="left"
|
|
1227
|
+
)
|
|
1228
|
+
annot_ms2_df_full = annot_ms2_df_full.drop_duplicates(
|
|
1229
|
+
subset=["mf_id", "query_spectrum_id", "ref_ms_id"]
|
|
1230
|
+
).copy()
|
|
1231
|
+
annot_ms2_df_full = annot_ms2_df_full.set_index("mf_id")
|
|
1232
|
+
annot_ms2_df_full.index.name = "mf_id"
|
|
1233
|
+
else:
|
|
1234
|
+
annot_ms2_df_full = None
|
|
1235
|
+
# Warn that no ms2 annotations were found (unless suppressed)
|
|
1236
|
+
if not suppress_warnings:
|
|
1237
|
+
warnings.warn(
|
|
1238
|
+
"No MS2 annotations found for mass features in dataset, were MS2 spectra added and searched against a database?",
|
|
1239
|
+
UserWarning,
|
|
1240
|
+
)
|
|
1241
|
+
|
|
1242
|
+
return annot_ms2_df_full
|
|
1243
|
+
|
|
1244
|
+
def plot_composite_mz_features(self, binsize = 1e-4, ph_int_min_thresh = 0.001, mf_plot = True, ms2_plot = True, return_fig = False):
|
|
1245
|
+
"""Returns a figure displaying
|
|
1246
|
+
(1) thresholded, unprocessed data
|
|
1247
|
+
(2) the m/z features
|
|
1248
|
+
(3) which m/z features are associated with MS2 spectra
|
|
1249
|
+
|
|
1250
|
+
Parameters
|
|
1251
|
+
-----------
|
|
1252
|
+
binsize : float
|
|
1253
|
+
Desired binsize for the m/z axis of the composite feature map. Defaults to 1e-4.
|
|
1254
|
+
mf_plot : boolean
|
|
1255
|
+
Indicates whether to plot the m/z features. Defaults to True.
|
|
1256
|
+
ms2_plot : boolean
|
|
1257
|
+
Indicates whether to identify m/z features with associated MS2 spectra. Defaults to True.
|
|
1258
|
+
return_fig : boolean
|
|
1259
|
+
Indicates whether to plot composite feature map (False) or return figure object (True). Defaults to False.
|
|
1260
|
+
|
|
1261
|
+
Returns
|
|
1262
|
+
--------
|
|
1263
|
+
matplotlib.pyplot.Figure
|
|
1264
|
+
A figure with the thresholded, unprocessed data on an axis of m/z value with respect to
|
|
1265
|
+
scan time. Unprocessed data is displayed in gray scale with darker colors indicating
|
|
1266
|
+
higher intensities. If m/z features are plotted, they are displayed in cyan. If m/z
|
|
1267
|
+
features with associated with MS2 spectra are plotted, they are displayed in red.
|
|
1268
|
+
|
|
1269
|
+
Raises
|
|
1270
|
+
------
|
|
1271
|
+
Warning
|
|
1272
|
+
If m/z features are set to be plot but aren't in the dataset.
|
|
1273
|
+
If m/z features with associated MS2 data are set to be plot but no MS2 annotations
|
|
1274
|
+
were found for the m/z features in the dataset.
|
|
1275
|
+
"""
|
|
1276
|
+
if mf_plot:
|
|
1277
|
+
# Check if mass_features is set, raise error if not
|
|
1278
|
+
if self.mass_features is None:
|
|
1279
|
+
raise ValueError(
|
|
1280
|
+
"mass_features not set, must run find_mass_features() first"
|
|
1281
|
+
)
|
|
1282
|
+
## call mass feature data
|
|
1283
|
+
mf_df = self.mass_features_to_df()
|
|
1284
|
+
|
|
1285
|
+
if ms2_plot:
|
|
1286
|
+
if not mf_plot:
|
|
1287
|
+
# Check if mass_features is set, raise error if not
|
|
1288
|
+
if self.mass_features is None:
|
|
1289
|
+
raise ValueError(
|
|
1290
|
+
"mass_features not set, must run find_mass_features() first"
|
|
1291
|
+
)
|
|
1292
|
+
|
|
1293
|
+
## call m/z feature data
|
|
1294
|
+
mf_df = self.mass_features_to_df()
|
|
1295
|
+
|
|
1296
|
+
# Check if ms2_spectrum is set, raise error if not
|
|
1297
|
+
if 'ms2_spectrum' not in mf_df.columns:
|
|
1298
|
+
raise ValueError(
|
|
1299
|
+
"ms2_spectrum not set, must run add_associated_ms2_dda() first"
|
|
1300
|
+
)
|
|
1301
|
+
|
|
1302
|
+
## threshold and grid unprocessed data
|
|
1303
|
+
df = self._ms_unprocessed[1].copy()
|
|
1304
|
+
df = df.dropna(subset=['intensity']).reset_index(drop = True)
|
|
1305
|
+
threshold = ph_int_min_thresh * df.intensity.max()
|
|
1306
|
+
df_thres = df[df["intensity"] > threshold].reset_index(drop = True).copy()
|
|
1307
|
+
df = self.grid_data(df_thres)
|
|
1308
|
+
|
|
1309
|
+
## format unprocessed data for plotting
|
|
1310
|
+
df = df.merge(self.scan_df[['scan', 'scan_time']], on = 'scan')
|
|
1311
|
+
mz_grid = np.arange(0, np.max(df.mz), binsize)
|
|
1312
|
+
mz_data = np.array(df.mz)
|
|
1313
|
+
df['mz_bin'] = find_closest(mz_grid, mz_data)
|
|
1314
|
+
df['ab_bin'] = df.groupby(['mz_bin', 'scan_time']).intensity.transform(sum)
|
|
1315
|
+
unproc_df = df[['scan_time', 'mz_bin', 'ab_bin']].drop_duplicates(ignore_index = True)
|
|
1316
|
+
|
|
1317
|
+
## generate figure
|
|
1318
|
+
fig = plt.figure()
|
|
1319
|
+
plt.scatter(
|
|
1320
|
+
unproc_df.scan_time,
|
|
1321
|
+
unproc_df.mz_bin*binsize,
|
|
1322
|
+
c = unproc_df.ab_bin/np.max(unproc_df.ab_bin),
|
|
1323
|
+
alpha = unproc_df.ab_bin/np.max(unproc_df.ab_bin),
|
|
1324
|
+
cmap = 'Greys_r',
|
|
1325
|
+
s = 1
|
|
1326
|
+
)
|
|
1327
|
+
|
|
1328
|
+
if mf_plot:
|
|
1329
|
+
if ms2_plot:
|
|
1330
|
+
plt.scatter(
|
|
1331
|
+
mf_df[mf_df.ms2_spectrum.isna()].scan_time,
|
|
1332
|
+
mf_df[mf_df.ms2_spectrum.isna()].mz,
|
|
1333
|
+
c = 'c',
|
|
1334
|
+
s = 4,
|
|
1335
|
+
label = 'M/Z features without MS2'
|
|
1336
|
+
)
|
|
1337
|
+
else:
|
|
1338
|
+
plt.scatter(
|
|
1339
|
+
mf_df.scan_time,
|
|
1340
|
+
mf_df.mz,
|
|
1341
|
+
c = 'c',
|
|
1342
|
+
s = 4,
|
|
1343
|
+
label = 'M/Z features'
|
|
1344
|
+
)
|
|
1345
|
+
|
|
1346
|
+
if ms2_plot:
|
|
1347
|
+
plt.scatter(
|
|
1348
|
+
mf_df[~mf_df.ms2_spectrum.isna()].scan_time,
|
|
1349
|
+
mf_df[~mf_df.ms2_spectrum.isna()].mz,
|
|
1350
|
+
c = 'r',
|
|
1351
|
+
s = 2,
|
|
1352
|
+
label = 'M/Z features with MS2'
|
|
1353
|
+
)
|
|
1354
|
+
|
|
1355
|
+
if mf_plot == True or ms2_plot == True:
|
|
1356
|
+
plt.legend(loc = 'lower center', bbox_to_anchor = (0.5, -0.25), ncol = 2)
|
|
1357
|
+
plt.xlabel('Scan time')
|
|
1358
|
+
plt.ylabel('m/z')
|
|
1359
|
+
plt.ylim(0, np.ceil(np.max(df.mz)))
|
|
1360
|
+
plt.xlim(0, np.ceil(np.max(df.scan_time)))
|
|
1361
|
+
plt.title('Composite Feature Map')
|
|
1362
|
+
|
|
1363
|
+
if return_fig:
|
|
1364
|
+
plt.close(fig)
|
|
1365
|
+
return fig
|
|
1366
|
+
|
|
1367
|
+
else:
|
|
1368
|
+
plt.show()
|
|
1369
|
+
|
|
1370
|
+
def search_for_targeted_mass_features_batch(
|
|
1371
|
+
self,
|
|
1372
|
+
ms1df,
|
|
1373
|
+
mz_mins,
|
|
1374
|
+
mz_maxs,
|
|
1375
|
+
st_mins,
|
|
1376
|
+
st_maxs,
|
|
1377
|
+
set_ids,
|
|
1378
|
+
obj_idx=0,
|
|
1379
|
+
st_aligned=False
|
|
1380
|
+
):
|
|
1381
|
+
"""
|
|
1382
|
+
Returns multiple LCMSMassFeatures from a specific sample within specific mass and time ranges.
|
|
1383
|
+
Vectorized batch version of search_for_targeted_mass_feature for improved performance.
|
|
1384
|
+
|
|
1385
|
+
Parameters
|
|
1386
|
+
-----------
|
|
1387
|
+
ms1df : pd.DataFrame
|
|
1388
|
+
Dataframe containing all the possible MS1 values to consider, collected by calling _ms_unprocessed[1] on the sample.
|
|
1389
|
+
mz_mins : np.ndarray
|
|
1390
|
+
Array of lower bounds of m/z values to use to find peaks.
|
|
1391
|
+
mz_maxs : np.ndarray
|
|
1392
|
+
Array of upper bounds of m/z values to use to find peaks.
|
|
1393
|
+
st_mins : np.ndarray
|
|
1394
|
+
Array of lower bounds of scan times to use to find peaks.
|
|
1395
|
+
st_maxs : np.ndarray
|
|
1396
|
+
Array of upper bounds of scan times to use to find peaks.
|
|
1397
|
+
set_ids : np.ndarray or list
|
|
1398
|
+
Array of strings used as IDs in LCMSMassFeatures.
|
|
1399
|
+
obj_idx : int
|
|
1400
|
+
Identifies index of sample in a collection. Defaults to 0.
|
|
1401
|
+
st_aligned : bool
|
|
1402
|
+
Whether to use scan_time_aligned or scan_time. Defaults to False.
|
|
1403
|
+
|
|
1404
|
+
Returns
|
|
1405
|
+
--------
|
|
1406
|
+
dict
|
|
1407
|
+
Dictionary mapping set_id to LCMSMassFeature objects.
|
|
1408
|
+
|
|
1409
|
+
Raises
|
|
1410
|
+
------
|
|
1411
|
+
ValueError
|
|
1412
|
+
If appropriate scan time data is not contained in ms1df or if array lengths don't match.
|
|
1413
|
+
"""
|
|
1414
|
+
# Validate inputs
|
|
1415
|
+
n_features = len(mz_mins)
|
|
1416
|
+
if not all(len(arr) == n_features for arr in [mz_maxs, st_mins, st_maxs, set_ids]):
|
|
1417
|
+
raise ValueError("All input arrays must have the same length")
|
|
1418
|
+
|
|
1419
|
+
# Validate scan time column
|
|
1420
|
+
time_col = 'scan_time_aligned' if st_aligned else 'scan_time'
|
|
1421
|
+
if time_col not in ms1df.columns:
|
|
1422
|
+
raise ValueError(f"{time_col} not contained in ms1df")
|
|
1423
|
+
|
|
1424
|
+
# Pre-extract columns for faster access
|
|
1425
|
+
mz_vals = ms1df.mz.values
|
|
1426
|
+
st_vals = ms1df[time_col].values
|
|
1427
|
+
scan_vals = ms1df.scan.values
|
|
1428
|
+
intensity_vals = ms1df.intensity.values
|
|
1429
|
+
|
|
1430
|
+
# Process all features
|
|
1431
|
+
results = {}
|
|
1432
|
+
for i in range(n_features):
|
|
1433
|
+
# Vectorized filtering
|
|
1434
|
+
mask = (
|
|
1435
|
+
(mz_vals >= mz_mins[i]) & (mz_vals <= mz_maxs[i]) &
|
|
1436
|
+
(st_vals >= st_mins[i]) & (st_vals <= st_maxs[i])
|
|
1437
|
+
)
|
|
1438
|
+
|
|
1439
|
+
if not mask.any():
|
|
1440
|
+
row_dict = {
|
|
1441
|
+
'apex_scan': -99,
|
|
1442
|
+
'mz': np.nan,
|
|
1443
|
+
'intensity': np.nan,
|
|
1444
|
+
'retention_time': np.nan,
|
|
1445
|
+
'persistence': np.nan,
|
|
1446
|
+
'id': set_ids[i]
|
|
1447
|
+
}
|
|
1448
|
+
else:
|
|
1449
|
+
# Find max intensity within filtered region
|
|
1450
|
+
filtered_intensities = intensity_vals[mask]
|
|
1451
|
+
max_idx = np.argmax(filtered_intensities)
|
|
1452
|
+
|
|
1453
|
+
# Get indices of filtered data
|
|
1454
|
+
filtered_indices = np.where(mask)[0]
|
|
1455
|
+
peak_idx = filtered_indices[max_idx]
|
|
1456
|
+
|
|
1457
|
+
row_dict = {
|
|
1458
|
+
'apex_scan': scan_vals[peak_idx],
|
|
1459
|
+
'mz': mz_vals[peak_idx],
|
|
1460
|
+
'intensity': intensity_vals[peak_idx],
|
|
1461
|
+
'retention_time': st_vals[peak_idx],
|
|
1462
|
+
'persistence': np.nan,
|
|
1463
|
+
'id': set_ids[i]
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
results[set_ids[i]] = LCMSMassFeature(self, **row_dict)
|
|
1467
|
+
|
|
1468
|
+
return results
|
|
1469
|
+
|
|
1470
|
+
def search_for_targeted_mass_feature(
|
|
1471
|
+
self,
|
|
1472
|
+
ms1df,
|
|
1473
|
+
mz_min,
|
|
1474
|
+
mz_max,
|
|
1475
|
+
st_min,
|
|
1476
|
+
st_max,
|
|
1477
|
+
set_id,
|
|
1478
|
+
obj_idx = 0,
|
|
1479
|
+
st_aligned = False
|
|
1480
|
+
):
|
|
1481
|
+
"""
|
|
1482
|
+
Returns an LCMSMassFeature from a specific sample within a specific mass and time range. Returns an empty
|
|
1483
|
+
LCMSMassFeature if no satisfactory peak is found in the given window.
|
|
1484
|
+
|
|
1485
|
+
Parameters
|
|
1486
|
+
-----------
|
|
1487
|
+
ms1df : Pandas DataFrame
|
|
1488
|
+
Dataframe containing all the possible MS1 values to consider, collected by calling _ms_unprocessed[1] on the sample.
|
|
1489
|
+
mz_min : float
|
|
1490
|
+
Identifies lower bound of the weights to use to find a peak.
|
|
1491
|
+
mz_max : float
|
|
1492
|
+
Identifies upper bound of the weights to use to find a peak.
|
|
1493
|
+
st_min : float
|
|
1494
|
+
Identifies lower bound of the scan times to use to find a peak.
|
|
1495
|
+
st_max : float
|
|
1496
|
+
Identifies upper bound of the scan times to use to find a peak.
|
|
1497
|
+
set_id : str
|
|
1498
|
+
Indicates string used as ID in LCMSMassFeature.
|
|
1499
|
+
obj_idx : int
|
|
1500
|
+
Identifies index of sample in a collection that LCMSMassFeature should be assigned to. Defaults to 0 and is not used
|
|
1501
|
+
if data provided is an LCMSBase instead of an LCMSCollection.
|
|
1502
|
+
st_aligned : boolean
|
|
1503
|
+
Indicates whether to call scan time from scan_time or from scan_time_aligned if using a collection. Defaults to False.
|
|
1504
|
+
|
|
1505
|
+
Returns
|
|
1506
|
+
--------
|
|
1507
|
+
LCMSMassFeature
|
|
1508
|
+
Object from ChromaPeak that contains data on selected MS1 peak. If no peak is found, will contain missing
|
|
1509
|
+
information and list the apex scan value as -99.
|
|
1510
|
+
|
|
1511
|
+
Raises
|
|
1512
|
+
------
|
|
1513
|
+
Warning
|
|
1514
|
+
If appropriate scan time data is not contained in ms1df.
|
|
1515
|
+
"""
|
|
1516
|
+
# Convert single feature to arrays and call batch method
|
|
1517
|
+
results = self.search_for_targeted_mass_features_batch(
|
|
1518
|
+
ms1df,
|
|
1519
|
+
np.array([mz_min]),
|
|
1520
|
+
np.array([mz_max]),
|
|
1521
|
+
np.array([st_min]),
|
|
1522
|
+
np.array([st_max]),
|
|
1523
|
+
[set_id],
|
|
1524
|
+
obj_idx=obj_idx,
|
|
1525
|
+
st_aligned=st_aligned
|
|
1526
|
+
)
|
|
1527
|
+
return results[set_id]
|
|
1528
|
+
|
|
1529
|
+
|
|
1530
|
+
def __len__(self):
|
|
1531
|
+
"""
|
|
1532
|
+
Returns the number of mass spectra in the dataset.
|
|
1533
|
+
|
|
1534
|
+
Returns
|
|
1535
|
+
--------
|
|
1536
|
+
int
|
|
1537
|
+
The number of mass spectra in the dataset.
|
|
1538
|
+
"""
|
|
1539
|
+
return len(self._ms)
|
|
1540
|
+
|
|
1541
|
+
def __getitem__(self, scan_number):
|
|
1542
|
+
"""
|
|
1543
|
+
Returns the mass spectrum corresponding to the specified scan number.
|
|
1544
|
+
|
|
1545
|
+
Parameters
|
|
1546
|
+
-----------
|
|
1547
|
+
scan_number : int
|
|
1548
|
+
The scan number of the desired mass spectrum.
|
|
1549
|
+
|
|
1550
|
+
Returns
|
|
1551
|
+
--------
|
|
1552
|
+
MassSpectrum
|
|
1553
|
+
The mass spectrum corresponding to the specified scan number.
|
|
1554
|
+
"""
|
|
1555
|
+
return self._ms.get(scan_number)
|
|
1556
|
+
|
|
1557
|
+
def __iter__(self):
|
|
1558
|
+
"""Returns an iterator over the mass spectra in the dataset.
|
|
1559
|
+
|
|
1560
|
+
Returns
|
|
1561
|
+
--------
|
|
1562
|
+
iterator
|
|
1563
|
+
An iterator over the mass spectra in the dataset.
|
|
1564
|
+
"""
|
|
1565
|
+
return iter(self._ms.values())
|
|
1566
|
+
|
|
1567
|
+
def set_tic_list_from_data(self, overwrite=False):
|
|
1568
|
+
"""Sets the TIC list from the mass spectrum objects within the _ms dictionary.
|
|
1569
|
+
|
|
1570
|
+
Parameters
|
|
1571
|
+
-----------
|
|
1572
|
+
overwrite : bool, optional
|
|
1573
|
+
If True, overwrites the TIC list if it is already set. Defaults to False.
|
|
1574
|
+
|
|
1575
|
+
Notes
|
|
1576
|
+
-----
|
|
1577
|
+
If the _ms dictionary is incomplete, sets the TIC list to an empty list.
|
|
1578
|
+
|
|
1579
|
+
Raises
|
|
1580
|
+
------
|
|
1581
|
+
ValueError
|
|
1582
|
+
If no mass spectra are found in the dataset.
|
|
1583
|
+
If the TIC list is already set and overwrite is False.
|
|
1584
|
+
"""
|
|
1585
|
+
# Check if _ms is empty and raise error if so
|
|
1586
|
+
if len(self._ms) == 0:
|
|
1587
|
+
raise ValueError("No mass spectra found in dataset")
|
|
1588
|
+
|
|
1589
|
+
# Check if tic_list is already set and raise error if so
|
|
1590
|
+
if len(self.tic) > 0 and not overwrite:
|
|
1591
|
+
raise ValueError("TIC list already set, use overwrite=True to overwrite")
|
|
1592
|
+
|
|
1593
|
+
self.tic = [self._ms.get(i).tic for i in self.scans_number]
|
|
1594
|
+
|
|
1595
|
+
def set_retention_time_from_data(self, overwrite=False):
|
|
1596
|
+
"""Sets the retention time list from the data in the _ms dictionary.
|
|
1597
|
+
|
|
1598
|
+
Parameters
|
|
1599
|
+
-----------
|
|
1600
|
+
overwrite : bool, optional
|
|
1601
|
+
If True, overwrites the retention time list if it is already set. Defaults to False.
|
|
1602
|
+
|
|
1603
|
+
Notes
|
|
1604
|
+
-----
|
|
1605
|
+
If the _ms dictionary is empty or incomplete, sets the retention time list to an empty list.
|
|
1606
|
+
|
|
1607
|
+
Raises
|
|
1608
|
+
------
|
|
1609
|
+
ValueError
|
|
1610
|
+
If no mass spectra are found in the dataset.
|
|
1611
|
+
If the retention time list is already set and overwrite is False.
|
|
1612
|
+
"""
|
|
1613
|
+
# Check if _ms is empty and raise error if so
|
|
1614
|
+
if len(self._ms) == 0:
|
|
1615
|
+
raise ValueError("No mass spectra found in dataset")
|
|
1616
|
+
|
|
1617
|
+
# Check if retention_time_list is already set and raise error if so
|
|
1618
|
+
if len(self.retention_time) > 0 and not overwrite:
|
|
1619
|
+
raise ValueError(
|
|
1620
|
+
"Retention time list already set, use overwrite=True to overwrite"
|
|
1621
|
+
)
|
|
1622
|
+
|
|
1623
|
+
retention_time_list = []
|
|
1624
|
+
for key_ms in sorted(self._ms.keys()):
|
|
1625
|
+
retention_time_list.append(self._ms.get(key_ms).retention_time)
|
|
1626
|
+
self.retention_time = retention_time_list
|
|
1627
|
+
|
|
1628
|
+
def set_scans_number_from_data(self, overwrite=False):
|
|
1629
|
+
"""Sets the scan number list from the data in the _ms dictionary.
|
|
1630
|
+
|
|
1631
|
+
Notes
|
|
1632
|
+
-----
|
|
1633
|
+
If the _ms dictionary is empty or incomplete, sets the scan number list to an empty list.
|
|
1634
|
+
|
|
1635
|
+
Raises
|
|
1636
|
+
------
|
|
1637
|
+
ValueError
|
|
1638
|
+
If no mass spectra are found in the dataset.
|
|
1639
|
+
If the scan number list is already set and overwrite is False.
|
|
1640
|
+
"""
|
|
1641
|
+
# Check if _ms is empty and raise error if so
|
|
1642
|
+
if len(self._ms) == 0:
|
|
1643
|
+
raise ValueError("No mass spectra found in dataset")
|
|
1644
|
+
|
|
1645
|
+
# Check if scans_number_list is already set and raise error if so
|
|
1646
|
+
if len(self.scans_number) > 0 and not overwrite:
|
|
1647
|
+
raise ValueError(
|
|
1648
|
+
"Scan number list already set, use overwrite=True to overwrite"
|
|
1649
|
+
)
|
|
1650
|
+
|
|
1651
|
+
self.scans_number = sorted(self._ms.keys())
|
|
1652
|
+
|
|
1653
|
+
@property
|
|
1654
|
+
def ms1_scans(self):
|
|
1655
|
+
"""
|
|
1656
|
+
list : A list of MS1 scan numbers for the dataset.
|
|
1657
|
+
"""
|
|
1658
|
+
return self.scan_df[self.scan_df.ms_level == 1].index.tolist()
|
|
1659
|
+
|
|
1660
|
+
@property
|
|
1661
|
+
def parameters(self):
|
|
1662
|
+
"""
|
|
1663
|
+
LCMSParameters : The parameters used for the LC-MS analysis.
|
|
1664
|
+
"""
|
|
1665
|
+
return self._parameters
|
|
1666
|
+
|
|
1667
|
+
@parameters.setter
|
|
1668
|
+
def parameters(self, paramsinstance):
|
|
1669
|
+
"""
|
|
1670
|
+
Sets the parameters used for the LC-MS analysis.
|
|
1671
|
+
|
|
1672
|
+
Parameters
|
|
1673
|
+
-----------
|
|
1674
|
+
paramsinstance : LCMSParameters
|
|
1675
|
+
The parameters used for the LC-MS analysis.
|
|
1676
|
+
"""
|
|
1677
|
+
self._parameters = paramsinstance
|
|
1678
|
+
|
|
1679
|
+
@property
|
|
1680
|
+
def scans_number(self):
|
|
1681
|
+
"""
|
|
1682
|
+
list : A list of scan numbers for the dataset.
|
|
1683
|
+
"""
|
|
1684
|
+
return self._scans_number_list
|
|
1685
|
+
|
|
1686
|
+
@scans_number.setter
|
|
1687
|
+
def scans_number(self, scan_numbers_list):
|
|
1688
|
+
"""
|
|
1689
|
+
Sets the scan numbers for the dataset.
|
|
1690
|
+
|
|
1691
|
+
Parameters
|
|
1692
|
+
-----------
|
|
1693
|
+
scan_numbers_list : list
|
|
1694
|
+
A list of scan numbers for the dataset.
|
|
1695
|
+
"""
|
|
1696
|
+
self._scans_number_list = scan_numbers_list
|
|
1697
|
+
|
|
1698
|
+
@property
|
|
1699
|
+
def retention_time(self):
|
|
1700
|
+
"""
|
|
1701
|
+
numpy.ndarray : An array of retention times for the dataset.
|
|
1702
|
+
"""
|
|
1703
|
+
return self._retention_time_list
|
|
1704
|
+
|
|
1705
|
+
@retention_time.setter
|
|
1706
|
+
def retention_time(self, rt_list):
|
|
1707
|
+
"""
|
|
1708
|
+
Sets the retention times for the dataset.
|
|
1709
|
+
|
|
1710
|
+
Parameters
|
|
1711
|
+
-----------
|
|
1712
|
+
rt_list : list
|
|
1713
|
+
A list of retention times for the dataset.
|
|
1714
|
+
"""
|
|
1715
|
+
self._retention_time_list = np.array(rt_list)
|
|
1716
|
+
|
|
1717
|
+
@property
|
|
1718
|
+
def tic(self):
|
|
1719
|
+
"""
|
|
1720
|
+
numpy.ndarray : An array of TIC values for the dataset.
|
|
1721
|
+
"""
|
|
1722
|
+
return self._tic_list
|
|
1723
|
+
|
|
1724
|
+
@tic.setter
|
|
1725
|
+
def tic(self, tic_list):
|
|
1726
|
+
"""
|
|
1727
|
+
Sets the TIC values for the dataset.
|
|
1728
|
+
|
|
1729
|
+
Parameters
|
|
1730
|
+
-----------
|
|
1731
|
+
tic_list : list
|
|
1732
|
+
A list of TIC values for the dataset.
|
|
1733
|
+
"""
|
|
1734
|
+
self._tic_list = np.array(tic_list)
|
|
1735
|
+
|
|
1736
|
+
class LCMSCollection(LCMSCollectionCalculations):
|
|
1737
|
+
"""A class representing a collection of liquid chromatography-mass spectrometry (LC-MS) runs.
|
|
1738
|
+
These runs can be from the same or different samples, but must be from the same instrument and have the same parameters
|
|
1739
|
+
for the initial processing steps. The LCMS objects are stored in an ordered dictionary with the sample name as the key.
|
|
1740
|
+
|
|
1741
|
+
Parameters
|
|
1742
|
+
-----------
|
|
1743
|
+
|
|
1744
|
+
Attributes
|
|
1745
|
+
-----------
|
|
1746
|
+
|
|
1747
|
+
Methods
|
|
1748
|
+
--------
|
|
1749
|
+
|
|
1750
|
+
Notes
|
|
1751
|
+
------
|
|
1752
|
+
This class is not intended to be instantiated directly, but rather instantiated using a parser object and then interacted with.
|
|
1753
|
+
#TODO KRH: add docstrings
|
|
1754
|
+
"""
|
|
1755
|
+
|
|
1756
|
+
def __init__(
|
|
1757
|
+
self,
|
|
1758
|
+
collection_location,
|
|
1759
|
+
manifest,
|
|
1760
|
+
collection_parser=None
|
|
1761
|
+
):
|
|
1762
|
+
self.collection_location = collection_location
|
|
1763
|
+
self._manifest_dict = manifest
|
|
1764
|
+
self.collection_parser = collection_parser
|
|
1765
|
+
self.raw_files_relocated = False
|
|
1766
|
+
|
|
1767
|
+
# These attributes are generally set by the parser during instantiation of this class
|
|
1768
|
+
self._lcms = {}
|
|
1769
|
+
self._combined_mass_features = None
|
|
1770
|
+
self._combined_induced_mass_features = None
|
|
1771
|
+
self.consensus_mass_features = {}
|
|
1772
|
+
self._parameters = LCMSCollectionParameters()
|
|
1773
|
+
self.isotopes_dropped = False
|
|
1774
|
+
self._mass_features_locked = False # Prevents rebuilding mass_features_dataframe from samples
|
|
1775
|
+
|
|
1776
|
+
# These attributes are set during processing
|
|
1777
|
+
self.rt_aligned = False
|
|
1778
|
+
self.rt_alignment_attempted = False
|
|
1779
|
+
self.missing_mass_features_searched = False
|
|
1780
|
+
|
|
1781
|
+
def _reorder_lcms_objects(self):
|
|
1782
|
+
"""
|
|
1783
|
+
Reorders the LCMS objects in the collection based on the order in the manifest.
|
|
1784
|
+
"""
|
|
1785
|
+
ordered_samples = self.samples
|
|
1786
|
+
self._lcms = {k: self._lcms[k] for k in ordered_samples}
|
|
1787
|
+
|
|
1788
|
+
def __getitem__(self, index):
|
|
1789
|
+
if isinstance(index, (float, np.floating, np.ndarray)):
|
|
1790
|
+
index = int(index)
|
|
1791
|
+
elif isinstance(index, np.integer):
|
|
1792
|
+
index = int(index)
|
|
1793
|
+
samp_name = self.samples[index]
|
|
1794
|
+
self._lcms[samp_name]
|
|
1795
|
+
return self._lcms[samp_name]
|
|
1796
|
+
|
|
1797
|
+
def __len__(self):
|
|
1798
|
+
return len(self.samples)
|
|
1799
|
+
|
|
1800
|
+
def _prepare_lcms_mass_features_for_combination(self, lcms_obj, induced_features = False):
|
|
1801
|
+
"""
|
|
1802
|
+
Prepares the mass features in the LCMS objects in the collection for combination.
|
|
1803
|
+
"""
|
|
1804
|
+
if induced_features:
|
|
1805
|
+
mf_df = lcms_obj.mass_features_to_df(induced_features = True)
|
|
1806
|
+
# Check if lcms_obj has attribute light_mf_df
|
|
1807
|
+
elif hasattr(lcms_obj, "light_mf_df"):
|
|
1808
|
+
mf_df = lcms_obj.light_mf_df
|
|
1809
|
+
else:
|
|
1810
|
+
mf_df = lcms_obj.mass_features_to_df()
|
|
1811
|
+
|
|
1812
|
+
# If dataframe is empty, add minimal required columns and return
|
|
1813
|
+
if len(mf_df) == 0:
|
|
1814
|
+
import pandas as pd
|
|
1815
|
+
mf_df["sample_name"] = []
|
|
1816
|
+
mf_df["sample_id"] = []
|
|
1817
|
+
mf_df["coll_mf_id"] = []
|
|
1818
|
+
mf_df["mf_id"] = []
|
|
1819
|
+
mf_df["_eic_mz"] = [] # Include _eic_mz for consistency with non-empty dataframes
|
|
1820
|
+
if induced_features:
|
|
1821
|
+
mf_df["cluster"] = []
|
|
1822
|
+
return mf_df
|
|
1823
|
+
|
|
1824
|
+
# Remove index
|
|
1825
|
+
mf_df = mf_df.reset_index(drop=False)
|
|
1826
|
+
# Add sample name and sample id to the dataframe
|
|
1827
|
+
mf_df["sample_name"] = lcms_obj.sample_name
|
|
1828
|
+
# Ensure sample_id is stored as an integer to avoid float indices later
|
|
1829
|
+
try:
|
|
1830
|
+
mf_df["sample_id"] = int(self.manifest[lcms_obj.sample_name]["collection_id"])
|
|
1831
|
+
except Exception:
|
|
1832
|
+
mf_df["sample_id"] = self.manifest[lcms_obj.sample_name]["collection_id"]
|
|
1833
|
+
mf_df["coll_mf_id"] = mf_df["sample_id"].astype(str) + "_" + mf_df["mf_id"].astype(str)
|
|
1834
|
+
|
|
1835
|
+
# For induced features, extract cluster from mf_id (format: c{cluster}_{index}_i)
|
|
1836
|
+
# and add as a column since cluster_index attribute may not be set on the object
|
|
1837
|
+
if induced_features:
|
|
1838
|
+
def extract_cluster(mf_id):
|
|
1839
|
+
# mf_id format: c{cluster}_{index}_i
|
|
1840
|
+
# Example: c123_5_i -> cluster 123
|
|
1841
|
+
if isinstance(mf_id, str) and mf_id.startswith('c') and '_i' in mf_id:
|
|
1842
|
+
parts = mf_id.split('_')
|
|
1843
|
+
if len(parts) >= 2:
|
|
1844
|
+
cluster_str = parts[0][1:] # Remove 'c' prefix
|
|
1845
|
+
try:
|
|
1846
|
+
return int(cluster_str)
|
|
1847
|
+
except ValueError:
|
|
1848
|
+
return None
|
|
1849
|
+
return None
|
|
1850
|
+
|
|
1851
|
+
mf_df['cluster'] = mf_df['mf_id'].apply(extract_cluster)
|
|
1852
|
+
|
|
1853
|
+
# Check if scan_df has scan_time_aligned and add to mf_df if so
|
|
1854
|
+
if "scan_time_aligned" in lcms_obj.scan_df.columns:
|
|
1855
|
+
scan_df = lcms_obj.scan_df[["scan", "scan_time_aligned"]].copy()
|
|
1856
|
+
scan_df = scan_df.rename(columns={"scan": "apex_scan"})
|
|
1857
|
+
mf_df = mf_df.merge(scan_df, on="apex_scan")
|
|
1858
|
+
|
|
1859
|
+
return mf_df
|
|
1860
|
+
|
|
1861
|
+
def _combine_mass_features(self, induced_features = False):
|
|
1862
|
+
"""
|
|
1863
|
+
Concatenates the mass features from all the LCMS objects in the collection.
|
|
1864
|
+
|
|
1865
|
+
Returns
|
|
1866
|
+
--------
|
|
1867
|
+
None, sets the _combined_mass_features or _combined_induced_mass_feature attribute.
|
|
1868
|
+
|
|
1869
|
+
Notes
|
|
1870
|
+
-----
|
|
1871
|
+
If _mass_features_locked is True (e.g., when only representative features are loaded),
|
|
1872
|
+
this method will skip rebuilding the regular mass features dataframe to preserve
|
|
1873
|
+
the full collection-level dataframe. Induced features are always rebuilt since they
|
|
1874
|
+
are created during processing.
|
|
1875
|
+
"""
|
|
1876
|
+
|
|
1877
|
+
# Skip rebuilding regular mass features if locked (preserves full dataframe)
|
|
1878
|
+
if not induced_features and self._mass_features_locked:
|
|
1879
|
+
return
|
|
1880
|
+
|
|
1881
|
+
## TODO: See why this function runs slower on multiprocessing,
|
|
1882
|
+
## especially for induced features
|
|
1883
|
+
## has only been considered so far on ~20 samples
|
|
1884
|
+
# if self.parameters.lcms_collection.cores == 1:
|
|
1885
|
+
# # Prepare mass features for combination sequentially
|
|
1886
|
+
# mf_df_list = []
|
|
1887
|
+
# for lcms_obj in self:
|
|
1888
|
+
# mf_df = self._prepare_lcms_mass_features_for_combination(lcms_obj, induced_features)
|
|
1889
|
+
# mf_df_list.append(mf_df)
|
|
1890
|
+
|
|
1891
|
+
# if self.parameters.lcms_collection.cores > 1:
|
|
1892
|
+
# # Parallelize the mass feature preparation
|
|
1893
|
+
# if self.parameters.lcms_collection.cores > len(self):
|
|
1894
|
+
# ncores = len(self)
|
|
1895
|
+
# else:
|
|
1896
|
+
# ncores = self.parameters.lcms_collection.cores
|
|
1897
|
+
# pool = multiprocessing.Pool(ncores)
|
|
1898
|
+
# mf_df_list = pool.starmap(
|
|
1899
|
+
# self._prepare_lcms_mass_features_for_combination,
|
|
1900
|
+
# [(lcms_obj, induced_features) for lcms_obj in self]
|
|
1901
|
+
# )
|
|
1902
|
+
|
|
1903
|
+
# Prepare mass features for combination sequentially
|
|
1904
|
+
mf_df_list = []
|
|
1905
|
+
for lcms_obj in self:
|
|
1906
|
+
# Skip samples with no induced mass features if processing induced features
|
|
1907
|
+
if induced_features:
|
|
1908
|
+
if not hasattr(lcms_obj, 'induced_mass_features') or len(lcms_obj.induced_mass_features) == 0:
|
|
1909
|
+
continue
|
|
1910
|
+
mf_df = self._prepare_lcms_mass_features_for_combination(lcms_obj, induced_features)
|
|
1911
|
+
mf_df_list.append(mf_df)
|
|
1912
|
+
|
|
1913
|
+
# If no mass features were collected (e.g., no induced features exist), return early
|
|
1914
|
+
if len(mf_df_list) == 0:
|
|
1915
|
+
# Add a warning here, not sure how one might reach this state, clearly saying if they are induced features or not
|
|
1916
|
+
warnings.warn("No mass features found to combine in the collection.", UserWarning)
|
|
1917
|
+
if induced_features:
|
|
1918
|
+
self._combined_induced_mass_features = None
|
|
1919
|
+
else:
|
|
1920
|
+
self._combined_mass_features = None
|
|
1921
|
+
return
|
|
1922
|
+
|
|
1923
|
+
combined_mass_features = pd.concat(mf_df_list)
|
|
1924
|
+
# Ensure sample_id and cluster columns have integer dtypes where possible
|
|
1925
|
+
if "sample_id" in combined_mass_features.columns:
|
|
1926
|
+
try:
|
|
1927
|
+
combined_mass_features["sample_id"] = combined_mass_features["sample_id"].astype(int)
|
|
1928
|
+
except Exception:
|
|
1929
|
+
combined_mass_features["sample_id"] = pd.to_numeric(
|
|
1930
|
+
combined_mass_features["sample_id"], errors="coerce"
|
|
1931
|
+
).astype("Int64")
|
|
1932
|
+
if "cluster" in combined_mass_features.columns:
|
|
1933
|
+
try:
|
|
1934
|
+
combined_mass_features["cluster"] = combined_mass_features["cluster"].astype(int)
|
|
1935
|
+
except Exception:
|
|
1936
|
+
combined_mass_features["cluster"] = pd.to_numeric(
|
|
1937
|
+
combined_mass_features["cluster"], errors="coerce"
|
|
1938
|
+
).astype("Int64")
|
|
1939
|
+
# Move coll_mf_id, sample_name, sample_id, and mf_id to front
|
|
1940
|
+
cols = combined_mass_features.columns.tolist()
|
|
1941
|
+
top_cols = ["coll_mf_id", "sample_name", "sample_id", "mf_id", "mz", "scan_time_aligned", "cluster"]
|
|
1942
|
+
cols = [x for x in top_cols + [col for col in cols if col not in top_cols] if x in cols]
|
|
1943
|
+
combined_mass_features = combined_mass_features[cols]
|
|
1944
|
+
# Make coll_mf_id the index
|
|
1945
|
+
combined_mass_features = combined_mass_features.set_index("coll_mf_id")
|
|
1946
|
+
if induced_features == True:
|
|
1947
|
+
self._combined_induced_mass_features = combined_mass_features
|
|
1948
|
+
else:
|
|
1949
|
+
self._combined_mass_features = combined_mass_features
|
|
1950
|
+
|
|
1951
|
+
def _check_mass_features_df(self, induced_features = False):
|
|
1952
|
+
"""Checks if the mass features dataframe has expected columns. If not, adds them.
|
|
1953
|
+
|
|
1954
|
+
Returns
|
|
1955
|
+
--------
|
|
1956
|
+
pandas.DataFrame
|
|
1957
|
+
A pandas dataframe of mass features in the collection.
|
|
1958
|
+
|
|
1959
|
+
Notes
|
|
1960
|
+
------
|
|
1961
|
+
If scan_time_aligned is not in the _combined_mass_features or
|
|
1962
|
+
_combined_induced_mass_features, tries to add it.
|
|
1963
|
+
|
|
1964
|
+
"""
|
|
1965
|
+
|
|
1966
|
+
if induced_features:
|
|
1967
|
+
cmf_df = self._combined_induced_mass_features
|
|
1968
|
+
else:
|
|
1969
|
+
cmf_df = self._combined_mass_features
|
|
1970
|
+
# Check if parameters are set to drop isotopologues and drop if so
|
|
1971
|
+
if self.parameters.lcms_collection.drop_isotopologues:
|
|
1972
|
+
if not self.isotopes_dropped:
|
|
1973
|
+
self._drop_isotopologues()
|
|
1974
|
+
# Check if scan_time_aligned is in combined_mass_features, try to add if not
|
|
1975
|
+
if cmf_df is not None and "scan_time_aligned" not in cmf_df.columns:
|
|
1976
|
+
cmb_mf = cmf_df.copy()
|
|
1977
|
+
cmb_mf = cmb_mf.reset_index(drop=False)
|
|
1978
|
+
lcms_aligned = [True for x in self if "scan_time_aligned" in x.scan_df.columns]
|
|
1979
|
+
if len(lcms_aligned) == len(self):
|
|
1980
|
+
# Add scan_time_aligned to combined_mass_features dataframe
|
|
1981
|
+
scan_time_aligned_list = []
|
|
1982
|
+
for lcms_obj in self:
|
|
1983
|
+
scan_time_df_i = lcms_obj.scan_df[["scan", "scan_time_aligned"]]
|
|
1984
|
+
scan_time_df_i["sample_name"] = lcms_obj.sample_name
|
|
1985
|
+
scan_time_aligned_list.append(scan_time_df_i)
|
|
1986
|
+
scan_time_aligned_df = pd.concat(scan_time_aligned_list)
|
|
1987
|
+
# Rename scan to apex_scan
|
|
1988
|
+
scan_time_aligned_df = scan_time_aligned_df.rename(columns={"scan": "apex_scan"})
|
|
1989
|
+
cmb_mf_merged = cmb_mf.merge(scan_time_aligned_df, on=["apex_scan", "sample_name"])
|
|
1990
|
+
cmb_mf_merged = cmb_mf_merged.set_index("coll_mf_id")
|
|
1991
|
+
# Merge scan_time_aligned_df with combined_mass_features on apex_scan and sample_name
|
|
1992
|
+
if induced_features:
|
|
1993
|
+
self._combined_induced_mass_features = cmb_mf_merged
|
|
1994
|
+
else:
|
|
1995
|
+
self._combined_mass_features = cmb_mf_merged
|
|
1996
|
+
|
|
1997
|
+
def plot_tics(self, ms_level=1, type = "raw", plot_legend=False):
|
|
1998
|
+
"""Plots the TICs for all the LCMS objects in the collection.
|
|
1999
|
+
|
|
2000
|
+
Parameters
|
|
2001
|
+
-----------
|
|
2002
|
+
ms_level : int, optional
|
|
2003
|
+
The MS level to plot the TICs for. Defaults to 1.
|
|
2004
|
+
type : str, optional
|
|
2005
|
+
The type of TIC to plot, either "raw" or "corrected" or "both". Defaults to "raw".
|
|
2006
|
+
plot_legend : bool, optional
|
|
2007
|
+
If True, plots a legend on the TIC plot that labels each sample. Defaults to False.
|
|
2008
|
+
"""
|
|
2009
|
+
to_plot = []
|
|
2010
|
+
if type == "both":
|
|
2011
|
+
to_plot = ["raw", "corrected"]
|
|
2012
|
+
else:
|
|
2013
|
+
to_plot = [type]
|
|
2014
|
+
|
|
2015
|
+
fig, axs = plt.subplots(
|
|
2016
|
+
len(to_plot), 1, figsize=(10, 5 * len(to_plot)), sharex=True, squeeze=False
|
|
2017
|
+
)
|
|
2018
|
+
|
|
2019
|
+
for i, plot_type in enumerate(to_plot):
|
|
2020
|
+
ax = axs[i, 0]
|
|
2021
|
+
colors = iter(plt.cm.rainbow(np.linspace(0, 1, len(self))))
|
|
2022
|
+
for lcms_obj in self:
|
|
2023
|
+
c = next(colors)
|
|
2024
|
+
# check if lcms_obj is the center of the collection
|
|
2025
|
+
self.manifest_dataframe[self.manifest_dataframe['center']].collection_id.values
|
|
2026
|
+
|
|
2027
|
+
|
|
2028
|
+
scan_df = lcms_obj.scan_df
|
|
2029
|
+
scan_df = scan_df[scan_df.ms_level == ms_level]
|
|
2030
|
+
if plot_type == "corrected":
|
|
2031
|
+
# Check that scan_time_aligned is key in scan_df
|
|
2032
|
+
if "scan_time_aligned" not in scan_df.columns:
|
|
2033
|
+
raise ValueError(f"scan_time_aligned not found in scan_df for {lcms_obj.sample_name}")
|
|
2034
|
+
else:
|
|
2035
|
+
ax.plot(scan_df.scan_time_aligned, scan_df.tic, label=lcms_obj.sample_name, c=c, linewidth=0.3)
|
|
2036
|
+
elif plot_type == "raw":
|
|
2037
|
+
ax.plot(scan_df.scan_time, scan_df.tic, label=lcms_obj.sample_name, c=c, linewidth=0.3)
|
|
2038
|
+
ax.set_xlabel("Retention Time (min," + f" {plot_type})" )
|
|
2039
|
+
ax.set_ylabel("TIC")
|
|
2040
|
+
if plot_legend:
|
|
2041
|
+
ax.legend()
|
|
2042
|
+
plt.show()
|
|
2043
|
+
|
|
2044
|
+
def plot_alignments(self, plot_legend=False):
|
|
2045
|
+
"""Plots the alignment of the LCMS objects in the collection.
|
|
2046
|
+
|
|
2047
|
+
Parameters
|
|
2048
|
+
-----------
|
|
2049
|
+
plot_legend : bool, optional
|
|
2050
|
+
If True, plots a legend on the alignment plot that labels each sample. Defaults to False.
|
|
2051
|
+
"""
|
|
2052
|
+
fig, ax = plt.subplots(figsize=(10, 5))
|
|
2053
|
+
colors = iter(plt.cm.rainbow(np.linspace(0, 1, len(self))))
|
|
2054
|
+
|
|
2055
|
+
for lcms_obj in self:
|
|
2056
|
+
c = next(colors)
|
|
2057
|
+
scan_df = lcms_obj.scan_df
|
|
2058
|
+
if "scan_time_aligned" not in scan_df.columns:
|
|
2059
|
+
raise ValueError(f"scan_time_aligned not found in scan_df for {lcms_obj.sample_name}")
|
|
2060
|
+
scan_df['time_diff'] = scan_df.scan_time - scan_df.scan_time_aligned
|
|
2061
|
+
ax.plot(scan_df.scan_time_aligned, scan_df.time_diff, label=lcms_obj.sample_name, c=c, linewidth=0.3)
|
|
2062
|
+
|
|
2063
|
+
ax.set_xlabel("Aligned Retention Time (min)")
|
|
2064
|
+
ax.set_ylabel("Time Difference (min)")
|
|
2065
|
+
if plot_legend:
|
|
2066
|
+
ax.legend()
|
|
2067
|
+
plt.show()
|
|
2068
|
+
|
|
2069
|
+
def _drop_isotopologues(self):
|
|
2070
|
+
"""Drops isotopologues from the mass features in combined_mass_features dataframe."""
|
|
2071
|
+
cmb_mf_df = self._combined_mass_features
|
|
2072
|
+
|
|
2073
|
+
# Keep monos or if no monos
|
|
2074
|
+
cmb_monos = cmb_mf_df[cmb_mf_df.monoisotopic_mf_id == cmb_mf_df.mf_id]
|
|
2075
|
+
cmb_nomonos = cmb_mf_df[cmb_mf_df.monoisotopic_mf_id.isnull()]
|
|
2076
|
+
# Keep deconvoluted parent or if no deconvoluted parent
|
|
2077
|
+
cmb_decon_parent = cmb_mf_df[cmb_mf_df.mass_spectrum_deconvoluted_parent | cmb_mf_df.monoisotopic_mf_id.isnull()]
|
|
2078
|
+
|
|
2079
|
+
cmb_mf_df2 = pd.concat([cmb_monos, cmb_nomonos, cmb_decon_parent])
|
|
2080
|
+
cmb_mf_df2 = cmb_mf_df2[~cmb_mf_df2.index.duplicated(keep='first')]
|
|
2081
|
+
self.isotopes_dropped = True
|
|
2082
|
+
self._combined_mass_features = cmb_mf_df2
|
|
2083
|
+
|
|
2084
|
+
|
|
2085
|
+
def load_raw_data(self, sample_idx: int, ms_level = 1, time_range = None) -> None:
|
|
2086
|
+
"""Load raw data for a specific sample index in the collection.
|
|
2087
|
+
|
|
2088
|
+
Parameters
|
|
2089
|
+
-----------
|
|
2090
|
+
sample_idx : int
|
|
2091
|
+
The index of the sample in the collection.
|
|
2092
|
+
ms_level : int, optional
|
|
2093
|
+
The MS level to load raw data for. Defaults to 1.
|
|
2094
|
+
time_range : tuple or list of tuples, optional
|
|
2095
|
+
Retention time range(s) to load. Can be a single tuple (min, max) or
|
|
2096
|
+
a list of tuples for multiple ranges. If None, loads all data. Defaults to None.
|
|
2097
|
+
|
|
2098
|
+
Raises
|
|
2099
|
+
-------
|
|
2100
|
+
IndexError
|
|
2101
|
+
If the sample index is out of range.
|
|
2102
|
+
ValueError
|
|
2103
|
+
If raw data for the specified MS level is already loaded for the sample index.
|
|
2104
|
+
ValueError
|
|
2105
|
+
If the spectra parser is not set for the LCMS object or if the parser type does not support loading raw data.
|
|
2106
|
+
|
|
2107
|
+
Returns
|
|
2108
|
+
--------
|
|
2109
|
+
None, but updates the LCMS object with the raw data for the specified MS level.
|
|
2110
|
+
"""
|
|
2111
|
+
if sample_idx < 0 or sample_idx >= len(self.samples):
|
|
2112
|
+
raise IndexError("Sample index out of range.")
|
|
2113
|
+
|
|
2114
|
+
# Check that the sample does not already have raw data loaded
|
|
2115
|
+
if ms_level in self[sample_idx]._ms_unprocessed:
|
|
2116
|
+
raise ValueError(f"Raw data for MS{ms_level} already loaded for sample index {sample_idx}. Drop data first if you want to reload it.")
|
|
2117
|
+
|
|
2118
|
+
# Check the parser type of the LCMS object
|
|
2119
|
+
if self[sample_idx].spectra_parser is None:
|
|
2120
|
+
raise ValueError("Spectra parser is not set for this LCMS object.")
|
|
2121
|
+
|
|
2122
|
+
# Instantiate the parser and load the raw data using the correct method
|
|
2123
|
+
parser = self[sample_idx].spectra_parser
|
|
2124
|
+
parser_class_name = self[sample_idx].spectra_parser_class.__name__
|
|
2125
|
+
scan_df = self[sample_idx].scan_df
|
|
2126
|
+
|
|
2127
|
+
# Get raw data for the specified MS level using the appropriate method
|
|
2128
|
+
if parser_class_name == "ImportMassSpectraThermoMSFileReader":
|
|
2129
|
+
self[sample_idx]._ms_unprocessed[ms_level] = parser.get_ms_raw(
|
|
2130
|
+
spectra=f"ms{ms_level}",
|
|
2131
|
+
scan_df=scan_df,
|
|
2132
|
+
time_range=time_range
|
|
2133
|
+
)[f"ms{ms_level}"]
|
|
2134
|
+
|
|
2135
|
+
elif parser_class_name == "MZMLSpectraParser":
|
|
2136
|
+
data = parser.load()
|
|
2137
|
+
self[sample_idx]._ms_unprocessed[ms_level] = parser.get_ms_raw(
|
|
2138
|
+
spectra=f"ms{ms_level}",
|
|
2139
|
+
scan_df=scan_df,
|
|
2140
|
+
data=data,
|
|
2141
|
+
time_range=time_range
|
|
2142
|
+
)[f"ms{ms_level}"]
|
|
2143
|
+
|
|
2144
|
+
elif parser_class_name == "ReadCoreMSHDFMassSpectra":
|
|
2145
|
+
raise ValueError(
|
|
2146
|
+
"ReadCoreMSHDFMassSpectra does not have a method to load raw data. Need to instantiate the original parser to access the raw data."
|
|
2147
|
+
)
|
|
2148
|
+
|
|
2149
|
+
def drop_raw_data(self, sample_idx: int, ms_level = 1) -> None:
|
|
2150
|
+
"""Drop raw data for a specific sample index in the collection.
|
|
2151
|
+
|
|
2152
|
+
Parameters
|
|
2153
|
+
-----------
|
|
2154
|
+
sample_idx : int
|
|
2155
|
+
The index of the sample in the collection.
|
|
2156
|
+
ms_level : int, optional
|
|
2157
|
+
The MS level to drop raw data for. Defaults to 1.
|
|
2158
|
+
|
|
2159
|
+
Raises
|
|
2160
|
+
-------
|
|
2161
|
+
IndexError
|
|
2162
|
+
If the sample index is out of range.
|
|
2163
|
+
ValueError
|
|
2164
|
+
If raw data for the specified MS level is not loaded for the sample index.
|
|
2165
|
+
|
|
2166
|
+
Returns
|
|
2167
|
+
--------
|
|
2168
|
+
None
|
|
2169
|
+
"""
|
|
2170
|
+
if sample_idx < 0 or sample_idx >= len(self.samples):
|
|
2171
|
+
raise IndexError("Sample index out of range.")
|
|
2172
|
+
|
|
2173
|
+
# Check that the sample has raw data loaded
|
|
2174
|
+
if ms_level not in self[sample_idx]._ms_unprocessed:
|
|
2175
|
+
raise ValueError(f"No raw data for MS{ms_level} found for sample index {sample_idx}. Load data first if you want to drop it.")
|
|
2176
|
+
|
|
2177
|
+
# Drop the raw data
|
|
2178
|
+
del self[sample_idx]._ms_unprocessed[ms_level]
|
|
2179
|
+
|
|
2180
|
+
def update_raw_file_locations(self, new_raw_folder):
|
|
2181
|
+
"""Update the raw file locations for all LCMS objects in the collection.
|
|
2182
|
+
|
|
2183
|
+
This method updates the path to the original raw data files (.raw, .mzML, etc.)
|
|
2184
|
+
that were used to create the processed HDF5 files stored in .corems folders.
|
|
2185
|
+
|
|
2186
|
+
Parameters
|
|
2187
|
+
-----------
|
|
2188
|
+
new_raw_folder : str or Path
|
|
2189
|
+
The new folder location containing the raw data files (.raw, .mzML, etc.).
|
|
2190
|
+
The method will look for raw files with the same base name as each sample.
|
|
2191
|
+
|
|
2192
|
+
Raises
|
|
2193
|
+
-------
|
|
2194
|
+
FileNotFoundError
|
|
2195
|
+
If the new raw folder does not exist.
|
|
2196
|
+
FileNotFoundError
|
|
2197
|
+
If a raw file for a sample is not found in the new folder.
|
|
2198
|
+
|
|
2199
|
+
Returns
|
|
2200
|
+
--------
|
|
2201
|
+
None, but updates the raw_file_location for each LCMS object in the collection.
|
|
2202
|
+
|
|
2203
|
+
Examples
|
|
2204
|
+
--------
|
|
2205
|
+
If raw files were moved from /old/path/ to /new/path/:
|
|
2206
|
+
>>> lcms_collection.update_raw_file_locations("/new/path/")
|
|
2207
|
+
"""
|
|
2208
|
+
from pathlib import Path
|
|
2209
|
+
|
|
2210
|
+
if isinstance(new_raw_folder, str):
|
|
2211
|
+
new_raw_folder = Path(new_raw_folder)
|
|
2212
|
+
|
|
2213
|
+
if not new_raw_folder.exists():
|
|
2214
|
+
raise FileNotFoundError(f"Raw data folder does not exist: {new_raw_folder}")
|
|
2215
|
+
|
|
2216
|
+
# Common raw file extensions
|
|
2217
|
+
raw_extensions = ['.raw', '.mzML', '.mzml']
|
|
2218
|
+
|
|
2219
|
+
for sample_name in self.samples:
|
|
2220
|
+
lcms_obj = self._lcms[sample_name]
|
|
2221
|
+
|
|
2222
|
+
# Try to find the raw file with common extensions
|
|
2223
|
+
new_raw_file = None
|
|
2224
|
+
for ext in raw_extensions:
|
|
2225
|
+
candidate = new_raw_folder / f"{sample_name}{ext}"
|
|
2226
|
+
if candidate.exists():
|
|
2227
|
+
new_raw_file = candidate
|
|
2228
|
+
break
|
|
2229
|
+
|
|
2230
|
+
if new_raw_file is None:
|
|
2231
|
+
raise FileNotFoundError(
|
|
2232
|
+
f"Raw file for sample '{sample_name}' not found in {new_raw_folder}. "
|
|
2233
|
+
f"Tried extensions: {', '.join(raw_extensions)}"
|
|
2234
|
+
)
|
|
2235
|
+
|
|
2236
|
+
# Update the raw file location and set flag that raw files have been relocated
|
|
2237
|
+
lcms_obj.raw_file_location = new_raw_file
|
|
2238
|
+
self.raw_files_relocated = True
|
|
2239
|
+
|
|
2240
|
+
def collection_pivot_table(self, attribute = 'coll_mf_id', verbose = True):
|
|
2241
|
+
"""Generate a pivot table of all regular and induced mass features in
|
|
2242
|
+
a collection. Default attribute presented is the mass feature ID, also
|
|
2243
|
+
prints a list of other available attributes.
|
|
2244
|
+
|
|
2245
|
+
Parameters
|
|
2246
|
+
-----------
|
|
2247
|
+
attribute : str
|
|
2248
|
+
The desired attribute to be presented in the pivot table. Defaults
|
|
2249
|
+
to mass feature ID
|
|
2250
|
+
verbose : boolean
|
|
2251
|
+
Print out all the possible values the fill the pivot table and list
|
|
2252
|
+
attributes that are not collected for induced mass features
|
|
2253
|
+
|
|
2254
|
+
Returns
|
|
2255
|
+
--------
|
|
2256
|
+
pd.DataFrame
|
|
2257
|
+
A DataFrame that displays one given attribute across all clusters
|
|
2258
|
+
and samples in a collection
|
|
2259
|
+
|
|
2260
|
+
"""
|
|
2261
|
+
|
|
2262
|
+
mf_pivot = self.mass_features_dataframe.copy()
|
|
2263
|
+
mf_pivot.reset_index(inplace = True)
|
|
2264
|
+
|
|
2265
|
+
# Only include induced mass features if gap-filling has been performed
|
|
2266
|
+
if self.induced_mass_features_dataframe is not None:
|
|
2267
|
+
imf_pivot = self.induced_mass_features_dataframe.copy()
|
|
2268
|
+
imf_pivot.reset_index(inplace = True)
|
|
2269
|
+
# Cluster column extracted from mf_id in _prepare_lcms_mass_features_for_combination
|
|
2270
|
+
mf_pivot = pd.concat([mf_pivot, imf_pivot], axis = 0)
|
|
2271
|
+
mf_pivot.reset_index(drop = True, inplace = True)
|
|
2272
|
+
else:
|
|
2273
|
+
imf_pivot = None
|
|
2274
|
+
|
|
2275
|
+
mf_pivot['cluster'] = mf_pivot['cluster'].astype(int)
|
|
2276
|
+
|
|
2277
|
+
if verbose:
|
|
2278
|
+
print(
|
|
2279
|
+
'Attributes available for pivot table:\n',
|
|
2280
|
+
[x for x in mf_pivot.columns if x not in ['cluster', 'sample_name', 'mf_id', 'partition_idx', 'idx']]
|
|
2281
|
+
)
|
|
2282
|
+
if imf_pivot is not None:
|
|
2283
|
+
print(
|
|
2284
|
+
'\nAttributes that have no value for induced mass features:\n',
|
|
2285
|
+
imf_pivot.columns[imf_pivot.isna().all()].tolist()
|
|
2286
|
+
)
|
|
2287
|
+
|
|
2288
|
+
# Create pivot table and reindex to include all samples (even those with no features)
|
|
2289
|
+
pivot = mf_pivot.pivot(index = 'cluster', columns = 'sample_name', values = attribute)
|
|
2290
|
+
|
|
2291
|
+
# Reindex columns to include all samples in the collection
|
|
2292
|
+
all_samples = self.samples
|
|
2293
|
+
pivot = pivot.reindex(columns=all_samples)
|
|
2294
|
+
|
|
2295
|
+
return pivot
|
|
2296
|
+
|
|
2297
|
+
def cluster_representatives_table(self):
|
|
2298
|
+
"""Generate a table of representative mass features from each consensus cluster.
|
|
2299
|
+
|
|
2300
|
+
This method returns a DataFrame containing all attributes for the
|
|
2301
|
+
representative mass feature from each consensus cluster. The representative
|
|
2302
|
+
is selected using the same logic as process_consensus_features().
|
|
2303
|
+
|
|
2304
|
+
Returns
|
|
2305
|
+
--------
|
|
2306
|
+
pd.DataFrame
|
|
2307
|
+
A DataFrame with one row per cluster containing all attributes for
|
|
2308
|
+
each cluster's representative mass feature. Includes:
|
|
2309
|
+
- cluster: cluster ID (as a column for easy joining)
|
|
2310
|
+
- polarity: ionization polarity from the collection
|
|
2311
|
+
- n_samples_detected: number of samples where the cluster was detected
|
|
2312
|
+
- All other mass feature attributes from the representative
|
|
2313
|
+
|
|
2314
|
+
Notes
|
|
2315
|
+
-----
|
|
2316
|
+
The representative metric used is determined by
|
|
2317
|
+
self.parameters.lcms_collection.consensus_representative_metric and
|
|
2318
|
+
is the same metric used by process_consensus_features() for consistency.
|
|
2319
|
+
Common options include 'intensity' (highest intensity) or
|
|
2320
|
+
'intensity_prefer_ms2' (highest intensity with preference for MS2 data).
|
|
2321
|
+
"""
|
|
2322
|
+
|
|
2323
|
+
mf_df = self.mass_features_dataframe.copy()
|
|
2324
|
+
mf_df.reset_index(inplace = True)
|
|
2325
|
+
|
|
2326
|
+
# Include induced mass features if they exist (from gap-filling)
|
|
2327
|
+
if self.induced_mass_features_dataframe is not None:
|
|
2328
|
+
imf_df = self.induced_mass_features_dataframe.copy()
|
|
2329
|
+
imf_df.reset_index(inplace = True)
|
|
2330
|
+
# Cluster column extracted from mf_id in _prepare_lcms_mass_features_for_combination
|
|
2331
|
+
mf_df = pd.concat([mf_df, imf_df], axis = 0)
|
|
2332
|
+
mf_df.reset_index(drop = True, inplace = True)
|
|
2333
|
+
mf_df['cluster'] = mf_df['cluster'].astype(int)
|
|
2334
|
+
|
|
2335
|
+
# Calculate number of samples per cluster
|
|
2336
|
+
cluster_sample_counts = mf_df.groupby('cluster')['sample_id'].nunique().to_dict()
|
|
2337
|
+
|
|
2338
|
+
# Use the same representative selection logic as process_consensus_features
|
|
2339
|
+
# This uses the configured representative_metric from parameters
|
|
2340
|
+
representatives = self.get_representative_mass_features_for_all_clusters()
|
|
2341
|
+
|
|
2342
|
+
# Get the coll_mf_ids of the representatives
|
|
2343
|
+
representative_ids = representatives['coll_mf_id'].tolist()
|
|
2344
|
+
|
|
2345
|
+
# Filter mf_df to only include representative features
|
|
2346
|
+
consensus_report = mf_df[mf_df.coll_mf_id.isin(representative_ids)].copy()
|
|
2347
|
+
|
|
2348
|
+
# Add polarity (get from first sample in collection)
|
|
2349
|
+
if len(self) > 0:
|
|
2350
|
+
polarity = self[0].polarity
|
|
2351
|
+
else:
|
|
2352
|
+
polarity = 'unknown'
|
|
2353
|
+
consensus_report['polarity'] = polarity
|
|
2354
|
+
|
|
2355
|
+
# Add number of samples detected
|
|
2356
|
+
consensus_report['n_samples_detected'] = consensus_report['cluster'].map(cluster_sample_counts)
|
|
2357
|
+
|
|
2358
|
+
# Reorder columns to put cluster at the front
|
|
2359
|
+
cols = consensus_report.columns.tolist()
|
|
2360
|
+
if 'cluster' in cols:
|
|
2361
|
+
cols.remove('cluster')
|
|
2362
|
+
cols = ['cluster'] + cols
|
|
2363
|
+
consensus_report = consensus_report[cols]
|
|
2364
|
+
|
|
2365
|
+
# Sort by cluster and return with cluster as a regular column
|
|
2366
|
+
return consensus_report.sort_values(by='cluster')
|
|
2367
|
+
|
|
2368
|
+
def feature_annotations_table(
|
|
2369
|
+
self,
|
|
2370
|
+
molecular_metadata=None,
|
|
2371
|
+
drop_unannotated=False,
|
|
2372
|
+
report_best_only=False
|
|
2373
|
+
):
|
|
2374
|
+
"""Generate a comprehensive annotation table for all loaded mass features across samples.
|
|
2375
|
+
|
|
2376
|
+
This method consolidates MS1 molecular formula assignments and MS2 spectral
|
|
2377
|
+
search results for all mass features across all samples in the collection.
|
|
2378
|
+
Only includes representative mass features (one per cluster per sample).
|
|
2379
|
+
|
|
2380
|
+
Parameters
|
|
2381
|
+
----------
|
|
2382
|
+
molecular_metadata : dict, optional
|
|
2383
|
+
Dictionary of MolecularMetadata objects, keyed by metabref_mol_id.
|
|
2384
|
+
Required for including molecular metadata in MS2 annotations.
|
|
2385
|
+
Default is None.
|
|
2386
|
+
drop_unannotated : bool, optional
|
|
2387
|
+
If True, drops rows where all annotation columns (everything except
|
|
2388
|
+
cluster, MS2 Spectrum, and representative_sample) are NaN.
|
|
2389
|
+
Default is False.
|
|
2390
|
+
report_best_only : bool, optional
|
|
2391
|
+
If True, only includes the best MS2 annotation per mass feature based on confidence score.
|
|
2392
|
+
Default is False, which includes all MS2 annotations for each mass feature.
|
|
2393
|
+
|
|
2394
|
+
Returns
|
|
2395
|
+
-------
|
|
2396
|
+
pd.DataFrame
|
|
2397
|
+
Consolidated annotation report with columns including:
|
|
2398
|
+
- cluster: cluster ID
|
|
2399
|
+
- sample_name: sample name
|
|
2400
|
+
- sample_id: sample ID
|
|
2401
|
+
- Mass Feature ID: mass feature ID within the sample
|
|
2402
|
+
- Mass feature attributes (mz, scan_time, intensity, etc.)
|
|
2403
|
+
- MS1 annotations (if molecular_formula_search was run)
|
|
2404
|
+
- MS2 annotations (if ms2_spectral_search was run)
|
|
2405
|
+
|
|
2406
|
+
Notes
|
|
2407
|
+
-----
|
|
2408
|
+
This method uses the standard LCMSMetabolomicsExport.to_report() workflow
|
|
2409
|
+
for each sample, then consolidates all results and adds cluster information.
|
|
2410
|
+
|
|
2411
|
+
Only mass features that are loaded in each sample's mass_features dict
|
|
2412
|
+
are included (typically the representative features if load_representatives
|
|
2413
|
+
was used in process_consensus_features).
|
|
2414
|
+
|
|
2415
|
+
Raises
|
|
2416
|
+
------
|
|
2417
|
+
ValueError
|
|
2418
|
+
If no representative features have been loaded. Call process_consensus_features
|
|
2419
|
+
with load_representatives=True first.
|
|
2420
|
+
ValueError
|
|
2421
|
+
If no samples with loaded mass features are found in the collection.
|
|
2422
|
+
"""
|
|
2423
|
+
from corems.mass_spectra.output.export import LCMSMetabolomicsExport
|
|
2424
|
+
import warnings
|
|
2425
|
+
|
|
2426
|
+
# Check if representative features have been loaded
|
|
2427
|
+
# Count samples with mass features loaded
|
|
2428
|
+
samples_with_features = sum(
|
|
2429
|
+
1 for lcms_obj in self
|
|
2430
|
+
if hasattr(lcms_obj, 'mass_features') and len(lcms_obj.mass_features) > 0
|
|
2431
|
+
)
|
|
2432
|
+
|
|
2433
|
+
if samples_with_features == 0:
|
|
2434
|
+
raise ValueError(
|
|
2435
|
+
"No representative mass features have been loaded into individual samples. "
|
|
2436
|
+
"Call process_consensus_features() with load_representatives=True before "
|
|
2437
|
+
"calling feature_annotations_table()."
|
|
2438
|
+
)
|
|
2439
|
+
|
|
2440
|
+
# Collect reports from all samples
|
|
2441
|
+
all_sample_reports = []
|
|
2442
|
+
has_any_ms2_annotations = False
|
|
2443
|
+
|
|
2444
|
+
for sample_id, lcms_obj in enumerate(self):
|
|
2445
|
+
# Skip samples with no loaded mass features
|
|
2446
|
+
if not hasattr(lcms_obj, 'mass_features') or len(lcms_obj.mass_features) == 0:
|
|
2447
|
+
continue
|
|
2448
|
+
|
|
2449
|
+
sample_name = self.samples[sample_id]
|
|
2450
|
+
|
|
2451
|
+
# Create exporter and generate report using standard workflow
|
|
2452
|
+
# Suppress individual warnings - we'll warn at collection level if needed
|
|
2453
|
+
exporter = LCMSMetabolomicsExport("temp", lcms_obj)
|
|
2454
|
+
sample_report = exporter.to_report(molecular_metadata=molecular_metadata, suppress_warnings=True)
|
|
2455
|
+
|
|
2456
|
+
# Check if this sample has any MS2 annotations
|
|
2457
|
+
ms2_cols = [col for col in sample_report.columns if 'Entropy Similarity' in col or 'spectral_similarity' in col.lower()]
|
|
2458
|
+
if ms2_cols and sample_report[ms2_cols].notna().any().any():
|
|
2459
|
+
has_any_ms2_annotations = True
|
|
2460
|
+
|
|
2461
|
+
# Add sample information
|
|
2462
|
+
sample_report['representative_sample'] = sample_name
|
|
2463
|
+
sample_report['sample_id'] = sample_id
|
|
2464
|
+
|
|
2465
|
+
# Get cluster information from the mass_features_dataframe
|
|
2466
|
+
# Build coll_mf_id for each row to look up cluster
|
|
2467
|
+
sample_report['coll_mf_id'] = sample_report['sample_id'].astype(str) + "_" + sample_report['Mass Feature ID'].astype(str)
|
|
2468
|
+
|
|
2469
|
+
# Get cluster from mass_features_dataframe
|
|
2470
|
+
if self.mass_features_dataframe is not None and 'cluster' in self.mass_features_dataframe.columns:
|
|
2471
|
+
mf_df = self.mass_features_dataframe.reset_index()
|
|
2472
|
+
cluster_lookup = mf_df.set_index('coll_mf_id')['cluster'].to_dict()
|
|
2473
|
+
sample_report['cluster'] = sample_report['coll_mf_id'].map(cluster_lookup)
|
|
2474
|
+
else:
|
|
2475
|
+
sample_report['cluster'] = None
|
|
2476
|
+
|
|
2477
|
+
# Drop temporary coll_mf_id column
|
|
2478
|
+
sample_report = sample_report.drop(columns=['coll_mf_id'])
|
|
2479
|
+
|
|
2480
|
+
all_sample_reports.append(sample_report)
|
|
2481
|
+
|
|
2482
|
+
# Combine all sample reports
|
|
2483
|
+
if len(all_sample_reports) == 0:
|
|
2484
|
+
raise ValueError("No samples with loaded mass features found in collection")
|
|
2485
|
+
|
|
2486
|
+
collection_report = pd.concat(all_sample_reports, ignore_index=True)
|
|
2487
|
+
|
|
2488
|
+
# Warn only if NO samples in the collection have MS2 annotations
|
|
2489
|
+
if not has_any_ms2_annotations:
|
|
2490
|
+
warnings.warn(
|
|
2491
|
+
"No MS2 annotations found across any samples in collection, were MS2 spectra added and searched against a database?",
|
|
2492
|
+
UserWarning,
|
|
2493
|
+
)
|
|
2494
|
+
|
|
2495
|
+
# Reorder columns to match specified order
|
|
2496
|
+
desired_cols = [
|
|
2497
|
+
'cluster',
|
|
2498
|
+
'Isotopologue Type',
|
|
2499
|
+
'Is Largest Ion after Deconvolution',
|
|
2500
|
+
'MS2 Spectrum',
|
|
2501
|
+
'Calculated m/z',
|
|
2502
|
+
'm/z Error (ppm)',
|
|
2503
|
+
'm/z Error Score',
|
|
2504
|
+
'Isotopologue Similarity',
|
|
2505
|
+
'Confidence Score',
|
|
2506
|
+
'Ion Formula',
|
|
2507
|
+
'Ion Type',
|
|
2508
|
+
'Molecular Formula',
|
|
2509
|
+
'inchikey',
|
|
2510
|
+
'name',
|
|
2511
|
+
'ref_ms_id',
|
|
2512
|
+
'Entropy Similarity',
|
|
2513
|
+
'Library mzs in Query (fraction)',
|
|
2514
|
+
'Spectra with Annotation (n)',
|
|
2515
|
+
'representative_sample'
|
|
2516
|
+
]
|
|
2517
|
+
|
|
2518
|
+
# Include only desired columns that exist, maintaining order
|
|
2519
|
+
cols = [col for col in desired_cols if col in collection_report.columns]
|
|
2520
|
+
collection_report = collection_report[cols]
|
|
2521
|
+
|
|
2522
|
+
# Optionally drop rows without any annotations
|
|
2523
|
+
if drop_unannotated:
|
|
2524
|
+
# Columns to exclude from the "all NA" check
|
|
2525
|
+
exclude_cols = ['cluster', 'MS2 Spectrum', 'representative_sample']
|
|
2526
|
+
# Get annotation columns (everything except the excluded ones)
|
|
2527
|
+
annot_cols = [col for col in collection_report.columns if col not in exclude_cols]
|
|
2528
|
+
# Keep rows where at least one annotation column is not NA
|
|
2529
|
+
if len(annot_cols) > 0:
|
|
2530
|
+
collection_report = collection_report[collection_report[annot_cols].notna().any(axis=1)]
|
|
2531
|
+
|
|
2532
|
+
# Sort by cluster, then by annotation quality
|
|
2533
|
+
sort_cols = ['cluster']
|
|
2534
|
+
if 'Entropy Similarity' in collection_report.columns:
|
|
2535
|
+
sort_cols.extend(['Entropy Similarity', 'Confidence Score'])
|
|
2536
|
+
collection_report = collection_report.sort_values(
|
|
2537
|
+
by=sort_cols,
|
|
2538
|
+
ascending=[True, False, False]
|
|
2539
|
+
)
|
|
2540
|
+
elif 'Confidence Score' in collection_report.columns:
|
|
2541
|
+
sort_cols.append('Confidence Score')
|
|
2542
|
+
collection_report = collection_report.sort_values(
|
|
2543
|
+
by=sort_cols,
|
|
2544
|
+
ascending=[True, False]
|
|
2545
|
+
)
|
|
2546
|
+
else:
|
|
2547
|
+
collection_report = collection_report.sort_values(by=sort_cols)
|
|
2548
|
+
|
|
2549
|
+
if report_best_only:
|
|
2550
|
+
# Keep only the best annotation per cluster based on the first annotation column available
|
|
2551
|
+
if 'Entropy Similarity' in collection_report.columns:
|
|
2552
|
+
best_annot_col = 'Entropy Similarity'
|
|
2553
|
+
elif 'Confidence Score' in collection_report.columns:
|
|
2554
|
+
best_annot_col = 'Confidence Score'
|
|
2555
|
+
else:
|
|
2556
|
+
best_annot_col = None
|
|
2557
|
+
|
|
2558
|
+
if best_annot_col is not None:
|
|
2559
|
+
collection_report = collection_report.sort_values(by=['cluster', best_annot_col], ascending=[True, False])
|
|
2560
|
+
collection_report = collection_report.drop_duplicates(subset=['cluster'], keep='first')
|
|
2561
|
+
|
|
2562
|
+
return collection_report
|
|
2563
|
+
|
|
2564
|
+
@property
|
|
2565
|
+
def parameters(self):
|
|
2566
|
+
"""
|
|
2567
|
+
LCMSCollectionParameters : The parameters used for the LCMS collection.
|
|
2568
|
+
"""
|
|
2569
|
+
return self._parameters
|
|
2570
|
+
|
|
2571
|
+
@parameters.setter
|
|
2572
|
+
def parameters(self, paramsinstance):
|
|
2573
|
+
"""
|
|
2574
|
+
Sets the parameters used for the LCMS analysis collection.
|
|
2575
|
+
|
|
2576
|
+
Parameters
|
|
2577
|
+
-----------
|
|
2578
|
+
paramsinstance : LCMSCollectionParameters
|
|
2579
|
+
The parameters used for the LC-MS analysis.
|
|
2580
|
+
"""
|
|
2581
|
+
self._parameters = paramsinstance
|
|
2582
|
+
|
|
2583
|
+
@property
|
|
2584
|
+
def mass_features_dataframe(self):
|
|
2585
|
+
self._check_mass_features_df()
|
|
2586
|
+
return self._combined_mass_features
|
|
2587
|
+
|
|
2588
|
+
@mass_features_dataframe.setter
|
|
2589
|
+
def mass_features_dataframe(self, df):
|
|
2590
|
+
# Check that the dataframe has the expected columns
|
|
2591
|
+
expected_cols = ["sample_name", "sample_id", "mz", "scan_time"]
|
|
2592
|
+
if not all([col in df.columns for col in expected_cols]):
|
|
2593
|
+
raise ValueError(f"Expected columns not found in dataframe: {expected_cols}")
|
|
2594
|
+
|
|
2595
|
+
# Check that coll_mf_id is the index and it is unique
|
|
2596
|
+
if df.index.name != "coll_mf_id":
|
|
2597
|
+
raise ValueError("coll_mf_id must be the index of the dataframe")
|
|
2598
|
+
if not df.index.is_unique:
|
|
2599
|
+
raise ValueError("coll_mf_id must be unique")
|
|
2600
|
+
self._combined_mass_features = df
|
|
2601
|
+
|
|
2602
|
+
@property
|
|
2603
|
+
def induced_mass_features_dataframe(self):
|
|
2604
|
+
self._check_mass_features_df(induced_features = True)
|
|
2605
|
+
if self._combined_induced_mass_features is not None and len(self._combined_induced_mass_features) > 0:
|
|
2606
|
+
# The cluster column is extracted from mf_id in _prepare_lcms_mass_features_for_combination
|
|
2607
|
+
# mf_id format for induced features: c{cluster}_{index}_i
|
|
2608
|
+
pass
|
|
2609
|
+
return self._combined_induced_mass_features
|
|
2610
|
+
|
|
2611
|
+
@induced_mass_features_dataframe.setter
|
|
2612
|
+
def induced_mass_features_dataframe(self, df):
|
|
2613
|
+
# Check that the dataframe has the expected columns
|
|
2614
|
+
expected_cols = ["sample_name", "sample_id", "mz", "scan_time"]
|
|
2615
|
+
if not all([col in df.columns for col in expected_cols]):
|
|
2616
|
+
raise ValueError(f"Expected columns not found in dataframe: {expected_cols}")
|
|
2617
|
+
|
|
2618
|
+
# Check that coll_mf_id is the index and it is unique
|
|
2619
|
+
if df.index.name != "coll_mf_id":
|
|
2620
|
+
raise ValueError("coll_mf_id must be the index of the dataframe")
|
|
2621
|
+
if not df.index.is_unique:
|
|
2622
|
+
raise ValueError("coll_mf_id must be unique")
|
|
2623
|
+
self._combined_induced_mass_features = df
|
|
2624
|
+
|
|
2625
|
+
@property
|
|
2626
|
+
def cluster_summary_dataframe(self):
|
|
2627
|
+
return self.summarize_clusters()
|
|
2628
|
+
|
|
2629
|
+
@property
|
|
2630
|
+
def samples(self):
|
|
2631
|
+
manifest_df = self.manifest_dataframe
|
|
2632
|
+
# order by batch, then by order
|
|
2633
|
+
manifest_df = manifest_df.sort_values(by=['batch', 'order'])
|
|
2634
|
+
return manifest_df.index.tolist()
|
|
2635
|
+
|
|
2636
|
+
@property
|
|
2637
|
+
def manifest(self):
|
|
2638
|
+
return self._manifest_dict
|
|
2639
|
+
|
|
2640
|
+
@property
|
|
2641
|
+
def manifest_dataframe(self):
|
|
2642
|
+
return pd.DataFrame(self._manifest_dict).T
|
|
2643
|
+
|
|
2644
|
+
@property
|
|
2645
|
+
def raw_files(self):
|
|
2646
|
+
"""Returns a list of raw files in the collection."""
|
|
2647
|
+
return [x.raw_file_location for x in self]
|
|
2648
|
+
|
|
2649
|
+
@property
|
|
2650
|
+
def rt_alignments(self):
|
|
2651
|
+
"""Returns a dictionary of retention time alignments for the collection."""
|
|
2652
|
+
if self.rt_aligned:
|
|
2653
|
+
_rt_alignments = {}
|
|
2654
|
+
# Construct a dictionary of aligned retention times (stored on each LCMS object within the collection, not the collection itself)
|
|
2655
|
+
for i, lcms_obj in enumerate(self):
|
|
2656
|
+
aligned_times = [x for k, x in sorted(lcms_obj._scan_info["scan_time_aligned"].items())]
|
|
2657
|
+
_rt_alignments[i] = aligned_times
|
|
2658
|
+
return _rt_alignments
|
|
2659
|
+
else:
|
|
2660
|
+
return None
|
|
2661
|
+
|
|
2662
|
+
@property
|
|
2663
|
+
def cluster_feature_dictionary(self):
|
|
2664
|
+
"""Generates a dictionary with clusters for keys and mass feature IDs as entries"""
|
|
2665
|
+
df = self.mass_features_dataframe
|
|
2666
|
+
cluster_dict = df.groupby('cluster').apply(lambda x: x.index.tolist()).to_dict()
|
|
2667
|
+
return cluster_dict
|
|
2668
|
+
|
|
2669
|
+
def get_eics_for_cluster(self, cluster_id):
|
|
2670
|
+
"""
|
|
2671
|
+
Retrieve all EICs for mass features in a specific cluster across all samples.
|
|
2672
|
+
|
|
2673
|
+
Returns a dictionary mapping sample names to EIC_Data objects for the given cluster.
|
|
2674
|
+
Useful for visualizing and comparing chromatographic peaks across samples.
|
|
2675
|
+
|
|
2676
|
+
Parameters
|
|
2677
|
+
----------
|
|
2678
|
+
cluster_id : int
|
|
2679
|
+
The cluster ID to retrieve EICs for
|
|
2680
|
+
|
|
2681
|
+
Returns
|
|
2682
|
+
-------
|
|
2683
|
+
dict
|
|
2684
|
+
Dictionary with structure: {sample_name: EIC_Data object}
|
|
2685
|
+
Only includes samples where the EIC was loaded.
|
|
2686
|
+
|
|
2687
|
+
Examples
|
|
2688
|
+
--------
|
|
2689
|
+
>>> # Load EICs first
|
|
2690
|
+
>>> collection.process_consensus_features(gather_eics=True, ...)
|
|
2691
|
+
>>>
|
|
2692
|
+
>>> # Get all EICs for cluster 5
|
|
2693
|
+
>>> eics = collection.get_eics_for_cluster(5)
|
|
2694
|
+
>>> for sample_name, eic_data in eics.items():
|
|
2695
|
+
... print(f"{sample_name}: {len(eic_data.scans)} scans")
|
|
2696
|
+
|
|
2697
|
+
Notes
|
|
2698
|
+
-----
|
|
2699
|
+
Requires that EICs have been loaded using gather_eics=True in
|
|
2700
|
+
process_consensus_features() or manually loaded via LoadEICsOperation.
|
|
2701
|
+
"""
|
|
2702
|
+
eics_by_sample = {}
|
|
2703
|
+
|
|
2704
|
+
# Iterate through all samples
|
|
2705
|
+
for sample_id, sample in enumerate(self):
|
|
2706
|
+
sample_name = self.samples[sample_id]
|
|
2707
|
+
|
|
2708
|
+
# Check if sample has EICs loaded
|
|
2709
|
+
if not hasattr(sample, 'eics') or not sample.eics:
|
|
2710
|
+
continue
|
|
2711
|
+
|
|
2712
|
+
# Find mass features in this cluster for this sample
|
|
2713
|
+
# Check both regular and induced mass features
|
|
2714
|
+
for mf in list(sample.mass_features.values()) + list(sample.induced_mass_features.values()):
|
|
2715
|
+
if hasattr(mf, 'cluster_index') and mf.cluster_index == cluster_id:
|
|
2716
|
+
# Get the EIC for this mass feature's m/z
|
|
2717
|
+
if mf.mz in sample.eics:
|
|
2718
|
+
eics_by_sample[sample_name] = sample.eics[mf.mz]
|
|
2719
|
+
break # Found the EIC for this sample, move to next sample
|
|
2720
|
+
|
|
2721
|
+
return eics_by_sample
|