CoreMS 4.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. corems/__init__.py +63 -0
  2. corems/chroma_peak/__init__.py +0 -0
  3. corems/chroma_peak/calc/ChromaPeakCalc.py +480 -0
  4. corems/chroma_peak/calc/__init__.py +0 -0
  5. corems/chroma_peak/calc/subset.py +196 -0
  6. corems/chroma_peak/factory/__init__.py +0 -0
  7. corems/chroma_peak/factory/chroma_peak_classes.py +1178 -0
  8. corems/encapsulation/__init__.py +0 -0
  9. corems/encapsulation/constant.py +1283 -0
  10. corems/encapsulation/factory/__init__.py +0 -0
  11. corems/encapsulation/factory/parameters.py +392 -0
  12. corems/encapsulation/factory/processingSetting.py +1244 -0
  13. corems/encapsulation/input/__init__.py +0 -0
  14. corems/encapsulation/input/parameter_from_json.py +579 -0
  15. corems/encapsulation/output/__init__.py +0 -0
  16. corems/encapsulation/output/parameter_to_dict.py +142 -0
  17. corems/encapsulation/output/parameter_to_json.py +332 -0
  18. corems/mass_spectra/__init__.py +0 -0
  19. corems/mass_spectra/calc/GC_Calc.py +84 -0
  20. corems/mass_spectra/calc/GC_Deconvolution.py +558 -0
  21. corems/mass_spectra/calc/GC_RI_Calibration.py +44 -0
  22. corems/mass_spectra/calc/MZSearch.py +249 -0
  23. corems/mass_spectra/calc/SignalProcessing.py +656 -0
  24. corems/mass_spectra/calc/__init__.py +0 -0
  25. corems/mass_spectra/calc/lc_calc.py +5790 -0
  26. corems/mass_spectra/calc/lc_calc_operations.py +1127 -0
  27. corems/mass_spectra/factory/GC_Class.py +791 -0
  28. corems/mass_spectra/factory/__init__.py +0 -0
  29. corems/mass_spectra/factory/chromat_data.py +51 -0
  30. corems/mass_spectra/factory/lc_class.py +2721 -0
  31. corems/mass_spectra/input/__init__.py +0 -0
  32. corems/mass_spectra/input/andiNetCDF.py +200 -0
  33. corems/mass_spectra/input/boosterHDF5.py +216 -0
  34. corems/mass_spectra/input/brukerSolarix.py +153 -0
  35. corems/mass_spectra/input/brukerSolarix_utils.py +73 -0
  36. corems/mass_spectra/input/corems_hdf5.py +1710 -0
  37. corems/mass_spectra/input/massList.py +133 -0
  38. corems/mass_spectra/input/mzml.py +668 -0
  39. corems/mass_spectra/input/parserbase.py +239 -0
  40. corems/mass_spectra/input/rawFileReader.py +1839 -0
  41. corems/mass_spectra/output/__init__.py +0 -0
  42. corems/mass_spectra/output/export.py +2800 -0
  43. corems/mass_spectrum/__init__.py +0 -0
  44. corems/mass_spectrum/calc/AutoRecalibration.py +237 -0
  45. corems/mass_spectrum/calc/Calibration.py +602 -0
  46. corems/mass_spectrum/calc/CalibrationCalc.py +253 -0
  47. corems/mass_spectrum/calc/KendrickGroup.py +239 -0
  48. corems/mass_spectrum/calc/MassErrorPrediction.py +436 -0
  49. corems/mass_spectrum/calc/MassSpectrumCalc.py +303 -0
  50. corems/mass_spectrum/calc/MeanResolvingPowerFilter.py +212 -0
  51. corems/mass_spectrum/calc/NoiseCalc.py +371 -0
  52. corems/mass_spectrum/calc/NoiseCalc_Bayes.py +93 -0
  53. corems/mass_spectrum/calc/PeakPicking.py +994 -0
  54. corems/mass_spectrum/calc/__init__.py +0 -0
  55. corems/mass_spectrum/factory/MassSpectrumClasses.py +1753 -0
  56. corems/mass_spectrum/factory/__init__.py +0 -0
  57. corems/mass_spectrum/input/__init__.py +0 -0
  58. corems/mass_spectrum/input/baseClass.py +531 -0
  59. corems/mass_spectrum/input/boosterHDF5.py +161 -0
  60. corems/mass_spectrum/input/coremsHDF5.py +475 -0
  61. corems/mass_spectrum/input/massList.py +402 -0
  62. corems/mass_spectrum/input/numpyArray.py +133 -0
  63. corems/mass_spectrum/output/__init__.py +0 -0
  64. corems/mass_spectrum/output/export.py +841 -0
  65. corems/molecular_formula/__init__.py +0 -0
  66. corems/molecular_formula/calc/MolecularFormulaCalc.py +885 -0
  67. corems/molecular_formula/calc/__init__.py +0 -0
  68. corems/molecular_formula/factory/MolecularFormulaFactory.py +895 -0
  69. corems/molecular_formula/factory/__init__.py +0 -0
  70. corems/molecular_formula/input/__init__.py +0 -0
  71. corems/molecular_formula/input/masslist_ref.py +355 -0
  72. corems/molecular_id/__init__.py +0 -0
  73. corems/molecular_id/calc/ClusterFilter.py +251 -0
  74. corems/molecular_id/calc/MolecularFilter.py +122 -0
  75. corems/molecular_id/calc/SpectralSimilarity.py +632 -0
  76. corems/molecular_id/calc/__init__.py +0 -0
  77. corems/molecular_id/calc/math_distance.py +1637 -0
  78. corems/molecular_id/data/FAMES_REF.msp +980 -0
  79. corems/molecular_id/data/PNNLMetV20191015.msp +157267 -0
  80. corems/molecular_id/factory/EI_SQL.py +650 -0
  81. corems/molecular_id/factory/MolecularLookupTable.py +914 -0
  82. corems/molecular_id/factory/__init__.py +0 -0
  83. corems/molecular_id/factory/classification.py +884 -0
  84. corems/molecular_id/factory/lipid_molecular_metadata.py +50 -0
  85. corems/molecular_id/factory/molecularSQL.py +827 -0
  86. corems/molecular_id/factory/spectrum_search_results.py +119 -0
  87. corems/molecular_id/input/__init__.py +0 -0
  88. corems/molecular_id/input/nistMSI.py +148 -0
  89. corems/molecular_id/search/__init__.py +0 -0
  90. corems/molecular_id/search/compoundSearch.py +214 -0
  91. corems/molecular_id/search/database_interfaces.py +1527 -0
  92. corems/molecular_id/search/findOxygenPeaks.py +330 -0
  93. corems/molecular_id/search/lcms_spectral_search.py +348 -0
  94. corems/molecular_id/search/molecularFormulaSearch.py +1117 -0
  95. corems/molecular_id/search/priorityAssignment.py +723 -0
  96. corems/ms_peak/__init__.py +0 -0
  97. corems/ms_peak/calc/MSPeakCalc.py +1010 -0
  98. corems/ms_peak/calc/__init__.py +0 -0
  99. corems/ms_peak/factory/MSPeakClasses.py +542 -0
  100. corems/ms_peak/factory/__init__.py +0 -0
  101. corems/transient/__init__.py +0 -0
  102. corems/transient/calc/TransientCalc.py +362 -0
  103. corems/transient/calc/__init__.py +0 -0
  104. corems/transient/factory/TransientClasses.py +457 -0
  105. corems/transient/factory/__init__.py +0 -0
  106. corems/transient/input/__init__.py +0 -0
  107. corems/transient/input/brukerSolarix.py +461 -0
  108. corems/transient/input/midasDatFile.py +172 -0
  109. corems-4.0.0.dist-info/METADATA +475 -0
  110. corems-4.0.0.dist-info/RECORD +171 -0
  111. corems-4.0.0.dist-info/WHEEL +5 -0
  112. corems-4.0.0.dist-info/licenses/LICENSE +22 -0
  113. corems-4.0.0.dist-info/top_level.txt +4 -0
  114. examples/archive/scripts/CoreMS_tutorial.py +94 -0
  115. examples/archive/scripts/DI HR-MS Halogens Bruker.py +196 -0
  116. examples/archive/scripts/DI HR-MS MassList.py +385 -0
  117. examples/archive/scripts/GC-MS MetabRef.py +213 -0
  118. examples/archive/scripts/GC-MS NetCDF.py +217 -0
  119. examples/archive/scripts/HR-MS Thermo Raw 21T.py +136 -0
  120. examples/archive/scripts/LC-ICPMS_metal_peaks.py +297 -0
  121. examples/archive/scripts/LCMS-Thermo.py +460 -0
  122. examples/archive/scripts/LCMS_isotopes.py +283 -0
  123. examples/archive/scripts/MSParams_example.py +21 -0
  124. examples/archive/scripts/Molecular Formula Data Aggreation.py +84 -0
  125. examples/archive/scripts/Single Mz Search.py +69 -0
  126. examples/test_notebooks.py +145 -0
  127. ext_lib/ChemstationMSFileReader.dll +0 -0
  128. ext_lib/ChemstationMSFileReader.xml +126 -0
  129. ext_lib/RawFileReaderLicense.doc +0 -0
  130. ext_lib/ThermoFisher.CommonCore.BackgroundSubtraction.dll +0 -0
  131. ext_lib/ThermoFisher.CommonCore.BackgroundSubtraction.xml +2307 -0
  132. ext_lib/ThermoFisher.CommonCore.Data.dll +0 -0
  133. ext_lib/ThermoFisher.CommonCore.Data.xml +28974 -0
  134. ext_lib/ThermoFisher.CommonCore.MassPrecisionEstimator.dll +0 -0
  135. ext_lib/ThermoFisher.CommonCore.MassPrecisionEstimator.xml +241 -0
  136. ext_lib/ThermoFisher.CommonCore.RawFileReader.dll +0 -0
  137. ext_lib/ThermoFisher.CommonCore.RawFileReader.xml +31174 -0
  138. ext_lib/__init__.py +0 -0
  139. ext_lib/dotnet/OpenMcdf.Extensions.dll +0 -0
  140. ext_lib/dotnet/OpenMcdf.dll +0 -0
  141. ext_lib/dotnet/OpenMcdf.xml +1154 -0
  142. ext_lib/dotnet/System.IO.FileSystem.AccessControl.dll +0 -0
  143. ext_lib/dotnet/System.IO.FileSystem.AccessControl.xml +506 -0
  144. ext_lib/dotnet/System.Security.AccessControl.dll +0 -0
  145. ext_lib/dotnet/System.Security.AccessControl.xml +2043 -0
  146. ext_lib/dotnet/System.Security.Principal.Windows.dll +0 -0
  147. ext_lib/dotnet/System.Security.Principal.Windows.xml +1011 -0
  148. ext_lib/dotnet/ThermoFisher.CommonCore.BackgroundSubtraction.dll +0 -0
  149. ext_lib/dotnet/ThermoFisher.CommonCore.BackgroundSubtraction.xml +2307 -0
  150. ext_lib/dotnet/ThermoFisher.CommonCore.Data.dll +0 -0
  151. ext_lib/dotnet/ThermoFisher.CommonCore.Data.xml +29148 -0
  152. ext_lib/dotnet/ThermoFisher.CommonCore.MassPrecisionEstimator.dll +0 -0
  153. ext_lib/dotnet/ThermoFisher.CommonCore.MassPrecisionEstimator.xml +241 -0
  154. ext_lib/dotnet/ThermoFisher.CommonCore.RawFileReader.dll +0 -0
  155. ext_lib/dotnet/ThermoFisher.CommonCore.RawFileReader.xml +31492 -0
  156. ext_lib/version +18 -0
  157. support_code/atom_parsers/AtomsDescription.py +359 -0
  158. support_code/atom_parsers/CreateAtomsDescription.py +56 -0
  159. support_code/nmdc/filefinder.py +90 -0
  160. support_code/nmdc/lipidomics/lipidomics_workflow.py +748 -0
  161. support_code/nmdc/lipidomics/manifest_examples.py +49 -0
  162. support_code/nmdc/metabolomics/gcms_workflow.py +158 -0
  163. support_code/nmdc/metabolomics/lcms_metabolomics_targeted_search.py +59 -0
  164. support_code/nmdc/metabolomics/lcms_metabolomics_workflow.py +248 -0
  165. support_code/nmdc/metabolomics/metabolomics_collection.py +628 -0
  166. support_code/nmdc/metadata/dms_api.py +42 -0
  167. support_code/nmdc/nom/archived_scripts/nmdc_metadata_gen.py +288 -0
  168. support_code/nmdc/nom/archived_scripts/nom_grow_workflow.py +209 -0
  169. support_code/nmdc/nom/nom_workflow.py +312 -0
  170. support_code/windows_only_importers/BrukerCompassXtract.py +180 -0
  171. support_code/windows_only_importers/ThermoMSFileReader.py +405 -0
@@ -0,0 +1,914 @@
1
+ __author__ = "Yuri E. Corilo"
2
+ __date__ = "Jul 02, 2019"
3
+
4
+ import contextlib
5
+ import cProfile
6
+ import io
7
+ import itertools
8
+ import json
9
+ import multiprocessing
10
+ import pstats
11
+ from copy import deepcopy
12
+ from typing import Dict
13
+
14
+ from sqlalchemy import create_engine, func
15
+ from sqlalchemy.orm import sessionmaker
16
+ from tqdm import tqdm
17
+
18
+ from corems import chunks, timeit
19
+ from corems.encapsulation.constant import Atoms
20
+ from corems.encapsulation.factory.parameters import MSParameters
21
+ from corems.encapsulation.factory.processingSetting import MolecularLookupDictSettings
22
+ from corems.molecular_id.factory.molecularSQL import (
23
+ CarbonHydrogen,
24
+ HeteroAtoms,
25
+ MolecularFormulaLink,
26
+ MolForm_SQL,
27
+ )
28
+
29
+
30
+ @contextlib.contextmanager
31
+ def profiled():
32
+ """A context manager for profiling."""
33
+ pr = cProfile.Profile()
34
+ pr.enable()
35
+ yield
36
+ pr.disable()
37
+ s = io.StringIO()
38
+ ps = pstats.Stats(pr, stream=s).sort_stats("cumulative")
39
+ ps.print_stats()
40
+ # uncomment this to see who's calling what
41
+ # ps.print_callers()
42
+ print(s.getvalue())
43
+
44
+
45
+ def insert_database_worker(args):
46
+ """Inserts data into the database."""
47
+ results, url = args
48
+
49
+ if not url:
50
+ url = "sqlite:///db/molformulas.sqlite"
51
+
52
+ if url[0:6] == "sqlite":
53
+ engine = create_engine(url, echo=False)
54
+ else:
55
+ engine = create_engine(url, echo=False, isolation_level="AUTOCOMMIT")
56
+
57
+ session_factory = sessionmaker(bind=engine)
58
+ session = session_factory()
59
+ insert_query = MolecularFormulaLink.__table__.insert().values(results)
60
+ session.execute(insert_query)
61
+ session.commit()
62
+ session.close()
63
+ engine.dispose()
64
+
65
+
66
+ class MolecularCombinations:
67
+ """A class for generating molecular formula combinations.
68
+
69
+ Parameters
70
+ ----------
71
+ molecular_search_settings : object
72
+ An object containing user-defined settings.
73
+
74
+ Attributes
75
+ ----------
76
+ sql_db : MolForm_SQL
77
+ The SQLite database object.
78
+ len_existing_classes : int
79
+ The number of existing classes in the SQLite database.
80
+ odd_ch_id : list
81
+ A list of odd carbon and hydrogen atom IDs.
82
+ odd_ch_dict : list
83
+ A list of odd carbon and hydrogen atom dictionaries.
84
+ odd_ch_mass : list
85
+ A list of odd carbon and hydrogen atom masses.
86
+ odd_ch_dbe : list
87
+ A list of odd carbon and hydrogen atom double bond equivalents.
88
+ even_ch_id : list
89
+ A list of even carbon and hydrogen atom IDs.
90
+ even_ch_dict : list
91
+ A list of even carbon and hydrogen atom dictionaries.
92
+ even_ch_mass : list
93
+ A list of even carbon and hydrogen atom masses.
94
+ even_ch_dbe : list
95
+ A list of even carbon and hydrogen atom double bond equivalents.
96
+
97
+ Methods
98
+ -------
99
+ * cProfile_worker(args)
100
+ A cProfile worker for the get_mol_formulas function.
101
+ * check_database_get_class_list(molecular_search_settings)
102
+ Checks if the database has all the classes, if not create the missing classes.
103
+ * get_carbonsHydrogens(settings, odd_even)
104
+ Retrieves carbon and hydrogen atoms from the molecular lookup table based on user-defined settings.
105
+ * add_carbonsHydrogens(settings, existing_classes_objs)
106
+ Adds carbon and hydrogen atoms to the molecular lookup table based on user-defined settings.
107
+ * runworker(molecular_search_settings)
108
+ Runs the molecular formula lookup table worker.
109
+ * get_classes_in_order(molecular_search_settings)
110
+ Gets the classes in order.
111
+ * sort_classes(atoms_in_order, combination_dict)
112
+ Sorts the classes in order.
113
+ * get_fixed_initial_number_of_hydrogen(min_h, odd_even)
114
+ Gets the fixed initial number of hydrogen atoms.
115
+ * calc_mz(datadict, class_mass=0)
116
+ Calculates the mass-to-charge ratio (m/z) of a molecular formula.
117
+ * calc_dbe_class(datadict)
118
+ Calculates the double bond equivalent (DBE) of a molecular formula.
119
+ * populate_combinations(classe_tuple, settings)
120
+ Populates the combinations.
121
+ * get_or_add(SomeClass, kw)
122
+ Gets or adds a class.
123
+ * get_mol_formulas(odd_even_tag, classe_tuple, settings)
124
+ Gets the molecular formulas.
125
+ * get_h_odd_or_even(class_dict)
126
+ Gets the hydrogen odd or even.
127
+ * get_total_halogen_atoms(class_dict)
128
+ Gets the total number of halogen atoms.
129
+ * get_total_hetero_valence(class_dict)
130
+ Gets the total valence of heteroatoms other than N, F, Cl, and Br
131
+ """
132
+
133
+ def __init__(self, sql_db=None):
134
+ if not sql_db:
135
+ self.sql_db = MolForm_SQL()
136
+ else:
137
+ self.sql_db = sql_db
138
+
139
+ def cProfile_worker(self, args):
140
+ """cProfile worker for the get_mol_formulas function"""
141
+ cProfile.runctx(
142
+ "self.get_mol_formulas(*args)",
143
+ globals(),
144
+ locals(),
145
+ "mf_database_cprofile.prof",
146
+ )
147
+
148
+ def check_database_get_class_list(self, molecular_search_settings):
149
+ """check if the database has all the classes, if not create the missing classes
150
+
151
+ Parameters
152
+ ----------
153
+ molecular_search_settings : object
154
+ An object containing user-defined settings.
155
+
156
+ Returns
157
+ -------
158
+ list
159
+ list of tuples with the class name and the class dictionary
160
+ """
161
+ all_class_to_create = []
162
+
163
+ classes_dict = self.get_classes_in_order(molecular_search_settings)
164
+
165
+ class_str_set = set(classes_dict.keys())
166
+
167
+ existing_classes_objs = self.sql_db.session.query(HeteroAtoms).distinct().all()
168
+
169
+ existing_classes_str = set([classe.name for classe in existing_classes_objs])
170
+
171
+ self.len_existing_classes = len(existing_classes_str)
172
+
173
+ class_to_create = class_str_set - existing_classes_str
174
+
175
+ class_count = len(existing_classes_objs)
176
+
177
+ data_classes = list()
178
+ for index, class_str in enumerate(class_to_create):
179
+ class_dict = classes_dict.get(class_str)
180
+ halogen_count = self.get_total_halogen_atoms(class_dict)
181
+ data_classes.append(
182
+ {
183
+ "name": class_str,
184
+ "id": class_count + index + 1,
185
+ "halogensCount": halogen_count,
186
+ }
187
+ )
188
+
189
+ # data_classes = [{"name":class_str, "id":class_count+ index + 1} for index, class_str in enumerate(class_to_create)]
190
+
191
+ if data_classes:
192
+ list_insert_chunks = chunks(data_classes, self.sql_db.chunks_count)
193
+ for insert_chunk in list_insert_chunks:
194
+ insert_query = HeteroAtoms.__table__.insert().values(insert_chunk)
195
+ self.sql_db.session.execute(insert_query)
196
+
197
+ for index, class_str in enumerate(class_to_create):
198
+ class_tuple = (
199
+ class_str,
200
+ classes_dict.get(class_str),
201
+ class_count + index + 1,
202
+ )
203
+
204
+ all_class_to_create.append(class_tuple)
205
+
206
+ return (
207
+ [(c_s, c_d) for c_s, c_d in classes_dict.items()],
208
+ all_class_to_create,
209
+ existing_classes_objs,
210
+ )
211
+
212
+ def get_carbonsHydrogens(self, settings, odd_even):
213
+ """Retrieve carbon and hydrogen atoms from the molecular lookup table based on user-defined settings.
214
+
215
+ Parameters
216
+ ----------
217
+ settings : object
218
+ An object containing user-defined settings.
219
+ odd_even : str
220
+ A string indicating whether to retrieve even or odd hydrogen atoms.
221
+
222
+ Returns
223
+ -------
224
+ list
225
+ A list of CarbonHydrogen objects that satisfy the specified conditions.
226
+ """
227
+ operator = "==" if odd_even == "even" else "!="
228
+ usedAtoms = settings.usedAtoms
229
+ user_min_c, user_max_c = usedAtoms.get("C")
230
+ user_min_h, user_max_h = usedAtoms.get("H")
231
+
232
+ return eval(
233
+ "self.sql_db.session.query(CarbonHydrogen).filter("
234
+ "CarbonHydrogen.C >= user_min_c,"
235
+ "CarbonHydrogen.H >= user_min_h,"
236
+ "CarbonHydrogen.C <= user_max_c,"
237
+ "CarbonHydrogen.H <= user_max_h,"
238
+ "CarbonHydrogen.H % 2" + operator + "0).all()"
239
+ )
240
+
241
+ def add_carbonsHydrogens(self, settings, existing_classes_objs):
242
+ """Add carbon and hydrogen atoms to the molecular lookup table based on user-defined settings.
243
+
244
+ Parameters
245
+ ----------
246
+ settings : object
247
+ An object containing user-defined settings.
248
+ existing_classes_objs : list
249
+ A list of HeteroAtoms objects.
250
+ """
251
+ usedAtoms = settings.usedAtoms
252
+
253
+ user_min_c, user_max_c = usedAtoms.get("C")
254
+ user_min_h, user_max_h = usedAtoms.get("H")
255
+
256
+ query_obj = self.sql_db.session.query(
257
+ func.max(CarbonHydrogen.C).label("max_c"),
258
+ func.min(CarbonHydrogen.C).label("min_c"),
259
+ func.max(CarbonHydrogen.H).label("max_h"),
260
+ func.min(CarbonHydrogen.H).label("min_h"),
261
+ )
262
+
263
+ database = query_obj.first()
264
+ if (
265
+ database.max_c == user_max_c
266
+ and database.min_c == user_min_c
267
+ and database.max_h == user_max_h
268
+ and database.min_h == user_min_h
269
+ ):
270
+ # all data is already available at the database
271
+ pass
272
+
273
+ else:
274
+ current_count = self.sql_db.session.query(CarbonHydrogen.C).count()
275
+
276
+ databaseCarbonHydrogen = self.sql_db.session.query(CarbonHydrogen).all()
277
+
278
+ userCarbon = set(range(user_min_c, user_max_c + 1))
279
+ userHydrogen = set(range(user_min_h, user_max_h + 1))
280
+
281
+ carbon_hydrogen_objs_database = {}
282
+ for obj in databaseCarbonHydrogen:
283
+ str_data = "C:{},H:{}".format(obj.C, obj.H)
284
+ carbon_hydrogen_objs_database[str_data] = str_data
285
+
286
+ carbon_hydrogen_objs_to_create = {"even": {}, "odd": {}}
287
+
288
+ list_ch_obj_to_add = list()
289
+ i = 0
290
+ for comb in itertools.product(userCarbon, userHydrogen):
291
+ C = comb[0]
292
+ H = comb[1]
293
+ data = {
294
+ "C": C,
295
+ "H": H,
296
+ }
297
+
298
+ data_insert = {
299
+ "C": C,
300
+ "H": H,
301
+ }
302
+
303
+ str_data = "C:{},H:{}".format(C, H)
304
+
305
+ if not str_data in carbon_hydrogen_objs_database.keys():
306
+ label = "even" if comb[1] % 2 == 0 else "odd"
307
+ data["mass"] = (C * Atoms.atomic_masses.get("C")) + (
308
+ H * Atoms.atomic_masses.get("H")
309
+ )
310
+ data["dbe"] = C - (H / 2) + 1
311
+ data["id"] = i + current_count + 1
312
+ data_insert["id"] = i + current_count + 1
313
+ i = i + 1
314
+ carbon_hydrogen_objs_to_create[label][str_data] = data
315
+
316
+ list_ch_obj_to_add.append(data_insert)
317
+
318
+ if list_ch_obj_to_add:
319
+ # insert carbon hydrogen objs
320
+ list_insert_chunks = chunks(
321
+ list_ch_obj_to_add, self.sql_db.chunks_count
322
+ )
323
+ for insert_chunk in list_insert_chunks:
324
+ insert_query = CarbonHydrogen.__table__.insert().values(
325
+ insert_chunk
326
+ )
327
+ self.sql_db.session.execute(insert_query)
328
+ self.sql_db.session.commit()
329
+
330
+ list_molecular_form = list()
331
+ for classe_obj in existing_classes_objs:
332
+ classe_dict = classe_obj.to_dict()
333
+ classe_mass = self.calc_mz(classe_dict)
334
+ classe_dbe = self.calc_dbe_class(classe_dict)
335
+
336
+ odd_even_label = self.get_h_odd_or_even(classe_dict)
337
+
338
+ ch_datalist = carbon_hydrogen_objs_to_create.get(
339
+ odd_even_label
340
+ ).values()
341
+
342
+ for ch_dict in ch_datalist:
343
+ mass = ch_dict.get("mass") + classe_mass
344
+ dbe = ch_dict.get("dbe") + classe_dbe
345
+
346
+ if settings.min_mz <= mass <= settings.max_mz:
347
+ if settings.min_dbe <= dbe <= settings.max_dbe:
348
+ list_molecular_form.append(
349
+ {
350
+ "heteroAtoms_id": classe_obj.id,
351
+ "carbonHydrogen_id": ch_dict.get("id"),
352
+ "mass": mass,
353
+ "DBE": dbe,
354
+ }
355
+ )
356
+
357
+ list_insert_chunks = chunks(
358
+ list_molecular_form, self.sql_db.chunks_count
359
+ )
360
+ for insert_chunk in list_insert_chunks:
361
+ insert_query = MolecularFormulaLink.__table__.insert().values(
362
+ insert_chunk
363
+ )
364
+ self.sql_db.session.execute(insert_query)
365
+ self.sql_db.session.commit()
366
+
367
+ @timeit(print_time=True)
368
+ def runworker(self, molecular_search_settings, **kwargs):
369
+ """Run the molecular formula lookup table worker.
370
+
371
+ Parameters
372
+ ----------
373
+ molecular_search_settings : object
374
+ An object containing user-defined settings.
375
+ kwargs : dict
376
+ A dictionary of keyword arguments.
377
+ Most notably, the print_time argument which is passed to the timeit decorator.
378
+
379
+ Returns
380
+ -------
381
+ list
382
+ A list of tuples with the class name and the class dictionary.
383
+
384
+
385
+ """
386
+ verbose = molecular_search_settings.verbose_processing
387
+
388
+ classes_list, class_to_create, existing_classes_objs = (
389
+ self.check_database_get_class_list(molecular_search_settings)
390
+ )
391
+
392
+ settings = MolecularLookupDictSettings()
393
+ settings.usedAtoms = deepcopy(molecular_search_settings.usedAtoms)
394
+ settings.url_database = molecular_search_settings.url_database
395
+ settings.db_jobs = molecular_search_settings.db_jobs
396
+
397
+ self.add_carbonsHydrogens(settings, existing_classes_objs)
398
+
399
+ if class_to_create:
400
+ settings = MolecularLookupDictSettings()
401
+ settings.usedAtoms = deepcopy(molecular_search_settings.usedAtoms)
402
+ settings.url_database = molecular_search_settings.url_database
403
+ settings.db_jobs = molecular_search_settings.db_jobs
404
+
405
+ self.sql_db.session.commit()
406
+ odd_ch_obj = self.get_carbonsHydrogens(settings, "odd")
407
+ self.odd_ch_id = [obj.id for obj in odd_ch_obj]
408
+ self.odd_ch_dict = [{"C": obj.C, "H": obj.H} for obj in odd_ch_obj]
409
+ self.odd_ch_mass = [obj.mass for obj in odd_ch_obj]
410
+ self.odd_ch_dbe = [obj.dbe for obj in odd_ch_obj]
411
+
412
+ even_ch_obj = self.get_carbonsHydrogens(settings, "even")
413
+ self.even_ch_id = [obj.id for obj in even_ch_obj]
414
+ self.even_ch_dict = [{"C": obj.C, "H": obj.H} for obj in even_ch_obj]
415
+ self.even_ch_mass = [obj.mass for obj in even_ch_obj]
416
+ self.even_ch_dbe = [obj.dbe for obj in even_ch_obj]
417
+
418
+ all_results = list()
419
+ for class_tuple in tqdm(class_to_create, disable = not verbose):
420
+ results = self.populate_combinations(class_tuple, settings)
421
+ all_results.extend(results)
422
+ if settings.db_jobs == 1:
423
+ # if len(all_results) >= self.sql_db.chunks_count:
424
+ list_insert_chunks = list(chunks(results, self.sql_db.chunks_count))
425
+ for chunk in list_insert_chunks:
426
+ insert_query = MolecularFormulaLink.__table__.insert().values(
427
+ chunk
428
+ )
429
+ self.sql_db.session.execute(insert_query)
430
+ # all_results = list()
431
+ self.sql_db.session.commit()
432
+ # each chunk takes ~600Mb of memory, so if using 8 processes the total free memory needs to be 5GB
433
+ if settings.db_jobs > 1:
434
+ list_insert_chunks = list(chunks(all_results, self.sql_db.chunks_count))
435
+ if verbose:
436
+ print(
437
+ "Started database insert using {} iterations for a total of {} rows".format(
438
+ len(list_insert_chunks), len(all_results)
439
+ )
440
+ )
441
+ worker_args = [
442
+ (chunk, settings.url_database) for chunk in list_insert_chunks
443
+ ]
444
+ p = multiprocessing.Pool(settings.db_jobs)
445
+ for class_list in tqdm(
446
+ p.imap_unordered(insert_database_worker, worker_args), disable= not verbose
447
+ ):
448
+ pass
449
+ p.close()
450
+ p.join()
451
+
452
+ return classes_list
453
+
454
+ def get_classes_in_order(self, molecular_search_settings):
455
+ """Get the classes in order
456
+
457
+ Parameters
458
+ ----------
459
+ molecular_search_settings : object
460
+ An object containing user-defined settings.
461
+
462
+ Returns
463
+ -------
464
+ dict
465
+ A dictionary of classes in order.
466
+ structure is ('HC', {'HC': 1})
467
+ """
468
+
469
+ usedAtoms = deepcopy(molecular_search_settings.usedAtoms)
470
+
471
+ usedAtoms.pop("C")
472
+ usedAtoms.pop("H")
473
+
474
+ min_n, max_n = usedAtoms.get("N") if usedAtoms.get("N") else (0, 0)
475
+ min_o, max_o = usedAtoms.get("O") if usedAtoms.get("O") else (0, 0)
476
+ min_s, max_s = usedAtoms.get("S") if usedAtoms.get("S") else (0, 0)
477
+ min_p, max_p = usedAtoms.get("P") if usedAtoms.get("P") else (0, 0)
478
+
479
+ possible_n = [n for n in range(min_n, max_n + 1)]
480
+ possible_o = [o for o in range(min_o, max_o + 1)]
481
+ possible_s = [s for s in range(min_s, max_s + 1)]
482
+ possible_p = [p for p in range(min_p, max_p + 1)]
483
+
484
+ atoms_in_order = ["N", "O", "S", "P"]
485
+
486
+ classe_in_order = {}
487
+
488
+ all_atoms_tuples = itertools.product(
489
+ possible_n, possible_o, possible_s, possible_p
490
+ )
491
+
492
+ for atom in atoms_in_order:
493
+ usedAtoms.pop(atom, None)
494
+
495
+ for selected_atom, min_max_tuple in usedAtoms.items():
496
+ min_x = min_max_tuple[0]
497
+ max_x = min_max_tuple[1]
498
+
499
+ possible_x = [x for x in range(min_x, max_x + 1)]
500
+
501
+ all_atoms_tuples = itertools.product(all_atoms_tuples, possible_x)
502
+ all_atoms_tuples = [
503
+ all_atoms_combined[0] + (all_atoms_combined[1],)
504
+ for all_atoms_combined in all_atoms_tuples
505
+ ]
506
+ atoms_in_order.append(selected_atom)
507
+
508
+ for all_atoms_tuple in all_atoms_tuples:
509
+ classe_str = ""
510
+ classe_dict = {}
511
+
512
+ for each_atoms_index, atom_number in enumerate(all_atoms_tuple):
513
+ if atom_number != 0:
514
+ classe_dict[atoms_in_order[each_atoms_index]] = atom_number
515
+
516
+ if not classe_dict:
517
+ classe_in_order["HC"] = {"HC": ""}
518
+ continue
519
+
520
+ classe_str = json.dumps(classe_dict)
521
+
522
+ if len(classe_str) > 0:
523
+ classe_in_order[classe_str] = classe_dict
524
+
525
+ classe_in_order_dict = self.sort_classes(atoms_in_order, classe_in_order)
526
+
527
+ return classe_in_order_dict
528
+
529
+ @staticmethod
530
+ def sort_classes(atoms_in_order, combination_dict) -> Dict[str, Dict[str, int]]:
531
+ """Sort the classes in order
532
+
533
+ Parameters
534
+ ----------
535
+ atoms_in_order : list
536
+ A list of atoms in order.
537
+ combination_dict : dict
538
+ A dictionary of classes.
539
+
540
+ Returns
541
+ -------
542
+ dict
543
+ A dictionary of classes in order.
544
+ """
545
+ # ensures atoms are always in the order defined at atoms_in_order list
546
+ join_dict_classes = dict()
547
+ atoms_in_order = ["N", "S", "P", "O"] + atoms_in_order[4:] + ["HC"]
548
+
549
+ sort_method = lambda atoms_keys: [atoms_in_order.index(atoms_keys)]
550
+ for class_str, class_dict in combination_dict.items():
551
+ sorted_dict_keys = sorted(class_dict, key=sort_method)
552
+ class_dict = {atom: class_dict[atom] for atom in sorted_dict_keys}
553
+ class_str = json.dumps(class_dict)
554
+ # using json for the new database, class
555
+ # class_str = ' '.join([atom + str(class_dict[atom]) for atom in sorted_dict_keys])
556
+ join_dict_classes[class_str] = class_dict
557
+
558
+ return join_dict_classes
559
+
560
+ @staticmethod
561
+ def get_fixed_initial_number_of_hydrogen(min_h, odd_even):
562
+ """Get the fixed initial number of hydrogen atoms
563
+
564
+ Parameters
565
+ ----------
566
+ min_h : int
567
+ The minimum number of hydrogen atoms.
568
+ odd_even : str
569
+ A string indicating whether to retrieve even or odd hydrogen atoms.
570
+ """
571
+ remaining_h = min_h % 2
572
+
573
+ if odd_even == "even":
574
+ if remaining_h == 0:
575
+ return remaining_h
576
+
577
+ else:
578
+ return remaining_h + 1
579
+
580
+ else:
581
+ if remaining_h == 0:
582
+ return remaining_h + 1
583
+
584
+ else:
585
+ return remaining_h
586
+
587
+ def calc_mz(self, datadict, class_mass=0):
588
+ """Calculate the mass-to-charge ratio (m/z) of a molecular formula.
589
+
590
+ Parameters
591
+ ----------
592
+ datadict : dict
593
+ A dictionary of classes.
594
+ class_mass : int
595
+ The mass of the class.
596
+
597
+ Returns
598
+ -------
599
+ float
600
+ The mass-to-charge ratio (m/z) of a molecular formula.
601
+ """
602
+ mass = class_mass
603
+
604
+ for atom in datadict.keys():
605
+ if atom != "HC":
606
+ mass = mass + Atoms.atomic_masses[atom] * datadict.get(atom)
607
+
608
+ return mass
609
+
610
+ def calc_dbe_class(self, datadict):
611
+ """Calculate the double bond equivalent (DBE) of a molecular formula.
612
+
613
+ Parameters
614
+ ----------
615
+ datadict : dict
616
+ A dictionary of classes.
617
+
618
+ Returns
619
+ -------
620
+ float
621
+ The double bond equivalent (DBE) of a molecular formula.
622
+ """
623
+ init_dbe = 0
624
+ for atom in datadict.keys():
625
+ if atom == "HC":
626
+ continue
627
+
628
+ n_atom = int(datadict.get(atom))
629
+
630
+ clean_atom = "".join([i for i in atom if not i.isdigit()])
631
+
632
+ valencia = MSParameters.molecular_search.used_atom_valences.get(clean_atom)
633
+
634
+ if type(valencia) is tuple:
635
+ valencia = valencia[0]
636
+ if valencia > 0:
637
+ # print atom, valencia, n_atom, init_dbe
638
+ init_dbe = init_dbe + (n_atom * (valencia - 2))
639
+ else:
640
+ continue
641
+
642
+ return 0.5 * init_dbe
643
+
644
+ def populate_combinations(self, classe_tuple, settings):
645
+ """Populate the combinations
646
+
647
+ Parameters
648
+ ----------
649
+ classe_tuple : tuple
650
+ A tuple containing the class name, the class dictionary, and the class ID.
651
+ settings : object
652
+ An object containing user-defined settings.
653
+
654
+ Returns
655
+ -------
656
+ list
657
+ A list of molecular formula data dictionaries.
658
+ """
659
+ ion_charge = 0
660
+
661
+ class_dict = classe_tuple[1]
662
+ odd_or_even = self.get_h_odd_or_even(class_dict)
663
+
664
+ return self.get_mol_formulas(odd_or_even, classe_tuple, settings)
665
+
666
+ def get_or_add(self, SomeClass, kw):
667
+ """Get or add a class
668
+
669
+ Parameters
670
+ ----------
671
+ SomeClass : object
672
+ A class object.
673
+ kw : dict
674
+ A dictionary of classes.
675
+
676
+ Returns
677
+ -------
678
+ object
679
+ A class object.
680
+ """
681
+ obj = self.sql_db.session.query(SomeClass).filter_by(**kw).first()
682
+ if not obj:
683
+ obj = SomeClass(**kw)
684
+ return obj
685
+
686
+ def get_mol_formulas(self, odd_even_tag, classe_tuple, settings):
687
+ """Get the molecular formulas
688
+
689
+ Parameters
690
+ ----------
691
+ odd_even_tag : str
692
+ A string indicating whether to retrieve even or odd hydrogen atoms.
693
+ classe_tuple : tuple
694
+
695
+ settings : object
696
+ An object containing user-defined settings.
697
+
698
+ Returns
699
+ -------
700
+ list
701
+ A list of molecular formula data dictionaries.
702
+
703
+ """
704
+ class_str = classe_tuple[0]
705
+ class_dict = classe_tuple[1]
706
+ classe_id = classe_tuple[2]
707
+
708
+ results = list()
709
+
710
+ if "HC" in class_dict:
711
+ del class_dict["HC"]
712
+
713
+ class_dbe = self.calc_dbe_class(class_dict)
714
+ class_mass = self.calc_mz(class_dict)
715
+
716
+ carbonHydrogen_mass = (
717
+ self.odd_ch_mass if odd_even_tag == "odd" else self.even_ch_mass
718
+ )
719
+ carbonHydrogen_dbe = (
720
+ self.odd_ch_dbe if odd_even_tag == "odd" else self.even_ch_dbe
721
+ )
722
+ carbonHydrogen_id = self.odd_ch_id if odd_even_tag == "odd" else self.even_ch_id
723
+
724
+ for index, carbonHydrogen_obj in enumerate(carbonHydrogen_id):
725
+ mass = carbonHydrogen_mass[index] + class_mass
726
+ dbe = carbonHydrogen_dbe[index] + class_dbe
727
+
728
+ if settings.min_mz <= mass <= settings.max_mz:
729
+ if settings.min_dbe <= dbe <= settings.max_dbe:
730
+ molecularFormula = {
731
+ "heteroAtoms_id": classe_id,
732
+ "carbonHydrogen_id": carbonHydrogen_id[index],
733
+ "mass": mass,
734
+ "DBE": dbe,
735
+ }
736
+
737
+ results.append(molecularFormula)
738
+
739
+ return results
740
+
741
+ def get_h_odd_or_even(self, class_dict):
742
+ """Get the hydrogen odd or even
743
+
744
+ Parameters
745
+ ----------
746
+ class_dict : dict
747
+ A dictionary of classes.
748
+
749
+ Returns
750
+ -------
751
+ str
752
+ A string indicating whether to retrieve even or odd hydrogen atoms.
753
+ """
754
+
755
+ HAS_NITROGEN = "N" in class_dict.keys()
756
+
757
+ number_of_halogen = self.get_total_halogen_atoms(class_dict)
758
+ number_of_hetero = self.get_total_heteroatoms(class_dict)
759
+
760
+ if number_of_halogen > 0:
761
+ HAS_HALOGEN = True
762
+
763
+ else:
764
+ HAS_HALOGEN = False
765
+
766
+ if HAS_HALOGEN:
767
+ remaining_halogen = number_of_halogen % 2
768
+
769
+ else:
770
+ remaining_halogen = 0
771
+
772
+ if number_of_hetero > 0:
773
+ HAS_OTHER_HETERO = True
774
+
775
+ total_hetero_valence = self.get_total_hetero_valence(class_dict)
776
+
777
+ else:
778
+ HAS_OTHER_HETERO = False
779
+
780
+ total_hetero_valence = 0
781
+
782
+ if HAS_OTHER_HETERO:
783
+ remaining_hetero_valence = total_hetero_valence % 2
784
+
785
+ else:
786
+ remaining_hetero_valence = 0
787
+
788
+ if HAS_NITROGEN and not HAS_OTHER_HETERO:
789
+ number_of_n = class_dict.get("N")
790
+ remaining_n = number_of_n % 2
791
+
792
+ elif HAS_NITROGEN and HAS_OTHER_HETERO:
793
+ number_of_n = class_dict.get("N")
794
+ remaining_n = (number_of_n + remaining_hetero_valence) % 2
795
+
796
+ elif HAS_OTHER_HETERO and not HAS_NITROGEN:
797
+ remaining_n = remaining_hetero_valence
798
+
799
+ else:
800
+ remaining_n = -1
801
+
802
+ if remaining_n > 0.0:
803
+ if HAS_NITROGEN or HAS_OTHER_HETERO:
804
+ if HAS_HALOGEN:
805
+ if remaining_halogen == 0:
806
+ return "odd"
807
+ else:
808
+ return "even"
809
+
810
+ else:
811
+ return "odd"
812
+
813
+ elif remaining_n == 0.0:
814
+ if HAS_NITROGEN or HAS_OTHER_HETERO:
815
+ if HAS_HALOGEN:
816
+ if remaining_halogen == 0:
817
+ return "even"
818
+ else:
819
+ return "odd"
820
+
821
+ else:
822
+ return "even"
823
+
824
+ else:
825
+ if HAS_HALOGEN:
826
+ if remaining_halogen == 0:
827
+ return "even"
828
+ else:
829
+ return "odd"
830
+
831
+ else:
832
+ return "even"
833
+
834
+ @staticmethod
835
+ def get_total_heteroatoms(class_dict):
836
+ """Get the total number of heteroatoms other than N, F, Cl, Br
837
+
838
+ Parameters
839
+ ----------
840
+ class_dict : dict
841
+ A dictionary of classes.
842
+
843
+ Returns
844
+ -------
845
+ int
846
+ The total number of heteroatoms.
847
+ """
848
+
849
+ total_number = 0
850
+
851
+ for atom in class_dict.keys():
852
+ if atom not in ["HC", "C", "H", "O", "N", "F", "Cl", "Br"]:
853
+ total_number = total_number + class_dict.get(atom)
854
+
855
+ return total_number
856
+
857
+ @staticmethod
858
+ def get_total_hetero_valence(class_dict):
859
+ """Get the total valence of heteroatoms other than N, F, Cl, Br
860
+
861
+ Parameters
862
+ ----------
863
+ class_dict : dict
864
+ A dictionary of classes.
865
+
866
+ Returns
867
+ -------
868
+ int
869
+ The total heteroatom valence.
870
+ """
871
+ total_valence = 0
872
+
873
+ for atom in class_dict.keys():
874
+ if atom not in ["HC", "C", "H", "O", "N", "F", "Cl", "Br"]:
875
+ clean_atom = "".join([i for i in atom if not i.isdigit()])
876
+
877
+ atom_valence = MSParameters.molecular_search.used_atom_valences.get(
878
+ clean_atom
879
+ )
880
+
881
+ if type(atom_valence) is tuple:
882
+ atom_valence = atom_valence[0]
883
+
884
+ n_atom = int(class_dict.get(atom))
885
+
886
+ n_atom_valence = atom_valence * n_atom
887
+
888
+ total_valence = total_valence + n_atom_valence
889
+
890
+ return total_valence
891
+
892
+ @staticmethod
893
+ def get_total_halogen_atoms(class_dict):
894
+ """Get the total number of halogen atoms
895
+
896
+ Parameters
897
+ ----------
898
+ class_dict : dict
899
+ A dictionary of classes.
900
+
901
+ Returns
902
+ -------
903
+ int
904
+ The total number of halogen atoms.
905
+ """
906
+ atoms = ["F", "Cl", "Br"]
907
+
908
+ total_number = 0
909
+
910
+ for atom in atoms:
911
+ if atom in class_dict.keys():
912
+ total_number = total_number + class_dict.get(atom)
913
+
914
+ return total_number