msfiddle 2.2.1__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {msfiddle-2.2.1 → msfiddle-2.3.0}/PKG-INFO +12 -7
- {msfiddle-2.2.1 → msfiddle-2.3.0}/README.md +11 -6
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/api.py +220 -73
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/config/fiddle_tcn_casmi.yml +10 -9
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/config/fiddle_tcn_demo.yml +12 -10
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/config/fiddle_tcn_demo_wo_da.yml +12 -10
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/config/fiddle_tcn_embl.yml +29 -22
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/config/fiddle_tcn_orbitrap.yml +18 -19
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/config/fiddle_tcn_qtof.yml +24 -26
- msfiddle-2.3.0/msfiddle/config/fiddle_tcn_timstof.yml +142 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/dataset.py +13 -5
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/download.py +59 -81
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/external_results.py +35 -6
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/main.py +10 -4
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/model_tcn.py +17 -0
- msfiddle-2.3.0/msfiddle/settings.py +75 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/utils/__init__.py +6 -0
- msfiddle-2.3.0/msfiddle/utils/casmi2026.py +231 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/utils/mol_utils.py +0 -23
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/utils/msms_utils.py +96 -48
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/utils/pkl_utils.py +59 -48
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/utils/refine_utils.py +1 -49
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle.egg-info/PKG-INFO +12 -7
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle.egg-info/SOURCES.txt +5 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/setup.py +1 -1
- {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_api.py +122 -1
- msfiddle-2.3.0/tests/test_casmi2026.py +305 -0
- msfiddle-2.3.0/tests/test_config.py +35 -0
- msfiddle-2.3.0/tests/test_download.py +93 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_external_results.py +46 -0
- msfiddle-2.3.0/tests/test_msms_utils.py +118 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_pkl_utils.py +22 -1
- msfiddle-2.2.1/tests/test_download.py +0 -27
- msfiddle-2.2.1/tests/test_msms_utils.py +0 -53
- {msfiddle-2.2.1 → msfiddle-2.3.0}/LICENSE +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/__init__.py +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/__main__.py +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/demo/input_msms.mgf +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/demo/output_buddy.csv +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/demo/output_fiddle.csv +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/demo/output_fiddle_all.csv +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/demo/output_sirius.csv +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/evaluate.py +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/optim.py +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle.egg-info/dependency_links.txt +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle.egg-info/entry_points.txt +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle.egg-info/requires.txt +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle.egg-info/top_level.txt +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/setup.cfg +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_evaluate.py +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_main.py +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_mol_utils.py +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_muon.py +0 -0
- {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_refine_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: msfiddle
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.0
|
|
4
4
|
Summary: A package for predicting chemical formulas from tandem mass spectra
|
|
5
5
|
Home-page: https://github.com/JosieHong/FIDDLE
|
|
6
6
|
Author: Yuhui Hong
|
|
@@ -69,11 +69,11 @@ msfiddle predict --demo --result_path demo.csv # 21 bundled example spect
|
|
|
69
69
|
msfiddle predict --test_data spectra.mgf --instrument_type orbitrap --result_path results.csv
|
|
70
70
|
```
|
|
71
71
|
|
|
72
|
-
Use `--instrument_type qtof` for Q-TOF data. Each spectrum needs `TITLE`, `PRECURSOR_MZ`, `PRECURSOR_TYPE` and `COLLISION_ENERGY`. The results list ranked candidates per spectrum in `Refined Formula (0)`, `Refined Formula (1)`, … with their scores in `Rescore (k)`.
|
|
72
|
+
Use `--instrument_type qtof` for Q-TOF data and `--instrument_type timstof` for timsTOF data. Each spectrum needs `TITLE`, `PRECURSOR_MZ`, `PRECURSOR_TYPE` and `COLLISION_ENERGY`. The results list ranked candidates per spectrum in `Refined Formula (0)`, `Refined Formula (1)`, … with their scores in `Rescore (k)`.
|
|
73
73
|
|
|
74
74
|
When the true formulas are known (a `SMILES` or `FORMULA` field in each spectrum), add `--eval` to also report top-k accuracy, saved as `results_eval.csv`.
|
|
75
75
|
|
|
76
|
-
The [documentation](https://msfiddle.readthedocs.io) covers the Python API, BUDDY and SIRIUS integration, custom checkpoints,
|
|
76
|
+
The [documentation](https://msfiddle.readthedocs.io) covers the Python API, BUDDY and SIRIUS integration, custom checkpoints, all output columns, and [benchmark results](https://msfiddle.readthedocs.io/en/latest/benchmarks.html).
|
|
77
77
|
|
|
78
78
|
## Reproduce the paper
|
|
79
79
|
|
|
@@ -85,7 +85,12 @@ This repository also holds the research code for training and evaluation. Run ev
|
|
|
85
85
|
conda env create -f environment.yml
|
|
86
86
|
```
|
|
87
87
|
|
|
88
|
-
2. (optional) Install [BUDDY](https://github.com/Philipbear/msbuddy) and [SIRIUS](https://v6.docs.sirius-ms.io/) for the comparison experiments.
|
|
88
|
+
2. (optional) Install [BUDDY](https://github.com/Philipbear/msbuddy) and [SIRIUS](https://v6.docs.sirius-ms.io/) for the comparison experiments. SIRIUS needs a free account; log in once as `install_sirius.sh` prints at the end.
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
bash running_scripts/install_buddy.sh
|
|
92
|
+
bash running_scripts/install_sirius.sh
|
|
93
|
+
```
|
|
89
94
|
|
|
90
95
|
3. Download the models into `./check_point/`, where the scripts expect them:
|
|
91
96
|
|
|
@@ -95,15 +100,15 @@ msfiddle download -d ./check_point
|
|
|
95
100
|
|
|
96
101
|
| Script | Description |
|
|
97
102
|
|---|---|
|
|
98
|
-
| `running_scripts/train_release_v2.
|
|
103
|
+
| `running_scripts/train_release_v2.3.sh` | Build the data, train the release models (TCN and rescore) for Q-TOF, Orbitrap and timsTOF in parallel and evaluate them; `INSTRUMENTS=timstof` runs one of them. timsTOF trains on the Enveda CASMI 2026 library at `data/origin/casmi2026/train.parquet` |
|
|
99
104
|
| `running_scripts/experiments_test_benchmark.sh` | Evaluate on external benchmarks (CASMI 2016, CASMI 2017, EMBL-MCF 2.0) |
|
|
100
105
|
| `running_scripts/experiments_test_nist23.sh` | Train and evaluate with compounds new in NIST23 held out |
|
|
101
106
|
| `running_scripts/experiments_test_chimeric.sh` | Evaluate on chimeric spectra |
|
|
102
107
|
| `running_scripts/experiments_test_noised.sh` | Evaluate on spectra with added noise peaks |
|
|
103
108
|
|
|
104
|
-
The benchmark, chimeric and noise experiments use the released models; the chimeric and noise ones also need the data
|
|
109
|
+
The benchmark, chimeric and noise experiments use the released models; the chimeric and noise ones also need the data those models were trained on.
|
|
105
110
|
|
|
106
|
-
To train from scratch, see `scripts/
|
|
111
|
+
To train from scratch, see `scripts/train_tcn_gpus_cl.py` and `scripts/train_rescore.py` with the configs in [`msfiddle/config/`](https://github.com/JosieHong/FIDDLE/tree/main/msfiddle/config).
|
|
107
112
|
|
|
108
113
|
## Citation
|
|
109
114
|
|
|
@@ -28,11 +28,11 @@ msfiddle predict --demo --result_path demo.csv # 21 bundled example spect
|
|
|
28
28
|
msfiddle predict --test_data spectra.mgf --instrument_type orbitrap --result_path results.csv
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
-
Use `--instrument_type qtof` for Q-TOF data. Each spectrum needs `TITLE`, `PRECURSOR_MZ`, `PRECURSOR_TYPE` and `COLLISION_ENERGY`. The results list ranked candidates per spectrum in `Refined Formula (0)`, `Refined Formula (1)`, … with their scores in `Rescore (k)`.
|
|
31
|
+
Use `--instrument_type qtof` for Q-TOF data and `--instrument_type timstof` for timsTOF data. Each spectrum needs `TITLE`, `PRECURSOR_MZ`, `PRECURSOR_TYPE` and `COLLISION_ENERGY`. The results list ranked candidates per spectrum in `Refined Formula (0)`, `Refined Formula (1)`, … with their scores in `Rescore (k)`.
|
|
32
32
|
|
|
33
33
|
When the true formulas are known (a `SMILES` or `FORMULA` field in each spectrum), add `--eval` to also report top-k accuracy, saved as `results_eval.csv`.
|
|
34
34
|
|
|
35
|
-
The [documentation](https://msfiddle.readthedocs.io) covers the Python API, BUDDY and SIRIUS integration, custom checkpoints,
|
|
35
|
+
The [documentation](https://msfiddle.readthedocs.io) covers the Python API, BUDDY and SIRIUS integration, custom checkpoints, all output columns, and [benchmark results](https://msfiddle.readthedocs.io/en/latest/benchmarks.html).
|
|
36
36
|
|
|
37
37
|
## Reproduce the paper
|
|
38
38
|
|
|
@@ -44,7 +44,12 @@ This repository also holds the research code for training and evaluation. Run ev
|
|
|
44
44
|
conda env create -f environment.yml
|
|
45
45
|
```
|
|
46
46
|
|
|
47
|
-
2. (optional) Install [BUDDY](https://github.com/Philipbear/msbuddy) and [SIRIUS](https://v6.docs.sirius-ms.io/) for the comparison experiments.
|
|
47
|
+
2. (optional) Install [BUDDY](https://github.com/Philipbear/msbuddy) and [SIRIUS](https://v6.docs.sirius-ms.io/) for the comparison experiments. SIRIUS needs a free account; log in once as `install_sirius.sh` prints at the end.
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
bash running_scripts/install_buddy.sh
|
|
51
|
+
bash running_scripts/install_sirius.sh
|
|
52
|
+
```
|
|
48
53
|
|
|
49
54
|
3. Download the models into `./check_point/`, where the scripts expect them:
|
|
50
55
|
|
|
@@ -54,15 +59,15 @@ msfiddle download -d ./check_point
|
|
|
54
59
|
|
|
55
60
|
| Script | Description |
|
|
56
61
|
|---|---|
|
|
57
|
-
| `running_scripts/train_release_v2.
|
|
62
|
+
| `running_scripts/train_release_v2.3.sh` | Build the data, train the release models (TCN and rescore) for Q-TOF, Orbitrap and timsTOF in parallel and evaluate them; `INSTRUMENTS=timstof` runs one of them. timsTOF trains on the Enveda CASMI 2026 library at `data/origin/casmi2026/train.parquet` |
|
|
58
63
|
| `running_scripts/experiments_test_benchmark.sh` | Evaluate on external benchmarks (CASMI 2016, CASMI 2017, EMBL-MCF 2.0) |
|
|
59
64
|
| `running_scripts/experiments_test_nist23.sh` | Train and evaluate with compounds new in NIST23 held out |
|
|
60
65
|
| `running_scripts/experiments_test_chimeric.sh` | Evaluate on chimeric spectra |
|
|
61
66
|
| `running_scripts/experiments_test_noised.sh` | Evaluate on spectra with added noise peaks |
|
|
62
67
|
|
|
63
|
-
The benchmark, chimeric and noise experiments use the released models; the chimeric and noise ones also need the data
|
|
68
|
+
The benchmark, chimeric and noise experiments use the released models; the chimeric and noise ones also need the data those models were trained on.
|
|
64
69
|
|
|
65
|
-
To train from scratch, see `scripts/
|
|
70
|
+
To train from scratch, see `scripts/train_tcn_gpus_cl.py` and `scripts/train_rescore.py` with the configs in [`msfiddle/config/`](https://github.com/JosieHong/FIDDLE/tree/main/msfiddle/config).
|
|
66
71
|
|
|
67
72
|
## Citation
|
|
68
73
|
|
|
@@ -15,9 +15,9 @@ from typing import Any
|
|
|
15
15
|
|
|
16
16
|
import numpy as np
|
|
17
17
|
import pandas as pd
|
|
18
|
-
import yaml
|
|
19
18
|
from pyteomics import mgf
|
|
20
19
|
|
|
20
|
+
from .settings import load_config
|
|
21
21
|
from .download import download_models as download_pretrained_models
|
|
22
22
|
from .download import get_checkpoint_dir
|
|
23
23
|
from .external_results import load_buddy_results, load_sirius_results
|
|
@@ -27,11 +27,21 @@ from .utils.mol_utils import (
|
|
|
27
27
|
formula_to_vector,
|
|
28
28
|
vector_to_formula,
|
|
29
29
|
)
|
|
30
|
-
from .utils.msms_utils import
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
from .utils.msms_utils import (
|
|
31
|
+
isotope_mass_shift,
|
|
32
|
+
mass_calculator,
|
|
33
|
+
precursor_window_da,
|
|
34
|
+
)
|
|
35
|
+
from .utils.pkl_utils import (
|
|
36
|
+
generate_ms,
|
|
37
|
+
is_stepped_collision_energy,
|
|
38
|
+
parse_collision_energy,
|
|
39
|
+
resolve_collision_energy,
|
|
40
|
+
unify_precursor_type,
|
|
41
|
+
)
|
|
42
|
+
from .utils.refine_utils import ISOTOPE_MASS, formula_refinement, parse_formula
|
|
33
43
|
|
|
34
|
-
_INSTRUMENT_TYPES = {"orbitrap", "qtof"}
|
|
44
|
+
_INSTRUMENT_TYPES = {"orbitrap", "qtof", "timstof"}
|
|
35
45
|
_NEUTRAL_FORMULAS = ("CH4O2", "CH2O2", "H2O", "NH3", "CO2")
|
|
36
46
|
|
|
37
47
|
|
|
@@ -223,46 +233,53 @@ class MsFiddlePredictor:
|
|
|
223
233
|
encoded = self._load_mgf_spectra(test_data_path)
|
|
224
234
|
predictions = self._predict_encoded(encoded, batch_size=batch_size)
|
|
225
235
|
|
|
226
|
-
|
|
227
|
-
|
|
236
|
+
# Indexed by ID: looked up once per spectrum
|
|
237
|
+
buddy_df = _index_by_id(load_buddy_results(buddy_path)) if buddy_path else None
|
|
238
|
+
sirius_df = (
|
|
239
|
+
_index_by_id(load_sirius_results(sirius_path)) if sirius_path else None
|
|
240
|
+
)
|
|
241
|
+
ids = {prediction.spectrum.id for prediction in predictions}
|
|
242
|
+
_warn_if_unmatched("BUDDY", buddy_df, ids)
|
|
243
|
+
_warn_if_unmatched("SIRIUS", sirius_df, ids)
|
|
228
244
|
|
|
229
245
|
refined_formula = {f"Refined Formula ({k})": [] for k in range(top_k)}
|
|
230
246
|
refined_mass = {f"Refined Mass ({k})": [] for k in range(top_k)}
|
|
231
247
|
refined_rescore = {f"Rescore ({k})": [] for k in range(top_k)}
|
|
248
|
+
isotope_shift = {f"Isotope Shift ({k})": [] for k in range(top_k)}
|
|
249
|
+
offsets = self.config["post_processing"]["isotope_errors"]
|
|
232
250
|
running_time: list[float] = []
|
|
233
|
-
neutral_masses: list[float] = []
|
|
234
251
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
prediction.spectrum.id,
|
|
241
|
-
buddy_df=buddy_df,
|
|
242
|
-
sirius_df=sirius_df,
|
|
243
|
-
),
|
|
252
|
+
# One refinement per spectrum and isotope offset
|
|
253
|
+
jobs = [] # (spectrum index, offset, refine args, neutral mass)
|
|
254
|
+
for i, prediction in enumerate(predictions):
|
|
255
|
+
extra_formulas = self._external_formula_candidates(
|
|
256
|
+
prediction.spectrum.id, buddy_df=buddy_df, sirius_df=sirius_df
|
|
244
257
|
)
|
|
245
|
-
for
|
|
246
|
-
|
|
258
|
+
for k in offsets:
|
|
259
|
+
refine_args, mass = self._refinement_args(
|
|
260
|
+
prediction, top_k, extra_formulas=extra_formulas, isotope_shift=k
|
|
261
|
+
)
|
|
262
|
+
jobs.append((i, k, refine_args, mass))
|
|
247
263
|
refined = _run_refinement(
|
|
248
|
-
[refine_args for refine_args, _ in jobs], num_workers, self.verbose
|
|
264
|
+
[refine_args for _, _, refine_args, _ in jobs], num_workers, self.verbose
|
|
249
265
|
)
|
|
250
|
-
|
|
251
|
-
|
|
266
|
+
parts: list[list] = [[] for _ in predictions]
|
|
267
|
+
refine_times = [0.0] * len(predictions)
|
|
268
|
+
neutral_masses: list[float] = [0.0] * len(predictions)
|
|
269
|
+
for (i, k, _, mass), (refined_results, refine_time) in zip(jobs, refined):
|
|
270
|
+
parts[i].append((k, refined_results))
|
|
271
|
+
refine_times[i] += refine_time
|
|
272
|
+
if k == 0:
|
|
273
|
+
neutral_masses[i] = mass
|
|
274
|
+
|
|
275
|
+
rescore_jobs = zip(predictions, parts, refine_times)
|
|
252
276
|
if self.verbose:
|
|
253
277
|
from tqdm import tqdm
|
|
254
278
|
|
|
255
|
-
rescore_jobs = tqdm(rescore_jobs, total=len(
|
|
256
|
-
for
|
|
257
|
-
prediction,
|
|
258
|
-
(_, neutral_mass),
|
|
259
|
-
(refined_results, refine_time),
|
|
260
|
-
) in rescore_jobs:
|
|
279
|
+
rescore_jobs = tqdm(rescore_jobs, total=len(parts), desc="Rescore")
|
|
280
|
+
for prediction, spectrum_parts, refine_time in rescore_jobs:
|
|
261
281
|
start_time = time.time()
|
|
262
|
-
results = self.
|
|
263
|
-
prediction.rescore_embedding, refined_results, top_k
|
|
264
|
-
)
|
|
265
|
-
neutral_masses.append(neutral_mass)
|
|
282
|
+
results = self._rank_candidates(prediction, spectrum_parts, top_k)
|
|
266
283
|
running_time.append(
|
|
267
284
|
prediction.prediction_time + refine_time + time.time() - start_time
|
|
268
285
|
)
|
|
@@ -273,6 +290,10 @@ class MsFiddlePredictor:
|
|
|
273
290
|
refined_formula[f"Refined Formula ({idx})"].append(formula)
|
|
274
291
|
refined_mass[f"Refined Mass ({idx})"].append(mass)
|
|
275
292
|
refined_rescore[f"Rescore ({idx})"].append(score)
|
|
293
|
+
if offsets != [0]:
|
|
294
|
+
isotope_shift[f"Isotope Shift ({idx})"].append(
|
|
295
|
+
results["isotope_shift"][idx]
|
|
296
|
+
)
|
|
276
297
|
|
|
277
298
|
base_columns = {
|
|
278
299
|
"ID": [prediction.spectrum.id for prediction in predictions],
|
|
@@ -284,9 +305,10 @@ class MsFiddlePredictor:
|
|
|
284
305
|
"Pred H/C Num": [prediction.pred_hc_count for prediction in predictions],
|
|
285
306
|
"Running Time": running_time,
|
|
286
307
|
}
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
308
|
+
columns = {**base_columns, **refined_formula, **refined_mass, **refined_rescore}
|
|
309
|
+
if offsets != [0]:
|
|
310
|
+
columns.update(isotope_shift)
|
|
311
|
+
return pd.DataFrame(columns)
|
|
290
312
|
|
|
291
313
|
def _encode_spectrum_record(
|
|
292
314
|
self,
|
|
@@ -345,7 +367,16 @@ class MsFiddlePredictor:
|
|
|
345
367
|
f"{adjusted_precursor_type}"
|
|
346
368
|
)
|
|
347
369
|
|
|
348
|
-
ce_str =
|
|
370
|
+
ce_str = resolve_collision_energy(
|
|
371
|
+
_collision_energy_to_string(collision_energy),
|
|
372
|
+
encoder.get("stepped_collision_energy", "drop"),
|
|
373
|
+
)
|
|
374
|
+
if ce_str is None:
|
|
375
|
+
raise ValueError(
|
|
376
|
+
f"Stepped collision energy {collision_energy!r} is not supported: "
|
|
377
|
+
"this model was trained on single-energy spectra (the timsTOF "
|
|
378
|
+
"config sets encoding.stepped_collision_energy to 'mean')"
|
|
379
|
+
)
|
|
349
380
|
ce, nce = parse_collision_energy(
|
|
350
381
|
ce_str=ce_str,
|
|
351
382
|
precursor_mz=precursor_mz,
|
|
@@ -388,8 +419,10 @@ class MsFiddlePredictor:
|
|
|
388
419
|
if self.config["encoding"]["use_simulated_precursor_mz"]
|
|
389
420
|
else "precursor_mz"
|
|
390
421
|
)
|
|
422
|
+
stepped_mode = self.config["encoding"].get("stepped_collision_energy", "drop")
|
|
391
423
|
encoded: list[_EncodedSpectrum] = []
|
|
392
424
|
invalid_count = 0
|
|
425
|
+
stepped: list[str] = []
|
|
393
426
|
for spectrum in spectra:
|
|
394
427
|
params = spectrum.get("params", {})
|
|
395
428
|
missing = [
|
|
@@ -413,6 +446,12 @@ class MsFiddlePredictor:
|
|
|
413
446
|
f"('title', '{precursor_mz_key}', 'precursor_type', 'collision_energy')"
|
|
414
447
|
)
|
|
415
448
|
continue
|
|
449
|
+
if stepped_mode == "drop" and is_stepped_collision_energy(
|
|
450
|
+
params["collision_energy"]
|
|
451
|
+
):
|
|
452
|
+
stepped.append(str(params["collision_energy"]))
|
|
453
|
+
invalid_count += 1
|
|
454
|
+
continue
|
|
416
455
|
try:
|
|
417
456
|
encoded.append(
|
|
418
457
|
self._encode_spectrum(
|
|
@@ -428,6 +467,13 @@ class MsFiddlePredictor:
|
|
|
428
467
|
invalid_count += 1
|
|
429
468
|
continue
|
|
430
469
|
|
|
470
|
+
if stepped:
|
|
471
|
+
warnings.warn(
|
|
472
|
+
f"Skipped {len(stepped)} spectra with a stepped collision energy "
|
|
473
|
+
f"(e.g. {stepped[0]!r}): FIDDLE was trained on single-energy "
|
|
474
|
+
"spectra. Set COLLISION_ENERGY to a single energy to predict them.",
|
|
475
|
+
stacklevel=3,
|
|
476
|
+
)
|
|
431
477
|
if self.verbose:
|
|
432
478
|
print(len(spectra) - invalid_count, "spectra left after filtering")
|
|
433
479
|
print(len(encoded), "spectra loaded into the dataset")
|
|
@@ -532,12 +578,7 @@ class MsFiddlePredictor:
|
|
|
532
578
|
if not path.exists()
|
|
533
579
|
]
|
|
534
580
|
if missing_paths and self.download_models:
|
|
535
|
-
download_pretrained_models(
|
|
536
|
-
models=[
|
|
537
|
-
f"fiddle_tcn_{self.instrument_type}",
|
|
538
|
-
f"fiddle_rescore_{self.instrument_type}",
|
|
539
|
-
]
|
|
540
|
-
)
|
|
581
|
+
download_pretrained_models(instruments=[self.instrument_type])
|
|
541
582
|
missing_paths = [
|
|
542
583
|
path
|
|
543
584
|
for path in (self.resume_path, self.rescore_resume_path)
|
|
@@ -574,7 +615,7 @@ class MsFiddlePredictor:
|
|
|
574
615
|
assert torch is not None
|
|
575
616
|
assert nn is not None
|
|
576
617
|
|
|
577
|
-
from .model_tcn import FormulaEncoder, MS2FNet_tcn, RescoreHead
|
|
618
|
+
from .model_tcn import FormulaEncoder, MS2FNet_tcn, RescoreHead, tcn_hash
|
|
578
619
|
|
|
579
620
|
self.device, self.device_ids = _resolve_torch_device(
|
|
580
621
|
torch,
|
|
@@ -622,6 +663,15 @@ class MsFiddlePredictor:
|
|
|
622
663
|
)
|
|
623
664
|
formula_encoder.load_state_dict(ckpt["formula_encoder_state_dict"])
|
|
624
665
|
rescore_head.load_state_dict(ckpt["rescore_head_state_dict"])
|
|
666
|
+
expected = ckpt.get("tcn_hash") # absent in older checkpoints
|
|
667
|
+
if expected and expected != tcn_hash(state_dict):
|
|
668
|
+
warnings.warn(
|
|
669
|
+
f"The rescore model {self.rescore_resume_path} was trained with a "
|
|
670
|
+
f"different TCN than {self.resume_path}; its scores are not "
|
|
671
|
+
"meaningful with this TCN. Use the two checkpoints of one release "
|
|
672
|
+
"or training run.",
|
|
673
|
+
stacklevel=2,
|
|
674
|
+
)
|
|
625
675
|
formula_encoder.eval()
|
|
626
676
|
rescore_head.eval()
|
|
627
677
|
|
|
@@ -636,16 +686,39 @@ class MsFiddlePredictor:
|
|
|
636
686
|
*,
|
|
637
687
|
extra_formulas: Sequence[str] | None = None,
|
|
638
688
|
) -> tuple[dict[str, list[Any]], float]:
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
689
|
+
parts = []
|
|
690
|
+
for k in self.config["post_processing"]["isotope_errors"]:
|
|
691
|
+
refine_args, mass = self._refinement_args(
|
|
692
|
+
prediction, top_k, extra_formulas=extra_formulas, isotope_shift=k
|
|
693
|
+
)
|
|
694
|
+
parts.append((k, formula_refinement(*refine_args)))
|
|
695
|
+
if k == 0:
|
|
696
|
+
neutral_mass = mass
|
|
697
|
+
return self._rank_candidates(prediction, parts, top_k), neutral_mass
|
|
698
|
+
|
|
699
|
+
def _rank_candidates(
|
|
700
|
+
self,
|
|
701
|
+
prediction: _ModelPrediction,
|
|
702
|
+
parts: Sequence[tuple[int, dict[str, list[Any]]]],
|
|
703
|
+
top_k: int,
|
|
704
|
+
) -> dict[str, list[Any]]:
|
|
705
|
+
"""Rescore the candidates refined at each isotope offset k and rank them.
|
|
706
|
+
|
|
707
|
+
With the single offset 0 this is plain rescoring. Otherwise the
|
|
708
|
+
candidates are pooled (a formula found at several offsets keeps the
|
|
709
|
+
smallest k) and ranked by rescore - isotope_penalty * k.
|
|
710
|
+
"""
|
|
711
|
+
if len(parts) == 1:
|
|
712
|
+
return self._rescore_candidates(
|
|
713
|
+
prediction.rescore_embedding, parts[0][1], top_k
|
|
714
|
+
)
|
|
715
|
+
merged, shift_of = _merge_isotope_refinements(parts)
|
|
716
|
+
results = self._rescore_candidates(prediction.rescore_embedding, merged, top_k)
|
|
717
|
+
return _apply_isotope_penalty(
|
|
718
|
+
results,
|
|
719
|
+
shift_of,
|
|
720
|
+
self.config["post_processing"]["isotope_penalty"],
|
|
721
|
+
top_k,
|
|
649
722
|
)
|
|
650
723
|
|
|
651
724
|
def _refinement_args(
|
|
@@ -654,13 +727,23 @@ class MsFiddlePredictor:
|
|
|
654
727
|
top_k: int,
|
|
655
728
|
*,
|
|
656
729
|
extra_formulas: Sequence[str] | None = None,
|
|
730
|
+
isotope_shift: int = 0,
|
|
657
731
|
) -> tuple[tuple, float]:
|
|
658
|
-
"""Positional arguments of formula_refinement for one prediction
|
|
732
|
+
"""Positional arguments of formula_refinement for one prediction, and
|
|
733
|
+
the neutral mass it refines around. ``isotope_shift`` = k assumes the
|
|
734
|
+
precursor m/z was read from the k-th 13C isotope peak."""
|
|
735
|
+
spectrum = prediction.spectrum
|
|
736
|
+
# Aliases such as [M+FA-H]- are encoded under their canonical name
|
|
737
|
+
adduct = unify_precursor_type(spectrum.precursor_type)
|
|
659
738
|
neutral_mass = float(
|
|
660
|
-
mass_calculator(
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
739
|
+
mass_calculator(adduct, spectrum.precursor_mz)
|
|
740
|
+
) - isotope_mass_shift(adduct, isotope_shift)
|
|
741
|
+
pp = self.config["post_processing"]
|
|
742
|
+
# A ppm tolerance applies to the precursor m/z, like the data filters
|
|
743
|
+
window = (
|
|
744
|
+
precursor_window_da(adduct, spectrum.precursor_mz, pp["mass_tolerance"])
|
|
745
|
+
if pp["ppm_mode"]
|
|
746
|
+
else pp["mass_tolerance"]
|
|
664
747
|
)
|
|
665
748
|
f0_list = [prediction.pred_formula]
|
|
666
749
|
if extra_formulas:
|
|
@@ -679,8 +762,8 @@ class MsFiddlePredictor:
|
|
|
679
762
|
refine_args = (
|
|
680
763
|
f0_list,
|
|
681
764
|
neutral_mass,
|
|
682
|
-
|
|
683
|
-
|
|
765
|
+
window,
|
|
766
|
+
False, # window is in Da
|
|
684
767
|
top_k,
|
|
685
768
|
self.config["post_processing"]["maxium_miss_atom_num"],
|
|
686
769
|
self.config["post_processing"]["time_out"],
|
|
@@ -748,15 +831,20 @@ class MsFiddlePredictor:
|
|
|
748
831
|
) -> list[dict[str, Any]]:
|
|
749
832
|
candidates = []
|
|
750
833
|
base_metadata = self._metadata_for_prediction(prediction, neutral_mass)
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
834
|
+
shifts = refined_results.get("isotope_shift", [])
|
|
835
|
+
for idx, (formula, mass, raw_score) in enumerate(
|
|
836
|
+
zip(
|
|
837
|
+
refined_results["formula"],
|
|
838
|
+
refined_results["mass"],
|
|
839
|
+
refined_results["rescore"],
|
|
840
|
+
)
|
|
755
841
|
):
|
|
756
842
|
if formula is None:
|
|
757
843
|
continue
|
|
758
844
|
metadata = dict(base_metadata)
|
|
759
845
|
metadata["raw_rescore"] = float(raw_score)
|
|
846
|
+
if shifts:
|
|
847
|
+
metadata["isotope_shift"] = shifts[idx]
|
|
760
848
|
candidates.append(
|
|
761
849
|
{
|
|
762
850
|
"formula": formula,
|
|
@@ -797,9 +885,8 @@ class MsFiddlePredictor:
|
|
|
797
885
|
) -> list[str]:
|
|
798
886
|
candidates: list[str] = []
|
|
799
887
|
if buddy_df is not None:
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
row = rows.iloc[0]
|
|
888
|
+
if spec_id in buddy_df.index:
|
|
889
|
+
row = buddy_df.loc[spec_id]
|
|
803
890
|
formulas = row[
|
|
804
891
|
[
|
|
805
892
|
"Pred Formula (1)",
|
|
@@ -826,9 +913,8 @@ class MsFiddlePredictor:
|
|
|
826
913
|
and score < self.config["post_processing"]["buddy_fdr_thr"]
|
|
827
914
|
)
|
|
828
915
|
if sirius_df is not None:
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
row = rows.iloc[0]
|
|
916
|
+
if spec_id in sirius_df.index:
|
|
917
|
+
row = sirius_df.loc[spec_id]
|
|
832
918
|
formulas = row[
|
|
833
919
|
[
|
|
834
920
|
"Pred Formula (1)",
|
|
@@ -857,6 +943,63 @@ class MsFiddlePredictor:
|
|
|
857
943
|
return candidates
|
|
858
944
|
|
|
859
945
|
|
|
946
|
+
def _index_by_id(df: pd.DataFrame) -> pd.DataFrame:
|
|
947
|
+
"""One row per spectrum ID (the first, as before), indexed by ID."""
|
|
948
|
+
return df.drop_duplicates("ID").set_index("ID")
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
def _warn_if_unmatched(tool: str, df: pd.DataFrame | None, ids: set[str]) -> None:
|
|
952
|
+
"""Warn when none of a tool's results belongs to a spectrum being predicted."""
|
|
953
|
+
if df is not None and len(df) and ids.isdisjoint(df.index):
|
|
954
|
+
warnings.warn(
|
|
955
|
+
f"No {tool} result matches a spectrum title, so {tool} adds no "
|
|
956
|
+
f"candidates. Its IDs look like {df.index[0]!r}.",
|
|
957
|
+
stacklevel=3,
|
|
958
|
+
)
|
|
959
|
+
|
|
960
|
+
|
|
961
|
+
def _merge_isotope_refinements(
|
|
962
|
+
parts: Sequence[tuple[int, dict[str, list[Any]]]],
|
|
963
|
+
) -> tuple[dict[str, list[Any]], dict[str, int]]:
|
|
964
|
+
"""Pool the candidates refined at each isotope offset. A formula found at
|
|
965
|
+
several offsets keeps the smallest one."""
|
|
966
|
+
formulas, masses, shift_of = [], [], {}
|
|
967
|
+
for k, refined in sorted(parts, key=lambda part: part[0]):
|
|
968
|
+
for formula, mass in zip(refined["formula"], refined["mass"]):
|
|
969
|
+
if formula is not None and formula not in shift_of:
|
|
970
|
+
shift_of[formula] = k
|
|
971
|
+
formulas.append(formula)
|
|
972
|
+
masses.append(mass)
|
|
973
|
+
return {"formula": formulas, "mass": masses}, shift_of
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
def _apply_isotope_penalty(
|
|
977
|
+
results: dict[str, list[Any]],
|
|
978
|
+
shift_of: Mapping[str, int],
|
|
979
|
+
penalty: float,
|
|
980
|
+
top_k: int,
|
|
981
|
+
) -> dict[str, list[Any]]:
|
|
982
|
+
"""Rank rescored candidates by rescore - penalty * isotope shift, keep top_k."""
|
|
983
|
+
ranked = sorted(
|
|
984
|
+
(
|
|
985
|
+
(rescore - penalty * shift_of[formula], formula, mass, rescore)
|
|
986
|
+
for formula, mass, rescore in zip(
|
|
987
|
+
results["formula"], results["mass"], results["rescore"]
|
|
988
|
+
)
|
|
989
|
+
if formula is not None
|
|
990
|
+
),
|
|
991
|
+
key=lambda item: item[0],
|
|
992
|
+
reverse=True,
|
|
993
|
+
)[:top_k]
|
|
994
|
+
pad = [None] * (top_k - len(ranked))
|
|
995
|
+
return {
|
|
996
|
+
"formula": [r[1] for r in ranked] + pad,
|
|
997
|
+
"mass": [r[2] for r in ranked] + pad,
|
|
998
|
+
"rescore": [r[3] for r in ranked] + [0.0] * len(pad),
|
|
999
|
+
"isotope_shift": [shift_of[r[1]] for r in ranked] + pad,
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
|
|
860
1003
|
def _timed_refinement(refine_args: tuple) -> tuple[dict[str, list[Any]], float]:
|
|
861
1004
|
start_time = time.time()
|
|
862
1005
|
return formula_refinement(*refine_args), time.time() - start_time
|
|
@@ -1001,16 +1144,20 @@ def _normalize_instrument_type(instrument_type: str) -> str:
|
|
|
1001
1144
|
|
|
1002
1145
|
|
|
1003
1146
|
def _is_external_formula(value: object) -> bool:
|
|
1147
|
+
"""A BUDDY or SIRIUS formula that refinement can use: present, and made only
|
|
1148
|
+
of elements FIDDLE models (SIRIUS may propose Si or Se, for example)."""
|
|
1004
1149
|
if pd.isna(value):
|
|
1005
1150
|
return False
|
|
1006
|
-
|
|
1151
|
+
text = str(value).strip()
|
|
1152
|
+
if text.lower() in {"", "na", "nan", "none", "null", "<na>"}:
|
|
1153
|
+
return False
|
|
1154
|
+
return parse_formula(text).keys() <= ISOTOPE_MASS.keys()
|
|
1007
1155
|
|
|
1008
1156
|
|
|
1009
1157
|
def _load_config(config_path: Path) -> dict[str, Any]:
|
|
1010
1158
|
if not config_path.exists():
|
|
1011
1159
|
raise FileNotFoundError(f"Config file not found: {config_path}")
|
|
1012
|
-
|
|
1013
|
-
return yaml.safe_load(handle)
|
|
1160
|
+
return load_config(config_path)
|
|
1014
1161
|
|
|
1015
1162
|
|
|
1016
1163
|
def _validate_top_k(top_k: int) -> int:
|
|
@@ -68,7 +68,7 @@ encoding:
|
|
|
68
68
|
model:
|
|
69
69
|
resolution: 0.2
|
|
70
70
|
max_mz: 1500
|
|
71
|
-
padding_dim: 20 #
|
|
71
|
+
padding_dim: 20 # pads 7500 m/z bins to 7520, a multiple of 2**5 for the 5 pooling layers
|
|
72
72
|
num_add: 6
|
|
73
73
|
input_channels: 1
|
|
74
74
|
|
|
@@ -77,7 +77,6 @@ model:
|
|
|
77
77
|
tcn_kernel_sizes: [45, 43, 41, 39, 37, 35]
|
|
78
78
|
tcn_dropout: 0.2
|
|
79
79
|
|
|
80
|
-
# metadata_dim: 16
|
|
81
80
|
add_embedding_dim: 4
|
|
82
81
|
ce_embedding_dim: 4
|
|
83
82
|
mass_embedding_dim: 4
|
|
@@ -94,20 +93,22 @@ model:
|
|
|
94
93
|
# ------------------------
|
|
95
94
|
# training settings
|
|
96
95
|
# ------------------------
|
|
97
|
-
#
|
|
96
|
+
# not used: this config is for testing only
|
|
98
97
|
|
|
99
98
|
# ------------------------
|
|
100
99
|
# postprocessing settings
|
|
101
100
|
# ------------------------
|
|
102
101
|
post_processing:
|
|
103
|
-
refine_atom_type: ['C', 'O', 'N', 'H']
|
|
104
|
-
refine_atom_num: [3, 3, 3, -1]
|
|
102
|
+
refine_atom_type: ['C', 'O', 'N', 'H'] # atom types refinement may change
|
|
103
|
+
refine_atom_num: [3, 3, 3, -1] # maximum change per atom type from the predicted formula; -1: no limit
|
|
105
104
|
|
|
106
|
-
mass_tolerance:
|
|
105
|
+
mass_tolerance: 10.0 # ppm of the precursor m/z, the unit of the data filters' ppm_tolerance
|
|
107
106
|
ppm_mode: True
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
107
|
+
isotope_errors: [0] # 13C isotope offsets k to search; [0, 1] also tries a precursor read from the M+1 peak
|
|
108
|
+
isotope_penalty: 0.5 # subtracted from the rescore per unit of k
|
|
109
|
+
time_out: -1 # seconds per spectrum; 0 or less: no limit
|
|
110
|
+
maxium_miss_atom_num: 50 # maximum atom-change steps from the predicted formula when the search cannot be enumerated
|
|
111
|
+
top_k: 5 # candidates kept per spectrum, at inference and in rescore training pairs
|
|
111
112
|
buddy_fdr_thr: 0.6
|
|
112
113
|
sirius_score_thr: -5
|
|
113
114
|
|
|
@@ -8,9 +8,9 @@ mona_qtof:
|
|
|
8
8
|
ms_level: 'MS2'
|
|
9
9
|
atom_type: ['C', 'O', 'N', 'H', 'P', 'S', 'F', 'Cl', 'B', 'Br', 'I', 'Na', 'K']
|
|
10
10
|
precursor_type: ['[M+H]+', '[M+Na]+', '[2M+H]+', # +1
|
|
11
|
-
'[M+H-H2O]+', '[M-H2O+H]+', '[M+H-2H2O]+', '[M+H-NH3]+', '[M+H+NH3]+', '[M+NH4]+', '[M+H-CH2O2]+', '[M+H-CH4O2]+', # +1
|
|
11
|
+
'[M+H-H2O]+', '[M-H2O+H]+', '[M+H-2H2O]+', '[M+H-NH3]+', '[M+H+NH3]+', '[M+NH4]+', '[M+H-CH2O2]+', '[M+H-CH4O2]+', # +1, with a neutral loss or gain
|
|
12
12
|
'[M-H]-', '[2M-H]-', # -1
|
|
13
|
-
'[M-H-CO2]-', '[M-CHO2]-', '[M-H-H2O]-', '[M-H2O-H]-', # -1
|
|
13
|
+
'[M-H-CO2]-', '[M-CHO2]-', '[M-H-H2O]-', '[M-H2O-H]-', # -1, with a neutral loss or gain
|
|
14
14
|
'[M+2H]2+', ] # +2
|
|
15
15
|
min_mz: 50
|
|
16
16
|
max_mz: 1500
|
|
@@ -61,7 +61,7 @@ encoding:
|
|
|
61
61
|
model:
|
|
62
62
|
resolution: 0.2
|
|
63
63
|
max_mz: 1500
|
|
64
|
-
padding_dim: 20 #
|
|
64
|
+
padding_dim: 20 # pads 7500 m/z bins to 7520, a multiple of 2**5 for the 5 pooling layers
|
|
65
65
|
num_add: 6
|
|
66
66
|
input_channels: 1
|
|
67
67
|
|
|
@@ -70,7 +70,6 @@ model:
|
|
|
70
70
|
tcn_kernel_sizes: [45, 43, 41, 39, 37, 35]
|
|
71
71
|
tcn_dropout: 0.2
|
|
72
72
|
|
|
73
|
-
# metadata_dim: 16
|
|
74
73
|
add_embedding_dim: 4
|
|
75
74
|
ce_embedding_dim: 4
|
|
76
75
|
mass_embedding_dim: 4
|
|
@@ -104,18 +103,21 @@ train_rescore:
|
|
|
104
103
|
num_workers: 8
|
|
105
104
|
lr: 0.0001
|
|
106
105
|
early_stop_step: 10
|
|
106
|
+
patience: 3 # epochs without improvement before the LR halves
|
|
107
107
|
|
|
108
108
|
# ------------------------
|
|
109
109
|
# postprocessing settings
|
|
110
110
|
# ------------------------
|
|
111
111
|
post_processing:
|
|
112
|
-
refine_atom_type: ['C', 'O', 'N', 'H']
|
|
113
|
-
refine_atom_num: [5, 5, 5, -1]
|
|
112
|
+
refine_atom_type: ['C', 'O', 'N', 'H'] # atom types refinement may change
|
|
113
|
+
refine_atom_num: [5, 5, 5, -1] # maximum change per atom type from the predicted formula; -1: no limit
|
|
114
114
|
|
|
115
|
-
mass_tolerance: 10.0 #
|
|
115
|
+
mass_tolerance: 10.0 # ppm of the precursor m/z, the unit of the data filters' ppm_tolerance
|
|
116
116
|
ppm_mode: True
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
117
|
+
isotope_errors: [0] # 13C isotope offsets k to search; [0, 1] also tries a precursor read from the M+1 peak
|
|
118
|
+
isotope_penalty: 0.5 # subtracted from the rescore per unit of k
|
|
119
|
+
time_out: -1 # seconds per spectrum; 0 or less: no limit
|
|
120
|
+
maxium_miss_atom_num: 50 # maximum atom-change steps from the predicted formula when the search cannot be enumerated
|
|
121
|
+
top_k: 5 # candidates kept per spectrum, at inference and in rescore training pairs
|
|
120
122
|
buddy_fdr_thr: 0.6
|
|
121
123
|
sirius_score_thr: -5
|