msfiddle 2.3.0__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {msfiddle-2.3.0 → msfiddle-2.4.0}/PKG-INFO +7 -4
- {msfiddle-2.3.0 → msfiddle-2.4.0}/README.md +4 -1
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/__init__.py +2 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/api.py +276 -63
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/config/fiddle_tcn_casmi.yml +1 -1
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/config/fiddle_tcn_embl.yml +4 -4
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/config/fiddle_tcn_timstof.yml +2 -2
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/demo/input_msms.mgf +21 -21
- msfiddle-2.4.0/msfiddle/evaluate.py +378 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/external_results.py +8 -1
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/main.py +88 -6
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/utils/msms_utils.py +124 -26
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle.egg-info/PKG-INFO +7 -4
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle.egg-info/SOURCES.txt +2 -1
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle.egg-info/requires.txt +2 -2
- {msfiddle-2.3.0 → msfiddle-2.4.0}/setup.py +3 -3
- msfiddle-2.4.0/tests/test_evaluate.py +185 -0
- msfiddle-2.4.0/tests/test_main.py +79 -0
- msfiddle-2.4.0/tests/test_msms_utils.py +208 -0
- msfiddle-2.4.0/tests/test_validate.py +154 -0
- msfiddle-2.3.0/msfiddle/evaluate.py +0 -152
- msfiddle-2.3.0/tests/test_evaluate.py +0 -62
- msfiddle-2.3.0/tests/test_main.py +0 -43
- msfiddle-2.3.0/tests/test_msms_utils.py +0 -118
- {msfiddle-2.3.0 → msfiddle-2.4.0}/LICENSE +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/__main__.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/config/fiddle_tcn_demo.yml +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/config/fiddle_tcn_demo_wo_da.yml +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/config/fiddle_tcn_orbitrap.yml +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/config/fiddle_tcn_qtof.yml +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/dataset.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/demo/output_buddy.csv +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/demo/output_fiddle.csv +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/demo/output_fiddle_all.csv +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/demo/output_sirius.csv +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/download.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/model_tcn.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/optim.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/settings.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/utils/__init__.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/utils/casmi2026.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/utils/mol_utils.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/utils/pkl_utils.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle/utils/refine_utils.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle.egg-info/dependency_links.txt +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle.egg-info/entry_points.txt +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/msfiddle.egg-info/top_level.txt +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/setup.cfg +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/tests/test_api.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/tests/test_casmi2026.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/tests/test_config.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/tests/test_download.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/tests/test_external_results.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/tests/test_mol_utils.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/tests/test_muon.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/tests/test_pkl_utils.py +0 -0
- {msfiddle-2.3.0 → msfiddle-2.4.0}/tests/test_refine_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: msfiddle
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.4.0
|
|
4
4
|
Summary: A package for predicting chemical formulas from tandem mass spectra
|
|
5
5
|
Home-page: https://github.com/JosieHong/FIDDLE
|
|
6
6
|
Author: Yuhui Hong
|
|
@@ -15,14 +15,14 @@ Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
|
|
|
15
15
|
Requires-Python: >=3.8
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
License-File: LICENSE
|
|
18
|
-
Requires-Dist: numpy
|
|
18
|
+
Requires-Dist: numpy>=1.20.0
|
|
19
19
|
Requires-Dist: pandas>=2.0.0
|
|
20
20
|
Requires-Dist: tqdm>=4.60.0
|
|
21
21
|
Requires-Dist: pyyaml>=6.0
|
|
22
22
|
Requires-Dist: scikit-learn>=1.0.0
|
|
23
23
|
Requires-Dist: scipy>=1.8.0
|
|
24
24
|
Requires-Dist: pyarrow>=10.0.0
|
|
25
|
-
Requires-Dist: rdkit
|
|
25
|
+
Requires-Dist: rdkit!=2026.9.2,>=2022.03.5
|
|
26
26
|
Requires-Dist: molmass
|
|
27
27
|
Requires-Dist: pyteomics
|
|
28
28
|
Provides-Extra: inference
|
|
@@ -69,9 +69,11 @@ msfiddle predict --demo --result_path demo.csv # 21 bundled example spect
|
|
|
69
69
|
msfiddle predict --test_data spectra.mgf --instrument_type orbitrap --result_path results.csv
|
|
70
70
|
```
|
|
71
71
|
|
|
72
|
+
The `inference` extra installs PyTorch. For a particular PyTorch build (another CUDA version, CPU only), install it [from pytorch.org](https://pytorch.org/get-started/locally/) first, then `pip install msfiddle`.
|
|
73
|
+
|
|
72
74
|
Use `--instrument_type qtof` for Q-TOF data and `--instrument_type timstof` for timsTOF data. Each spectrum needs `TITLE`, `PRECURSOR_MZ`, `PRECURSOR_TYPE` and `COLLISION_ENERGY`. The results list ranked candidates per spectrum in `Refined Formula (0)`, `Refined Formula (1)`, … with their scores in `Rescore (k)`.
|
|
73
75
|
|
|
74
|
-
When the true formulas are known (a `SMILES` or `FORMULA` field in each spectrum), add `--eval` to also report top-k accuracy, saved as `results_eval.csv`.
|
|
76
|
+
When the true formulas are known (a `SMILES` or `FORMULA` field in each spectrum), add `--eval` to also report top-k accuracy, saved as `results_eval.csv`. `msfiddle validate --test_data your.mgf` checks a file without the models and says why any spectrum would be skipped; if accuracy is lower than expected, see [troubleshooting](https://msfiddle.readthedocs.io/en/latest/troubleshooting.html).
|
|
75
77
|
|
|
76
78
|
The [documentation](https://msfiddle.readthedocs.io) covers the Python API, BUDDY and SIRIUS integration, custom checkpoints, all output columns, and [benchmark results](https://msfiddle.readthedocs.io/en/latest/benchmarks.html).
|
|
77
79
|
|
|
@@ -100,6 +102,7 @@ msfiddle download -d ./check_point
|
|
|
100
102
|
|
|
101
103
|
| Script | Description |
|
|
102
104
|
|---|---|
|
|
105
|
+
| `running_scripts/train_release_v2.4.sh` | As `train_release_v2.3.sh`, but as in the paper the theoretical precursor m/z of NIST20 and NIST23 is replaced by the theoretical m/z plus a seeded Gaussian error within the instrument's tolerance; trains one instrument after another on two GPUs |
|
|
103
106
|
| `running_scripts/train_release_v2.3.sh` | Build the data, train the release models (TCN and rescore) for Q-TOF, Orbitrap and timsTOF in parallel and evaluate them; `INSTRUMENTS=timstof` runs one of them. timsTOF trains on the Enveda CASMI 2026 library at `data/origin/casmi2026/train.parquet` |
|
|
104
107
|
| `running_scripts/experiments_test_benchmark.sh` | Evaluate on external benchmarks (CASMI 2016, CASMI 2017, EMBL-MCF 2.0) |
|
|
105
108
|
| `running_scripts/experiments_test_nist23.sh` | Train and evaluate with compounds new in NIST23 held out |
|
|
@@ -28,9 +28,11 @@ msfiddle predict --demo --result_path demo.csv # 21 bundled example spect
|
|
|
28
28
|
msfiddle predict --test_data spectra.mgf --instrument_type orbitrap --result_path results.csv
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
+
The `inference` extra installs PyTorch. For a particular PyTorch build (another CUDA version, CPU only), install it [from pytorch.org](https://pytorch.org/get-started/locally/) first, then `pip install msfiddle`.
|
|
32
|
+
|
|
31
33
|
Use `--instrument_type qtof` for Q-TOF data and `--instrument_type timstof` for timsTOF data. Each spectrum needs `TITLE`, `PRECURSOR_MZ`, `PRECURSOR_TYPE` and `COLLISION_ENERGY`. The results list ranked candidates per spectrum in `Refined Formula (0)`, `Refined Formula (1)`, … with their scores in `Rescore (k)`.
|
|
32
34
|
|
|
33
|
-
When the true formulas are known (a `SMILES` or `FORMULA` field in each spectrum), add `--eval` to also report top-k accuracy, saved as `results_eval.csv`.
|
|
35
|
+
When the true formulas are known (a `SMILES` or `FORMULA` field in each spectrum), add `--eval` to also report top-k accuracy, saved as `results_eval.csv`. `msfiddle validate --test_data your.mgf` checks a file without the models and says why any spectrum would be skipped; if accuracy is lower than expected, see [troubleshooting](https://msfiddle.readthedocs.io/en/latest/troubleshooting.html).
|
|
34
36
|
|
|
35
37
|
The [documentation](https://msfiddle.readthedocs.io) covers the Python API, BUDDY and SIRIUS integration, custom checkpoints, all output columns, and [benchmark results](https://msfiddle.readthedocs.io/en/latest/benchmarks.html).
|
|
36
38
|
|
|
@@ -59,6 +61,7 @@ msfiddle download -d ./check_point
|
|
|
59
61
|
|
|
60
62
|
| Script | Description |
|
|
61
63
|
|---|---|
|
|
64
|
+
| `running_scripts/train_release_v2.4.sh` | As `train_release_v2.3.sh`, but as in the paper the theoretical precursor m/z of NIST20 and NIST23 is replaced by the theoretical m/z plus a seeded Gaussian error within the instrument's tolerance; trains one instrument after another on two GPUs |
|
|
62
65
|
| `running_scripts/train_release_v2.3.sh` | Build the data, train the release models (TCN and rescore) for Q-TOF, Orbitrap and timsTOF in parallel and evaluate them; `INSTRUMENTS=timstof` runs one of them. timsTOF trains on the Enveda CASMI 2026 library at `data/origin/casmi2026/train.parquet` |
|
|
63
66
|
| `running_scripts/experiments_test_benchmark.sh` | Evaluate on external benchmarks (CASMI 2016, CASMI 2017, EMBL-MCF 2.0) |
|
|
64
67
|
| `running_scripts/experiments_test_nist23.sh` | Train and evaluate with compounds new in NIST23 held out |
|
|
@@ -14,6 +14,7 @@ from .api import (
|
|
|
14
14
|
predict_batch_from_spectra,
|
|
15
15
|
predict_from_mgf,
|
|
16
16
|
predict_from_spectrum,
|
|
17
|
+
validate_mgf,
|
|
17
18
|
)
|
|
18
19
|
from .utils.mol_utils import vector_to_formula, formula_to_vector
|
|
19
20
|
from .utils.msms_utils import mass_calculator
|
|
@@ -39,6 +40,7 @@ __all__ = [
|
|
|
39
40
|
"predict_from_spectrum",
|
|
40
41
|
"predict_batch_from_spectra",
|
|
41
42
|
"predict_from_mgf",
|
|
43
|
+
"validate_mgf",
|
|
42
44
|
"MS2FNet_tcn",
|
|
43
45
|
"FormulaEncoder",
|
|
44
46
|
"RescoreHead",
|
|
@@ -45,6 +45,29 @@ _INSTRUMENT_TYPES = {"orbitrap", "qtof", "timstof"}
|
|
|
45
45
|
_NEUTRAL_FORMULAS = ("CH4O2", "CH2O2", "H2O", "NH3", "CO2")
|
|
46
46
|
|
|
47
47
|
|
|
48
|
+
# Why a spectrum is left out of prediction. predict and validate share them.
|
|
49
|
+
REJECTION_REASONS = (
|
|
50
|
+
"missing_metadata",
|
|
51
|
+
"duplicate_id",
|
|
52
|
+
"invalid_precursor_mz",
|
|
53
|
+
"unsupported_adduct",
|
|
54
|
+
"insufficient_peaks",
|
|
55
|
+
"invalid_peaks",
|
|
56
|
+
"peak_mz_out_of_range",
|
|
57
|
+
"stepped_collision_energy",
|
|
58
|
+
"invalid_collision_energy",
|
|
59
|
+
"encoding_failure",
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class SpectrumRejected(ValueError):
|
|
64
|
+
"""A spectrum the model cannot take; ``reason`` is one of REJECTION_REASONS."""
|
|
65
|
+
|
|
66
|
+
def __init__(self, reason: str, message: str) -> None:
|
|
67
|
+
super().__init__(message)
|
|
68
|
+
self.reason = reason
|
|
69
|
+
|
|
70
|
+
|
|
48
71
|
@dataclass
|
|
49
72
|
class _EncodedSpectrum:
|
|
50
73
|
id: str
|
|
@@ -56,6 +79,20 @@ class _EncodedSpectrum:
|
|
|
56
79
|
neutral_add: np.ndarray
|
|
57
80
|
|
|
58
81
|
|
|
82
|
+
@dataclass
|
|
83
|
+
class _SpectrumCheck:
|
|
84
|
+
index: int
|
|
85
|
+
id: str
|
|
86
|
+
status: str # ok, warning or rejected
|
|
87
|
+
reason: str
|
|
88
|
+
message: str
|
|
89
|
+
value: str = "" # the offending metadata value, for summaries
|
|
90
|
+
|
|
91
|
+
@classmethod
|
|
92
|
+
def rejected(cls, index, id, reason, message, value=""):
|
|
93
|
+
return cls(index, id, "rejected", reason, message, value)
|
|
94
|
+
|
|
95
|
+
|
|
59
96
|
@dataclass
|
|
60
97
|
class _ModelPrediction:
|
|
61
98
|
spectrum: _EncodedSpectrum
|
|
@@ -358,13 +395,18 @@ class MsFiddlePredictor:
|
|
|
358
395
|
charge=charge,
|
|
359
396
|
)
|
|
360
397
|
if not good_spec:
|
|
361
|
-
raise
|
|
398
|
+
raise SpectrumRejected(
|
|
399
|
+
"encoding_failure",
|
|
400
|
+
"Invalid spectrum: no peaks left to encode after removing the "
|
|
401
|
+
"precursor isotope peaks, or all intensities are equal",
|
|
402
|
+
)
|
|
362
403
|
|
|
363
404
|
neutral_add, adjusted_precursor_type = _melt_neutral_precursor(precursor_type)
|
|
364
405
|
if adjusted_precursor_type not in encoder["precursor_type"]:
|
|
365
|
-
raise
|
|
406
|
+
raise SpectrumRejected(
|
|
407
|
+
"unsupported_adduct",
|
|
366
408
|
"Unsupported adduct after neutral-loss normalization: "
|
|
367
|
-
f"{adjusted_precursor_type}"
|
|
409
|
+
f"{adjusted_precursor_type}",
|
|
368
410
|
)
|
|
369
411
|
|
|
370
412
|
ce_str = resolve_collision_energy(
|
|
@@ -372,10 +414,11 @@ class MsFiddlePredictor:
|
|
|
372
414
|
encoder.get("stepped_collision_energy", "drop"),
|
|
373
415
|
)
|
|
374
416
|
if ce_str is None:
|
|
375
|
-
raise
|
|
417
|
+
raise SpectrumRejected(
|
|
418
|
+
"stepped_collision_energy",
|
|
376
419
|
f"Stepped collision energy {collision_energy!r} is not supported: "
|
|
377
420
|
"this model was trained on single-energy spectra (the timsTOF "
|
|
378
|
-
"config sets encoding.stepped_collision_energy to 'mean')"
|
|
421
|
+
"config sets encoding.stepped_collision_energy to 'mean')",
|
|
379
422
|
)
|
|
380
423
|
ce, nce = parse_collision_energy(
|
|
381
424
|
ce_str=ce_str,
|
|
@@ -383,7 +426,10 @@ class MsFiddlePredictor:
|
|
|
383
426
|
charge=abs(charge),
|
|
384
427
|
)
|
|
385
428
|
if ce is None and nce is None:
|
|
386
|
-
raise
|
|
429
|
+
raise SpectrumRejected(
|
|
430
|
+
"invalid_collision_energy",
|
|
431
|
+
f"Could not parse collision energy: {collision_energy!r}",
|
|
432
|
+
)
|
|
387
433
|
|
|
388
434
|
env_arr = np.array(
|
|
389
435
|
[
|
|
@@ -403,80 +449,188 @@ class MsFiddlePredictor:
|
|
|
403
449
|
neutral_add=np.array(neutral_add, dtype=np.float32),
|
|
404
450
|
)
|
|
405
451
|
|
|
406
|
-
def
|
|
452
|
+
def validate_mgf(self, test_data_path: str | os.PathLike[str]) -> pd.DataFrame:
|
|
453
|
+
"""Check every spectrum of an MGF file without loading the models.
|
|
454
|
+
|
|
455
|
+
Returns one row per spectrum in file order: ``index``, ``id``,
|
|
456
|
+
``status`` (``ok``, ``warning`` or ``rejected``), ``reason`` (one of
|
|
457
|
+
``REJECTION_REASONS`` for a rejected spectrum, else the warning codes
|
|
458
|
+
joined by ``;``) and ``message``. ``predict_mgf`` keeps exactly the
|
|
459
|
+
spectra whose status is not ``rejected``.
|
|
460
|
+
"""
|
|
461
|
+
rows = [
|
|
462
|
+
{
|
|
463
|
+
"index": check.index,
|
|
464
|
+
"id": check.id,
|
|
465
|
+
"status": check.status,
|
|
466
|
+
"reason": check.reason,
|
|
467
|
+
"message": check.message,
|
|
468
|
+
}
|
|
469
|
+
for _, check in self._check_mgf_spectra(test_data_path)
|
|
470
|
+
]
|
|
471
|
+
return pd.DataFrame(
|
|
472
|
+
rows, columns=["index", "id", "status", "reason", "message"]
|
|
473
|
+
)
|
|
474
|
+
|
|
475
|
+
def _check_mgf_spectra(
|
|
407
476
|
self, test_data_path: str | os.PathLike[str]
|
|
408
|
-
) -> list[_EncodedSpectrum]:
|
|
477
|
+
) -> list[tuple[_EncodedSpectrum | None, _SpectrumCheck]]:
|
|
409
478
|
path = Path(test_data_path)
|
|
410
479
|
if not path.exists():
|
|
411
480
|
raise FileNotFoundError(f"MGF file not found: {path}")
|
|
412
481
|
|
|
413
|
-
|
|
482
|
+
# Read sequentially: the indexed reader that mgf.read returns by default
|
|
483
|
+
# silently drops untitled spectra and all but the last of a repeated
|
|
484
|
+
# TITLE. Those are now rejected with a reason, keeping the same spectra.
|
|
485
|
+
spectra = list(mgf.read(str(path), use_index=False))
|
|
414
486
|
if self.verbose:
|
|
415
487
|
print(len(spectra), "spectra loaded from", str(path))
|
|
416
488
|
|
|
489
|
+
encoder = self.config["encoding"]
|
|
417
490
|
precursor_mz_key = (
|
|
418
491
|
"simulated_precursor_mz"
|
|
419
|
-
if
|
|
492
|
+
if encoder["use_simulated_precursor_mz"]
|
|
420
493
|
else "precursor_mz"
|
|
421
494
|
)
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
495
|
+
required = ("title", precursor_mz_key, "precursor_type", "collision_energy")
|
|
496
|
+
stepped_mode = encoder.get("stepped_collision_energy", "drop")
|
|
497
|
+
titles = [str(s.get("params", {}).get("title", "")) for s in spectra]
|
|
498
|
+
title_counts: dict[str, int] = {}
|
|
499
|
+
last_index: dict[str, int] = {}
|
|
500
|
+
for idx, title in enumerate(titles):
|
|
501
|
+
title_counts[title] = title_counts.get(title, 0) + 1
|
|
502
|
+
last_index[title] = idx
|
|
503
|
+
|
|
504
|
+
results = []
|
|
505
|
+
for idx, spectrum in enumerate(spectra):
|
|
427
506
|
params = spectrum.get("params", {})
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
for key in (
|
|
431
|
-
"title",
|
|
432
|
-
precursor_mz_key,
|
|
433
|
-
"precursor_type",
|
|
434
|
-
"collision_energy",
|
|
435
|
-
)
|
|
436
|
-
if key not in params
|
|
437
|
-
]
|
|
507
|
+
spec_id = titles[idx]
|
|
508
|
+
missing = [key for key in required if key not in params]
|
|
438
509
|
if missing:
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
510
|
+
results.append(
|
|
511
|
+
(
|
|
512
|
+
None,
|
|
513
|
+
_SpectrumCheck.rejected(
|
|
514
|
+
idx,
|
|
515
|
+
spec_id,
|
|
516
|
+
"missing_metadata",
|
|
517
|
+
"Missing MGF field(s): "
|
|
518
|
+
+ ", ".join(key.upper() for key in missing),
|
|
519
|
+
),
|
|
443
520
|
)
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
521
|
+
)
|
|
522
|
+
continue
|
|
523
|
+
if last_index[spec_id] != idx:
|
|
524
|
+
results.append(
|
|
525
|
+
(
|
|
526
|
+
None,
|
|
527
|
+
_SpectrumCheck.rejected(
|
|
528
|
+
idx,
|
|
529
|
+
spec_id,
|
|
530
|
+
"duplicate_id",
|
|
531
|
+
f"TITLE {spec_id!r} is repeated; only its last "
|
|
532
|
+
"spectrum is predicted",
|
|
533
|
+
),
|
|
447
534
|
)
|
|
535
|
+
)
|
|
448
536
|
continue
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
537
|
+
ce = params["collision_energy"]
|
|
538
|
+
if stepped_mode == "drop" and is_stepped_collision_energy(ce):
|
|
539
|
+
results.append(
|
|
540
|
+
(
|
|
541
|
+
None,
|
|
542
|
+
_SpectrumCheck.rejected(
|
|
543
|
+
idx,
|
|
544
|
+
spec_id,
|
|
545
|
+
"stepped_collision_energy",
|
|
546
|
+
f"Stepped collision energy {str(ce)!r}: this model "
|
|
547
|
+
"was trained on single-energy spectra",
|
|
548
|
+
str(ce),
|
|
549
|
+
),
|
|
550
|
+
)
|
|
551
|
+
)
|
|
454
552
|
continue
|
|
455
553
|
try:
|
|
456
|
-
encoded.
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
554
|
+
encoded = self._encode_spectrum(
|
|
555
|
+
mz_array=spectrum.get("m/z array", []),
|
|
556
|
+
intensity_array=spectrum.get("intensity array", []),
|
|
557
|
+
precursor_mz=params[precursor_mz_key],
|
|
558
|
+
adduct=params["precursor_type"],
|
|
559
|
+
collision_energy=ce,
|
|
560
|
+
spec_id=spec_id,
|
|
561
|
+
)
|
|
562
|
+
except SpectrumRejected as exc:
|
|
563
|
+
results.append(
|
|
564
|
+
(None, _SpectrumCheck.rejected(idx, spec_id, exc.reason, str(exc)))
|
|
565
|
+
)
|
|
566
|
+
continue
|
|
567
|
+
except ValueError as exc:
|
|
568
|
+
results.append(
|
|
569
|
+
(
|
|
570
|
+
None,
|
|
571
|
+
_SpectrumCheck.rejected(
|
|
572
|
+
idx, spec_id, "encoding_failure", str(exc)
|
|
573
|
+
),
|
|
464
574
|
)
|
|
465
575
|
)
|
|
466
|
-
except ValueError:
|
|
467
|
-
invalid_count += 1
|
|
468
576
|
continue
|
|
469
577
|
|
|
578
|
+
warning_codes, warning_messages = [], []
|
|
579
|
+
if is_stepped_collision_energy(ce):
|
|
580
|
+
warning_codes.append("stepped_collision_energy_averaged")
|
|
581
|
+
warning_messages.append(
|
|
582
|
+
f"Stepped collision energy {str(ce)!r} encoded at its mean "
|
|
583
|
+
f"({encoded.collision_energy})"
|
|
584
|
+
)
|
|
585
|
+
if title_counts[spec_id] > 1:
|
|
586
|
+
warning_codes.append("duplicate_id")
|
|
587
|
+
warning_messages.append(
|
|
588
|
+
f"TITLE {spec_id!r} is shared by {title_counts[spec_id]} "
|
|
589
|
+
"spectra; only this last one is predicted"
|
|
590
|
+
)
|
|
591
|
+
check = _SpectrumCheck(
|
|
592
|
+
idx,
|
|
593
|
+
spec_id,
|
|
594
|
+
"warning" if warning_codes else "ok",
|
|
595
|
+
";".join(warning_codes),
|
|
596
|
+
"; ".join(warning_messages),
|
|
597
|
+
)
|
|
598
|
+
results.append((encoded, check))
|
|
599
|
+
return results
|
|
600
|
+
|
|
601
|
+
def _load_mgf_spectra(
|
|
602
|
+
self, test_data_path: str | os.PathLike[str]
|
|
603
|
+
) -> list[_EncodedSpectrum]:
|
|
604
|
+
results = self._check_mgf_spectra(test_data_path)
|
|
605
|
+
# A repeated TITLE keeps its last spectrum at its first position, as
|
|
606
|
+
# the indexed reader did, so prediction output is unchanged.
|
|
607
|
+
first_index: dict[str, int] = {}
|
|
608
|
+
for _, check in results:
|
|
609
|
+
first_index.setdefault(check.id, check.index)
|
|
610
|
+
encoded = sorted(
|
|
611
|
+
(spectrum for spectrum, _ in results if spectrum is not None),
|
|
612
|
+
key=lambda spectrum: first_index[spectrum.id],
|
|
613
|
+
)
|
|
614
|
+
rejected = [check for _, check in results if check.status == "rejected"]
|
|
615
|
+
|
|
616
|
+
stepped = [c for c in rejected if c.reason == "stepped_collision_energy"]
|
|
470
617
|
if stepped:
|
|
471
618
|
warnings.warn(
|
|
472
619
|
f"Skipped {len(stepped)} spectra with a stepped collision energy "
|
|
473
|
-
f"(e.g. {stepped[0]!r}): FIDDLE was trained on
|
|
474
|
-
"spectra. Set COLLISION_ENERGY to a single energy
|
|
620
|
+
f"(e.g. {stepped[0].value!r}): FIDDLE was trained on "
|
|
621
|
+
"single-energy spectra. Set COLLISION_ENERGY to a single energy "
|
|
622
|
+
"to predict them.",
|
|
475
623
|
stacklevel=3,
|
|
476
624
|
)
|
|
477
625
|
if self.verbose:
|
|
478
|
-
|
|
479
|
-
|
|
626
|
+
counts: dict[str, int] = {}
|
|
627
|
+
for check in rejected:
|
|
628
|
+
counts[check.reason] = counts.get(check.reason, 0) + 1
|
|
629
|
+
for reason, count in sorted(counts.items()):
|
|
630
|
+
print(f"Skipped {count} spectra: {reason}")
|
|
631
|
+
if counts:
|
|
632
|
+
print("Run 'msfiddle validate' for the reason for each spectrum")
|
|
633
|
+
print(len(encoded), "spectra left after filtering")
|
|
480
634
|
return encoded
|
|
481
635
|
|
|
482
636
|
def _predict_encoded(
|
|
@@ -1133,6 +1287,42 @@ def predict_from_mgf(
|
|
|
1133
1287
|
)
|
|
1134
1288
|
|
|
1135
1289
|
|
|
1290
|
+
def validate_mgf(
|
|
1291
|
+
test_data_path: str | os.PathLike[str],
|
|
1292
|
+
*,
|
|
1293
|
+
instrument_type: str = "orbitrap",
|
|
1294
|
+
config_path: str | os.PathLike[str] | None = None,
|
|
1295
|
+
) -> pd.DataFrame:
|
|
1296
|
+
"""Check each spectrum of an MGF file against an instrument's encoding,
|
|
1297
|
+
without loading or downloading checkpoints. See
|
|
1298
|
+
``MsFiddlePredictor.validate_mgf`` for the returned columns."""
|
|
1299
|
+
predictor = MsFiddlePredictor(
|
|
1300
|
+
instrument_type=instrument_type, config_path=config_path
|
|
1301
|
+
)
|
|
1302
|
+
return predictor.validate_mgf(test_data_path)
|
|
1303
|
+
|
|
1304
|
+
|
|
1305
|
+
def summarize_validation(report: pd.DataFrame) -> str:
|
|
1306
|
+
"""A short text summary of a ``validate_mgf`` report."""
|
|
1307
|
+
n = len(report)
|
|
1308
|
+
rejected = report[report["status"] == "rejected"]
|
|
1309
|
+
warned = report[report["status"] == "warning"]
|
|
1310
|
+
lines = [
|
|
1311
|
+
f"{n} spectra: {n - len(rejected)} usable "
|
|
1312
|
+
f"({len(warned)} with warnings), {len(rejected)} rejected"
|
|
1313
|
+
]
|
|
1314
|
+
if len(rejected):
|
|
1315
|
+
lines.append("Rejected (prediction skips these):")
|
|
1316
|
+
for reason, count in rejected["reason"].value_counts().items():
|
|
1317
|
+
lines.append(f" {reason:<28s}{count}")
|
|
1318
|
+
if len(warned):
|
|
1319
|
+
codes = warned["reason"].str.split(";").explode().value_counts()
|
|
1320
|
+
lines.append("Warnings (predicted, but may be less reliable):")
|
|
1321
|
+
for code, count in codes.items():
|
|
1322
|
+
lines.append(f" {code:<28s}{count}")
|
|
1323
|
+
return "\n".join(lines)
|
|
1324
|
+
|
|
1325
|
+
|
|
1136
1326
|
def _normalize_instrument_type(instrument_type: str) -> str:
|
|
1137
1327
|
normalized = str(instrument_type).lower()
|
|
1138
1328
|
if normalized not in _INSTRUMENT_TYPES:
|
|
@@ -1194,32 +1384,55 @@ def _validate_arrays(
|
|
|
1194
1384
|
mz_values = np.asarray(mz_array, dtype=float)
|
|
1195
1385
|
intensity_values = np.asarray(intensity_array, dtype=float)
|
|
1196
1386
|
if mz_values.ndim != 1 or intensity_values.ndim != 1:
|
|
1197
|
-
raise
|
|
1387
|
+
raise SpectrumRejected(
|
|
1388
|
+
"invalid_peaks", "mz_array and intensity_array must be one-dimensional"
|
|
1389
|
+
)
|
|
1198
1390
|
if len(mz_values) != len(intensity_values):
|
|
1199
|
-
raise
|
|
1391
|
+
raise SpectrumRejected(
|
|
1392
|
+
"invalid_peaks",
|
|
1200
1393
|
"mz_array and intensity_array must have the same length: "
|
|
1201
|
-
f"{len(mz_values)} != {len(intensity_values)}"
|
|
1394
|
+
f"{len(mz_values)} != {len(intensity_values)}",
|
|
1202
1395
|
)
|
|
1203
1396
|
if len(mz_values) < min_peak_num:
|
|
1204
|
-
raise
|
|
1397
|
+
raise SpectrumRejected(
|
|
1398
|
+
"insufficient_peaks",
|
|
1399
|
+
f"Spectrum must contain at least {min_peak_num} peaks "
|
|
1400
|
+
f"(has {len(mz_values)})",
|
|
1401
|
+
)
|
|
1205
1402
|
if not np.isfinite(mz_values).all() or not np.isfinite(intensity_values).all():
|
|
1206
|
-
raise
|
|
1403
|
+
raise SpectrumRejected(
|
|
1404
|
+
"invalid_peaks",
|
|
1405
|
+
"mz_array and intensity_array must contain only finite values",
|
|
1406
|
+
)
|
|
1207
1407
|
if not np.isfinite(intensity_values.max() - intensity_values.min()):
|
|
1208
|
-
raise
|
|
1408
|
+
raise SpectrumRejected("invalid_peaks", "Invalid intensity array")
|
|
1409
|
+
# A negative m/z would bin at a negative index, i.e. among the highest m/z
|
|
1410
|
+
if float(np.min(mz_values)) < 0:
|
|
1411
|
+
raise SpectrumRejected("invalid_peaks", "m/z values must not be negative")
|
|
1209
1412
|
if float(np.max(mz_values)) < min_mz:
|
|
1210
|
-
raise
|
|
1413
|
+
raise SpectrumRejected(
|
|
1414
|
+
"peak_mz_out_of_range", f"Spectrum maximum m/z must be at least {min_mz}"
|
|
1415
|
+
)
|
|
1211
1416
|
if float(np.max(mz_values)) > max_mz:
|
|
1212
|
-
raise
|
|
1417
|
+
raise SpectrumRejected(
|
|
1418
|
+
"peak_mz_out_of_range",
|
|
1419
|
+
f"Spectrum maximum m/z must be no greater than {max_mz}",
|
|
1420
|
+
)
|
|
1213
1421
|
return mz_values, intensity_values
|
|
1214
1422
|
|
|
1215
1423
|
|
|
1216
1424
|
def _validate_float(name: str, value: Any) -> float:
|
|
1425
|
+
reason = f"invalid_{name}"
|
|
1217
1426
|
try:
|
|
1218
1427
|
result = float(value)
|
|
1219
1428
|
except (TypeError, ValueError) as exc:
|
|
1220
|
-
raise
|
|
1221
|
-
|
|
1222
|
-
|
|
1429
|
+
raise SpectrumRejected(
|
|
1430
|
+
reason, f"{name} must be numeric, got {value!r}"
|
|
1431
|
+
) from exc
|
|
1432
|
+
if not math.isfinite(result) or result <= 0:
|
|
1433
|
+
raise SpectrumRejected(
|
|
1434
|
+
reason, f"{name} must be finite and positive, got {value!r}"
|
|
1435
|
+
)
|
|
1223
1436
|
return result
|
|
1224
1437
|
|
|
1225
1438
|
|
|
@@ -1227,7 +1440,7 @@ def _validate_adduct(adduct: Any, encoder: Mapping[str, Any]) -> str:
|
|
|
1227
1440
|
precursor_type = str(adduct)
|
|
1228
1441
|
normalized = unify_precursor_type(precursor_type)
|
|
1229
1442
|
if normalized not in encoder["type2charge"]:
|
|
1230
|
-
raise
|
|
1443
|
+
raise SpectrumRejected("unsupported_adduct", f"Unsupported adduct: {adduct!r}")
|
|
1231
1444
|
return precursor_type
|
|
1232
1445
|
|
|
1233
1446
|
|
|
@@ -34,7 +34,7 @@ encoding:
|
|
|
34
34
|
resolution: 0.2
|
|
35
35
|
max_mz: 1500
|
|
36
36
|
max_atom_num: 300
|
|
37
|
-
use_simulated_precursor_mz:
|
|
37
|
+
use_simulated_precursor_mz: False # benchmarks use the measured PRECURSOR_MZ; the simulated one is drawn around the true mass
|
|
38
38
|
precursor_type:
|
|
39
39
|
'[M+H]+': 1
|
|
40
40
|
'[M-H]-': 2
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# ------------------------
|
|
2
|
-
# EMBL MCF 2.0 (Orbitrap Exploris 240):
|
|
3
|
-
#
|
|
4
|
-
#
|
|
2
|
+
# EMBL MCF 2.0 (Orbitrap Exploris 240): scripts/embl2mgf.py filters the
|
|
3
|
+
# library with the `embl` section. The benchmark predicts with the Orbitrap
|
|
4
|
+
# release config (fiddle_tcn_orbitrap.yml); the settings below follow it
|
|
5
5
|
# ------------------------
|
|
6
6
|
embl:
|
|
7
7
|
atom_type: ['C', 'O', 'N', 'H', 'P', 'S', 'F', 'Cl', 'B', 'Br', 'I', 'Na', 'K']
|
|
@@ -96,7 +96,7 @@ post_processing:
|
|
|
96
96
|
refine_atom_type: ['C', 'O', 'N', 'H', 'Cl', 'Br', 'F', 'I', 'S', 'P'] # atom types refinement may change
|
|
97
97
|
refine_atom_num: [5, 5, 5, -1, 2, 1, 2, 1, 2, 1] # maximum change per atom type from the predicted formula; -1: no limit
|
|
98
98
|
|
|
99
|
-
mass_tolerance: 50.0 # ppm of the precursor m/z;
|
|
99
|
+
mass_tolerance: 50.0 # ppm of the precursor m/z; the benchmark uses fiddle_tcn_orbitrap.yml (5 ppm)
|
|
100
100
|
ppm_mode: True
|
|
101
101
|
isotope_errors: [0] # 13C isotope offsets k to search; [0, 1] also tries a precursor read from the M+1 peak
|
|
102
102
|
isotope_penalty: 0.5 # subtracted from the rescore per unit of k
|
|
@@ -106,8 +106,8 @@ model:
|
|
|
106
106
|
# ------------------------
|
|
107
107
|
train:
|
|
108
108
|
epochs: 200
|
|
109
|
-
batch_size:
|
|
110
|
-
grad_accum_steps:
|
|
109
|
+
batch_size: 256 # per step; 512 takes about 25 GB with --amp, 256 split over two 11 GB GPUs about 6 GB each
|
|
110
|
+
grad_accum_steps: 2 # optimizer step every N batches; effective batch = batch_size x N = 512, as for v2.3
|
|
111
111
|
num_workers: 8
|
|
112
112
|
lr: 0.002 # as Q-TOF; 0.001 when fine-tuning from a Q-TOF checkpoint
|
|
113
113
|
muon_lr: 0.005 # Muon's rate for weight matrices; the default 0.02 diverged on timsTOF
|