msfiddle 2.2.1__tar.gz → 2.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {msfiddle-2.2.1 → msfiddle-2.3.0}/PKG-INFO +12 -7
  2. {msfiddle-2.2.1 → msfiddle-2.3.0}/README.md +11 -6
  3. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/api.py +220 -73
  4. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/config/fiddle_tcn_casmi.yml +10 -9
  5. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/config/fiddle_tcn_demo.yml +12 -10
  6. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/config/fiddle_tcn_demo_wo_da.yml +12 -10
  7. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/config/fiddle_tcn_embl.yml +29 -22
  8. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/config/fiddle_tcn_orbitrap.yml +18 -19
  9. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/config/fiddle_tcn_qtof.yml +24 -26
  10. msfiddle-2.3.0/msfiddle/config/fiddle_tcn_timstof.yml +142 -0
  11. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/dataset.py +13 -5
  12. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/download.py +59 -81
  13. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/external_results.py +35 -6
  14. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/main.py +10 -4
  15. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/model_tcn.py +17 -0
  16. msfiddle-2.3.0/msfiddle/settings.py +75 -0
  17. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/utils/__init__.py +6 -0
  18. msfiddle-2.3.0/msfiddle/utils/casmi2026.py +231 -0
  19. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/utils/mol_utils.py +0 -23
  20. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/utils/msms_utils.py +96 -48
  21. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/utils/pkl_utils.py +59 -48
  22. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/utils/refine_utils.py +1 -49
  23. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle.egg-info/PKG-INFO +12 -7
  24. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle.egg-info/SOURCES.txt +5 -0
  25. {msfiddle-2.2.1 → msfiddle-2.3.0}/setup.py +1 -1
  26. {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_api.py +122 -1
  27. msfiddle-2.3.0/tests/test_casmi2026.py +305 -0
  28. msfiddle-2.3.0/tests/test_config.py +35 -0
  29. msfiddle-2.3.0/tests/test_download.py +93 -0
  30. {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_external_results.py +46 -0
  31. msfiddle-2.3.0/tests/test_msms_utils.py +118 -0
  32. {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_pkl_utils.py +22 -1
  33. msfiddle-2.2.1/tests/test_download.py +0 -27
  34. msfiddle-2.2.1/tests/test_msms_utils.py +0 -53
  35. {msfiddle-2.2.1 → msfiddle-2.3.0}/LICENSE +0 -0
  36. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/__init__.py +0 -0
  37. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/__main__.py +0 -0
  38. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/demo/input_msms.mgf +0 -0
  39. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/demo/output_buddy.csv +0 -0
  40. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/demo/output_fiddle.csv +0 -0
  41. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/demo/output_fiddle_all.csv +0 -0
  42. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/demo/output_sirius.csv +0 -0
  43. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/evaluate.py +0 -0
  44. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle/optim.py +0 -0
  45. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle.egg-info/dependency_links.txt +0 -0
  46. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle.egg-info/entry_points.txt +0 -0
  47. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle.egg-info/requires.txt +0 -0
  48. {msfiddle-2.2.1 → msfiddle-2.3.0}/msfiddle.egg-info/top_level.txt +0 -0
  49. {msfiddle-2.2.1 → msfiddle-2.3.0}/setup.cfg +0 -0
  50. {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_evaluate.py +0 -0
  51. {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_main.py +0 -0
  52. {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_mol_utils.py +0 -0
  53. {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_muon.py +0 -0
  54. {msfiddle-2.2.1 → msfiddle-2.3.0}/tests/test_refine_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: msfiddle
3
- Version: 2.2.1
3
+ Version: 2.3.0
4
4
  Summary: A package for predicting chemical formulas from tandem mass spectra
5
5
  Home-page: https://github.com/JosieHong/FIDDLE
6
6
  Author: Yuhui Hong
@@ -69,11 +69,11 @@ msfiddle predict --demo --result_path demo.csv # 21 bundled example spect
69
69
  msfiddle predict --test_data spectra.mgf --instrument_type orbitrap --result_path results.csv
70
70
  ```
71
71
 
72
- Use `--instrument_type qtof` for Q-TOF data. Each spectrum needs `TITLE`, `PRECURSOR_MZ`, `PRECURSOR_TYPE` and `COLLISION_ENERGY`. The results list ranked candidates per spectrum in `Refined Formula (0)`, `Refined Formula (1)`, … with their scores in `Rescore (k)`.
72
+ Use `--instrument_type qtof` for Q-TOF data and `--instrument_type timstof` for timsTOF data. Each spectrum needs `TITLE`, `PRECURSOR_MZ`, `PRECURSOR_TYPE` and `COLLISION_ENERGY`. The results list ranked candidates per spectrum in `Refined Formula (0)`, `Refined Formula (1)`, … with their scores in `Rescore (k)`.
73
73
 
74
74
  When the true formulas are known (a `SMILES` or `FORMULA` field in each spectrum), add `--eval` to also report top-k accuracy, saved as `results_eval.csv`.
75
75
 
76
- The [documentation](https://msfiddle.readthedocs.io) covers the Python API, BUDDY and SIRIUS integration, custom checkpoints, and all output columns.
76
+ The [documentation](https://msfiddle.readthedocs.io) covers the Python API, BUDDY and SIRIUS integration, custom checkpoints, all output columns, and [benchmark results](https://msfiddle.readthedocs.io/en/latest/benchmarks.html).
77
77
 
78
78
  ## Reproduce the paper
79
79
 
@@ -85,7 +85,12 @@ This repository also holds the research code for training and evaluation. Run ev
85
85
  conda env create -f environment.yml
86
86
  ```
87
87
 
88
- 2. (optional) Install [BUDDY](https://github.com/Philipbear/msbuddy) and [SIRIUS](https://v6.docs.sirius-ms.io/) for the comparison experiments.
88
+ 2. (optional) Install [BUDDY](https://github.com/Philipbear/msbuddy) and [SIRIUS](https://v6.docs.sirius-ms.io/) for the comparison experiments. SIRIUS needs a free account; log in once as `install_sirius.sh` prints at the end.
89
+
90
+ ```bash
91
+ bash running_scripts/install_buddy.sh
92
+ bash running_scripts/install_sirius.sh
93
+ ```
89
94
 
90
95
  3. Download the models into `./check_point/`, where the scripts expect them:
91
96
 
@@ -95,15 +100,15 @@ msfiddle download -d ./check_point
95
100
 
96
101
  | Script | Description |
97
102
  |---|---|
98
- | `running_scripts/train_release_v2.2.sh` | Build the data and train the release models (TCN and rescore, Q-TOF and Orbitrap) |
103
+ | `running_scripts/train_release_v2.3.sh` | Build the data, train the release models (TCN and rescore) for Q-TOF, Orbitrap and timsTOF in parallel and evaluate them; `INSTRUMENTS=timstof` runs one of them. timsTOF trains on the Enveda CASMI 2026 library at `data/origin/casmi2026/train.parquet` |
99
104
  | `running_scripts/experiments_test_benchmark.sh` | Evaluate on external benchmarks (CASMI 2016, CASMI 2017, EMBL-MCF 2.0) |
100
105
  | `running_scripts/experiments_test_nist23.sh` | Train and evaluate with compounds new in NIST23 held out |
101
106
  | `running_scripts/experiments_test_chimeric.sh` | Evaluate on chimeric spectra |
102
107
  | `running_scripts/experiments_test_noised.sh` | Evaluate on spectra with added noise peaks |
103
108
 
104
- The benchmark, chimeric and noise experiments use the released models; the chimeric and noise ones also need the data built by `train_release_v2.2.sh`.
109
+ The benchmark, chimeric and noise experiments use the released models; the chimeric and noise ones also need the data those models were trained on.
105
110
 
106
- To train from scratch, see `scripts/train_tcn_gpus.py`, `scripts/train_tcn_gpus_cl.py` and `scripts/train_rescore.py` with the configs in [`msfiddle/config/`](https://github.com/JosieHong/FIDDLE/tree/main/msfiddle/config).
111
+ To train from scratch, see `scripts/train_tcn_gpus_cl.py` and `scripts/train_rescore.py` with the configs in [`msfiddle/config/`](https://github.com/JosieHong/FIDDLE/tree/main/msfiddle/config).
107
112
 
108
113
  ## Citation
109
114
 
@@ -28,11 +28,11 @@ msfiddle predict --demo --result_path demo.csv # 21 bundled example spect
28
28
  msfiddle predict --test_data spectra.mgf --instrument_type orbitrap --result_path results.csv
29
29
  ```
30
30
 
31
- Use `--instrument_type qtof` for Q-TOF data. Each spectrum needs `TITLE`, `PRECURSOR_MZ`, `PRECURSOR_TYPE` and `COLLISION_ENERGY`. The results list ranked candidates per spectrum in `Refined Formula (0)`, `Refined Formula (1)`, … with their scores in `Rescore (k)`.
31
+ Use `--instrument_type qtof` for Q-TOF data and `--instrument_type timstof` for timsTOF data. Each spectrum needs `TITLE`, `PRECURSOR_MZ`, `PRECURSOR_TYPE` and `COLLISION_ENERGY`. The results list ranked candidates per spectrum in `Refined Formula (0)`, `Refined Formula (1)`, … with their scores in `Rescore (k)`.
32
32
 
33
33
  When the true formulas are known (a `SMILES` or `FORMULA` field in each spectrum), add `--eval` to also report top-k accuracy, saved as `results_eval.csv`.
34
34
 
35
- The [documentation](https://msfiddle.readthedocs.io) covers the Python API, BUDDY and SIRIUS integration, custom checkpoints, and all output columns.
35
+ The [documentation](https://msfiddle.readthedocs.io) covers the Python API, BUDDY and SIRIUS integration, custom checkpoints, all output columns, and [benchmark results](https://msfiddle.readthedocs.io/en/latest/benchmarks.html).
36
36
 
37
37
  ## Reproduce the paper
38
38
 
@@ -44,7 +44,12 @@ This repository also holds the research code for training and evaluation. Run ev
44
44
  conda env create -f environment.yml
45
45
  ```
46
46
 
47
- 2. (optional) Install [BUDDY](https://github.com/Philipbear/msbuddy) and [SIRIUS](https://v6.docs.sirius-ms.io/) for the comparison experiments.
47
+ 2. (optional) Install [BUDDY](https://github.com/Philipbear/msbuddy) and [SIRIUS](https://v6.docs.sirius-ms.io/) for the comparison experiments. SIRIUS needs a free account; log in once as `install_sirius.sh` prints at the end.
48
+
49
+ ```bash
50
+ bash running_scripts/install_buddy.sh
51
+ bash running_scripts/install_sirius.sh
52
+ ```
48
53
 
49
54
  3. Download the models into `./check_point/`, where the scripts expect them:
50
55
 
@@ -54,15 +59,15 @@ msfiddle download -d ./check_point
54
59
 
55
60
  | Script | Description |
56
61
  |---|---|
57
- | `running_scripts/train_release_v2.2.sh` | Build the data and train the release models (TCN and rescore, Q-TOF and Orbitrap) |
62
+ | `running_scripts/train_release_v2.3.sh` | Build the data, train the release models (TCN and rescore) for Q-TOF, Orbitrap and timsTOF in parallel and evaluate them; `INSTRUMENTS=timstof` runs one of them. timsTOF trains on the Enveda CASMI 2026 library at `data/origin/casmi2026/train.parquet` |
58
63
  | `running_scripts/experiments_test_benchmark.sh` | Evaluate on external benchmarks (CASMI 2016, CASMI 2017, EMBL-MCF 2.0) |
59
64
  | `running_scripts/experiments_test_nist23.sh` | Train and evaluate with compounds new in NIST23 held out |
60
65
  | `running_scripts/experiments_test_chimeric.sh` | Evaluate on chimeric spectra |
61
66
  | `running_scripts/experiments_test_noised.sh` | Evaluate on spectra with added noise peaks |
62
67
 
63
- The benchmark, chimeric and noise experiments use the released models; the chimeric and noise ones also need the data built by `train_release_v2.2.sh`.
68
+ The benchmark, chimeric and noise experiments use the released models; the chimeric and noise ones also need the data those models were trained on.
64
69
 
65
- To train from scratch, see `scripts/train_tcn_gpus.py`, `scripts/train_tcn_gpus_cl.py` and `scripts/train_rescore.py` with the configs in [`msfiddle/config/`](https://github.com/JosieHong/FIDDLE/tree/main/msfiddle/config).
70
+ To train from scratch, see `scripts/train_tcn_gpus_cl.py` and `scripts/train_rescore.py` with the configs in [`msfiddle/config/`](https://github.com/JosieHong/FIDDLE/tree/main/msfiddle/config).
66
71
 
67
72
  ## Citation
68
73
 
@@ -15,9 +15,9 @@ from typing import Any
15
15
 
16
16
  import numpy as np
17
17
  import pandas as pd
18
- import yaml
19
18
  from pyteomics import mgf
20
19
 
20
+ from .settings import load_config
21
21
  from .download import download_models as download_pretrained_models
22
22
  from .download import get_checkpoint_dir
23
23
  from .external_results import load_buddy_results, load_sirius_results
@@ -27,11 +27,21 @@ from .utils.mol_utils import (
27
27
  formula_to_vector,
28
28
  vector_to_formula,
29
29
  )
30
- from .utils.msms_utils import mass_calculator
31
- from .utils.pkl_utils import generate_ms, parse_collision_energy, unify_precursor_type
32
- from .utils.refine_utils import formula_refinement
30
+ from .utils.msms_utils import (
31
+ isotope_mass_shift,
32
+ mass_calculator,
33
+ precursor_window_da,
34
+ )
35
+ from .utils.pkl_utils import (
36
+ generate_ms,
37
+ is_stepped_collision_energy,
38
+ parse_collision_energy,
39
+ resolve_collision_energy,
40
+ unify_precursor_type,
41
+ )
42
+ from .utils.refine_utils import ISOTOPE_MASS, formula_refinement, parse_formula
33
43
 
34
- _INSTRUMENT_TYPES = {"orbitrap", "qtof"}
44
+ _INSTRUMENT_TYPES = {"orbitrap", "qtof", "timstof"}
35
45
  _NEUTRAL_FORMULAS = ("CH4O2", "CH2O2", "H2O", "NH3", "CO2")
36
46
 
37
47
 
@@ -223,46 +233,53 @@ class MsFiddlePredictor:
223
233
  encoded = self._load_mgf_spectra(test_data_path)
224
234
  predictions = self._predict_encoded(encoded, batch_size=batch_size)
225
235
 
226
- buddy_df = load_buddy_results(buddy_path) if buddy_path else None
227
- sirius_df = load_sirius_results(sirius_path) if sirius_path else None
236
+ # Indexed by ID: looked up once per spectrum
237
+ buddy_df = _index_by_id(load_buddy_results(buddy_path)) if buddy_path else None
238
+ sirius_df = (
239
+ _index_by_id(load_sirius_results(sirius_path)) if sirius_path else None
240
+ )
241
+ ids = {prediction.spectrum.id for prediction in predictions}
242
+ _warn_if_unmatched("BUDDY", buddy_df, ids)
243
+ _warn_if_unmatched("SIRIUS", sirius_df, ids)
228
244
 
229
245
  refined_formula = {f"Refined Formula ({k})": [] for k in range(top_k)}
230
246
  refined_mass = {f"Refined Mass ({k})": [] for k in range(top_k)}
231
247
  refined_rescore = {f"Rescore ({k})": [] for k in range(top_k)}
248
+ isotope_shift = {f"Isotope Shift ({k})": [] for k in range(top_k)}
249
+ offsets = self.config["post_processing"]["isotope_errors"]
232
250
  running_time: list[float] = []
233
- neutral_masses: list[float] = []
234
251
 
235
- jobs = [
236
- self._refinement_args(
237
- prediction,
238
- top_k,
239
- extra_formulas=self._external_formula_candidates(
240
- prediction.spectrum.id,
241
- buddy_df=buddy_df,
242
- sirius_df=sirius_df,
243
- ),
252
+ # One refinement per spectrum and isotope offset
253
+ jobs = [] # (spectrum index, offset, refine args, neutral mass)
254
+ for i, prediction in enumerate(predictions):
255
+ extra_formulas = self._external_formula_candidates(
256
+ prediction.spectrum.id, buddy_df=buddy_df, sirius_df=sirius_df
244
257
  )
245
- for prediction in predictions
246
- ]
258
+ for k in offsets:
259
+ refine_args, mass = self._refinement_args(
260
+ prediction, top_k, extra_formulas=extra_formulas, isotope_shift=k
261
+ )
262
+ jobs.append((i, k, refine_args, mass))
247
263
  refined = _run_refinement(
248
- [refine_args for refine_args, _ in jobs], num_workers, self.verbose
264
+ [refine_args for _, _, refine_args, _ in jobs], num_workers, self.verbose
249
265
  )
250
-
251
- rescore_jobs = zip(predictions, jobs, refined)
266
+ parts: list[list] = [[] for _ in predictions]
267
+ refine_times = [0.0] * len(predictions)
268
+ neutral_masses: list[float] = [0.0] * len(predictions)
269
+ for (i, k, _, mass), (refined_results, refine_time) in zip(jobs, refined):
270
+ parts[i].append((k, refined_results))
271
+ refine_times[i] += refine_time
272
+ if k == 0:
273
+ neutral_masses[i] = mass
274
+
275
+ rescore_jobs = zip(predictions, parts, refine_times)
252
276
  if self.verbose:
253
277
  from tqdm import tqdm
254
278
 
255
- rescore_jobs = tqdm(rescore_jobs, total=len(jobs), desc="Rescore")
256
- for (
257
- prediction,
258
- (_, neutral_mass),
259
- (refined_results, refine_time),
260
- ) in rescore_jobs:
279
+ rescore_jobs = tqdm(rescore_jobs, total=len(parts), desc="Rescore")
280
+ for prediction, spectrum_parts, refine_time in rescore_jobs:
261
281
  start_time = time.time()
262
- results = self._rescore_candidates(
263
- prediction.rescore_embedding, refined_results, top_k
264
- )
265
- neutral_masses.append(neutral_mass)
282
+ results = self._rank_candidates(prediction, spectrum_parts, top_k)
266
283
  running_time.append(
267
284
  prediction.prediction_time + refine_time + time.time() - start_time
268
285
  )
@@ -273,6 +290,10 @@ class MsFiddlePredictor:
273
290
  refined_formula[f"Refined Formula ({idx})"].append(formula)
274
291
  refined_mass[f"Refined Mass ({idx})"].append(mass)
275
292
  refined_rescore[f"Rescore ({idx})"].append(score)
293
+ if offsets != [0]:
294
+ isotope_shift[f"Isotope Shift ({idx})"].append(
295
+ results["isotope_shift"][idx]
296
+ )
276
297
 
277
298
  base_columns = {
278
299
  "ID": [prediction.spectrum.id for prediction in predictions],
@@ -284,9 +305,10 @@ class MsFiddlePredictor:
284
305
  "Pred H/C Num": [prediction.pred_hc_count for prediction in predictions],
285
306
  "Running Time": running_time,
286
307
  }
287
- return pd.DataFrame(
288
- {**base_columns, **refined_formula, **refined_mass, **refined_rescore}
289
- )
308
+ columns = {**base_columns, **refined_formula, **refined_mass, **refined_rescore}
309
+ if offsets != [0]:
310
+ columns.update(isotope_shift)
311
+ return pd.DataFrame(columns)
290
312
 
291
313
  def _encode_spectrum_record(
292
314
  self,
@@ -345,7 +367,16 @@ class MsFiddlePredictor:
345
367
  f"{adjusted_precursor_type}"
346
368
  )
347
369
 
348
- ce_str = _collision_energy_to_string(collision_energy)
370
+ ce_str = resolve_collision_energy(
371
+ _collision_energy_to_string(collision_energy),
372
+ encoder.get("stepped_collision_energy", "drop"),
373
+ )
374
+ if ce_str is None:
375
+ raise ValueError(
376
+ f"Stepped collision energy {collision_energy!r} is not supported: "
377
+ "this model was trained on single-energy spectra (the timsTOF "
378
+ "config sets encoding.stepped_collision_energy to 'mean')"
379
+ )
349
380
  ce, nce = parse_collision_energy(
350
381
  ce_str=ce_str,
351
382
  precursor_mz=precursor_mz,
@@ -388,8 +419,10 @@ class MsFiddlePredictor:
388
419
  if self.config["encoding"]["use_simulated_precursor_mz"]
389
420
  else "precursor_mz"
390
421
  )
422
+ stepped_mode = self.config["encoding"].get("stepped_collision_energy", "drop")
391
423
  encoded: list[_EncodedSpectrum] = []
392
424
  invalid_count = 0
425
+ stepped: list[str] = []
393
426
  for spectrum in spectra:
394
427
  params = spectrum.get("params", {})
395
428
  missing = [
@@ -413,6 +446,12 @@ class MsFiddlePredictor:
413
446
  f"('title', '{precursor_mz_key}', 'precursor_type', 'collision_energy')"
414
447
  )
415
448
  continue
449
+ if stepped_mode == "drop" and is_stepped_collision_energy(
450
+ params["collision_energy"]
451
+ ):
452
+ stepped.append(str(params["collision_energy"]))
453
+ invalid_count += 1
454
+ continue
416
455
  try:
417
456
  encoded.append(
418
457
  self._encode_spectrum(
@@ -428,6 +467,13 @@ class MsFiddlePredictor:
428
467
  invalid_count += 1
429
468
  continue
430
469
 
470
+ if stepped:
471
+ warnings.warn(
472
+ f"Skipped {len(stepped)} spectra with a stepped collision energy "
473
+ f"(e.g. {stepped[0]!r}): FIDDLE was trained on single-energy "
474
+ "spectra. Set COLLISION_ENERGY to a single energy to predict them.",
475
+ stacklevel=3,
476
+ )
431
477
  if self.verbose:
432
478
  print(len(spectra) - invalid_count, "spectra left after filtering")
433
479
  print(len(encoded), "spectra loaded into the dataset")
@@ -532,12 +578,7 @@ class MsFiddlePredictor:
532
578
  if not path.exists()
533
579
  ]
534
580
  if missing_paths and self.download_models:
535
- download_pretrained_models(
536
- models=[
537
- f"fiddle_tcn_{self.instrument_type}",
538
- f"fiddle_rescore_{self.instrument_type}",
539
- ]
540
- )
581
+ download_pretrained_models(instruments=[self.instrument_type])
541
582
  missing_paths = [
542
583
  path
543
584
  for path in (self.resume_path, self.rescore_resume_path)
@@ -574,7 +615,7 @@ class MsFiddlePredictor:
574
615
  assert torch is not None
575
616
  assert nn is not None
576
617
 
577
- from .model_tcn import FormulaEncoder, MS2FNet_tcn, RescoreHead
618
+ from .model_tcn import FormulaEncoder, MS2FNet_tcn, RescoreHead, tcn_hash
578
619
 
579
620
  self.device, self.device_ids = _resolve_torch_device(
580
621
  torch,
@@ -622,6 +663,15 @@ class MsFiddlePredictor:
622
663
  )
623
664
  formula_encoder.load_state_dict(ckpt["formula_encoder_state_dict"])
624
665
  rescore_head.load_state_dict(ckpt["rescore_head_state_dict"])
666
+ expected = ckpt.get("tcn_hash") # absent in older checkpoints
667
+ if expected and expected != tcn_hash(state_dict):
668
+ warnings.warn(
669
+ f"The rescore model {self.rescore_resume_path} was trained with a "
670
+ f"different TCN than {self.resume_path}; its scores are not "
671
+ "meaningful with this TCN. Use the two checkpoints of one release "
672
+ "or training run.",
673
+ stacklevel=2,
674
+ )
625
675
  formula_encoder.eval()
626
676
  rescore_head.eval()
627
677
 
@@ -636,16 +686,39 @@ class MsFiddlePredictor:
636
686
  *,
637
687
  extra_formulas: Sequence[str] | None = None,
638
688
  ) -> tuple[dict[str, list[Any]], float]:
639
- refine_args, neutral_mass = self._refinement_args(
640
- prediction, top_k, extra_formulas=extra_formulas
641
- )
642
- return (
643
- self._rescore_candidates(
644
- prediction.rescore_embedding,
645
- formula_refinement(*refine_args),
646
- top_k,
647
- ),
648
- neutral_mass,
689
+ parts = []
690
+ for k in self.config["post_processing"]["isotope_errors"]:
691
+ refine_args, mass = self._refinement_args(
692
+ prediction, top_k, extra_formulas=extra_formulas, isotope_shift=k
693
+ )
694
+ parts.append((k, formula_refinement(*refine_args)))
695
+ if k == 0:
696
+ neutral_mass = mass
697
+ return self._rank_candidates(prediction, parts, top_k), neutral_mass
698
+
699
+ def _rank_candidates(
700
+ self,
701
+ prediction: _ModelPrediction,
702
+ parts: Sequence[tuple[int, dict[str, list[Any]]]],
703
+ top_k: int,
704
+ ) -> dict[str, list[Any]]:
705
+ """Rescore the candidates refined at each isotope offset k and rank them.
706
+
707
+ With the single offset 0 this is plain rescoring. Otherwise the
708
+ candidates are pooled (a formula found at several offsets keeps the
709
+ smallest k) and ranked by rescore - isotope_penalty * k.
710
+ """
711
+ if len(parts) == 1:
712
+ return self._rescore_candidates(
713
+ prediction.rescore_embedding, parts[0][1], top_k
714
+ )
715
+ merged, shift_of = _merge_isotope_refinements(parts)
716
+ results = self._rescore_candidates(prediction.rescore_embedding, merged, top_k)
717
+ return _apply_isotope_penalty(
718
+ results,
719
+ shift_of,
720
+ self.config["post_processing"]["isotope_penalty"],
721
+ top_k,
649
722
  )
650
723
 
651
724
  def _refinement_args(
@@ -654,13 +727,23 @@ class MsFiddlePredictor:
654
727
  top_k: int,
655
728
  *,
656
729
  extra_formulas: Sequence[str] | None = None,
730
+ isotope_shift: int = 0,
657
731
  ) -> tuple[tuple, float]:
658
- """Positional arguments of formula_refinement for one prediction."""
732
+ """Positional arguments of formula_refinement for one prediction, and
733
+ the neutral mass it refines around. ``isotope_shift`` = k assumes the
734
+ precursor m/z was read from the k-th 13C isotope peak."""
735
+ spectrum = prediction.spectrum
736
+ # Aliases such as [M+FA-H]- are encoded under their canonical name
737
+ adduct = unify_precursor_type(spectrum.precursor_type)
659
738
  neutral_mass = float(
660
- mass_calculator(
661
- prediction.spectrum.precursor_type,
662
- prediction.spectrum.precursor_mz,
663
- )
739
+ mass_calculator(adduct, spectrum.precursor_mz)
740
+ ) - isotope_mass_shift(adduct, isotope_shift)
741
+ pp = self.config["post_processing"]
742
+ # A ppm tolerance applies to the precursor m/z, like the data filters
743
+ window = (
744
+ precursor_window_da(adduct, spectrum.precursor_mz, pp["mass_tolerance"])
745
+ if pp["ppm_mode"]
746
+ else pp["mass_tolerance"]
664
747
  )
665
748
  f0_list = [prediction.pred_formula]
666
749
  if extra_formulas:
@@ -679,8 +762,8 @@ class MsFiddlePredictor:
679
762
  refine_args = (
680
763
  f0_list,
681
764
  neutral_mass,
682
- self.config["post_processing"]["mass_tolerance"],
683
- self.config["post_processing"]["ppm_mode"],
765
+ window,
766
+ False, # window is in Da
684
767
  top_k,
685
768
  self.config["post_processing"]["maxium_miss_atom_num"],
686
769
  self.config["post_processing"]["time_out"],
@@ -748,15 +831,20 @@ class MsFiddlePredictor:
748
831
  ) -> list[dict[str, Any]]:
749
832
  candidates = []
750
833
  base_metadata = self._metadata_for_prediction(prediction, neutral_mass)
751
- for formula, mass, raw_score in zip(
752
- refined_results["formula"],
753
- refined_results["mass"],
754
- refined_results["rescore"],
834
+ shifts = refined_results.get("isotope_shift", [])
835
+ for idx, (formula, mass, raw_score) in enumerate(
836
+ zip(
837
+ refined_results["formula"],
838
+ refined_results["mass"],
839
+ refined_results["rescore"],
840
+ )
755
841
  ):
756
842
  if formula is None:
757
843
  continue
758
844
  metadata = dict(base_metadata)
759
845
  metadata["raw_rescore"] = float(raw_score)
846
+ if shifts:
847
+ metadata["isotope_shift"] = shifts[idx]
760
848
  candidates.append(
761
849
  {
762
850
  "formula": formula,
@@ -797,9 +885,8 @@ class MsFiddlePredictor:
797
885
  ) -> list[str]:
798
886
  candidates: list[str] = []
799
887
  if buddy_df is not None:
800
- rows = buddy_df.loc[buddy_df["ID"] == spec_id]
801
- if len(rows) > 0:
802
- row = rows.iloc[0]
888
+ if spec_id in buddy_df.index:
889
+ row = buddy_df.loc[spec_id]
803
890
  formulas = row[
804
891
  [
805
892
  "Pred Formula (1)",
@@ -826,9 +913,8 @@ class MsFiddlePredictor:
826
913
  and score < self.config["post_processing"]["buddy_fdr_thr"]
827
914
  )
828
915
  if sirius_df is not None:
829
- rows = sirius_df.loc[sirius_df["ID"] == spec_id]
830
- if len(rows) > 0:
831
- row = rows.iloc[0]
916
+ if spec_id in sirius_df.index:
917
+ row = sirius_df.loc[spec_id]
832
918
  formulas = row[
833
919
  [
834
920
  "Pred Formula (1)",
@@ -857,6 +943,63 @@ class MsFiddlePredictor:
857
943
  return candidates
858
944
 
859
945
 
946
+ def _index_by_id(df: pd.DataFrame) -> pd.DataFrame:
947
+ """One row per spectrum ID (the first, as before), indexed by ID."""
948
+ return df.drop_duplicates("ID").set_index("ID")
949
+
950
+
951
+ def _warn_if_unmatched(tool: str, df: pd.DataFrame | None, ids: set[str]) -> None:
952
+ """Warn when none of a tool's results belongs to a spectrum being predicted."""
953
+ if df is not None and len(df) and ids.isdisjoint(df.index):
954
+ warnings.warn(
955
+ f"No {tool} result matches a spectrum title, so {tool} adds no "
956
+ f"candidates. Its IDs look like {df.index[0]!r}.",
957
+ stacklevel=3,
958
+ )
959
+
960
+
961
+ def _merge_isotope_refinements(
962
+ parts: Sequence[tuple[int, dict[str, list[Any]]]],
963
+ ) -> tuple[dict[str, list[Any]], dict[str, int]]:
964
+ """Pool the candidates refined at each isotope offset. A formula found at
965
+ several offsets keeps the smallest one."""
966
+ formulas, masses, shift_of = [], [], {}
967
+ for k, refined in sorted(parts, key=lambda part: part[0]):
968
+ for formula, mass in zip(refined["formula"], refined["mass"]):
969
+ if formula is not None and formula not in shift_of:
970
+ shift_of[formula] = k
971
+ formulas.append(formula)
972
+ masses.append(mass)
973
+ return {"formula": formulas, "mass": masses}, shift_of
974
+
975
+
976
+ def _apply_isotope_penalty(
977
+ results: dict[str, list[Any]],
978
+ shift_of: Mapping[str, int],
979
+ penalty: float,
980
+ top_k: int,
981
+ ) -> dict[str, list[Any]]:
982
+ """Rank rescored candidates by rescore - penalty * isotope shift, keep top_k."""
983
+ ranked = sorted(
984
+ (
985
+ (rescore - penalty * shift_of[formula], formula, mass, rescore)
986
+ for formula, mass, rescore in zip(
987
+ results["formula"], results["mass"], results["rescore"]
988
+ )
989
+ if formula is not None
990
+ ),
991
+ key=lambda item: item[0],
992
+ reverse=True,
993
+ )[:top_k]
994
+ pad = [None] * (top_k - len(ranked))
995
+ return {
996
+ "formula": [r[1] for r in ranked] + pad,
997
+ "mass": [r[2] for r in ranked] + pad,
998
+ "rescore": [r[3] for r in ranked] + [0.0] * len(pad),
999
+ "isotope_shift": [shift_of[r[1]] for r in ranked] + pad,
1000
+ }
1001
+
1002
+
860
1003
  def _timed_refinement(refine_args: tuple) -> tuple[dict[str, list[Any]], float]:
861
1004
  start_time = time.time()
862
1005
  return formula_refinement(*refine_args), time.time() - start_time
@@ -1001,16 +1144,20 @@ def _normalize_instrument_type(instrument_type: str) -> str:
1001
1144
 
1002
1145
 
1003
1146
  def _is_external_formula(value: object) -> bool:
1147
+ """A BUDDY or SIRIUS formula that refinement can use: present, and made only
1148
+ of elements FIDDLE models (SIRIUS may propose Si or Se, for example)."""
1004
1149
  if pd.isna(value):
1005
1150
  return False
1006
- return str(value).strip().lower() not in {"", "na", "nan", "none", "null", "<na>"}
1151
+ text = str(value).strip()
1152
+ if text.lower() in {"", "na", "nan", "none", "null", "<na>"}:
1153
+ return False
1154
+ return parse_formula(text).keys() <= ISOTOPE_MASS.keys()
1007
1155
 
1008
1156
 
1009
1157
  def _load_config(config_path: Path) -> dict[str, Any]:
1010
1158
  if not config_path.exists():
1011
1159
  raise FileNotFoundError(f"Config file not found: {config_path}")
1012
- with config_path.open("r") as handle:
1013
- return yaml.safe_load(handle)
1160
+ return load_config(config_path)
1014
1161
 
1015
1162
 
1016
1163
  def _validate_top_k(top_k: int) -> int:
@@ -68,7 +68,7 @@ encoding:
68
68
  model:
69
69
  resolution: 0.2
70
70
  max_mz: 1500
71
- padding_dim: 20 # 1500 / 0.2 = 7500; 7500 / 2**5 (# pooling layer) = 234.375; 235 * (2**5) = 7520
71
+ padding_dim: 20 # pads 7500 m/z bins to 7520, a multiple of 2**5 for the 5 pooling layers
72
72
  num_add: 6
73
73
  input_channels: 1
74
74
 
@@ -77,7 +77,6 @@ model:
77
77
  tcn_kernel_sizes: [45, 43, 41, 39, 37, 35]
78
78
  tcn_dropout: 0.2
79
79
 
80
- # metadata_dim: 16
81
80
  add_embedding_dim: 4
82
81
  ce_embedding_dim: 4
83
82
  mass_embedding_dim: 4
@@ -94,20 +93,22 @@ model:
94
93
  # ------------------------
95
94
  # training settings
96
95
  # ------------------------
97
- # unimplemented
96
+ # not used: this config is for testing only
98
97
 
99
98
  # ------------------------
100
99
  # postprocessing settings
101
100
  # ------------------------
102
101
  post_processing:
103
- refine_atom_type: ['C', 'O', 'N', 'H']
104
- refine_atom_num: [3, 3, 3, -1]
102
+ refine_atom_type: ['C', 'O', 'N', 'H'] # atom types refinement may change
103
+ refine_atom_num: [3, 3, 3, -1] # maximum change per atom type from the predicted formula; -1: no limit
105
104
 
106
- mass_tolerance: 5.0 # target space
105
+ mass_tolerance: 10.0 # ppm of the precursor m/z, the unit of the data filters' ppm_tolerance
107
106
  ppm_mode: True
108
- time_out: -1 # do not limit the time
109
- maxium_miss_atom_num: 50 # search space
110
- top_k: 5
107
+ isotope_errors: [0] # 13C isotope offsets k to search; [0, 1] also tries a precursor read from the M+1 peak
108
+ isotope_penalty: 0.5 # subtracted from the rescore per unit of k
109
+ time_out: -1 # seconds per spectrum; 0 or less: no limit
110
+ maxium_miss_atom_num: 50 # maximum atom-change steps from the predicted formula when the search cannot be enumerated
111
+ top_k: 5 # candidates kept per spectrum, at inference and in rescore training pairs
111
112
  buddy_fdr_thr: 0.6
112
113
  sirius_score_thr: -5
113
114
 
@@ -8,9 +8,9 @@ mona_qtof:
8
8
  ms_level: 'MS2'
9
9
  atom_type: ['C', 'O', 'N', 'H', 'P', 'S', 'F', 'Cl', 'B', 'Br', 'I', 'Na', 'K']
10
10
  precursor_type: ['[M+H]+', '[M+Na]+', '[2M+H]+', # +1
11
- '[M+H-H2O]+', '[M-H2O+H]+', '[M+H-2H2O]+', '[M+H-NH3]+', '[M+H+NH3]+', '[M+NH4]+', '[M+H-CH2O2]+', '[M+H-CH4O2]+', # +1+-neutrue
11
+ '[M+H-H2O]+', '[M-H2O+H]+', '[M+H-2H2O]+', '[M+H-NH3]+', '[M+H+NH3]+', '[M+NH4]+', '[M+H-CH2O2]+', '[M+H-CH4O2]+', # +1, with a neutral loss or gain
12
12
  '[M-H]-', '[2M-H]-', # -1
13
- '[M-H-CO2]-', '[M-CHO2]-', '[M-H-H2O]-', '[M-H2O-H]-', # -1+-neutrue
13
+ '[M-H-CO2]-', '[M-CHO2]-', '[M-H-H2O]-', '[M-H2O-H]-', # -1, with a neutral loss or gain
14
14
  '[M+2H]2+', ] # +2
15
15
  min_mz: 50
16
16
  max_mz: 1500
@@ -61,7 +61,7 @@ encoding:
61
61
  model:
62
62
  resolution: 0.2
63
63
  max_mz: 1500
64
- padding_dim: 20 # 1500 / 0.2 = 7500; 7500 / 2**5 (# pooling layer) = 234.375; 235 * (2**5) = 7520
64
+ padding_dim: 20 # pads 7500 m/z bins to 7520, a multiple of 2**5 for the 5 pooling layers
65
65
  num_add: 6
66
66
  input_channels: 1
67
67
 
@@ -70,7 +70,6 @@ model:
70
70
  tcn_kernel_sizes: [45, 43, 41, 39, 37, 35]
71
71
  tcn_dropout: 0.2
72
72
 
73
- # metadata_dim: 16
74
73
  add_embedding_dim: 4
75
74
  ce_embedding_dim: 4
76
75
  mass_embedding_dim: 4
@@ -104,18 +103,21 @@ train_rescore:
104
103
  num_workers: 8
105
104
  lr: 0.0001
106
105
  early_stop_step: 10
106
+ patience: 3 # epochs without improvement before the LR halves
107
107
 
108
108
  # ------------------------
109
109
  # postprocessing settings
110
110
  # ------------------------
111
111
  post_processing:
112
- refine_atom_type: ['C', 'O', 'N', 'H']
113
- refine_atom_num: [5, 5, 5, -1]
112
+ refine_atom_type: ['C', 'O', 'N', 'H'] # atom types refinement may change
113
+ refine_atom_num: [5, 5, 5, -1] # maximum change per atom type from the predicted formula; -1: no limit
114
114
 
115
- mass_tolerance: 10.0 # target space
115
+ mass_tolerance: 10.0 # ppm of the precursor m/z, the unit of the data filters' ppm_tolerance
116
116
  ppm_mode: True
117
- time_out: -1 # do not limit the time
118
- maxium_miss_atom_num: 50 # search space
119
- top_k: 5
117
+ isotope_errors: [0] # 13C isotope offsets k to search; [0, 1] also tries a precursor read from the M+1 peak
118
+ isotope_penalty: 0.5 # subtracted from the rescore per unit of k
119
+ time_out: -1 # seconds per spectrum; 0 or less: no limit
120
+ maxium_miss_atom_num: 50 # maximum atom-change steps from the predicted formula when the search cannot be enumerated
121
+ top_k: 5 # candidates kept per spectrum, at inference and in rescore training pairs
120
122
  buddy_fdr_thr: 0.6
121
123
  sirius_score_thr: -5