msfiddle 2.2.0__tar.gz → 2.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- msfiddle-2.2.1/PKG-INFO +123 -0
- msfiddle-2.2.1/README.md +82 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/__init__.py +0 -8
- msfiddle-2.2.1/msfiddle/__main__.py +5 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/api.py +101 -42
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/dataset.py +1 -38
- msfiddle-2.2.1/msfiddle/evaluate.py +152 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/main.py +90 -110
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/utils/msms_utils.py +0 -15
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/utils/refine_utils.py +0 -11
- msfiddle-2.2.1/msfiddle.egg-info/PKG-INFO +123 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle.egg-info/SOURCES.txt +4 -4
- {msfiddle-2.2.0 → msfiddle-2.2.1}/setup.py +1 -1
- {msfiddle-2.2.0 → msfiddle-2.2.1}/tests/test_api.py +32 -71
- msfiddle-2.2.1/tests/test_download.py +27 -0
- msfiddle-2.2.1/tests/test_evaluate.py +62 -0
- msfiddle-2.2.1/tests/test_external_results.py +163 -0
- msfiddle-2.2.1/tests/test_main.py +43 -0
- msfiddle-2.2.1/tests/test_mol_utils.py +47 -0
- msfiddle-2.2.1/tests/test_msms_utils.py +53 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/tests/test_muon.py +4 -25
- msfiddle-2.2.1/tests/test_pkl_utils.py +43 -0
- msfiddle-2.2.1/tests/test_refine_utils.py +130 -0
- msfiddle-2.2.0/PKG-INFO +0 -169
- msfiddle-2.2.0/README.md +0 -128
- msfiddle-2.2.0/msfiddle.egg-info/PKG-INFO +0 -169
- msfiddle-2.2.0/tests/test_download.py +0 -29
- msfiddle-2.2.0/tests/test_external_results.py +0 -282
- msfiddle-2.2.0/tests/test_generate_ms.py +0 -52
- msfiddle-2.2.0/tests/test_main.py +0 -17
- msfiddle-2.2.0/tests/test_mol_utils.py +0 -94
- msfiddle-2.2.0/tests/test_msms_utils.py +0 -86
- msfiddle-2.2.0/tests/test_preprocessing_speedups.py +0 -148
- msfiddle-2.2.0/tests/test_refine_enumeration.py +0 -136
- msfiddle-2.2.0/tests/test_refine_fast_mass.py +0 -106
- msfiddle-2.2.0/tests/test_refine_utils.py +0 -64
- {msfiddle-2.2.0 → msfiddle-2.2.1}/LICENSE +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/config/fiddle_tcn_casmi.yml +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/config/fiddle_tcn_demo.yml +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/config/fiddle_tcn_demo_wo_da.yml +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/config/fiddle_tcn_embl.yml +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/config/fiddle_tcn_orbitrap.yml +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/config/fiddle_tcn_qtof.yml +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/demo/input_msms.mgf +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/demo/output_buddy.csv +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/demo/output_fiddle.csv +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/demo/output_fiddle_all.csv +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/demo/output_sirius.csv +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/download.py +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/external_results.py +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/model_tcn.py +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/optim.py +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/utils/__init__.py +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/utils/mol_utils.py +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle/utils/pkl_utils.py +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle.egg-info/dependency_links.txt +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle.egg-info/entry_points.txt +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle.egg-info/requires.txt +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/msfiddle.egg-info/top_level.txt +0 -0
- {msfiddle-2.2.0 → msfiddle-2.2.1}/setup.cfg +0 -0
msfiddle-2.2.1/PKG-INFO
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: msfiddle
|
|
3
|
+
Version: 2.2.1
|
|
4
|
+
Summary: A package for predicting chemical formulas from tandem mass spectra
|
|
5
|
+
Home-page: https://github.com/JosieHong/FIDDLE
|
|
6
|
+
Author: Yuhui Hong
|
|
7
|
+
Author-email: josieexception@outlook.com
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Chemistry
|
|
12
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
|
|
15
|
+
Requires-Python: >=3.8
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: numpy<2.0.0,>=1.20.0
|
|
19
|
+
Requires-Dist: pandas>=2.0.0
|
|
20
|
+
Requires-Dist: tqdm>=4.60.0
|
|
21
|
+
Requires-Dist: pyyaml>=6.0
|
|
22
|
+
Requires-Dist: scikit-learn>=1.0.0
|
|
23
|
+
Requires-Dist: scipy>=1.8.0
|
|
24
|
+
Requires-Dist: pyarrow>=10.0.0
|
|
25
|
+
Requires-Dist: rdkit>=2022.03.5
|
|
26
|
+
Requires-Dist: molmass
|
|
27
|
+
Requires-Dist: pyteomics
|
|
28
|
+
Provides-Extra: inference
|
|
29
|
+
Requires-Dist: torch; extra == "inference"
|
|
30
|
+
Dynamic: author
|
|
31
|
+
Dynamic: author-email
|
|
32
|
+
Dynamic: classifier
|
|
33
|
+
Dynamic: description
|
|
34
|
+
Dynamic: description-content-type
|
|
35
|
+
Dynamic: home-page
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
Dynamic: provides-extra
|
|
38
|
+
Dynamic: requires-dist
|
|
39
|
+
Dynamic: requires-python
|
|
40
|
+
Dynamic: summary
|
|
41
|
+
|
|
42
|
+
<p align="center">
|
|
43
|
+
<a href="https://github.com/JosieHong/FIDDLE">
|
|
44
|
+
<picture>
|
|
45
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/JosieHong/FIDDLE/main/img/fiddle-logo-dark@2x.png">
|
|
46
|
+
<img src="https://raw.githubusercontent.com/JosieHong/FIDDLE/main/img/fiddle-logo-light@2x.png" alt="FIDDLE" width="480">
|
|
47
|
+
</picture>
|
|
48
|
+
</a>
|
|
49
|
+
</p>
|
|
50
|
+
|
|
51
|
+
[](https://opensource.org/licenses/Apache-2.0)
|
|
52
|
+
[](https://doi.org/10.5281/zenodo.19181279)
|
|
53
|
+
[](https://github.com/JosieHong/FIDDLE/releases)
|
|
54
|
+
|
|
55
|
+
FIDDLE predicts molecular formulas from MS/MS spectra with deep learning.
|
|
56
|
+
|
|
57
|
+
**[Paper](https://www.nature.com/articles/s41467-025-66060-9)** · **[Documentation](https://msfiddle.readthedocs.io)** · **[Try the demo on Hugging Face](https://huggingface.co/spaces/J0siee/FIDDLE)**
|
|
58
|
+
|
|
59
|
+
## Quick start
|
|
60
|
+
|
|
61
|
+
Install the [`msfiddle`](https://pypi.org/project/msfiddle/) package, download the pre-trained models once, then predict formulas:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
pip install "msfiddle[inference]"
|
|
65
|
+
msfiddle download
|
|
66
|
+
msfiddle predict --demo --result_path demo.csv # 21 bundled example spectra
|
|
67
|
+
|
|
68
|
+
# your own spectra
|
|
69
|
+
msfiddle predict --test_data spectra.mgf --instrument_type orbitrap --result_path results.csv
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Use `--instrument_type qtof` for Q-TOF data. Each spectrum needs `TITLE`, `PRECURSOR_MZ`, `PRECURSOR_TYPE` and `COLLISION_ENERGY`. The results list ranked candidates per spectrum in `Refined Formula (0)`, `Refined Formula (1)`, … with their scores in `Rescore (k)`.
|
|
73
|
+
|
|
74
|
+
When the true formulas are known (a `SMILES` or `FORMULA` field in each spectrum), add `--eval` to also report top-k accuracy, saved as `results_eval.csv`.
|
|
75
|
+
|
|
76
|
+
The [documentation](https://msfiddle.readthedocs.io) covers the Python API, BUDDY and SIRIUS integration, custom checkpoints, and all output columns.
|
|
77
|
+
|
|
78
|
+
## Reproduce the paper
|
|
79
|
+
|
|
80
|
+
This repository also holds the research code for training and evaluation. Run everything from the repository root (`FIDDLE/`).
|
|
81
|
+
|
|
82
|
+
1. Create the environment with [Anaconda](https://docs.anaconda.com/anaconda/install/) (use `environment_mac.yml` on Apple Silicon):
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
conda env create -f environment.yml
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
2. (optional) Install [BUDDY](https://github.com/Philipbear/msbuddy) and [SIRIUS](https://v6.docs.sirius-ms.io/) for the comparison experiments.
|
|
89
|
+
|
|
90
|
+
3. Download the models into `./check_point/`, where the scripts expect them:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
msfiddle download -d ./check_point
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
| Script | Description |
|
|
97
|
+
|---|---|
|
|
98
|
+
| `running_scripts/train_release_v2.2.sh` | Build the data and train the release models (TCN and rescore, Q-TOF and Orbitrap) |
|
|
99
|
+
| `running_scripts/experiments_test_benchmark.sh` | Evaluate on external benchmarks (CASMI 2016, CASMI 2017, EMBL-MCF 2.0) |
|
|
100
|
+
| `running_scripts/experiments_test_nist23.sh` | Train and evaluate with compounds new in NIST23 held out |
|
|
101
|
+
| `running_scripts/experiments_test_chimeric.sh` | Evaluate on chimeric spectra |
|
|
102
|
+
| `running_scripts/experiments_test_noised.sh` | Evaluate on spectra with added noise peaks |
|
|
103
|
+
|
|
104
|
+
The benchmark, chimeric and noise experiments use the released models; the chimeric and noise ones also need the data built by `train_release_v2.2.sh`.
|
|
105
|
+
|
|
106
|
+
To train from scratch, see `scripts/train_tcn_gpus.py`, `scripts/train_tcn_gpus_cl.py` and `scripts/train_rescore.py` with the configs in [`msfiddle/config/`](https://github.com/JosieHong/FIDDLE/tree/main/msfiddle/config).
|
|
107
|
+
|
|
108
|
+
## Citation
|
|
109
|
+
|
|
110
|
+
If you use FIDDLE, please cite:
|
|
111
|
+
|
|
112
|
+
```
|
|
113
|
+
@article{hong2025fiddle,
|
|
114
|
+
title={FIDDLE: a deep learning method for chemical formulas prediction from tandem mass spectra},
|
|
115
|
+
author={Hong, Yuhui and Li, Sujun and Ye, Yuzhen and Tang, Haixu},
|
|
116
|
+
journal={Nature Communications},
|
|
117
|
+
volume={16},
|
|
118
|
+
number={1},
|
|
119
|
+
pages={11102},
|
|
120
|
+
year={2025},
|
|
121
|
+
publisher={Nature Publishing Group UK London}
|
|
122
|
+
}
|
|
123
|
+
```
|
msfiddle-2.2.1/README.md
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<a href="https://github.com/JosieHong/FIDDLE">
|
|
3
|
+
<picture>
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/JosieHong/FIDDLE/main/img/fiddle-logo-dark@2x.png">
|
|
5
|
+
<img src="https://raw.githubusercontent.com/JosieHong/FIDDLE/main/img/fiddle-logo-light@2x.png" alt="FIDDLE" width="480">
|
|
6
|
+
</picture>
|
|
7
|
+
</a>
|
|
8
|
+
</p>
|
|
9
|
+
|
|
10
|
+
[](https://opensource.org/licenses/Apache-2.0)
|
|
11
|
+
[](https://doi.org/10.5281/zenodo.19181279)
|
|
12
|
+
[](https://github.com/JosieHong/FIDDLE/releases)
|
|
13
|
+
|
|
14
|
+
FIDDLE predicts molecular formulas from MS/MS spectra with deep learning.
|
|
15
|
+
|
|
16
|
+
**[Paper](https://www.nature.com/articles/s41467-025-66060-9)** · **[Documentation](https://msfiddle.readthedocs.io)** · **[Try the demo on Hugging Face](https://huggingface.co/spaces/J0siee/FIDDLE)**
|
|
17
|
+
|
|
18
|
+
## Quick start
|
|
19
|
+
|
|
20
|
+
Install the [`msfiddle`](https://pypi.org/project/msfiddle/) package, download the pre-trained models once, then predict formulas:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install "msfiddle[inference]"
|
|
24
|
+
msfiddle download
|
|
25
|
+
msfiddle predict --demo --result_path demo.csv # 21 bundled example spectra
|
|
26
|
+
|
|
27
|
+
# your own spectra
|
|
28
|
+
msfiddle predict --test_data spectra.mgf --instrument_type orbitrap --result_path results.csv
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Use `--instrument_type qtof` for Q-TOF data. Each spectrum needs `TITLE`, `PRECURSOR_MZ`, `PRECURSOR_TYPE` and `COLLISION_ENERGY`. The results list ranked candidates per spectrum in `Refined Formula (0)`, `Refined Formula (1)`, … with their scores in `Rescore (k)`.
|
|
32
|
+
|
|
33
|
+
When the true formulas are known (a `SMILES` or `FORMULA` field in each spectrum), add `--eval` to also report top-k accuracy, saved as `results_eval.csv`.
|
|
34
|
+
|
|
35
|
+
The [documentation](https://msfiddle.readthedocs.io) covers the Python API, BUDDY and SIRIUS integration, custom checkpoints, and all output columns.
|
|
36
|
+
|
|
37
|
+
## Reproduce the paper
|
|
38
|
+
|
|
39
|
+
This repository also holds the research code for training and evaluation. Run everything from the repository root (`FIDDLE/`).
|
|
40
|
+
|
|
41
|
+
1. Create the environment with [Anaconda](https://docs.anaconda.com/anaconda/install/) (use `environment_mac.yml` on Apple Silicon):
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
conda env create -f environment.yml
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
2. (optional) Install [BUDDY](https://github.com/Philipbear/msbuddy) and [SIRIUS](https://v6.docs.sirius-ms.io/) for the comparison experiments.
|
|
48
|
+
|
|
49
|
+
3. Download the models into `./check_point/`, where the scripts expect them:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
msfiddle download -d ./check_point
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
| Script | Description |
|
|
56
|
+
|---|---|
|
|
57
|
+
| `running_scripts/train_release_v2.2.sh` | Build the data and train the release models (TCN and rescore, Q-TOF and Orbitrap) |
|
|
58
|
+
| `running_scripts/experiments_test_benchmark.sh` | Evaluate on external benchmarks (CASMI 2016, CASMI 2017, EMBL-MCF 2.0) |
|
|
59
|
+
| `running_scripts/experiments_test_nist23.sh` | Train and evaluate with compounds new in NIST23 held out |
|
|
60
|
+
| `running_scripts/experiments_test_chimeric.sh` | Evaluate on chimeric spectra |
|
|
61
|
+
| `running_scripts/experiments_test_noised.sh` | Evaluate on spectra with added noise peaks |
|
|
62
|
+
|
|
63
|
+
The benchmark, chimeric and noise experiments use the released models; the chimeric and noise ones also need the data built by `train_release_v2.2.sh`.
|
|
64
|
+
|
|
65
|
+
To train from scratch, see `scripts/train_tcn_gpus.py`, `scripts/train_tcn_gpus_cl.py` and `scripts/train_rescore.py` with the configs in [`msfiddle/config/`](https://github.com/JosieHong/FIDDLE/tree/main/msfiddle/config).
|
|
66
|
+
|
|
67
|
+
## Citation
|
|
68
|
+
|
|
69
|
+
If you use FIDDLE, please cite:
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
@article{hong2025fiddle,
|
|
73
|
+
title={FIDDLE: a deep learning method for chemical formulas prediction from tandem mass spectra},
|
|
74
|
+
author={Hong, Yuhui and Li, Sujun and Ye, Yuzhen and Tang, Haixu},
|
|
75
|
+
journal={Nature Communications},
|
|
76
|
+
volume={16},
|
|
77
|
+
number={1},
|
|
78
|
+
pages={11102},
|
|
79
|
+
year={2025},
|
|
80
|
+
publisher={Nature Publishing Group UK London}
|
|
81
|
+
}
|
|
82
|
+
```
|
|
@@ -30,12 +30,6 @@ def __getattr__(name):
|
|
|
30
30
|
globals()["FormulaEncoder"] = FormulaEncoder
|
|
31
31
|
globals()["RescoreHead"] = RescoreHead
|
|
32
32
|
return globals()[name]
|
|
33
|
-
if name in ("test_step", "rescore_candidates"):
|
|
34
|
-
from .main import rescore_candidates, test_step
|
|
35
|
-
|
|
36
|
-
globals()["test_step"] = test_step
|
|
37
|
-
globals()["rescore_candidates"] = rescore_candidates
|
|
38
|
-
return globals()[name]
|
|
39
33
|
raise AttributeError(f"module 'msfiddle' has no attribute {name!r}")
|
|
40
34
|
|
|
41
35
|
|
|
@@ -52,8 +46,6 @@ __all__ = [
|
|
|
52
46
|
"mass_calculator",
|
|
53
47
|
"vector_to_formula",
|
|
54
48
|
"formula_to_vector",
|
|
55
|
-
"test_step",
|
|
56
|
-
"rescore_candidates",
|
|
57
49
|
"download_models",
|
|
58
50
|
"check_models_exist",
|
|
59
51
|
"get_model_path",
|
|
@@ -55,6 +55,8 @@ class _ModelPrediction:
|
|
|
55
55
|
pred_atom_count: float
|
|
56
56
|
pred_hc_count: float
|
|
57
57
|
prediction_time: float
|
|
58
|
+
# L2-normalised embedding with precursor m/z zeroed, as the rescorer sees it
|
|
59
|
+
rescore_embedding: np.ndarray | None = None
|
|
58
60
|
|
|
59
61
|
|
|
60
62
|
class MsFiddlePredictor:
|
|
@@ -196,6 +198,7 @@ class MsFiddlePredictor:
|
|
|
196
198
|
sirius_path: str | os.PathLike[str] | None = "",
|
|
197
199
|
top_k: int | None = None,
|
|
198
200
|
batch_size: int | None = None,
|
|
201
|
+
num_workers: int = 1,
|
|
199
202
|
) -> pd.DataFrame:
|
|
200
203
|
"""Predict formulas for spectra in an MGF file.
|
|
201
204
|
|
|
@@ -203,6 +206,9 @@ class MsFiddlePredictor:
|
|
|
203
206
|
BUDDY/msbuddy and SIRIUS formula-identification outputs or legacy
|
|
204
207
|
msfiddle-normalized CSV files.
|
|
205
208
|
|
|
209
|
+
``num_workers`` > 1 runs formula refinement (CPU-bound, per spectrum)
|
|
210
|
+
in that many processes; results are identical to a single process.
|
|
211
|
+
|
|
206
212
|
The returned DataFrame uses the same result columns as the CLI CSV.
|
|
207
213
|
"""
|
|
208
214
|
|
|
@@ -226,26 +232,40 @@ class MsFiddlePredictor:
|
|
|
226
232
|
running_time: list[float] = []
|
|
227
233
|
neutral_masses: list[float] = []
|
|
228
234
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
235
|
+
jobs = [
|
|
236
|
+
self._refinement_args(
|
|
237
|
+
prediction,
|
|
238
|
+
top_k,
|
|
239
|
+
extra_formulas=self._external_formula_candidates(
|
|
240
|
+
prediction.spectrum.id,
|
|
241
|
+
buddy_df=buddy_df,
|
|
242
|
+
sirius_df=sirius_df,
|
|
243
|
+
),
|
|
244
|
+
)
|
|
245
|
+
for prediction in predictions
|
|
246
|
+
]
|
|
247
|
+
refined = _run_refinement(
|
|
248
|
+
[refine_args for refine_args, _ in jobs], num_workers, self.verbose
|
|
249
|
+
)
|
|
232
250
|
|
|
233
|
-
|
|
251
|
+
rescore_jobs = zip(predictions, jobs, refined)
|
|
252
|
+
if self.verbose:
|
|
253
|
+
from tqdm import tqdm
|
|
234
254
|
|
|
235
|
-
|
|
255
|
+
rescore_jobs = tqdm(rescore_jobs, total=len(jobs), desc="Rescore")
|
|
256
|
+
for (
|
|
257
|
+
prediction,
|
|
258
|
+
(_, neutral_mass),
|
|
259
|
+
(refined_results, refine_time),
|
|
260
|
+
) in rescore_jobs:
|
|
236
261
|
start_time = time.time()
|
|
237
|
-
|
|
238
|
-
prediction.
|
|
239
|
-
buddy_df=buddy_df,
|
|
240
|
-
sirius_df=sirius_df,
|
|
241
|
-
)
|
|
242
|
-
results, neutral_mass = self._refine_and_rescore(
|
|
243
|
-
prediction,
|
|
244
|
-
top_k,
|
|
245
|
-
extra_formulas=extra_formulas,
|
|
262
|
+
results = self._rescore_candidates(
|
|
263
|
+
prediction.rescore_embedding, refined_results, top_k
|
|
246
264
|
)
|
|
247
265
|
neutral_masses.append(neutral_mass)
|
|
248
|
-
running_time.append(
|
|
266
|
+
running_time.append(
|
|
267
|
+
prediction.prediction_time + refine_time + time.time() - start_time
|
|
268
|
+
)
|
|
249
269
|
|
|
250
270
|
for idx, (formula, mass, score) in enumerate(
|
|
251
271
|
zip(results["formula"], results["mass"], results["rescore"])
|
|
@@ -430,6 +450,7 @@ class MsFiddlePredictor:
|
|
|
430
450
|
mass_predictions: list[np.ndarray] = []
|
|
431
451
|
atom_predictions: list[np.ndarray] = []
|
|
432
452
|
hc_predictions: list[np.ndarray] = []
|
|
453
|
+
rescore_embeddings: list[np.ndarray] = []
|
|
433
454
|
|
|
434
455
|
batch_starts: Sequence[int] = range(0, len(encoded), batch_size)
|
|
435
456
|
if self.verbose:
|
|
@@ -461,6 +482,11 @@ class MsFiddlePredictor:
|
|
|
461
482
|
_, pred_f, pred_mass, pred_atomnum, pred_hcnum = self.model(
|
|
462
483
|
spec_t, env_t
|
|
463
484
|
)
|
|
485
|
+
env_t[:, 0] = 0.0 # the rescorer never sees the precursor m/z
|
|
486
|
+
z_spec, _, _, _, _ = self.model(spec_t, env_t)
|
|
487
|
+
rescore_embeddings.append(
|
|
488
|
+
self._F.normalize(z_spec, dim=1).cpu().numpy()
|
|
489
|
+
)
|
|
464
490
|
pred_f = pred_f - neutral_t
|
|
465
491
|
formula_vectors.append(pred_f.detach().cpu().numpy())
|
|
466
492
|
mass_predictions.append(pred_mass.detach().cpu().numpy())
|
|
@@ -472,6 +498,7 @@ class MsFiddlePredictor:
|
|
|
472
498
|
mass_arr = np.concatenate(mass_predictions, axis=0)
|
|
473
499
|
atom_arr = np.concatenate(atom_predictions, axis=0)
|
|
474
500
|
hc_arr = np.concatenate(hc_predictions, axis=0)
|
|
501
|
+
z_arr = np.concatenate(rescore_embeddings, axis=0)
|
|
475
502
|
|
|
476
503
|
predictions = []
|
|
477
504
|
for idx, item in enumerate(encoded):
|
|
@@ -485,6 +512,7 @@ class MsFiddlePredictor:
|
|
|
485
512
|
pred_atom_count=float(atom_arr[idx]),
|
|
486
513
|
pred_hc_count=float(hc_arr[idx]),
|
|
487
514
|
prediction_time=prediction_time,
|
|
515
|
+
rescore_embedding=z_arr[idx],
|
|
488
516
|
)
|
|
489
517
|
)
|
|
490
518
|
return predictions
|
|
@@ -608,6 +636,26 @@ class MsFiddlePredictor:
|
|
|
608
636
|
*,
|
|
609
637
|
extra_formulas: Sequence[str] | None = None,
|
|
610
638
|
) -> tuple[dict[str, list[Any]], float]:
|
|
639
|
+
refine_args, neutral_mass = self._refinement_args(
|
|
640
|
+
prediction, top_k, extra_formulas=extra_formulas
|
|
641
|
+
)
|
|
642
|
+
return (
|
|
643
|
+
self._rescore_candidates(
|
|
644
|
+
prediction.rescore_embedding,
|
|
645
|
+
formula_refinement(*refine_args),
|
|
646
|
+
top_k,
|
|
647
|
+
),
|
|
648
|
+
neutral_mass,
|
|
649
|
+
)
|
|
650
|
+
|
|
651
|
+
def _refinement_args(
|
|
652
|
+
self,
|
|
653
|
+
prediction: _ModelPrediction,
|
|
654
|
+
top_k: int,
|
|
655
|
+
*,
|
|
656
|
+
extra_formulas: Sequence[str] | None = None,
|
|
657
|
+
) -> tuple[tuple, float]:
|
|
658
|
+
"""Positional arguments of formula_refinement for one prediction."""
|
|
611
659
|
neutral_mass = float(
|
|
612
660
|
mass_calculator(
|
|
613
661
|
prediction.spectrum.precursor_type,
|
|
@@ -628,7 +676,7 @@ class MsFiddlePredictor:
|
|
|
628
676
|
refine_atom_type.append(atom)
|
|
629
677
|
refine_atom_num.append(max(1, int(count)))
|
|
630
678
|
|
|
631
|
-
|
|
679
|
+
refine_args = (
|
|
632
680
|
f0_list,
|
|
633
681
|
neutral_mass,
|
|
634
682
|
self.config["post_processing"]["mass_tolerance"],
|
|
@@ -639,26 +687,16 @@ class MsFiddlePredictor:
|
|
|
639
687
|
refine_atom_type,
|
|
640
688
|
refine_atom_num,
|
|
641
689
|
)
|
|
642
|
-
return
|
|
643
|
-
self._rescore_candidates(
|
|
644
|
-
prediction.spectrum,
|
|
645
|
-
refined_results,
|
|
646
|
-
top_k,
|
|
647
|
-
),
|
|
648
|
-
neutral_mass,
|
|
649
|
-
)
|
|
690
|
+
return refine_args, neutral_mass
|
|
650
691
|
|
|
651
692
|
def _rescore_candidates(
|
|
652
693
|
self,
|
|
653
|
-
|
|
694
|
+
z_spec: np.ndarray,
|
|
654
695
|
refined_results: dict[str, list[Any]],
|
|
655
696
|
top_k: int,
|
|
656
697
|
) -> dict[str, list[Any]]:
|
|
657
698
|
torch = self._torch
|
|
658
|
-
F = self._F
|
|
659
699
|
assert torch is not None
|
|
660
|
-
assert F is not None
|
|
661
|
-
assert self.model is not None
|
|
662
700
|
assert self.formula_encoder is not None
|
|
663
701
|
assert self.rescore_head is not None
|
|
664
702
|
|
|
@@ -679,23 +717,14 @@ class MsFiddlePredictor:
|
|
|
679
717
|
f_vecs = torch.from_numpy(
|
|
680
718
|
np.array([formula_to_vector(formula) for formula in refine_f])
|
|
681
719
|
)
|
|
682
|
-
spec_t = torch.from_numpy(spectrum.spec[None, :]).to(
|
|
683
|
-
self.device, dtype=torch.float32
|
|
684
|
-
)
|
|
685
|
-
env_t = (
|
|
686
|
-
torch.from_numpy(spectrum.env[None, :])
|
|
687
|
-
.to(self.device, dtype=torch.float32)
|
|
688
|
-
.clone()
|
|
689
|
-
)
|
|
690
|
-
env_t[:, 0] = 0.0
|
|
691
|
-
|
|
692
|
-
self.model.eval()
|
|
693
720
|
self.formula_encoder.eval()
|
|
694
721
|
self.rescore_head.eval()
|
|
695
722
|
with torch.no_grad():
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
723
|
+
z_spec_rep = (
|
|
724
|
+
torch.from_numpy(z_spec[None, :])
|
|
725
|
+
.to(self.device)
|
|
726
|
+
.expand(len(refine_f), -1)
|
|
727
|
+
)
|
|
699
728
|
f_t = f_vecs.to(self.device, dtype=torch.float32)
|
|
700
729
|
z_form = self.formula_encoder(f_t)
|
|
701
730
|
logits = self.rescore_head(z_spec_rep * z_form)
|
|
@@ -828,6 +857,34 @@ class MsFiddlePredictor:
|
|
|
828
857
|
return candidates
|
|
829
858
|
|
|
830
859
|
|
|
860
|
+
def _timed_refinement(refine_args: tuple) -> tuple[dict[str, list[Any]], float]:
|
|
861
|
+
start_time = time.time()
|
|
862
|
+
return formula_refinement(*refine_args), time.time() - start_time
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
def _run_refinement(
|
|
866
|
+
jobs: Sequence[tuple], num_workers: int, verbose: bool
|
|
867
|
+
) -> list[tuple[dict[str, list[Any]], float]]:
|
|
868
|
+
"""Refine every job, in num_workers processes when > 1, preserving order."""
|
|
869
|
+
if verbose and jobs:
|
|
870
|
+
from tqdm import tqdm
|
|
871
|
+
|
|
872
|
+
def progress(it):
|
|
873
|
+
return tqdm(it, total=len(jobs), desc="Refine")
|
|
874
|
+
|
|
875
|
+
else:
|
|
876
|
+
|
|
877
|
+
def progress(it):
|
|
878
|
+
return it
|
|
879
|
+
|
|
880
|
+
if num_workers > 1 and len(jobs) > 1:
|
|
881
|
+
from multiprocessing import Pool
|
|
882
|
+
|
|
883
|
+
with Pool(num_workers) as pool:
|
|
884
|
+
return list(progress(pool.imap(_timed_refinement, jobs, chunksize=16)))
|
|
885
|
+
return [_timed_refinement(job) for job in progress(jobs)]
|
|
886
|
+
|
|
887
|
+
|
|
831
888
|
def predict_from_spectrum(
|
|
832
889
|
mz_array: Sequence[float],
|
|
833
890
|
intensity_array: Sequence[float],
|
|
@@ -909,6 +966,7 @@ def predict_from_mgf(
|
|
|
909
966
|
download_models: bool = False,
|
|
910
967
|
top_k: int | None = None,
|
|
911
968
|
batch_size: int = 32,
|
|
969
|
+
num_workers: int = 1,
|
|
912
970
|
) -> pd.DataFrame:
|
|
913
971
|
"""Predict formulas for spectra in an MGF file."""
|
|
914
972
|
|
|
@@ -928,6 +986,7 @@ def predict_from_mgf(
|
|
|
928
986
|
sirius_path=sirius_path,
|
|
929
987
|
top_k=top_k,
|
|
930
988
|
batch_size=batch_size,
|
|
989
|
+
num_workers=num_workers,
|
|
931
990
|
)
|
|
932
991
|
|
|
933
992
|
|
|
@@ -9,7 +9,7 @@ from pyteomics import mgf
|
|
|
9
9
|
from torch.utils.data import Dataset
|
|
10
10
|
from tqdm import tqdm
|
|
11
11
|
|
|
12
|
-
from .utils.mol_utils import ATOMS_INDEX, formula_to_dict
|
|
12
|
+
from .utils.mol_utils import ATOMS_INDEX, formula_to_dict
|
|
13
13
|
from .utils.pkl_utils import generate_ms, parse_collision_energy, unify_precursor_type
|
|
14
14
|
|
|
15
15
|
|
|
@@ -414,40 +414,3 @@ class MGFDataset(Dataset):
|
|
|
414
414
|
vector[index] = int(count) if count else 1
|
|
415
415
|
|
|
416
416
|
return vector
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
class RescoreDataset(Dataset):
|
|
420
|
-
"""Dataset for FDR model training and evaluation.
|
|
421
|
-
|
|
422
|
-
Loads a pkl file produced by prepare_rescore.py, where each item contains a
|
|
423
|
-
spectrum array, environment vector, a predicted formula string, and a binary
|
|
424
|
-
label (1 = correct formula, 0 = decoy). The formula string is converted to a
|
|
425
|
-
fixed-length atom-count vector at load time.
|
|
426
|
-
|
|
427
|
-
Args:
|
|
428
|
-
path: Path to the FDR pkl file (e.g. 'orbitrap_maxmin_fdr_train.pkl').
|
|
429
|
-
"""
|
|
430
|
-
|
|
431
|
-
def __init__(self, path):
|
|
432
|
-
with open(path, "rb") as file:
|
|
433
|
-
self.data = pickle.load(file)
|
|
434
|
-
|
|
435
|
-
# convert formula string to formula vector
|
|
436
|
-
for d in self.data:
|
|
437
|
-
f_vec = np.array(formula_to_vector(d["pred_formula"]))
|
|
438
|
-
# f_vec = (f_vec - f_vec.min()) / (f_vec.max() - f_vec.min()) # normalize formula vector
|
|
439
|
-
d["f"] = f_vec
|
|
440
|
-
|
|
441
|
-
print("Load {} data from {}".format(len(self.data), path))
|
|
442
|
-
|
|
443
|
-
def __len__(self):
|
|
444
|
-
return len(self.data)
|
|
445
|
-
|
|
446
|
-
def __getitem__(self, idx):
|
|
447
|
-
return (
|
|
448
|
-
self.data[idx]["title"],
|
|
449
|
-
self.data[idx]["spec"][:, 0],
|
|
450
|
-
self.data[idx]["env"],
|
|
451
|
-
self.data[idx]["f"],
|
|
452
|
-
self.data[idx]["label"],
|
|
453
|
-
)
|