papyrus-scripts 2.1.2__tar.gz → 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {papyrus_scripts-2.1.2 → papyrus_scripts-3.0.0}/LICENSE +21 -21
- papyrus_scripts-3.0.0/PKG-INFO +245 -0
- papyrus_scripts-3.0.0/README.md +193 -0
- papyrus_scripts-3.0.0/pyproject.toml +173 -0
- papyrus_scripts-3.0.0/setup.cfg +4 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts/__init__.py +38 -0
- {papyrus_scripts-2.1.2 → papyrus_scripts-3.0.0}/src/papyrus_scripts/__main__.py +15 -15
- papyrus_scripts-3.0.0/src/papyrus_scripts/cli.py +565 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts/download.py +1237 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts/fingerprint.py +338 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts/matchRCSB.py +283 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts/modelling.py +924 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts/neuralnet.py +368 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts/oop.py +1881 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts/preprocess.py +1133 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts/reader.py +613 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts/subsim_search.py +1949 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts/utils/IO.py +1570 -0
- {papyrus_scripts-2.1.2 → papyrus_scripts-3.0.0}/src/papyrus_scripts/utils/UniprotMatch.py +278 -259
- {papyrus_scripts-2.1.2 → papyrus_scripts-3.0.0}/src/papyrus_scripts/utils/__init__.py +3 -3
- {papyrus_scripts-2.1.2 → papyrus_scripts-3.0.0}/src/papyrus_scripts/utils/aliases.json +75 -75
- papyrus_scripts-3.0.0/src/papyrus_scripts/utils/links.json +1524 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts/utils/mol_reader.py +757 -0
- papyrus_scripts-3.0.0/src/papyrus_scripts.egg-info/PKG-INFO +245 -0
- {papyrus_scripts-2.1.2 → papyrus_scripts-3.0.0}/src/papyrus_scripts.egg-info/SOURCES.txt +18 -3
- papyrus_scripts-3.0.0/src/papyrus_scripts.egg-info/requires.txt +40 -0
- papyrus_scripts-3.0.0/tests/test_cli.py +304 -0
- papyrus_scripts-3.0.0/tests/test_download.py +2458 -0
- papyrus_scripts-3.0.0/tests/test_fingerprint.py +243 -0
- papyrus_scripts-3.0.0/tests/test_io.py +1366 -0
- papyrus_scripts-3.0.0/tests/test_matchRCSB.py +448 -0
- papyrus_scripts-3.0.0/tests/test_modelling.py +726 -0
- papyrus_scripts-3.0.0/tests/test_modelling_real_data.py +205 -0
- papyrus_scripts-3.0.0/tests/test_mol_reader.py +490 -0
- papyrus_scripts-3.0.0/tests/test_neuralnet.py +385 -0
- {papyrus_scripts-2.1.2 → papyrus_scripts-3.0.0}/tests/test_oop.py +244 -240
- papyrus_scripts-3.0.0/tests/test_oop_unit.py +1386 -0
- papyrus_scripts-3.0.0/tests/test_preprocess.py +826 -0
- papyrus_scripts-3.0.0/tests/test_reader.py +72 -0
- papyrus_scripts-3.0.0/tests/test_reader_offline.py +644 -0
- papyrus_scripts-3.0.0/tests/test_reader_real_data.py +262 -0
- papyrus_scripts-3.0.0/tests/test_subsim_search.py +1775 -0
- papyrus_scripts-3.0.0/tests/test_uniprot_match.py +409 -0
- papyrus_scripts-2.1.2/PKG-INFO +0 -246
- papyrus_scripts-2.1.2/README.md +0 -206
- papyrus_scripts-2.1.2/setup.cfg +0 -70
- papyrus_scripts-2.1.2/setup.py +0 -8
- papyrus_scripts-2.1.2/src/papyrus_scripts/__init__.py +0 -23
- papyrus_scripts-2.1.2/src/papyrus_scripts/cli.py +0 -384
- papyrus_scripts-2.1.2/src/papyrus_scripts/download.py +0 -405
- papyrus_scripts-2.1.2/src/papyrus_scripts/fingerprint.py +0 -223
- papyrus_scripts-2.1.2/src/papyrus_scripts/matchRCSB.py +0 -161
- papyrus_scripts-2.1.2/src/papyrus_scripts/modelling.py +0 -1074
- papyrus_scripts-2.1.2/src/papyrus_scripts/neuralnet.py +0 -526
- papyrus_scripts-2.1.2/src/papyrus_scripts/oop.py +0 -801
- papyrus_scripts-2.1.2/src/papyrus_scripts/preprocess.py +0 -934
- papyrus_scripts-2.1.2/src/papyrus_scripts/reader.py +0 -384
- papyrus_scripts-2.1.2/src/papyrus_scripts/subsim_search.py +0 -1022
- papyrus_scripts-2.1.2/src/papyrus_scripts/utils/IO.py +0 -464
- papyrus_scripts-2.1.2/src/papyrus_scripts/utils/links.json +0 -408
- papyrus_scripts-2.1.2/src/papyrus_scripts/utils/mol_reader.py +0 -326
- papyrus_scripts-2.1.2/src/papyrus_scripts.egg-info/PKG-INFO +0 -246
- papyrus_scripts-2.1.2/src/papyrus_scripts.egg-info/requires.txt +0 -21
- {papyrus_scripts-2.1.2 → papyrus_scripts-3.0.0}/src/papyrus_scripts.egg-info/dependency_links.txt +0 -0
- {papyrus_scripts-2.1.2 → papyrus_scripts-3.0.0}/src/papyrus_scripts.egg-info/entry_points.txt +0 -0
- {papyrus_scripts-2.1.2 → papyrus_scripts-3.0.0}/src/papyrus_scripts.egg-info/top_level.txt +0 -0
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2021 OlivierBeq
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE.
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2021-2026 OlivierBeq
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: papyrus_scripts
|
|
3
|
+
Version: 3.0.0
|
|
4
|
+
Summary: A collection of scripts to handle the Papyrus bioactivity dataset
|
|
5
|
+
Author: Brandon J. Bongers, Willem Jespers
|
|
6
|
+
Author-email: "Olivier J. M. Béquignon" <olivier.bequignon.maintainer@gmail.com>
|
|
7
|
+
Maintainer-email: "Olivier J. M. Béquignon" <olivier.bequignon.maintainer@gmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Project-URL: Homepage, https://github.com/OlivierBeq/Papyrus-scripts
|
|
10
|
+
Keywords: bioactivity data,QSAR,proteochemometrics,cheminformatics,modelling,machine learning
|
|
11
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
12
|
+
Classifier: Programming Language :: Python
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: click
|
|
20
|
+
Requires-Dist: numpy>=2.0.0
|
|
21
|
+
Requires-Dist: pandas
|
|
22
|
+
Requires-Dist: polars
|
|
23
|
+
Requires-Dist: pyarrow
|
|
24
|
+
Requires-Dist: rdkit
|
|
25
|
+
Requires-Dist: requests
|
|
26
|
+
Requires-Dist: fake-useragent
|
|
27
|
+
Requires-Dist: tqdm
|
|
28
|
+
Requires-Dist: ipywidgets
|
|
29
|
+
Requires-Dist: colorama; platform_system == "Windows"
|
|
30
|
+
Requires-Dist: mordred
|
|
31
|
+
Requires-Dist: scikit-learn
|
|
32
|
+
Requires-Dist: xgboost
|
|
33
|
+
Requires-Dist: pystow
|
|
34
|
+
Requires-Dist: prodec
|
|
35
|
+
Requires-Dist: defusedxml
|
|
36
|
+
Provides-Extra: testing
|
|
37
|
+
Requires-Dist: pytest; extra == "testing"
|
|
38
|
+
Requires-Dist: parameterized; extra == "testing"
|
|
39
|
+
Provides-Extra: dnn
|
|
40
|
+
Requires-Dist: torch; extra == "dnn"
|
|
41
|
+
Requires-Dist: skorch; extra == "dnn"
|
|
42
|
+
Provides-Extra: subsim
|
|
43
|
+
Requires-Dist: tables; extra == "subsim"
|
|
44
|
+
Requires-Dist: FPSim2; extra == "subsim"
|
|
45
|
+
Provides-Extra: gpu
|
|
46
|
+
Requires-Dist: cupy-cuda12x[ctk]; extra == "gpu"
|
|
47
|
+
Provides-Extra: simgpu
|
|
48
|
+
Requires-Dist: papyrus_scripts[gpu,subsim]; extra == "simgpu"
|
|
49
|
+
Provides-Extra: all
|
|
50
|
+
Requires-Dist: papyrus_scripts[dnn,simgpu]; extra == "all"
|
|
51
|
+
Dynamic: license-file
|
|
52
|
+
|
|
53
|
+
<div align="center">
|
|
54
|
+
<img src="https://raw.githubusercontent.com/OlivierBeq/Papyrus-scripts/refs/heads/master/figures/logo/Papyrus_trnsp-bg.svg" alt="Papyrus logo" width="200">
|
|
55
|
+
|
|
56
|
+
# 📜 Papyrus-scripts
|
|
57
|
+
|
|
58
|
+
[](https://pypi.org/project/papyrus-scripts/)
|
|
59
|
+
[](https://pypi.org/project/papyrus-scripts/)
|
|
60
|
+
[](https://opensource.org/licenses/MIT)
|
|
61
|
+
[](https://github.com/OlivierBeq/Papyrus-scripts/actions/workflows/ci.yml)
|
|
62
|
+
[](https://github.com/astral-sh/ruff)
|
|
63
|
+
[](https://doi.org/10.1186/s13321-022-00672-x)
|
|
64
|
+
</div>
|
|
65
|
+
|
|
66
|
+
A Python library for working with **Papyrus**, a large-scale curated dataset of bioactivity data aimed at machine learning applications. It handles downloading, versioning, and caching the dataset, and provides a fluent API to filter, search, and export exactly the subset you need — without ever loading the full dataset into memory.
|
|
67
|
+
|
|
68
|
+
## ✨ Features
|
|
69
|
+
|
|
70
|
+
- 🗂️ **Versioned dataset access** — download and cache specific Papyrus releases from Zenodo or 4TU, with automatic integrity checks.
|
|
71
|
+
- 🔗 **Fluent filtering API** — chain quality, source, organism, protein-class and molecular filters over a lazy Polars pipeline; only what you keep gets materialized.
|
|
72
|
+
- 🧬 **Proteins, structures & descriptors** — retrieve matching UniProt targets, 2D/3D compound structures, and precomputed molecular/protein descriptors in one call.
|
|
73
|
+
- 🔍 **Similarity & substructure search** — build an indexed FPSim2/RDKit search database over the dataset, with CPU, GPU (CUDA), or auto-fallback search engines.
|
|
74
|
+
- 🧠 **DNN-ready** — train PyTorch-based QSAR/PCM models on curated subsets, with y-scrambling and repeated seeds built in.
|
|
75
|
+
- 💻 **CLI included** — download, convert and clean up dataset files without writing any Python.
|
|
76
|
+
- 🔄 **Format-friendly** — transparent LZMA ↔ Gzip conversion and Parquet caching for tools that don't handle `.xz`.
|
|
77
|
+
|
|
78
|
+
## 📦 Installation
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
pip install papyrus-scripts
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
<details>
|
|
85
|
+
<summary><strong>⚠️ Troubleshooting <code>pip</code> installation</strong></summary>
|
|
86
|
+
|
|
87
|
+
If you see `Defaulting to user installation because normal site-packages is not writeable` followed by import errors, reinstall with:
|
|
88
|
+
```bash
|
|
89
|
+
pip uninstall -y papyrus-scripts
|
|
90
|
+
python -m pip install papyrus-scripts
|
|
91
|
+
```
|
|
92
|
+
</details>
|
|
93
|
+
|
|
94
|
+
Optional extras enable additional functionality:
|
|
95
|
+
|
|
96
|
+
| Extra | Enables |
|
|
97
|
+
|---|---|
|
|
98
|
+
| `papyrus-scripts[subsim]` | CPU similarity & substructure search (`tables`, `FPSim2`) |
|
|
99
|
+
| `papyrus-scripts[gpu]` | GPU-accelerated similarity search (`cupy-cuda12x[ctk]`, bundles the CUDA 12.x toolkit) |
|
|
100
|
+
| `papyrus-scripts[simgpu]` | CPU + GPU-accelerated similarity search (`subsim` + `gpu`) |
|
|
101
|
+
| `papyrus-scripts[dnn]` | DNN model training (`torch`, `skorch`) |
|
|
102
|
+
| `papyrus-scripts[all]` | Everything above |
|
|
103
|
+
|
|
104
|
+
> **Note:** `openbabel` (needed only for FP2/FP3/FP4 fingerprints) must be installed via conda-forge, not pip, when used alongside RDKit/FPSim2/cupy in the same environment: `conda install -c conda-forge openbabel`.
|
|
105
|
+
>
|
|
106
|
+
> **Note:** the `gpu` extra pins `cupy-cuda12x[ctk]`, whose `ctk` extra pulls in the CUDA 12.x toolkit as a pip package — no separate toolkit install needed, only an NVIDIA driver compatible with CUDA 12.x.
|
|
107
|
+
|
|
108
|
+
## 🛠️ Requirements
|
|
109
|
+
|
|
110
|
+
- Python 3.11+
|
|
111
|
+
- [RDKit](https://www.rdkit.org/docs/Install.html)
|
|
112
|
+
|
|
113
|
+
## 💡 Usage
|
|
114
|
+
|
|
115
|
+
### Quickstart: the object-oriented API
|
|
116
|
+
|
|
117
|
+
The recommended way to interact with the dataset. It downloads and caches data automatically as needed.
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from papyrus_scripts import PapyrusDataset
|
|
121
|
+
|
|
122
|
+
dataset = PapyrusDataset(version='2024.09.2', plusplus=True)
|
|
123
|
+
|
|
124
|
+
filtered = (dataset
|
|
125
|
+
.keep_source(['chembl', 'sharma'])
|
|
126
|
+
.keep_quality('high'))
|
|
127
|
+
|
|
128
|
+
df = filtered.to_dataframe()
|
|
129
|
+
proteins = filtered.proteins().to_dataframe()
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
### Downloading data: the CLI
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
# Download Papyrus++ bioactivities & targets for the latest version
|
|
136
|
+
papyrus download -V latest
|
|
137
|
+
|
|
138
|
+
# Download the full (all-quality) dataset with all precomputed descriptors for a specific revision
|
|
139
|
+
papyrus download -V 2022.11.3 --more -d all
|
|
140
|
+
|
|
141
|
+
# Download Papyrus++ data & compound structures for two versions
|
|
142
|
+
papyrus download -V 2022.11.3 -V 2022.04.2 -S
|
|
143
|
+
|
|
144
|
+
papyrus download --help
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
By default, data is downloaded to [pystow](https://github.com/cthoyt/pystow)'s home directory; override it with `-o`.
|
|
148
|
+
|
|
149
|
+
<details>
|
|
150
|
+
<summary><strong>Legacy functional API</strong></summary>
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
from papyrus_scripts import (read_papyrus, read_protein_set,
|
|
154
|
+
keep_quality, keep_source, consume_chunks)
|
|
155
|
+
|
|
156
|
+
chunks = read_papyrus(version='2024.09.2', plusplus=True, chunksize=1_000_000)
|
|
157
|
+
proteins = read_protein_set(version='2024.09.2')
|
|
158
|
+
|
|
159
|
+
filtered = keep_quality(keep_source(chunks, source=['chembl', 'sharma']), min_quality='high')
|
|
160
|
+
df = consume_chunks(filtered)
|
|
161
|
+
```
|
|
162
|
+
</details>
|
|
163
|
+
|
|
164
|
+
<details>
|
|
165
|
+
<summary><strong>Similarity & substructure search</strong></summary>
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
from papyrus_scripts.subsim_search import FPSubSim2
|
|
169
|
+
|
|
170
|
+
fpss = FPSubSim2()
|
|
171
|
+
fpss.create_from_papyrus(version='2024.09.2', njobs=-1) # builds a search database using all CPU cores
|
|
172
|
+
|
|
173
|
+
# cuda=False (default, CPU) | True (GPU, raises if unavailable) | 'auto' (GPU with CPU fallback)
|
|
174
|
+
engine = fpss.get_similarity_lib(cuda='auto')
|
|
175
|
+
hits = engine.similarity('CCO', threshold=0.7)
|
|
176
|
+
|
|
177
|
+
sub_lib = fpss.get_substructure_lib()
|
|
178
|
+
matches = sub_lib.substructure('c1ccccc1')
|
|
179
|
+
```
|
|
180
|
+
</details>
|
|
181
|
+
|
|
182
|
+
## 📊 Dataset versions
|
|
183
|
+
|
|
184
|
+
Papyrus releases are hosted on Zenodo:
|
|
185
|
+
|
|
186
|
+
| Papyrus version | Revisions | Legacy alias | ChEMBL version | Zenodo |
|
|
187
|
+
|:---:|:---:|:---:|:---:|:---:|
|
|
188
|
+
| **2022.04** | 2 | 05.4 | 29 | [link](https://zenodo.org/records/10943992) |
|
|
189
|
+
| **2022.08** | 3 | 05.5 | 30 | [link](https://zenodo.org/records/7019873) |
|
|
190
|
+
| **2022.11** | 4 | 05.6 | 31 | [link](https://zenodo.org/records/7373213) |
|
|
191
|
+
| **2024.09** | 2 | 05.7 | 34 | [link](https://zenodo.org/records/13787633) |
|
|
192
|
+
|
|
193
|
+
> **Note:** for machine learning use cases, we recommend the datasets without stereochemistry (files marked "2D" and/or "without_stereochemistry").
|
|
194
|
+
|
|
195
|
+
## ⚙️ Advanced utilities
|
|
196
|
+
|
|
197
|
+
<details>
|
|
198
|
+
<summary><strong>Compression conversion</strong></summary>
|
|
199
|
+
|
|
200
|
+
Data is distributed as LZMA-compressed files (`.xz`), which some tools (e.g. Pipeline Pilot) don't support. Convert to Gzip (or back) without manually decompressing:
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
papyrus convert -v latest
|
|
204
|
+
```
|
|
205
|
+
</details>
|
|
206
|
+
|
|
207
|
+
<details>
|
|
208
|
+
<summary><strong>Removing downloaded data</strong></summary>
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
papyrus clean --remove_root
|
|
212
|
+
```
|
|
213
|
+
```python
|
|
214
|
+
from papyrus_scripts import remove_papyrus
|
|
215
|
+
|
|
216
|
+
remove_papyrus(papyrus_root=True)
|
|
217
|
+
```
|
|
218
|
+
</details>
|
|
219
|
+
|
|
220
|
+
## 📚 Learn more
|
|
221
|
+
|
|
222
|
+
- [`simple_examples.ipynb`](https://github.com/OlivierBeq/Papyrus-scripts/blob/master/notebook_examples/simple_examples.ipynb) [](https://colab.research.google.com/github/OlivierBeq/Papyrus-scripts/blob/master/notebook_examples/simple_examples.ipynb)
|
|
223
|
+
- [`matchRCSB.ipynb`](https://github.com/OlivierBeq/Papyrus-scripts/blob/master/notebook_examples/matchRCSB.ipynb) — matching Papyrus data against the Protein Data Bank [](https://colab.research.google.com/github/OlivierBeq/Papyrus-scripts/blob/master/notebook_examples/matchRCSB.ipynb)
|
|
224
|
+
- [`advanced_querying.ipynb`](https://github.com/OlivierBeq/Papyrus-scripts/blob/master/notebook_examples/advanced_querying.ipynb)
|
|
225
|
+
- To reproduce the models, extraction pipeline and visualizations from the original publication, see [Papyrus-modelling](https://github.com/OlivierBeq/Papyrus-modelling).
|
|
226
|
+
|
|
227
|
+
## 🖋️ Citation
|
|
228
|
+
|
|
229
|
+
If you use `papyrus-scripts` or the Papyrus dataset in your research, please cite:
|
|
230
|
+
|
|
231
|
+
```bibtex
|
|
232
|
+
@article{Bequignon2023Papyrus,
|
|
233
|
+
title={Papyrus - A large scale curated dataset aimed at bioactivity predictions},
|
|
234
|
+
author={B{\'e}quignon, Olivier J.M. and Bongers, Bart J. and Jespers, Willem and IJzerman, Adriaan P. and van de Water, Bob and van Westen, Gerard J.P.},
|
|
235
|
+
journal={Journal of Cheminformatics},
|
|
236
|
+
volume={15},
|
|
237
|
+
number={3},
|
|
238
|
+
year={2023},
|
|
239
|
+
doi={10.1186/s13321-022-00672-x}
|
|
240
|
+
}
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
## 📄 License
|
|
244
|
+
|
|
245
|
+
This project is licensed under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/OlivierBeq/Papyrus-scripts/refs/heads/master/figures/logo/Papyrus_trnsp-bg.svg" alt="Papyrus logo" width="200">
|
|
3
|
+
|
|
4
|
+
# 📜 Papyrus-scripts
|
|
5
|
+
|
|
6
|
+
[](https://pypi.org/project/papyrus-scripts/)
|
|
7
|
+
[](https://pypi.org/project/papyrus-scripts/)
|
|
8
|
+
[](https://opensource.org/licenses/MIT)
|
|
9
|
+
[](https://github.com/OlivierBeq/Papyrus-scripts/actions/workflows/ci.yml)
|
|
10
|
+
[](https://github.com/astral-sh/ruff)
|
|
11
|
+
[](https://doi.org/10.1186/s13321-022-00672-x)
|
|
12
|
+
</div>
|
|
13
|
+
|
|
14
|
+
A Python library for working with **Papyrus**, a large-scale curated dataset of bioactivity data aimed at machine learning applications. It handles downloading, versioning, and caching the dataset, and provides a fluent API to filter, search, and export exactly the subset you need — without ever loading the full dataset into memory.
|
|
15
|
+
|
|
16
|
+
## ✨ Features
|
|
17
|
+
|
|
18
|
+
- 🗂️ **Versioned dataset access** — download and cache specific Papyrus releases from Zenodo or 4TU, with automatic integrity checks.
|
|
19
|
+
- 🔗 **Fluent filtering API** — chain quality, source, organism, protein-class and molecular filters over a lazy Polars pipeline; only what you keep gets materialized.
|
|
20
|
+
- 🧬 **Proteins, structures & descriptors** — retrieve matching UniProt targets, 2D/3D compound structures, and precomputed molecular/protein descriptors in one call.
|
|
21
|
+
- 🔍 **Similarity & substructure search** — build an indexed FPSim2/RDKit search database over the dataset, with CPU, GPU (CUDA), or auto-fallback search engines.
|
|
22
|
+
- 🧠 **DNN-ready** — train PyTorch-based QSAR/PCM models on curated subsets, with y-scrambling and repeated seeds built in.
|
|
23
|
+
- 💻 **CLI included** — download, convert and clean up dataset files without writing any Python.
|
|
24
|
+
- 🔄 **Format-friendly** — transparent LZMA ↔ Gzip conversion and Parquet caching for tools that don't handle `.xz`.
|
|
25
|
+
|
|
26
|
+
## 📦 Installation
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install papyrus-scripts
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
<details>
|
|
33
|
+
<summary><strong>⚠️ Troubleshooting <code>pip</code> installation</strong></summary>
|
|
34
|
+
|
|
35
|
+
If you see `Defaulting to user installation because normal site-packages is not writeable` followed by import errors, reinstall with:
|
|
36
|
+
```bash
|
|
37
|
+
pip uninstall -y papyrus-scripts
|
|
38
|
+
python -m pip install papyrus-scripts
|
|
39
|
+
```
|
|
40
|
+
</details>
|
|
41
|
+
|
|
42
|
+
Optional extras enable additional functionality:
|
|
43
|
+
|
|
44
|
+
| Extra | Enables |
|
|
45
|
+
|---|---|
|
|
46
|
+
| `papyrus-scripts[subsim]` | CPU similarity & substructure search (`tables`, `FPSim2`) |
|
|
47
|
+
| `papyrus-scripts[gpu]` | GPU-accelerated similarity search (`cupy-cuda12x[ctk]`, bundles the CUDA 12.x toolkit) |
|
|
48
|
+
| `papyrus-scripts[simgpu]` | CPU + GPU-accelerated similarity search (`subsim` + `gpu`) |
|
|
49
|
+
| `papyrus-scripts[dnn]` | DNN model training (`torch`, `skorch`) |
|
|
50
|
+
| `papyrus-scripts[all]` | Everything above |
|
|
51
|
+
|
|
52
|
+
> **Note:** `openbabel` (needed only for FP2/FP3/FP4 fingerprints) must be installed via conda-forge, not pip, when used alongside RDKit/FPSim2/cupy in the same environment: `conda install -c conda-forge openbabel`.
|
|
53
|
+
>
|
|
54
|
+
> **Note:** the `gpu` extra pins `cupy-cuda12x[ctk]`, whose `ctk` extra pulls in the CUDA 12.x toolkit as a pip package — no separate toolkit install needed, only an NVIDIA driver compatible with CUDA 12.x.
|
|
55
|
+
|
|
56
|
+
## 🛠️ Requirements
|
|
57
|
+
|
|
58
|
+
- Python 3.11+
|
|
59
|
+
- [RDKit](https://www.rdkit.org/docs/Install.html)
|
|
60
|
+
|
|
61
|
+
## 💡 Usage
|
|
62
|
+
|
|
63
|
+
### Quickstart: the object-oriented API
|
|
64
|
+
|
|
65
|
+
The recommended way to interact with the dataset. It downloads and caches data automatically as needed.
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from papyrus_scripts import PapyrusDataset
|
|
69
|
+
|
|
70
|
+
dataset = PapyrusDataset(version='2024.09.2', plusplus=True)
|
|
71
|
+
|
|
72
|
+
filtered = (dataset
|
|
73
|
+
.keep_source(['chembl', 'sharma'])
|
|
74
|
+
.keep_quality('high'))
|
|
75
|
+
|
|
76
|
+
df = filtered.to_dataframe()
|
|
77
|
+
proteins = filtered.proteins().to_dataframe()
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
### Downloading data: the CLI
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
# Download Papyrus++ bioactivities & targets for the latest version
|
|
84
|
+
papyrus download -V latest
|
|
85
|
+
|
|
86
|
+
# Download the full (all-quality) dataset with all precomputed descriptors for a specific revision
|
|
87
|
+
papyrus download -V 2022.11.3 --more -d all
|
|
88
|
+
|
|
89
|
+
# Download Papyrus++ data & compound structures for two versions
|
|
90
|
+
papyrus download -V 2022.11.3 -V 2022.04.2 -S
|
|
91
|
+
|
|
92
|
+
papyrus download --help
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
By default, data is downloaded to [pystow](https://github.com/cthoyt/pystow)'s home directory; override it with `-o`.
|
|
96
|
+
|
|
97
|
+
<details>
|
|
98
|
+
<summary><strong>Legacy functional API</strong></summary>
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from papyrus_scripts import (read_papyrus, read_protein_set,
|
|
102
|
+
keep_quality, keep_source, consume_chunks)
|
|
103
|
+
|
|
104
|
+
chunks = read_papyrus(version='2024.09.2', plusplus=True, chunksize=1_000_000)
|
|
105
|
+
proteins = read_protein_set(version='2024.09.2')
|
|
106
|
+
|
|
107
|
+
filtered = keep_quality(keep_source(chunks, source=['chembl', 'sharma']), min_quality='high')
|
|
108
|
+
df = consume_chunks(filtered)
|
|
109
|
+
```
|
|
110
|
+
</details>
|
|
111
|
+
|
|
112
|
+
<details>
|
|
113
|
+
<summary><strong>Similarity & substructure search</strong></summary>
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
from papyrus_scripts.subsim_search import FPSubSim2
|
|
117
|
+
|
|
118
|
+
fpss = FPSubSim2()
|
|
119
|
+
fpss.create_from_papyrus(version='2024.09.2', njobs=-1) # builds a search database using all CPU cores
|
|
120
|
+
|
|
121
|
+
# cuda=False (default, CPU) | True (GPU, raises if unavailable) | 'auto' (GPU with CPU fallback)
|
|
122
|
+
engine = fpss.get_similarity_lib(cuda='auto')
|
|
123
|
+
hits = engine.similarity('CCO', threshold=0.7)
|
|
124
|
+
|
|
125
|
+
sub_lib = fpss.get_substructure_lib()
|
|
126
|
+
matches = sub_lib.substructure('c1ccccc1')
|
|
127
|
+
```
|
|
128
|
+
</details>
|
|
129
|
+
|
|
130
|
+
## 📊 Dataset versions
|
|
131
|
+
|
|
132
|
+
Papyrus releases are hosted on Zenodo:
|
|
133
|
+
|
|
134
|
+
| Papyrus version | Revisions | Legacy alias | ChEMBL version | Zenodo |
|
|
135
|
+
|:---:|:---:|:---:|:---:|:---:|
|
|
136
|
+
| **2022.04** | 2 | 05.4 | 29 | [link](https://zenodo.org/records/10943992) |
|
|
137
|
+
| **2022.08** | 3 | 05.5 | 30 | [link](https://zenodo.org/records/7019873) |
|
|
138
|
+
| **2022.11** | 4 | 05.6 | 31 | [link](https://zenodo.org/records/7373213) |
|
|
139
|
+
| **2024.09** | 2 | 05.7 | 34 | [link](https://zenodo.org/records/13787633) |
|
|
140
|
+
|
|
141
|
+
> **Note:** for machine learning use cases, we recommend the datasets without stereochemistry (files marked "2D" and/or "without_stereochemistry").
|
|
142
|
+
|
|
143
|
+
## ⚙️ Advanced utilities
|
|
144
|
+
|
|
145
|
+
<details>
|
|
146
|
+
<summary><strong>Compression conversion</strong></summary>
|
|
147
|
+
|
|
148
|
+
Data is distributed as LZMA-compressed files (`.xz`), which some tools (e.g. Pipeline Pilot) don't support. Convert to Gzip (or back) without manually decompressing:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
papyrus convert -v latest
|
|
152
|
+
```
|
|
153
|
+
</details>
|
|
154
|
+
|
|
155
|
+
<details>
|
|
156
|
+
<summary><strong>Removing downloaded data</strong></summary>
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
papyrus clean --remove_root
|
|
160
|
+
```
|
|
161
|
+
```python
|
|
162
|
+
from papyrus_scripts import remove_papyrus
|
|
163
|
+
|
|
164
|
+
remove_papyrus(papyrus_root=True)
|
|
165
|
+
```
|
|
166
|
+
</details>
|
|
167
|
+
|
|
168
|
+
## 📚 Learn more
|
|
169
|
+
|
|
170
|
+
- [`simple_examples.ipynb`](https://github.com/OlivierBeq/Papyrus-scripts/blob/master/notebook_examples/simple_examples.ipynb) [](https://colab.research.google.com/github/OlivierBeq/Papyrus-scripts/blob/master/notebook_examples/simple_examples.ipynb)
|
|
171
|
+
- [`matchRCSB.ipynb`](https://github.com/OlivierBeq/Papyrus-scripts/blob/master/notebook_examples/matchRCSB.ipynb) — matching Papyrus data against the Protein Data Bank [](https://colab.research.google.com/github/OlivierBeq/Papyrus-scripts/blob/master/notebook_examples/matchRCSB.ipynb)
|
|
172
|
+
- [`advanced_querying.ipynb`](https://github.com/OlivierBeq/Papyrus-scripts/blob/master/notebook_examples/advanced_querying.ipynb)
|
|
173
|
+
- To reproduce the models, extraction pipeline and visualizations from the original publication, see [Papyrus-modelling](https://github.com/OlivierBeq/Papyrus-modelling).
|
|
174
|
+
|
|
175
|
+
## 🖋️ Citation
|
|
176
|
+
|
|
177
|
+
If you use `papyrus-scripts` or the Papyrus dataset in your research, please cite:
|
|
178
|
+
|
|
179
|
+
```bibtex
|
|
180
|
+
@article{Bequignon2023Papyrus,
|
|
181
|
+
title={Papyrus - A large scale curated dataset aimed at bioactivity predictions},
|
|
182
|
+
author={B{\'e}quignon, Olivier J.M. and Bongers, Bart J. and Jespers, Willem and IJzerman, Adriaan P. and van de Water, Bob and van Westen, Gerard J.P.},
|
|
183
|
+
journal={Journal of Cheminformatics},
|
|
184
|
+
volume={15},
|
|
185
|
+
number={3},
|
|
186
|
+
year={2023},
|
|
187
|
+
doi={10.1186/s13321-022-00672-x}
|
|
188
|
+
}
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
## 📄 License
|
|
192
|
+
|
|
193
|
+
This project is licensed under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77.0.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "papyrus_scripts"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "A collection of scripts to handle the Papyrus bioactivity dataset"
|
|
9
|
+
requires-python = ">=3.11"
|
|
10
|
+
readme = { file = "README.md", content-type = "text/markdown" }
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [
|
|
14
|
+
{ name = "Olivier J. M. Béquignon", email = "olivier.bequignon.maintainer@gmail.com" },
|
|
15
|
+
{ name = "Brandon J. Bongers"},
|
|
16
|
+
{ name = "Willem Jespers"},
|
|
17
|
+
|
|
18
|
+
]
|
|
19
|
+
maintainers = [
|
|
20
|
+
{ name = "Olivier J. M. Béquignon", email = "olivier.bequignon.maintainer@gmail.com" },
|
|
21
|
+
]
|
|
22
|
+
classifiers = [
|
|
23
|
+
"Development Status :: 2 - Pre-Alpha",
|
|
24
|
+
"Programming Language :: Python",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
"Programming Language :: Python :: 3.13",
|
|
28
|
+
]
|
|
29
|
+
keywords = [
|
|
30
|
+
"bioactivity data",
|
|
31
|
+
"QSAR",
|
|
32
|
+
"proteochemometrics",
|
|
33
|
+
"cheminformatics",
|
|
34
|
+
"modelling",
|
|
35
|
+
"machine learning",
|
|
36
|
+
]
|
|
37
|
+
dependencies = [
|
|
38
|
+
"click",
|
|
39
|
+
"numpy>=2.0.0",
|
|
40
|
+
"pandas",
|
|
41
|
+
"polars",
|
|
42
|
+
"pyarrow",
|
|
43
|
+
"rdkit",
|
|
44
|
+
"requests",
|
|
45
|
+
"fake-useragent",
|
|
46
|
+
"tqdm",
|
|
47
|
+
"ipywidgets",
|
|
48
|
+
"colorama; platform_system == 'Windows'",
|
|
49
|
+
"mordred",
|
|
50
|
+
"scikit-learn",
|
|
51
|
+
"xgboost",
|
|
52
|
+
"pystow",
|
|
53
|
+
"prodec",
|
|
54
|
+
"defusedxml",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
[project.urls]
|
|
58
|
+
Homepage = "https://github.com/OlivierBeq/Papyrus-scripts"
|
|
59
|
+
|
|
60
|
+
[project.scripts]
|
|
61
|
+
papyrus = "papyrus_scripts.cli:main"
|
|
62
|
+
|
|
63
|
+
[project.optional-dependencies]
|
|
64
|
+
testing = [
|
|
65
|
+
"pytest",
|
|
66
|
+
"parameterized",
|
|
67
|
+
]
|
|
68
|
+
dnn = [
|
|
69
|
+
"torch",
|
|
70
|
+
"skorch",
|
|
71
|
+
]
|
|
72
|
+
subsim = [
|
|
73
|
+
"tables",
|
|
74
|
+
"FPSim2",
|
|
75
|
+
]
|
|
76
|
+
gpu = [
|
|
77
|
+
"cupy-cuda12x[ctk]",
|
|
78
|
+
]
|
|
79
|
+
simgpu = [
|
|
80
|
+
"papyrus_scripts[subsim,gpu]"
|
|
81
|
+
]
|
|
82
|
+
all = [
|
|
83
|
+
"papyrus_scripts[dnn,simgpu]"
|
|
84
|
+
]
|
|
85
|
+
|
|
86
|
+
[tool.coverage.run]
|
|
87
|
+
source = ["src/papyrus_scripts"]
|
|
88
|
+
omit = ["*/__main__.py"]
|
|
89
|
+
|
|
90
|
+
[tool.coverage.report]
|
|
91
|
+
fail_under = 95
|
|
92
|
+
exclude_also = [
|
|
93
|
+
"if __name__ == .__main__.:",
|
|
94
|
+
"if TYPE_CHECKING:",
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
[tool.setuptools.dynamic]
|
|
98
|
+
version = { attr = "papyrus_scripts.__version__" }
|
|
99
|
+
|
|
100
|
+
[tool.setuptools.packages.find]
|
|
101
|
+
where = ["src"]
|
|
102
|
+
|
|
103
|
+
[tool.setuptools.package-data]
|
|
104
|
+
"*" = ["*.json"]
|
|
105
|
+
|
|
106
|
+
[tool.ruff]
|
|
107
|
+
line-length = 120
|
|
108
|
+
target-version = "py311"
|
|
109
|
+
extend-exclude = ["tests/fixtures"]
|
|
110
|
+
|
|
111
|
+
[tool.ruff.lint]
|
|
112
|
+
select = [
|
|
113
|
+
"E", # pycodestyle errors (flake8)
|
|
114
|
+
"W", # pycodestyle warnings (flake8)
|
|
115
|
+
"F", # pyflakes (flake8)
|
|
116
|
+
"I", # import sorting (isort, flake8-isort)
|
|
117
|
+
"N", # naming conventions (pep8-naming)
|
|
118
|
+
"D", # docstring conventions (pydocstyle, flake8-docstrings)
|
|
119
|
+
"UP", # pyupgrade
|
|
120
|
+
"B", # bugbear (flake8-bugbear)
|
|
121
|
+
"A", # shadowed builtins (flake8-builtins)
|
|
122
|
+
"COM", # trailing commas (flake8-commas)
|
|
123
|
+
"C4", # comprehensions (flake8-comprehensions)
|
|
124
|
+
"T20", # print statements (flake8-print)
|
|
125
|
+
"S", # security (flake8-bandit)
|
|
126
|
+
]
|
|
127
|
+
ignore = [
|
|
128
|
+
"S310", # URL open with a variable scheme (blanket-ignored under flake8 too)
|
|
129
|
+
"UP009", # UTF-8 coding declaration: kept deliberately in every file, do not strip
|
|
130
|
+
"N999", # module naming: renaming existing public modules (IO.py, matchRCSB.py, ...) would break imports
|
|
131
|
+
"N802", # function/method naming: mirrors external APIs (torch.autograd.Variable,
|
|
132
|
+
# RDKit's GetMatches, unittest's assertX convention), not a mistake
|
|
133
|
+
"N803", # argument naming: mirrors sklearn/RDKit conventions (X, y, nBits, isQuery, is3D, ...)
|
|
134
|
+
"N806", # in-function variable naming: CHUNKSIZE/_ROOT_FTYPES-style caps used for local constants
|
|
135
|
+
"N812", # import aliasing: torch's own convention (`import torch as T`, `functional as F`)
|
|
136
|
+
"T201", # print(): this is a CLI tool and training loop that deliberately prints progress/output
|
|
137
|
+
"A001", # builtin-shadowing variables: descriptive local names (format, input, id) used harmlessly
|
|
138
|
+
"A002", # same, for function arguments
|
|
139
|
+
"C408", # dict(...) calls: used deliberately for readability when building keyword-style dicts
|
|
140
|
+
"E701", # multiple statements on one line: used for compact optional-dependency stub classes
|
|
141
|
+
]
|
|
142
|
+
|
|
143
|
+
[tool.ruff.lint.per-file-ignores]
|
|
144
|
+
"tests/*" = [
|
|
145
|
+
"S101", # assert usage is expected in tests
|
|
146
|
+
"D", # tests are not held to docstring conventions
|
|
147
|
+
]
|
|
148
|
+
"src/papyrus_scripts/__init__.py" = [
|
|
149
|
+
"F401", # this is a barrel file: imports are re-exports of the public API, not unused
|
|
150
|
+
]
|
|
151
|
+
|
|
152
|
+
[tool.ruff.lint.isort]
|
|
153
|
+
known-first-party = ["papyrus_scripts", "tests"]
|
|
154
|
+
|
|
155
|
+
[tool.ruff.lint.pydocstyle]
|
|
156
|
+
convention = "pep257"
|
|
157
|
+
|
|
158
|
+
[tool.mypy]
|
|
159
|
+
python_version = "3.11"
|
|
160
|
+
exclude = ["tests/fixtures", "build", "dist"]
|
|
161
|
+
|
|
162
|
+
[[tool.mypy.overrides]]
|
|
163
|
+
module = ["rdkit", "rdkit.*"]
|
|
164
|
+
follow_imports = "skip"
|
|
165
|
+
ignore_missing_imports = true
|
|
166
|
+
|
|
167
|
+
[[tool.mypy.overrides]]
|
|
168
|
+
module = [
|
|
169
|
+
"pandas.*", "requests.*", "tqdm.*", "sklearn.*", "prodec.*", "skorch.*",
|
|
170
|
+
"defusedxml.*", "scipy.*", "mordred.*", "xgboost.*", "pystow.*",
|
|
171
|
+
"fake_useragent.*", "FPSim2.*", "cupy.*", "tables.*", "openbabel.*",
|
|
172
|
+
]
|
|
173
|
+
ignore_missing_imports = true
|