pandera-chem 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pandera_chem-0.1.0/LICENSE +21 -0
- pandera_chem-0.1.0/PKG-INFO +215 -0
- pandera_chem-0.1.0/README.md +189 -0
- pandera_chem-0.1.0/pyproject.toml +86 -0
- pandera_chem-0.1.0/pyproject.toml.orig +68 -0
- pandera_chem-0.1.0/src/pandera_chem/__init__.py +56 -0
- pandera_chem-0.1.0/src/pandera_chem/_mol.py +77 -0
- pandera_chem-0.1.0/src/pandera_chem/checks.py +393 -0
- pandera_chem-0.1.0/src/pandera_chem/py.typed +0 -0
- pandera_chem-0.1.0/src/pandera_chem/quarantine.py +163 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Monstrous Data (David Obeh Alobo)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pandera-chem
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Chemistry checks for Pandera: validate tables of molecules with RDKit, and quarantine the bad rows with their reasons
|
|
5
|
+
Keywords: cheminformatics,pandera,rdkit,data-quality,data-validation,smiles,chemistry
|
|
6
|
+
Author: David.Alobo
|
|
7
|
+
Author-email: David.Alobo <davidalobo01@gmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Chemistry
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Dist: pandera[pandas]>=0.24
|
|
20
|
+
Requires-Dist: rdkit>=2024.9
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Project-URL: Homepage, https://github.com/David-Alobo/pandera-chem
|
|
23
|
+
Project-URL: Issues, https://github.com/David-Alobo/pandera-chem/issues
|
|
24
|
+
Project-URL: Course, https://monstrous.com.ng/learn/
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
<p align="center">
|
|
28
|
+
<img src=".github/banner.png" alt="pandera-chem: chemistry checks for Pandera. Validate tables of molecules with RDKit, and quarantine the bad rows with their reasons." width="100%">
|
|
29
|
+
</p>
|
|
30
|
+
|
|
31
|
+
<p align="center">
|
|
32
|
+
<a href="https://github.com/David-Alobo/pandera-chem/actions/workflows/ci.yml"><img src="https://github.com/David-Alobo/pandera-chem/actions/workflows/ci.yml/badge.svg" alt="Tests"></a>
|
|
33
|
+
<img src="https://img.shields.io/badge/python-3.11_%7C_3.12_%7C_3.13-3776ab" alt="Python 3.11, 3.12, 3.13">
|
|
34
|
+
<a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json" alt="Ruff"></a>
|
|
35
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="MIT licence"></a>
|
|
36
|
+
<a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-7c5cff" alt="Cite"></a>
|
|
37
|
+
</p>
|
|
38
|
+
|
|
39
|
+
**Chemistry checks for [Pandera](https://pandera.readthedocs.io/).** Validate tables of molecules
|
|
40
|
+
with [RDKit](https://www.rdkit.org/) inside the data pipeline you already have, and set the bad rows
|
|
41
|
+
aside with the reason each one failed, instead of crashing or losing them.
|
|
42
|
+
|
|
43
|
+
## Why
|
|
44
|
+
|
|
45
|
+
Chemical tables arrive with problems a type check cannot see: SMILES that RDKit cannot read, salts
|
|
46
|
+
and mixtures, unexpected elements, charged forms, missing stereochemistry, and the same molecule
|
|
47
|
+
twice under different SMILES. The usual fix is a loop that drops what fails to parse, and the
|
|
48
|
+
records vanish with no reason recorded.
|
|
49
|
+
|
|
50
|
+
pandera-chem turns those problems into Pandera checks, so they sit in the same schema as your other
|
|
51
|
+
rules (types, ranges, units), and adds `split()`, which keeps the good rows, sets the bad ones aside
|
|
52
|
+
with their reasons, and gives you counts that must add up.
|
|
53
|
+
|
|
54
|
+
## Install
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
pip install pandera-chem
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
or `uv add pandera-chem`. It needs Python 3.11 or newer, and installs Pandera and RDKit.
|
|
61
|
+
|
|
62
|
+
## Example
|
|
63
|
+
|
|
64
|
+
A table of measured compounds, with a few of the problems real data has:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
import pandas as pd
|
|
68
|
+
import pandera.pandas as pa
|
|
69
|
+
|
|
70
|
+
import pandera_chem as pc
|
|
71
|
+
|
|
72
|
+
table = pd.DataFrame(
|
|
73
|
+
{
|
|
74
|
+
"compound_id": ["C1", "C2", "C3", "C4", "C5", "C6", "C7"],
|
|
75
|
+
"smiles": [
|
|
76
|
+
"CC(=O)Oc1ccccc1C(=O)O", # aspirin
|
|
77
|
+
"OC(=O)c1ccccc1OC(C)=O", # aspirin again, written differently
|
|
78
|
+
"c1ccc2ccccc2c1", # naphthalene
|
|
79
|
+
"c1ccccc", # a ring that never closes
|
|
80
|
+
"C[Se]c1ccccc1", # selenium
|
|
81
|
+
"CC(=O)[O-].[Na+]", # a salt
|
|
82
|
+
"CC(C)Cc1ccc(cc1)C(C)C(=O)O", # ibuprofen, stereocentre not given
|
|
83
|
+
],
|
|
84
|
+
"pic50": [5.2, 5.4, 4.1, 6.0, 5.0, 3.2, 15.0],
|
|
85
|
+
}
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
schema = pa.DataFrameSchema(
|
|
89
|
+
{
|
|
90
|
+
"compound_id": pa.Column(str, unique=True),
|
|
91
|
+
"smiles": pa.Column(
|
|
92
|
+
str,
|
|
93
|
+
[
|
|
94
|
+
pc.valid_smiles(),
|
|
95
|
+
pc.allowed_elements(),
|
|
96
|
+
pc.single_fragment(),
|
|
97
|
+
pc.unique_molecules(),
|
|
98
|
+
pc.stereo_defined(raise_warning=True), # a warning, not a failure
|
|
99
|
+
],
|
|
100
|
+
),
|
|
101
|
+
"pic50": pa.Column(float, pa.Check.in_range(0, 14)),
|
|
102
|
+
}
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
result = pc.split(table, schema)
|
|
106
|
+
print(result.report)
|
|
107
|
+
print()
|
|
108
|
+
print(result.quarantined[["compound_id", "reasons"]].to_string(index=False))
|
|
109
|
+
print()
|
|
110
|
+
print(pc.problems("c1ccccc"))
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
It prints:
|
|
114
|
+
|
|
115
|
+
```text
|
|
116
|
+
7 rows in: 2 valid, 5 quarantined
|
|
117
|
+
2 smiles: allowed_elements
|
|
118
|
+
1 smiles: valid_smiles
|
|
119
|
+
1 smiles: single_fragment
|
|
120
|
+
1 smiles: unique_molecules
|
|
121
|
+
1 pic50: in_range(0, 14)
|
|
122
|
+
warning: Column 'smiles' failed element-wise validator number 4: <Check stereo_defined> failure cases: CC(C)Cc1ccc(cc1)C(C)C(=O)O
|
|
123
|
+
|
|
124
|
+
compound_id reasons
|
|
125
|
+
C2 smiles: unique_molecules
|
|
126
|
+
C4 smiles: valid_smiles
|
|
127
|
+
C5 smiles: allowed_elements
|
|
128
|
+
C6 smiles: allowed_elements; smiles: single_fragment
|
|
129
|
+
C7 pic50: in_range(0, 14)
|
|
130
|
+
|
|
131
|
+
['the SMILES could not be read']
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
- **`result.valid`**: the rows that passed (aspirin and naphthalene), ready for the next step.
|
|
135
|
+
- **`result.quarantined`**: the rest, unchanged, with a `reasons` column. Save it, fix the records
|
|
136
|
+
and run them again.
|
|
137
|
+
- **`result.report`**: the counts. `result.report.reconciles` is `True` when every row is accounted
|
|
138
|
+
for (rows in = valid + quarantined); log it with every run.
|
|
139
|
+
- Ibuprofen passed with a **warning**: its stereocentre is not given, which you may accept for a
|
|
140
|
+
racemate. Any check becomes a warning with `raise_warning=True`.
|
|
141
|
+
|
|
142
|
+
The checks work in plain Pandera too: `schema.validate(table)` raises an error naming the check that
|
|
143
|
+
failed, as Pandera always does.
|
|
144
|
+
|
|
145
|
+
## The checks
|
|
146
|
+
|
|
147
|
+
On a column of SMILES:
|
|
148
|
+
|
|
149
|
+
| Check | Fails when |
|
|
150
|
+
|---|---|
|
|
151
|
+
| `valid_smiles()` | RDKit cannot read or sanitise the SMILES (syntax, valence, aromaticity) |
|
|
152
|
+
| `allowed_elements(elements=ORGANIC)` | an atom is not in the allowed set (default: H, B, C, N, O, F, P, S, Cl, Br, I) |
|
|
153
|
+
| `single_fragment()` | the record holds more than one molecule: a salt, a solvent or a mixture |
|
|
154
|
+
| `neutral()` | the net formal charge is not zero (charges that cancel, as in a nitro group, are fine) |
|
|
155
|
+
| `stereo_defined()` | a stereocentre or stereogenic double bond has no configuration given |
|
|
156
|
+
| `heavy_atoms(min_value, max_value)` | the largest fragment has too few or too many heavy atoms |
|
|
157
|
+
| `mol_weight(max_value, min_value=0)` | the largest fragment is too light or too heavy (g/mol) |
|
|
158
|
+
| `no_substructures({"name": "SMARTS", ...})` | the molecule contains one of the given groups |
|
|
159
|
+
| `unique_molecules()` | the same molecule appeared in an earlier row, even as a different SMILES |
|
|
160
|
+
|
|
161
|
+
On InChIKeys:
|
|
162
|
+
|
|
163
|
+
| Check | Fails when |
|
|
164
|
+
|---|---|
|
|
165
|
+
| `inchikey_format()` | the value is not shaped like a standard InChIKey |
|
|
166
|
+
| `inchikey_matches("smiles", "inchikey")` | (table-wide) a row's InChIKey is not the one its SMILES gives |
|
|
167
|
+
|
|
168
|
+
Helpers: `pc.problems(smiles)` says why a SMILES cannot be read, in RDKit's words;
|
|
169
|
+
`pc.inchikey(smiles)` gives its standard InChIKey; `pc.clear_cache()` frees the parsed molecules.
|
|
170
|
+
|
|
171
|
+
## How it works
|
|
172
|
+
|
|
173
|
+
- **One reason per problem.** Only `valid_smiles()` fails a SMILES that cannot be read; every other
|
|
174
|
+
check lets it pass. A broken record is reported once, as unreadable, not under every check.
|
|
175
|
+
- **Each SMILES is read once.** All the checks share one cache of parsed molecules, so five checks on
|
|
176
|
+
a column cost about one parse per distinct SMILES. Call `pc.clear_cache()` after a very large table.
|
|
177
|
+
- **Pandera's own options work.** Every check accepts them: `raise_warning=True`,
|
|
178
|
+
`n_failure_cases=10` and so on.
|
|
179
|
+
- **Nothing is changed.** pandera-chem checks molecules; it never alters them. To repair records
|
|
180
|
+
(strip salts, neutralise charges, pick one tautomer), use RDKit's
|
|
181
|
+
[MolStandardize](https://www.rdkit.org/docs/source/rdkit.Chem.MolStandardize.rdMolStandardize.html)
|
|
182
|
+
or the [ChEMBL structure pipeline](https://github.com/chembl/ChEMBL_Structure_Pipeline) first, then
|
|
183
|
+
check the result. For a web page that validates an uploaded file, see
|
|
184
|
+
[ChemAudit](https://nfdi4chem.de/introducing-chemaudit/).
|
|
185
|
+
|
|
186
|
+
## Roadmap
|
|
187
|
+
|
|
188
|
+
Ideas for later versions; [suggest one](https://github.com/David-Alobo/pandera-chem/issues/new?template=feature.yml).
|
|
189
|
+
|
|
190
|
+
- Polars data frames, which Pandera also supports.
|
|
191
|
+
- The ChEMBL structure checker as a check, with a penalty threshold.
|
|
192
|
+
- Pydantic types for single records (`SMILES`, `InChIKey`).
|
|
193
|
+
- Checks registered by name, so schemas can be written in YAML.
|
|
194
|
+
|
|
195
|
+
## Learn the ideas behind it
|
|
196
|
+
|
|
197
|
+
pandera-chem grew out of the free course
|
|
198
|
+
[Data Engineering for Cheminformatics](https://monstrous.com.ng/learn/), which builds these checks by
|
|
199
|
+
hand: validating molecules in Module 6, the quarantine in Module 10, and validating whole tables with
|
|
200
|
+
Pydantic and Pandera in Module 12.
|
|
201
|
+
|
|
202
|
+
## Contributing
|
|
203
|
+
|
|
204
|
+
Bug reports, suggestions and pull requests are welcome: see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
205
|
+
Everyone taking part follows the [code of conduct](CODE_OF_CONDUCT.md). The example above is run by
|
|
206
|
+
the tests, which check that this README shows what it really prints.
|
|
207
|
+
|
|
208
|
+
## Citing
|
|
209
|
+
|
|
210
|
+
If pandera-chem helps your research, please cite it: GitHub's **Cite this repository** button (from
|
|
211
|
+
[CITATION.cff](CITATION.cff)) gives the reference in APA and BibTeX.
|
|
212
|
+
|
|
213
|
+
## Licence
|
|
214
|
+
|
|
215
|
+
MIT ([LICENSE](LICENSE)). Made by [Monstrous Data](https://monstrous.com.ng).
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src=".github/banner.png" alt="pandera-chem: chemistry checks for Pandera. Validate tables of molecules with RDKit, and quarantine the bad rows with their reasons." width="100%">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center">
|
|
6
|
+
<a href="https://github.com/David-Alobo/pandera-chem/actions/workflows/ci.yml"><img src="https://github.com/David-Alobo/pandera-chem/actions/workflows/ci.yml/badge.svg" alt="Tests"></a>
|
|
7
|
+
<img src="https://img.shields.io/badge/python-3.11_%7C_3.12_%7C_3.13-3776ab" alt="Python 3.11, 3.12, 3.13">
|
|
8
|
+
<a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json" alt="Ruff"></a>
|
|
9
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="MIT licence"></a>
|
|
10
|
+
<a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-7c5cff" alt="Cite"></a>
|
|
11
|
+
</p>
|
|
12
|
+
|
|
13
|
+
**Chemistry checks for [Pandera](https://pandera.readthedocs.io/).** Validate tables of molecules
|
|
14
|
+
with [RDKit](https://www.rdkit.org/) inside the data pipeline you already have, and set the bad rows
|
|
15
|
+
aside with the reason each one failed, instead of crashing or losing them.
|
|
16
|
+
|
|
17
|
+
## Why
|
|
18
|
+
|
|
19
|
+
Chemical tables arrive with problems a type check cannot see: SMILES that RDKit cannot read, salts
|
|
20
|
+
and mixtures, unexpected elements, charged forms, missing stereochemistry, and the same molecule
|
|
21
|
+
twice under different SMILES. The usual fix is a loop that drops what fails to parse, and the
|
|
22
|
+
records vanish with no reason recorded.
|
|
23
|
+
|
|
24
|
+
pandera-chem turns those problems into Pandera checks, so they sit in the same schema as your other
|
|
25
|
+
rules (types, ranges, units), and adds `split()`, which keeps the good rows, sets the bad ones aside
|
|
26
|
+
with their reasons, and gives you counts that must add up.
|
|
27
|
+
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install pandera-chem
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
or `uv add pandera-chem`. It needs Python 3.11 or newer, and installs Pandera and RDKit.
|
|
35
|
+
|
|
36
|
+
## Example
|
|
37
|
+
|
|
38
|
+
A table of measured compounds, with a few of the problems real data has:
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
import pandas as pd
|
|
42
|
+
import pandera.pandas as pa
|
|
43
|
+
|
|
44
|
+
import pandera_chem as pc
|
|
45
|
+
|
|
46
|
+
table = pd.DataFrame(
|
|
47
|
+
{
|
|
48
|
+
"compound_id": ["C1", "C2", "C3", "C4", "C5", "C6", "C7"],
|
|
49
|
+
"smiles": [
|
|
50
|
+
"CC(=O)Oc1ccccc1C(=O)O", # aspirin
|
|
51
|
+
"OC(=O)c1ccccc1OC(C)=O", # aspirin again, written differently
|
|
52
|
+
"c1ccc2ccccc2c1", # naphthalene
|
|
53
|
+
"c1ccccc", # a ring that never closes
|
|
54
|
+
"C[Se]c1ccccc1", # selenium
|
|
55
|
+
"CC(=O)[O-].[Na+]", # a salt
|
|
56
|
+
"CC(C)Cc1ccc(cc1)C(C)C(=O)O", # ibuprofen, stereocentre not given
|
|
57
|
+
],
|
|
58
|
+
"pic50": [5.2, 5.4, 4.1, 6.0, 5.0, 3.2, 15.0],
|
|
59
|
+
}
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
schema = pa.DataFrameSchema(
|
|
63
|
+
{
|
|
64
|
+
"compound_id": pa.Column(str, unique=True),
|
|
65
|
+
"smiles": pa.Column(
|
|
66
|
+
str,
|
|
67
|
+
[
|
|
68
|
+
pc.valid_smiles(),
|
|
69
|
+
pc.allowed_elements(),
|
|
70
|
+
pc.single_fragment(),
|
|
71
|
+
pc.unique_molecules(),
|
|
72
|
+
pc.stereo_defined(raise_warning=True), # a warning, not a failure
|
|
73
|
+
],
|
|
74
|
+
),
|
|
75
|
+
"pic50": pa.Column(float, pa.Check.in_range(0, 14)),
|
|
76
|
+
}
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
result = pc.split(table, schema)
|
|
80
|
+
print(result.report)
|
|
81
|
+
print()
|
|
82
|
+
print(result.quarantined[["compound_id", "reasons"]].to_string(index=False))
|
|
83
|
+
print()
|
|
84
|
+
print(pc.problems("c1ccccc"))
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
It prints:
|
|
88
|
+
|
|
89
|
+
```text
|
|
90
|
+
7 rows in: 2 valid, 5 quarantined
|
|
91
|
+
2 smiles: allowed_elements
|
|
92
|
+
1 smiles: valid_smiles
|
|
93
|
+
1 smiles: single_fragment
|
|
94
|
+
1 smiles: unique_molecules
|
|
95
|
+
1 pic50: in_range(0, 14)
|
|
96
|
+
warning: Column 'smiles' failed element-wise validator number 4: <Check stereo_defined> failure cases: CC(C)Cc1ccc(cc1)C(C)C(=O)O
|
|
97
|
+
|
|
98
|
+
compound_id reasons
|
|
99
|
+
C2 smiles: unique_molecules
|
|
100
|
+
C4 smiles: valid_smiles
|
|
101
|
+
C5 smiles: allowed_elements
|
|
102
|
+
C6 smiles: allowed_elements; smiles: single_fragment
|
|
103
|
+
C7 pic50: in_range(0, 14)
|
|
104
|
+
|
|
105
|
+
['the SMILES could not be read']
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
- **`result.valid`**: the rows that passed (aspirin and naphthalene), ready for the next step.
|
|
109
|
+
- **`result.quarantined`**: the rest, unchanged, with a `reasons` column. Save it, fix the records
|
|
110
|
+
and run them again.
|
|
111
|
+
- **`result.report`**: the counts. `result.report.reconciles` is `True` when every row is accounted
|
|
112
|
+
for (rows in = valid + quarantined); log it with every run.
|
|
113
|
+
- Ibuprofen passed with a **warning**: its stereocentre is not given, which you may accept for a
|
|
114
|
+
racemate. Any check becomes a warning with `raise_warning=True`.
|
|
115
|
+
|
|
116
|
+
The checks work in plain Pandera too: `schema.validate(table)` raises an error naming the check that
|
|
117
|
+
failed, as Pandera always does.
|
|
118
|
+
|
|
119
|
+
## The checks
|
|
120
|
+
|
|
121
|
+
On a column of SMILES:
|
|
122
|
+
|
|
123
|
+
| Check | Fails when |
|
|
124
|
+
|---|---|
|
|
125
|
+
| `valid_smiles()` | RDKit cannot read or sanitise the SMILES (syntax, valence, aromaticity) |
|
|
126
|
+
| `allowed_elements(elements=ORGANIC)` | an atom is not in the allowed set (default: H, B, C, N, O, F, P, S, Cl, Br, I) |
|
|
127
|
+
| `single_fragment()` | the record holds more than one molecule: a salt, a solvent or a mixture |
|
|
128
|
+
| `neutral()` | the net formal charge is not zero (charges that cancel, as in a nitro group, are fine) |
|
|
129
|
+
| `stereo_defined()` | a stereocentre or stereogenic double bond has no configuration given |
|
|
130
|
+
| `heavy_atoms(min_value, max_value)` | the largest fragment has too few or too many heavy atoms |
|
|
131
|
+
| `mol_weight(max_value, min_value=0)` | the largest fragment is too light or too heavy (g/mol) |
|
|
132
|
+
| `no_substructures({"name": "SMARTS", ...})` | the molecule contains one of the given groups |
|
|
133
|
+
| `unique_molecules()` | the same molecule appeared in an earlier row, even as a different SMILES |
|
|
134
|
+
|
|
135
|
+
On InChIKeys:
|
|
136
|
+
|
|
137
|
+
| Check | Fails when |
|
|
138
|
+
|---|---|
|
|
139
|
+
| `inchikey_format()` | the value is not shaped like a standard InChIKey |
|
|
140
|
+
| `inchikey_matches("smiles", "inchikey")` | (table-wide) a row's InChIKey is not the one its SMILES gives |
|
|
141
|
+
|
|
142
|
+
Helpers: `pc.problems(smiles)` says why a SMILES cannot be read, in RDKit's words;
|
|
143
|
+
`pc.inchikey(smiles)` gives its standard InChIKey; `pc.clear_cache()` frees the parsed molecules.
|
|
144
|
+
|
|
145
|
+
## How it works
|
|
146
|
+
|
|
147
|
+
- **One reason per problem.** Only `valid_smiles()` fails a SMILES that cannot be read; every other
|
|
148
|
+
check lets it pass. A broken record is reported once, as unreadable, not under every check.
|
|
149
|
+
- **Each SMILES is read once.** All the checks share one cache of parsed molecules, so five checks on
|
|
150
|
+
a column cost about one parse per distinct SMILES. Call `pc.clear_cache()` after a very large table.
|
|
151
|
+
- **Pandera's own options work.** Every check accepts them: `raise_warning=True`,
|
|
152
|
+
`n_failure_cases=10` and so on.
|
|
153
|
+
- **Nothing is changed.** pandera-chem checks molecules; it never alters them. To repair records
|
|
154
|
+
(strip salts, neutralise charges, pick one tautomer), use RDKit's
|
|
155
|
+
[MolStandardize](https://www.rdkit.org/docs/source/rdkit.Chem.MolStandardize.rdMolStandardize.html)
|
|
156
|
+
or the [ChEMBL structure pipeline](https://github.com/chembl/ChEMBL_Structure_Pipeline) first, then
|
|
157
|
+
check the result. For a web page that validates an uploaded file, see
|
|
158
|
+
[ChemAudit](https://nfdi4chem.de/introducing-chemaudit/).
|
|
159
|
+
|
|
160
|
+
## Roadmap
|
|
161
|
+
|
|
162
|
+
Ideas for later versions; [suggest one](https://github.com/David-Alobo/pandera-chem/issues/new?template=feature.yml).
|
|
163
|
+
|
|
164
|
+
- Polars data frames, which Pandera also supports.
|
|
165
|
+
- The ChEMBL structure checker as a check, with a penalty threshold.
|
|
166
|
+
- Pydantic types for single records (`SMILES`, `InChIKey`).
|
|
167
|
+
- Checks registered by name, so schemas can be written in YAML.
|
|
168
|
+
|
|
169
|
+
## Learn the ideas behind it
|
|
170
|
+
|
|
171
|
+
pandera-chem grew out of the free course
|
|
172
|
+
[Data Engineering for Cheminformatics](https://monstrous.com.ng/learn/), which builds these checks by
|
|
173
|
+
hand: validating molecules in Module 6, the quarantine in Module 10, and validating whole tables with
|
|
174
|
+
Pydantic and Pandera in Module 12.
|
|
175
|
+
|
|
176
|
+
## Contributing
|
|
177
|
+
|
|
178
|
+
Bug reports, suggestions and pull requests are welcome: see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
179
|
+
Everyone taking part follows the [code of conduct](CODE_OF_CONDUCT.md). The example above is run by
|
|
180
|
+
the tests, which check that this README shows what it really prints.
|
|
181
|
+
|
|
182
|
+
## Citing
|
|
183
|
+
|
|
184
|
+
If pandera-chem helps your research, please cite it: GitHub's **Cite this repository** button (from
|
|
185
|
+
[CITATION.cff](CITATION.cff)) gives the reference in APA and BibTeX.
|
|
186
|
+
|
|
187
|
+
## Licence
|
|
188
|
+
|
|
189
|
+
MIT ([LICENSE](LICENSE)). Made by [Monstrous Data](https://monstrous.com.ng).
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "pandera-chem"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Chemistry checks for Pandera: validate tables of molecules with RDKit, and quarantine the bad rows with their reasons"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
license-files = ["LICENSE"]
|
|
9
|
+
keywords = [
|
|
10
|
+
"cheminformatics",
|
|
11
|
+
"pandera",
|
|
12
|
+
"rdkit",
|
|
13
|
+
"data-quality",
|
|
14
|
+
"data-validation",
|
|
15
|
+
"smiles",
|
|
16
|
+
"chemistry",
|
|
17
|
+
]
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Development Status :: 3 - Alpha",
|
|
20
|
+
"Intended Audience :: Science/Research",
|
|
21
|
+
"Intended Audience :: Developers",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Topic :: Scientific/Engineering :: Chemistry",
|
|
27
|
+
"Typing :: Typed",
|
|
28
|
+
]
|
|
29
|
+
dependencies = [
|
|
30
|
+
"pandera[pandas]>=0.24",
|
|
31
|
+
"rdkit>=2024.9",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[[project.authors]]
|
|
35
|
+
name = "David.Alobo"
|
|
36
|
+
email = "davidalobo01@gmail.com"
|
|
37
|
+
|
|
38
|
+
[project.urls]
|
|
39
|
+
Homepage = "https://github.com/David-Alobo/pandera-chem"
|
|
40
|
+
Issues = "https://github.com/David-Alobo/pandera-chem/issues"
|
|
41
|
+
Course = "https://monstrous.com.ng/learn/"
|
|
42
|
+
|
|
43
|
+
[build-system]
|
|
44
|
+
requires = ["uv_build>=0.13.4,<0.14.0"]
|
|
45
|
+
build-backend = "uv_build"
|
|
46
|
+
|
|
47
|
+
[dependency-groups]
|
|
48
|
+
dev = [
|
|
49
|
+
"hypothesis>=6.170.17",
|
|
50
|
+
"pytest>=9.2.0",
|
|
51
|
+
"pytest-cov>=7.1.0",
|
|
52
|
+
"ruff>=0.17.6",
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
[tool.ruff]
|
|
56
|
+
line-length = 100
|
|
57
|
+
target-version = "py311"
|
|
58
|
+
|
|
59
|
+
[tool.ruff.lint]
|
|
60
|
+
select = [
|
|
61
|
+
"E",
|
|
62
|
+
"F",
|
|
63
|
+
"W",
|
|
64
|
+
"I",
|
|
65
|
+
"UP",
|
|
66
|
+
"B",
|
|
67
|
+
"D",
|
|
68
|
+
"SIM",
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
[tool.ruff.lint.pydocstyle]
|
|
72
|
+
convention = "google"
|
|
73
|
+
|
|
74
|
+
[tool.ruff.lint.per-file-ignores]
|
|
75
|
+
"tests/*" = ["D"]
|
|
76
|
+
|
|
77
|
+
[tool.pytest.ini_options]
|
|
78
|
+
testpaths = ["tests"]
|
|
79
|
+
|
|
80
|
+
[tool.coverage.run]
|
|
81
|
+
source = ["pandera_chem"]
|
|
82
|
+
branch = true
|
|
83
|
+
|
|
84
|
+
[tool.coverage.report]
|
|
85
|
+
show_missing = true
|
|
86
|
+
fail_under = 95
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "pandera-chem"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Chemistry checks for Pandera: validate tables of molecules with RDKit, and quarantine the bad rows with their reasons"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [
|
|
7
|
+
{ name = "David.Alobo", email = "davidalobo01@gmail.com" }
|
|
8
|
+
]
|
|
9
|
+
requires-python = ">=3.11"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
keywords = ["cheminformatics", "pandera", "rdkit", "data-quality", "data-validation", "smiles", "chemistry"]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"Intended Audience :: Science/Research",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Topic :: Scientific/Engineering :: Chemistry",
|
|
22
|
+
"Typing :: Typed",
|
|
23
|
+
]
|
|
24
|
+
dependencies = [
|
|
25
|
+
"pandera[pandas]>=0.24",
|
|
26
|
+
"rdkit>=2024.9",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
[build-system]
|
|
30
|
+
requires = ["uv_build>=0.13.4,<0.14.0"]
|
|
31
|
+
build-backend = "uv_build"
|
|
32
|
+
|
|
33
|
+
[dependency-groups]
|
|
34
|
+
dev = [
|
|
35
|
+
"hypothesis>=6.170.17",
|
|
36
|
+
"pytest>=9.2.0",
|
|
37
|
+
"pytest-cov>=7.1.0",
|
|
38
|
+
"ruff>=0.17.6",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
[project.urls]
|
|
42
|
+
Homepage = "https://github.com/David-Alobo/pandera-chem"
|
|
43
|
+
Issues = "https://github.com/David-Alobo/pandera-chem/issues"
|
|
44
|
+
Course = "https://monstrous.com.ng/learn/"
|
|
45
|
+
|
|
46
|
+
[tool.ruff]
|
|
47
|
+
line-length = 100
|
|
48
|
+
target-version = "py311"
|
|
49
|
+
|
|
50
|
+
[tool.ruff.lint]
|
|
51
|
+
select = ["E", "F", "W", "I", "UP", "B", "D", "SIM"]
|
|
52
|
+
|
|
53
|
+
[tool.ruff.lint.pydocstyle]
|
|
54
|
+
convention = "google"
|
|
55
|
+
|
|
56
|
+
[tool.ruff.lint.per-file-ignores]
|
|
57
|
+
"tests/*" = ["D"]
|
|
58
|
+
|
|
59
|
+
[tool.pytest.ini_options]
|
|
60
|
+
testpaths = ["tests"]
|
|
61
|
+
|
|
62
|
+
[tool.coverage.run]
|
|
63
|
+
source = ["pandera_chem"]
|
|
64
|
+
branch = true
|
|
65
|
+
|
|
66
|
+
[tool.coverage.report]
|
|
67
|
+
show_missing = true
|
|
68
|
+
fail_under = 95
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Chemistry checks for Pandera: validate tables of molecules, and quarantine the bad rows.
|
|
2
|
+
|
|
3
|
+
Quick start::
|
|
4
|
+
|
|
5
|
+
import pandera.pandas as pa
|
|
6
|
+
import pandera_chem as pc
|
|
7
|
+
|
|
8
|
+
schema = pa.DataFrameSchema({
|
|
9
|
+
"smiles": pa.Column(str, [pc.valid_smiles(), pc.allowed_elements(), pc.single_fragment()]),
|
|
10
|
+
"pic50": pa.Column(float, pa.Check.in_range(0, 14)),
|
|
11
|
+
})
|
|
12
|
+
result = pc.split(table, schema)
|
|
13
|
+
print(result.report)
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from ._mol import clear_cache, inchikey, problems
|
|
17
|
+
from .checks import (
|
|
18
|
+
INCHIKEY,
|
|
19
|
+
ORGANIC,
|
|
20
|
+
allowed_elements,
|
|
21
|
+
heavy_atoms,
|
|
22
|
+
inchikey_format,
|
|
23
|
+
inchikey_matches,
|
|
24
|
+
mol_weight,
|
|
25
|
+
neutral,
|
|
26
|
+
no_substructures,
|
|
27
|
+
single_fragment,
|
|
28
|
+
stereo_defined,
|
|
29
|
+
unique_molecules,
|
|
30
|
+
valid_smiles,
|
|
31
|
+
)
|
|
32
|
+
from .quarantine import Report, Split, split
|
|
33
|
+
|
|
34
|
+
__version__ = "0.1.0"
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"INCHIKEY",
|
|
38
|
+
"ORGANIC",
|
|
39
|
+
"Report",
|
|
40
|
+
"Split",
|
|
41
|
+
"allowed_elements",
|
|
42
|
+
"clear_cache",
|
|
43
|
+
"heavy_atoms",
|
|
44
|
+
"inchikey",
|
|
45
|
+
"inchikey_format",
|
|
46
|
+
"inchikey_matches",
|
|
47
|
+
"mol_weight",
|
|
48
|
+
"neutral",
|
|
49
|
+
"no_substructures",
|
|
50
|
+
"problems",
|
|
51
|
+
"single_fragment",
|
|
52
|
+
"split",
|
|
53
|
+
"stereo_defined",
|
|
54
|
+
"unique_molecules",
|
|
55
|
+
"valid_smiles",
|
|
56
|
+
]
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Reading molecules once: a cached parser shared by every check.
|
|
2
|
+
|
|
3
|
+
A schema often runs several chemistry checks on the same column, and a table often repeats the
|
|
4
|
+
same SMILES. Parsing is the slow part, so each distinct SMILES is parsed and sanitised once and
|
|
5
|
+
the result is reused. The cached molecules are only read, never changed, by the checks.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from functools import lru_cache
|
|
11
|
+
|
|
12
|
+
from rdkit import Chem, rdBase
|
|
13
|
+
|
|
14
|
+
CACHE_SIZE = 65_536 # distinct SMILES kept; about 100-200 MB at drug-like sizes
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@lru_cache(maxsize=CACHE_SIZE)
|
|
18
|
+
def parse(smiles: str) -> Chem.Mol | None:
|
|
19
|
+
"""Read and sanitise one SMILES, quietly.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
smiles: The SMILES to read.
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
The sanitised molecule, or None if it cannot be read or sanitised. RDKit's own error
|
|
26
|
+
messages are not printed; use :func:`pandera_chem.problems` to see why one failed.
|
|
27
|
+
"""
|
|
28
|
+
if not isinstance(smiles, str) or not smiles.strip():
|
|
29
|
+
return None
|
|
30
|
+
with rdBase.BlockLogs():
|
|
31
|
+
return Chem.MolFromSmiles(smiles)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@lru_cache(maxsize=CACHE_SIZE)
|
|
35
|
+
def inchikey(smiles: str) -> str | None:
|
|
36
|
+
"""The standard InChIKey of a SMILES, or None if it cannot be read or has no key.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
smiles: The SMILES.
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
The InChIKey, or None.
|
|
43
|
+
"""
|
|
44
|
+
mol = parse(smiles)
|
|
45
|
+
if mol is None:
|
|
46
|
+
return None
|
|
47
|
+
with rdBase.BlockLogs():
|
|
48
|
+
key = Chem.MolToInchiKey(mol)
|
|
49
|
+
return key or None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def problems(smiles: str) -> list[str]:
|
|
53
|
+
"""Why a SMILES cannot be read or sanitised, in RDKit's words.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
smiles: The SMILES.
|
|
57
|
+
|
|
58
|
+
Returns:
|
|
59
|
+
One message per problem; empty if the SMILES reads and sanitises.
|
|
60
|
+
"""
|
|
61
|
+
if not isinstance(smiles, str) or not smiles.strip():
|
|
62
|
+
return ["empty SMILES"]
|
|
63
|
+
with rdBase.BlockLogs():
|
|
64
|
+
mol = Chem.MolFromSmiles(smiles, sanitize=False)
|
|
65
|
+
if mol is None:
|
|
66
|
+
return ["the SMILES could not be read"]
|
|
67
|
+
return [p.Message() for p in Chem.DetectChemistryProblems(mol)]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def clear_cache() -> None:
|
|
71
|
+
"""Forget every parsed molecule and InChIKey (to free memory after a large table).
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
None.
|
|
75
|
+
"""
|
|
76
|
+
parse.cache_clear()
|
|
77
|
+
inchikey.cache_clear()
|
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
"""Chemistry checks for Pandera schemas.
|
|
2
|
+
|
|
3
|
+
Each function returns a ``pandera.Check`` to put on a column of SMILES (or of InChIKeys), next
|
|
4
|
+
to Pandera's own checks::
|
|
5
|
+
|
|
6
|
+
import pandera.pandas as pa
|
|
7
|
+
import pandera_chem as pc
|
|
8
|
+
|
|
9
|
+
schema = pa.DataFrameSchema({
|
|
10
|
+
"smiles": pa.Column(str, [pc.valid_smiles(), pc.allowed_elements(), pc.neutral()]),
|
|
11
|
+
})
|
|
12
|
+
|
|
13
|
+
One rule keeps the reasons clear: only :func:`valid_smiles` fails a SMILES that cannot be read.
|
|
14
|
+
Every other check passes it, so each bad record is reported once, for its real problem.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import re
|
|
20
|
+
from collections.abc import Iterable, Mapping
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
import pandas as pd
|
|
24
|
+
import pandera.pandas as pa
|
|
25
|
+
from rdkit import Chem
|
|
26
|
+
from rdkit.Chem import Descriptors
|
|
27
|
+
|
|
28
|
+
from ._mol import inchikey, parse
|
|
29
|
+
|
|
30
|
+
# the elements of most drug-like and screening libraries; pass your own to allowed_elements()
|
|
31
|
+
ORGANIC = frozenset({"H", "B", "C", "N", "O", "F", "P", "S", "Cl", "Br", "I"})
|
|
32
|
+
INCHIKEY = re.compile(r"^[A-Z]{14}-[A-Z]{10}-[A-Z]$")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def valid_smiles(**options: Any) -> pa.Check:
|
|
36
|
+
"""The SMILES can be read and sanitised by RDKit.
|
|
37
|
+
|
|
38
|
+
Catches syntax errors, unclosed rings, impossible valences and aromatic rings that cannot
|
|
39
|
+
be kekulised. :func:`pandera_chem.problems` says why a given SMILES fails.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
**options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
|
|
43
|
+
instead of failing, or ``n_failure_cases=10``.
|
|
44
|
+
|
|
45
|
+
Returns:
|
|
46
|
+
The check.
|
|
47
|
+
"""
|
|
48
|
+
return pa.Check(
|
|
49
|
+
lambda smiles: parse(smiles) is not None,
|
|
50
|
+
element_wise=True,
|
|
51
|
+
name="valid_smiles",
|
|
52
|
+
description="the SMILES cannot be read or sanitised by RDKit",
|
|
53
|
+
**options,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def allowed_elements(elements: Iterable[str] = ORGANIC, **options: Any) -> pa.Check:
|
|
58
|
+
"""Every atom is one of the allowed elements.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
elements: The element symbols allowed, e.g. ``{"C", "N", "O", "H"}``. Defaults to
|
|
62
|
+
:data:`ORGANIC`: H, B, C, N, O, F, P, S, Cl, Br and I.
|
|
63
|
+
**options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
|
|
64
|
+
instead of failing, or ``n_failure_cases=10``.
|
|
65
|
+
|
|
66
|
+
Returns:
|
|
67
|
+
The check.
|
|
68
|
+
"""
|
|
69
|
+
allowed = frozenset(elements)
|
|
70
|
+
|
|
71
|
+
def ok(smiles: str) -> bool:
|
|
72
|
+
"""Whether every atom of one SMILES is allowed (unreadable SMILES pass).
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
smiles: The SMILES.
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
True if allowed or unreadable.
|
|
79
|
+
"""
|
|
80
|
+
mol = parse(smiles)
|
|
81
|
+
return mol is None or all(a.GetSymbol() in allowed for a in mol.GetAtoms())
|
|
82
|
+
|
|
83
|
+
return pa.Check(
|
|
84
|
+
ok,
|
|
85
|
+
element_wise=True,
|
|
86
|
+
name="allowed_elements",
|
|
87
|
+
description=f"an element outside {sorted(allowed)}",
|
|
88
|
+
statistics={"elements": sorted(allowed)},
|
|
89
|
+
**options,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def single_fragment(**options: Any) -> pa.Check:
|
|
94
|
+
"""The record is one connected molecule: no salts, solvents or mixtures.
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
**options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
|
|
98
|
+
instead of failing, or ``n_failure_cases=10``.
|
|
99
|
+
|
|
100
|
+
Returns:
|
|
101
|
+
The check.
|
|
102
|
+
"""
|
|
103
|
+
|
|
104
|
+
def ok(smiles: str) -> bool:
|
|
105
|
+
"""Whether one SMILES is a single piece (unreadable SMILES pass).
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
smiles: The SMILES.
|
|
109
|
+
|
|
110
|
+
Returns:
|
|
111
|
+
True if one piece or unreadable.
|
|
112
|
+
"""
|
|
113
|
+
mol = parse(smiles)
|
|
114
|
+
return mol is None or len(Chem.GetMolFrags(mol)) == 1
|
|
115
|
+
|
|
116
|
+
return pa.Check(
|
|
117
|
+
ok,
|
|
118
|
+
element_wise=True,
|
|
119
|
+
name="single_fragment",
|
|
120
|
+
description="more than one molecule in the record (a salt, solvent or mixture)",
|
|
121
|
+
**options,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def neutral(**options: Any) -> pa.Check:
|
|
126
|
+
"""The net formal charge is zero (charges that cancel, as in a nitro group, are fine).
|
|
127
|
+
|
|
128
|
+
Args:
|
|
129
|
+
**options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
|
|
130
|
+
instead of failing, or ``n_failure_cases=10``.
|
|
131
|
+
|
|
132
|
+
Returns:
|
|
133
|
+
The check.
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
def ok(smiles: str) -> bool:
|
|
137
|
+
"""Whether one SMILES has no net charge (unreadable SMILES pass).
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
smiles: The SMILES.
|
|
141
|
+
|
|
142
|
+
Returns:
|
|
143
|
+
True if neutral or unreadable.
|
|
144
|
+
"""
|
|
145
|
+
mol = parse(smiles)
|
|
146
|
+
return mol is None or Chem.GetFormalCharge(mol) == 0
|
|
147
|
+
|
|
148
|
+
return pa.Check(
|
|
149
|
+
ok, element_wise=True, name="neutral", description="a net formal charge", **options
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def stereo_defined(**options: Any) -> pa.Check:
|
|
154
|
+
"""Every stereocentre and stereogenic double bond has its configuration given.
|
|
155
|
+
|
|
156
|
+
Often used as a warning rather than an error: ``stereo_defined(raise_warning=True)``.
|
|
157
|
+
|
|
158
|
+
Args:
|
|
159
|
+
**options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
|
|
160
|
+
instead of failing, or ``n_failure_cases=10``.
|
|
161
|
+
|
|
162
|
+
Returns:
|
|
163
|
+
The check.
|
|
164
|
+
"""
|
|
165
|
+
|
|
166
|
+
def ok(smiles: str) -> bool:
|
|
167
|
+
"""Whether one SMILES specifies all its stereochemistry (unreadable SMILES pass).
|
|
168
|
+
|
|
169
|
+
Args:
|
|
170
|
+
smiles: The SMILES.
|
|
171
|
+
|
|
172
|
+
Returns:
|
|
173
|
+
True if fully specified or unreadable.
|
|
174
|
+
"""
|
|
175
|
+
mol = parse(smiles)
|
|
176
|
+
if mol is None:
|
|
177
|
+
return True
|
|
178
|
+
return all(
|
|
179
|
+
info.specified != Chem.StereoSpecified.Unspecified
|
|
180
|
+
for info in Chem.FindPotentialStereo(mol)
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
return pa.Check(
|
|
184
|
+
ok,
|
|
185
|
+
element_wise=True,
|
|
186
|
+
name="stereo_defined",
|
|
187
|
+
description="a stereocentre or double bond with no configuration given",
|
|
188
|
+
**options,
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def heavy_atoms(min_value: int = 1, max_value: int | None = None, **options: Any) -> pa.Check:
|
|
193
|
+
"""The number of heavy (non-hydrogen) atoms of the largest fragment is within limits.
|
|
194
|
+
|
|
195
|
+
Args:
|
|
196
|
+
min_value: The fewest heavy atoms allowed.
|
|
197
|
+
max_value: The most heavy atoms allowed, or None for no upper limit.
|
|
198
|
+
**options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
|
|
199
|
+
instead of failing, or ``n_failure_cases=10``.
|
|
200
|
+
|
|
201
|
+
Returns:
|
|
202
|
+
The check.
|
|
203
|
+
"""
|
|
204
|
+
top = float("inf") if max_value is None else max_value
|
|
205
|
+
|
|
206
|
+
def ok(smiles: str) -> bool:
|
|
207
|
+
"""Whether one SMILES' largest fragment is within the limits (unreadable SMILES pass).
|
|
208
|
+
|
|
209
|
+
Args:
|
|
210
|
+
smiles: The SMILES.
|
|
211
|
+
|
|
212
|
+
Returns:
|
|
213
|
+
True if within the limits or unreadable.
|
|
214
|
+
"""
|
|
215
|
+
mol = parse(smiles)
|
|
216
|
+
if mol is None:
|
|
217
|
+
return True
|
|
218
|
+
n = max(f.GetNumHeavyAtoms() for f in Chem.GetMolFrags(mol, asMols=True))
|
|
219
|
+
return min_value <= n <= top
|
|
220
|
+
|
|
221
|
+
return pa.Check(
|
|
222
|
+
ok,
|
|
223
|
+
element_wise=True,
|
|
224
|
+
name="heavy_atoms",
|
|
225
|
+
description=f"heavy atoms outside {min_value} to {max_value}",
|
|
226
|
+
statistics={"min_value": min_value, "max_value": max_value},
|
|
227
|
+
**options,
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def mol_weight(max_value: float, min_value: float = 0.0, **options: Any) -> pa.Check:
|
|
232
|
+
"""The molecular weight (g/mol) of the largest fragment is within limits.
|
|
233
|
+
|
|
234
|
+
Args:
|
|
235
|
+
max_value: The largest weight allowed.
|
|
236
|
+
min_value: The smallest weight allowed.
|
|
237
|
+
**options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
|
|
238
|
+
instead of failing, or ``n_failure_cases=10``.
|
|
239
|
+
|
|
240
|
+
Returns:
|
|
241
|
+
The check.
|
|
242
|
+
"""
|
|
243
|
+
|
|
244
|
+
def ok(smiles: str) -> bool:
|
|
245
|
+
"""Whether one SMILES' largest fragment weighs within the limits (unreadable pass).
|
|
246
|
+
|
|
247
|
+
Args:
|
|
248
|
+
smiles: The SMILES.
|
|
249
|
+
|
|
250
|
+
Returns:
|
|
251
|
+
True if within the limits or unreadable.
|
|
252
|
+
"""
|
|
253
|
+
mol = parse(smiles)
|
|
254
|
+
if mol is None:
|
|
255
|
+
return True
|
|
256
|
+
piece = max(Chem.GetMolFrags(mol, asMols=True), key=lambda f: f.GetNumHeavyAtoms())
|
|
257
|
+
return min_value <= Descriptors.MolWt(piece) <= max_value
|
|
258
|
+
|
|
259
|
+
return pa.Check(
|
|
260
|
+
ok,
|
|
261
|
+
element_wise=True,
|
|
262
|
+
name="mol_weight",
|
|
263
|
+
description=f"molecular weight outside {min_value} to {max_value} g/mol",
|
|
264
|
+
statistics={"min_value": min_value, "max_value": max_value},
|
|
265
|
+
**options,
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def no_substructures(patterns: Mapping[str, str], **options: Any) -> pa.Check:
|
|
270
|
+
"""None of the given substructures is present (e.g. reactive groups you want kept out).
|
|
271
|
+
|
|
272
|
+
Args:
|
|
273
|
+
patterns: Each pattern's name and its SMARTS, e.g. ``{"acyl halide": "C(=O)[Cl,Br,I]"}``.
|
|
274
|
+
**options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
|
|
275
|
+
instead of failing, or ``n_failure_cases=10``.
|
|
276
|
+
|
|
277
|
+
Returns:
|
|
278
|
+
The check.
|
|
279
|
+
|
|
280
|
+
Raises:
|
|
281
|
+
ValueError: If a SMARTS cannot be read.
|
|
282
|
+
"""
|
|
283
|
+
compiled = {}
|
|
284
|
+
for label, smarts in patterns.items():
|
|
285
|
+
query = Chem.MolFromSmarts(smarts)
|
|
286
|
+
if query is None:
|
|
287
|
+
raise ValueError(f"cannot read the SMARTS for {label!r}: {smarts}")
|
|
288
|
+
compiled[label] = query
|
|
289
|
+
|
|
290
|
+
def ok(smiles: str) -> bool:
|
|
291
|
+
"""Whether one SMILES contains none of the patterns (unreadable SMILES pass).
|
|
292
|
+
|
|
293
|
+
Args:
|
|
294
|
+
smiles: The SMILES.
|
|
295
|
+
|
|
296
|
+
Returns:
|
|
297
|
+
True if no pattern matches or unreadable.
|
|
298
|
+
"""
|
|
299
|
+
mol = parse(smiles)
|
|
300
|
+
return mol is None or not any(mol.HasSubstructMatch(q) for q in compiled.values())
|
|
301
|
+
|
|
302
|
+
return pa.Check(
|
|
303
|
+
ok,
|
|
304
|
+
element_wise=True,
|
|
305
|
+
name="no_substructures",
|
|
306
|
+
description=f"contains one of: {', '.join(compiled)}",
|
|
307
|
+
statistics={"patterns": dict(patterns)},
|
|
308
|
+
**options,
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def unique_molecules(**options: Any) -> pa.Check:
|
|
313
|
+
"""No molecule appears twice, even when written as different SMILES.
|
|
314
|
+
|
|
315
|
+
Molecules are compared by standard InChIKey, so ``OCC`` and ``CCO`` are the same. The first
|
|
316
|
+
record of each molecule passes; every later one fails. Unreadable SMILES pass.
|
|
317
|
+
|
|
318
|
+
Args:
|
|
319
|
+
**options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
|
|
320
|
+
instead of failing, or ``n_failure_cases=10``.
|
|
321
|
+
|
|
322
|
+
Returns:
|
|
323
|
+
The check.
|
|
324
|
+
"""
|
|
325
|
+
|
|
326
|
+
def ok(smiles: pd.Series) -> pd.Series:
|
|
327
|
+
"""Which records are the first (or only) record of their molecule.
|
|
328
|
+
|
|
329
|
+
Args:
|
|
330
|
+
smiles: The column of SMILES.
|
|
331
|
+
|
|
332
|
+
Returns:
|
|
333
|
+
True for first occurrences and unreadable SMILES.
|
|
334
|
+
"""
|
|
335
|
+
keys = smiles.map(inchikey)
|
|
336
|
+
return ~(keys.notna() & keys.duplicated(keep="first"))
|
|
337
|
+
|
|
338
|
+
return pa.Check(
|
|
339
|
+
ok,
|
|
340
|
+
name="unique_molecules",
|
|
341
|
+
description="the same molecule as an earlier record",
|
|
342
|
+
**options,
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def inchikey_format(**options: Any) -> pa.Check:
|
|
347
|
+
"""The value is shaped like a standard InChIKey (14 letters, 10 letters, 1 letter).
|
|
348
|
+
|
|
349
|
+
Args:
|
|
350
|
+
**options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
|
|
351
|
+
instead of failing, or ``n_failure_cases=10``.
|
|
352
|
+
|
|
353
|
+
Returns:
|
|
354
|
+
The check.
|
|
355
|
+
"""
|
|
356
|
+
return pa.Check.str_matches(INCHIKEY.pattern, name="inchikey_format", **options)
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def inchikey_matches(smiles_column: str, inchikey_column: str, **options: Any) -> pa.Check:
|
|
360
|
+
"""A table-wide check: each record's InChIKey is the one its SMILES gives.
|
|
361
|
+
|
|
362
|
+
Records whose SMILES cannot be read, or whose InChIKey is malformed, pass: the column
|
|
363
|
+
checks report those.
|
|
364
|
+
|
|
365
|
+
Args:
|
|
366
|
+
smiles_column: The name of the SMILES column.
|
|
367
|
+
inchikey_column: The name of the InChIKey column.
|
|
368
|
+
**options: Passed on to ``pandera.Check``.
|
|
369
|
+
|
|
370
|
+
Returns:
|
|
371
|
+
The check, to put in ``DataFrameSchema(checks=[...])``.
|
|
372
|
+
"""
|
|
373
|
+
|
|
374
|
+
def ok(df: pd.DataFrame) -> pd.Series:
|
|
375
|
+
"""Which records' InChIKeys agree with their SMILES.
|
|
376
|
+
|
|
377
|
+
Args:
|
|
378
|
+
df: The table.
|
|
379
|
+
|
|
380
|
+
Returns:
|
|
381
|
+
True where they agree, or where either value cannot be compared.
|
|
382
|
+
"""
|
|
383
|
+
expected = df[smiles_column].map(inchikey)
|
|
384
|
+
given = df[inchikey_column]
|
|
385
|
+
comparable = expected.notna() & given.astype(str).str.match(INCHIKEY.pattern)
|
|
386
|
+
return ~comparable | (expected == given)
|
|
387
|
+
|
|
388
|
+
return pa.Check(
|
|
389
|
+
ok,
|
|
390
|
+
name="inchikey_matches",
|
|
391
|
+
description=f"{inchikey_column} is not the InChIKey of {smiles_column}",
|
|
392
|
+
**options,
|
|
393
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Quarantine: keep the rows that pass, set the others aside with their reasons.
|
|
2
|
+
|
|
3
|
+
Pandera either accepts a table or raises an error listing what failed. In a pipeline you usually
|
|
4
|
+
want neither: you want to carry on with the good rows and keep the bad ones, each with the
|
|
5
|
+
reason it was set aside, so someone can fix them later. :func:`split` does that, and its report
|
|
6
|
+
gives counts that must add up.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import warnings
|
|
12
|
+
from collections import Counter
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
|
|
15
|
+
import pandas as pd
|
|
16
|
+
import pandera.pandas as pa
|
|
17
|
+
from pandera.errors import SchemaErrors, SchemaWarning
|
|
18
|
+
|
|
19
|
+
REASONS = "reasons" # the column added to the quarantined rows
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class Report:
|
|
24
|
+
"""What happened to a table: how many rows came in, passed and were set aside, and why.
|
|
25
|
+
|
|
26
|
+
Attributes:
|
|
27
|
+
rows_in: Rows in the table given to :func:`split`.
|
|
28
|
+
rows_valid: Rows that passed every check.
|
|
29
|
+
rows_quarantined: Rows set aside.
|
|
30
|
+
by_reason: For each reason ("column: check"), how many rows failed it. A row that fails
|
|
31
|
+
two checks is counted under both, so these can add up to more than
|
|
32
|
+
``rows_quarantined``.
|
|
33
|
+
warnings: The messages of checks that only warn (``raise_warning=True``).
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
rows_in: int
|
|
37
|
+
rows_valid: int
|
|
38
|
+
rows_quarantined: int
|
|
39
|
+
by_reason: dict[str, int] = field(default_factory=dict)
|
|
40
|
+
warnings: list[str] = field(default_factory=list)
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def reconciles(self) -> bool:
|
|
44
|
+
"""Whether every row is accounted for: rows in = valid + quarantined.
|
|
45
|
+
|
|
46
|
+
Returns:
|
|
47
|
+
True if the counts add up.
|
|
48
|
+
"""
|
|
49
|
+
return self.rows_in == self.rows_valid + self.rows_quarantined
|
|
50
|
+
|
|
51
|
+
def __str__(self) -> str:
|
|
52
|
+
"""A short summary to print or log.
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
The summary, one fact per line.
|
|
56
|
+
"""
|
|
57
|
+
lines = [
|
|
58
|
+
f"{self.rows_in} rows in: {self.rows_valid} valid, {self.rows_quarantined} quarantined",
|
|
59
|
+
]
|
|
60
|
+
lines += [f" {n:>6} {reason}" for reason, n in self.by_reason.items()]
|
|
61
|
+
lines += [f" warning: {w}" for w in self.warnings]
|
|
62
|
+
return "\n".join(lines)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class Split:
|
|
67
|
+
"""The result of :func:`split`.
|
|
68
|
+
|
|
69
|
+
Attributes:
|
|
70
|
+
valid: The rows that passed, as validated (and coerced) by the schema.
|
|
71
|
+
quarantined: The rows that failed, unchanged, with a ``reasons`` column added.
|
|
72
|
+
report: The counts.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
valid: pd.DataFrame
|
|
76
|
+
quarantined: pd.DataFrame
|
|
77
|
+
report: Report
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _reason(column: object, check: object) -> str:
|
|
81
|
+
"""One failure's reason, as "column: check" (or just the check for a table-wide one).
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
column: The column that failed, or None/NaN for a table-wide check.
|
|
85
|
+
check: The check's name.
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
The reason.
|
|
89
|
+
"""
|
|
90
|
+
return str(check) if pd.isna(column) else f"{column}: {check}"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _warnings(caught: list[warnings.WarningMessage]) -> list[str]:
|
|
94
|
+
"""The distinct messages of the checks that only warn, in the order they came.
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
caught: Warnings recorded while validating.
|
|
98
|
+
|
|
99
|
+
Returns:
|
|
100
|
+
Each Pandera warning's message, once.
|
|
101
|
+
"""
|
|
102
|
+
messages = [str(w.message) for w in caught if issubclass(w.category, SchemaWarning)]
|
|
103
|
+
return list(dict.fromkeys(messages))
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _all_valid(df: pd.DataFrame, valid: pd.DataFrame, caught: list) -> Split:
|
|
107
|
+
"""The split of a table in which every row passed.
|
|
108
|
+
|
|
109
|
+
Args:
|
|
110
|
+
df: The table given to :func:`split`.
|
|
111
|
+
valid: The table as validated by the schema.
|
|
112
|
+
caught: Warnings recorded while validating.
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
Every row valid, none quarantined.
|
|
116
|
+
"""
|
|
117
|
+
empty = df.iloc[0:0].assign(**{REASONS: pd.Series(dtype=str)})
|
|
118
|
+
return Split(valid, empty, Report(len(df), len(df), 0, {}, _warnings(caught)))
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def split(df: pd.DataFrame, schema: pa.DataFrameSchema) -> Split:
|
|
122
|
+
"""Validate a table and divide it into the rows that pass and the rows that fail.
|
|
123
|
+
|
|
124
|
+
Args:
|
|
125
|
+
df: The table. Its index must be unique, so each failure can be traced to one row.
|
|
126
|
+
schema: The Pandera schema to check it against.
|
|
127
|
+
|
|
128
|
+
Returns:
|
|
129
|
+
The valid rows, the quarantined rows with their reasons, and a report.
|
|
130
|
+
|
|
131
|
+
Raises:
|
|
132
|
+
ValueError: If the index of ``df`` has repeated labels.
|
|
133
|
+
pandera.errors.SchemaErrors: If something fails that no row can be blamed for, such as
|
|
134
|
+
a missing column; the whole table is then unusable, and the error says why.
|
|
135
|
+
"""
|
|
136
|
+
if not df.index.is_unique:
|
|
137
|
+
raise ValueError("the table's index has repeated labels; reset it first")
|
|
138
|
+
with warnings.catch_warnings(record=True) as caught:
|
|
139
|
+
warnings.simplefilter("always", SchemaWarning)
|
|
140
|
+
try:
|
|
141
|
+
return _all_valid(df, schema.validate(df, lazy=True), caught)
|
|
142
|
+
except SchemaErrors as error:
|
|
143
|
+
failures = error.failure_cases
|
|
144
|
+
if failures["index"].isna().any():
|
|
145
|
+
raise # a failure of the whole table, not of particular rows
|
|
146
|
+
reasons = (
|
|
147
|
+
failures.assign(
|
|
148
|
+
reason=[
|
|
149
|
+
_reason(c, k)
|
|
150
|
+
for c, k in zip(failures["column"], failures["check"], strict=True)
|
|
151
|
+
]
|
|
152
|
+
)
|
|
153
|
+
.drop_duplicates(["index", "reason"])
|
|
154
|
+
.groupby("index", sort=False)["reason"]
|
|
155
|
+
.agg("; ".join)
|
|
156
|
+
)
|
|
157
|
+
bad = df.index.isin(reasons.index)
|
|
158
|
+
quarantined = df[bad].assign(**{REASONS: df.index[bad].map(reasons)})
|
|
159
|
+
valid = schema.validate(df[~bad], lazy=True) if (~bad).any() else df[~bad]
|
|
160
|
+
warned = _warnings(caught)
|
|
161
|
+
counts = Counter(r for joined in reasons for r in joined.split("; "))
|
|
162
|
+
report = Report(len(df), int((~bad).sum()), int(bad.sum()), dict(counts.most_common()), warned)
|
|
163
|
+
return Split(valid, quarantined, report)
|