pandera-chem 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Monstrous Data (David Obeh Alobo)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,215 @@
1
+ Metadata-Version: 2.4
2
+ Name: pandera-chem
3
+ Version: 0.1.0
4
+ Summary: Chemistry checks for Pandera: validate tables of molecules with RDKit, and quarantine the bad rows with their reasons
5
+ Keywords: cheminformatics,pandera,rdkit,data-quality,data-validation,smiles,chemistry
6
+ Author: David.Alobo
7
+ Author-email: David.Alobo <davidalobo01@gmail.com>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Topic :: Scientific/Engineering :: Chemistry
18
+ Classifier: Typing :: Typed
19
+ Requires-Dist: pandera[pandas]>=0.24
20
+ Requires-Dist: rdkit>=2024.9
21
+ Requires-Python: >=3.11
22
+ Project-URL: Homepage, https://github.com/David-Alobo/pandera-chem
23
+ Project-URL: Issues, https://github.com/David-Alobo/pandera-chem/issues
24
+ Project-URL: Course, https://monstrous.com.ng/learn/
25
+ Description-Content-Type: text/markdown
26
+
27
+ <p align="center">
28
+ <img src=".github/banner.png" alt="pandera-chem: chemistry checks for Pandera. Validate tables of molecules with RDKit, and quarantine the bad rows with their reasons." width="100%">
29
+ </p>
30
+
31
+ <p align="center">
32
+ <a href="https://github.com/David-Alobo/pandera-chem/actions/workflows/ci.yml"><img src="https://github.com/David-Alobo/pandera-chem/actions/workflows/ci.yml/badge.svg" alt="Tests"></a>
33
+ <img src="https://img.shields.io/badge/python-3.11_%7C_3.12_%7C_3.13-3776ab" alt="Python 3.11, 3.12, 3.13">
34
+ <a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json" alt="Ruff"></a>
35
+ <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="MIT licence"></a>
36
+ <a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-7c5cff" alt="Cite"></a>
37
+ </p>
38
+
39
+ **Chemistry checks for [Pandera](https://pandera.readthedocs.io/).** Validate tables of molecules
40
+ with [RDKit](https://www.rdkit.org/) inside the data pipeline you already have, and set the bad rows
41
+ aside with the reason each one failed, instead of crashing or losing them.
42
+
43
+ ## Why
44
+
45
+ Chemical tables arrive with problems a type check cannot see: SMILES that RDKit cannot read, salts
46
+ and mixtures, unexpected elements, charged forms, missing stereochemistry, and the same molecule
47
+ twice under different SMILES. The usual fix is a loop that drops what fails to parse, and the
48
+ records vanish with no reason recorded.
49
+
50
+ pandera-chem turns those problems into Pandera checks, so they sit in the same schema as your other
51
+ rules (types, ranges, units), and adds `split()`, which keeps the good rows, sets the bad ones aside
52
+ with their reasons, and gives you counts that must add up.
53
+
54
+ ## Install
55
+
56
+ ```bash
57
+ pip install pandera-chem
58
+ ```
59
+
60
+ or `uv add pandera-chem`. It needs Python 3.11 or newer, and installs Pandera and RDKit.
61
+
62
+ ## Example
63
+
64
+ A table of measured compounds, with a few of the problems real data has:
65
+
66
+ ```python
67
+ import pandas as pd
68
+ import pandera.pandas as pa
69
+
70
+ import pandera_chem as pc
71
+
72
+ table = pd.DataFrame(
73
+ {
74
+ "compound_id": ["C1", "C2", "C3", "C4", "C5", "C6", "C7"],
75
+ "smiles": [
76
+ "CC(=O)Oc1ccccc1C(=O)O", # aspirin
77
+ "OC(=O)c1ccccc1OC(C)=O", # aspirin again, written differently
78
+ "c1ccc2ccccc2c1", # naphthalene
79
+ "c1ccccc", # a ring that never closes
80
+ "C[Se]c1ccccc1", # selenium
81
+ "CC(=O)[O-].[Na+]", # a salt
82
+ "CC(C)Cc1ccc(cc1)C(C)C(=O)O", # ibuprofen, stereocentre not given
83
+ ],
84
+ "pic50": [5.2, 5.4, 4.1, 6.0, 5.0, 3.2, 15.0],
85
+ }
86
+ )
87
+
88
+ schema = pa.DataFrameSchema(
89
+ {
90
+ "compound_id": pa.Column(str, unique=True),
91
+ "smiles": pa.Column(
92
+ str,
93
+ [
94
+ pc.valid_smiles(),
95
+ pc.allowed_elements(),
96
+ pc.single_fragment(),
97
+ pc.unique_molecules(),
98
+ pc.stereo_defined(raise_warning=True), # a warning, not a failure
99
+ ],
100
+ ),
101
+ "pic50": pa.Column(float, pa.Check.in_range(0, 14)),
102
+ }
103
+ )
104
+
105
+ result = pc.split(table, schema)
106
+ print(result.report)
107
+ print()
108
+ print(result.quarantined[["compound_id", "reasons"]].to_string(index=False))
109
+ print()
110
+ print(pc.problems("c1ccccc"))
111
+ ```
112
+
113
+ It prints:
114
+
115
+ ```text
116
+ 7 rows in: 2 valid, 5 quarantined
117
+ 2 smiles: allowed_elements
118
+ 1 smiles: valid_smiles
119
+ 1 smiles: single_fragment
120
+ 1 smiles: unique_molecules
121
+ 1 pic50: in_range(0, 14)
122
+ warning: Column 'smiles' failed element-wise validator number 4: <Check stereo_defined> failure cases: CC(C)Cc1ccc(cc1)C(C)C(=O)O
123
+
124
+ compound_id reasons
125
+ C2 smiles: unique_molecules
126
+ C4 smiles: valid_smiles
127
+ C5 smiles: allowed_elements
128
+ C6 smiles: allowed_elements; smiles: single_fragment
129
+ C7 pic50: in_range(0, 14)
130
+
131
+ ['the SMILES could not be read']
132
+ ```
133
+
134
+ - **`result.valid`**: the rows that passed (aspirin and naphthalene), ready for the next step.
135
+ - **`result.quarantined`**: the rest, unchanged, with a `reasons` column. Save it, fix the records
136
+ and run them again.
137
+ - **`result.report`**: the counts. `result.report.reconciles` is `True` when every row is accounted
138
+ for (rows in = valid + quarantined); log it with every run.
139
+ - Ibuprofen passed with a **warning**: its stereocentre is not given, which you may accept for a
140
+ racemate. Any check becomes a warning with `raise_warning=True`.
141
+
142
+ The checks work in plain Pandera too: `schema.validate(table)` raises an error naming the check that
143
+ failed, as Pandera always does.
144
+
145
+ ## The checks
146
+
147
+ On a column of SMILES:
148
+
149
+ | Check | Fails when |
150
+ |---|---|
151
+ | `valid_smiles()` | RDKit cannot read or sanitise the SMILES (syntax, valence, aromaticity) |
152
+ | `allowed_elements(elements=ORGANIC)` | an atom is not in the allowed set (default: H, B, C, N, O, F, P, S, Cl, Br, I) |
153
+ | `single_fragment()` | the record holds more than one molecule: a salt, a solvent or a mixture |
154
+ | `neutral()` | the net formal charge is not zero (charges that cancel, as in a nitro group, are fine) |
155
+ | `stereo_defined()` | a stereocentre or stereogenic double bond has no configuration given |
156
+ | `heavy_atoms(min_value, max_value)` | the largest fragment has too few or too many heavy atoms |
157
+ | `mol_weight(max_value, min_value=0)` | the largest fragment is too light or too heavy (g/mol) |
158
+ | `no_substructures({"name": "SMARTS", ...})` | the molecule contains one of the given groups |
159
+ | `unique_molecules()` | the same molecule appeared in an earlier row, even as a different SMILES |
160
+
161
+ On InChIKeys:
162
+
163
+ | Check | Fails when |
164
+ |---|---|
165
+ | `inchikey_format()` | the value is not shaped like a standard InChIKey |
166
+ | `inchikey_matches("smiles", "inchikey")` | (table-wide) a row's InChIKey is not the one its SMILES gives |
167
+
168
+ Helpers: `pc.problems(smiles)` says why a SMILES cannot be read, in RDKit's words;
169
+ `pc.inchikey(smiles)` gives its standard InChIKey; `pc.clear_cache()` frees the parsed molecules.
170
+
171
+ ## How it works
172
+
173
+ - **One reason per problem.** Only `valid_smiles()` fails a SMILES that cannot be read; every other
174
+ check lets it pass. A broken record is reported once, as unreadable, not under every check.
175
+ - **Each SMILES is read once.** All the checks share one cache of parsed molecules, so five checks on
176
+ a column cost about one parse per distinct SMILES. Call `pc.clear_cache()` after a very large table.
177
+ - **Pandera's own options work.** Every check accepts them: `raise_warning=True`,
178
+ `n_failure_cases=10` and so on.
179
+ - **Nothing is changed.** pandera-chem checks molecules; it never alters them. To repair records
180
+ (strip salts, neutralise charges, pick one tautomer), use RDKit's
181
+ [MolStandardize](https://www.rdkit.org/docs/source/rdkit.Chem.MolStandardize.rdMolStandardize.html)
182
+ or the [ChEMBL structure pipeline](https://github.com/chembl/ChEMBL_Structure_Pipeline) first, then
183
+ check the result. For a web page that validates an uploaded file, see
184
+ [ChemAudit](https://nfdi4chem.de/introducing-chemaudit/).
185
+
186
+ ## Roadmap
187
+
188
+ Ideas for later versions; [suggest one](https://github.com/David-Alobo/pandera-chem/issues/new?template=feature.yml).
189
+
190
+ - Polars data frames, which Pandera also supports.
191
+ - The ChEMBL structure checker as a check, with a penalty threshold.
192
+ - Pydantic types for single records (`SMILES`, `InChIKey`).
193
+ - Checks registered by name, so schemas can be written in YAML.
194
+
195
+ ## Learn the ideas behind it
196
+
197
+ pandera-chem grew out of the free course
198
+ [Data Engineering for Cheminformatics](https://monstrous.com.ng/learn/), which builds these checks by
199
+ hand: validating molecules in Module 6, the quarantine in Module 10, and validating whole tables with
200
+ Pydantic and Pandera in Module 12.
201
+
202
+ ## Contributing
203
+
204
+ Bug reports, suggestions and pull requests are welcome: see [CONTRIBUTING.md](CONTRIBUTING.md).
205
+ Everyone taking part follows the [code of conduct](CODE_OF_CONDUCT.md). The example above is run by
206
+ the tests, which check that this README shows what it really prints.
207
+
208
+ ## Citing
209
+
210
+ If pandera-chem helps your research, please cite it: GitHub's **Cite this repository** button (from
211
+ [CITATION.cff](CITATION.cff)) gives the reference in APA and BibTeX.
212
+
213
+ ## Licence
214
+
215
+ MIT ([LICENSE](LICENSE)). Made by [Monstrous Data](https://monstrous.com.ng).
@@ -0,0 +1,189 @@
1
+ <p align="center">
2
+ <img src=".github/banner.png" alt="pandera-chem: chemistry checks for Pandera. Validate tables of molecules with RDKit, and quarantine the bad rows with their reasons." width="100%">
3
+ </p>
4
+
5
+ <p align="center">
6
+ <a href="https://github.com/David-Alobo/pandera-chem/actions/workflows/ci.yml"><img src="https://github.com/David-Alobo/pandera-chem/actions/workflows/ci.yml/badge.svg" alt="Tests"></a>
7
+ <img src="https://img.shields.io/badge/python-3.11_%7C_3.12_%7C_3.13-3776ab" alt="Python 3.11, 3.12, 3.13">
8
+ <a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json" alt="Ruff"></a>
9
+ <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="MIT licence"></a>
10
+ <a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-7c5cff" alt="Cite"></a>
11
+ </p>
12
+
13
+ **Chemistry checks for [Pandera](https://pandera.readthedocs.io/).** Validate tables of molecules
14
+ with [RDKit](https://www.rdkit.org/) inside the data pipeline you already have, and set the bad rows
15
+ aside with the reason each one failed, instead of crashing or losing them.
16
+
17
+ ## Why
18
+
19
+ Chemical tables arrive with problems a type check cannot see: SMILES that RDKit cannot read, salts
20
+ and mixtures, unexpected elements, charged forms, missing stereochemistry, and the same molecule
21
+ twice under different SMILES. The usual fix is a loop that drops what fails to parse, and the
22
+ records vanish with no reason recorded.
23
+
24
+ pandera-chem turns those problems into Pandera checks, so they sit in the same schema as your other
25
+ rules (types, ranges, units), and adds `split()`, which keeps the good rows, sets the bad ones aside
26
+ with their reasons, and gives you counts that must add up.
27
+
28
+ ## Install
29
+
30
+ ```bash
31
+ pip install pandera-chem
32
+ ```
33
+
34
+ or `uv add pandera-chem`. It needs Python 3.11 or newer, and installs Pandera and RDKit.
35
+
36
+ ## Example
37
+
38
+ A table of measured compounds, with a few of the problems real data has:
39
+
40
+ ```python
41
+ import pandas as pd
42
+ import pandera.pandas as pa
43
+
44
+ import pandera_chem as pc
45
+
46
+ table = pd.DataFrame(
47
+ {
48
+ "compound_id": ["C1", "C2", "C3", "C4", "C5", "C6", "C7"],
49
+ "smiles": [
50
+ "CC(=O)Oc1ccccc1C(=O)O", # aspirin
51
+ "OC(=O)c1ccccc1OC(C)=O", # aspirin again, written differently
52
+ "c1ccc2ccccc2c1", # naphthalene
53
+ "c1ccccc", # a ring that never closes
54
+ "C[Se]c1ccccc1", # selenium
55
+ "CC(=O)[O-].[Na+]", # a salt
56
+ "CC(C)Cc1ccc(cc1)C(C)C(=O)O", # ibuprofen, stereocentre not given
57
+ ],
58
+ "pic50": [5.2, 5.4, 4.1, 6.0, 5.0, 3.2, 15.0],
59
+ }
60
+ )
61
+
62
+ schema = pa.DataFrameSchema(
63
+ {
64
+ "compound_id": pa.Column(str, unique=True),
65
+ "smiles": pa.Column(
66
+ str,
67
+ [
68
+ pc.valid_smiles(),
69
+ pc.allowed_elements(),
70
+ pc.single_fragment(),
71
+ pc.unique_molecules(),
72
+ pc.stereo_defined(raise_warning=True), # a warning, not a failure
73
+ ],
74
+ ),
75
+ "pic50": pa.Column(float, pa.Check.in_range(0, 14)),
76
+ }
77
+ )
78
+
79
+ result = pc.split(table, schema)
80
+ print(result.report)
81
+ print()
82
+ print(result.quarantined[["compound_id", "reasons"]].to_string(index=False))
83
+ print()
84
+ print(pc.problems("c1ccccc"))
85
+ ```
86
+
87
+ It prints:
88
+
89
+ ```text
90
+ 7 rows in: 2 valid, 5 quarantined
91
+ 2 smiles: allowed_elements
92
+ 1 smiles: valid_smiles
93
+ 1 smiles: single_fragment
94
+ 1 smiles: unique_molecules
95
+ 1 pic50: in_range(0, 14)
96
+ warning: Column 'smiles' failed element-wise validator number 4: <Check stereo_defined> failure cases: CC(C)Cc1ccc(cc1)C(C)C(=O)O
97
+
98
+ compound_id reasons
99
+ C2 smiles: unique_molecules
100
+ C4 smiles: valid_smiles
101
+ C5 smiles: allowed_elements
102
+ C6 smiles: allowed_elements; smiles: single_fragment
103
+ C7 pic50: in_range(0, 14)
104
+
105
+ ['the SMILES could not be read']
106
+ ```
107
+
108
+ - **`result.valid`**: the rows that passed (aspirin and naphthalene), ready for the next step.
109
+ - **`result.quarantined`**: the rest, unchanged, with a `reasons` column. Save it, fix the records
110
+ and run them again.
111
+ - **`result.report`**: the counts. `result.report.reconciles` is `True` when every row is accounted
112
+ for (rows in = valid + quarantined); log it with every run.
113
+ - Ibuprofen passed with a **warning**: its stereocentre is not given, which you may accept for a
114
+ racemate. Any check becomes a warning with `raise_warning=True`.
115
+
116
+ The checks work in plain Pandera too: `schema.validate(table)` raises an error naming the check that
117
+ failed, as Pandera always does.
118
+
119
+ ## The checks
120
+
121
+ On a column of SMILES:
122
+
123
+ | Check | Fails when |
124
+ |---|---|
125
+ | `valid_smiles()` | RDKit cannot read or sanitise the SMILES (syntax, valence, aromaticity) |
126
+ | `allowed_elements(elements=ORGANIC)` | an atom is not in the allowed set (default: H, B, C, N, O, F, P, S, Cl, Br, I) |
127
+ | `single_fragment()` | the record holds more than one molecule: a salt, a solvent or a mixture |
128
+ | `neutral()` | the net formal charge is not zero (charges that cancel, as in a nitro group, are fine) |
129
+ | `stereo_defined()` | a stereocentre or stereogenic double bond has no configuration given |
130
+ | `heavy_atoms(min_value, max_value)` | the largest fragment has too few or too many heavy atoms |
131
+ | `mol_weight(max_value, min_value=0)` | the largest fragment is too light or too heavy (g/mol) |
132
+ | `no_substructures({"name": "SMARTS", ...})` | the molecule contains one of the given groups |
133
+ | `unique_molecules()` | the same molecule appeared in an earlier row, even as a different SMILES |
134
+
135
+ On InChIKeys:
136
+
137
+ | Check | Fails when |
138
+ |---|---|
139
+ | `inchikey_format()` | the value is not shaped like a standard InChIKey |
140
+ | `inchikey_matches("smiles", "inchikey")` | (table-wide) a row's InChIKey is not the one its SMILES gives |
141
+
142
+ Helpers: `pc.problems(smiles)` says why a SMILES cannot be read, in RDKit's words;
143
+ `pc.inchikey(smiles)` gives its standard InChIKey; `pc.clear_cache()` frees the parsed molecules.
144
+
145
+ ## How it works
146
+
147
+ - **One reason per problem.** Only `valid_smiles()` fails a SMILES that cannot be read; every other
148
+ check lets it pass. A broken record is reported once, as unreadable, not under every check.
149
+ - **Each SMILES is read once.** All the checks share one cache of parsed molecules, so five checks on
150
+ a column cost about one parse per distinct SMILES. Call `pc.clear_cache()` after a very large table.
151
+ - **Pandera's own options work.** Every check accepts them: `raise_warning=True`,
152
+ `n_failure_cases=10` and so on.
153
+ - **Nothing is changed.** pandera-chem checks molecules; it never alters them. To repair records
154
+ (strip salts, neutralise charges, pick one tautomer), use RDKit's
155
+ [MolStandardize](https://www.rdkit.org/docs/source/rdkit.Chem.MolStandardize.rdMolStandardize.html)
156
+ or the [ChEMBL structure pipeline](https://github.com/chembl/ChEMBL_Structure_Pipeline) first, then
157
+ check the result. For a web page that validates an uploaded file, see
158
+ [ChemAudit](https://nfdi4chem.de/introducing-chemaudit/).
159
+
160
+ ## Roadmap
161
+
162
+ Ideas for later versions; [suggest one](https://github.com/David-Alobo/pandera-chem/issues/new?template=feature.yml).
163
+
164
+ - Polars data frames, which Pandera also supports.
165
+ - The ChEMBL structure checker as a check, with a penalty threshold.
166
+ - Pydantic types for single records (`SMILES`, `InChIKey`).
167
+ - Checks registered by name, so schemas can be written in YAML.
168
+
169
+ ## Learn the ideas behind it
170
+
171
+ pandera-chem grew out of the free course
172
+ [Data Engineering for Cheminformatics](https://monstrous.com.ng/learn/), which builds these checks by
173
+ hand: validating molecules in Module 6, the quarantine in Module 10, and validating whole tables with
174
+ Pydantic and Pandera in Module 12.
175
+
176
+ ## Contributing
177
+
178
+ Bug reports, suggestions and pull requests are welcome: see [CONTRIBUTING.md](CONTRIBUTING.md).
179
+ Everyone taking part follows the [code of conduct](CODE_OF_CONDUCT.md). The example above is run by
180
+ the tests, which check that this README shows what it really prints.
181
+
182
+ ## Citing
183
+
184
+ If pandera-chem helps your research, please cite it: GitHub's **Cite this repository** button (from
185
+ [CITATION.cff](CITATION.cff)) gives the reference in APA and BibTeX.
186
+
187
+ ## Licence
188
+
189
+ MIT ([LICENSE](LICENSE)). Made by [Monstrous Data](https://monstrous.com.ng).
@@ -0,0 +1,86 @@
1
+ [project]
2
+ name = "pandera-chem"
3
+ version = "0.1.0"
4
+ description = "Chemistry checks for Pandera: validate tables of molecules with RDKit, and quarantine the bad rows with their reasons"
5
+ readme = "README.md"
6
+ requires-python = ">=3.11"
7
+ license = "MIT"
8
+ license-files = ["LICENSE"]
9
+ keywords = [
10
+ "cheminformatics",
11
+ "pandera",
12
+ "rdkit",
13
+ "data-quality",
14
+ "data-validation",
15
+ "smiles",
16
+ "chemistry",
17
+ ]
18
+ classifiers = [
19
+ "Development Status :: 3 - Alpha",
20
+ "Intended Audience :: Science/Research",
21
+ "Intended Audience :: Developers",
22
+ "Programming Language :: Python :: 3",
23
+ "Programming Language :: Python :: 3.11",
24
+ "Programming Language :: Python :: 3.12",
25
+ "Programming Language :: Python :: 3.13",
26
+ "Topic :: Scientific/Engineering :: Chemistry",
27
+ "Typing :: Typed",
28
+ ]
29
+ dependencies = [
30
+ "pandera[pandas]>=0.24",
31
+ "rdkit>=2024.9",
32
+ ]
33
+
34
+ [[project.authors]]
35
+ name = "David.Alobo"
36
+ email = "davidalobo01@gmail.com"
37
+
38
+ [project.urls]
39
+ Homepage = "https://github.com/David-Alobo/pandera-chem"
40
+ Issues = "https://github.com/David-Alobo/pandera-chem/issues"
41
+ Course = "https://monstrous.com.ng/learn/"
42
+
43
+ [build-system]
44
+ requires = ["uv_build>=0.13.4,<0.14.0"]
45
+ build-backend = "uv_build"
46
+
47
+ [dependency-groups]
48
+ dev = [
49
+ "hypothesis>=6.170.17",
50
+ "pytest>=9.2.0",
51
+ "pytest-cov>=7.1.0",
52
+ "ruff>=0.17.6",
53
+ ]
54
+
55
+ [tool.ruff]
56
+ line-length = 100
57
+ target-version = "py311"
58
+
59
+ [tool.ruff.lint]
60
+ select = [
61
+ "E",
62
+ "F",
63
+ "W",
64
+ "I",
65
+ "UP",
66
+ "B",
67
+ "D",
68
+ "SIM",
69
+ ]
70
+
71
+ [tool.ruff.lint.pydocstyle]
72
+ convention = "google"
73
+
74
+ [tool.ruff.lint.per-file-ignores]
75
+ "tests/*" = ["D"]
76
+
77
+ [tool.pytest.ini_options]
78
+ testpaths = ["tests"]
79
+
80
+ [tool.coverage.run]
81
+ source = ["pandera_chem"]
82
+ branch = true
83
+
84
+ [tool.coverage.report]
85
+ show_missing = true
86
+ fail_under = 95
@@ -0,0 +1,68 @@
1
+ [project]
2
+ name = "pandera-chem"
3
+ version = "0.1.0"
4
+ description = "Chemistry checks for Pandera: validate tables of molecules with RDKit, and quarantine the bad rows with their reasons"
5
+ readme = "README.md"
6
+ authors = [
7
+ { name = "David.Alobo", email = "davidalobo01@gmail.com" }
8
+ ]
9
+ requires-python = ">=3.11"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ keywords = ["cheminformatics", "pandera", "rdkit", "data-quality", "data-validation", "smiles", "chemistry"]
13
+ classifiers = [
14
+ "Development Status :: 3 - Alpha",
15
+ "Intended Audience :: Science/Research",
16
+ "Intended Audience :: Developers",
17
+ "Programming Language :: Python :: 3",
18
+ "Programming Language :: Python :: 3.11",
19
+ "Programming Language :: Python :: 3.12",
20
+ "Programming Language :: Python :: 3.13",
21
+ "Topic :: Scientific/Engineering :: Chemistry",
22
+ "Typing :: Typed",
23
+ ]
24
+ dependencies = [
25
+ "pandera[pandas]>=0.24",
26
+ "rdkit>=2024.9",
27
+ ]
28
+
29
+ [build-system]
30
+ requires = ["uv_build>=0.13.4,<0.14.0"]
31
+ build-backend = "uv_build"
32
+
33
+ [dependency-groups]
34
+ dev = [
35
+ "hypothesis>=6.170.17",
36
+ "pytest>=9.2.0",
37
+ "pytest-cov>=7.1.0",
38
+ "ruff>=0.17.6",
39
+ ]
40
+
41
+ [project.urls]
42
+ Homepage = "https://github.com/David-Alobo/pandera-chem"
43
+ Issues = "https://github.com/David-Alobo/pandera-chem/issues"
44
+ Course = "https://monstrous.com.ng/learn/"
45
+
46
+ [tool.ruff]
47
+ line-length = 100
48
+ target-version = "py311"
49
+
50
+ [tool.ruff.lint]
51
+ select = ["E", "F", "W", "I", "UP", "B", "D", "SIM"]
52
+
53
+ [tool.ruff.lint.pydocstyle]
54
+ convention = "google"
55
+
56
+ [tool.ruff.lint.per-file-ignores]
57
+ "tests/*" = ["D"]
58
+
59
+ [tool.pytest.ini_options]
60
+ testpaths = ["tests"]
61
+
62
+ [tool.coverage.run]
63
+ source = ["pandera_chem"]
64
+ branch = true
65
+
66
+ [tool.coverage.report]
67
+ show_missing = true
68
+ fail_under = 95
@@ -0,0 +1,56 @@
1
+ """Chemistry checks for Pandera: validate tables of molecules, and quarantine the bad rows.
2
+
3
+ Quick start::
4
+
5
+ import pandera.pandas as pa
6
+ import pandera_chem as pc
7
+
8
+ schema = pa.DataFrameSchema({
9
+ "smiles": pa.Column(str, [pc.valid_smiles(), pc.allowed_elements(), pc.single_fragment()]),
10
+ "pic50": pa.Column(float, pa.Check.in_range(0, 14)),
11
+ })
12
+ result = pc.split(table, schema)
13
+ print(result.report)
14
+ """
15
+
16
+ from ._mol import clear_cache, inchikey, problems
17
+ from .checks import (
18
+ INCHIKEY,
19
+ ORGANIC,
20
+ allowed_elements,
21
+ heavy_atoms,
22
+ inchikey_format,
23
+ inchikey_matches,
24
+ mol_weight,
25
+ neutral,
26
+ no_substructures,
27
+ single_fragment,
28
+ stereo_defined,
29
+ unique_molecules,
30
+ valid_smiles,
31
+ )
32
+ from .quarantine import Report, Split, split
33
+
34
+ __version__ = "0.1.0"
35
+
36
+ __all__ = [
37
+ "INCHIKEY",
38
+ "ORGANIC",
39
+ "Report",
40
+ "Split",
41
+ "allowed_elements",
42
+ "clear_cache",
43
+ "heavy_atoms",
44
+ "inchikey",
45
+ "inchikey_format",
46
+ "inchikey_matches",
47
+ "mol_weight",
48
+ "neutral",
49
+ "no_substructures",
50
+ "problems",
51
+ "single_fragment",
52
+ "split",
53
+ "stereo_defined",
54
+ "unique_molecules",
55
+ "valid_smiles",
56
+ ]
@@ -0,0 +1,77 @@
1
+ """Reading molecules once: a cached parser shared by every check.
2
+
3
+ A schema often runs several chemistry checks on the same column, and a table often repeats the
4
+ same SMILES. Parsing is the slow part, so each distinct SMILES is parsed and sanitised once and
5
+ the result is reused. The cached molecules are only read, never changed, by the checks.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from functools import lru_cache
11
+
12
+ from rdkit import Chem, rdBase
13
+
14
+ CACHE_SIZE = 65_536 # distinct SMILES kept; about 100-200 MB at drug-like sizes
15
+
16
+
17
+ @lru_cache(maxsize=CACHE_SIZE)
18
+ def parse(smiles: str) -> Chem.Mol | None:
19
+ """Read and sanitise one SMILES, quietly.
20
+
21
+ Args:
22
+ smiles: The SMILES to read.
23
+
24
+ Returns:
25
+ The sanitised molecule, or None if it cannot be read or sanitised. RDKit's own error
26
+ messages are not printed; use :func:`pandera_chem.problems` to see why one failed.
27
+ """
28
+ if not isinstance(smiles, str) or not smiles.strip():
29
+ return None
30
+ with rdBase.BlockLogs():
31
+ return Chem.MolFromSmiles(smiles)
32
+
33
+
34
+ @lru_cache(maxsize=CACHE_SIZE)
35
+ def inchikey(smiles: str) -> str | None:
36
+ """The standard InChIKey of a SMILES, or None if it cannot be read or has no key.
37
+
38
+ Args:
39
+ smiles: The SMILES.
40
+
41
+ Returns:
42
+ The InChIKey, or None.
43
+ """
44
+ mol = parse(smiles)
45
+ if mol is None:
46
+ return None
47
+ with rdBase.BlockLogs():
48
+ key = Chem.MolToInchiKey(mol)
49
+ return key or None
50
+
51
+
52
+ def problems(smiles: str) -> list[str]:
53
+ """Why a SMILES cannot be read or sanitised, in RDKit's words.
54
+
55
+ Args:
56
+ smiles: The SMILES.
57
+
58
+ Returns:
59
+ One message per problem; empty if the SMILES reads and sanitises.
60
+ """
61
+ if not isinstance(smiles, str) or not smiles.strip():
62
+ return ["empty SMILES"]
63
+ with rdBase.BlockLogs():
64
+ mol = Chem.MolFromSmiles(smiles, sanitize=False)
65
+ if mol is None:
66
+ return ["the SMILES could not be read"]
67
+ return [p.Message() for p in Chem.DetectChemistryProblems(mol)]
68
+
69
+
70
+ def clear_cache() -> None:
71
+ """Forget every parsed molecule and InChIKey (to free memory after a large table).
72
+
73
+ Returns:
74
+ None.
75
+ """
76
+ parse.cache_clear()
77
+ inchikey.cache_clear()
@@ -0,0 +1,393 @@
1
+ """Chemistry checks for Pandera schemas.
2
+
3
+ Each function returns a ``pandera.Check`` to put on a column of SMILES (or of InChIKeys), next
4
+ to Pandera's own checks::
5
+
6
+ import pandera.pandas as pa
7
+ import pandera_chem as pc
8
+
9
+ schema = pa.DataFrameSchema({
10
+ "smiles": pa.Column(str, [pc.valid_smiles(), pc.allowed_elements(), pc.neutral()]),
11
+ })
12
+
13
+ One rule keeps the reasons clear: only :func:`valid_smiles` fails a SMILES that cannot be read.
14
+ Every other check passes it, so each bad record is reported once, for its real problem.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import re
20
+ from collections.abc import Iterable, Mapping
21
+ from typing import Any
22
+
23
+ import pandas as pd
24
+ import pandera.pandas as pa
25
+ from rdkit import Chem
26
+ from rdkit.Chem import Descriptors
27
+
28
+ from ._mol import inchikey, parse
29
+
30
+ # the elements of most drug-like and screening libraries; pass your own to allowed_elements()
31
+ ORGANIC = frozenset({"H", "B", "C", "N", "O", "F", "P", "S", "Cl", "Br", "I"})
32
+ INCHIKEY = re.compile(r"^[A-Z]{14}-[A-Z]{10}-[A-Z]$")
33
+
34
+
35
+ def valid_smiles(**options: Any) -> pa.Check:
36
+ """The SMILES can be read and sanitised by RDKit.
37
+
38
+ Catches syntax errors, unclosed rings, impossible valences and aromatic rings that cannot
39
+ be kekulised. :func:`pandera_chem.problems` says why a given SMILES fails.
40
+
41
+ Args:
42
+ **options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
43
+ instead of failing, or ``n_failure_cases=10``.
44
+
45
+ Returns:
46
+ The check.
47
+ """
48
+ return pa.Check(
49
+ lambda smiles: parse(smiles) is not None,
50
+ element_wise=True,
51
+ name="valid_smiles",
52
+ description="the SMILES cannot be read or sanitised by RDKit",
53
+ **options,
54
+ )
55
+
56
+
57
+ def allowed_elements(elements: Iterable[str] = ORGANIC, **options: Any) -> pa.Check:
58
+ """Every atom is one of the allowed elements.
59
+
60
+ Args:
61
+ elements: The element symbols allowed, e.g. ``{"C", "N", "O", "H"}``. Defaults to
62
+ :data:`ORGANIC`: H, B, C, N, O, F, P, S, Cl, Br and I.
63
+ **options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
64
+ instead of failing, or ``n_failure_cases=10``.
65
+
66
+ Returns:
67
+ The check.
68
+ """
69
+ allowed = frozenset(elements)
70
+
71
+ def ok(smiles: str) -> bool:
72
+ """Whether every atom of one SMILES is allowed (unreadable SMILES pass).
73
+
74
+ Args:
75
+ smiles: The SMILES.
76
+
77
+ Returns:
78
+ True if allowed or unreadable.
79
+ """
80
+ mol = parse(smiles)
81
+ return mol is None or all(a.GetSymbol() in allowed for a in mol.GetAtoms())
82
+
83
+ return pa.Check(
84
+ ok,
85
+ element_wise=True,
86
+ name="allowed_elements",
87
+ description=f"an element outside {sorted(allowed)}",
88
+ statistics={"elements": sorted(allowed)},
89
+ **options,
90
+ )
91
+
92
+
93
+ def single_fragment(**options: Any) -> pa.Check:
94
+ """The record is one connected molecule: no salts, solvents or mixtures.
95
+
96
+ Args:
97
+ **options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
98
+ instead of failing, or ``n_failure_cases=10``.
99
+
100
+ Returns:
101
+ The check.
102
+ """
103
+
104
+ def ok(smiles: str) -> bool:
105
+ """Whether one SMILES is a single piece (unreadable SMILES pass).
106
+
107
+ Args:
108
+ smiles: The SMILES.
109
+
110
+ Returns:
111
+ True if one piece or unreadable.
112
+ """
113
+ mol = parse(smiles)
114
+ return mol is None or len(Chem.GetMolFrags(mol)) == 1
115
+
116
+ return pa.Check(
117
+ ok,
118
+ element_wise=True,
119
+ name="single_fragment",
120
+ description="more than one molecule in the record (a salt, solvent or mixture)",
121
+ **options,
122
+ )
123
+
124
+
125
+ def neutral(**options: Any) -> pa.Check:
126
+ """The net formal charge is zero (charges that cancel, as in a nitro group, are fine).
127
+
128
+ Args:
129
+ **options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
130
+ instead of failing, or ``n_failure_cases=10``.
131
+
132
+ Returns:
133
+ The check.
134
+ """
135
+
136
+ def ok(smiles: str) -> bool:
137
+ """Whether one SMILES has no net charge (unreadable SMILES pass).
138
+
139
+ Args:
140
+ smiles: The SMILES.
141
+
142
+ Returns:
143
+ True if neutral or unreadable.
144
+ """
145
+ mol = parse(smiles)
146
+ return mol is None or Chem.GetFormalCharge(mol) == 0
147
+
148
+ return pa.Check(
149
+ ok, element_wise=True, name="neutral", description="a net formal charge", **options
150
+ )
151
+
152
+
153
+ def stereo_defined(**options: Any) -> pa.Check:
154
+ """Every stereocentre and stereogenic double bond has its configuration given.
155
+
156
+ Often used as a warning rather than an error: ``stereo_defined(raise_warning=True)``.
157
+
158
+ Args:
159
+ **options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
160
+ instead of failing, or ``n_failure_cases=10``.
161
+
162
+ Returns:
163
+ The check.
164
+ """
165
+
166
+ def ok(smiles: str) -> bool:
167
+ """Whether one SMILES specifies all its stereochemistry (unreadable SMILES pass).
168
+
169
+ Args:
170
+ smiles: The SMILES.
171
+
172
+ Returns:
173
+ True if fully specified or unreadable.
174
+ """
175
+ mol = parse(smiles)
176
+ if mol is None:
177
+ return True
178
+ return all(
179
+ info.specified != Chem.StereoSpecified.Unspecified
180
+ for info in Chem.FindPotentialStereo(mol)
181
+ )
182
+
183
+ return pa.Check(
184
+ ok,
185
+ element_wise=True,
186
+ name="stereo_defined",
187
+ description="a stereocentre or double bond with no configuration given",
188
+ **options,
189
+ )
190
+
191
+
192
+ def heavy_atoms(min_value: int = 1, max_value: int | None = None, **options: Any) -> pa.Check:
193
+ """The number of heavy (non-hydrogen) atoms of the largest fragment is within limits.
194
+
195
+ Args:
196
+ min_value: The fewest heavy atoms allowed.
197
+ max_value: The most heavy atoms allowed, or None for no upper limit.
198
+ **options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
199
+ instead of failing, or ``n_failure_cases=10``.
200
+
201
+ Returns:
202
+ The check.
203
+ """
204
+ top = float("inf") if max_value is None else max_value
205
+
206
+ def ok(smiles: str) -> bool:
207
+ """Whether one SMILES' largest fragment is within the limits (unreadable SMILES pass).
208
+
209
+ Args:
210
+ smiles: The SMILES.
211
+
212
+ Returns:
213
+ True if within the limits or unreadable.
214
+ """
215
+ mol = parse(smiles)
216
+ if mol is None:
217
+ return True
218
+ n = max(f.GetNumHeavyAtoms() for f in Chem.GetMolFrags(mol, asMols=True))
219
+ return min_value <= n <= top
220
+
221
+ return pa.Check(
222
+ ok,
223
+ element_wise=True,
224
+ name="heavy_atoms",
225
+ description=f"heavy atoms outside {min_value} to {max_value}",
226
+ statistics={"min_value": min_value, "max_value": max_value},
227
+ **options,
228
+ )
229
+
230
+
231
+ def mol_weight(max_value: float, min_value: float = 0.0, **options: Any) -> pa.Check:
232
+ """The molecular weight (g/mol) of the largest fragment is within limits.
233
+
234
+ Args:
235
+ max_value: The largest weight allowed.
236
+ min_value: The smallest weight allowed.
237
+ **options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
238
+ instead of failing, or ``n_failure_cases=10``.
239
+
240
+ Returns:
241
+ The check.
242
+ """
243
+
244
+ def ok(smiles: str) -> bool:
245
+ """Whether one SMILES' largest fragment weighs within the limits (unreadable pass).
246
+
247
+ Args:
248
+ smiles: The SMILES.
249
+
250
+ Returns:
251
+ True if within the limits or unreadable.
252
+ """
253
+ mol = parse(smiles)
254
+ if mol is None:
255
+ return True
256
+ piece = max(Chem.GetMolFrags(mol, asMols=True), key=lambda f: f.GetNumHeavyAtoms())
257
+ return min_value <= Descriptors.MolWt(piece) <= max_value
258
+
259
+ return pa.Check(
260
+ ok,
261
+ element_wise=True,
262
+ name="mol_weight",
263
+ description=f"molecular weight outside {min_value} to {max_value} g/mol",
264
+ statistics={"min_value": min_value, "max_value": max_value},
265
+ **options,
266
+ )
267
+
268
+
269
+ def no_substructures(patterns: Mapping[str, str], **options: Any) -> pa.Check:
270
+ """None of the given substructures is present (e.g. reactive groups you want kept out).
271
+
272
+ Args:
273
+ patterns: Each pattern's name and its SMARTS, e.g. ``{"acyl halide": "C(=O)[Cl,Br,I]"}``.
274
+ **options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
275
+ instead of failing, or ``n_failure_cases=10``.
276
+
277
+ Returns:
278
+ The check.
279
+
280
+ Raises:
281
+ ValueError: If a SMARTS cannot be read.
282
+ """
283
+ compiled = {}
284
+ for label, smarts in patterns.items():
285
+ query = Chem.MolFromSmarts(smarts)
286
+ if query is None:
287
+ raise ValueError(f"cannot read the SMARTS for {label!r}: {smarts}")
288
+ compiled[label] = query
289
+
290
+ def ok(smiles: str) -> bool:
291
+ """Whether one SMILES contains none of the patterns (unreadable SMILES pass).
292
+
293
+ Args:
294
+ smiles: The SMILES.
295
+
296
+ Returns:
297
+ True if no pattern matches or unreadable.
298
+ """
299
+ mol = parse(smiles)
300
+ return mol is None or not any(mol.HasSubstructMatch(q) for q in compiled.values())
301
+
302
+ return pa.Check(
303
+ ok,
304
+ element_wise=True,
305
+ name="no_substructures",
306
+ description=f"contains one of: {', '.join(compiled)}",
307
+ statistics={"patterns": dict(patterns)},
308
+ **options,
309
+ )
310
+
311
+
312
+ def unique_molecules(**options: Any) -> pa.Check:
313
+ """No molecule appears twice, even when written as different SMILES.
314
+
315
+ Molecules are compared by standard InChIKey, so ``OCC`` and ``CCO`` are the same. The first
316
+ record of each molecule passes; every later one fails. Unreadable SMILES pass.
317
+
318
+ Args:
319
+ **options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
320
+ instead of failing, or ``n_failure_cases=10``.
321
+
322
+ Returns:
323
+ The check.
324
+ """
325
+
326
+ def ok(smiles: pd.Series) -> pd.Series:
327
+ """Which records are the first (or only) record of their molecule.
328
+
329
+ Args:
330
+ smiles: The column of SMILES.
331
+
332
+ Returns:
333
+ True for first occurrences and unreadable SMILES.
334
+ """
335
+ keys = smiles.map(inchikey)
336
+ return ~(keys.notna() & keys.duplicated(keep="first"))
337
+
338
+ return pa.Check(
339
+ ok,
340
+ name="unique_molecules",
341
+ description="the same molecule as an earlier record",
342
+ **options,
343
+ )
344
+
345
+
346
+ def inchikey_format(**options: Any) -> pa.Check:
347
+ """The value is shaped like a standard InChIKey (14 letters, 10 letters, 1 letter).
348
+
349
+ Args:
350
+ **options: Passed on to ``pandera.Check``, e.g. ``raise_warning=True`` to warn
351
+ instead of failing, or ``n_failure_cases=10``.
352
+
353
+ Returns:
354
+ The check.
355
+ """
356
+ return pa.Check.str_matches(INCHIKEY.pattern, name="inchikey_format", **options)
357
+
358
+
359
+ def inchikey_matches(smiles_column: str, inchikey_column: str, **options: Any) -> pa.Check:
360
+ """A table-wide check: each record's InChIKey is the one its SMILES gives.
361
+
362
+ Records whose SMILES cannot be read, or whose InChIKey is malformed, pass: the column
363
+ checks report those.
364
+
365
+ Args:
366
+ smiles_column: The name of the SMILES column.
367
+ inchikey_column: The name of the InChIKey column.
368
+ **options: Passed on to ``pandera.Check``.
369
+
370
+ Returns:
371
+ The check, to put in ``DataFrameSchema(checks=[...])``.
372
+ """
373
+
374
+ def ok(df: pd.DataFrame) -> pd.Series:
375
+ """Which records' InChIKeys agree with their SMILES.
376
+
377
+ Args:
378
+ df: The table.
379
+
380
+ Returns:
381
+ True where they agree, or where either value cannot be compared.
382
+ """
383
+ expected = df[smiles_column].map(inchikey)
384
+ given = df[inchikey_column]
385
+ comparable = expected.notna() & given.astype(str).str.match(INCHIKEY.pattern)
386
+ return ~comparable | (expected == given)
387
+
388
+ return pa.Check(
389
+ ok,
390
+ name="inchikey_matches",
391
+ description=f"{inchikey_column} is not the InChIKey of {smiles_column}",
392
+ **options,
393
+ )
File without changes
@@ -0,0 +1,163 @@
1
+ """Quarantine: keep the rows that pass, set the others aside with their reasons.
2
+
3
+ Pandera either accepts a table or raises an error listing what failed. In a pipeline you usually
4
+ want neither: you want to carry on with the good rows and keep the bad ones, each with the
5
+ reason it was set aside, so someone can fix them later. :func:`split` does that, and its report
6
+ gives counts that must add up.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import warnings
12
+ from collections import Counter
13
+ from dataclasses import dataclass, field
14
+
15
+ import pandas as pd
16
+ import pandera.pandas as pa
17
+ from pandera.errors import SchemaErrors, SchemaWarning
18
+
19
+ REASONS = "reasons" # the column added to the quarantined rows
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class Report:
24
+ """What happened to a table: how many rows came in, passed and were set aside, and why.
25
+
26
+ Attributes:
27
+ rows_in: Rows in the table given to :func:`split`.
28
+ rows_valid: Rows that passed every check.
29
+ rows_quarantined: Rows set aside.
30
+ by_reason: For each reason ("column: check"), how many rows failed it. A row that fails
31
+ two checks is counted under both, so these can add up to more than
32
+ ``rows_quarantined``.
33
+ warnings: The messages of checks that only warn (``raise_warning=True``).
34
+ """
35
+
36
+ rows_in: int
37
+ rows_valid: int
38
+ rows_quarantined: int
39
+ by_reason: dict[str, int] = field(default_factory=dict)
40
+ warnings: list[str] = field(default_factory=list)
41
+
42
+ @property
43
+ def reconciles(self) -> bool:
44
+ """Whether every row is accounted for: rows in = valid + quarantined.
45
+
46
+ Returns:
47
+ True if the counts add up.
48
+ """
49
+ return self.rows_in == self.rows_valid + self.rows_quarantined
50
+
51
+ def __str__(self) -> str:
52
+ """A short summary to print or log.
53
+
54
+ Returns:
55
+ The summary, one fact per line.
56
+ """
57
+ lines = [
58
+ f"{self.rows_in} rows in: {self.rows_valid} valid, {self.rows_quarantined} quarantined",
59
+ ]
60
+ lines += [f" {n:>6} {reason}" for reason, n in self.by_reason.items()]
61
+ lines += [f" warning: {w}" for w in self.warnings]
62
+ return "\n".join(lines)
63
+
64
+
65
+ @dataclass(frozen=True)
66
+ class Split:
67
+ """The result of :func:`split`.
68
+
69
+ Attributes:
70
+ valid: The rows that passed, as validated (and coerced) by the schema.
71
+ quarantined: The rows that failed, unchanged, with a ``reasons`` column added.
72
+ report: The counts.
73
+ """
74
+
75
+ valid: pd.DataFrame
76
+ quarantined: pd.DataFrame
77
+ report: Report
78
+
79
+
80
+ def _reason(column: object, check: object) -> str:
81
+ """One failure's reason, as "column: check" (or just the check for a table-wide one).
82
+
83
+ Args:
84
+ column: The column that failed, or None/NaN for a table-wide check.
85
+ check: The check's name.
86
+
87
+ Returns:
88
+ The reason.
89
+ """
90
+ return str(check) if pd.isna(column) else f"{column}: {check}"
91
+
92
+
93
+ def _warnings(caught: list[warnings.WarningMessage]) -> list[str]:
94
+ """The distinct messages of the checks that only warn, in the order they came.
95
+
96
+ Args:
97
+ caught: Warnings recorded while validating.
98
+
99
+ Returns:
100
+ Each Pandera warning's message, once.
101
+ """
102
+ messages = [str(w.message) for w in caught if issubclass(w.category, SchemaWarning)]
103
+ return list(dict.fromkeys(messages))
104
+
105
+
106
+ def _all_valid(df: pd.DataFrame, valid: pd.DataFrame, caught: list) -> Split:
107
+ """The split of a table in which every row passed.
108
+
109
+ Args:
110
+ df: The table given to :func:`split`.
111
+ valid: The table as validated by the schema.
112
+ caught: Warnings recorded while validating.
113
+
114
+ Returns:
115
+ Every row valid, none quarantined.
116
+ """
117
+ empty = df.iloc[0:0].assign(**{REASONS: pd.Series(dtype=str)})
118
+ return Split(valid, empty, Report(len(df), len(df), 0, {}, _warnings(caught)))
119
+
120
+
121
+ def split(df: pd.DataFrame, schema: pa.DataFrameSchema) -> Split:
122
+ """Validate a table and divide it into the rows that pass and the rows that fail.
123
+
124
+ Args:
125
+ df: The table. Its index must be unique, so each failure can be traced to one row.
126
+ schema: The Pandera schema to check it against.
127
+
128
+ Returns:
129
+ The valid rows, the quarantined rows with their reasons, and a report.
130
+
131
+ Raises:
132
+ ValueError: If the index of ``df`` has repeated labels.
133
+ pandera.errors.SchemaErrors: If something fails that no row can be blamed for, such as
134
+ a missing column; the whole table is then unusable, and the error says why.
135
+ """
136
+ if not df.index.is_unique:
137
+ raise ValueError("the table's index has repeated labels; reset it first")
138
+ with warnings.catch_warnings(record=True) as caught:
139
+ warnings.simplefilter("always", SchemaWarning)
140
+ try:
141
+ return _all_valid(df, schema.validate(df, lazy=True), caught)
142
+ except SchemaErrors as error:
143
+ failures = error.failure_cases
144
+ if failures["index"].isna().any():
145
+ raise # a failure of the whole table, not of particular rows
146
+ reasons = (
147
+ failures.assign(
148
+ reason=[
149
+ _reason(c, k)
150
+ for c, k in zip(failures["column"], failures["check"], strict=True)
151
+ ]
152
+ )
153
+ .drop_duplicates(["index", "reason"])
154
+ .groupby("index", sort=False)["reason"]
155
+ .agg("; ".join)
156
+ )
157
+ bad = df.index.isin(reasons.index)
158
+ quarantined = df[bad].assign(**{REASONS: df.index[bad].map(reasons)})
159
+ valid = schema.validate(df[~bad], lazy=True) if (~bad).any() else df[~bad]
160
+ warned = _warnings(caught)
161
+ counts = Counter(r for joined in reasons for r in joined.split("; "))
162
+ report = Report(len(df), int((~bad).sum()), int(bad.sum()), dict(counts.most_common()), warned)
163
+ return Split(valid, quarantined, report)