tablassert 19.5.0__tar.gz → 19.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-19.5.0 → tablassert-19.6.0}/PKG-INFO +49 -16
- {tablassert-19.5.0 → tablassert-19.6.0}/README.md +47 -14
- {tablassert-19.5.0 → tablassert-19.6.0}/pyproject.toml +2 -2
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/src/fullmap.rs +5 -5
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/biolink.py +209 -72
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/cli.py +65 -20
- {tablassert-19.5.0 → tablassert-19.6.0}/LICENSE +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/Cargo.lock +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/Cargo.toml +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/src/json.rs +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/src/lib.rs +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/src/ndjson.rs +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/src/nlp.rs +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/src/uuid.rs +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/__init__.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/agent.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/cli_entry.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/coerce.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/distill.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/distill_reward.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/enums.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/errors.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/extras.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/fullmap.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/ingests.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/lib.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/log.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/models.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/net.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/nlp.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/progress.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/qc.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/rig.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/rs.pyi +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/runcache.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/study.py +0 -0
- {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version: 19.
|
|
3
|
+
Version: 19.6.0
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -22,7 +22,7 @@ Classifier: Framework :: Pydantic :: 2
|
|
|
22
22
|
Classifier: Operating System :: POSIX :: Linux
|
|
23
23
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
24
24
|
Classifier: Environment :: Console
|
|
25
|
-
Requires-Dist: biolink-model>=4.4.
|
|
25
|
+
Requires-Dist: biolink-model>=4.4.5
|
|
26
26
|
Requires-Dist: polars>=2.0.0rc2
|
|
27
27
|
Requires-Dist: pyarrow>=21.0.0
|
|
28
28
|
Requires-Dist: rapidfuzz>=3.14.3
|
|
@@ -77,6 +77,36 @@ KGX-compliant nodes and edges.
|
|
|
77
77
|
**[Full Documentation](https://skyeav.github.io/Tablassert/)**: installation guides, tutorial,
|
|
78
78
|
configuration reference, and API docs.
|
|
79
79
|
|
|
80
|
+
## Statement of need
|
|
81
|
+
|
|
82
|
+
Biomedical knowledge lives in spreadsheets: association tables, assay results, curated gene-disease
|
|
83
|
+
lists. Getting those rows into an NCATS Translator knowledge graph means writing an ingest that maps
|
|
84
|
+
columns to Biolink statements, resolves free text ("TP53", "lung cancer") to standard CURIEs, and
|
|
85
|
+
records provenance, terms of use, and statistical annotations. Today that ingest is Python code
|
|
86
|
+
maintained per source, and entity resolution usually means calling a hosted name-resolution service
|
|
87
|
+
at build time.
|
|
88
|
+
|
|
89
|
+
Tablassert replaces the per-source code with a declarative YAML mapping plus an embedded, offline
|
|
90
|
+
entity-resolution database built from RENCI BABEL exports, so a build is reproducible, auditable,
|
|
91
|
+
and network-free. It is aimed at Translator ingest authors and knowledge-graph data engineers who
|
|
92
|
+
hold tabular biomedical sources, and at bioinformatics groups that want a validated KGX output
|
|
93
|
+
without writing a transform pipeline. The configuration model and the Biolink/KGX contracts it
|
|
94
|
+
emits follow current [`NCATSTranslator/translator-ingests`](https://github.com/NCATSTranslator/translator-ingests)
|
|
95
|
+
practice, and each build can emit the Resource Ingest Guide (RIG) metadata those submissions require.
|
|
96
|
+
|
|
97
|
+
## Getting help
|
|
98
|
+
|
|
99
|
+
- **Questions and bug reports:** open an issue at
|
|
100
|
+
[github.com/SkyeAv/Tablassert/issues](https://github.com/SkyeAv/Tablassert/issues) (use the
|
|
101
|
+
`question` label for usage questions). Include the command you ran, its output, and your
|
|
102
|
+
`tablassert --version`.
|
|
103
|
+
- **Contributing code or docs:** see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, the local quality
|
|
104
|
+
gates, and pull-request expectations. By participating in this project you agree to follow the
|
|
105
|
+
[Code of Conduct](CODE_OF_CONDUCT.md).
|
|
106
|
+
- **Reference:** the [CLI Reference](https://skyeav.github.io/Tablassert/cli/) and
|
|
107
|
+
[configuration guides](https://skyeav.github.io/Tablassert/configuration/graph/) document every
|
|
108
|
+
flag and field.
|
|
109
|
+
|
|
80
110
|
## Quick Start
|
|
81
111
|
|
|
82
112
|
```bash
|
|
@@ -105,14 +135,17 @@ template:
|
|
|
105
135
|
```
|
|
106
136
|
|
|
107
137
|
Wrap it in a graph config (`graph.yaml`) pointing at your fullmap entity-resolution database
|
|
108
|
-
and carrying the required `rig:` metadata for the generated Resource Ingest Guide
|
|
138
|
+
and carrying the required `rig:` metadata for the generated Resource Ingest Guide. Build or
|
|
139
|
+
download that database once first (`tablassert build-fullmap`, a multi-GB download; see the
|
|
140
|
+
[Fullmap guide](https://skyeav.github.io/Tablassert/fullmap/)), which makes `./fullmap` below the
|
|
141
|
+
right path:
|
|
109
142
|
|
|
110
143
|
```yaml
|
|
111
144
|
name: MY_KG
|
|
112
145
|
version: 1.0.0
|
|
113
146
|
tables:
|
|
114
147
|
- ./table.yaml
|
|
115
|
-
fullmap:
|
|
148
|
+
fullmap: ./fullmap
|
|
116
149
|
rig:
|
|
117
150
|
source_info:
|
|
118
151
|
infores_id: infores:my-kg
|
|
@@ -159,7 +192,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
159
192
|
low-confidence mappings
|
|
160
193
|
- **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
|
|
161
194
|
and predicates
|
|
162
|
-
- **Autonomous agent**: `tablassert agent` derives, builds, and refines configs for
|
|
195
|
+
- **Autonomous agent (experimental)**: `tablassert agent` derives, builds, and refines configs for
|
|
196
|
+
whole papers
|
|
163
197
|
- **Performance & reproducibility**: lazy Polars pipelines and a deterministic UV-based
|
|
164
198
|
development environment
|
|
165
199
|
|
|
@@ -169,8 +203,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
169
203
|
pip install tablassert
|
|
170
204
|
```
|
|
171
205
|
|
|
172
|
-
Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API;
|
|
173
|
-
|
|
206
|
+
Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API; `[cli]`
|
|
207
|
+
provides the `tablassert` command. Other extras are opt-in:
|
|
174
208
|
|
|
175
209
|
| Extra | Adds | Install |
|
|
176
210
|
| ----- | ---- | ------- |
|
|
@@ -178,17 +212,16 @@ install `[cli]` to use the `tablassert` command and optional extras for addition
|
|
|
178
212
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
179
213
|
| `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
180
214
|
| `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
181
|
-
| `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
|
|
182
|
-
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
|
|
183
|
-
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
|
|
215
|
+
| `agent` | autonomous agent (smolagents, litellm, article/table context) -- experimental, API may change | `pip install "tablassert[agent]"` |
|
|
216
|
+
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) -- experimental, API may change | `pip install "tablassert[optimize]"` |
|
|
217
|
+
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) -- experimental, API may change | `pip install "tablassert[distill]"` |
|
|
184
218
|
| `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
|
|
185
219
|
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
[Installation guide](https://skyeav.github.io/Tablassert/installation/) for the full matrix and the
|
|
220
|
+
Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure
|
|
221
|
+
names the missing package and the install command that fixes it. QC is opt-in at build time
|
|
222
|
+
(`build-kg --qc`). See the
|
|
223
|
+
[Installation guide](https://skyeav.github.io/Tablassert/installation/) for the full matrix,
|
|
224
|
+
per-command preflight behavior, and the
|
|
192
225
|
[CLI Reference](https://skyeav.github.io/Tablassert/cli/) for every flag.
|
|
193
226
|
|
|
194
227
|
## Entity Resolution API
|
|
@@ -16,6 +16,36 @@ KGX-compliant nodes and edges.
|
|
|
16
16
|
**[Full Documentation](https://skyeav.github.io/Tablassert/)**: installation guides, tutorial,
|
|
17
17
|
configuration reference, and API docs.
|
|
18
18
|
|
|
19
|
+
## Statement of need
|
|
20
|
+
|
|
21
|
+
Biomedical knowledge lives in spreadsheets: association tables, assay results, curated gene-disease
|
|
22
|
+
lists. Getting those rows into an NCATS Translator knowledge graph means writing an ingest that maps
|
|
23
|
+
columns to Biolink statements, resolves free text ("TP53", "lung cancer") to standard CURIEs, and
|
|
24
|
+
records provenance, terms of use, and statistical annotations. Today that ingest is Python code
|
|
25
|
+
maintained per source, and entity resolution usually means calling a hosted name-resolution service
|
|
26
|
+
at build time.
|
|
27
|
+
|
|
28
|
+
Tablassert replaces the per-source code with a declarative YAML mapping plus an embedded, offline
|
|
29
|
+
entity-resolution database built from RENCI BABEL exports, so a build is reproducible, auditable,
|
|
30
|
+
and network-free. It is aimed at Translator ingest authors and knowledge-graph data engineers who
|
|
31
|
+
hold tabular biomedical sources, and at bioinformatics groups that want a validated KGX output
|
|
32
|
+
without writing a transform pipeline. The configuration model and the Biolink/KGX contracts it
|
|
33
|
+
emits follow current [`NCATSTranslator/translator-ingests`](https://github.com/NCATSTranslator/translator-ingests)
|
|
34
|
+
practice, and each build can emit the Resource Ingest Guide (RIG) metadata those submissions require.
|
|
35
|
+
|
|
36
|
+
## Getting help
|
|
37
|
+
|
|
38
|
+
- **Questions and bug reports:** open an issue at
|
|
39
|
+
[github.com/SkyeAv/Tablassert/issues](https://github.com/SkyeAv/Tablassert/issues) (use the
|
|
40
|
+
`question` label for usage questions). Include the command you ran, its output, and your
|
|
41
|
+
`tablassert --version`.
|
|
42
|
+
- **Contributing code or docs:** see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, the local quality
|
|
43
|
+
gates, and pull-request expectations. By participating in this project you agree to follow the
|
|
44
|
+
[Code of Conduct](CODE_OF_CONDUCT.md).
|
|
45
|
+
- **Reference:** the [CLI Reference](https://skyeav.github.io/Tablassert/cli/) and
|
|
46
|
+
[configuration guides](https://skyeav.github.io/Tablassert/configuration/graph/) document every
|
|
47
|
+
flag and field.
|
|
48
|
+
|
|
19
49
|
## Quick Start
|
|
20
50
|
|
|
21
51
|
```bash
|
|
@@ -44,14 +74,17 @@ template:
|
|
|
44
74
|
```
|
|
45
75
|
|
|
46
76
|
Wrap it in a graph config (`graph.yaml`) pointing at your fullmap entity-resolution database
|
|
47
|
-
and carrying the required `rig:` metadata for the generated Resource Ingest Guide
|
|
77
|
+
and carrying the required `rig:` metadata for the generated Resource Ingest Guide. Build or
|
|
78
|
+
download that database once first (`tablassert build-fullmap`, a multi-GB download; see the
|
|
79
|
+
[Fullmap guide](https://skyeav.github.io/Tablassert/fullmap/)), which makes `./fullmap` below the
|
|
80
|
+
right path:
|
|
48
81
|
|
|
49
82
|
```yaml
|
|
50
83
|
name: MY_KG
|
|
51
84
|
version: 1.0.0
|
|
52
85
|
tables:
|
|
53
86
|
- ./table.yaml
|
|
54
|
-
fullmap:
|
|
87
|
+
fullmap: ./fullmap
|
|
55
88
|
rig:
|
|
56
89
|
source_info:
|
|
57
90
|
infores_id: infores:my-kg
|
|
@@ -98,7 +131,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
98
131
|
low-confidence mappings
|
|
99
132
|
- **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
|
|
100
133
|
and predicates
|
|
101
|
-
- **Autonomous agent**: `tablassert agent` derives, builds, and refines configs for
|
|
134
|
+
- **Autonomous agent (experimental)**: `tablassert agent` derives, builds, and refines configs for
|
|
135
|
+
whole papers
|
|
102
136
|
- **Performance & reproducibility**: lazy Polars pipelines and a deterministic UV-based
|
|
103
137
|
development environment
|
|
104
138
|
|
|
@@ -108,8 +142,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
108
142
|
pip install tablassert
|
|
109
143
|
```
|
|
110
144
|
|
|
111
|
-
Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API;
|
|
112
|
-
|
|
145
|
+
Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API; `[cli]`
|
|
146
|
+
provides the `tablassert` command. Other extras are opt-in:
|
|
113
147
|
|
|
114
148
|
| Extra | Adds | Install |
|
|
115
149
|
| ----- | ---- | ------- |
|
|
@@ -117,17 +151,16 @@ install `[cli]` to use the `tablassert` command and optional extras for addition
|
|
|
117
151
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
118
152
|
| `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
119
153
|
| `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
120
|
-
| `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
|
|
121
|
-
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
|
|
122
|
-
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
|
|
154
|
+
| `agent` | autonomous agent (smolagents, litellm, article/table context) -- experimental, API may change | `pip install "tablassert[agent]"` |
|
|
155
|
+
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) -- experimental, API may change | `pip install "tablassert[optimize]"` |
|
|
156
|
+
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) -- experimental, API may change | `pip install "tablassert[distill]"` |
|
|
123
157
|
| `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
|
|
124
158
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
[Installation guide](https://skyeav.github.io/Tablassert/installation/) for the full matrix and the
|
|
159
|
+
Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure
|
|
160
|
+
names the missing package and the install command that fixes it. QC is opt-in at build time
|
|
161
|
+
(`build-kg --qc`). See the
|
|
162
|
+
[Installation guide](https://skyeav.github.io/Tablassert/installation/) for the full matrix,
|
|
163
|
+
per-command preflight behavior, and the
|
|
131
164
|
[CLI Reference](https://skyeav.github.io/Tablassert/cli/) for every flag.
|
|
132
165
|
|
|
133
166
|
## Entity Resolution API
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "19.
|
|
3
|
+
version = "19.6.0"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -47,7 +47,7 @@ classifiers = [
|
|
|
47
47
|
]
|
|
48
48
|
requires-python = ">=3.11"
|
|
49
49
|
dependencies = [
|
|
50
|
-
"biolink-model>=4.4.
|
|
50
|
+
"biolink-model>=4.4.5",
|
|
51
51
|
"polars>=2.0.0rc2",
|
|
52
52
|
"pyarrow>=21.0.0",
|
|
53
53
|
"rapidfuzz>=3.14.3",
|
|
@@ -2950,11 +2950,11 @@ fn extract_validate_rename(
|
|
|
2950
2950
|
archive.display()
|
|
2951
2951
|
))
|
|
2952
2952
|
};
|
|
2953
|
-
let database = open_read_only(&primary).map_err(
|
|
2954
|
-
validate_schema(&database).map_err(
|
|
2955
|
-
let build_id = read_build_id(&database).map_err(
|
|
2956
|
-
let shard_count = shard_count_of(&database).map_err(
|
|
2957
|
-
let recorded_allowlist = read_taxon_allowlist_identity(&database).map_err(
|
|
2953
|
+
let database = open_read_only(&primary).map_err(validation_ctx)?;
|
|
2954
|
+
validate_schema(&database).map_err(validation_ctx)?;
|
|
2955
|
+
let build_id = read_build_id(&database).map_err(validation_ctx)?;
|
|
2956
|
+
let shard_count = shard_count_of(&database).map_err(validation_ctx)?;
|
|
2957
|
+
let recorded_allowlist = read_taxon_allowlist_identity(&database).map_err(validation_ctx)?;
|
|
2958
2958
|
drop(database);
|
|
2959
2959
|
|
|
2960
2960
|
// A published archive built WITHOUT the required filter is not the database
|
|
@@ -46,6 +46,7 @@ from __future__ import annotations
|
|
|
46
46
|
|
|
47
47
|
import inspect
|
|
48
48
|
import json
|
|
49
|
+
import os
|
|
49
50
|
import re
|
|
50
51
|
from collections import Counter
|
|
51
52
|
from enum import Enum
|
|
@@ -86,6 +87,7 @@ __all__ = [
|
|
|
86
87
|
"legal_predicates",
|
|
87
88
|
"node_class",
|
|
88
89
|
"numeric_slot_kind",
|
|
90
|
+
"prune_kgx_edges",
|
|
89
91
|
"resolve_association_class",
|
|
90
92
|
"resolve_node_category",
|
|
91
93
|
"resolve_node_class",
|
|
@@ -591,28 +593,8 @@ release from making ``species_context_qualifier`` silently emittable again.
|
|
|
591
593
|
|
|
592
594
|
|
|
593
595
|
CLASS_FIELD_OVERRIDES: dict[str, frozenset[str]] = {
|
|
594
|
-
"EntityToDiseaseAssociation": frozenset(
|
|
595
|
-
|
|
596
|
-
"anatomical_context_qualifier",
|
|
597
|
-
"disease_context_qualifier",
|
|
598
|
-
"frequency_qualifier",
|
|
599
|
-
"population_context_qualifier",
|
|
600
|
-
"regulatory_approvals",
|
|
601
|
-
"sex_qualifier",
|
|
602
|
-
"temporal_context_qualifier",
|
|
603
|
-
}
|
|
604
|
-
),
|
|
605
|
-
"EntityToPhenotypicFeatureAssociation": frozenset(
|
|
606
|
-
{
|
|
607
|
-
"anatomical_context_qualifier",
|
|
608
|
-
"disease_context_qualifier",
|
|
609
|
-
"frequency_qualifier",
|
|
610
|
-
"population_context_qualifier",
|
|
611
|
-
"regulatory_approvals",
|
|
612
|
-
"sex_qualifier",
|
|
613
|
-
"temporal_context_qualifier",
|
|
614
|
-
}
|
|
615
|
-
),
|
|
596
|
+
"EntityToDiseaseAssociation": frozenset({"population_context_qualifier", "sex_qualifier", "temporal_context_qualifier"}),
|
|
597
|
+
"EntityToPhenotypicFeatureAssociation": frozenset({"population_context_qualifier", "temporal_context_qualifier"}),
|
|
616
598
|
}
|
|
617
599
|
"""Per-class grants of edge fields the resolved association class does not declare.
|
|
618
600
|
|
|
@@ -620,22 +602,21 @@ Keys are bare association class names (``association_class(cat).__name__``), val
|
|
|
620
602
|
the slots ``lib.prune_to_class`` keeps on rows resolved to that class even though the
|
|
621
603
|
installed model attaches them elsewhere.
|
|
622
604
|
|
|
623
|
-
The motivating case is a DAKP contraindication edge
|
|
624
|
-
|
|
625
|
-
``
|
|
626
|
-
``
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
``
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
``
|
|
634
|
-
|
|
635
|
-
a grant could never validate). Tablassert
|
|
636
|
-
deliberately emits the granted fields on the pinned classes ahead of the pinned model
|
|
605
|
+
The motivating case is a DAKP contraindication edge carrying the sparse qualifier
|
|
606
|
+
stack ahead of the pinned model: ``population_context_qualifier`` and
|
|
607
|
+
``temporal_context_qualifier`` are satisfiable qualifier slots the pinned classes do
|
|
608
|
+
not declare, and ``sex_qualifier`` is declared only on the phenotypic-feature side of
|
|
609
|
+
the family -- so without a grant ``prune_to_class`` would null them off the edge.
|
|
610
|
+
Upstream Biolink has absorbed most of the original stack (the biolink-model 4.4.5
|
|
611
|
+
mixin consolidation attached ``disease_context_qualifier``,
|
|
612
|
+
``anatomical_context_qualifier``, and ``frequency_qualifier`` to the whole
|
|
613
|
+
disease/phenotype family, renamed ``FDA_regulatory_approvals`` to the canonical
|
|
614
|
+
``regulatory_approvals`` on the pinned classes, and added ``sex_qualifier`` to
|
|
615
|
+
``EntityToPhenotypicFeatureAssociation``); the tripwire removed those grants the day
|
|
616
|
+
the release landed, and the remaining ones stay deliberately ahead of the model
|
|
637
617
|
(pending an upstream Biolink widening). ``_validation_record`` strips granted fields
|
|
638
|
-
before record validation so the deliberate gap is not reported as
|
|
618
|
+
before record validation so the deliberate gap is not reported as
|
|
619
|
+
``extra_forbidden``.
|
|
639
620
|
|
|
640
621
|
A tripwire test asserts every granted field is still absent from its class: the moment
|
|
641
622
|
a biolink-model release attaches the slot, the suite fails and the stale grant is
|
|
@@ -965,45 +946,201 @@ def validate_kgx(nodes_path: Path, edges_path: Path, limit: int = 20) -> dict[st
|
|
|
965
946
|
"""
|
|
966
947
|
report: dict[str, Any] = {"biolink_version": BIOLINK_VERSION, "ok": True, "ok_excluding_pending": True}
|
|
967
948
|
for label, path, edge in (("nodes", nodes_path, False), ("edges", edges_path, True)):
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
949
|
+
section: dict[str, Any] = _validate_file(path, edge=edge, limit=limit)
|
|
950
|
+
report[label] = section
|
|
951
|
+
if section["missing"] or section["total"] != section["valid"]:
|
|
952
|
+
report["ok"] = False
|
|
953
|
+
if section["missing"] or section["total"] != section["valid_excluding_pending"]:
|
|
954
|
+
report["ok_excluding_pending"] = False
|
|
955
|
+
return report
|
|
956
|
+
|
|
957
|
+
|
|
958
|
+
def _validate_file(path: Path, *, edge: bool, limit: int) -> dict[str, Any]:
|
|
959
|
+
"""Validate one KGX NDJSON file record by record (the per-file half of ``validate_kgx``).
|
|
960
|
+
|
|
961
|
+
Args:
|
|
962
|
+
path: Path to the ``.ndjson`` file; a missing path reports ``missing`` instead of
|
|
963
|
+
counting zero records out of zero as a pass.
|
|
964
|
+
edge: ``True`` to dispatch on the association family, ``False`` for nodes.
|
|
965
|
+
limit: Maximum number of example failures to retain.
|
|
966
|
+
|
|
967
|
+
Returns:
|
|
968
|
+
The same per-file section ``validate_kgx`` nests under ``"nodes"`` / ``"edges"``.
|
|
969
|
+
"""
|
|
970
|
+
total: int = 0
|
|
971
|
+
valid: int = 0
|
|
972
|
+
valid_excluding_pending: int = 0
|
|
973
|
+
problems: Counter[str] = Counter()
|
|
974
|
+
examples: list[dict[str, Any]] = []
|
|
975
|
+
# A missing path must never read as a clean bill of health: counting zero records
|
|
976
|
+
# out of zero would otherwise exit 0 on a typo'd filename and hide a broken build.
|
|
977
|
+
missing: bool = not path.is_file()
|
|
978
|
+
if not missing:
|
|
979
|
+
with path.open(encoding="utf-8") as handle:
|
|
980
|
+
for line in handle:
|
|
981
|
+
if not line.strip():
|
|
982
|
+
continue
|
|
983
|
+
total += 1
|
|
984
|
+
record: dict[str, Any] = json.loads(line)
|
|
985
|
+
errors: list[str] = validate_record(record, edge=edge)
|
|
986
|
+
if not errors:
|
|
987
|
+
valid += 1
|
|
988
|
+
valid_excluding_pending += 1
|
|
989
|
+
continue
|
|
990
|
+
if _record_has_only_pending_problems(record, errors):
|
|
991
|
+
valid_excluding_pending += 1
|
|
992
|
+
problems.update(errors)
|
|
993
|
+
if len(examples) < limit:
|
|
994
|
+
examples.append({"id": record.get("id"), "errors": errors})
|
|
995
|
+
return {
|
|
996
|
+
"total": total,
|
|
997
|
+
"valid": valid,
|
|
998
|
+
"valid_excluding_pending": valid_excluding_pending,
|
|
999
|
+
"failures": total - valid,
|
|
1000
|
+
"missing": missing,
|
|
1001
|
+
"problems": dict(problems.most_common()),
|
|
1002
|
+
"examples": examples,
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
|
|
1006
|
+
def prune_kgx_edges(edges_path: Path, limit: int = 20) -> dict[str, Any]:
|
|
1007
|
+
"""Drop every edge record that is not strictly KGX/Biolink-compliant, in place.
|
|
1008
|
+
|
|
1009
|
+
A record survives only when it validates against the Biolink Pydantic class named
|
|
1010
|
+
by its own ``category`` with zero errors -- the same strict bar a plain
|
|
1011
|
+
``validate_kgx`` run uses for its ``ok`` verdict, so a prune can never disagree
|
|
1012
|
+
with what validation reports:
|
|
1013
|
+
|
|
1014
|
+
* a strictly valid record is kept;
|
|
1015
|
+
* a record whose every failure is a known, intentional model gap (the KGX
|
|
1016
|
+
denormalized carryovers in :data:`KNOWN_PENDING_EDGE_FIELDS`, or a
|
|
1017
|
+
:data:`PREDICATE_OVERRIDES` predicate the pinned classes reject) is dropped --
|
|
1018
|
+
these carryovers are most of a real graph, so the caller asked for a
|
|
1019
|
+
strictly-compliant graph over maximal data retention when invoking this;
|
|
1020
|
+
* every other failing record is a real defect and is dropped;
|
|
1021
|
+
* a line that is not valid JSON at all is dropped and counted separately in
|
|
1022
|
+
``malformed``.
|
|
1023
|
+
|
|
1024
|
+
Nodes are never touched: dropping one would strand references, and cascading that
|
|
1025
|
+
into the edges is a modelling decision, not a validation repair.
|
|
1026
|
+
|
|
1027
|
+
The rewrite is ATOMIC and lossless for kept records: kept lines are written verbatim
|
|
1028
|
+
(byte-identical, order preserved, blank lines passed through) into a sibling
|
|
1029
|
+
``.{name}.tmp`` and swapped in with :func:`os.replace` only once every line was
|
|
1030
|
+
written, so a crash never leaves a torn edges file. When nothing is prunable the
|
|
1031
|
+
destination is not rewritten at all (``rewritten`` is ``False`` and the bytes are
|
|
1032
|
+
untouched), and a missing file is reported (``missing``) without ever being created
|
|
1033
|
+
-- a typo'd path must never read as a prune.
|
|
1034
|
+
|
|
1035
|
+
Args:
|
|
1036
|
+
edges_path: Path to ``<name>_<version>.edges.ndjson``.
|
|
1037
|
+
limit: Maximum number of example failures to retain per report section.
|
|
1038
|
+
|
|
1039
|
+
Returns:
|
|
1040
|
+
Mapping with ``before`` (the same per-file section ``validate_kgx`` builds over
|
|
1041
|
+
the original file), ``after`` (the same shape over the kept records, i.e. the
|
|
1042
|
+
file's post-prune state, equal to what ``validate_kgx`` reports on the rewritten
|
|
1043
|
+
file), ``dropped`` (the ``pruned`` records' problem histogram and examples --
|
|
1044
|
+
the records that were removed), ``pruned`` / ``kept`` / ``malformed`` /
|
|
1045
|
+
``rewritten`` counts, and ``ok`` / ``ok_excluding_pending`` computed over the
|
|
1046
|
+
post-prune state. For a readable file ``ok`` is therefore always true -- every
|
|
1047
|
+
kept record is strictly valid -- and ``ok_excluding_pending`` agrees, because
|
|
1048
|
+
nothing pending survives the prune.
|
|
1049
|
+
|
|
1050
|
+
Raises:
|
|
1051
|
+
OSError: the destination's directory is unwritable; the error propagates and the
|
|
1052
|
+
original file keeps its previous content.
|
|
1053
|
+
"""
|
|
1054
|
+
total: int = 0
|
|
1055
|
+
valid: int = 0
|
|
1056
|
+
pending: int = 0 # pending-only failures: counted for the BEFORE report, never kept
|
|
1057
|
+
pruned: int = 0
|
|
1058
|
+
malformed: int = 0
|
|
1059
|
+
all_problems: Counter[str] = Counter()
|
|
1060
|
+
all_examples: list[dict[str, Any]] = []
|
|
1061
|
+
pruned_problems: Counter[str] = Counter()
|
|
1062
|
+
pruned_examples: list[dict[str, Any]] = []
|
|
1063
|
+
# A missing path must never read as a clean bill of health: reporting zero records
|
|
1064
|
+
# out of zero would otherwise exit 0 on a typo'd filename and hide a broken build.
|
|
1065
|
+
missing: bool = not edges_path.is_file()
|
|
1066
|
+
if not missing:
|
|
1067
|
+
tmp: Path = edges_path.with_name(f".{edges_path.name}.tmp")
|
|
1068
|
+
try:
|
|
1069
|
+
# newline="" on both ends keeps every kept line byte-identical: the default
|
|
1070
|
+
# universal-newlines mode would silently rewrite "\r\n" as "\n".
|
|
1071
|
+
with edges_path.open(encoding="utf-8", newline="") as src, tmp.open("w", encoding="utf-8", newline="") as dst:
|
|
1072
|
+
for line in src:
|
|
979
1073
|
if not line.strip():
|
|
1074
|
+
dst.write(line) # blank lines are not records; pass through verbatim
|
|
980
1075
|
continue
|
|
981
1076
|
total += 1
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
1077
|
+
try:
|
|
1078
|
+
record: dict[str, Any] = json.loads(line)
|
|
1079
|
+
except json.JSONDecodeError:
|
|
1080
|
+
malformed += 1
|
|
1081
|
+
pruned += 1
|
|
1082
|
+
pruned_problems["record: json_parse_error"] += 1
|
|
1083
|
+
if len(pruned_examples) < limit:
|
|
1084
|
+
pruned_examples.append({"id": None, "errors": ["record: json_parse_error"]})
|
|
987
1085
|
continue
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1086
|
+
errors: list[str] = validate_record(record, edge=True)
|
|
1087
|
+
all_problems.update(errors)
|
|
1088
|
+
if len(all_examples) < limit:
|
|
1089
|
+
all_examples.append({"id": record.get("id"), "errors": errors})
|
|
1090
|
+
if errors:
|
|
1091
|
+
# The before-section keeps validate_kgx's pending score so the
|
|
1092
|
+
# pre-report matches a plain validation run; the STRICT bar still
|
|
1093
|
+
# drops the record -- real defects AND deliberate pending gaps.
|
|
1094
|
+
if _record_has_only_pending_problems(record, errors):
|
|
1095
|
+
pending += 1
|
|
1096
|
+
pruned += 1
|
|
1097
|
+
pruned_problems.update(errors)
|
|
1098
|
+
if len(pruned_examples) < limit:
|
|
1099
|
+
pruned_examples.append({"id": record.get("id"), "errors": errors})
|
|
1100
|
+
continue
|
|
1101
|
+
valid += 1
|
|
1102
|
+
dst.write(line)
|
|
1103
|
+
# ATOMIC: the swap happens only after every kept line was written, so a crash
|
|
1104
|
+
# mid-write leaves the original file untouched and the temp never read back.
|
|
1105
|
+
rewritten: bool = pruned > 0
|
|
1106
|
+
if rewritten:
|
|
1107
|
+
os.replace(tmp, edges_path)
|
|
1108
|
+
finally:
|
|
1109
|
+
tmp.unlink(missing_ok=True) # no-op after a successful replace; cleanup after a raise
|
|
1110
|
+
else:
|
|
1111
|
+
rewritten = False
|
|
1112
|
+
kept: int = total - pruned
|
|
1113
|
+
before: dict[str, Any] = {
|
|
1114
|
+
"total": total,
|
|
1115
|
+
"valid": valid,
|
|
1116
|
+
"valid_excluding_pending": valid + pending,
|
|
1117
|
+
"failures": total - valid,
|
|
1118
|
+
"missing": missing,
|
|
1119
|
+
"problems": dict(all_problems.most_common()),
|
|
1120
|
+
"examples": all_examples,
|
|
1121
|
+
}
|
|
1122
|
+
dropped: dict[str, Any] = {"total": pruned, "problems": dict(pruned_problems.most_common()), "examples": pruned_examples}
|
|
1123
|
+
after: dict[str, Any] = {
|
|
1124
|
+
"total": kept,
|
|
1125
|
+
"valid": valid,
|
|
1126
|
+
"valid_excluding_pending": valid, # nothing pending survives, so the scores agree
|
|
1127
|
+
"failures": kept - valid,
|
|
1128
|
+
"missing": missing,
|
|
1129
|
+
"problems": {},
|
|
1130
|
+
"examples": [],
|
|
1131
|
+
}
|
|
1132
|
+
return {
|
|
1133
|
+
"biolink_version": BIOLINK_VERSION,
|
|
1134
|
+
"before": before,
|
|
1135
|
+
"after": after,
|
|
1136
|
+
"dropped": dropped,
|
|
1137
|
+
"pruned": pruned,
|
|
1138
|
+
"kept": kept,
|
|
1139
|
+
"malformed": malformed,
|
|
1140
|
+
"rewritten": rewritten,
|
|
1141
|
+
"ok": not missing and kept == valid,
|
|
1142
|
+
"ok_excluding_pending": not missing and kept == valid,
|
|
1143
|
+
}
|
|
1007
1144
|
|
|
1008
1145
|
|
|
1009
1146
|
# Node slots Tablassert can always populate from a fullmap hit. A class requiring
|
|
@@ -955,11 +955,29 @@ def validate(
|
|
|
955
955
|
run(3, validate_pipeline, configuration_file)
|
|
956
956
|
|
|
957
957
|
|
|
958
|
+
def _print_kgx_section(label: str, section: dict[str, Any], path: Path) -> None:
|
|
959
|
+
"""Print one ``validate-kgx`` per-file section (shared by the plain and ``--prune`` paths)."""
|
|
960
|
+
if section["missing"]:
|
|
961
|
+
# Never let a typo'd path read as a pass: 0/0 valid would otherwise exit 0.
|
|
962
|
+
print(f"{label}: file not found ({path})", file=sys.stderr)
|
|
963
|
+
logger.info(f"validate-kgx {label}: file not found")
|
|
964
|
+
return
|
|
965
|
+
pending: int = section["valid_excluding_pending"] - section["valid"]
|
|
966
|
+
suffix: str = f"; {pending} pending biolink-model support" if pending else ""
|
|
967
|
+
print(f"{label}: {section['valid']}/{section['total']} valid ({section['failures']} failures{suffix})", file=sys.stderr)
|
|
968
|
+
for problem, count in section["problems"].items():
|
|
969
|
+
print(f" {count:>9} {problem}", file=sys.stderr)
|
|
970
|
+
for example in section["examples"][:3]:
|
|
971
|
+
print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
|
|
972
|
+
logger.info(f"validate-kgx {label}: {section['valid']}/{section['total']} valid")
|
|
973
|
+
|
|
974
|
+
|
|
958
975
|
@APP.command(name="validate-kgx")
|
|
959
976
|
def validate_kgx_command(
|
|
960
977
|
nodes: Annotated[Path, cyclopts.Parameter(name=["--nodes", "-n"])],
|
|
961
978
|
edges: Annotated[Path, cyclopts.Parameter(name=["--edges", "-e"])],
|
|
962
979
|
limit: Annotated[int, cyclopts.Parameter(name=["--limit"])] = 20,
|
|
980
|
+
prune: Annotated[bool, cyclopts.Parameter(name=["--prune", "-p"])] = False,
|
|
963
981
|
) -> None:
|
|
964
982
|
"""Validate built KGX NDJSON against the Biolink Model.
|
|
965
983
|
|
|
@@ -968,33 +986,60 @@ def validate_kgx_command(
|
|
|
968
986
|
and reports failures grouped by field and error type. Exits non-zero when any
|
|
969
987
|
record fails, so a build can be gated in CI.
|
|
970
988
|
|
|
989
|
+
``--prune`` rewrites the EDGES file in place, keeping only edges that validate
|
|
990
|
+
strictly. Real defects (non-coercible values, extras the model will never declare,
|
|
991
|
+
rejected predicates), the deliberate pending carryovers (``synonym``/``xref``/
|
|
992
|
+
``relation``/``provided_by``, ...), and lines that are not JSON at all are all
|
|
993
|
+
removed. On a real graph the carryovers are most of the edges, so ``--prune``
|
|
994
|
+
trades them for a strictly-compliant graph. The rewrite is atomic and kept lines
|
|
995
|
+
stay byte-identical; a clean file is not rewritten at all, and a missing edges file
|
|
996
|
+
is reported without ever being pruned. Nodes are validated and reported, never
|
|
997
|
+
pruned. The exit code stays strict over the post-prune state: after a prune, every
|
|
998
|
+
kept edge is strictly valid, so the run is compliant exactly when the nodes are.
|
|
999
|
+
|
|
971
1000
|
Args:
|
|
972
1001
|
nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
|
|
973
1002
|
edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
|
|
974
1003
|
limit: Maximum example failures retained per file for the ``e.g.`` report lines;
|
|
975
1004
|
every record is still validated regardless of this cap. A missing file is
|
|
976
1005
|
reported and fails the run rather than reading as a pass.
|
|
1006
|
+
prune: Rewrite the edges file in place, removing every edge record that is not
|
|
1007
|
+
strictly KGX/Biolink-compliant, including the deliberate pending carryovers.
|
|
1008
|
+
The nodes file is never modified; a clean edges file is left untouched.
|
|
977
1009
|
"""
|
|
978
|
-
from tablassert.biolink import validate_kgx
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
print(
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1010
|
+
from tablassert.biolink import _validate_file, prune_kgx_edges, validate_kgx
|
|
1011
|
+
|
|
1012
|
+
if not prune:
|
|
1013
|
+
report: dict[str, Any] = validate_kgx(nodes, edges, limit=limit)
|
|
1014
|
+
print(f"biolink-model {report['biolink_version']}", file=sys.stderr)
|
|
1015
|
+
_print_kgx_section("nodes", report["nodes"], nodes)
|
|
1016
|
+
_print_kgx_section("edges", report["edges"], edges)
|
|
1017
|
+
if not report["ok"]:
|
|
1018
|
+
print("KGX output is not Biolink-compliant.", file=sys.stderr)
|
|
1019
|
+
raise SystemExit(1)
|
|
1020
|
+
print("KGX output is Biolink-compliant.", file=sys.stderr)
|
|
1021
|
+
return
|
|
1022
|
+
|
|
1023
|
+
# The prune path validates nodes-only itself and drives prune_kgx_edges' own single
|
|
1024
|
+
# pass for both edges sections, so the edges file is never read and written twice.
|
|
1025
|
+
nodes_section: dict[str, Any] = _validate_file(nodes, edge=False, limit=limit)
|
|
1026
|
+
pruned_report: dict[str, Any] = prune_kgx_edges(edges, limit=limit)
|
|
1027
|
+
print(f"biolink-model {pruned_report['biolink_version']}", file=sys.stderr)
|
|
1028
|
+
_print_kgx_section("nodes", nodes_section, nodes)
|
|
1029
|
+
_print_kgx_section("edges", pruned_report["before"], edges)
|
|
1030
|
+
if pruned_report["before"]["missing"]:
|
|
1031
|
+
print("KGX output is not Biolink-compliant.", file=sys.stderr)
|
|
1032
|
+
raise SystemExit(1)
|
|
1033
|
+
malformed_note: str = f" ({pruned_report['malformed']} malformed lines)" if pruned_report["malformed"] else ""
|
|
1034
|
+
print(f"edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} non-compliant records{malformed_note}", file=sys.stderr)
|
|
1035
|
+
logger.info(f"validate-kgx edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} (rewritten={pruned_report['rewritten']})")
|
|
1036
|
+
for problem, count in pruned_report["dropped"]["problems"].items():
|
|
1037
|
+
print(f" {count:>9} {problem}", file=sys.stderr)
|
|
1038
|
+
for example in pruned_report["dropped"]["examples"][:3]:
|
|
1039
|
+
print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
|
|
1040
|
+
_print_kgx_section("edges", pruned_report["after"], edges)
|
|
1041
|
+
nodes_ok: bool = not nodes_section["missing"] and nodes_section["total"] == nodes_section["valid"]
|
|
1042
|
+
if not (nodes_ok and pruned_report["ok"]):
|
|
998
1043
|
print("KGX output is not Biolink-compliant.", file=sys.stderr)
|
|
999
1044
|
raise SystemExit(1)
|
|
1000
1045
|
print("KGX output is Biolink-compliant.", file=sys.stderr)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|