tablassert 19.4.0__tar.gz → 19.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-19.4.0 → tablassert-19.5.1}/PKG-INFO +49 -16
- {tablassert-19.4.0 → tablassert-19.5.1}/README.md +47 -14
- {tablassert-19.4.0 → tablassert-19.5.1}/pyproject.toml +2 -2
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/src/fullmap.rs +5 -5
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/agent.py +15 -7
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/biolink.py +16 -37
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/enums.py +1 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/lib.py +22 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/models.py +8 -2
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/runcache.py +1 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/LICENSE +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/Cargo.lock +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/Cargo.toml +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/examples/count_tables.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/src/json.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/src/lib.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/src/ndjson.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/src/nlp.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/src/uuid.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/tests/build_golden.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/tests/common/mod.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/__init__.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/_lazy.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/cli.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/cli_entry.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/coerce.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/distill.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/distill_reward.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/errors.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/extras.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/fullmap.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/graph_target.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/ingests.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/log.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/net.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/nlp.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/progress.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/qc.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/rig.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/rs.pyi +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/study.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.1}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version: 19.
|
|
3
|
+
Version: 19.5.1
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -22,7 +22,7 @@ Classifier: Framework :: Pydantic :: 2
|
|
|
22
22
|
Classifier: Operating System :: POSIX :: Linux
|
|
23
23
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
24
24
|
Classifier: Environment :: Console
|
|
25
|
-
Requires-Dist: biolink-model>=4.4.
|
|
25
|
+
Requires-Dist: biolink-model>=4.4.5
|
|
26
26
|
Requires-Dist: polars>=2.0.0rc2
|
|
27
27
|
Requires-Dist: pyarrow>=21.0.0
|
|
28
28
|
Requires-Dist: rapidfuzz>=3.14.3
|
|
@@ -77,6 +77,36 @@ KGX-compliant nodes and edges.
|
|
|
77
77
|
**[Full Documentation](https://skyeav.github.io/Tablassert/)**: installation guides, tutorial,
|
|
78
78
|
configuration reference, and API docs.
|
|
79
79
|
|
|
80
|
+
## Statement of need
|
|
81
|
+
|
|
82
|
+
Biomedical knowledge lives in spreadsheets: association tables, assay results, curated gene-disease
|
|
83
|
+
lists. Getting those rows into an NCATS Translator knowledge graph means writing an ingest that maps
|
|
84
|
+
columns to Biolink statements, resolves free text ("TP53", "lung cancer") to standard CURIEs, and
|
|
85
|
+
records provenance, terms of use, and statistical annotations. Today that ingest is Python code
|
|
86
|
+
maintained per source, and entity resolution usually means calling a hosted name-resolution service
|
|
87
|
+
at build time.
|
|
88
|
+
|
|
89
|
+
Tablassert replaces the per-source code with a declarative YAML mapping plus an embedded, offline
|
|
90
|
+
entity-resolution database built from RENCI BABEL exports, so a build is reproducible, auditable,
|
|
91
|
+
and network-free. It is aimed at Translator ingest authors and knowledge-graph data engineers who
|
|
92
|
+
hold tabular biomedical sources, and at bioinformatics groups that want a validated KGX output
|
|
93
|
+
without writing a transform pipeline. The configuration model and the Biolink/KGX contracts it
|
|
94
|
+
emits follow current [`NCATSTranslator/translator-ingests`](https://github.com/NCATSTranslator/translator-ingests)
|
|
95
|
+
practice, and each build can emit the Resource Ingest Guide (RIG) metadata those submissions require.
|
|
96
|
+
|
|
97
|
+
## Getting help
|
|
98
|
+
|
|
99
|
+
- **Questions and bug reports:** open an issue at
|
|
100
|
+
[github.com/SkyeAv/Tablassert/issues](https://github.com/SkyeAv/Tablassert/issues) (use the
|
|
101
|
+
`question` label for usage questions). Include the command you ran, its output, and your
|
|
102
|
+
`tablassert --version`.
|
|
103
|
+
- **Contributing code or docs:** see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, the local quality
|
|
104
|
+
gates, and pull-request expectations. By participating in this project you agree to follow the
|
|
105
|
+
[Code of Conduct](CODE_OF_CONDUCT.md).
|
|
106
|
+
- **Reference:** the [CLI Reference](https://skyeav.github.io/Tablassert/cli/) and
|
|
107
|
+
[configuration guides](https://skyeav.github.io/Tablassert/configuration/graph/) document every
|
|
108
|
+
flag and field.
|
|
109
|
+
|
|
80
110
|
## Quick Start
|
|
81
111
|
|
|
82
112
|
```bash
|
|
@@ -105,14 +135,17 @@ template:
|
|
|
105
135
|
```
|
|
106
136
|
|
|
107
137
|
Wrap it in a graph config (`graph.yaml`) pointing at your fullmap entity-resolution database
|
|
108
|
-
and carrying the required `rig:` metadata for the generated Resource Ingest Guide
|
|
138
|
+
and carrying the required `rig:` metadata for the generated Resource Ingest Guide. Build or
|
|
139
|
+
download that database once first (`tablassert build-fullmap`, a multi-GB download; see the
|
|
140
|
+
[Fullmap guide](https://skyeav.github.io/Tablassert/fullmap/)), which makes `./fullmap` below the
|
|
141
|
+
right path:
|
|
109
142
|
|
|
110
143
|
```yaml
|
|
111
144
|
name: MY_KG
|
|
112
145
|
version: 1.0.0
|
|
113
146
|
tables:
|
|
114
147
|
- ./table.yaml
|
|
115
|
-
fullmap:
|
|
148
|
+
fullmap: ./fullmap
|
|
116
149
|
rig:
|
|
117
150
|
source_info:
|
|
118
151
|
infores_id: infores:my-kg
|
|
@@ -159,7 +192,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
159
192
|
low-confidence mappings
|
|
160
193
|
- **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
|
|
161
194
|
and predicates
|
|
162
|
-
- **Autonomous agent**: `tablassert agent` derives, builds, and refines configs for
|
|
195
|
+
- **Autonomous agent (experimental)**: `tablassert agent` derives, builds, and refines configs for
|
|
196
|
+
whole papers
|
|
163
197
|
- **Performance & reproducibility**: lazy Polars pipelines and a deterministic UV-based
|
|
164
198
|
development environment
|
|
165
199
|
|
|
@@ -169,8 +203,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
169
203
|
pip install tablassert
|
|
170
204
|
```
|
|
171
205
|
|
|
172
|
-
Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API;
|
|
173
|
-
|
|
206
|
+
Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API; `[cli]`
|
|
207
|
+
provides the `tablassert` command. Other extras are opt-in:
|
|
174
208
|
|
|
175
209
|
| Extra | Adds | Install |
|
|
176
210
|
| ----- | ---- | ------- |
|
|
@@ -178,17 +212,16 @@ install `[cli]` to use the `tablassert` command and optional extras for addition
|
|
|
178
212
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
179
213
|
| `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
180
214
|
| `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
181
|
-
| `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
|
|
182
|
-
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
|
|
183
|
-
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
|
|
215
|
+
| `agent` | autonomous agent (smolagents, litellm, article/table context) -- experimental, API may change | `pip install "tablassert[agent]"` |
|
|
216
|
+
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) -- experimental, API may change | `pip install "tablassert[optimize]"` |
|
|
217
|
+
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) -- experimental, API may change | `pip install "tablassert[distill]"` |
|
|
184
218
|
| `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
|
|
185
219
|
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
[Installation guide](https://skyeav.github.io/Tablassert/installation/) for the full matrix and the
|
|
220
|
+
Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure
|
|
221
|
+
names the missing package and the install command that fixes it. QC is opt-in at build time
|
|
222
|
+
(`build-kg --qc`). See the
|
|
223
|
+
[Installation guide](https://skyeav.github.io/Tablassert/installation/) for the full matrix,
|
|
224
|
+
per-command preflight behavior, and the
|
|
192
225
|
[CLI Reference](https://skyeav.github.io/Tablassert/cli/) for every flag.
|
|
193
226
|
|
|
194
227
|
## Entity Resolution API
|
|
@@ -16,6 +16,36 @@ KGX-compliant nodes and edges.
|
|
|
16
16
|
**[Full Documentation](https://skyeav.github.io/Tablassert/)**: installation guides, tutorial,
|
|
17
17
|
configuration reference, and API docs.
|
|
18
18
|
|
|
19
|
+
## Statement of need
|
|
20
|
+
|
|
21
|
+
Biomedical knowledge lives in spreadsheets: association tables, assay results, curated gene-disease
|
|
22
|
+
lists. Getting those rows into an NCATS Translator knowledge graph means writing an ingest that maps
|
|
23
|
+
columns to Biolink statements, resolves free text ("TP53", "lung cancer") to standard CURIEs, and
|
|
24
|
+
records provenance, terms of use, and statistical annotations. Today that ingest is Python code
|
|
25
|
+
maintained per source, and entity resolution usually means calling a hosted name-resolution service
|
|
26
|
+
at build time.
|
|
27
|
+
|
|
28
|
+
Tablassert replaces the per-source code with a declarative YAML mapping plus an embedded, offline
|
|
29
|
+
entity-resolution database built from RENCI BABEL exports, so a build is reproducible, auditable,
|
|
30
|
+
and network-free. It is aimed at Translator ingest authors and knowledge-graph data engineers who
|
|
31
|
+
hold tabular biomedical sources, and at bioinformatics groups that want a validated KGX output
|
|
32
|
+
without writing a transform pipeline. The configuration model and the Biolink/KGX contracts it
|
|
33
|
+
emits follow current [`NCATSTranslator/translator-ingests`](https://github.com/NCATSTranslator/translator-ingests)
|
|
34
|
+
practice, and each build can emit the Resource Ingest Guide (RIG) metadata those submissions require.
|
|
35
|
+
|
|
36
|
+
## Getting help
|
|
37
|
+
|
|
38
|
+
- **Questions and bug reports:** open an issue at
|
|
39
|
+
[github.com/SkyeAv/Tablassert/issues](https://github.com/SkyeAv/Tablassert/issues) (use the
|
|
40
|
+
`question` label for usage questions). Include the command you ran, its output, and your
|
|
41
|
+
`tablassert --version`.
|
|
42
|
+
- **Contributing code or docs:** see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, the local quality
|
|
43
|
+
gates, and pull-request expectations. By participating in this project you agree to follow the
|
|
44
|
+
[Code of Conduct](CODE_OF_CONDUCT.md).
|
|
45
|
+
- **Reference:** the [CLI Reference](https://skyeav.github.io/Tablassert/cli/) and
|
|
46
|
+
[configuration guides](https://skyeav.github.io/Tablassert/configuration/graph/) document every
|
|
47
|
+
flag and field.
|
|
48
|
+
|
|
19
49
|
## Quick Start
|
|
20
50
|
|
|
21
51
|
```bash
|
|
@@ -44,14 +74,17 @@ template:
|
|
|
44
74
|
```
|
|
45
75
|
|
|
46
76
|
Wrap it in a graph config (`graph.yaml`) pointing at your fullmap entity-resolution database
|
|
47
|
-
and carrying the required `rig:` metadata for the generated Resource Ingest Guide
|
|
77
|
+
and carrying the required `rig:` metadata for the generated Resource Ingest Guide. Build or
|
|
78
|
+
download that database once first (`tablassert build-fullmap`, a multi-GB download; see the
|
|
79
|
+
[Fullmap guide](https://skyeav.github.io/Tablassert/fullmap/)), which makes `./fullmap` below the
|
|
80
|
+
right path:
|
|
48
81
|
|
|
49
82
|
```yaml
|
|
50
83
|
name: MY_KG
|
|
51
84
|
version: 1.0.0
|
|
52
85
|
tables:
|
|
53
86
|
- ./table.yaml
|
|
54
|
-
fullmap:
|
|
87
|
+
fullmap: ./fullmap
|
|
55
88
|
rig:
|
|
56
89
|
source_info:
|
|
57
90
|
infores_id: infores:my-kg
|
|
@@ -98,7 +131,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
98
131
|
low-confidence mappings
|
|
99
132
|
- **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
|
|
100
133
|
and predicates
|
|
101
|
-
- **Autonomous agent**: `tablassert agent` derives, builds, and refines configs for
|
|
134
|
+
- **Autonomous agent (experimental)**: `tablassert agent` derives, builds, and refines configs for
|
|
135
|
+
whole papers
|
|
102
136
|
- **Performance & reproducibility**: lazy Polars pipelines and a deterministic UV-based
|
|
103
137
|
development environment
|
|
104
138
|
|
|
@@ -108,8 +142,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
108
142
|
pip install tablassert
|
|
109
143
|
```
|
|
110
144
|
|
|
111
|
-
Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API;
|
|
112
|
-
|
|
145
|
+
Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API; `[cli]`
|
|
146
|
+
provides the `tablassert` command. Other extras are opt-in:
|
|
113
147
|
|
|
114
148
|
| Extra | Adds | Install |
|
|
115
149
|
| ----- | ---- | ------- |
|
|
@@ -117,17 +151,16 @@ install `[cli]` to use the `tablassert` command and optional extras for addition
|
|
|
117
151
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
118
152
|
| `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
119
153
|
| `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
120
|
-
| `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
|
|
121
|
-
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
|
|
122
|
-
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
|
|
154
|
+
| `agent` | autonomous agent (smolagents, litellm, article/table context) -- experimental, API may change | `pip install "tablassert[agent]"` |
|
|
155
|
+
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) -- experimental, API may change | `pip install "tablassert[optimize]"` |
|
|
156
|
+
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) -- experimental, API may change | `pip install "tablassert[distill]"` |
|
|
123
157
|
| `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
|
|
124
158
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
[Installation guide](https://skyeav.github.io/Tablassert/installation/) for the full matrix and the
|
|
159
|
+
Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure
|
|
160
|
+
names the missing package and the install command that fixes it. QC is opt-in at build time
|
|
161
|
+
(`build-kg --qc`). See the
|
|
162
|
+
[Installation guide](https://skyeav.github.io/Tablassert/installation/) for the full matrix,
|
|
163
|
+
per-command preflight behavior, and the
|
|
131
164
|
[CLI Reference](https://skyeav.github.io/Tablassert/cli/) for every flag.
|
|
132
165
|
|
|
133
166
|
## Entity Resolution API
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "19.
|
|
3
|
+
version = "19.5.1"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -47,7 +47,7 @@ classifiers = [
|
|
|
47
47
|
]
|
|
48
48
|
requires-python = ">=3.11"
|
|
49
49
|
dependencies = [
|
|
50
|
-
"biolink-model>=4.4.
|
|
50
|
+
"biolink-model>=4.4.5",
|
|
51
51
|
"polars>=2.0.0rc2",
|
|
52
52
|
"pyarrow>=21.0.0",
|
|
53
53
|
"rapidfuzz>=3.14.3",
|
|
@@ -2950,11 +2950,11 @@ fn extract_validate_rename(
|
|
|
2950
2950
|
archive.display()
|
|
2951
2951
|
))
|
|
2952
2952
|
};
|
|
2953
|
-
let database = open_read_only(&primary).map_err(
|
|
2954
|
-
validate_schema(&database).map_err(
|
|
2955
|
-
let build_id = read_build_id(&database).map_err(
|
|
2956
|
-
let shard_count = shard_count_of(&database).map_err(
|
|
2957
|
-
let recorded_allowlist = read_taxon_allowlist_identity(&database).map_err(
|
|
2953
|
+
let database = open_read_only(&primary).map_err(validation_ctx)?;
|
|
2954
|
+
validate_schema(&database).map_err(validation_ctx)?;
|
|
2955
|
+
let build_id = read_build_id(&database).map_err(validation_ctx)?;
|
|
2956
|
+
let shard_count = shard_count_of(&database).map_err(validation_ctx)?;
|
|
2957
|
+
let recorded_allowlist = read_taxon_allowlist_identity(&database).map_err(validation_ctx)?;
|
|
2958
2958
|
drop(database);
|
|
2959
2959
|
|
|
2960
2960
|
// A published archive built WITHOUT the required filter is not the database
|
|
@@ -115,7 +115,7 @@ def is_lazy() -> bool:
|
|
|
115
115
|
PMC_BUCKET: str = "pmc-oa-opendata"
|
|
116
116
|
PMC_HTTPS_BASE: str = "https://pmc-oa-opendata.s3.amazonaws.com"
|
|
117
117
|
PMC_S3API_BASE: str = "https://pmc-oa-opendata.s3.us-east-1.amazonaws.com"
|
|
118
|
-
TABLE_EXTENSIONS: frozenset[str] = frozenset({".xlsx", ".xls", ".csv", ".tsv"})
|
|
118
|
+
TABLE_EXTENSIONS: frozenset[str] = frozenset({".xlsx", ".xls", ".csv", ".tsv", ".parquet"})
|
|
119
119
|
#: Default minimum number of non-empty data rows for agent table candidates.
|
|
120
120
|
#:
|
|
121
121
|
#: Counts use the same header inference as the production readers, so the header is not included.
|
|
@@ -662,8 +662,9 @@ def _load_table(path: Path, sheet: str | None = None) -> pl.DataFrame:
|
|
|
662
662
|
"""Dispatch a local table file to the right polars reader by suffix.
|
|
663
663
|
|
|
664
664
|
``.csv`` -> ``read_csv``; ``.tsv``/``.txt`` -> ``read_csv(separator="\\t")``;
|
|
665
|
-
``.
|
|
666
|
-
|
|
665
|
+
``.parquet`` -> ``read_parquet`` (typed columns read as-is); ``.xlsx``/``.xls`` ->
|
|
666
|
+
:func:`_read_excel` (``sheet`` selects a worksheet by name; ignored for
|
|
667
|
+
csv/tsv/parquet). Any polars parse failure becomes a clear ``ValueError``; an
|
|
667
668
|
unknown suffix is a ``ValueError`` too (never a silent mis-read).
|
|
668
669
|
"""
|
|
669
670
|
suffix: str = path.suffix.lower()
|
|
@@ -674,6 +675,8 @@ def _load_table(path: Path, sheet: str | None = None) -> pl.DataFrame:
|
|
|
674
675
|
return pl.read_csv(path)
|
|
675
676
|
if suffix in {".tsv", ".txt"}:
|
|
676
677
|
return pl.read_csv(path, separator="\t")
|
|
678
|
+
if suffix == ".parquet":
|
|
679
|
+
return pl.read_parquet(path)
|
|
677
680
|
except Exception as e:
|
|
678
681
|
raise ValueError(f"Could not read table {path}: {e}") from e
|
|
679
682
|
raise ValueError(f"Could not read table {path}: unsupported extension {suffix!r}")
|
|
@@ -685,8 +688,9 @@ def _effective_rows_cached(path: str, mtime_ns: int, size: int, sheet: str | Non
|
|
|
685
688
|
|
|
686
689
|
``path``, ``mtime_ns``, and ``size`` form the cache key so a re-fetched or rewritten file
|
|
687
690
|
cannot reuse a stale count. Delimited files are counted by the parser rather than physical
|
|
688
|
-
lines, which handles quoted embedded newlines correctly. Excel
|
|
689
|
-
:func:`_load_table` and
|
|
691
|
+
lines, which handles quoted embedded newlines correctly. Excel and parquet use the same
|
|
692
|
+
reader as :func:`_load_table` and remove rows that are null in every column (formatted
|
|
693
|
+
blank rows).
|
|
690
694
|
"""
|
|
691
695
|
table_path: Path = Path(path)
|
|
692
696
|
suffix: str = table_path.suffix.lower()
|
|
@@ -694,6 +698,10 @@ def _effective_rows_cached(path: str, mtime_ns: int, size: int, sheet: str | Non
|
|
|
694
698
|
frame: pl.DataFrame = _read_excel(table_path, sheet)
|
|
695
699
|
return frame.filter(~pl.all_horizontal(pl.all().is_null())).height
|
|
696
700
|
|
|
701
|
+
if suffix == ".parquet":
|
|
702
|
+
pq: pl.DataFrame = pl.read_parquet(table_path)
|
|
703
|
+
return pq.filter(~pl.all_horizontal(pl.all().is_null())).height
|
|
704
|
+
|
|
697
705
|
if suffix == ".csv":
|
|
698
706
|
separator: str = ","
|
|
699
707
|
elif suffix in {".tsv", ".txt"}:
|
|
@@ -1233,7 +1241,7 @@ def normalize_agent_table_config(config_yaml: str, *, base_dirs: Sequence[Path]
|
|
|
1233
1241
|
# hand-maintained guess table), so a model default change automatically changes
|
|
1234
1242
|
# what counts as removable. Semantic guards: the ``provenance`` subtree is never
|
|
1235
1243
|
# touched (legal attribution), ``kind`` is never dropped (it discriminates the
|
|
1236
|
-
# ``Excel | Text`` source union), a non-default null such as ``taxon: null``
|
|
1244
|
+
# ``Excel | Text | Parquet`` source union), a non-default null such as ``taxon: null``
|
|
1237
1245
|
# (default 9606) is preserved, and in ``{template, sections}`` configs a section
|
|
1238
1246
|
# entry equal to a model default is only removed when the template cannot change
|
|
1239
1247
|
# the merged result (fastmerge lets section scalars override template values, so
|
|
@@ -1245,7 +1253,7 @@ _COMPACT_ABSENT: object = object()
|
|
|
1245
1253
|
|
|
1246
1254
|
#: Keys compaction never removes and never recurses into: ``provenance`` values are
|
|
1247
1255
|
#: the edge's legal attribution (untouched even when they equal a model default),
|
|
1248
|
-
#: and ``kind`` discriminates the ``Excel | Text`` source union — dropping it could
|
|
1256
|
+
#: and ``kind`` discriminates the ``Excel | Text | Parquet`` source union — dropping it could
|
|
1249
1257
|
#: flip which model a re-parsed source validates as.
|
|
1250
1258
|
_COMPACT_UNTOUCHED_KEYS: frozenset[str] = frozenset({"provenance", "kind"})
|
|
1251
1259
|
|
|
@@ -591,28 +591,8 @@ release from making ``species_context_qualifier`` silently emittable again.
|
|
|
591
591
|
|
|
592
592
|
|
|
593
593
|
CLASS_FIELD_OVERRIDES: dict[str, frozenset[str]] = {
|
|
594
|
-
"EntityToDiseaseAssociation": frozenset(
|
|
595
|
-
|
|
596
|
-
"anatomical_context_qualifier",
|
|
597
|
-
"disease_context_qualifier",
|
|
598
|
-
"frequency_qualifier",
|
|
599
|
-
"population_context_qualifier",
|
|
600
|
-
"regulatory_approvals",
|
|
601
|
-
"sex_qualifier",
|
|
602
|
-
"temporal_context_qualifier",
|
|
603
|
-
}
|
|
604
|
-
),
|
|
605
|
-
"EntityToPhenotypicFeatureAssociation": frozenset(
|
|
606
|
-
{
|
|
607
|
-
"anatomical_context_qualifier",
|
|
608
|
-
"disease_context_qualifier",
|
|
609
|
-
"frequency_qualifier",
|
|
610
|
-
"population_context_qualifier",
|
|
611
|
-
"regulatory_approvals",
|
|
612
|
-
"sex_qualifier",
|
|
613
|
-
"temporal_context_qualifier",
|
|
614
|
-
}
|
|
615
|
-
),
|
|
594
|
+
"EntityToDiseaseAssociation": frozenset({"population_context_qualifier", "sex_qualifier", "temporal_context_qualifier"}),
|
|
595
|
+
"EntityToPhenotypicFeatureAssociation": frozenset({"population_context_qualifier", "temporal_context_qualifier"}),
|
|
616
596
|
}
|
|
617
597
|
"""Per-class grants of edge fields the resolved association class does not declare.
|
|
618
598
|
|
|
@@ -620,22 +600,21 @@ Keys are bare association class names (``association_class(cat).__name__``), val
|
|
|
620
600
|
the slots ``lib.prune_to_class`` keeps on rows resolved to that class even though the
|
|
621
601
|
installed model attaches them elsewhere.
|
|
622
602
|
|
|
623
|
-
The motivating case is a DAKP contraindication edge
|
|
624
|
-
|
|
625
|
-
``
|
|
626
|
-
``
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
``
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
``
|
|
634
|
-
|
|
635
|
-
a grant could never validate). Tablassert
|
|
636
|
-
deliberately emits the granted fields on the pinned classes ahead of the pinned model
|
|
603
|
+
The motivating case is a DAKP contraindication edge carrying the sparse qualifier
|
|
604
|
+
stack ahead of the pinned model: ``population_context_qualifier`` and
|
|
605
|
+
``temporal_context_qualifier`` are satisfiable qualifier slots the pinned classes do
|
|
606
|
+
not declare, and ``sex_qualifier`` is declared only on the phenotypic-feature side of
|
|
607
|
+
the family -- so without a grant ``prune_to_class`` would null them off the edge.
|
|
608
|
+
Upstream Biolink has absorbed most of the original stack (the biolink-model 4.4.5
|
|
609
|
+
mixin consolidation attached ``disease_context_qualifier``,
|
|
610
|
+
``anatomical_context_qualifier``, and ``frequency_qualifier`` to the whole
|
|
611
|
+
disease/phenotype family, renamed ``FDA_regulatory_approvals`` to the canonical
|
|
612
|
+
``regulatory_approvals`` on the pinned classes, and added ``sex_qualifier`` to
|
|
613
|
+
``EntityToPhenotypicFeatureAssociation``); the tripwire removed those grants the day
|
|
614
|
+
the release landed, and the remaining ones stay deliberately ahead of the model
|
|
637
615
|
(pending an upstream Biolink widening). ``_validation_record`` strips granted fields
|
|
638
|
-
before record validation so the deliberate gap is not reported as
|
|
616
|
+
before record validation so the deliberate gap is not reported as
|
|
617
|
+
``extra_forbidden``.
|
|
639
618
|
|
|
640
619
|
A tripwire test asserts every granted field is still absent from its class: the moment
|
|
641
620
|
a biolink-model release attaches the slot, the suite fails and the stale grant is
|
|
@@ -953,6 +953,26 @@ def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame:
|
|
|
953
953
|
return df.lazy()
|
|
954
954
|
|
|
955
955
|
|
|
956
|
+
def parquet(p: Path) -> pl.LazyFrame:
|
|
957
|
+
"""Read a parquet source as a LazyFrame with positional ``column_<n>`` names.
|
|
958
|
+
|
|
959
|
+
Args:
|
|
960
|
+
p: Path to the parquet file.
|
|
961
|
+
|
|
962
|
+
Returns:
|
|
963
|
+
LazyFrame whose columns are renamed to ``column_1..`` in schema order.
|
|
964
|
+
|
|
965
|
+
Notes:
|
|
966
|
+
Parquet carries its own column names, but the config contract addresses columns
|
|
967
|
+
positionally (Excel-style ``A-ZZ`` encoding letters resolve to ``column_<n>``), so
|
|
968
|
+
the reader renames whatever names the file carries to ``column_1..`` -- the same
|
|
969
|
+
convention the headerless ``csv``/``excel`` readers produce.
|
|
970
|
+
"""
|
|
971
|
+
lf: pl.LazyFrame = pl.scan_parquet(source=p)
|
|
972
|
+
names: list[str] = list(lf.collect_schema().names())
|
|
973
|
+
return lf.rename(dict(zip(names, (f"column_{i + 1}" for i in range(len(names))), strict=True)))
|
|
974
|
+
|
|
975
|
+
|
|
956
976
|
def crop(lf: pl.LazyFrame, row_slice: list[NonNegativeInt | Literal[Tokens.AUTO]]) -> pl.LazyFrame:
|
|
957
977
|
"""Take a contiguous slice from a LazyFrame.
|
|
958
978
|
|
|
@@ -1296,6 +1316,7 @@ class Tcode(Section):
|
|
|
1296
1316
|
return [
|
|
1297
1317
|
(csv, (self.source.local, self.source.delimiter)) if self.source.kind == Files.TEXT else None, # pyright: ignore
|
|
1298
1318
|
(excel, (self.source.local, self.source.sheet)) if self.source.kind == Files.EXCEL else None, # pyright: ignore
|
|
1319
|
+
(parquet, (self.source.local,)) if self.source.kind == Files.PARQUET else None, # pyright: ignore
|
|
1299
1320
|
(idx, ()),
|
|
1300
1321
|
(crop, (self.source.row_slice,)) if self.source.row_slice else None,
|
|
1301
1322
|
(pick, (self.source.rows,)) if self.source.rows else None,
|
|
@@ -1507,6 +1528,7 @@ PHASE_OF: dict[Callable, str] = {
|
|
|
1507
1528
|
# Maps each Tcode op callable to a short lowercase phase label for progress UX.
|
|
1508
1529
|
csv: "load",
|
|
1509
1530
|
excel: "load",
|
|
1531
|
+
parquet: "load",
|
|
1510
1532
|
idx: "load",
|
|
1511
1533
|
crop: "filter",
|
|
1512
1534
|
pick: "filter",
|
|
@@ -67,7 +67,7 @@ def _shown(name: str, target: str) -> str:
|
|
|
67
67
|
return f"`{name}` (coerced to `{target}`)" if target != name else f"`{name}`"
|
|
68
68
|
|
|
69
69
|
|
|
70
|
-
def _section_source_label(source: Excel | Text) -> str:
|
|
70
|
+
def _section_source_label(source: Excel | Text | Parquet) -> str:
|
|
71
71
|
"""Render a section's source for a diagnostic message: the local path, plus the sheet for Excel.
|
|
72
72
|
|
|
73
73
|
Args:
|
|
@@ -174,6 +174,12 @@ class Text(BaseSource):
|
|
|
174
174
|
delimiter: str | None = Field(",", description="Field delimiter for headerless text/CSV scanning.", examples=[",", "\t", "|"])
|
|
175
175
|
|
|
176
176
|
|
|
177
|
+
# Parquet is self-describing (typed columns, real headers, no sheets or delimiters), so unlike
|
|
178
|
+
# `Text`/`Excel` it adds no format-specific fields beyond `BaseSource`.
|
|
179
|
+
class Parquet(BaseSource):
|
|
180
|
+
kind: Literal[Files.PARQUET] = Field(Files.PARQUET, description="Source kind; must be 'parquet'.")
|
|
181
|
+
|
|
182
|
+
|
|
177
183
|
class Regex(TablaBase):
|
|
178
184
|
pattern: int | float | str = Field(..., description="Regex pattern passed to string replacement.", examples=["\\s+", "\\.$"])
|
|
179
185
|
|
|
@@ -791,7 +797,7 @@ class Annotation(Encoding):
|
|
|
791
797
|
class Section(TablaBase):
|
|
792
798
|
"""Pydantic section model and coercion target for a single table configuration."""
|
|
793
799
|
|
|
794
|
-
source: Excel | Text = Field(..., description="Input source definition for reading tabular rows.")
|
|
800
|
+
source: Excel | Text | Parquet = Field(..., description="Input source definition for reading tabular rows.")
|
|
795
801
|
statement: Statement = Field(..., description="Subject-object statement mapping for this section.")
|
|
796
802
|
provenance: Provenance = Field(..., description="Provenance metadata applied to all produced edges.")
|
|
797
803
|
annotations: list[Annotation] | None = Field(None, description="Optional extra encoded columns added to each row.")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|