tablassert 19.2.0__tar.gz → 19.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-19.2.0 → tablassert-19.3.0}/PKG-INFO +14 -16
- {tablassert-19.2.0 → tablassert-19.3.0}/README.md +9 -12
- {tablassert-19.2.0 → tablassert-19.3.0}/pyproject.toml +5 -4
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/fullmap.py +4 -1
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/lib.py +30 -4
- {tablassert-19.2.0 → tablassert-19.3.0}/LICENSE +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/Cargo.lock +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/Cargo.toml +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/src/fullmap.rs +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/src/json.rs +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/src/lib.rs +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/src/ndjson.rs +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/src/nlp.rs +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/src/uuid.rs +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/__init__.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/agent.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/biolink.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/cli.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/cli_entry.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/coerce.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/distill.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/distill_reward.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/enums.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/errors.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/extras.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/ingests.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/log.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/models.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/net.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/nlp.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/progress.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/qc.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/rig.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/rs.pyi +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/runcache.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/study.py +0 -0
- {tablassert-19.2.0 → tablassert-19.3.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version: 19.
|
|
3
|
+
Version: 19.3.0
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -23,11 +23,12 @@ Classifier: Operating System :: POSIX :: Linux
|
|
|
23
23
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
24
24
|
Classifier: Environment :: Console
|
|
25
25
|
Requires-Dist: biolink-model>=4.4.4
|
|
26
|
-
Requires-Dist: polars>=
|
|
26
|
+
Requires-Dist: polars>=2.0.0rc2
|
|
27
|
+
Requires-Dist: pyarrow>=21.0.0
|
|
27
28
|
Requires-Dist: rapidfuzz>=3.14.3
|
|
28
29
|
Requires-Dist: pydantic>=2.12.5
|
|
29
30
|
Requires-Dist: pyyaml>=6.0.3
|
|
30
|
-
Requires-Dist: fastexcel>=0.
|
|
31
|
+
Requires-Dist: fastexcel>=0.21.0
|
|
31
32
|
Requires-Dist: smolagents>=1.26.0 ; extra == 'agent'
|
|
32
33
|
Requires-Dist: litellm>=1.93.0 ; extra == 'agent'
|
|
33
34
|
Requires-Dist: aria2==0.0.1b0 ; extra == 'aria2'
|
|
@@ -38,7 +39,7 @@ Requires-Dist: loguru>=0.7.3 ; extra == 'log'
|
|
|
38
39
|
Requires-Dist: dspy>=3.2.1 ; extra == 'optimize'
|
|
39
40
|
Requires-Dist: scikit-learn>=1.8.0 ; extra == 'qc'
|
|
40
41
|
Requires-Dist: sentence-transformers>=5.3.0 ; extra == 'qc'
|
|
41
|
-
Requires-Dist: polars[rtcompat]>=
|
|
42
|
+
Requires-Dist: polars[rtcompat]>=2.0.0rc2 ; extra == 'rt'
|
|
42
43
|
Provides-Extra: agent
|
|
43
44
|
Provides-Extra: aria2
|
|
44
45
|
Provides-Extra: cli
|
|
@@ -60,13 +61,10 @@ Project-URL: Source, https://github.com/SkyeAv/Tablassert
|
|
|
60
61
|
|
|
61
62
|
# Tablassert
|
|
62
63
|
|
|
63
|
-
[](https://opensource.org/licenses/Apache-2.0)
|
|
64
|
-
[](https://www.python.org/downloads/)
|
|
65
64
|
[](https://badge.fury.io/py/tablassert)
|
|
66
|
-
[](https://pepy.tech/project/tablassert)
|
|
67
65
|
[](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
|
|
68
66
|
[](https://skyeav.github.io/Tablassert/)
|
|
69
|
-
[](https://opensource.org/licenses/Apache-2.0)
|
|
70
68
|
|
|
71
69
|
> Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON,
|
|
72
70
|
> declaratively, with entity resolution built in and optional quality control.
|
|
@@ -82,7 +80,7 @@ configuration reference, and API docs.
|
|
|
82
80
|
## Quick Start
|
|
83
81
|
|
|
84
82
|
```bash
|
|
85
|
-
pip install tablassert
|
|
83
|
+
pip install "tablassert[cli]"
|
|
86
84
|
```
|
|
87
85
|
|
|
88
86
|
Given a CSV of gene-disease associations with p-values and sample sizes, declare the mapping in a
|
|
@@ -147,7 +145,7 @@ Output is one JSON object per line: nodes with Biolink categories, edges with an
|
|
|
147
145
|
```
|
|
148
146
|
|
|
149
147
|
```json
|
|
150
|
-
{"subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}}}
|
|
148
|
+
{"id":"2cfea591-0f8f-33af-a7df-03da531d3359","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
|
|
151
149
|
```
|
|
152
150
|
|
|
153
151
|
See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full walkthrough.
|
|
@@ -157,7 +155,7 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
157
155
|
- **Declarative YAML configuration**: define data transformations without writing code
|
|
158
156
|
- **Built-in entity resolution**: map free text to genes, diseases, and chemicals with standard
|
|
159
157
|
CURIEs, taxonomic filtering, and provenance, backed by an embedded redb database
|
|
160
|
-
- **Optional quality control**: a four-stage audit (exact
|
|
158
|
+
- **Optional quality control**: a four-stage audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) flags
|
|
161
159
|
low-confidence mappings
|
|
162
160
|
- **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
|
|
163
161
|
and predicates
|
|
@@ -179,7 +177,7 @@ install `[cli]` to use the `tablassert` command and optional extras for addition
|
|
|
179
177
|
| `cli` | `tablassert` command and rich terminal progress | `pip install "tablassert[cli]"` |
|
|
180
178
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
181
179
|
| `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
182
|
-
| `qc` | four-stage QC audit (exact
|
|
180
|
+
| `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
183
181
|
| `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
|
|
184
182
|
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
|
|
185
183
|
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
|
|
@@ -216,11 +214,11 @@ no LazyFrame setup or NLP preprocessing required. See the
|
|
|
216
214
|
|
|
217
215
|
- **[Installation](https://skyeav.github.io/Tablassert/installation/)**: install methods, extras, and development setup
|
|
218
216
|
- **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
|
|
219
|
-
- **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
|
|
220
217
|
- **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
|
|
218
|
+
- **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
|
|
221
219
|
- **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
|
|
222
|
-
- **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
|
|
223
220
|
- **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
|
|
221
|
+
- **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
|
|
224
222
|
- **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
|
|
225
223
|
- **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
|
|
226
224
|
- **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
|
|
@@ -241,8 +239,8 @@ request guidelines.
|
|
|
241
239
|
If you use Tablassert, please cite it as described in [CITATION.cff](CITATION.cff). The approach is
|
|
242
240
|
described in:
|
|
243
241
|
|
|
244
|
-
> Skye Lane Goetz,
|
|
245
|
-
> and host health through knowledge graphs
|
|
242
|
+
> Skye Lane Goetz, Amy K. Glen, and Gwênlyn Glusman. "MicrobiomeKG: bridging microbiome research
|
|
243
|
+
> and host health through knowledge graphs." *Frontiers in Systems Biology* 5 (2025).
|
|
246
244
|
> [doi:10.3389/fsysb.2025.1544432](https://doi.org/10.3389/fsysb.2025.1544432)
|
|
247
245
|
|
|
248
246
|
## License
|
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
# Tablassert
|
|
2
2
|
|
|
3
|
-
[](https://opensource.org/licenses/Apache-2.0)
|
|
4
|
-
[](https://www.python.org/downloads/)
|
|
5
3
|
[](https://badge.fury.io/py/tablassert)
|
|
6
|
-
[](https://pepy.tech/project/tablassert)
|
|
7
4
|
[](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
|
|
8
5
|
[](https://skyeav.github.io/Tablassert/)
|
|
9
|
-
[](https://opensource.org/licenses/Apache-2.0)
|
|
10
7
|
|
|
11
8
|
> Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON,
|
|
12
9
|
> declaratively, with entity resolution built in and optional quality control.
|
|
@@ -22,7 +19,7 @@ configuration reference, and API docs.
|
|
|
22
19
|
## Quick Start
|
|
23
20
|
|
|
24
21
|
```bash
|
|
25
|
-
pip install tablassert
|
|
22
|
+
pip install "tablassert[cli]"
|
|
26
23
|
```
|
|
27
24
|
|
|
28
25
|
Given a CSV of gene-disease associations with p-values and sample sizes, declare the mapping in a
|
|
@@ -87,7 +84,7 @@ Output is one JSON object per line: nodes with Biolink categories, edges with an
|
|
|
87
84
|
```
|
|
88
85
|
|
|
89
86
|
```json
|
|
90
|
-
{"subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}}}
|
|
87
|
+
{"id":"2cfea591-0f8f-33af-a7df-03da531d3359","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
|
|
91
88
|
```
|
|
92
89
|
|
|
93
90
|
See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full walkthrough.
|
|
@@ -97,7 +94,7 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
97
94
|
- **Declarative YAML configuration**: define data transformations without writing code
|
|
98
95
|
- **Built-in entity resolution**: map free text to genes, diseases, and chemicals with standard
|
|
99
96
|
CURIEs, taxonomic filtering, and provenance, backed by an embedded redb database
|
|
100
|
-
- **Optional quality control**: a four-stage audit (exact
|
|
97
|
+
- **Optional quality control**: a four-stage audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) flags
|
|
101
98
|
low-confidence mappings
|
|
102
99
|
- **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
|
|
103
100
|
and predicates
|
|
@@ -119,7 +116,7 @@ install `[cli]` to use the `tablassert` command and optional extras for addition
|
|
|
119
116
|
| `cli` | `tablassert` command and rich terminal progress | `pip install "tablassert[cli]"` |
|
|
120
117
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
121
118
|
| `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
122
|
-
| `qc` | four-stage QC audit (exact
|
|
119
|
+
| `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
123
120
|
| `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
|
|
124
121
|
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
|
|
125
122
|
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
|
|
@@ -156,11 +153,11 @@ no LazyFrame setup or NLP preprocessing required. See the
|
|
|
156
153
|
|
|
157
154
|
- **[Installation](https://skyeav.github.io/Tablassert/installation/)**: install methods, extras, and development setup
|
|
158
155
|
- **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
|
|
159
|
-
- **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
|
|
160
156
|
- **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
|
|
157
|
+
- **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
|
|
161
158
|
- **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
|
|
162
|
-
- **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
|
|
163
159
|
- **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
|
|
160
|
+
- **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
|
|
164
161
|
- **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
|
|
165
162
|
- **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
|
|
166
163
|
- **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
|
|
@@ -181,8 +178,8 @@ request guidelines.
|
|
|
181
178
|
If you use Tablassert, please cite it as described in [CITATION.cff](CITATION.cff). The approach is
|
|
182
179
|
described in:
|
|
183
180
|
|
|
184
|
-
> Skye Lane Goetz,
|
|
185
|
-
> and host health through knowledge graphs
|
|
181
|
+
> Skye Lane Goetz, Amy K. Glen, and Gwênlyn Glusman. "MicrobiomeKG: bridging microbiome research
|
|
182
|
+
> and host health through knowledge graphs." *Frontiers in Systems Biology* 5 (2025).
|
|
186
183
|
> [doi:10.3389/fsysb.2025.1544432](https://doi.org/10.3389/fsysb.2025.1544432)
|
|
187
184
|
|
|
188
185
|
## License
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "19.
|
|
3
|
+
version = "19.3.0"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -48,11 +48,12 @@ classifiers = [
|
|
|
48
48
|
requires-python = ">=3.11"
|
|
49
49
|
dependencies = [
|
|
50
50
|
"biolink-model>=4.4.4",
|
|
51
|
-
"polars>=
|
|
51
|
+
"polars>=2.0.0rc2",
|
|
52
|
+
"pyarrow>=21.0.0",
|
|
52
53
|
"rapidfuzz>=3.14.3",
|
|
53
54
|
"pydantic>=2.12.5",
|
|
54
55
|
"pyyaml>=6.0.3",
|
|
55
|
-
"fastexcel>=0.
|
|
56
|
+
"fastexcel>=0.21.0",
|
|
56
57
|
]
|
|
57
58
|
|
|
58
59
|
[project.urls]
|
|
@@ -81,7 +82,7 @@ cli = [
|
|
|
81
82
|
"rich>=13.0.0",
|
|
82
83
|
]
|
|
83
84
|
rt = [
|
|
84
|
-
"polars[rtcompat]>=
|
|
85
|
+
"polars[rtcompat]>=2.0.0rc2",
|
|
85
86
|
]
|
|
86
87
|
# PyPI distribution "aria2" exposes import module "aria2c" with a bundled aria2c binary.
|
|
87
88
|
aria2 = [
|
|
@@ -292,7 +292,10 @@ def distinct(lf: pl.LazyFrame, l1: str, l2: str, col: str = "term") -> pl.LazyFr
|
|
|
292
292
|
t2: pl.LazyFrame = lf.select(pl.col(l2).alias(col)).unique()
|
|
293
293
|
t2 = t2.with_columns(pl.lit(2).alias("nlp_level"))
|
|
294
294
|
|
|
295
|
-
|
|
295
|
+
# Level-1 wins per term. Group-by min instead of concat + unique(keep="first"):
|
|
296
|
+
# the streaming engine (polars 2.0 default) does not preserve row order for
|
|
297
|
+
# concat/unique, so first-seen tier selection would be scheduling-dependent.
|
|
298
|
+
terms: pl.LazyFrame = pl.concat([t1, t2]).group_by(col).agg(pl.col("nlp_level").min())
|
|
296
299
|
|
|
297
300
|
bad: str = r"^\d+$|^(none|nan|na|null|unknown|not applicable|p_value|variable|result|exposure|expression|symbol)$|^$"
|
|
298
301
|
return terms.filter(~pl.col(col).str.contains(bad))
|
|
@@ -892,6 +892,16 @@ def idx(lf: pl.LazyFrame, col: str = "extracted_from_row_number") -> pl.LazyFram
|
|
|
892
892
|
return lf.with_row_index(col, offset=1)
|
|
893
893
|
|
|
894
894
|
|
|
895
|
+
def _polars_auto_column_names(names: list[str]) -> bool:
|
|
896
|
+
"""Return True when ``names`` is exactly polars' headerless auto-naming scheme.
|
|
897
|
+
|
|
898
|
+
Polars 2.0 autogenerates headerless column names as ``column_0``.. ``column_{n-1}``
|
|
899
|
+
(1.x used ``column_1``.. ``column_n``). Matching the full consecutive run keeps the
|
|
900
|
+
remap below strictly scoped to machine-generated names, never user data.
|
|
901
|
+
"""
|
|
902
|
+
return bool(names) and all(name == f"column_{i}" for i, name in enumerate(names))
|
|
903
|
+
|
|
904
|
+
|
|
895
905
|
def csv(p: Path, sep: str) -> pl.LazyFrame:
|
|
896
906
|
"""Read a CSV or TSV source as a LazyFrame.
|
|
897
907
|
|
|
@@ -901,8 +911,17 @@ def csv(p: Path, sep: str) -> pl.LazyFrame:
|
|
|
901
911
|
|
|
902
912
|
Returns:
|
|
903
913
|
LazyFrame over the file (no header inference; all columns raw).
|
|
914
|
+
|
|
915
|
+
Notes:
|
|
916
|
+
Headerless auto column names are remapped to the 1-based ``column_1``..
|
|
917
|
+
convention so existing table configs keep referencing the same names as
|
|
918
|
+
under polars 1.x (output parity).
|
|
904
919
|
"""
|
|
905
|
-
|
|
920
|
+
lf: pl.LazyFrame = pl.scan_csv(source=p, separator=sep, has_header=False, infer_schema_length=None, truncate_ragged_lines=True)
|
|
921
|
+
schema_names: list[str] = list(lf.collect_schema().names())
|
|
922
|
+
if _polars_auto_column_names(schema_names):
|
|
923
|
+
lf = lf.rename(dict(zip(schema_names, (f"column_{i + 1}" for i in range(len(schema_names))), strict=True)))
|
|
924
|
+
return lf
|
|
906
925
|
|
|
907
926
|
|
|
908
927
|
def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame:
|
|
@@ -915,6 +934,11 @@ def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame:
|
|
|
915
934
|
|
|
916
935
|
Returns:
|
|
917
936
|
LazyFrame over the sheet contents (no header inference).
|
|
937
|
+
|
|
938
|
+
Notes:
|
|
939
|
+
Headerless auto column names are remapped to the 1-based ``column_1``..
|
|
940
|
+
convention so existing table configs keep referencing the same names as
|
|
941
|
+
under polars 1.x (output parity).
|
|
918
942
|
"""
|
|
919
943
|
df: pl.DataFrame = pl.read_excel(
|
|
920
944
|
source=p,
|
|
@@ -923,6 +947,8 @@ def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame:
|
|
|
923
947
|
has_header=False,
|
|
924
948
|
infer_schema_length=None,
|
|
925
949
|
)
|
|
950
|
+
if _polars_auto_column_names(df.columns):
|
|
951
|
+
df = df.rename(dict(zip(df.columns, (f"column_{i + 1}" for i in range(len(df.columns))), strict=True)))
|
|
926
952
|
return df.lazy()
|
|
927
953
|
|
|
928
954
|
|
|
@@ -1624,7 +1650,7 @@ def normalize(edges: pl.LazyFrame, col: str, names: list[str] | None = None, inf
|
|
|
1624
1650
|
available: list[str] = edges.collect_schema().names()
|
|
1625
1651
|
# `<col>_taxon_label` is not produced by every fullmap revision.
|
|
1626
1652
|
pairs: list[tuple[str, str]] = [(c, n) for c, n in zip(cols, names, strict=True) if c in available]
|
|
1627
|
-
nodes: pl.LazyFrame = edges.select([c for c, _ in pairs]).unique().rename(dict(pairs))
|
|
1653
|
+
nodes: pl.LazyFrame = edges.select([c for c, _ in pairs]).unique(maintain_order=True).rename(dict(pairs))
|
|
1628
1654
|
# Ensures category has biolink: prefix.
|
|
1629
1655
|
nodes = nodes.with_columns(
|
|
1630
1656
|
pl.when(pl.col("category").str.starts_with("biolink:"))
|
|
@@ -1917,7 +1943,7 @@ def _write_ndjson(
|
|
|
1917
1943
|
on_phase("write-nodes")
|
|
1918
1944
|
with nodes_tmp.open("a", encoding="utf-8") as f:
|
|
1919
1945
|
for subnode in subnodes:
|
|
1920
|
-
eagernode: pl.DataFrame = subnode.collect().unique()
|
|
1946
|
+
eagernode: pl.DataFrame = subnode.collect().unique(maintain_order=True)
|
|
1921
1947
|
eagernode.write_ndjson(f)
|
|
1922
1948
|
|
|
1923
1949
|
# Phase: write-edges.
|
|
@@ -1925,7 +1951,7 @@ def _write_ndjson(
|
|
|
1925
1951
|
on_phase("write-edges")
|
|
1926
1952
|
with edges_tmp.open("a", encoding="utf-8") as f:
|
|
1927
1953
|
for subedge in subedges:
|
|
1928
|
-
eageredge: pl.DataFrame = subedge.collect().unique()
|
|
1954
|
+
eageredge: pl.DataFrame = subedge.collect().unique(maintain_order=True)
|
|
1929
1955
|
eageredge.write_ndjson(f)
|
|
1930
1956
|
|
|
1931
1957
|
# Phase: dedup.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|