tablassert 19.1.0__tar.gz → 19.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-19.1.0 → tablassert-19.3.0}/PKG-INFO +14 -16
- {tablassert-19.1.0 → tablassert-19.3.0}/README.md +9 -12
- {tablassert-19.1.0 → tablassert-19.3.0}/pyproject.toml +5 -4
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/src/ndjson.rs +132 -32
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/cli.py +4 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/fullmap.py +4 -1
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/lib.py +58 -5
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/models.py +4 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/qc.py +37 -8
- {tablassert-19.1.0 → tablassert-19.3.0}/LICENSE +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/Cargo.lock +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/Cargo.toml +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/src/fullmap.rs +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/src/json.rs +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/src/lib.rs +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/src/nlp.rs +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/src/uuid.rs +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/__init__.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/agent.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/biolink.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/cli_entry.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/coerce.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/distill.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/distill_reward.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/enums.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/errors.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/extras.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/ingests.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/log.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/net.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/nlp.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/progress.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/rig.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/rs.pyi +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/runcache.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/study.py +0 -0
- {tablassert-19.1.0 → tablassert-19.3.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version: 19.
|
|
3
|
+
Version: 19.3.0
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -23,11 +23,12 @@ Classifier: Operating System :: POSIX :: Linux
|
|
|
23
23
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
24
24
|
Classifier: Environment :: Console
|
|
25
25
|
Requires-Dist: biolink-model>=4.4.4
|
|
26
|
-
Requires-Dist: polars>=
|
|
26
|
+
Requires-Dist: polars>=2.0.0rc2
|
|
27
|
+
Requires-Dist: pyarrow>=21.0.0
|
|
27
28
|
Requires-Dist: rapidfuzz>=3.14.3
|
|
28
29
|
Requires-Dist: pydantic>=2.12.5
|
|
29
30
|
Requires-Dist: pyyaml>=6.0.3
|
|
30
|
-
Requires-Dist: fastexcel>=0.
|
|
31
|
+
Requires-Dist: fastexcel>=0.21.0
|
|
31
32
|
Requires-Dist: smolagents>=1.26.0 ; extra == 'agent'
|
|
32
33
|
Requires-Dist: litellm>=1.93.0 ; extra == 'agent'
|
|
33
34
|
Requires-Dist: aria2==0.0.1b0 ; extra == 'aria2'
|
|
@@ -38,7 +39,7 @@ Requires-Dist: loguru>=0.7.3 ; extra == 'log'
|
|
|
38
39
|
Requires-Dist: dspy>=3.2.1 ; extra == 'optimize'
|
|
39
40
|
Requires-Dist: scikit-learn>=1.8.0 ; extra == 'qc'
|
|
40
41
|
Requires-Dist: sentence-transformers>=5.3.0 ; extra == 'qc'
|
|
41
|
-
Requires-Dist: polars[rtcompat]>=
|
|
42
|
+
Requires-Dist: polars[rtcompat]>=2.0.0rc2 ; extra == 'rt'
|
|
42
43
|
Provides-Extra: agent
|
|
43
44
|
Provides-Extra: aria2
|
|
44
45
|
Provides-Extra: cli
|
|
@@ -60,13 +61,10 @@ Project-URL: Source, https://github.com/SkyeAv/Tablassert
|
|
|
60
61
|
|
|
61
62
|
# Tablassert
|
|
62
63
|
|
|
63
|
-
[](https://opensource.org/licenses/Apache-2.0)
|
|
64
|
-
[](https://www.python.org/downloads/)
|
|
65
64
|
[](https://badge.fury.io/py/tablassert)
|
|
66
|
-
[](https://pepy.tech/project/tablassert)
|
|
67
65
|
[](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
|
|
68
66
|
[](https://skyeav.github.io/Tablassert/)
|
|
69
|
-
[](https://opensource.org/licenses/Apache-2.0)
|
|
70
68
|
|
|
71
69
|
> Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON,
|
|
72
70
|
> declaratively, with entity resolution built in and optional quality control.
|
|
@@ -82,7 +80,7 @@ configuration reference, and API docs.
|
|
|
82
80
|
## Quick Start
|
|
83
81
|
|
|
84
82
|
```bash
|
|
85
|
-
pip install tablassert
|
|
83
|
+
pip install "tablassert[cli]"
|
|
86
84
|
```
|
|
87
85
|
|
|
88
86
|
Given a CSV of gene-disease associations with p-values and sample sizes, declare the mapping in a
|
|
@@ -147,7 +145,7 @@ Output is one JSON object per line: nodes with Biolink categories, edges with an
|
|
|
147
145
|
```
|
|
148
146
|
|
|
149
147
|
```json
|
|
150
|
-
{"subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}}}
|
|
148
|
+
{"id":"2cfea591-0f8f-33af-a7df-03da531d3359","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
|
|
151
149
|
```
|
|
152
150
|
|
|
153
151
|
See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full walkthrough.
|
|
@@ -157,7 +155,7 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
157
155
|
- **Declarative YAML configuration**: define data transformations without writing code
|
|
158
156
|
- **Built-in entity resolution**: map free text to genes, diseases, and chemicals with standard
|
|
159
157
|
CURIEs, taxonomic filtering, and provenance, backed by an embedded redb database
|
|
160
|
-
- **Optional quality control**: a four-stage audit (exact
|
|
158
|
+
- **Optional quality control**: a four-stage audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) flags
|
|
161
159
|
low-confidence mappings
|
|
162
160
|
- **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
|
|
163
161
|
and predicates
|
|
@@ -179,7 +177,7 @@ install `[cli]` to use the `tablassert` command and optional extras for addition
|
|
|
179
177
|
| `cli` | `tablassert` command and rich terminal progress | `pip install "tablassert[cli]"` |
|
|
180
178
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
181
179
|
| `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
182
|
-
| `qc` | four-stage QC audit (exact
|
|
180
|
+
| `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
183
181
|
| `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
|
|
184
182
|
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
|
|
185
183
|
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
|
|
@@ -216,11 +214,11 @@ no LazyFrame setup or NLP preprocessing required. See the
|
|
|
216
214
|
|
|
217
215
|
- **[Installation](https://skyeav.github.io/Tablassert/installation/)**: install methods, extras, and development setup
|
|
218
216
|
- **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
|
|
219
|
-
- **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
|
|
220
217
|
- **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
|
|
218
|
+
- **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
|
|
221
219
|
- **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
|
|
222
|
-
- **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
|
|
223
220
|
- **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
|
|
221
|
+
- **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
|
|
224
222
|
- **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
|
|
225
223
|
- **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
|
|
226
224
|
- **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
|
|
@@ -241,8 +239,8 @@ request guidelines.
|
|
|
241
239
|
If you use Tablassert, please cite it as described in [CITATION.cff](CITATION.cff). The approach is
|
|
242
240
|
described in:
|
|
243
241
|
|
|
244
|
-
> Skye Lane Goetz,
|
|
245
|
-
> and host health through knowledge graphs
|
|
242
|
+
> Skye Lane Goetz, Amy K. Glen, and Gwênlyn Glusman. "MicrobiomeKG: bridging microbiome research
|
|
243
|
+
> and host health through knowledge graphs." *Frontiers in Systems Biology* 5 (2025).
|
|
246
244
|
> [doi:10.3389/fsysb.2025.1544432](https://doi.org/10.3389/fsysb.2025.1544432)
|
|
247
245
|
|
|
248
246
|
## License
|
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
# Tablassert
|
|
2
2
|
|
|
3
|
-
[](https://opensource.org/licenses/Apache-2.0)
|
|
4
|
-
[](https://www.python.org/downloads/)
|
|
5
3
|
[](https://badge.fury.io/py/tablassert)
|
|
6
|
-
[](https://pepy.tech/project/tablassert)
|
|
7
4
|
[](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
|
|
8
5
|
[](https://skyeav.github.io/Tablassert/)
|
|
9
|
-
[](https://opensource.org/licenses/Apache-2.0)
|
|
10
7
|
|
|
11
8
|
> Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON,
|
|
12
9
|
> declaratively, with entity resolution built in and optional quality control.
|
|
@@ -22,7 +19,7 @@ configuration reference, and API docs.
|
|
|
22
19
|
## Quick Start
|
|
23
20
|
|
|
24
21
|
```bash
|
|
25
|
-
pip install tablassert
|
|
22
|
+
pip install "tablassert[cli]"
|
|
26
23
|
```
|
|
27
24
|
|
|
28
25
|
Given a CSV of gene-disease associations with p-values and sample sizes, declare the mapping in a
|
|
@@ -87,7 +84,7 @@ Output is one JSON object per line: nodes with Biolink categories, edges with an
|
|
|
87
84
|
```
|
|
88
85
|
|
|
89
86
|
```json
|
|
90
|
-
{"subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}}}
|
|
87
|
+
{"id":"2cfea591-0f8f-33af-a7df-03da531d3359","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
|
|
91
88
|
```
|
|
92
89
|
|
|
93
90
|
See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full walkthrough.
|
|
@@ -97,7 +94,7 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
97
94
|
- **Declarative YAML configuration**: define data transformations without writing code
|
|
98
95
|
- **Built-in entity resolution**: map free text to genes, diseases, and chemicals with standard
|
|
99
96
|
CURIEs, taxonomic filtering, and provenance, backed by an embedded redb database
|
|
100
|
-
- **Optional quality control**: a four-stage audit (exact
|
|
97
|
+
- **Optional quality control**: a four-stage audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) flags
|
|
101
98
|
low-confidence mappings
|
|
102
99
|
- **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
|
|
103
100
|
and predicates
|
|
@@ -119,7 +116,7 @@ install `[cli]` to use the `tablassert` command and optional extras for addition
|
|
|
119
116
|
| `cli` | `tablassert` command and rich terminal progress | `pip install "tablassert[cli]"` |
|
|
120
117
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
121
118
|
| `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
122
|
-
| `qc` | four-stage QC audit (exact
|
|
119
|
+
| `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
123
120
|
| `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
|
|
124
121
|
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
|
|
125
122
|
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
|
|
@@ -156,11 +153,11 @@ no LazyFrame setup or NLP preprocessing required. See the
|
|
|
156
153
|
|
|
157
154
|
- **[Installation](https://skyeav.github.io/Tablassert/installation/)**: install methods, extras, and development setup
|
|
158
155
|
- **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
|
|
159
|
-
- **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
|
|
160
156
|
- **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
|
|
157
|
+
- **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
|
|
161
158
|
- **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
|
|
162
|
-
- **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
|
|
163
159
|
- **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
|
|
160
|
+
- **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
|
|
164
161
|
- **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
|
|
165
162
|
- **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
|
|
166
163
|
- **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
|
|
@@ -181,8 +178,8 @@ request guidelines.
|
|
|
181
178
|
If you use Tablassert, please cite it as described in [CITATION.cff](CITATION.cff). The approach is
|
|
182
179
|
described in:
|
|
183
180
|
|
|
184
|
-
> Skye Lane Goetz,
|
|
185
|
-
> and host health through knowledge graphs
|
|
181
|
+
> Skye Lane Goetz, Amy K. Glen, and Gwênlyn Glusman. "MicrobiomeKG: bridging microbiome research
|
|
182
|
+
> and host health through knowledge graphs." *Frontiers in Systems Biology* 5 (2025).
|
|
186
183
|
> [doi:10.3389/fsysb.2025.1544432](https://doi.org/10.3389/fsysb.2025.1544432)
|
|
187
184
|
|
|
188
185
|
## License
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "19.
|
|
3
|
+
version = "19.3.0"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -48,11 +48,12 @@ classifiers = [
|
|
|
48
48
|
requires-python = ">=3.11"
|
|
49
49
|
dependencies = [
|
|
50
50
|
"biolink-model>=4.4.4",
|
|
51
|
-
"polars>=
|
|
51
|
+
"polars>=2.0.0rc2",
|
|
52
|
+
"pyarrow>=21.0.0",
|
|
52
53
|
"rapidfuzz>=3.14.3",
|
|
53
54
|
"pydantic>=2.12.5",
|
|
54
55
|
"pyyaml>=6.0.3",
|
|
55
|
-
"fastexcel>=0.
|
|
56
|
+
"fastexcel>=0.21.0",
|
|
56
57
|
]
|
|
57
58
|
|
|
58
59
|
[project.urls]
|
|
@@ -81,7 +82,7 @@ cli = [
|
|
|
81
82
|
"rich>=13.0.0",
|
|
82
83
|
]
|
|
83
84
|
rt = [
|
|
84
|
-
"polars[rtcompat]>=
|
|
85
|
+
"polars[rtcompat]>=2.0.0rc2",
|
|
85
86
|
]
|
|
86
87
|
# PyPI distribution "aria2" exposes import module "aria2c" with a bundled aria2c binary.
|
|
87
88
|
aria2 = [
|
|
@@ -5,7 +5,7 @@ use pyo3::prelude::*;
|
|
|
5
5
|
use rustc_hash::{FxHashMap, FxHashSet};
|
|
6
6
|
use serde_json::Value;
|
|
7
7
|
use std::collections::hash_map::Entry;
|
|
8
|
-
use std::fs::File;
|
|
8
|
+
use std::fs::{self, File};
|
|
9
9
|
use std::io::{BufRead, BufReader, BufWriter, Write};
|
|
10
10
|
use std::path::{Path, PathBuf};
|
|
11
11
|
use std::rc::Rc;
|
|
@@ -299,8 +299,9 @@ impl ListState {
|
|
|
299
299
|
/// membership are hash sets, and the per-fold re-canonicalization + re-sort of every
|
|
300
300
|
/// stored list item is gone -- lists sort ONCE, at write-out, and only if a union ran.
|
|
301
301
|
struct MergedRecord {
|
|
302
|
-
/// The live record: scalars
|
|
303
|
-
///
|
|
302
|
+
/// The live record: conflicting scalars keep the lexicographically smallest canonical
|
|
303
|
+
/// value (see `merge_records`), lists in original/append order until the deferred
|
|
304
|
+
/// write-out sort.
|
|
304
305
|
value: Value,
|
|
305
306
|
/// Content hashes of every record absorbed into this id: O(1) exact-repeat
|
|
306
307
|
/// suppression. Bare hashes keep the pre-US-002 `Vec<u64>` semantics exactly (a false
|
|
@@ -311,6 +312,9 @@ struct MergedRecord {
|
|
|
311
312
|
lists: FxHashMap<String, ListState>,
|
|
312
313
|
/// Distinct non-empty scalar source values for each `original_*` field.
|
|
313
314
|
original_values: OriginalValues,
|
|
315
|
+
/// Conflicting scalar field names, one entry per fold conflict (with multiplicity).
|
|
316
|
+
/// Consumed at write-out by the per-predicate fold report.
|
|
317
|
+
conflict_fields: Vec<String>,
|
|
314
318
|
}
|
|
315
319
|
|
|
316
320
|
impl MergedRecord {
|
|
@@ -332,6 +336,7 @@ impl MergedRecord {
|
|
|
332
336
|
value,
|
|
333
337
|
hashes,
|
|
334
338
|
lists,
|
|
339
|
+
conflict_fields: Vec::new(),
|
|
335
340
|
})
|
|
336
341
|
}
|
|
337
342
|
|
|
@@ -387,8 +392,9 @@ impl MergedRecord {
|
|
|
387
392
|
}
|
|
388
393
|
}
|
|
389
394
|
|
|
390
|
-
/// Fold `incoming` into `stored`, field-wise. Returns the
|
|
391
|
-
///
|
|
395
|
+
/// Fold `incoming` into `stored`, field-wise. Returns the conflicting scalar field names
|
|
396
|
+
/// (one entry per conflict, reported for the base record) so the caller can build the fold
|
|
397
|
+
/// report.
|
|
392
398
|
///
|
|
393
399
|
/// Near-linear implementation of the semantics frozen by `merge_records_reference` and
|
|
394
400
|
/// policed by `merge_fold_matches_reference_on_fuzz`:
|
|
@@ -398,25 +404,32 @@ impl MergedRecord {
|
|
|
398
404
|
/// checked against the field's `ListState` set in O(1); stored items are never
|
|
399
405
|
/// re-canonicalized, and the sort by canonical bytes is deferred to write-out
|
|
400
406
|
/// (`MergedRecord::finish`), where it runs only for fields that saw a real union;
|
|
401
|
-
/// - scalar fields:
|
|
407
|
+
/// - scalar fields: a conflicting scalar keeps the LEXICOGRAPHICALLY SMALLEST canonical
|
|
408
|
+
/// value -- a commutative, associative winner, so the merged record is identical under
|
|
409
|
+
/// any arrival order -- and each conflict is reported by field name for the fold report,
|
|
410
|
+
/// except `original_*` fields;
|
|
402
411
|
/// - `original_*` fields collect distinct non-empty strings and render sorted values as
|
|
403
412
|
/// `A`, `A|B`, or `A|B|C`;
|
|
404
413
|
/// - fields only on `incoming`: copied over (only a conflict when both sides disagree);
|
|
405
414
|
/// - `id` is never touched: both sides carry the same one by construction.
|
|
406
415
|
///
|
|
407
|
-
/// `number_of_cases` has one hardcoded exception to
|
|
408
|
-
/// carries `supporting_case_ids` (a build-internal `list[str]` of the case
|
|
409
|
-
/// the count -- allowed onto edge frames so it reaches this pass, then stripped
|
|
410
|
-
/// write) and either side carried a count, the count is recomputed as the length of
|
|
411
|
-
/// unioned ID list. A case ID shared by both records is one case, so
|
|
412
|
-
/// summing both over- and under-report; the union length is the exact count. The
|
|
413
|
-
/// superseded divergence is NOT reported as a scalar conflict. When neither side
|
|
414
|
-
///
|
|
415
|
-
fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<
|
|
416
|
+
/// `number_of_cases` has one hardcoded exception to the smallest-value rule: when the
|
|
417
|
+
/// MERGED record carries `supporting_case_ids` (a build-internal `list[str]` of the case
|
|
418
|
+
/// IDs behind the count -- allowed onto edge frames so it reaches this pass, then stripped
|
|
419
|
+
/// before write) and either side carried a count, the count is recomputed as the length of
|
|
420
|
+
/// the unioned ID list. A case ID shared by both records is one case, so smallest-value
|
|
421
|
+
/// and summing both over- and under-report; the union length is the exact count. The
|
|
422
|
+
/// superseded divergence is NOT reported as a scalar conflict. When neither side carries
|
|
423
|
+
/// the list, `number_of_cases` stays an ordinary smallest-value scalar.
|
|
424
|
+
fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<Vec<String>> {
|
|
416
425
|
let Some(incoming_map) = incoming.as_object() else {
|
|
417
426
|
return Err(runtime_error("expected JSON object"));
|
|
418
427
|
};
|
|
419
|
-
let mut
|
|
428
|
+
let mut conflict_fields: Vec<String> = Vec::new();
|
|
429
|
+
// Winner overrides are collected during the loop and applied after it: the stored
|
|
430
|
+
// value's mutable borrow is live while a conflict is detected, and the winner needs
|
|
431
|
+
// the canonical bytes of BOTH sides compared.
|
|
432
|
+
let mut winner_overrides: Vec<(String, Value)> = Vec::new();
|
|
420
433
|
// Read both counts BEFORE the fold: the fold may copy incoming's over a stored side
|
|
421
434
|
// that lacks it, and the recompute rule below needs to know each side contributed one.
|
|
422
435
|
let stored_cases: Option<Value> = stored.value.get("number_of_cases").cloned();
|
|
@@ -487,14 +500,37 @@ fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<u64> {
|
|
|
487
500
|
.map(str::to_owned),
|
|
488
501
|
);
|
|
489
502
|
} else if stored_value != incoming_value {
|
|
490
|
-
|
|
503
|
+
conflict_fields.push(key.clone());
|
|
504
|
+
let stored_wins = {
|
|
505
|
+
let stored_bytes =
|
|
506
|
+
canonical_json_bytes(stored_value).map_err(runtime_error)?;
|
|
507
|
+
let incoming_bytes =
|
|
508
|
+
canonical_json_bytes(incoming_value).map_err(runtime_error)?;
|
|
509
|
+
stored_bytes <= incoming_bytes
|
|
510
|
+
};
|
|
511
|
+
if !stored_wins {
|
|
512
|
+
winner_overrides.push((key.clone(), incoming_value.clone()));
|
|
513
|
+
}
|
|
491
514
|
}
|
|
492
515
|
} else if stored_value != incoming_value {
|
|
493
|
-
|
|
516
|
+
conflict_fields.push(key.clone());
|
|
517
|
+
let stored_wins = {
|
|
518
|
+
let stored_bytes =
|
|
519
|
+
canonical_json_bytes(stored_value).map_err(runtime_error)?;
|
|
520
|
+
let incoming_bytes =
|
|
521
|
+
canonical_json_bytes(incoming_value).map_err(runtime_error)?;
|
|
522
|
+
stored_bytes <= incoming_bytes
|
|
523
|
+
};
|
|
524
|
+
if !stored_wins {
|
|
525
|
+
winner_overrides.push((key.clone(), incoming_value.clone()));
|
|
526
|
+
}
|
|
494
527
|
}
|
|
495
528
|
}
|
|
496
529
|
}
|
|
497
530
|
}
|
|
531
|
+
for (key, value) in winner_overrides {
|
|
532
|
+
stored_map.insert(key, value);
|
|
533
|
+
}
|
|
498
534
|
// WHY: exact-unique `number_of_cases` semantics (see the docstring). Guarded on the
|
|
499
535
|
// merged record actually carrying the ID list: a one-sided carrier is fine (the union
|
|
500
536
|
// is just that side's list), while a carrier-less merge never recomputes.
|
|
@@ -511,7 +547,17 @@ fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<u64> {
|
|
|
511
547
|
// not both arrays (two arrays took the union path and were never counted).
|
|
512
548
|
if let (Some(left), Some(right)) = (&stored_cases, &incoming_cases) {
|
|
513
549
|
if left != right && !(left.is_array() && right.is_array()) {
|
|
514
|
-
|
|
550
|
+
// The superseded divergence was counted by an EARLIER fold when the
|
|
551
|
+
// counts first collided (not necessarily this one), so the pop is
|
|
552
|
+
// unconditional on this fold's pushes: at most one live
|
|
553
|
+
// number_of_cases conflict can exist per merged record, and the
|
|
554
|
+
// recompute retires it.
|
|
555
|
+
if let Some(pos) = conflict_fields
|
|
556
|
+
.iter()
|
|
557
|
+
.rposition(|field| field == "number_of_cases")
|
|
558
|
+
{
|
|
559
|
+
conflict_fields.remove(pos);
|
|
560
|
+
}
|
|
515
561
|
}
|
|
516
562
|
}
|
|
517
563
|
let Some(map) = stored.value.as_object_mut() else {
|
|
@@ -523,7 +569,7 @@ fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<u64> {
|
|
|
523
569
|
);
|
|
524
570
|
}
|
|
525
571
|
}
|
|
526
|
-
Ok(
|
|
572
|
+
Ok(conflict_fields)
|
|
527
573
|
}
|
|
528
574
|
|
|
529
575
|
/// Remove build-internal carrier fields from an edge record before it is written.
|
|
@@ -555,8 +601,10 @@ impl MergeIndex {
|
|
|
555
601
|
return Ok(());
|
|
556
602
|
}
|
|
557
603
|
self.merged += 1;
|
|
558
|
-
|
|
604
|
+
let fields: Vec<String> = merge_records(record, &value)?;
|
|
605
|
+
self.scalar_conflicts += fields.len() as u64;
|
|
559
606
|
record.hashes.insert(content);
|
|
607
|
+
record.conflict_fields.extend(fields);
|
|
560
608
|
Ok(())
|
|
561
609
|
}
|
|
562
610
|
}
|
|
@@ -573,7 +621,8 @@ impl MergeIndex {
|
|
|
573
621
|
/// Original semantics doc:
|
|
574
622
|
///
|
|
575
623
|
/// Fold `incoming` into `stored`, field-wise. Returns the number of conflicting scalar
|
|
576
|
-
/// fields (
|
|
624
|
+
/// fields (the base record's value wins; see `merge_records`) so the caller can report
|
|
625
|
+
/// them.
|
|
577
626
|
///
|
|
578
627
|
/// - list fields: union, deduped by canonical JSON bytes (so two `sources` objects that
|
|
579
628
|
/// differ only in key order collapse), then sorted by canonical bytes so the merged
|
|
@@ -582,20 +631,21 @@ impl MergeIndex {
|
|
|
582
631
|
/// - fields only on `incoming`: copied over (only a conflict when both sides disagree);
|
|
583
632
|
/// - `id` is never touched: both sides carry the same one by construction.
|
|
584
633
|
///
|
|
585
|
-
/// `number_of_cases` has one hardcoded exception to
|
|
634
|
+
/// `number_of_cases` has one hardcoded exception to base-wins: when the MERGED record
|
|
586
635
|
/// carries `supporting_case_ids` (a build-internal `list[str]` of the case IDs behind
|
|
587
636
|
/// the count -- allowed onto edge frames so it reaches this pass, then stripped before
|
|
588
637
|
/// write) and either side carried a count, the count is recomputed as the length of the
|
|
589
|
-
/// unioned ID list. A case ID shared by both records is one case, so
|
|
638
|
+
/// unioned ID list. A case ID shared by both records is one case, so base-wins and
|
|
590
639
|
/// summing both over- and under-report; the union length is the exact count. The
|
|
591
640
|
/// superseded divergence is NOT reported as a scalar conflict. When neither side
|
|
592
|
-
/// carries the list, `number_of_cases` stays an ordinary
|
|
641
|
+
/// carries the list, `number_of_cases` stays an ordinary base-wins scalar.
|
|
593
642
|
#[cfg(test)]
|
|
594
643
|
fn merge_records_reference(stored: &mut Value, incoming: &Value) -> PyResult<u64> {
|
|
595
644
|
let Some(incoming_map) = incoming.as_object() else {
|
|
596
645
|
return Err(runtime_error("expected JSON object"));
|
|
597
646
|
};
|
|
598
647
|
let mut conflicts: u64 = 0;
|
|
648
|
+
let mut winner_overrides: Vec<(String, Value)> = Vec::new();
|
|
599
649
|
let mut original_values: OriginalValues = collect_original_values(stored);
|
|
600
650
|
for (key, values) in collect_original_values(incoming) {
|
|
601
651
|
original_values.entry(key).or_default().extend(values);
|
|
@@ -640,10 +690,19 @@ fn merge_records_reference(stored: &mut Value, incoming: &Value) -> PyResult<u64
|
|
|
640
690
|
// Aggregated and rendered after this fold, once all values are known.
|
|
641
691
|
} else if stored_value != incoming_value {
|
|
642
692
|
conflicts += 1;
|
|
693
|
+
let stored_bytes = canonical_json_bytes(stored_value).map_err(runtime_error)?;
|
|
694
|
+
let incoming_bytes =
|
|
695
|
+
canonical_json_bytes(incoming_value).map_err(runtime_error)?;
|
|
696
|
+
if incoming_bytes < stored_bytes {
|
|
697
|
+
winner_overrides.push((key.clone(), incoming_value.clone()));
|
|
698
|
+
}
|
|
643
699
|
}
|
|
644
700
|
}
|
|
645
701
|
}
|
|
646
702
|
}
|
|
703
|
+
for (key, value) in winner_overrides {
|
|
704
|
+
stored_map.insert(key, value);
|
|
705
|
+
}
|
|
647
706
|
if let Some(map) = stored.as_object_mut() {
|
|
648
707
|
for (key, values) in &original_values {
|
|
649
708
|
if !values.is_empty() {
|
|
@@ -684,6 +743,12 @@ fn merge_records_reference(stored: &mut Value, incoming: &Value) -> PyResult<u64
|
|
|
684
743
|
/// FROZEN EQUIVALENCE ORACLE -- BYTE-VERBATIM copy of the pre-US-002 `MergeIndex` absorb
|
|
685
744
|
/// path (state shape included), compiled ONLY under `cfg(test)`; drives
|
|
686
745
|
/// `merge_records_reference`. Treat as read-only -- see that fn's docs.
|
|
746
|
+
///
|
|
747
|
+
/// AMENDED once, deliberately (fold determinism): the oracle's base election now keeps the
|
|
748
|
+
/// record with the smallest canonical bytes as the fold base, exactly mirroring the
|
|
749
|
+
/// production `MergeIndex::absorb`, so both sides still demand byte-identical merged
|
|
750
|
+
/// output under any arrival order. This is a semantic amendment agreed for the fold
|
|
751
|
+
/// report work, not an optimization -- the oracle remains the algorithm of record.
|
|
687
752
|
#[cfg(test)]
|
|
688
753
|
#[derive(Default)]
|
|
689
754
|
struct MergeIndexReference {
|
|
@@ -719,17 +784,24 @@ impl MergeIndexReference {
|
|
|
719
784
|
}
|
|
720
785
|
}
|
|
721
786
|
|
|
722
|
-
/// Merge-mode edge pass: buffer every unique edge, fold divergent same-id records into
|
|
723
|
-
///
|
|
724
|
-
/// scalar fields) for the summary log
|
|
725
|
-
///
|
|
726
|
-
///
|
|
787
|
+
/// Merge-mode edge pass: buffer every unique edge, fold divergent same-id records into
|
|
788
|
+
/// their smallest-canonical-bytes base, then write in first-seen id order. Returns
|
|
789
|
+
/// (divergent records merged, conflicting scalar fields) for the summary log, and writes a
|
|
790
|
+
/// fold report (`<output>.foldreport.json`: per-predicate, per-field conflict counts) when
|
|
791
|
+
/// any record folded. Runs ONLY under `uuid_on_collision: merge`; the default path stays
|
|
792
|
+
/// streaming and never buffers a record. The build-internal `supporting_case_ids` carrier
|
|
793
|
+
/// is stripped from each record right before the write.
|
|
727
794
|
fn dedup_edges_merge(
|
|
728
795
|
reader: BufReader<File>,
|
|
729
796
|
mut writer: BufWriter<File>,
|
|
730
797
|
domain: &str,
|
|
731
798
|
fields: Option<&[String]>,
|
|
799
|
+
output: &Path,
|
|
732
800
|
) -> PyResult<(u64, u64)> {
|
|
801
|
+
let mut fold_report: std::collections::BTreeMap<
|
|
802
|
+
String,
|
|
803
|
+
std::collections::BTreeMap<String, u64>,
|
|
804
|
+
> = std::collections::BTreeMap::new();
|
|
733
805
|
let mut index: MergeIndex = MergeIndex::default();
|
|
734
806
|
for line in reader.lines() {
|
|
735
807
|
let line: String = line.map_err(runtime_error)?;
|
|
@@ -744,9 +816,23 @@ fn dedup_edges_merge(
|
|
|
744
816
|
index.absorb(edge_id_bytes(&value)?, value, content)?;
|
|
745
817
|
}
|
|
746
818
|
for id in std::mem::take(&mut index.order) {
|
|
747
|
-
let Some(record) = index.records.remove(&id) else {
|
|
819
|
+
let Some(mut record) = index.records.remove(&id) else {
|
|
748
820
|
continue;
|
|
749
821
|
};
|
|
822
|
+
// Fold report: every conflict this id accumulated is attributed to the base
|
|
823
|
+
// record's predicate, so consumers can see WHERE divergent evidence merged.
|
|
824
|
+
if !record.conflict_fields.is_empty() {
|
|
825
|
+
let predicate = record
|
|
826
|
+
.value
|
|
827
|
+
.get("predicate")
|
|
828
|
+
.and_then(Value::as_str)
|
|
829
|
+
.unwrap_or("_unknown_predicate_")
|
|
830
|
+
.to_string();
|
|
831
|
+
let fields_report = fold_report.entry(predicate).or_default();
|
|
832
|
+
for field in std::mem::take(&mut record.conflict_fields) {
|
|
833
|
+
*fields_report.entry(field).or_insert(0) += 1;
|
|
834
|
+
}
|
|
835
|
+
}
|
|
750
836
|
// The one deferred sort: unioned lists sort here, everything else ships as-is.
|
|
751
837
|
let mut value: Value = record.finish()?;
|
|
752
838
|
strip_internal_edge_fields(&mut value);
|
|
@@ -756,6 +842,20 @@ fn dedup_edges_merge(
|
|
|
756
842
|
writer.write_all(b"\n").map_err(runtime_error)?;
|
|
757
843
|
}
|
|
758
844
|
writer.flush().map_err(runtime_error)?;
|
|
845
|
+
if !fold_report.is_empty() {
|
|
846
|
+
let report_path = output.with_file_name(format!(
|
|
847
|
+
"{}.foldreport.json",
|
|
848
|
+
output
|
|
849
|
+
.file_name()
|
|
850
|
+
.and_then(std::ffi::OsStr::to_str)
|
|
851
|
+
.unwrap_or_default()
|
|
852
|
+
));
|
|
853
|
+
fs::write(
|
|
854
|
+
&report_path,
|
|
855
|
+
serde_json::to_vec_pretty(&fold_report).map_err(runtime_error)?,
|
|
856
|
+
)
|
|
857
|
+
.map_err(runtime_error)?;
|
|
858
|
+
}
|
|
759
859
|
Ok((index.merged, index.scalar_conflicts))
|
|
760
860
|
}
|
|
761
861
|
|
|
@@ -783,7 +883,7 @@ pub fn dedup_ndjson(
|
|
|
783
883
|
let reader: BufReader<File> = BufReader::new(File::open(input).map_err(runtime_error)?);
|
|
784
884
|
let mut writer: BufWriter<File> = BufWriter::new(File::create(&output).map_err(runtime_error)?);
|
|
785
885
|
if is_edges && merge {
|
|
786
|
-
return dedup_edges_merge(reader, writer, &domain, fields);
|
|
886
|
+
return dedup_edges_merge(reader, writer, &domain, fields, &output);
|
|
787
887
|
}
|
|
788
888
|
let mut nodes: FxHashMap<u64, Vec<Vec<u8>>> = FxHashMap::default();
|
|
789
889
|
let mut edges: EdgeIndex = EdgeIndex::default();
|
|
@@ -368,6 +368,10 @@ def build_graph_pipeline(
|
|
|
368
368
|
"head": head,
|
|
369
369
|
"name": g.name,
|
|
370
370
|
"infores": g.rig.source_info.infores_id,
|
|
371
|
+
"qc_similarity_threshold": g.qc_similarity_threshold,
|
|
372
|
+
"qc_fuzzy_ratio_threshold": g.qc_fuzzy_ratio_threshold,
|
|
373
|
+
"qc_fuzzy_partial_threshold": g.qc_fuzzy_partial_threshold,
|
|
374
|
+
"qc_aliases": g.qc_aliases,
|
|
371
375
|
}
|
|
372
376
|
)
|
|
373
377
|
)
|
|
@@ -292,7 +292,10 @@ def distinct(lf: pl.LazyFrame, l1: str, l2: str, col: str = "term") -> pl.LazyFr
|
|
|
292
292
|
t2: pl.LazyFrame = lf.select(pl.col(l2).alias(col)).unique()
|
|
293
293
|
t2 = t2.with_columns(pl.lit(2).alias("nlp_level"))
|
|
294
294
|
|
|
295
|
-
|
|
295
|
+
# Level-1 wins per term. Group-by min instead of concat + unique(keep="first"):
|
|
296
|
+
# the streaming engine (polars 2.0 default) does not preserve row order for
|
|
297
|
+
# concat/unique, so first-seen tier selection would be scheduling-dependent.
|
|
298
|
+
terms: pl.LazyFrame = pl.concat([t1, t2]).group_by(col).agg(pl.col("nlp_level").min())
|
|
296
299
|
|
|
297
300
|
bad: str = r"^\d+$|^(none|nan|na|null|unknown|not applicable|p_value|variable|result|exposure|expression|symbol)$|^$"
|
|
298
301
|
return terms.filter(~pl.col(col).str.contains(bad))
|
|
@@ -892,6 +892,16 @@ def idx(lf: pl.LazyFrame, col: str = "extracted_from_row_number") -> pl.LazyFram
|
|
|
892
892
|
return lf.with_row_index(col, offset=1)
|
|
893
893
|
|
|
894
894
|
|
|
895
|
+
def _polars_auto_column_names(names: list[str]) -> bool:
|
|
896
|
+
"""Return True when ``names`` is exactly polars' headerless auto-naming scheme.
|
|
897
|
+
|
|
898
|
+
Polars 2.0 autogenerates headerless column names as ``column_0``.. ``column_{n-1}``
|
|
899
|
+
(1.x used ``column_1``.. ``column_n``). Matching the full consecutive run keeps the
|
|
900
|
+
remap below strictly scoped to machine-generated names, never user data.
|
|
901
|
+
"""
|
|
902
|
+
return bool(names) and all(name == f"column_{i}" for i, name in enumerate(names))
|
|
903
|
+
|
|
904
|
+
|
|
895
905
|
def csv(p: Path, sep: str) -> pl.LazyFrame:
|
|
896
906
|
"""Read a CSV or TSV source as a LazyFrame.
|
|
897
907
|
|
|
@@ -901,8 +911,17 @@ def csv(p: Path, sep: str) -> pl.LazyFrame:
|
|
|
901
911
|
|
|
902
912
|
Returns:
|
|
903
913
|
LazyFrame over the file (no header inference; all columns raw).
|
|
914
|
+
|
|
915
|
+
Notes:
|
|
916
|
+
Headerless auto column names are remapped to the 1-based ``column_1``..
|
|
917
|
+
convention so existing table configs keep referencing the same names as
|
|
918
|
+
under polars 1.x (output parity).
|
|
904
919
|
"""
|
|
905
|
-
|
|
920
|
+
lf: pl.LazyFrame = pl.scan_csv(source=p, separator=sep, has_header=False, infer_schema_length=None, truncate_ragged_lines=True)
|
|
921
|
+
schema_names: list[str] = list(lf.collect_schema().names())
|
|
922
|
+
if _polars_auto_column_names(schema_names):
|
|
923
|
+
lf = lf.rename(dict(zip(schema_names, (f"column_{i + 1}" for i in range(len(schema_names))), strict=True)))
|
|
924
|
+
return lf
|
|
906
925
|
|
|
907
926
|
|
|
908
927
|
def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame:
|
|
@@ -915,6 +934,11 @@ def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame:
|
|
|
915
934
|
|
|
916
935
|
Returns:
|
|
917
936
|
LazyFrame over the sheet contents (no header inference).
|
|
937
|
+
|
|
938
|
+
Notes:
|
|
939
|
+
Headerless auto column names are remapped to the 1-based ``column_1``..
|
|
940
|
+
convention so existing table configs keep referencing the same names as
|
|
941
|
+
under polars 1.x (output parity).
|
|
918
942
|
"""
|
|
919
943
|
df: pl.DataFrame = pl.read_excel(
|
|
920
944
|
source=p,
|
|
@@ -923,6 +947,8 @@ def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame:
|
|
|
923
947
|
has_header=False,
|
|
924
948
|
infer_schema_length=None,
|
|
925
949
|
)
|
|
950
|
+
if _polars_auto_column_names(df.columns):
|
|
951
|
+
df = df.rename(dict(zip(df.columns, (f"column_{i + 1}" for i in range(len(df.columns))), strict=True)))
|
|
926
952
|
return df.lazy()
|
|
927
953
|
|
|
928
954
|
|
|
@@ -1150,6 +1176,10 @@ class Tcode(Section):
|
|
|
1150
1176
|
head: bool = Field(False)
|
|
1151
1177
|
name: str | None = Field(None)
|
|
1152
1178
|
infores: str | None = Field(None)
|
|
1179
|
+
qc_similarity_threshold: float = Field(0.5)
|
|
1180
|
+
qc_fuzzy_ratio_threshold: float = Field(70.0)
|
|
1181
|
+
qc_fuzzy_partial_threshold: float = Field(80.0)
|
|
1182
|
+
qc_aliases: dict[str, str] = Field(default_factory=dict)
|
|
1153
1183
|
|
|
1154
1184
|
def encoding(self: Self, x: Encoding, col: str, table_literal: bool = False) -> list[Any]:
|
|
1155
1185
|
"""Collect helper for Encoding classes.
|
|
@@ -1302,7 +1332,30 @@ class Tcode(Section):
|
|
|
1302
1332
|
# QC audits only the strict columns: a nullable qualifier's nulls are expected
|
|
1303
1333
|
# (blank cell / no match), not resolution errors for the audit to delete.
|
|
1304
1334
|
[
|
|
1305
|
-
(
|
|
1335
|
+
(
|
|
1336
|
+
fullmap_audit,
|
|
1337
|
+
(
|
|
1338
|
+
(col, self.store.stem, self.config.name, "passed", True)
|
|
1339
|
+
if (
|
|
1340
|
+
self.qc_similarity_threshold == 0.5
|
|
1341
|
+
and self.qc_fuzzy_ratio_threshold == 70.0
|
|
1342
|
+
and self.qc_fuzzy_partial_threshold == 80.0
|
|
1343
|
+
and not self.qc_aliases
|
|
1344
|
+
)
|
|
1345
|
+
else (
|
|
1346
|
+
col,
|
|
1347
|
+
self.store.stem,
|
|
1348
|
+
self.config.name,
|
|
1349
|
+
"passed",
|
|
1350
|
+
True,
|
|
1351
|
+
None,
|
|
1352
|
+
self.qc_similarity_threshold,
|
|
1353
|
+
self.qc_fuzzy_ratio_threshold,
|
|
1354
|
+
self.qc_fuzzy_partial_threshold,
|
|
1355
|
+
self.qc_aliases,
|
|
1356
|
+
)
|
|
1357
|
+
),
|
|
1358
|
+
)
|
|
1306
1359
|
for x, col in node_columns
|
|
1307
1360
|
if not (isinstance(x, Qualifier) and x.nullable)
|
|
1308
1361
|
]
|
|
@@ -1597,7 +1650,7 @@ def normalize(edges: pl.LazyFrame, col: str, names: list[str] | None = None, inf
|
|
|
1597
1650
|
available: list[str] = edges.collect_schema().names()
|
|
1598
1651
|
# `<col>_taxon_label` is not produced by every fullmap revision.
|
|
1599
1652
|
pairs: list[tuple[str, str]] = [(c, n) for c, n in zip(cols, names, strict=True) if c in available]
|
|
1600
|
-
nodes: pl.LazyFrame = edges.select([c for c, _ in pairs]).unique().rename(dict(pairs))
|
|
1653
|
+
nodes: pl.LazyFrame = edges.select([c for c, _ in pairs]).unique(maintain_order=True).rename(dict(pairs))
|
|
1601
1654
|
# Ensures category has biolink: prefix.
|
|
1602
1655
|
nodes = nodes.with_columns(
|
|
1603
1656
|
pl.when(pl.col("category").str.starts_with("biolink:"))
|
|
@@ -1890,7 +1943,7 @@ def _write_ndjson(
|
|
|
1890
1943
|
on_phase("write-nodes")
|
|
1891
1944
|
with nodes_tmp.open("a", encoding="utf-8") as f:
|
|
1892
1945
|
for subnode in subnodes:
|
|
1893
|
-
eagernode: pl.DataFrame = subnode.collect().unique()
|
|
1946
|
+
eagernode: pl.DataFrame = subnode.collect().unique(maintain_order=True)
|
|
1894
1947
|
eagernode.write_ndjson(f)
|
|
1895
1948
|
|
|
1896
1949
|
# Phase: write-edges.
|
|
@@ -1898,7 +1951,7 @@ def _write_ndjson(
|
|
|
1898
1951
|
on_phase("write-edges")
|
|
1899
1952
|
with edges_tmp.open("a", encoding="utf-8") as f:
|
|
1900
1953
|
for subedge in subedges:
|
|
1901
|
-
eageredge: pl.DataFrame = subedge.collect().unique()
|
|
1954
|
+
eageredge: pl.DataFrame = subedge.collect().unique(maintain_order=True)
|
|
1902
1955
|
eageredge.write_ndjson(f)
|
|
1903
1956
|
|
|
1904
1957
|
# Phase: dedup.
|
|
@@ -1138,6 +1138,10 @@ class Graph(TablaBase):
|
|
|
1138
1138
|
description="Explicit UUID namespace. Defaults to `rig.source_info.infores_id` when `uuid_fields` is set, `TABLASSERT` otherwise. Set it only when graphs must deliberately share an id space.",
|
|
1139
1139
|
examples=["infores:multiomicskg"],
|
|
1140
1140
|
)
|
|
1141
|
+
qc_similarity_threshold: float = Field(default=0.5, ge=0.0, le=1.0, description="SapBERT cosine threshold for QC.")
|
|
1142
|
+
qc_fuzzy_ratio_threshold: float = Field(default=70.0, ge=0.0, le=100.0, description="RapidFuzz ratio threshold for QC.")
|
|
1143
|
+
qc_fuzzy_partial_threshold: float = Field(default=80.0, ge=0.0, le=100.0, description="RapidFuzz token threshold for QC.")
|
|
1144
|
+
qc_aliases: dict[str, str] = Field(default_factory=dict, description="QC source-text aliases applied before similarity scoring.")
|
|
1141
1145
|
uuid_on_collision: Literal["error", "merge"] = Field(
|
|
1142
1146
|
default="error",
|
|
1143
1147
|
description="What to do when two different edges derive one id under `uuid_fields`: `error` aborts the build with `uuid-fields-not-a-key`; `merge` folds the records into one edge (list fields unioned and sorted, scalars first-wins). Requires `uuid_fields`, and buffers one full record per unique id, which is why it is opt-in.",
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import re
|
|
3
4
|
from collections.abc import Callable
|
|
4
5
|
from functools import cache
|
|
5
6
|
from pathlib import Path
|
|
@@ -27,6 +28,18 @@ MODEL: Path = BASE / "sapbert"
|
|
|
27
28
|
# BioBERT gate; SapBERT's UMLS synonym-alignment training shifts true synonym
|
|
28
29
|
# pairs upward, so a tuning pass against real reject logs is the natural follow-up.
|
|
29
30
|
SIMILARITY_THRESHOLD: float = 0.5
|
|
31
|
+
FUZZ_RATIO_THRESHOLD: float = 70.0
|
|
32
|
+
FUZZ_PARTIAL_THRESHOLD: float = 80.0
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _qc_text(value: str) -> str:
|
|
36
|
+
"""Normalize separator noise before lexical QC scoring."""
|
|
37
|
+
return re.sub(r"[^\w]+", " ", value.replace("?", "-")).strip().casefold()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _qc_alias(value: str, aliases: dict[str, str]) -> str:
|
|
41
|
+
"""Resolve configured source-specific aliases before generic similarity stages."""
|
|
42
|
+
return aliases.get(value.casefold(), value)
|
|
30
43
|
|
|
31
44
|
|
|
32
45
|
def _cascade(passed: pl.DataFrame, scored: pl.DataFrame, out: str) -> tuple[pl.DataFrame, pl.DataFrame]:
|
|
@@ -121,6 +134,10 @@ def fullmap_audit(
|
|
|
121
134
|
out: str = "passed",
|
|
122
135
|
log: bool = True,
|
|
123
136
|
on_phase: Callable[[str], None] | None = None,
|
|
137
|
+
similarity_threshold: float = SIMILARITY_THRESHOLD,
|
|
138
|
+
fuzzy_ratio_threshold: float = FUZZ_RATIO_THRESHOLD,
|
|
139
|
+
fuzzy_partial_threshold: float = FUZZ_PARTIAL_THRESHOLD,
|
|
140
|
+
aliases: dict[str, str] | None = None,
|
|
124
141
|
) -> pl.LazyFrame:
|
|
125
142
|
"""Audit that fullmap correctly processed source strings into CURIEs.
|
|
126
143
|
|
|
@@ -182,8 +199,11 @@ def fullmap_audit(
|
|
|
182
199
|
on_phase("qc:exact")
|
|
183
200
|
# Collection point: pending pairs require eager.
|
|
184
201
|
df: pl.DataFrame = lf.collect()
|
|
202
|
+
aliases = {str(key).casefold(): str(value) for key, value in (aliases or {}).items()}
|
|
185
203
|
pairs: pl.DataFrame = df.select(cols).unique()
|
|
186
|
-
pairs = pairs.with_columns(
|
|
204
|
+
pairs = pairs.with_columns(
|
|
205
|
+
pl.struct(cols[1:]).map_elements(lambda row: _qc_alias(str(row[cols[1]]), aliases) == str(row[cols[2]]), return_dtype=pl.Boolean).alias(out)
|
|
206
|
+
)
|
|
187
207
|
|
|
188
208
|
passed: pl.DataFrame
|
|
189
209
|
pending: pl.DataFrame
|
|
@@ -203,15 +223,24 @@ def fullmap_audit(
|
|
|
203
223
|
# Stage 2: fuzzy matching via RapidFuzz (batched).
|
|
204
224
|
if on_phase is not None:
|
|
205
225
|
on_phase("qc:fuzzy")
|
|
206
|
-
originals: list[str] = pending.get_column(cols[1]).to_list()
|
|
207
|
-
preferreds: list[str] = pending.get_column(cols[2]).to_list()
|
|
226
|
+
originals: list[str] = [_qc_alias(str(value), aliases) for value in pending.get_column(cols[1]).to_list()]
|
|
227
|
+
preferreds: list[str] = [str(value) for value in pending.get_column(cols[2]).to_list()]
|
|
228
|
+
normalized_originals: list[str] = [_qc_text(value) for value in originals]
|
|
229
|
+
normalized_preferreds: list[str] = [_qc_text(value) for value in preferreds]
|
|
208
230
|
|
|
209
|
-
ratio_scores: object = cpdist(
|
|
210
|
-
partial_scores: object = cpdist(
|
|
231
|
+
ratio_scores: object = cpdist(normalized_originals, normalized_preferreds, scorer=fuzz.ratio)
|
|
232
|
+
partial_scores: object = cpdist(normalized_originals, normalized_preferreds, scorer=fuzz.partial_token_sort_ratio)
|
|
233
|
+
token_scores: object = cpdist(normalized_originals, normalized_preferreds, scorer=fuzz.token_set_ratio)
|
|
211
234
|
|
|
212
|
-
pending = pending.with_columns(
|
|
235
|
+
pending = pending.with_columns(
|
|
236
|
+
[pl.Series("fuzz_ratio", ratio_scores), pl.Series("fuzz_partial", partial_scores), pl.Series("fuzz_token_set", token_scores)]
|
|
237
|
+
)
|
|
213
238
|
|
|
214
|
-
fuzz_mask: pl.Series = pl.Series(
|
|
239
|
+
fuzz_mask: pl.Series = pl.Series(
|
|
240
|
+
out,
|
|
241
|
+
(ratio_scores >= fuzzy_ratio_threshold) | (partial_scores >= fuzzy_partial_threshold) | (token_scores >= fuzzy_partial_threshold),
|
|
242
|
+
dtype=pl.Boolean,
|
|
243
|
+
)
|
|
215
244
|
masked_fuzz: pl.DataFrame = pending.with_columns(fuzz_mask)
|
|
216
245
|
pairs = pl.concat((passed, masked_fuzz), how="diagonal")
|
|
217
246
|
|
|
@@ -253,7 +282,7 @@ def fullmap_audit(
|
|
|
253
282
|
|
|
254
283
|
pending = pending.with_columns(pl.Series("sapbert_similarity", similarity)) # pyright: ignore
|
|
255
284
|
|
|
256
|
-
sapbert_mask: pl.Series = pl.Series(out, similarity >=
|
|
285
|
+
sapbert_mask: pl.Series = pl.Series(out, similarity >= similarity_threshold, dtype=pl.Boolean) # pyright: ignore
|
|
257
286
|
masked_sapbert: pl.DataFrame = pending.with_columns(sapbert_mask)
|
|
258
287
|
pairs = pl.concat((passed, masked_sapbert), how="diagonal")
|
|
259
288
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|