tablassert 21.0.0__tar.gz → 22.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-21.0.0 → tablassert-22.0.1}/PKG-INFO +10 -8
- {tablassert-21.0.0 → tablassert-22.0.1}/README.md +9 -7
- {tablassert-21.0.0 → tablassert-22.0.1}/pyproject.toml +1 -1
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/errors.py +0 -3
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/fullmap.py +27 -2
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/lib.py +24 -95
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/models.py +21 -118
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/runcache.py +0 -3
- {tablassert-21.0.0 → tablassert-22.0.1}/LICENSE +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/Cargo.lock +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/Cargo.toml +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/examples/count_tables.rs +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/src/fullmap.rs +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/src/json.rs +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/src/lib.rs +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/src/ndjson.rs +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/src/nlp.rs +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/src/uuid.rs +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/tests/build_golden.rs +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/tests/common/mod.rs +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/__init__.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/_lazy.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/biolink.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/cli.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/cli_entry.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/coerce.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/data/infores_catalog.yaml +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/enums.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/extras.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/infores_registry.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/ingests.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/log.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/net.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/nlp.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/progress.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/qc.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/rig.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/rs.pyi +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/study.py +0 -0
- {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version:
|
|
3
|
+
Version: 22.0.1
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -163,15 +163,15 @@ Build the knowledge graph:
|
|
|
163
163
|
tablassert build-kg graph.yaml
|
|
164
164
|
```
|
|
165
165
|
|
|
166
|
-
Output is one JSON object per line: nodes with Biolink categories, edges with annotations.
|
|
166
|
+
Output is one JSON object per line: nodes with Biolink categories, edges with annotations. The edge sample below is abridged; real records also carry provenance fields such as `knowledge_level`, `agent_type`, `sources`, and `original_subject`/`original_object`.
|
|
167
167
|
|
|
168
168
|
```json
|
|
169
|
-
{"id":"HGNC:11998","name":"TP53","category":["biolink:Gene"],"
|
|
170
|
-
{"id":"MONDO:0008903","name":"lung cancer","category":["biolink:Disease"]}
|
|
169
|
+
{"id":"HGNC:11998","name":"TP53","category":["biolink:Gene"],"in_taxon":["NCBITaxon:9606"],"provided_by":["infores:my-kg"]}
|
|
170
|
+
{"id":"MONDO:0008903","name":"lung cancer","category":["biolink:Disease"],"in_taxon":["NCBITaxon:9606"],"provided_by":["infores:my-kg"]}
|
|
171
171
|
```
|
|
172
172
|
|
|
173
173
|
```json
|
|
174
|
-
{"id":"
|
|
174
|
+
{"id":"8f41c2d7-9b3e-35c1-a4e8-6d2f9a0c5b71","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"very_strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.yaml SECTION 1","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
|
|
175
175
|
```
|
|
176
176
|
|
|
177
177
|
See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full walkthrough.
|
|
@@ -194,8 +194,10 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
194
194
|
pip install tablassert
|
|
195
195
|
```
|
|
196
196
|
|
|
197
|
-
|
|
198
|
-
|
|
197
|
+
To use the Python API, install Tablassert into your project's environment with `pip install
|
|
198
|
+
"tablassert"` (or `uv add tablassert`). To get the `tablassert` command as a standalone tool, use
|
|
199
|
+
`uv tool install "tablassert[cli]"`; note that a tool install is isolated and does not expose the
|
|
200
|
+
Python API to your project. Other extras are opt-in:
|
|
199
201
|
|
|
200
202
|
| Extra | Adds | Install |
|
|
201
203
|
| ----- | ---- | ------- |
|
|
@@ -271,5 +273,5 @@ described in:
|
|
|
271
273
|
|
|
272
274
|
- [Skye Lane Goetz](mailto:sgoetz@isbscience.org), Institute for Systems Biology
|
|
273
275
|
- [Gwênlyn Glusman](mailto:gglusman@isbscience.org), Institute for Systems Biology
|
|
274
|
-
- Jared C. Roach, Institute for Systems Biology
|
|
276
|
+
- [Jared C. Roach](mailto:jroach@isbscience.org), Institute for Systems Biology
|
|
275
277
|
|
|
@@ -109,15 +109,15 @@ Build the knowledge graph:
|
|
|
109
109
|
tablassert build-kg graph.yaml
|
|
110
110
|
```
|
|
111
111
|
|
|
112
|
-
Output is one JSON object per line: nodes with Biolink categories, edges with annotations.
|
|
112
|
+
Output is one JSON object per line: nodes with Biolink categories, edges with annotations. The edge sample below is abridged; real records also carry provenance fields such as `knowledge_level`, `agent_type`, `sources`, and `original_subject`/`original_object`.
|
|
113
113
|
|
|
114
114
|
```json
|
|
115
|
-
{"id":"HGNC:11998","name":"TP53","category":["biolink:Gene"],"
|
|
116
|
-
{"id":"MONDO:0008903","name":"lung cancer","category":["biolink:Disease"]}
|
|
115
|
+
{"id":"HGNC:11998","name":"TP53","category":["biolink:Gene"],"in_taxon":["NCBITaxon:9606"],"provided_by":["infores:my-kg"]}
|
|
116
|
+
{"id":"MONDO:0008903","name":"lung cancer","category":["biolink:Disease"],"in_taxon":["NCBITaxon:9606"],"provided_by":["infores:my-kg"]}
|
|
117
117
|
```
|
|
118
118
|
|
|
119
119
|
```json
|
|
120
|
-
{"id":"
|
|
120
|
+
{"id":"8f41c2d7-9b3e-35c1-a4e8-6d2f9a0c5b71","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"very_strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.yaml SECTION 1","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
|
|
121
121
|
```
|
|
122
122
|
|
|
123
123
|
See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full walkthrough.
|
|
@@ -140,8 +140,10 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
140
140
|
pip install tablassert
|
|
141
141
|
```
|
|
142
142
|
|
|
143
|
-
|
|
144
|
-
|
|
143
|
+
To use the Python API, install Tablassert into your project's environment with `pip install
|
|
144
|
+
"tablassert"` (or `uv add tablassert`). To get the `tablassert` command as a standalone tool, use
|
|
145
|
+
`uv tool install "tablassert[cli]"`; note that a tool install is isolated and does not expose the
|
|
146
|
+
Python API to your project. Other extras are opt-in:
|
|
145
147
|
|
|
146
148
|
| Extra | Adds | Install |
|
|
147
149
|
| ----- | ---- | ------- |
|
|
@@ -217,4 +219,4 @@ described in:
|
|
|
217
219
|
|
|
218
220
|
- [Skye Lane Goetz](mailto:sgoetz@isbscience.org), Institute for Systems Biology
|
|
219
221
|
- [Gwênlyn Glusman](mailto:gglusman@isbscience.org), Institute for Systems Biology
|
|
220
|
-
- Jared C. Roach, Institute for Systems Biology
|
|
222
|
+
- [Jared C. Roach](mailto:jroach@isbscience.org), Institute for Systems Biology
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "
|
|
3
|
+
version = "22.0.1"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -63,9 +63,6 @@ TablassertErrorCodes = Literal[
|
|
|
63
63
|
"provenance-missing-publication",
|
|
64
64
|
"provenance-publication-and-override",
|
|
65
65
|
"encoding-list-method-removed",
|
|
66
|
-
"no-map-dead-config",
|
|
67
|
-
"no-map-bad-curie",
|
|
68
|
-
"no-map-curie-violation",
|
|
69
66
|
"annotation-split-by-requires-column",
|
|
70
67
|
"annotation-split-by-empty",
|
|
71
68
|
"field-disabled",
|
|
@@ -4,6 +4,7 @@ import time
|
|
|
4
4
|
from collections import OrderedDict
|
|
5
5
|
from collections.abc import Callable
|
|
6
6
|
from enum import Enum
|
|
7
|
+
from functools import cache
|
|
7
8
|
from pathlib import Path
|
|
8
9
|
from typing import TYPE_CHECKING, Any, NamedTuple, cast
|
|
9
10
|
|
|
@@ -269,6 +270,28 @@ def lookup_rows(db: Path, terms: list[str]) -> list[dict[str, object]]:
|
|
|
269
270
|
return rows
|
|
270
271
|
|
|
271
272
|
|
|
273
|
+
@cache
|
|
274
|
+
def _generic_probe_keys() -> frozenset[str]:
|
|
275
|
+
"""Apply the real lookup normalizers to the existing excluded generic labels."""
|
|
276
|
+
labels = [
|
|
277
|
+
"none",
|
|
278
|
+
"nan",
|
|
279
|
+
"na",
|
|
280
|
+
"null",
|
|
281
|
+
"unknown",
|
|
282
|
+
"not applicable",
|
|
283
|
+
"p_value",
|
|
284
|
+
"p value",
|
|
285
|
+
"variable",
|
|
286
|
+
"result",
|
|
287
|
+
"exposure",
|
|
288
|
+
"expression",
|
|
289
|
+
"symbol",
|
|
290
|
+
]
|
|
291
|
+
normalized = level_two(level_one(pl.LazyFrame({"term": labels}), "term"), "term").collect()
|
|
292
|
+
return frozenset(labels) | frozenset(normalized["term"]) | frozenset(normalized["term_two"])
|
|
293
|
+
|
|
294
|
+
|
|
272
295
|
def distinct(lf: pl.LazyFrame, l1: str, l2: str, col: str = "term") -> pl.LazyFrame:
|
|
273
296
|
"""Extract unique terms from two text-normalization columns as a LazyFrame.
|
|
274
297
|
|
|
@@ -297,8 +320,10 @@ def distinct(lf: pl.LazyFrame, l1: str, l2: str, col: str = "term") -> pl.LazyFr
|
|
|
297
320
|
# concat/unique, so first-seen tier selection would be scheduling-dependent.
|
|
298
321
|
terms: pl.LazyFrame = pl.concat([t1, t2]).group_by(col).agg(pl.col("nlp_level").min())
|
|
299
322
|
|
|
300
|
-
|
|
301
|
-
|
|
323
|
+
# Probe keys are stemmed/sorted at level one, then stripped at level two.
|
|
324
|
+
# Filtering only the original spellings lets headers such as "Expression"
|
|
325
|
+
# ("express") and "Not applicable" ("applic not") resolve as entities.
|
|
326
|
+
return terms.filter(~pl.col(col).str.contains(r"^\d+$|^$") & ~pl.col(col).is_in(_generic_probe_keys()))
|
|
302
327
|
|
|
303
328
|
|
|
304
329
|
def deduplicate_result(result: pl.DataFrame, column_context: bool) -> pl.DataFrame:
|
|
@@ -6,6 +6,7 @@ import re
|
|
|
6
6
|
from collections.abc import Callable, Iterable
|
|
7
7
|
from functools import cache
|
|
8
8
|
from pathlib import Path
|
|
9
|
+
from tempfile import TemporaryDirectory
|
|
9
10
|
from typing import TYPE_CHECKING, Any, Literal, Self
|
|
10
11
|
|
|
11
12
|
from pydantic import Field, NonNegativeInt
|
|
@@ -48,7 +49,7 @@ from tablassert.enums import EncodingMethods, Files, Functions, InformationResou
|
|
|
48
49
|
from tablassert.errors import TablassertError
|
|
49
50
|
from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
|
|
50
51
|
from tablassert.log import cat
|
|
51
|
-
from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
|
|
52
|
+
from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
|
|
52
53
|
from tablassert.nlp import level_one, level_two
|
|
53
54
|
from tablassert.qc import fullmap_audit
|
|
54
55
|
from tablassert.rig import (
|
|
@@ -1220,65 +1221,6 @@ def qualifier_vocabulary_audit(lf: pl.LazyFrame, col: str, vocabulary: Iterable[
|
|
|
1220
1221
|
)
|
|
1221
1222
|
|
|
1222
1223
|
|
|
1223
|
-
def curie_audit(lf: pl.LazyFrame, col: str, section: str, config_file: str) -> pl.LazyFrame:
|
|
1224
|
-
"""Finalize a ``no_map`` node column: blank rows drop, non-CURIE values fail, category is synthesized.
|
|
1225
|
-
|
|
1226
|
-
Columns configured with ``no_map: true`` bypass entity resolution, so nothing else
|
|
1227
|
-
(a) drops rows whose cell is blank (a missing node id can never form an edge, exactly
|
|
1228
|
-
like an unresolved subject/object), (b) rejects cells that are not CURIE-shaped -- the
|
|
1229
|
-
encoded value becomes the KGX node id verbatim, so garbage would ship as an identifier --
|
|
1230
|
-
or (c) produces the ``<col>_category`` column downstream stages require: without a
|
|
1231
|
-
fullmap lookup there is no role to derive, so the column is pinned to
|
|
1232
|
-
``biolink:NamedThing`` and edge derivation degrades to ``biolink:Association`` exactly
|
|
1233
|
-
like the documented no-role-lookup fallback in :func:`edge_category`.
|
|
1234
|
-
|
|
1235
|
-
Args:
|
|
1236
|
-
lf: Source LazyFrame.
|
|
1237
|
-
col: The no_map node column (``subject``/``object`` or a CURIE-ranged qualifier key).
|
|
1238
|
-
section: Short section hash (for log context).
|
|
1239
|
-
config_file: Originating config file (for log context).
|
|
1240
|
-
|
|
1241
|
-
Returns:
|
|
1242
|
-
LazyFrame with blank/null-like rows dropped and ``<col>_category`` added.
|
|
1243
|
-
|
|
1244
|
-
Notes:
|
|
1245
|
-
A non-blank value that is not CURIE-shaped raises a coded error naming the
|
|
1246
|
-
offending values (mirroring ``qualifier_vocabulary_audit``); blank and null-like
|
|
1247
|
-
cells (``""``, ``"NA"``, ...) are dropped with a loud per-column count instead,
|
|
1248
|
-
mirroring how an unresolvable subject/object row is dropped, never emitted.
|
|
1249
|
-
"""
|
|
1250
|
-
values: pl.Series = lf.select(pl.col(col)).collect().get_column(col)
|
|
1251
|
-
if values.dtype.base_type() in (pl.List, pl.Array):
|
|
1252
|
-
values = values.explode()
|
|
1253
|
-
blank_mask: pl.Series = values.is_null() | values.str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT))
|
|
1254
|
-
# Judge CURIE shape on the distinct non-blank values only: the audit runs once per
|
|
1255
|
-
# section, and map_elements is a per-element Python call that must not scale with
|
|
1256
|
-
# row count when one bad cell repeats a million times.
|
|
1257
|
-
distinct: pl.Series = values.filter(~blank_mask).unique()
|
|
1258
|
-
offending: pl.Series = distinct.filter(~distinct.str.strip_chars().map_elements(is_curie, return_dtype=pl.Boolean)).sort()
|
|
1259
|
-
if offending.len() > 0:
|
|
1260
|
-
shown: list[str] = offending.head(10).to_list()
|
|
1261
|
-
more: str = "..." if offending.len() > len(shown) else ""
|
|
1262
|
-
raise TablassertError(
|
|
1263
|
-
f"{col} is configured with `no_map: true` so its values are used verbatim as node CURIEs, but "
|
|
1264
|
-
f"{offending.len()} non-blank value(s) in the built column are not CURIE-shaped: "
|
|
1265
|
-
f"{', '.join(repr(x) for x in shown)}{more}. Fix the source table or the encoding "
|
|
1266
|
-
"(prefix/suffix can form a CURIE from raw cells).",
|
|
1267
|
-
code="no-map-curie-violation",
|
|
1268
|
-
)
|
|
1269
|
-
dropped: int = int(blank_mask.sum())
|
|
1270
|
-
if dropped:
|
|
1271
|
-
logger.info(
|
|
1272
|
-
"no_map column {col} in {config} ({hash}) dropped {n} blank row(s): a missing CURIE cannot form a node",
|
|
1273
|
-
col=col,
|
|
1274
|
-
config=config_file,
|
|
1275
|
-
hash=section,
|
|
1276
|
-
n=dropped,
|
|
1277
|
-
)
|
|
1278
|
-
present: pl.Expr = pl.col(col).is_not_null() & ~pl.col(col).str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT))
|
|
1279
|
-
return lf.filter(present).with_columns(pl.lit("biolink:NamedThing").alias(f"{col}_category"))
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
1224
|
def to_store(lf: pl.LazyFrame, p: Path, config_name: str) -> Path:
|
|
1283
1225
|
"""Collect and write a section LazyFrame to a parquet file.
|
|
1284
1226
|
|
|
@@ -1437,25 +1379,16 @@ class Tcode(Section):
|
|
|
1437
1379
|
enabled.
|
|
1438
1380
|
"""
|
|
1439
1381
|
# Subject/object/qualifiers share one resolve_batch call instead of one per column.
|
|
1440
|
-
#
|
|
1441
|
-
#
|
|
1442
|
-
#
|
|
1443
|
-
# rejects), and no_map nodes, whose values are already CURIEs used verbatim.
|
|
1382
|
+
# Enum-ranged qualifiers are excluded from resolution: their range is a closed
|
|
1383
|
+
# Biolink vocabulary, so sending them through the fullmap would turn the required
|
|
1384
|
+
# token `increased` into the CURIE `UMLS:C0205217`, which the slot rejects.
|
|
1444
1385
|
qualifiers: list[Qualifier] = self.statement.qualifiers or []
|
|
1445
|
-
subj_obj: list[tuple[NodeEncoding, str]] = [(self.statement.subject, "subject"), (self.statement.object, "object")]
|
|
1446
1386
|
node_columns: list[tuple[NodeEncoding, str]] = [
|
|
1447
|
-
|
|
1387
|
+
(self.statement.subject, "subject"),
|
|
1388
|
+
(self.statement.object, "object"),
|
|
1448
1389
|
*[(x, x.qualifier) for x in qualifiers if x.resolved],
|
|
1449
1390
|
]
|
|
1450
|
-
|
|
1451
|
-
# enum-ranged qualifier is rejected at config time). Their encoded values become
|
|
1452
|
-
# node ids verbatim: encode only, then curie_audit drops blank rows, fails on
|
|
1453
|
-
# non-CURIE cells, and synthesizes the category column edge derivation needs.
|
|
1454
|
-
verbatim: list[tuple[NodeEncoding, str]] = [
|
|
1455
|
-
*[(x, col) for x, col in subj_obj if x.no_map],
|
|
1456
|
-
*[(x, x.qualifier) for x in qualifiers if not x.resolved and x.no_map],
|
|
1457
|
-
]
|
|
1458
|
-
literals: list[Qualifier] = [x for x in qualifiers if not x.resolved and not x.no_map]
|
|
1391
|
+
literals: list[Qualifier] = [x for x in qualifiers if not x.resolved]
|
|
1459
1392
|
# A nullable qualifier keeps its edge when the cell is blank or unresolvable (the
|
|
1460
1393
|
# column stays null and the null-stripper omits the key); subject/object and
|
|
1461
1394
|
# strict qualifiers drop the row as before.
|
|
@@ -1478,18 +1411,6 @@ class Tcode(Section):
|
|
|
1478
1411
|
# ranged column ends with a vocabulary audit: nothing else validates its cell
|
|
1479
1412
|
# values against the closed Biolink range before KGX emission.
|
|
1480
1413
|
[[*self.encoding(x, x.qualifier), (qualifier_vocabulary_audit, (str(x.qualifier), sorted(x.vocabulary or ())))] for x in literals],
|
|
1481
|
-
# no_map columns are encoded exactly like enum-ranged literals (no NLP
|
|
1482
|
-
# normalization -- it exists only to feed entity resolution these columns
|
|
1483
|
-
# never undergo), then finalized by curie_audit: blank rows dropped,
|
|
1484
|
-
# non-CURIE cells failed loudly, and the pinned `biolink:NamedThing`
|
|
1485
|
-
# category added for downstream edge derivation. The `<col>_pre_resolution`
|
|
1486
|
-
# snapshot rides along because `_collect_subframes` discovers node columns
|
|
1487
|
-
# through it at graph-compile time; without it a verbatim subject/object
|
|
1488
|
-
# would never be split out of the edges into nodes.
|
|
1489
|
-
[
|
|
1490
|
-
[*self.encoding(x, col), (column, (f"{col}_pre_resolution", col)), (curie_audit, (col, self.store.stem, self.config.name))]
|
|
1491
|
-
for x, col in verbatim
|
|
1492
|
-
],
|
|
1493
1414
|
# Raw ``column_<n>`` columns are dead weight once the encodings above have
|
|
1494
1415
|
# copied them into named slots; trim before resolution so the frame that
|
|
1495
1416
|
# resolve_batch materializes and joins stays narrow.
|
|
@@ -1654,7 +1575,6 @@ PHASE_OF: dict[Callable, str] = {
|
|
|
1654
1575
|
fullmap_audit: "qc",
|
|
1655
1576
|
column: "encode",
|
|
1656
1577
|
qualifier_vocabulary_audit: "encode",
|
|
1657
|
-
curie_audit: "encode",
|
|
1658
1578
|
edge_category: "edge",
|
|
1659
1579
|
publications: "provenance",
|
|
1660
1580
|
retrieval_sources: "provenance",
|
|
@@ -1925,15 +1845,24 @@ def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fi
|
|
|
1925
1845
|
so it never ships in the final NDJSON.
|
|
1926
1846
|
|
|
1927
1847
|
Returns:
|
|
1928
|
-
``None``;
|
|
1929
|
-
``.tmp`` suffix, then deletes ``p_in``.
|
|
1848
|
+
``None``; atomically replaces the deduplicated stream alongside ``p_in`` with no
|
|
1849
|
+
``.tmp`` suffix, then deletes ``p_in``. Parsing, collision, and native writer
|
|
1850
|
+
failures preserve the existing output and the input for diagnosis.
|
|
1930
1851
|
"""
|
|
1931
1852
|
p_out: Path = p_in.with_suffix("")
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1853
|
+
if p_out == p_in:
|
|
1854
|
+
raise ValueError("dedup_stream requires an input filename with a removable suffix")
|
|
1855
|
+
|
|
1856
|
+
# Keep the previous artifact intact until native parsing/dedup completes.
|
|
1857
|
+
# A sibling directory keeps replacement on the same filesystem and retains
|
|
1858
|
+
# the public basename for native collision diagnostics and fold reports.
|
|
1859
|
+
with TemporaryDirectory(prefix=f".{p_out.name}.dedup-", dir=p_out.parent) as stage_dir:
|
|
1860
|
+
staged: Path = Path(stage_dir) / p_out.name
|
|
1861
|
+
merged, scalar_conflicts = rs.dedup_ndjson(p_in, staged, is_edges, domain, uuid_fields, on_collision)
|
|
1862
|
+
report: Path = staged.with_name(staged.name + ".foldreport.json")
|
|
1863
|
+
staged.replace(p_out)
|
|
1864
|
+
if report.exists():
|
|
1865
|
+
report.replace(p_out.with_name(p_out.name + ".foldreport.json"))
|
|
1937
1866
|
if merged:
|
|
1938
1867
|
logger.warning(
|
|
1939
1868
|
"uuid_on_collision=merge folded {merged} divergent edge records into their first-seen edges; {conflicts} conflicting scalar fields kept the first value",
|
|
@@ -47,11 +47,20 @@ else:
|
|
|
47
47
|
pl = LazyModule("polars")
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
# Deprecated config keys -> guidance.
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
DEPRECATED_KEYS: dict[str, str] = {
|
|
50
|
+
# Deprecated config keys -> guidance. The before-hook below only warns and returns the data
|
|
51
|
+
# unchanged: a key that is no longer a valid field is STILL rejected by extra="forbid" unless the
|
|
52
|
+
# hook also pops/translates it, so registering a key here supplies the warning half only. That is
|
|
53
|
+
# the right half for a removed feature: the config still fails, but it says why and what to do.
|
|
54
|
+
DEPRECATED_KEYS: dict[str, str] = {
|
|
55
|
+
"no_map": (
|
|
56
|
+
"`no_map` is removed: subject, object, and CURIE-ranged qualifier values resolve against the fullmap again. "
|
|
57
|
+
"The mode emitted node ids verbatim and pinned each one to `biolink:NamedThing`, which flattened every derived "
|
|
58
|
+
"edge to `biolink:Association`. Tables that already carry CURIEs need nothing else, because the fullmap indexes "
|
|
59
|
+
"CURIEs and their equivalent identifiers as terms, so a CURIE cell resolves back to its own CURIE. The usual "
|
|
60
|
+
"resolution rules still apply: `taxon`, `prioritize`/`avoid`, `exclude_prefixes`/`exclude_regex`, and a blank "
|
|
61
|
+
"cell still drops the row."
|
|
62
|
+
)
|
|
63
|
+
}
|
|
55
64
|
|
|
56
65
|
|
|
57
66
|
def _shown(name: str, target: str) -> str:
|
|
@@ -94,10 +103,11 @@ class TablaBase(BaseModel):
|
|
|
94
103
|
"""Soft-warn when a raw input key is registered in ``DEPRECATED_KEYS``.
|
|
95
104
|
|
|
96
105
|
Pure observation: returns ``data`` unchanged and never rejects (``extra="forbid"``
|
|
97
|
-
still governs
|
|
98
|
-
|
|
106
|
+
still governs keys that name no live field, so a removed feature's key warns and then
|
|
107
|
+
still fails; that pairing is the point, since silently dropping the key would build the
|
|
108
|
+
output the removal exists to prevent). ``no_map`` is the registered entry today.
|
|
99
109
|
Note: with ``validate_assignment=True`` pydantic also re-runs this on attribute set,
|
|
100
|
-
so a registered key warns on assignment too
|
|
110
|
+
so a registered key warns on assignment too.
|
|
101
111
|
"""
|
|
102
112
|
if isinstance(data, dict):
|
|
103
113
|
for key in data:
|
|
@@ -288,16 +298,6 @@ class Encoding(TablaBase):
|
|
|
288
298
|
)
|
|
289
299
|
|
|
290
300
|
|
|
291
|
-
#: CURIE shape enforced on verbatim (``no_map``) node values: non-empty ``PREFIX:local-id``.
|
|
292
|
-
#: Shared by config-time literal validation (models) and build-time cell audits (lib).
|
|
293
|
-
CURIE_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]*:[A-Za-z0-9_.\-/]+$")
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
def is_curie(value: object) -> bool:
|
|
297
|
-
"""Whether ``value`` is a CURIE-shaped string (``PREFIX:local-id``, both halves non-empty)."""
|
|
298
|
-
return isinstance(value, str) and CURIE_PATTERN.match(value.strip()) is not None
|
|
299
|
-
|
|
300
|
-
|
|
301
301
|
class NodeEncoding(Encoding):
|
|
302
302
|
taxon: PositiveInt | None = Field(
|
|
303
303
|
9606,
|
|
@@ -322,87 +322,6 @@ class NodeEncoding(Encoding):
|
|
|
322
322
|
examples=[["^OMIM:\\d+$"]],
|
|
323
323
|
)
|
|
324
324
|
|
|
325
|
-
no_map: bool = Field(
|
|
326
|
-
False,
|
|
327
|
-
description=(
|
|
328
|
-
"When True, skip fullmap entity resolution: the encoded value is used verbatim as the node CURIE, "
|
|
329
|
-
"for data that already carries its own CURIEs. No NLP normalization, taxon constraint, or category "
|
|
330
|
-
"lookup is applied; blank cells are dropped like unresolved nodes and non-CURIE cells fail the build."
|
|
331
|
-
),
|
|
332
|
-
examples=[True],
|
|
333
|
-
)
|
|
334
|
-
|
|
335
|
-
@model_validator(mode="before")
|
|
336
|
-
@classmethod
|
|
337
|
-
def no_map_rejects_explicit_taxon(cls, data: Any) -> Any:
|
|
338
|
-
"""Reject ``no_map`` with an explicit non-default ``taxon`` (dead resolution config).
|
|
339
|
-
|
|
340
|
-
``taxon`` defaults to 9606 and only steers fullmap ranking, so an explicit
|
|
341
|
-
non-default taxon alongside ``no_map: true`` is config that can never take
|
|
342
|
-
effect. The default itself stays indistinguishable from an explicit 9606 and
|
|
343
|
-
is therefore allowed. Checked ``before`` precisely so "explicitly set" is
|
|
344
|
-
detectable from the raw input dict.
|
|
345
|
-
"""
|
|
346
|
-
if isinstance(data, dict) and data.get("no_map") and "taxon" in data and data["taxon"] not in (None, 9606):
|
|
347
|
-
raise TablassertValidationError(
|
|
348
|
-
f"`no_map: true` skips entity resolution, so `taxon: {data['taxon']}` can never take effect; remove the taxon constraint.",
|
|
349
|
-
code="no-map-dead-config",
|
|
350
|
-
)
|
|
351
|
-
return data
|
|
352
|
-
|
|
353
|
-
@model_validator(mode="after")
|
|
354
|
-
def no_map_rejects_resolution_knobs(self: Self) -> Self:
|
|
355
|
-
"""Reject ``no_map`` combined with resolution-only knobs (dead config).
|
|
356
|
-
|
|
357
|
-
``prioritize``, ``avoid``, ``exclude_prefixes``, and ``exclude_regex`` only
|
|
358
|
-
steer fullmap ranking and filtering; with ``no_map: true`` no lookup happens,
|
|
359
|
-
so these fields mislead the reader. Fail loudly at config time, like the
|
|
360
|
-
``nullable``-on-literal check on :class:`Qualifier`.
|
|
361
|
-
"""
|
|
362
|
-
if not self.no_map:
|
|
363
|
-
return self
|
|
364
|
-
dead: list[str] = [
|
|
365
|
-
name
|
|
366
|
-
for name, value in (
|
|
367
|
-
("prioritize", self.prioritize),
|
|
368
|
-
("avoid", self.avoid),
|
|
369
|
-
("exclude_prefixes", self.exclude_prefixes),
|
|
370
|
-
("exclude_regex", self.exclude_regex),
|
|
371
|
-
)
|
|
372
|
-
if value
|
|
373
|
-
]
|
|
374
|
-
if dead:
|
|
375
|
-
raise TablassertValidationError(
|
|
376
|
-
f"`no_map: true` skips entity resolution, so {', '.join(f'`{name}`' for name in dead)} can never take effect; remove them.",
|
|
377
|
-
code="no-map-dead-config",
|
|
378
|
-
)
|
|
379
|
-
return self
|
|
380
|
-
|
|
381
|
-
@model_validator(mode="after")
|
|
382
|
-
def no_map_literal_must_be_curie(self: Self) -> Self:
|
|
383
|
-
"""Validate that a ``no_map`` literal encodes a CURIE at config time.
|
|
384
|
-
|
|
385
|
-
A ``method: value`` node with ``no_map: true`` becomes the node id verbatim,
|
|
386
|
-
so the literal (after ``prefix``/``suffix`` are applied, matching the build's
|
|
387
|
-
``encoding()``) must already be CURIE-shaped; otherwise every row would be
|
|
388
|
-
dropped at build time. ``method: column`` cells are audited per row at build
|
|
389
|
-
time instead.
|
|
390
|
-
"""
|
|
391
|
-
if not self.no_map or self.method != EncodingMethods.VALUE:
|
|
392
|
-
return self
|
|
393
|
-
# Enum-ranged qualifier literals are closed-vocabulary tokens (audited against
|
|
394
|
-
# the vocabulary, and `no_map` on them is rejected as dead config), not CURIEs.
|
|
395
|
-
if getattr(self, "vocabulary", None) is not None:
|
|
396
|
-
return self
|
|
397
|
-
literal: str = f"{self.prefix or ''}{self.encoding}{self.suffix or ''}".strip()
|
|
398
|
-
if not is_curie(literal):
|
|
399
|
-
raise TablassertValidationError(
|
|
400
|
-
f"`no_map: true` uses the encoded value verbatim as the node CURIE, but the literal {literal!r} is not CURIE-shaped "
|
|
401
|
-
"(expected `PREFIX:local-id`). Fix the literal, or add `prefix`/`suffix` to form a CURIE.",
|
|
402
|
-
code="no-map-bad-curie",
|
|
403
|
-
)
|
|
404
|
-
return self
|
|
405
|
-
|
|
406
325
|
@field_validator("exclude_regex", mode="after")
|
|
407
326
|
@classmethod
|
|
408
327
|
def polars_compatible_exclude_regex(cls, exclude_regex: list[str] | None) -> list[str] | None:
|
|
@@ -450,26 +369,10 @@ class Qualifier(NodeEncoding):
|
|
|
450
369
|
def resolved(self: Self) -> bool:
|
|
451
370
|
"""Whether this qualifier's values go through fullmap entity resolution.
|
|
452
371
|
|
|
453
|
-
|
|
454
|
-
and must be passed through verbatim.
|
|
455
|
-
unless they opt out with ``no_map: true`` (their values are already CURIEs).
|
|
456
|
-
"""
|
|
457
|
-
return self.vocabulary is None and not self.no_map
|
|
458
|
-
|
|
459
|
-
@model_validator(mode="after")
|
|
460
|
-
def reject_no_map_on_enum_ranged(self: Self) -> Self:
|
|
461
|
-
"""Reject ``no_map: true`` on enum-ranged qualifiers as dead config.
|
|
462
|
-
|
|
463
|
-
Enum-ranged qualifiers already pass their literal tokens through verbatim, so
|
|
464
|
-
``no_map`` would be a no-op that misleads the reader; fail loudly at config
|
|
465
|
-
time, like the ``nullable``-on-literal check below.
|
|
372
|
+
Only CURIE-ranged qualifiers are resolved. Enum-ranged ones carry a literal
|
|
373
|
+
token from a closed Biolink vocabulary and must be passed through verbatim.
|
|
466
374
|
"""
|
|
467
|
-
|
|
468
|
-
raise TablassertValidationError(
|
|
469
|
-
f"{self.qualifier} has a closed Biolink vocabulary and already passes values through verbatim, so `no_map` is redundant; remove it.",
|
|
470
|
-
code="no-map-dead-config",
|
|
471
|
-
)
|
|
472
|
-
return self
|
|
375
|
+
return self.vocabulary is None
|
|
473
376
|
|
|
474
377
|
@model_validator(mode="after")
|
|
475
378
|
def reject_disabled_qualifiers(self: Self) -> Self:
|
|
@@ -272,9 +272,6 @@ OP_COST: dict[Callable, int] = {
|
|
|
272
272
|
lib.split_list: 1,
|
|
273
273
|
# Vocabulary audit: one eager distinct-value pass over a single enum column.
|
|
274
274
|
lib.qualifier_vocabulary_audit: 1,
|
|
275
|
-
# CURIE audit: one eager pass over a single no_map column (distinct-value check,
|
|
276
|
-
# blank-row filter, category column synthesis).
|
|
277
|
-
lib.curie_audit: 1,
|
|
278
275
|
# Clean: coerce_columns applies five coercion rules (schema resolve + renames + value
|
|
279
276
|
# coercion) in one pass, so it outprices a single-expression clean op.
|
|
280
277
|
lib.coerce_columns: 3,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|