tablassert 21.0.0__tar.gz → 22.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {tablassert-21.0.0 → tablassert-22.0.1}/PKG-INFO +10 -8
  2. {tablassert-21.0.0 → tablassert-22.0.1}/README.md +9 -7
  3. {tablassert-21.0.0 → tablassert-22.0.1}/pyproject.toml +1 -1
  4. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/errors.py +0 -3
  5. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/fullmap.py +27 -2
  6. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/lib.py +24 -95
  7. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/models.py +21 -118
  8. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/runcache.py +0 -3
  9. {tablassert-21.0.0 → tablassert-22.0.1}/LICENSE +0 -0
  10. {tablassert-21.0.0 → tablassert-22.0.1}/rust/Cargo.lock +0 -0
  11. {tablassert-21.0.0 → tablassert-22.0.1}/rust/Cargo.toml +0 -0
  12. {tablassert-21.0.0 → tablassert-22.0.1}/rust/examples/count_tables.rs +0 -0
  13. {tablassert-21.0.0 → tablassert-22.0.1}/rust/src/fullmap.rs +0 -0
  14. {tablassert-21.0.0 → tablassert-22.0.1}/rust/src/json.rs +0 -0
  15. {tablassert-21.0.0 → tablassert-22.0.1}/rust/src/lib.rs +0 -0
  16. {tablassert-21.0.0 → tablassert-22.0.1}/rust/src/ndjson.rs +0 -0
  17. {tablassert-21.0.0 → tablassert-22.0.1}/rust/src/nlp.rs +0 -0
  18. {tablassert-21.0.0 → tablassert-22.0.1}/rust/src/uuid.rs +0 -0
  19. {tablassert-21.0.0 → tablassert-22.0.1}/rust/tests/build_golden.rs +0 -0
  20. {tablassert-21.0.0 → tablassert-22.0.1}/rust/tests/common/mod.rs +0 -0
  21. {tablassert-21.0.0 → tablassert-22.0.1}/rust/tests/extract_prebuilt.rs +0 -0
  22. {tablassert-21.0.0 → tablassert-22.0.1}/rust/tests/fixtures/nlp_golden.tsv +0 -0
  23. {tablassert-21.0.0 → tablassert-22.0.1}/rust/tests/nlp_golden.rs +0 -0
  24. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/__init__.py +0 -0
  25. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/_lazy.py +0 -0
  26. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/biolink.py +0 -0
  27. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/cli.py +0 -0
  28. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/cli_entry.py +0 -0
  29. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/coerce.py +0 -0
  30. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/data/experimental_taxa.yaml +0 -0
  31. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/data/infores_catalog.yaml +0 -0
  32. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/enums.py +0 -0
  33. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/extras.py +0 -0
  34. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/infores_registry.py +0 -0
  35. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/ingests.py +0 -0
  36. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/log.py +0 -0
  37. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/net.py +0 -0
  38. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/nlp.py +0 -0
  39. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/progress.py +0 -0
  40. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/qc.py +0 -0
  41. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/rig.py +0 -0
  42. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/rs.pyi +0 -0
  43. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/study.py +0 -0
  44. {tablassert-21.0.0 → tablassert-22.0.1}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 21.0.0
3
+ Version: 22.0.1
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -163,15 +163,15 @@ Build the knowledge graph:
163
163
  tablassert build-kg graph.yaml
164
164
  ```
165
165
 
166
- Output is one JSON object per line: nodes with Biolink categories, edges with annotations.
166
+ Output is one JSON object per line: nodes with Biolink categories, edges with annotations. The edge sample below is abridged; real records also carry provenance fields such as `knowledge_level`, `agent_type`, `sources`, and `original_subject`/`original_object`.
167
167
 
168
168
  ```json
169
- {"id":"HGNC:11998","name":"TP53","category":["biolink:Gene"],"taxon":"NCBITaxon:9606"}
170
- {"id":"MONDO:0008903","name":"lung cancer","category":["biolink:Disease"]}
169
+ {"id":"HGNC:11998","name":"TP53","category":["biolink:Gene"],"in_taxon":["NCBITaxon:9606"],"provided_by":["infores:my-kg"]}
170
+ {"id":"MONDO:0008903","name":"lung cancer","category":["biolink:Disease"],"in_taxon":["NCBITaxon:9606"],"provided_by":["infores:my-kg"]}
171
171
  ```
172
172
 
173
173
  ```json
174
- {"id":"2cfea591-0f8f-33af-a7df-03da531d3359","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
174
+ {"id":"8f41c2d7-9b3e-35c1-a4e8-6d2f9a0c5b71","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"very_strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.yaml SECTION 1","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
175
175
  ```
176
176
 
177
177
  See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full walkthrough.
@@ -194,8 +194,10 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
194
194
  pip install tablassert
195
195
  ```
196
196
 
197
- Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API; `[cli]`
198
- provides the `tablassert` command. Other extras are opt-in:
197
+ To use the Python API, install Tablassert into your project's environment with `pip install
198
+ "tablassert"` (or `uv add tablassert`). To get the `tablassert` command as a standalone tool, use
199
+ `uv tool install "tablassert[cli]"`; note that a tool install is isolated and does not expose the
200
+ Python API to your project. Other extras are opt-in:
199
201
 
200
202
  | Extra | Adds | Install |
201
203
  | ----- | ---- | ------- |
@@ -271,5 +273,5 @@ described in:
271
273
 
272
274
  - [Skye Lane Goetz](mailto:sgoetz@isbscience.org), Institute for Systems Biology
273
275
  - [Gwênlyn Glusman](mailto:gglusman@isbscience.org), Institute for Systems Biology
274
- - Jared C. Roach, Institute for Systems Biology
276
+ - [Jared C. Roach](mailto:jroach@isbscience.org), Institute for Systems Biology
275
277
 
@@ -109,15 +109,15 @@ Build the knowledge graph:
109
109
  tablassert build-kg graph.yaml
110
110
  ```
111
111
 
112
- Output is one JSON object per line: nodes with Biolink categories, edges with annotations.
112
+ Output is one JSON object per line: nodes with Biolink categories, edges with annotations. The edge sample below is abridged; real records also carry provenance fields such as `knowledge_level`, `agent_type`, `sources`, and `original_subject`/`original_object`.
113
113
 
114
114
  ```json
115
- {"id":"HGNC:11998","name":"TP53","category":["biolink:Gene"],"taxon":"NCBITaxon:9606"}
116
- {"id":"MONDO:0008903","name":"lung cancer","category":["biolink:Disease"]}
115
+ {"id":"HGNC:11998","name":"TP53","category":["biolink:Gene"],"in_taxon":["NCBITaxon:9606"],"provided_by":["infores:my-kg"]}
116
+ {"id":"MONDO:0008903","name":"lung cancer","category":["biolink:Disease"],"in_taxon":["NCBITaxon:9606"],"provided_by":["infores:my-kg"]}
117
117
  ```
118
118
 
119
119
  ```json
120
- {"id":"2cfea591-0f8f-33af-a7df-03da531d3359","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
120
+ {"id":"8f41c2d7-9b3e-35c1-a4e8-6d2f9a0c5b71","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"very_strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.yaml SECTION 1","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
121
121
  ```
122
122
 
123
123
  See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full walkthrough.
@@ -140,8 +140,10 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
140
140
  pip install tablassert
141
141
  ```
142
142
 
143
- Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API; `[cli]`
144
- provides the `tablassert` command. Other extras are opt-in:
143
+ To use the Python API, install Tablassert into your project's environment with `pip install
144
+ "tablassert"` (or `uv add tablassert`). To get the `tablassert` command as a standalone tool, use
145
+ `uv tool install "tablassert[cli]"`; note that a tool install is isolated and does not expose the
146
+ Python API to your project. Other extras are opt-in:
145
147
 
146
148
  | Extra | Adds | Install |
147
149
  | ----- | ---- | ------- |
@@ -217,4 +219,4 @@ described in:
217
219
 
218
220
  - [Skye Lane Goetz](mailto:sgoetz@isbscience.org), Institute for Systems Biology
219
221
  - [Gwênlyn Glusman](mailto:gglusman@isbscience.org), Institute for Systems Biology
220
- - Jared C. Roach, Institute for Systems Biology
222
+ - [Jared C. Roach](mailto:jroach@isbscience.org), Institute for Systems Biology
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "21.0.0"
3
+ version = "22.0.1"
4
4
  description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -63,9 +63,6 @@ TablassertErrorCodes = Literal[
63
63
  "provenance-missing-publication",
64
64
  "provenance-publication-and-override",
65
65
  "encoding-list-method-removed",
66
- "no-map-dead-config",
67
- "no-map-bad-curie",
68
- "no-map-curie-violation",
69
66
  "annotation-split-by-requires-column",
70
67
  "annotation-split-by-empty",
71
68
  "field-disabled",
@@ -4,6 +4,7 @@ import time
4
4
  from collections import OrderedDict
5
5
  from collections.abc import Callable
6
6
  from enum import Enum
7
+ from functools import cache
7
8
  from pathlib import Path
8
9
  from typing import TYPE_CHECKING, Any, NamedTuple, cast
9
10
 
@@ -269,6 +270,28 @@ def lookup_rows(db: Path, terms: list[str]) -> list[dict[str, object]]:
269
270
  return rows
270
271
 
271
272
 
273
+ @cache
274
+ def _generic_probe_keys() -> frozenset[str]:
275
+ """Apply the real lookup normalizers to the existing excluded generic labels."""
276
+ labels = [
277
+ "none",
278
+ "nan",
279
+ "na",
280
+ "null",
281
+ "unknown",
282
+ "not applicable",
283
+ "p_value",
284
+ "p value",
285
+ "variable",
286
+ "result",
287
+ "exposure",
288
+ "expression",
289
+ "symbol",
290
+ ]
291
+ normalized = level_two(level_one(pl.LazyFrame({"term": labels}), "term"), "term").collect()
292
+ return frozenset(labels) | frozenset(normalized["term"]) | frozenset(normalized["term_two"])
293
+
294
+
272
295
  def distinct(lf: pl.LazyFrame, l1: str, l2: str, col: str = "term") -> pl.LazyFrame:
273
296
  """Extract unique terms from two text-normalization columns as a LazyFrame.
274
297
 
@@ -297,8 +320,10 @@ def distinct(lf: pl.LazyFrame, l1: str, l2: str, col: str = "term") -> pl.LazyFr
297
320
  # concat/unique, so first-seen tier selection would be scheduling-dependent.
298
321
  terms: pl.LazyFrame = pl.concat([t1, t2]).group_by(col).agg(pl.col("nlp_level").min())
299
322
 
300
- bad: str = r"^\d+$|^(none|nan|na|null|unknown|not applicable|p_value|variable|result|exposure|expression|symbol)$|^$"
301
- return terms.filter(~pl.col(col).str.contains(bad))
323
+ # Probe keys are stemmed/sorted at level one, then stripped at level two.
324
+ # Filtering only the original spellings lets headers such as "Expression"
325
+ # ("express") and "Not applicable" ("applic not") resolve as entities.
326
+ return terms.filter(~pl.col(col).str.contains(r"^\d+$|^$") & ~pl.col(col).is_in(_generic_probe_keys()))
302
327
 
303
328
 
304
329
  def deduplicate_result(result: pl.DataFrame, column_context: bool) -> pl.DataFrame:
@@ -6,6 +6,7 @@ import re
6
6
  from collections.abc import Callable, Iterable
7
7
  from functools import cache
8
8
  from pathlib import Path
9
+ from tempfile import TemporaryDirectory
9
10
  from typing import TYPE_CHECKING, Any, Literal, Self
10
11
 
11
12
  from pydantic import Field, NonNegativeInt
@@ -48,7 +49,7 @@ from tablassert.enums import EncodingMethods, Files, Functions, InformationResou
48
49
  from tablassert.errors import TablassertError
49
50
  from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
50
51
  from tablassert.log import cat
51
- from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section, is_curie
52
+ from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
52
53
  from tablassert.nlp import level_one, level_two
53
54
  from tablassert.qc import fullmap_audit
54
55
  from tablassert.rig import (
@@ -1220,65 +1221,6 @@ def qualifier_vocabulary_audit(lf: pl.LazyFrame, col: str, vocabulary: Iterable[
1220
1221
  )
1221
1222
 
1222
1223
 
1223
- def curie_audit(lf: pl.LazyFrame, col: str, section: str, config_file: str) -> pl.LazyFrame:
1224
- """Finalize a ``no_map`` node column: blank rows drop, non-CURIE values fail, category is synthesized.
1225
-
1226
- Columns configured with ``no_map: true`` bypass entity resolution, so nothing else
1227
- (a) drops rows whose cell is blank (a missing node id can never form an edge, exactly
1228
- like an unresolved subject/object), (b) rejects cells that are not CURIE-shaped -- the
1229
- encoded value becomes the KGX node id verbatim, so garbage would ship as an identifier --
1230
- or (c) produces the ``<col>_category`` column downstream stages require: without a
1231
- fullmap lookup there is no role to derive, so the column is pinned to
1232
- ``biolink:NamedThing`` and edge derivation degrades to ``biolink:Association`` exactly
1233
- like the documented no-role-lookup fallback in :func:`edge_category`.
1234
-
1235
- Args:
1236
- lf: Source LazyFrame.
1237
- col: The no_map node column (``subject``/``object`` or a CURIE-ranged qualifier key).
1238
- section: Short section hash (for log context).
1239
- config_file: Originating config file (for log context).
1240
-
1241
- Returns:
1242
- LazyFrame with blank/null-like rows dropped and ``<col>_category`` added.
1243
-
1244
- Notes:
1245
- A non-blank value that is not CURIE-shaped raises a coded error naming the
1246
- offending values (mirroring ``qualifier_vocabulary_audit``); blank and null-like
1247
- cells (``""``, ``"NA"``, ...) are dropped with a loud per-column count instead,
1248
- mirroring how an unresolvable subject/object row is dropped, never emitted.
1249
- """
1250
- values: pl.Series = lf.select(pl.col(col)).collect().get_column(col)
1251
- if values.dtype.base_type() in (pl.List, pl.Array):
1252
- values = values.explode()
1253
- blank_mask: pl.Series = values.is_null() | values.str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT))
1254
- # Judge CURIE shape on the distinct non-blank values only: the audit runs once per
1255
- # section, and map_elements is a per-element Python call that must not scale with
1256
- # row count when one bad cell repeats a million times.
1257
- distinct: pl.Series = values.filter(~blank_mask).unique()
1258
- offending: pl.Series = distinct.filter(~distinct.str.strip_chars().map_elements(is_curie, return_dtype=pl.Boolean)).sort()
1259
- if offending.len() > 0:
1260
- shown: list[str] = offending.head(10).to_list()
1261
- more: str = "..." if offending.len() > len(shown) else ""
1262
- raise TablassertError(
1263
- f"{col} is configured with `no_map: true` so its values are used verbatim as node CURIEs, but "
1264
- f"{offending.len()} non-blank value(s) in the built column are not CURIE-shaped: "
1265
- f"{', '.join(repr(x) for x in shown)}{more}. Fix the source table or the encoding "
1266
- "(prefix/suffix can form a CURIE from raw cells).",
1267
- code="no-map-curie-violation",
1268
- )
1269
- dropped: int = int(blank_mask.sum())
1270
- if dropped:
1271
- logger.info(
1272
- "no_map column {col} in {config} ({hash}) dropped {n} blank row(s): a missing CURIE cannot form a node",
1273
- col=col,
1274
- config=config_file,
1275
- hash=section,
1276
- n=dropped,
1277
- )
1278
- present: pl.Expr = pl.col(col).is_not_null() & ~pl.col(col).str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT))
1279
- return lf.filter(present).with_columns(pl.lit("biolink:NamedThing").alias(f"{col}_category"))
1280
-
1281
-
1282
1224
  def to_store(lf: pl.LazyFrame, p: Path, config_name: str) -> Path:
1283
1225
  """Collect and write a section LazyFrame to a parquet file.
1284
1226
 
@@ -1437,25 +1379,16 @@ class Tcode(Section):
1437
1379
  enabled.
1438
1380
  """
1439
1381
  # Subject/object/qualifiers share one resolve_batch call instead of one per column.
1440
- # Two kinds of columns never reach it: enum-ranged qualifiers, whose range is a
1441
- # closed Biolink vocabulary (sending them through the fullmap would turn the
1442
- # required token `increased` into the CURIE `UMLS:C0205217`, which the slot
1443
- # rejects), and no_map nodes, whose values are already CURIEs used verbatim.
1382
+ # Enum-ranged qualifiers are excluded from resolution: their range is a closed
1383
+ # Biolink vocabulary, so sending them through the fullmap would turn the required
1384
+ # token `increased` into the CURIE `UMLS:C0205217`, which the slot rejects.
1444
1385
  qualifiers: list[Qualifier] = self.statement.qualifiers or []
1445
- subj_obj: list[tuple[NodeEncoding, str]] = [(self.statement.subject, "subject"), (self.statement.object, "object")]
1446
1386
  node_columns: list[tuple[NodeEncoding, str]] = [
1447
- *[(x, col) for x, col in subj_obj if not x.no_map],
1387
+ (self.statement.subject, "subject"),
1388
+ (self.statement.object, "object"),
1448
1389
  *[(x, x.qualifier) for x in qualifiers if x.resolved],
1449
1390
  ]
1450
- # no_map subject/object plus CURIE-ranged qualifiers that opted out (a no_map
1451
- # enum-ranged qualifier is rejected at config time). Their encoded values become
1452
- # node ids verbatim: encode only, then curie_audit drops blank rows, fails on
1453
- # non-CURIE cells, and synthesizes the category column edge derivation needs.
1454
- verbatim: list[tuple[NodeEncoding, str]] = [
1455
- *[(x, col) for x, col in subj_obj if x.no_map],
1456
- *[(x, x.qualifier) for x in qualifiers if not x.resolved and x.no_map],
1457
- ]
1458
- literals: list[Qualifier] = [x for x in qualifiers if not x.resolved and not x.no_map]
1391
+ literals: list[Qualifier] = [x for x in qualifiers if not x.resolved]
1459
1392
  # A nullable qualifier keeps its edge when the cell is blank or unresolvable (the
1460
1393
  # column stays null and the null-stripper omits the key); subject/object and
1461
1394
  # strict qualifiers drop the row as before.
@@ -1478,18 +1411,6 @@ class Tcode(Section):
1478
1411
  # ranged column ends with a vocabulary audit: nothing else validates its cell
1479
1412
  # values against the closed Biolink range before KGX emission.
1480
1413
  [[*self.encoding(x, x.qualifier), (qualifier_vocabulary_audit, (str(x.qualifier), sorted(x.vocabulary or ())))] for x in literals],
1481
- # no_map columns are encoded exactly like enum-ranged literals (no NLP
1482
- # normalization -- it exists only to feed entity resolution these columns
1483
- # never undergo), then finalized by curie_audit: blank rows dropped,
1484
- # non-CURIE cells failed loudly, and the pinned `biolink:NamedThing`
1485
- # category added for downstream edge derivation. The `<col>_pre_resolution`
1486
- # snapshot rides along because `_collect_subframes` discovers node columns
1487
- # through it at graph-compile time; without it a verbatim subject/object
1488
- # would never be split out of the edges into nodes.
1489
- [
1490
- [*self.encoding(x, col), (column, (f"{col}_pre_resolution", col)), (curie_audit, (col, self.store.stem, self.config.name))]
1491
- for x, col in verbatim
1492
- ],
1493
1414
  # Raw ``column_<n>`` columns are dead weight once the encodings above have
1494
1415
  # copied them into named slots; trim before resolution so the frame that
1495
1416
  # resolve_batch materializes and joins stays narrow.
@@ -1654,7 +1575,6 @@ PHASE_OF: dict[Callable, str] = {
1654
1575
  fullmap_audit: "qc",
1655
1576
  column: "encode",
1656
1577
  qualifier_vocabulary_audit: "encode",
1657
- curie_audit: "encode",
1658
1578
  edge_category: "edge",
1659
1579
  publications: "provenance",
1660
1580
  retrieval_sources: "provenance",
@@ -1925,15 +1845,24 @@ def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fi
1925
1845
  so it never ships in the final NDJSON.
1926
1846
 
1927
1847
  Returns:
1928
- ``None``; writes the deduplicated stream alongside ``p_in`` with no
1929
- ``.tmp`` suffix, then deletes ``p_in``.
1848
+ ``None``; atomically replaces the deduplicated stream alongside ``p_in`` with no
1849
+ ``.tmp`` suffix, then deletes ``p_in``. Parsing, collision, and native writer
1850
+ failures preserve the existing output and the input for diagnosis.
1930
1851
  """
1931
1852
  p_out: Path = p_in.with_suffix("")
1932
-
1933
- if p_out.is_file():
1934
- p_out.unlink()
1935
-
1936
- merged, scalar_conflicts = rs.dedup_ndjson(p_in, p_out, is_edges, domain, uuid_fields, on_collision)
1853
+ if p_out == p_in:
1854
+ raise ValueError("dedup_stream requires an input filename with a removable suffix")
1855
+
1856
+ # Keep the previous artifact intact until native parsing/dedup completes.
1857
+ # A sibling directory keeps replacement on the same filesystem and retains
1858
+ # the public basename for native collision diagnostics and fold reports.
1859
+ with TemporaryDirectory(prefix=f".{p_out.name}.dedup-", dir=p_out.parent) as stage_dir:
1860
+ staged: Path = Path(stage_dir) / p_out.name
1861
+ merged, scalar_conflicts = rs.dedup_ndjson(p_in, staged, is_edges, domain, uuid_fields, on_collision)
1862
+ report: Path = staged.with_name(staged.name + ".foldreport.json")
1863
+ staged.replace(p_out)
1864
+ if report.exists():
1865
+ report.replace(p_out.with_name(p_out.name + ".foldreport.json"))
1937
1866
  if merged:
1938
1867
  logger.warning(
1939
1868
  "uuid_on_collision=merge folded {merged} divergent edge records into their first-seen edges; {conflicts} conflicting scalar fields kept the first value",
@@ -47,11 +47,20 @@ else:
47
47
  pl = LazyModule("polars")
48
48
 
49
49
 
50
- # Deprecated config keys -> guidance. EMPTY today: the before-hook below is a silent no-op until a
51
- # future rename registers a key here to WARN on it. The hook only warns and returns the data
52
- # unchanged — a renamed key that is no longer a valid field is STILL rejected by extra="forbid"
53
- # unless the future hook also pops/translates it; registering a key here supplies the warning half only.
54
- DEPRECATED_KEYS: dict[str, str] = {}
50
+ # Deprecated config keys -> guidance. The before-hook below only warns and returns the data
51
+ # unchanged: a key that is no longer a valid field is STILL rejected by extra="forbid" unless the
52
+ # hook also pops/translates it, so registering a key here supplies the warning half only. That is
53
+ # the right half for a removed feature: the config still fails, but it says why and what to do.
54
+ DEPRECATED_KEYS: dict[str, str] = {
55
+ "no_map": (
56
+ "`no_map` is removed: subject, object, and CURIE-ranged qualifier values resolve against the fullmap again. "
57
+ "The mode emitted node ids verbatim and pinned each one to `biolink:NamedThing`, which flattened every derived "
58
+ "edge to `biolink:Association`. Tables that already carry CURIEs need nothing else, because the fullmap indexes "
59
+ "CURIEs and their equivalent identifiers as terms, so a CURIE cell resolves back to its own CURIE. The usual "
60
+ "resolution rules still apply: `taxon`, `prioritize`/`avoid`, `exclude_prefixes`/`exclude_regex`, and a blank "
61
+ "cell still drops the row."
62
+ )
63
+ }
55
64
 
56
65
 
57
66
  def _shown(name: str, target: str) -> str:
@@ -94,10 +103,11 @@ class TablaBase(BaseModel):
94
103
  """Soft-warn when a raw input key is registered in ``DEPRECATED_KEYS``.
95
104
 
96
105
  Pure observation: returns ``data`` unchanged and never rejects (``extra="forbid"``
97
- still governs truly-unknown keys). The registry is EMPTY today, so this is a silent
98
- no-op; a future field rename registers its old key here to warn instead of hard-fail.
106
+ still governs keys that name no live field, so a removed feature's key warns and then
107
+ still fails; that pairing is the point, since silently dropping the key would build the
108
+ output the removal exists to prevent). ``no_map`` is the registered entry today.
99
109
  Note: with ``validate_assignment=True`` pydantic also re-runs this on attribute set,
100
- so a registered key warns on assignment too — inert while the registry is empty.
110
+ so a registered key warns on assignment too.
101
111
  """
102
112
  if isinstance(data, dict):
103
113
  for key in data:
@@ -288,16 +298,6 @@ class Encoding(TablaBase):
288
298
  )
289
299
 
290
300
 
291
- #: CURIE shape enforced on verbatim (``no_map``) node values: non-empty ``PREFIX:local-id``.
292
- #: Shared by config-time literal validation (models) and build-time cell audits (lib).
293
- CURIE_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]*:[A-Za-z0-9_.\-/]+$")
294
-
295
-
296
- def is_curie(value: object) -> bool:
297
- """Whether ``value`` is a CURIE-shaped string (``PREFIX:local-id``, both halves non-empty)."""
298
- return isinstance(value, str) and CURIE_PATTERN.match(value.strip()) is not None
299
-
300
-
301
301
  class NodeEncoding(Encoding):
302
302
  taxon: PositiveInt | None = Field(
303
303
  9606,
@@ -322,87 +322,6 @@ class NodeEncoding(Encoding):
322
322
  examples=[["^OMIM:\\d+$"]],
323
323
  )
324
324
 
325
- no_map: bool = Field(
326
- False,
327
- description=(
328
- "When True, skip fullmap entity resolution: the encoded value is used verbatim as the node CURIE, "
329
- "for data that already carries its own CURIEs. No NLP normalization, taxon constraint, or category "
330
- "lookup is applied; blank cells are dropped like unresolved nodes and non-CURIE cells fail the build."
331
- ),
332
- examples=[True],
333
- )
334
-
335
- @model_validator(mode="before")
336
- @classmethod
337
- def no_map_rejects_explicit_taxon(cls, data: Any) -> Any:
338
- """Reject ``no_map`` with an explicit non-default ``taxon`` (dead resolution config).
339
-
340
- ``taxon`` defaults to 9606 and only steers fullmap ranking, so an explicit
341
- non-default taxon alongside ``no_map: true`` is config that can never take
342
- effect. The default itself stays indistinguishable from an explicit 9606 and
343
- is therefore allowed. Checked ``before`` precisely so "explicitly set" is
344
- detectable from the raw input dict.
345
- """
346
- if isinstance(data, dict) and data.get("no_map") and "taxon" in data and data["taxon"] not in (None, 9606):
347
- raise TablassertValidationError(
348
- f"`no_map: true` skips entity resolution, so `taxon: {data['taxon']}` can never take effect; remove the taxon constraint.",
349
- code="no-map-dead-config",
350
- )
351
- return data
352
-
353
- @model_validator(mode="after")
354
- def no_map_rejects_resolution_knobs(self: Self) -> Self:
355
- """Reject ``no_map`` combined with resolution-only knobs (dead config).
356
-
357
- ``prioritize``, ``avoid``, ``exclude_prefixes``, and ``exclude_regex`` only
358
- steer fullmap ranking and filtering; with ``no_map: true`` no lookup happens,
359
- so these fields mislead the reader. Fail loudly at config time, like the
360
- ``nullable``-on-literal check on :class:`Qualifier`.
361
- """
362
- if not self.no_map:
363
- return self
364
- dead: list[str] = [
365
- name
366
- for name, value in (
367
- ("prioritize", self.prioritize),
368
- ("avoid", self.avoid),
369
- ("exclude_prefixes", self.exclude_prefixes),
370
- ("exclude_regex", self.exclude_regex),
371
- )
372
- if value
373
- ]
374
- if dead:
375
- raise TablassertValidationError(
376
- f"`no_map: true` skips entity resolution, so {', '.join(f'`{name}`' for name in dead)} can never take effect; remove them.",
377
- code="no-map-dead-config",
378
- )
379
- return self
380
-
381
- @model_validator(mode="after")
382
- def no_map_literal_must_be_curie(self: Self) -> Self:
383
- """Validate that a ``no_map`` literal encodes a CURIE at config time.
384
-
385
- A ``method: value`` node with ``no_map: true`` becomes the node id verbatim,
386
- so the literal (after ``prefix``/``suffix`` are applied, matching the build's
387
- ``encoding()``) must already be CURIE-shaped; otherwise every row would be
388
- dropped at build time. ``method: column`` cells are audited per row at build
389
- time instead.
390
- """
391
- if not self.no_map or self.method != EncodingMethods.VALUE:
392
- return self
393
- # Enum-ranged qualifier literals are closed-vocabulary tokens (audited against
394
- # the vocabulary, and `no_map` on them is rejected as dead config), not CURIEs.
395
- if getattr(self, "vocabulary", None) is not None:
396
- return self
397
- literal: str = f"{self.prefix or ''}{self.encoding}{self.suffix or ''}".strip()
398
- if not is_curie(literal):
399
- raise TablassertValidationError(
400
- f"`no_map: true` uses the encoded value verbatim as the node CURIE, but the literal {literal!r} is not CURIE-shaped "
401
- "(expected `PREFIX:local-id`). Fix the literal, or add `prefix`/`suffix` to form a CURIE.",
402
- code="no-map-bad-curie",
403
- )
404
- return self
405
-
406
325
  @field_validator("exclude_regex", mode="after")
407
326
  @classmethod
408
327
  def polars_compatible_exclude_regex(cls, exclude_regex: list[str] | None) -> list[str] | None:
@@ -450,26 +369,10 @@ class Qualifier(NodeEncoding):
450
369
  def resolved(self: Self) -> bool:
451
370
  """Whether this qualifier's values go through fullmap entity resolution.
452
371
 
453
- Enum-ranged qualifiers carry a literal token from a closed Biolink vocabulary
454
- and must be passed through verbatim. CURIE-ranged qualifiers are resolved
455
- unless they opt out with ``no_map: true`` (their values are already CURIEs).
456
- """
457
- return self.vocabulary is None and not self.no_map
458
-
459
- @model_validator(mode="after")
460
- def reject_no_map_on_enum_ranged(self: Self) -> Self:
461
- """Reject ``no_map: true`` on enum-ranged qualifiers as dead config.
462
-
463
- Enum-ranged qualifiers already pass their literal tokens through verbatim, so
464
- ``no_map`` would be a no-op that misleads the reader; fail loudly at config
465
- time, like the ``nullable``-on-literal check below.
372
+ Only CURIE-ranged qualifiers are resolved. Enum-ranged ones carry a literal
373
+ token from a closed Biolink vocabulary and must be passed through verbatim.
466
374
  """
467
- if self.no_map and self.vocabulary is not None:
468
- raise TablassertValidationError(
469
- f"{self.qualifier} has a closed Biolink vocabulary and already passes values through verbatim, so `no_map` is redundant; remove it.",
470
- code="no-map-dead-config",
471
- )
472
- return self
375
+ return self.vocabulary is None
473
376
 
474
377
  @model_validator(mode="after")
475
378
  def reject_disabled_qualifiers(self: Self) -> Self:
@@ -272,9 +272,6 @@ OP_COST: dict[Callable, int] = {
272
272
  lib.split_list: 1,
273
273
  # Vocabulary audit: one eager distinct-value pass over a single enum column.
274
274
  lib.qualifier_vocabulary_audit: 1,
275
- # CURIE audit: one eager pass over a single no_map column (distinct-value check,
276
- # blank-row filter, category column synthesis).
277
- lib.curie_audit: 1,
278
275
  # Clean: coerce_columns applies five coercion rules (schema resolve + renames + value
279
276
  # coercion) in one pass, so it outprices a single-expression clean op.
280
277
  lib.coerce_columns: 3,
File without changes
File without changes
File without changes
File without changes
File without changes