tablassert 22.0.0__tar.gz → 22.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-22.0.0 → tablassert-22.1.0}/PKG-INFO +2 -2
- {tablassert-22.0.0 → tablassert-22.1.0}/README.md +1 -1
- {tablassert-22.0.0 → tablassert-22.1.0}/pyproject.toml +1 -1
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/biolink.py +8 -4
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/errors.py +6 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/fullmap.py +140 -8
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/lib.py +41 -8
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/models.py +77 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/LICENSE +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/Cargo.lock +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/Cargo.toml +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/src/fullmap.rs +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/src/json.rs +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/src/lib.rs +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/src/ndjson.rs +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/src/nlp.rs +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/src/uuid.rs +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/__init__.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/cli.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/cli_entry.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/coerce.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/data/infores_catalog.yaml +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/enums.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/extras.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/infores_registry.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/ingests.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/log.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/net.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/nlp.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/progress.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/qc.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/rig.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/rs.pyi +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/runcache.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/study.py +0 -0
- {tablassert-22.0.0 → tablassert-22.1.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version: 22.
|
|
3
|
+
Version: 22.1.0
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -273,5 +273,5 @@ described in:
|
|
|
273
273
|
|
|
274
274
|
- [Skye Lane Goetz](mailto:sgoetz@isbscience.org), Institute for Systems Biology
|
|
275
275
|
- [Gwênlyn Glusman](mailto:gglusman@isbscience.org), Institute for Systems Biology
|
|
276
|
-
- Jared C. Roach, Institute for Systems Biology
|
|
276
|
+
- [Jared C. Roach](mailto:jroach@isbscience.org), Institute for Systems Biology
|
|
277
277
|
|
|
@@ -219,4 +219,4 @@ described in:
|
|
|
219
219
|
|
|
220
220
|
- [Skye Lane Goetz](mailto:sgoetz@isbscience.org), Institute for Systems Biology
|
|
221
221
|
- [Gwênlyn Glusman](mailto:gglusman@isbscience.org), Institute for Systems Biology
|
|
222
|
-
- Jared C. Roach, Institute for Systems Biology
|
|
222
|
+
- [Jared C. Roach](mailto:jroach@isbscience.org), Institute for Systems Biology
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "22.
|
|
3
|
+
version = "22.1.0"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -582,13 +582,17 @@ description dump.
|
|
|
582
582
|
"""
|
|
583
583
|
|
|
584
584
|
|
|
585
|
-
DISABLED_EDGE_FIELDS: frozenset[str] = frozenset(
|
|
586
|
-
"""Edge fields Tablassert intentionally never emits or accepts.
|
|
585
|
+
DISABLED_EDGE_FIELDS: frozenset[str] = frozenset()
|
|
586
|
+
"""Edge fields Tablassert intentionally never emits or accepts, whatever the model says.
|
|
587
587
|
|
|
588
588
|
This policy is separate from :data:`UNSATISFIABLE_EDGE_FIELDS`: the latter tracks
|
|
589
589
|
Biolink Model attachment and may change with a dependency release, while this set is
|
|
590
|
-
a stable Tablassert product decision.
|
|
591
|
-
|
|
590
|
+
a stable Tablassert product decision. It is currently empty: the one entry it ever
|
|
591
|
+
held, ``species_context_qualifier``, was re-enabled as a declarable, CURIE-ranged
|
|
592
|
+
qualifier (Biolink Model 4.4.6 declares the slot on the root ``association`` class,
|
|
593
|
+
range ``organism taxon``) after its old node-taxonomy auto-derivation was removed; no
|
|
594
|
+
derivation returns, so the field only appears when a config declares it. The empty
|
|
595
|
+
set keeps the policy mechanism alive for future product decisions.
|
|
592
596
|
"""
|
|
593
597
|
|
|
594
598
|
|
|
@@ -47,6 +47,12 @@ TablassertErrorCodes = Literal[
|
|
|
47
47
|
"babel-download-failed",
|
|
48
48
|
"source-file-unreadable",
|
|
49
49
|
"resolve-bad-specs",
|
|
50
|
+
"taxon-from-conflict",
|
|
51
|
+
"taxon-from-unknown-column",
|
|
52
|
+
"taxon-from-self",
|
|
53
|
+
"taxon-from-cycle",
|
|
54
|
+
"taxon-from-unresolved-column",
|
|
55
|
+
"taxon-from-not-ncbitaxon",
|
|
50
56
|
"config-rows-and-row-slice-conflict",
|
|
51
57
|
"comparison-bad-comparator-type",
|
|
52
58
|
"comparison-nonnumeric-comparator",
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import re
|
|
3
4
|
import time
|
|
4
5
|
from collections import OrderedDict
|
|
5
6
|
from collections.abc import Callable
|
|
6
7
|
from enum import Enum
|
|
8
|
+
from functools import cache
|
|
7
9
|
from pathlib import Path
|
|
8
10
|
from typing import TYPE_CHECKING, Any, NamedTuple, cast
|
|
9
11
|
|
|
@@ -269,6 +271,28 @@ def lookup_rows(db: Path, terms: list[str]) -> list[dict[str, object]]:
|
|
|
269
271
|
return rows
|
|
270
272
|
|
|
271
273
|
|
|
274
|
+
@cache
|
|
275
|
+
def _generic_probe_keys() -> frozenset[str]:
|
|
276
|
+
"""Apply the real lookup normalizers to the existing excluded generic labels."""
|
|
277
|
+
labels = [
|
|
278
|
+
"none",
|
|
279
|
+
"nan",
|
|
280
|
+
"na",
|
|
281
|
+
"null",
|
|
282
|
+
"unknown",
|
|
283
|
+
"not applicable",
|
|
284
|
+
"p_value",
|
|
285
|
+
"p value",
|
|
286
|
+
"variable",
|
|
287
|
+
"result",
|
|
288
|
+
"exposure",
|
|
289
|
+
"expression",
|
|
290
|
+
"symbol",
|
|
291
|
+
]
|
|
292
|
+
normalized = level_two(level_one(pl.LazyFrame({"term": labels}), "term"), "term").collect()
|
|
293
|
+
return frozenset(labels) | frozenset(normalized["term"]) | frozenset(normalized["term_two"])
|
|
294
|
+
|
|
295
|
+
|
|
272
296
|
def distinct(lf: pl.LazyFrame, l1: str, l2: str, col: str = "term") -> pl.LazyFrame:
|
|
273
297
|
"""Extract unique terms from two text-normalization columns as a LazyFrame.
|
|
274
298
|
|
|
@@ -297,8 +321,10 @@ def distinct(lf: pl.LazyFrame, l1: str, l2: str, col: str = "term") -> pl.LazyFr
|
|
|
297
321
|
# concat/unique, so first-seen tier selection would be scheduling-dependent.
|
|
298
322
|
terms: pl.LazyFrame = pl.concat([t1, t2]).group_by(col).agg(pl.col("nlp_level").min())
|
|
299
323
|
|
|
300
|
-
|
|
301
|
-
|
|
324
|
+
# Probe keys are stemmed/sorted at level one, then stripped at level two.
|
|
325
|
+
# Filtering only the original spellings lets headers such as "Expression"
|
|
326
|
+
# ("express") and "Not applicable" ("applic not") resolve as entities.
|
|
327
|
+
return terms.filter(~pl.col(col).str.contains(r"^\d+$|^$") & ~pl.col(col).is_in(_generic_probe_keys()))
|
|
302
328
|
|
|
303
329
|
|
|
304
330
|
def deduplicate_result(result: pl.DataFrame, column_context: bool) -> pl.DataFrame:
|
|
@@ -550,7 +576,9 @@ def join_matches(lf: pl.LazyFrame, col: str, matches: pl.DataFrame, tag: str = "
|
|
|
550
576
|
return _join_matches_eager(lf.collect(), col, matches, tag, drop_unresolved).lazy()
|
|
551
577
|
|
|
552
578
|
|
|
553
|
-
def _join_matches_eager(
|
|
579
|
+
def _join_matches_eager(
|
|
580
|
+
df: pl.DataFrame, col: str, matches: pl.DataFrame, tag: str = "_two", drop_unresolved: bool = True, taxon_key: str | None = None
|
|
581
|
+
) -> pl.DataFrame:
|
|
554
582
|
"""Eager core of ``join_matches``: join ranked matches into an in-memory frame.
|
|
555
583
|
|
|
556
584
|
Args:
|
|
@@ -560,6 +588,10 @@ def _join_matches_eager(df: pl.DataFrame, col: str, matches: pl.DataFrame, tag:
|
|
|
560
588
|
tag: Suffix used to derive the level-two column name.
|
|
561
589
|
drop_unresolved: When True rows whose ``col`` did not match are dropped;
|
|
562
590
|
when False the row is kept and the resolved columns stay null.
|
|
591
|
+
taxon_key: Optional join-key column present on BOTH sides (``df`` and
|
|
592
|
+
``matches``) that must match in addition to the term, used by per-row
|
|
593
|
+
taxon scoping (``ResolveSpec.taxon_from``) so one term's matches from
|
|
594
|
+
different taxon scopes never leak across rows. Dropped from the output.
|
|
563
595
|
|
|
564
596
|
Returns:
|
|
565
597
|
DataFrame with resolved columns. Tied best-ranked CURIEs are returned as
|
|
@@ -569,14 +601,25 @@ def _join_matches_eager(df: pl.DataFrame, col: str, matches: pl.DataFrame, tag:
|
|
|
569
601
|
l1: str = col
|
|
570
602
|
l2: str = l1 + tag
|
|
571
603
|
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
604
|
+
if taxon_key is not None:
|
|
605
|
+
result: pl.DataFrame = df.join(
|
|
606
|
+
matches.filter(pl.col("NLP_LEVEL").eq(1)), left_on=[l1, taxon_key], right_on=["term", taxon_key], how="left", suffix="_l1"
|
|
607
|
+
)
|
|
608
|
+
l2_matches: pl.DataFrame = matches.filter(pl.col("NLP_LEVEL").eq(2))
|
|
609
|
+
result = result.join(l2_matches, left_on=[l2, taxon_key], right_on=["term", taxon_key], how="left", suffix="_l2")
|
|
610
|
+
else:
|
|
611
|
+
result = df.join(matches.filter(pl.col("NLP_LEVEL").eq(1)), left_on=l1, right_on="term", how="left", suffix="_l1")
|
|
612
|
+
l2_matches = matches.filter(pl.col("NLP_LEVEL").eq(2))
|
|
613
|
+
result = result.join(l2_matches, left_on=l2, right_on="term", how="left", suffix="_l2")
|
|
576
614
|
|
|
577
615
|
result = result.with_columns([_coalesce_expr(col, suffix, base, prefix, cast_str) for suffix, base, prefix, cast_str in _JOIN_COALESCE_SPECS])
|
|
578
616
|
|
|
579
|
-
|
|
617
|
+
exclude: str = r"^(CURIE|PREFERRED_NAME|CATEGORY_NAME|TAXON_ID|SOURCE_NAME|SOURCE_VERSION|NLP_LEVEL|PR|FREQUENCY)(_l2)?$"
|
|
618
|
+
if taxon_key is not None:
|
|
619
|
+
exclude = (
|
|
620
|
+
r"^(CURIE|PREFERRED_NAME|CATEGORY_NAME|TAXON_ID|SOURCE_NAME|SOURCE_VERSION|NLP_LEVEL|PR|FREQUENCY|" + re.escape(taxon_key) + r")(_l2)?$"
|
|
621
|
+
)
|
|
622
|
+
result = result.select(pl.exclude(exclude))
|
|
580
623
|
result = result.select(pl.exclude(col + tag))
|
|
581
624
|
result = result.with_columns(pl.col(f"{col}_taxon").replace("NCBITaxon:0", None))
|
|
582
625
|
if drop_unresolved:
|
|
@@ -602,6 +645,89 @@ class ResolveSpec(NamedTuple):
|
|
|
602
645
|
Set only for ``nullable`` qualifiers; subject/object always resolve strict so an
|
|
603
646
|
edge with a missing node is dropped, never emitted.
|
|
604
647
|
"""
|
|
648
|
+
taxon_from: str | None = None
|
|
649
|
+
"""When set, per-row taxon scoping from another resolved column of the same frame.
|
|
650
|
+
|
|
651
|
+
Names a node column that an earlier ``resolve_batch`` pass already resolved: its
|
|
652
|
+
``<col>_taxon`` metadata (an ``NCBITaxon:<id>`` string) scopes this column's
|
|
653
|
+
``filter_and_rank`` taxon filter row by row instead of the static ``taxon``.
|
|
654
|
+
Rows whose taxon cell is blank fall back to the 9606 default; a non-null cell
|
|
655
|
+
whose resolved taxon is not an NCBITaxon id fails loudly rather than silently
|
|
656
|
+
resolving everything under one species.
|
|
657
|
+
"""
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
_TAXON_SCOPE: str = "__taxon_scope"
|
|
661
|
+
"""Helper column carrying each row's per-row taxon scope during ``resolve_batch``.
|
|
662
|
+
|
|
663
|
+
Filled by :func:`_per_row_taxon_matches` from the ``taxon_from`` column's resolved
|
|
664
|
+
metadata (an NCBITaxon id string, or the ``9606`` fallback for blank taxon cells),
|
|
665
|
+
used as the second join key so one term's matches from different taxon scopes never
|
|
666
|
+
leak across rows, and dropped before the frame leaves the op.
|
|
667
|
+
"""
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def _per_row_taxon_matches(
|
|
671
|
+
result: pl.DataFrame, spec: ResolveSpec, raw: pl.DataFrame, tag: str, column_context: bool
|
|
672
|
+
) -> tuple[pl.DataFrame, pl.DataFrame]:
|
|
673
|
+
"""Scope this spec's matches per row by the ``taxon_from`` column's resolved taxon.
|
|
674
|
+
|
|
675
|
+
The ``taxon_from`` column was resolved by an earlier ``resolve_batch`` pass, so
|
|
676
|
+
``result`` already carries its resolved CURIE (``<col>``) and its taxon metadata
|
|
677
|
+
(``<col>_taxon``, ``NCBITaxon:<id>`` or null for a non-taxon-bearing concept).
|
|
678
|
+
Rows are bucketed by that scope; each bucket's terms go through the same
|
|
679
|
+
``filter_and_rank`` a static-taxon spec uses, keyed by the scope so the join back
|
|
680
|
+
picks each row's own bucket. Blank taxon cells fall back to the 9606 default; a
|
|
681
|
+
non-null cell whose taxon is not an NCBITaxon id fails the build instead of
|
|
682
|
+
silently resolving everything under one species.
|
|
683
|
+
|
|
684
|
+
Returns:
|
|
685
|
+
The frame with ``_TAXON_SCOPE`` appended (join key; the caller drops it after
|
|
686
|
+
the join) and the concatenated per-scope matches carrying the same key.
|
|
687
|
+
|
|
688
|
+
Raises:
|
|
689
|
+
TablassertError: ``taxon-from-unresolved-column`` when the taxon source's
|
|
690
|
+
resolved or metadata column is absent (the earlier pass never ran);
|
|
691
|
+
``taxon-from-not-ncbitaxon`` when a non-null taxon cell resolves without
|
|
692
|
+
an NCBITaxon taxon id.
|
|
693
|
+
"""
|
|
694
|
+
source: str = spec.taxon_from or ""
|
|
695
|
+
meta_col: str = f"{source}_taxon"
|
|
696
|
+
missing: list[str] = [c for c in (source, meta_col) if c not in result.columns]
|
|
697
|
+
if missing:
|
|
698
|
+
raise TablassertError(
|
|
699
|
+
f"ResolveSpec for {spec.col!r} sets taxon_from={source!r} but the frame lacks {', '.join(repr(c) for c in missing)}; "
|
|
700
|
+
"the named column must be resolved by an earlier resolve_batch pass in the same pipeline.",
|
|
701
|
+
code="taxon-from-unresolved-column",
|
|
702
|
+
)
|
|
703
|
+
value: pl.Series = result.select(pl.col(source).cast(pl.String).str.strip_chars().alias("v"))["v"]
|
|
704
|
+
meta: pl.Series = result.select(pl.col(meta_col).cast(pl.String).str.strip_chars().alias("m"))["m"]
|
|
705
|
+
has_value: pl.Series = value.is_not_null() & (value != "")
|
|
706
|
+
is_ncbitaxon: pl.Series = meta.str.contains(r"^NCBITaxon:\d+$").fill_null(False)
|
|
707
|
+
bad: pl.Series = has_value & ~is_ncbitaxon
|
|
708
|
+
if bad.any():
|
|
709
|
+
offenders: list[str] = sorted({f"{v!r} -> {m!r}" for v, m in zip(value.filter(bad).to_list(), meta.filter(bad).to_list(), strict=True)})[:5]
|
|
710
|
+
raise TablassertError(
|
|
711
|
+
f"taxon_from column {source!r} carries value(s) that did not resolve to an NCBITaxon taxon: "
|
|
712
|
+
f"{', '.join(offenders)}. Fix the source table so every non-blank taxon cell names a species, "
|
|
713
|
+
"or the row's entity resolution cannot be scoped.",
|
|
714
|
+
code="taxon-from-not-ncbitaxon",
|
|
715
|
+
)
|
|
716
|
+
scope: pl.Expr = (
|
|
717
|
+
pl.when(has_value & is_ncbitaxon)
|
|
718
|
+
.then(pl.col(meta_col).cast(pl.String).str.strip_chars().str.replace("NCBITaxon:", ""))
|
|
719
|
+
.otherwise(pl.lit("9606"))
|
|
720
|
+
)
|
|
721
|
+
result = result.with_columns(scope.alias(_TAXON_SCOPE))
|
|
722
|
+
parts: list[pl.DataFrame] = []
|
|
723
|
+
for taxon in sorted(set(result[_TAXON_SCOPE].to_list())):
|
|
724
|
+
bucket: pl.DataFrame = result.filter(pl.col(_TAXON_SCOPE) == taxon)
|
|
725
|
+
bucket_terms: pl.DataFrame = distinct(bucket.lazy(), spec.col, spec.col + tag).collect()
|
|
726
|
+
bucket_matches: pl.DataFrame = filter_and_rank(
|
|
727
|
+
raw, bucket_terms, taxon, spec.prioritize, spec.avoid, column_context, spec.exclude_prefixes, spec.exclude_regex
|
|
728
|
+
)
|
|
729
|
+
parts.append(bucket_matches.with_columns(pl.lit(taxon).alias(_TAXON_SCOPE)))
|
|
730
|
+
return result, (pl.concat(parts) if parts else empty_matches(column_context))
|
|
605
731
|
|
|
606
732
|
|
|
607
733
|
def resolve_batch(
|
|
@@ -696,6 +822,12 @@ def resolve_batch(
|
|
|
696
822
|
for spec in specs:
|
|
697
823
|
if on_phase is not None:
|
|
698
824
|
on_phase(f"resolve:{spec.col}")
|
|
825
|
+
if spec.taxon_from is not None:
|
|
826
|
+
scoped_result, matches = _per_row_taxon_matches(result, spec, raw, tag, column_context)
|
|
827
|
+
if log:
|
|
828
|
+
log_unmatched(spec.col, terms_by_col[spec.col].lazy(), matches, section_hash, config_file)
|
|
829
|
+
result = _join_matches_eager(scoped_result, spec.col, matches, tag, drop_unresolved=not spec.nullable, taxon_key=_TAXON_SCOPE)
|
|
830
|
+
continue
|
|
699
831
|
terms_df: pl.DataFrame = terms_by_col[spec.col]
|
|
700
832
|
matches: pl.DataFrame = filter_and_rank(
|
|
701
833
|
raw, terms_df, spec.taxon, spec.prioritize, spec.avoid, column_context, spec.exclude_prefixes, spec.exclude_regex
|
|
@@ -6,6 +6,7 @@ import re
|
|
|
6
6
|
from collections.abc import Callable, Iterable
|
|
7
7
|
from functools import cache
|
|
8
8
|
from pathlib import Path
|
|
9
|
+
from tempfile import TemporaryDirectory
|
|
9
10
|
from typing import TYPE_CHECKING, Any, Literal, Self
|
|
10
11
|
|
|
11
12
|
from pydantic import Field, NonNegativeInt
|
|
@@ -1400,9 +1401,32 @@ class Tcode(Section):
|
|
|
1400
1401
|
x.exclude_prefixes,
|
|
1401
1402
|
x.exclude_regex,
|
|
1402
1403
|
x.nullable if isinstance(x, Qualifier) else False,
|
|
1404
|
+
x.taxon_from,
|
|
1403
1405
|
)
|
|
1404
1406
|
for x, col in node_columns
|
|
1405
1407
|
]
|
|
1408
|
+
# Per-row taxon scoping (ResolveSpec.taxon_from): a scoped column reads the
|
|
1409
|
+
# taxon metadata its source column's resolution emits, so sources must resolve
|
|
1410
|
+
# first. Layer the specs by dependency (config validation has already rejected
|
|
1411
|
+
# cycles and unknown targets); with no taxon_from anywhere this is one batch
|
|
1412
|
+
# and the op list is exactly what it always was.
|
|
1413
|
+
spec_cols: set[str] = {spec.col for spec in specs}
|
|
1414
|
+
batches: list[list[ResolveSpec]] = []
|
|
1415
|
+
placed: set[str] = set()
|
|
1416
|
+
pending: list[ResolveSpec] = list(specs)
|
|
1417
|
+
while pending:
|
|
1418
|
+
ready: list[ResolveSpec] = [s for s in pending if s.taxon_from is None or s.taxon_from in placed or s.taxon_from not in spec_cols]
|
|
1419
|
+
if not ready:
|
|
1420
|
+
raise TablassertError(
|
|
1421
|
+
f"taxon_from references form a cycle among {', '.join(sorted(s.col for s in pending))}; "
|
|
1422
|
+
"no column can resolve first. This is rejected at config time, so a build reaching here means the "
|
|
1423
|
+
"section was assembled programmatically -- fix the spec layering.",
|
|
1424
|
+
code="taxon-from-cycle",
|
|
1425
|
+
)
|
|
1426
|
+
batches.append(ready)
|
|
1427
|
+
placed.update(s.col for s in ready)
|
|
1428
|
+
pending = [s for s in pending if s.col not in placed]
|
|
1429
|
+
resolve_ops: list[Any] = [(resolve_batch, (batch, db, self.log, self.store.stem, self.config.name, True, "_two")) for batch in batches]
|
|
1406
1430
|
return [
|
|
1407
1431
|
[self.node_prep(x, col) for x, col in node_columns],
|
|
1408
1432
|
# Encode only: no pre-resolution snapshot and no NLP normalization, both of
|
|
@@ -1414,7 +1438,7 @@ class Tcode(Section):
|
|
|
1414
1438
|
# copied them into named slots; trim before resolution so the frame that
|
|
1415
1439
|
# resolve_batch materializes and joins stays narrow.
|
|
1416
1440
|
(trim, ()),
|
|
1417
|
-
|
|
1441
|
+
*resolve_ops,
|
|
1418
1442
|
# QC audits only the strict columns: a nullable qualifier's nulls are expected
|
|
1419
1443
|
# (blank cell / no match), not resolution errors for the audit to delete.
|
|
1420
1444
|
[
|
|
@@ -1844,15 +1868,24 @@ def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fi
|
|
|
1844
1868
|
so it never ships in the final NDJSON.
|
|
1845
1869
|
|
|
1846
1870
|
Returns:
|
|
1847
|
-
``None``;
|
|
1848
|
-
``.tmp`` suffix, then deletes ``p_in``.
|
|
1871
|
+
``None``; atomically replaces the deduplicated stream alongside ``p_in`` with no
|
|
1872
|
+
``.tmp`` suffix, then deletes ``p_in``. Parsing, collision, and native writer
|
|
1873
|
+
failures preserve the existing output and the input for diagnosis.
|
|
1849
1874
|
"""
|
|
1850
1875
|
p_out: Path = p_in.with_suffix("")
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1876
|
+
if p_out == p_in:
|
|
1877
|
+
raise ValueError("dedup_stream requires an input filename with a removable suffix")
|
|
1878
|
+
|
|
1879
|
+
# Keep the previous artifact intact until native parsing/dedup completes.
|
|
1880
|
+
# A sibling directory keeps replacement on the same filesystem and retains
|
|
1881
|
+
# the public basename for native collision diagnostics and fold reports.
|
|
1882
|
+
with TemporaryDirectory(prefix=f".{p_out.name}.dedup-", dir=p_out.parent) as stage_dir:
|
|
1883
|
+
staged: Path = Path(stage_dir) / p_out.name
|
|
1884
|
+
merged, scalar_conflicts = rs.dedup_ndjson(p_in, staged, is_edges, domain, uuid_fields, on_collision)
|
|
1885
|
+
report: Path = staged.with_name(staged.name + ".foldreport.json")
|
|
1886
|
+
staged.replace(p_out)
|
|
1887
|
+
if report.exists():
|
|
1888
|
+
report.replace(p_out.with_name(p_out.name + ".foldreport.json"))
|
|
1856
1889
|
if merged:
|
|
1857
1890
|
logger.warning(
|
|
1858
1891
|
"uuid_on_collision=merge folded {merged} divergent edge records into their first-seen edges; {conflicts} conflicting scalar fields kept the first value",
|
|
@@ -307,6 +307,36 @@ class NodeEncoding(Encoding):
|
|
|
307
307
|
),
|
|
308
308
|
examples=[9606, 10090],
|
|
309
309
|
)
|
|
310
|
+
taxon_from: str | None = Field(
|
|
311
|
+
None,
|
|
312
|
+
description=(
|
|
313
|
+
"Per-row taxon scoping: name another resolved node column of the same statement (a CURIE-ranged qualifier key, "
|
|
314
|
+
"or `subject`/`object`) whose resolved NCBITaxon CURIE scopes this column's entity resolution row by row, instead of "
|
|
315
|
+
"the static `taxon`. The named column is resolved first; rows whose taxon cell is blank fall back to the 9606 "
|
|
316
|
+
"default, and a non-null cell that does not resolve to an NCBITaxon CURIE fails the build (`taxon-from-not-ncbitaxon`). "
|
|
317
|
+
"Mutually exclusive with an explicit `taxon`."
|
|
318
|
+
),
|
|
319
|
+
examples=["species_context_qualifier"],
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
@model_validator(mode="before")
|
|
323
|
+
@classmethod
|
|
324
|
+
def taxon_from_excludes_explicit_taxon(cls, data: Any) -> Any:
|
|
325
|
+
"""Reject `taxon_from` set together with an explicit `taxon`.
|
|
326
|
+
|
|
327
|
+
Two taxon knobs on one encoding would compete for the same filter, and the
|
|
328
|
+
winner would depend on evaluation order. `taxon` defaults to 9606, so the
|
|
329
|
+
check must run on the raw input: only an author-written `taxon` conflicts,
|
|
330
|
+
and `taxon_from` supplies its own default (9606 for blank taxon cells).
|
|
331
|
+
"""
|
|
332
|
+
if isinstance(data, dict) and data.get("taxon_from") is not None and "taxon" in data:
|
|
333
|
+
raise TablassertValidationError(
|
|
334
|
+
"`taxon_from` and an explicit `taxon` are mutually exclusive; remove one. `taxon_from` scopes resolution "
|
|
335
|
+
"per row from the named column and falls back to the 9606 default for blank taxon cells.",
|
|
336
|
+
code="taxon-from-conflict",
|
|
337
|
+
)
|
|
338
|
+
return data
|
|
339
|
+
|
|
310
340
|
prioritize: list[Categories] | None = Field(
|
|
311
341
|
None, description="Biolink categories ranked higher during entity resolution.", examples=[[Categories.GENE, Categories.PROTEIN]]
|
|
312
342
|
)
|
|
@@ -484,6 +514,53 @@ class Statement(TablaBase):
|
|
|
484
514
|
seen.add(key)
|
|
485
515
|
return self
|
|
486
516
|
|
|
517
|
+
@model_validator(mode="after")
|
|
518
|
+
def validate_taxon_from_targets(self: Self) -> Self:
|
|
519
|
+
"""Validate every `taxon_from` reference before a build can run.
|
|
520
|
+
|
|
521
|
+
`taxon_from` scopes one node column's resolution by another resolved node
|
|
522
|
+
column of the same statement, so the target must exist, must itself resolve
|
|
523
|
+
(enum-ranged qualifiers never resolve), must not be the column carrying the
|
|
524
|
+
reference, and must not participate in a reference cycle -- a cyclic pair
|
|
525
|
+
would have no valid resolution order. All four failures are config-time so a
|
|
526
|
+
bad reference dies at load, not hours into a build.
|
|
527
|
+
"""
|
|
528
|
+
resolved: dict[str, NodeEncoding | Qualifier] = {
|
|
529
|
+
"subject": self.subject,
|
|
530
|
+
"object": self.object,
|
|
531
|
+
**{str(q.qualifier): q for q in self.qualifiers or [] if q.resolved},
|
|
532
|
+
}
|
|
533
|
+
for name, encoding in resolved.items():
|
|
534
|
+
target: str | None = getattr(encoding, "taxon_from", None)
|
|
535
|
+
if target is None:
|
|
536
|
+
continue
|
|
537
|
+
if target not in resolved:
|
|
538
|
+
raise TablassertValidationError(
|
|
539
|
+
f"`taxon_from: {target}` on `{name}` does not name a resolved node column of this statement; "
|
|
540
|
+
"expected `subject`, `object`, or a CURIE-ranged qualifier key.",
|
|
541
|
+
code="taxon-from-unknown-column",
|
|
542
|
+
)
|
|
543
|
+
if target == name:
|
|
544
|
+
raise TablassertValidationError(
|
|
545
|
+
f"`taxon_from: {target}` on `{name}` references itself; a column cannot be scoped by its own resolution.", code="taxon-from-self"
|
|
546
|
+
)
|
|
547
|
+
# Chase the reference chain from the target: any path back to a column that
|
|
548
|
+
# already carries a reference is a cycle with no valid resolution order.
|
|
549
|
+
seen: set[str] = {name}
|
|
550
|
+
cursor: str = target
|
|
551
|
+
while cursor in resolved:
|
|
552
|
+
if cursor in seen:
|
|
553
|
+
raise TablassertValidationError(
|
|
554
|
+
f"`taxon_from` references form a cycle through `{cursor}`; there is no column that can resolve first.",
|
|
555
|
+
code="taxon-from-cycle",
|
|
556
|
+
)
|
|
557
|
+
seen.add(cursor)
|
|
558
|
+
nxt: str | None = getattr(resolved[cursor], "taxon_from", None)
|
|
559
|
+
if nxt is None:
|
|
560
|
+
break
|
|
561
|
+
cursor = nxt
|
|
562
|
+
return self
|
|
563
|
+
|
|
487
564
|
@model_validator(mode="after")
|
|
488
565
|
def warn_when_an_override_is_demoted_by_the_predicate(self: Self) -> Self:
|
|
489
566
|
"""Warn when a pinned association class cannot carry the section predicate.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|