tablassert 19.0.0__tar.gz → 19.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-19.0.0 → tablassert-19.1.0}/PKG-INFO +6 -5
- {tablassert-19.0.0 → tablassert-19.1.0}/README.md +4 -3
- {tablassert-19.0.0 → tablassert-19.1.0}/pyproject.toml +2 -2
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/biolink.py +83 -12
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/cli.py +123 -1
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/fullmap.py +104 -1
- {tablassert-19.0.0 → tablassert-19.1.0}/LICENSE +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/Cargo.lock +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/Cargo.toml +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/src/fullmap.rs +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/src/json.rs +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/src/lib.rs +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/src/ndjson.rs +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/src/nlp.rs +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/src/uuid.rs +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/__init__.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/agent.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/cli_entry.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/coerce.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/distill.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/distill_reward.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/enums.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/errors.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/extras.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/ingests.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/lib.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/log.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/models.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/net.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/nlp.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/progress.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/qc.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/rig.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/rs.pyi +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/runcache.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/study.py +0 -0
- {tablassert-19.0.0 → tablassert-19.1.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version: 19.
|
|
3
|
+
Version: 19.1.0
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -48,7 +48,7 @@ Provides-Extra: optimize
|
|
|
48
48
|
Provides-Extra: qc
|
|
49
49
|
Provides-Extra: rt
|
|
50
50
|
License-File: LICENSE
|
|
51
|
-
Summary:
|
|
51
|
+
Summary: Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges.
|
|
52
52
|
Keywords: knowledge graph,kgx,ncats translator,biolink,bioinformatics,biomedical data,entity resolution,tabular data,yaml configuration,declarative pipeline,quality control,llm agent
|
|
53
53
|
Author-email: Skye Lane Goetz <sgoetz@isbscience.org>
|
|
54
54
|
License-Expression: Apache-2.0
|
|
@@ -60,10 +60,11 @@ Project-URL: Source, https://github.com/SkyeAv/Tablassert
|
|
|
60
60
|
|
|
61
61
|
# Tablassert
|
|
62
62
|
|
|
63
|
-
[](https://opensource.org/licenses/Apache-2.0)
|
|
64
|
+
[](https://www.python.org/downloads/)
|
|
65
|
+
[](https://badge.fury.io/py/tablassert)
|
|
66
|
+
[](https://pepy.tech/project/tablassert)
|
|
65
67
|
[](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
|
|
66
|
-
[](https://github.com/SkyeAv/Tablassert/blob/main/LICENSE)
|
|
67
68
|
[](https://skyeav.github.io/Tablassert/)
|
|
68
69
|
[](https://github.com/SkyeAv/Tablassert/stargazers)
|
|
69
70
|
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
# Tablassert
|
|
2
2
|
|
|
3
|
-
[](https://opensource.org/licenses/Apache-2.0)
|
|
4
|
+
[](https://www.python.org/downloads/)
|
|
5
|
+
[](https://badge.fury.io/py/tablassert)
|
|
6
|
+
[](https://pepy.tech/project/tablassert)
|
|
5
7
|
[](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
|
|
6
|
-
[](https://github.com/SkyeAv/Tablassert/blob/main/LICENSE)
|
|
7
8
|
[](https://skyeav.github.io/Tablassert/)
|
|
8
9
|
[](https://github.com/SkyeAv/Tablassert/stargazers)
|
|
9
10
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "19.
|
|
4
|
-
description = "
|
|
3
|
+
version = "19.1.0"
|
|
4
|
+
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
7
7
|
]
|
|
@@ -11,7 +11,10 @@ Deriving these values from the model -- instead of hand-maintaining copies that
|
|
|
11
11
|
can drift -- guarantees Tablassert's KGX output uses only terms that are valid
|
|
12
12
|
for the pinned Biolink version. Downstream translator-ingests can therefore
|
|
13
13
|
consume Tablassert output without running Koza or re-validating against the
|
|
14
|
-
Biolink Model.
|
|
14
|
+
Biolink Model. A tiny curated set of named local extensions
|
|
15
|
+
(:data:`PREDICATE_OVERRIDES`, the DAKP prevention family) rides on top of that
|
|
16
|
+
model-derived vocabulary; each is a deliberate step ahead of the pinned model
|
|
17
|
+
and is dropped by a drift tripwire the moment the model adopts it.
|
|
15
18
|
|
|
16
19
|
Source mapping
|
|
17
20
|
--------------
|
|
@@ -22,7 +25,8 @@ Source mapping
|
|
|
22
25
|
- ``Predicates`` / ``Qualifiers``: walked from the bundled
|
|
23
26
|
``biolink_model.yaml`` slot hierarchy via ``linkml-runtime`` (a pinned
|
|
24
27
|
dependency of ``biolink-model``), because predicates and qualifiers are LinkML
|
|
25
|
-
*slots* rather than Pydantic classes.
|
|
28
|
+
*slots* rather than Pydantic classes. ``Predicates`` then unions
|
|
29
|
+
:data:`PREDICATE_OVERRIDES`, the named local extensions above.
|
|
26
30
|
- ``ALLOWED_EDGE_FIELDS``: the Biolink ``Association`` model fields (walked over
|
|
27
31
|
the MRO) unioned with the qualifier slot names and a curated set of KGX /
|
|
28
32
|
Tablassert edge columns that are not Biolink Association fields.
|
|
@@ -65,6 +69,7 @@ __all__ = [
|
|
|
65
69
|
"EFFECT_TYPE_VALUES",
|
|
66
70
|
"ENUM_RANGED_QUALIFIERS",
|
|
67
71
|
"KNOWN_PENDING_EDGE_FIELDS",
|
|
72
|
+
"PREDICATE_OVERRIDES",
|
|
68
73
|
"STUDY_METADATA_FIELDS",
|
|
69
74
|
"UNSATISFIABLE_EDGE_FIELDS",
|
|
70
75
|
"AgentTypes",
|
|
@@ -188,6 +193,22 @@ scan otherwise picks it up), the suite fails and the stale name is removed from
|
|
|
188
193
|
``CATEGORY_OVERRIDES`` -- the same philosophy as ``CLASS_FIELD_OVERRIDES``.
|
|
189
194
|
"""
|
|
190
195
|
|
|
196
|
+
PREDICATE_OVERRIDES: frozenset[str] = frozenset({"applied_to_prevent", "contraindicated_in_the_prevention_of", "prevents"})
|
|
197
|
+
"""Biolink predicate names Tablassert deliberately supports ahead of pinned model.
|
|
198
|
+
|
|
199
|
+
``Predicates`` normally comes directly from installed Biolink ``related to`` slots.
|
|
200
|
+
DAKP needs this prevention family before upstream adds it:
|
|
201
|
+
``applied_to_prevent`` expresses use of an intervention for prevention,
|
|
202
|
+
``prevents`` expresses preventive effect, and
|
|
203
|
+
``contraindicated_in_the_prevention_of`` expresses a prevention-specific
|
|
204
|
+
contraindication. Each emits as its ordinary ``biolink:`` CURIE, but remains a
|
|
205
|
+
local extension until Biolink adopts it.
|
|
206
|
+
|
|
207
|
+
A tripwire test asserts every member remains absent from installed ``related to``
|
|
208
|
+
slots. When Biolink adds one, that test fails and the stale member is removed from
|
|
209
|
+
this set, restoring fully model-derived vocabulary for that predicate.
|
|
210
|
+
"""
|
|
211
|
+
|
|
191
212
|
|
|
192
213
|
def _entity_category_names() -> list[str]:
|
|
193
214
|
"""Collect all Biolink entity category names from the Pydantic model.
|
|
@@ -265,8 +286,8 @@ def _slot_descendants(root: str) -> frozenset[str]:
|
|
|
265
286
|
|
|
266
287
|
|
|
267
288
|
def _predicate_values() -> list[str]:
|
|
268
|
-
"""All Biolink predicates
|
|
269
|
-
return sorted({_snake(p) for p in (_slot_descendants("related to") | {"related to"})})
|
|
289
|
+
"""All Biolink predicates plus deliberate pending-upstream local extensions."""
|
|
290
|
+
return sorted({_snake(p) for p in (_slot_descendants("related to") | {"related to"})} | PREDICATE_OVERRIDES)
|
|
270
291
|
|
|
271
292
|
|
|
272
293
|
def _qualifier_values() -> list[str]:
|
|
@@ -478,6 +499,9 @@ if TYPE_CHECKING:
|
|
|
478
499
|
CHEMICAL_GENE_INTERACTION_ASSOCIATION: EdgeCategories
|
|
479
500
|
|
|
480
501
|
class Predicates(str, Enum):
|
|
502
|
+
APPLIED_TO_PREVENT: Predicates
|
|
503
|
+
CONTRAINDICATED_IN_THE_PREVENTION_OF: Predicates
|
|
504
|
+
PREVENTS: Predicates
|
|
481
505
|
RELATED_TO: Predicates
|
|
482
506
|
TREATS: Predicates
|
|
483
507
|
|
|
@@ -856,17 +880,62 @@ def numeric_slot_kind(field: str) -> str | None:
|
|
|
856
880
|
return None
|
|
857
881
|
|
|
858
882
|
|
|
883
|
+
def _is_pending_edge_problem(problem: str, record: dict[str, Any]) -> bool:
|
|
884
|
+
"""Whether a single edge failure on ``record`` is a known, intentional model gap.
|
|
885
|
+
|
|
886
|
+
Two deliberate-gap kinds:
|
|
887
|
+
|
|
888
|
+
* ``extra_forbidden`` on a field in :data:`KNOWN_PENDING_EDGE_FIELDS` -- a curated
|
|
889
|
+
KGX / Tablassert column emitted ahead of the pinned model (same contract as
|
|
890
|
+
:func:`is_pending_problem`).
|
|
891
|
+
* ``literal_error`` / enum mismatch on the ``predicate`` slot when the record's
|
|
892
|
+
own predicate value is a member of :data:`PREDICATE_OVERRIDES` -- a local
|
|
893
|
+
predicate Tablassert emits ahead of the installed model. Most association
|
|
894
|
+
classes (``Association`` itself, the DAKP-pinned
|
|
895
|
+
``EntityToDisease`` / ``EntityToPhenotypicFeature`` classes) declare ``predicate``
|
|
896
|
+
as open ``str`` and accept it natively; the constrained ones -- e.g.
|
|
897
|
+
``ChemicalEntityToBiologicalProcessAssociation``'s exhaustive ``Literal`` or the
|
|
898
|
+
``GeneToDiseasePredicateEnum`` family -- reject it with exactly these error
|
|
899
|
+
types, which this arm forgives.
|
|
900
|
+
|
|
901
|
+
Every other failure is a real defect.
|
|
902
|
+
"""
|
|
903
|
+
field, _, error_type = problem.rpartition(": ")
|
|
904
|
+
if field == "predicate" and error_type in {"literal_error", "enum"}:
|
|
905
|
+
predicate: Any = record.get("predicate")
|
|
906
|
+
normalized: str = predicate if isinstance(predicate, str) else ""
|
|
907
|
+
if not normalized.startswith("biolink:"):
|
|
908
|
+
normalized = f"biolink:{normalized}"
|
|
909
|
+
return normalized.removeprefix("biolink:") in PREDICATE_OVERRIDES
|
|
910
|
+
return error_type == "extra_forbidden" and field in KNOWN_PENDING_EDGE_FIELDS
|
|
911
|
+
|
|
912
|
+
|
|
859
913
|
def is_pending_problem(problem: str) -> bool:
|
|
860
|
-
"""Whether a ``"field: error-type"`` problem is a known, intentional model gap.
|
|
914
|
+
"""Whether a ``"field: error-type"`` problem string is a known, intentional model gap.
|
|
861
915
|
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
916
|
+
Kept field-only for the existing extra-forbidden vocabulary. For predicate
|
|
917
|
+
failures the record's own predicate value is needed, so prefer
|
|
918
|
+
:func:`_is_pending_edge_problem` (or :func:`_record_has_only_pending_problems` for
|
|
919
|
+
a record's full error list).
|
|
865
920
|
"""
|
|
866
921
|
field, _, error_type = problem.rpartition(": ")
|
|
867
922
|
return error_type == "extra_forbidden" and field in KNOWN_PENDING_EDGE_FIELDS
|
|
868
923
|
|
|
869
924
|
|
|
925
|
+
def _record_has_only_pending_problems(record: dict[str, Any], problems: list[str]) -> bool:
|
|
926
|
+
"""True when every failure of ``record`` is a known, intentional model gap.
|
|
927
|
+
|
|
928
|
+
Applies the strict :func:`is_pending_problem` contract to edge records (where the
|
|
929
|
+
override-predicate exemption is meaningful) and keeps the field-only contract
|
|
930
|
+
everywhere else, so a predicate mismatch on a node record can never be forgiven.
|
|
931
|
+
"""
|
|
932
|
+
if not problems:
|
|
933
|
+
return False
|
|
934
|
+
if "predicate" in record:
|
|
935
|
+
return all(_is_pending_edge_problem(problem, record) for problem in problems)
|
|
936
|
+
return all(is_pending_problem(problem) for problem in problems)
|
|
937
|
+
|
|
938
|
+
|
|
870
939
|
def validate_kgx(nodes_path: Path, edges_path: Path, limit: int = 20) -> dict[str, Any]:
|
|
871
940
|
"""Validate emitted KGX NDJSON files against the Biolink Pydantic model.
|
|
872
941
|
|
|
@@ -877,9 +946,11 @@ def validate_kgx(nodes_path: Path, edges_path: Path, limit: int = 20) -> dict[st
|
|
|
877
946
|
|
|
878
947
|
Two pass rates are reported. ``valid`` is strict and drives ``ok`` (the CLI's
|
|
879
948
|
non-zero exit). ``valid_excluding_pending`` additionally counts records whose *every*
|
|
880
|
-
failure is a
|
|
881
|
-
|
|
882
|
-
|
|
949
|
+
failure is a known, intentional model gap -- an :func:`is_pending_problem` field
|
|
950
|
+
rejection, or a record-scoped :data:`PREDICATE_OVERRIDES` predicate rejection (see
|
|
951
|
+
:func:`_is_pending_edge_problem`) -- the score to optimize against, so a deliberate
|
|
952
|
+
gap like the KGX denormalized carryovers is not mistaken for a malformed record. The
|
|
953
|
+
two converge as future model releases absorb curated extras.
|
|
883
954
|
|
|
884
955
|
Args:
|
|
885
956
|
nodes_path: Path to ``<name>_<version>.nodes.ndjson``.
|
|
@@ -914,7 +985,7 @@ def validate_kgx(nodes_path: Path, edges_path: Path, limit: int = 20) -> dict[st
|
|
|
914
985
|
valid += 1
|
|
915
986
|
valid_excluding_pending += 1
|
|
916
987
|
continue
|
|
917
|
-
if
|
|
988
|
+
if _record_has_only_pending_problems(record, errors):
|
|
918
989
|
valid_excluding_pending += 1
|
|
919
990
|
problems.update(errors)
|
|
920
991
|
if len(examples) < limit:
|
|
@@ -17,7 +17,7 @@ from importlib.metadata import version as get_version
|
|
|
17
17
|
from itertools import chain, pairwise
|
|
18
18
|
from multiprocessing import Pool
|
|
19
19
|
from pathlib import Path
|
|
20
|
-
from typing import TYPE_CHECKING, Annotated, Any, BinaryIO, Literal, NoReturn
|
|
20
|
+
from typing import TYPE_CHECKING, Annotated, Any, BinaryIO, Literal, NoReturn, cast
|
|
21
21
|
from urllib.error import HTTPError, URLError
|
|
22
22
|
from urllib.request import Request, urlopen
|
|
23
23
|
|
|
@@ -32,6 +32,7 @@ if TYPE_CHECKING:
|
|
|
32
32
|
import polars as pl
|
|
33
33
|
import pydantic
|
|
34
34
|
|
|
35
|
+
from tablassert.biolink import Categories
|
|
35
36
|
from tablassert.lib import Tcode
|
|
36
37
|
from tablassert.models import Graph
|
|
37
38
|
from tablassert.progress import PipelineProgress
|
|
@@ -968,6 +969,127 @@ def validate_kgx_command(
|
|
|
968
969
|
print("KGX output is Biolink-compliant.", file=sys.stderr)
|
|
969
970
|
|
|
970
971
|
|
|
972
|
+
@APP.command(name="quick-map")
|
|
973
|
+
def quick_map_command(
|
|
974
|
+
terms: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False)],
|
|
975
|
+
*,
|
|
976
|
+
fullmap: Annotated[Path, cyclopts.Parameter(name=["--fullmap", "-f"])],
|
|
977
|
+
taxon: Annotated[int | None, cyclopts.Parameter(name=["--taxon", "-t"])] = 9606,
|
|
978
|
+
prioritize: Annotated[list[str] | None, cyclopts.Parameter(name=["--prioritize", "-p"])] = None,
|
|
979
|
+
avoid: Annotated[list[str] | None, cyclopts.Parameter(name=["--avoid", "-a"])] = None,
|
|
980
|
+
exclude_prefixes: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-prefixes", "-ep"])] = None,
|
|
981
|
+
exclude_regex: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-regex", "-er"])] = None,
|
|
982
|
+
) -> None:
|
|
983
|
+
"""Show what fullmap entity resolution does with one or more terms, as a build would see it.
|
|
984
|
+
|
|
985
|
+
Each term runs through the exact op chain a build runs per node column -- level-one/level-two
|
|
986
|
+
normalization, probe-key extraction, the redb fetch, then ``filter_and_rank``'s taxon filter,
|
|
987
|
+
category prioritize/avoid, prefix/regex exclusion, ``PR`` scoring, and best-tier dedup -- so the
|
|
988
|
+
printed rows are the rows ``build-kg`` would emit for a cell holding that term under a
|
|
989
|
+
``NodeEncoding`` carrying the same settings. Use it to debug a mapping ("why did this cell
|
|
990
|
+
resolve to that CURIE?") or to smoke-test a freshly built fullmap. The flags mirror the
|
|
991
|
+
``NodeEncoding`` config fields one-to-one; ``--taxon`` defaults to 9606 like the config does,
|
|
992
|
+
and ``--taxon 0`` disables the filter like ``taxon: null`` does.
|
|
993
|
+
|
|
994
|
+
``--prioritize`` / ``--avoid`` accept the live Biolink entity category names -- the same
|
|
995
|
+
vocabulary those keys accept in a table config -- validated at run time rather than enumerated
|
|
996
|
+
here (the enum is built dynamically from the installed biolink-model), so an invalid value
|
|
997
|
+
exits 2 naming the nearest valid names. A term with no matches prints an explicit "no matches"
|
|
998
|
+
line (with the normalized probe keys it became, so the miss is diagnosable) and still exits 0:
|
|
999
|
+
a miss is a finding, not a failure. Only usage errors and an unreadable fullmap exit 2.
|
|
1000
|
+
|
|
1001
|
+
Args:
|
|
1002
|
+
terms: One or more terms to resolve (positional); any casing or whitespace, and a CURIE
|
|
1003
|
+
string works too because the fullmap indexes CURIEs and their equivalent identifiers
|
|
1004
|
+
as terms.
|
|
1005
|
+
fullmap: Path to the fullmap redb file, or a directory holding one (resolved like the
|
|
1006
|
+
``Graph.fullmap`` config field: ``<dir>/fullmap.redb`` then ``<dir>/data/fullmap.redb``).
|
|
1007
|
+
taxon: NCBI taxon id constraining taxon-bearing matches; ``0`` disables the filter.
|
|
1008
|
+
prioritize: Biolink categories ranked higher (see the vocabulary note above).
|
|
1009
|
+
avoid: Biolink categories dropped entirely (see the vocabulary note above).
|
|
1010
|
+
exclude_prefixes: CURIE namespace prefixes (text before the first ':') to drop.
|
|
1011
|
+
exclude_regex: Regex patterns; any resolved CURIE matching one is dropped.
|
|
1012
|
+
"""
|
|
1013
|
+
from difflib import get_close_matches
|
|
1014
|
+
|
|
1015
|
+
from rich.console import Console
|
|
1016
|
+
from rich.table import Table
|
|
1017
|
+
|
|
1018
|
+
from tablassert import rs
|
|
1019
|
+
from tablassert.fullmap import _KNOWN_CATEGORIES, _probe_keys, fullmap_db_path, quick_map
|
|
1020
|
+
|
|
1021
|
+
def fail(message: str) -> NoReturn:
|
|
1022
|
+
"""Print one actionable user error and use the CLI's documented exit status."""
|
|
1023
|
+
print(f"tablassert quick-map: {message}", file=sys.stderr)
|
|
1024
|
+
raise SystemExit(2)
|
|
1025
|
+
|
|
1026
|
+
# Flag validation precedes any fullmap access: a typo'd category or regex is the user's fastest
|
|
1027
|
+
# loop to close, and these checks are pure so they fire before the redb is even resolved.
|
|
1028
|
+
for label, categories in (("prioritize", prioritize), ("avoid", avoid)):
|
|
1029
|
+
for category in categories or []:
|
|
1030
|
+
if category not in _KNOWN_CATEGORIES:
|
|
1031
|
+
nearest: list[str] = get_close_matches(category, _KNOWN_CATEGORIES, n=3, cutoff=0.5)
|
|
1032
|
+
hint: str = f"; did you mean {', '.join(repr(x) for x in nearest)}" if nearest else ""
|
|
1033
|
+
fail(
|
|
1034
|
+
f"--{label} value {category!r} is not a known Biolink entity category{hint}; accepted values are the same names the `prioritize`/`avoid` config keys accept"
|
|
1035
|
+
)
|
|
1036
|
+
for pattern in exclude_regex or []:
|
|
1037
|
+
# Mirrors the NodeEncoding.exclude_regex validator: an empty pattern matches EVERY CURIE
|
|
1038
|
+
# and would silently drop all candidates, and a non-polars regex would die mid-filter.
|
|
1039
|
+
if not str(pattern).strip():
|
|
1040
|
+
fail(f"--exclude-regex entries must be non-empty patterns (an empty pattern matches every CURIE), got {pattern!r}")
|
|
1041
|
+
try:
|
|
1042
|
+
pl.Series([""]).str.contains(str(pattern))
|
|
1043
|
+
except Exception as e:
|
|
1044
|
+
fail(f"--exclude-regex entries must be polars-compatible regular expressions, got {pattern!r}: {e}")
|
|
1045
|
+
|
|
1046
|
+
db: Path = fullmap_db_path(fullmap)
|
|
1047
|
+
if not db.is_file():
|
|
1048
|
+
fail(f"no fullmap database at {db} (resolved from {fullmap}) — build one with tablassert build-fullmap")
|
|
1049
|
+
|
|
1050
|
+
results: dict[str, pl.DataFrame] = quick_map(
|
|
1051
|
+
terms,
|
|
1052
|
+
db,
|
|
1053
|
+
taxon=str(taxon) if taxon else None,
|
|
1054
|
+
prioritize=cast("list[Categories] | None", prioritize),
|
|
1055
|
+
avoid=cast("list[Categories] | None", avoid),
|
|
1056
|
+
exclude_prefixes=exclude_prefixes,
|
|
1057
|
+
exclude_regex=exclude_regex,
|
|
1058
|
+
)
|
|
1059
|
+
|
|
1060
|
+
# One run-level header: the resolved database, its source version, the term count, and every
|
|
1061
|
+
# active filter, printed once so per-term output stays pure result. Joined as ONE string so a
|
|
1062
|
+
# wrapped line never dangles a separator at the break.
|
|
1063
|
+
console: Console = Console()
|
|
1064
|
+
active: list[str] = []
|
|
1065
|
+
if taxon:
|
|
1066
|
+
active.append(f"taxon={taxon}")
|
|
1067
|
+
for label, values in (("prioritize", prioritize), ("avoid", avoid), ("exclude-prefixes", exclude_prefixes), ("exclude-regex", exclude_regex)):
|
|
1068
|
+
if values:
|
|
1069
|
+
active.append(f"{label}={','.join(values)}")
|
|
1070
|
+
header: str = " · ".join(["[bold]quick-map[/bold]", str(db), f"source {rs.fullmap_source_version()}", f"{len(results)} term(s)", *active])
|
|
1071
|
+
console.print(header)
|
|
1072
|
+
|
|
1073
|
+
# Probe keys come from the SAME normalization quick_map probed with, so a title can never
|
|
1074
|
+
# diagnose a miss against a key the lookup never used.
|
|
1075
|
+
keys: pl.DataFrame = _probe_keys(terms)
|
|
1076
|
+
columns: tuple[str, ...] = ("CURIE", "PREFERRED_NAME", "CATEGORY_NAME", "TAXON_ID", "SOURCE_NAME", "NLP_LEVEL", "PR")
|
|
1077
|
+
for term, matches in results.items():
|
|
1078
|
+
position: int = terms.index(term)
|
|
1079
|
+
level_one_key: str = str(keys.get_column("term")[position])
|
|
1080
|
+
level_two_key: str = str(keys.get_column("term_two")[position])
|
|
1081
|
+
probe: str = level_one_key if level_one_key == level_two_key else f"{level_one_key} | {level_two_key}"
|
|
1082
|
+
if matches.height == 0:
|
|
1083
|
+
console.print(f"{term!r} → {probe} · no matches")
|
|
1084
|
+
continue
|
|
1085
|
+
table: Table = Table(title=f"{term!r} → {probe}", title_justify="left")
|
|
1086
|
+
for column in columns:
|
|
1087
|
+
table.add_column(column)
|
|
1088
|
+
for row in matches.iter_rows(named=True):
|
|
1089
|
+
table.add_row(*(str(row[column]) for column in columns))
|
|
1090
|
+
console.print(table)
|
|
1091
|
+
|
|
1092
|
+
|
|
971
1093
|
@APP.command(name="agent")
|
|
972
1094
|
def agent(
|
|
973
1095
|
pmc_ids: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False)],
|
|
@@ -12,7 +12,7 @@ from tablassert._lazy import LazyModule
|
|
|
12
12
|
from tablassert.biolink import Categories
|
|
13
13
|
from tablassert.errors import TablassertError
|
|
14
14
|
from tablassert.log import cat
|
|
15
|
-
from tablassert.nlp import _normalize_terms_series
|
|
15
|
+
from tablassert.nlp import _normalize_terms_series, level_one, level_two
|
|
16
16
|
|
|
17
17
|
logger = cat("FULLMAP")
|
|
18
18
|
|
|
@@ -754,3 +754,106 @@ def resolve(
|
|
|
754
754
|
column_context=column_context,
|
|
755
755
|
tag=tag,
|
|
756
756
|
)
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def _probe_keys(terms: list[str]) -> pl.DataFrame:
|
|
760
|
+
"""Normalize raw terms through the exact level-one/level-two steps a build applies.
|
|
761
|
+
|
|
762
|
+
This is ``Tcode.node_prep``'s normalization pair (``lib.py`` ``node_prep``) with no
|
|
763
|
+
encodings in front of it: ``level_one`` then ``level_two`` on one ``term`` column.
|
|
764
|
+
``quick_map`` builds its probe frame from the result, and callers use it to show a
|
|
765
|
+
user what their raw input became -- both need the SAME normalization the build ran,
|
|
766
|
+
never a Python-side reimplementation, or a miss would be diagnosed against the wrong
|
|
767
|
+
key.
|
|
768
|
+
|
|
769
|
+
Args:
|
|
770
|
+
terms: Raw input terms, in input order.
|
|
771
|
+
|
|
772
|
+
Returns:
|
|
773
|
+
DataFrame with ``term`` (level-one) and ``term_two`` (level-two) columns,
|
|
774
|
+
position-aligned with ``terms``.
|
|
775
|
+
"""
|
|
776
|
+
return level_two(level_one(pl.LazyFrame({"term": terms}), "term"), "term").collect()
|
|
777
|
+
|
|
778
|
+
|
|
779
|
+
def quick_map(
|
|
780
|
+
terms: list[str],
|
|
781
|
+
db: Path,
|
|
782
|
+
*,
|
|
783
|
+
taxon: str | None = "9606",
|
|
784
|
+
prioritize: list[Categories] | None = None,
|
|
785
|
+
avoid: list[Categories] | None = None,
|
|
786
|
+
exclude_prefixes: list[str] | None = None,
|
|
787
|
+
exclude_regex: list[str] | None = None,
|
|
788
|
+
) -> dict[str, pl.DataFrame]:
|
|
789
|
+
"""Resolve raw terms against a fullmap exactly as a build would, one entry per input.
|
|
790
|
+
|
|
791
|
+
This is the ``tablassert quick-map`` core: it answers "what will entity resolution do
|
|
792
|
+
with THIS term under THESE filters?" by running the same op chain a build runs per
|
|
793
|
+
node column -- normalize (``level_one``/``level_two``) -> extract probe keys
|
|
794
|
+
(``distinct``) -> fetch (``lookup_rows``) -> filter, rank, and dedup
|
|
795
|
+
(``filter_and_rank``) -- over a one-column frame instead of a table. Nothing is
|
|
796
|
+
reimplemented, so the returned rows are the rows a build would emit for a cell holding
|
|
797
|
+
that term under a ``NodeEncoding`` carrying the same settings.
|
|
798
|
+
|
|
799
|
+
Differences from a build, both deliberate:
|
|
800
|
+
|
|
801
|
+
- ``column_context`` is fixed ``False``. The frequency tiebreaker counts category
|
|
802
|
+
occurrences within one resolved column of a real table; a handful of probe terms is
|
|
803
|
+
not that population, so enabling it would invent a ranking signal no build would
|
|
804
|
+
have.
|
|
805
|
+
- The whole input is ONE batch through ONE ``lookup_rows`` call (the same pooling
|
|
806
|
+
``resolve_batch`` does across columns), not one round trip per term.
|
|
807
|
+
|
|
808
|
+
Grouping back to inputs: a ranked row's ``term`` is a normalized probe key, so each
|
|
809
|
+
input is reported under every probe key it produced that survived ``distinct``'s
|
|
810
|
+
junk-term filter. Two inputs that normalize to the same key both report that key's
|
|
811
|
+
matches -- the fullmap genuinely cannot distinguish them. An input whose keys were
|
|
812
|
+
all dropped (a purely numeric or sentinel term) maps to a zero-row frame, because a
|
|
813
|
+
build would never probe it either. Repeated identical inputs collapse to one entry.
|
|
814
|
+
|
|
815
|
+
Args:
|
|
816
|
+
terms: Raw input terms, in input order (any casing/whitespace; a CURIE string
|
|
817
|
+
works too, because the builder indexes CURIEs and their equivalent
|
|
818
|
+
identifiers as terms).
|
|
819
|
+
db: Path to the fullmap redb file (already resolved via ``fullmap_db_path``).
|
|
820
|
+
taxon: Optional NCBI taxon id constraining taxon-bearing matches; rows with
|
|
821
|
+
``TAXON_ID`` 0 are retained. ``None`` disables the filter.
|
|
822
|
+
prioritize: Categories to boost in ranking.
|
|
823
|
+
avoid: Categories to drop entirely (see :func:`filter_and_rank`, which also drops
|
|
824
|
+
categories the ``Categories`` enum cannot name).
|
|
825
|
+
exclude_prefixes: CURIE namespace prefixes (text before the first ':') to drop.
|
|
826
|
+
exclude_regex: Regex patterns; any CURIE matching one is dropped.
|
|
827
|
+
|
|
828
|
+
Returns:
|
|
829
|
+
Insertion-ordered mapping of each distinct input term to its ranked matches
|
|
830
|
+
DataFrame (``filter_and_rank``'s schema: ``term``, ``CURIE``, ``PREFERRED_NAME``,
|
|
831
|
+
``CATEGORY_NAME``, ``TAXON_ID``, ``SOURCE_NAME``, ``SOURCE_VERSION``,
|
|
832
|
+
``NLP_LEVEL``, ``PR``). A term with no matches maps to ``empty_matches(False)``,
|
|
833
|
+
never to a missing key, so callers never branch on presence.
|
|
834
|
+
"""
|
|
835
|
+
if not terms:
|
|
836
|
+
return {}
|
|
837
|
+
# Position-aligned (level-one, level-two) keys per input; ALSO the display truth for
|
|
838
|
+
# callers, so probe keys and titles can never drift apart.
|
|
839
|
+
keys: pl.DataFrame = _probe_keys(terms)
|
|
840
|
+
# distinct() dedups the probe keys across inputs, tags each with its NLP level, and
|
|
841
|
+
# drops numeric/sentinel/generic junk exactly as a build's term extraction does.
|
|
842
|
+
terms_df: pl.DataFrame = distinct(keys.lazy(), "term", "term_two").collect()
|
|
843
|
+
probe_keys: list[str] = terms_df.get_column("term").to_list()
|
|
844
|
+
probed: set[str] = set(probe_keys)
|
|
845
|
+
# ONE redb round trip for the whole batch (term cache, dimension maps, and lock retry
|
|
846
|
+
# all live inside lookup_rows), then ONE filter/rank pass whose per-term best-tier
|
|
847
|
+
# dedup already partitions results by probe key.
|
|
848
|
+
rows: list[dict[str, object]] = lookup_rows(db, probe_keys) if probe_keys else []
|
|
849
|
+
matches: pl.DataFrame = filter_and_rank(pl.DataFrame(rows), terms_df, taxon, prioritize, avoid, False, exclude_prefixes, exclude_regex)
|
|
850
|
+
level_one_col: pl.Series = keys.get_column("term")
|
|
851
|
+
level_two_col: pl.Series = keys.get_column("term_two")
|
|
852
|
+
out: dict[str, pl.DataFrame] = {}
|
|
853
|
+
for position, term in enumerate(terms):
|
|
854
|
+
candidates: list[str] = []
|
|
855
|
+
for key in (level_one_col[position], level_two_col[position]):
|
|
856
|
+
if key and key in probed and key not in candidates:
|
|
857
|
+
candidates.append(key)
|
|
858
|
+
out[term] = matches.filter(pl.col("term").is_in(candidates)) if candidates else empty_matches(False)
|
|
859
|
+
return out
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|