tablassert 22.3.1__tar.gz → 22.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {tablassert-22.3.1 → tablassert-22.3.3}/PKG-INFO +17 -15
  2. {tablassert-22.3.1 → tablassert-22.3.3}/README.md +15 -14
  3. {tablassert-22.3.1 → tablassert-22.3.3}/pyproject.toml +46 -1
  4. {tablassert-22.3.1 → tablassert-22.3.3}/rust/Cargo.lock +12 -12
  5. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/biolink.py +0 -1
  6. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/cli.py +23 -3
  7. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/models.py +0 -5
  8. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/net.py +0 -11
  9. {tablassert-22.3.1 → tablassert-22.3.3}/LICENSE +0 -0
  10. {tablassert-22.3.1 → tablassert-22.3.3}/rust/Cargo.toml +0 -0
  11. {tablassert-22.3.1 → tablassert-22.3.3}/rust/examples/count_tables.rs +0 -0
  12. {tablassert-22.3.1 → tablassert-22.3.3}/rust/src/fullmap.rs +0 -0
  13. {tablassert-22.3.1 → tablassert-22.3.3}/rust/src/json.rs +0 -0
  14. {tablassert-22.3.1 → tablassert-22.3.3}/rust/src/lib.rs +0 -0
  15. {tablassert-22.3.1 → tablassert-22.3.3}/rust/src/ndjson.rs +0 -0
  16. {tablassert-22.3.1 → tablassert-22.3.3}/rust/src/nlp.rs +0 -0
  17. {tablassert-22.3.1 → tablassert-22.3.3}/rust/src/uuid.rs +0 -0
  18. {tablassert-22.3.1 → tablassert-22.3.3}/rust/tests/build_golden.rs +0 -0
  19. {tablassert-22.3.1 → tablassert-22.3.3}/rust/tests/common/mod.rs +0 -0
  20. {tablassert-22.3.1 → tablassert-22.3.3}/rust/tests/extract_prebuilt.rs +0 -0
  21. {tablassert-22.3.1 → tablassert-22.3.3}/rust/tests/fixtures/nlp_golden.tsv +0 -0
  22. {tablassert-22.3.1 → tablassert-22.3.3}/rust/tests/nlp_golden.rs +0 -0
  23. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/__init__.py +0 -0
  24. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/_lazy.py +0 -0
  25. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/cli_entry.py +0 -0
  26. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/coerce.py +0 -0
  27. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/data/experimental_taxa.yaml +0 -0
  28. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/data/infores_catalog.yaml +0 -0
  29. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/enums.py +0 -0
  30. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/errors.py +0 -0
  31. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/extras.py +0 -0
  32. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/fullmap.py +0 -0
  33. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/infores_registry.py +0 -0
  34. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/ingests.py +0 -0
  35. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/lib.py +0 -0
  36. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/log.py +0 -0
  37. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/nlp.py +0 -0
  38. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/progress.py +0 -0
  39. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/qc.py +0 -0
  40. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/rig.py +0 -0
  41. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/rs.pyi +0 -0
  42. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/runcache.py +0 -0
  43. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/study.py +0 -0
  44. {tablassert-22.3.1 → tablassert-22.3.3}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 22.3.1
3
+ Version: 22.3.3
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -23,6 +23,7 @@ Classifier: Operating System :: POSIX :: Linux
23
23
  Classifier: Operating System :: MacOS :: MacOS X
24
24
  Classifier: Environment :: Console
25
25
  Requires-Dist: biolink-model>=4.4.6
26
+ Requires-Dist: linkml-runtime>=1.11
26
27
  Requires-Dist: polars>=2.0.0rc2
27
28
  Requires-Dist: pyarrow>=21.0.0
28
29
  Requires-Dist: rapidfuzz>=3.14.3
@@ -73,20 +74,21 @@ configuration reference, and API docs.
73
74
 
74
75
  ## Statement of need
75
76
 
76
- Biomedical knowledge lives in spreadsheets: association tables, assay results, curated gene-disease
77
- lists. Getting those rows into an NCATS Translator knowledge graph means writing an ingest that maps
78
- columns to Biolink statements, resolves free text ("TP53", "lung cancer") to standard CURIEs, and
79
- records provenance, terms of use, and statistical annotations. Today that ingest is Python code
80
- maintained per source, and entity resolution usually means calling a hosted name-resolution service
81
- at build time.
82
-
83
- Tablassert replaces the per-source code with a declarative YAML mapping plus an embedded, offline
84
- entity-resolution database built from RENCI BABEL exports, so a build is reproducible, auditable,
85
- and network-free. It is aimed at Translator ingest authors and knowledge-graph data engineers who
86
- hold tabular biomedical sources, and at bioinformatics groups that want a validated KGX output
87
- without writing a transform pipeline. The configuration model and the Biolink/KGX contracts it
88
- emits follow current [`NCATSTranslator/translator-ingests`](https://github.com/NCATSTranslator/translator-ingests)
89
- practice, and each build can emit the Resource Ingest Guide (RIG) metadata those submissions require.
77
+ Biomedical knowledge graphs classically rely on bespoke ingest scripts for each source. When data
78
+ sources lack standardization, such as the millions of tables published in biomedical literature,
79
+ bespoke scripts impose three costs that limit scalability. First, mapping each table's rows,
80
+ columns, and cells to biological assertions is idiosyncratic, making them hard to inspect, review,
81
+ or reuse. Second, entity resolution may depend on hosted network services, making graph builds
82
+ unreproducible. Third, and most importantly, because each source requires a new pipeline, most
83
+ published tables are never ingested and remain neither interoperable nor reusable.
84
+
85
+ Tablassert supports knowledge-graph engineers and bioinformaticians working with tabular biomedical
86
+ sources who need standards-conformant output. Declarative YAML recipes make data-to-statement
87
+ mapping scalable, inspectable, and reusable. Batched resolution against an embedded identifier
88
+ database removes runtime service dependencies. These recipes extend beyond journal supplemental
89
+ tables to any relational source (trial registries, drug approval records, or a researcher's own
90
+ tables). Each build can emit a standardized resource descriptor, like the Resource Ingest Guide
91
+ metadata required for integration with Translator.
90
92
 
91
93
  ## Getting help
92
94
 
@@ -19,20 +19,21 @@ configuration reference, and API docs.
19
19
 
20
20
  ## Statement of need
21
21
 
22
- Biomedical knowledge lives in spreadsheets: association tables, assay results, curated gene-disease
23
- lists. Getting those rows into an NCATS Translator knowledge graph means writing an ingest that maps
24
- columns to Biolink statements, resolves free text ("TP53", "lung cancer") to standard CURIEs, and
25
- records provenance, terms of use, and statistical annotations. Today that ingest is Python code
26
- maintained per source, and entity resolution usually means calling a hosted name-resolution service
27
- at build time.
28
-
29
- Tablassert replaces the per-source code with a declarative YAML mapping plus an embedded, offline
30
- entity-resolution database built from RENCI BABEL exports, so a build is reproducible, auditable,
31
- and network-free. It is aimed at Translator ingest authors and knowledge-graph data engineers who
32
- hold tabular biomedical sources, and at bioinformatics groups that want a validated KGX output
33
- without writing a transform pipeline. The configuration model and the Biolink/KGX contracts it
34
- emits follow current [`NCATSTranslator/translator-ingests`](https://github.com/NCATSTranslator/translator-ingests)
35
- practice, and each build can emit the Resource Ingest Guide (RIG) metadata those submissions require.
22
+ Biomedical knowledge graphs classically rely on bespoke ingest scripts for each source. When data
23
+ sources lack standardization, such as the millions of tables published in biomedical literature,
24
+ bespoke scripts impose three costs that limit scalability. First, mapping each table's rows,
25
+ columns, and cells to biological assertions is idiosyncratic, making them hard to inspect, review,
26
+ or reuse. Second, entity resolution may depend on hosted network services, making graph builds
27
+ unreproducible. Third, and most importantly, because each source requires a new pipeline, most
28
+ published tables are never ingested and remain neither interoperable nor reusable.
29
+
30
+ Tablassert supports knowledge-graph engineers and bioinformaticians working with tabular biomedical
31
+ sources who need standards-conformant output. Declarative YAML recipes make data-to-statement
32
+ mapping scalable, inspectable, and reusable. Batched resolution against an embedded identifier
33
+ database removes runtime service dependencies. These recipes extend beyond journal supplemental
34
+ tables to any relational source (trial registries, drug approval records, or a researcher's own
35
+ tables). Each build can emit a standardized resource descriptor, like the Resource Ingest Guide
36
+ metadata required for integration with Translator.
36
37
 
37
38
  ## Getting help
38
39
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "22.3.1"
3
+ version = "22.3.3"
4
4
  description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -47,6 +47,10 @@ classifiers = [
47
47
  requires-python = ">=3.11"
48
48
  dependencies = [
49
49
  "biolink-model>=4.4.6",
50
+ # Direct runtime import (`biolink._schema` builds a SchemaView); was only transitive
51
+ # via biolink-model. Declared for the same reason openpyxl is in the test group: a
52
+ # biolink-model bump must not be able to silently drop a module this package imports.
53
+ "linkml-runtime>=1.11",
50
54
  "polars>=2.0.0rc2",
51
55
  "pyarrow>=21.0.0",
52
56
  "rapidfuzz>=3.14.3",
@@ -120,6 +124,10 @@ docs = [
120
124
  ci = [
121
125
  { include-group = "test" },
122
126
  { include-group = "typecheck" },
127
+ # Dead-code gates run by the python-type CI job; live in `ci` so the lockfile pins
128
+ # the same versions the pre-commit hooks mirror (see .pre-commit-config.yaml).
129
+ "deptry>=0.25",
130
+ "vulture>=2.16",
123
131
  ]
124
132
  # Superset for local development, so `make setup` still installs everything.
125
133
  dev = [
@@ -185,3 +193,40 @@ reportMissingTypeStubs = false
185
193
  reportUnusedImport = true
186
194
  reportUnusedVariable = true
187
195
  reportDuplicateImport = true
196
+
197
+ [tool.vulture]
198
+ # Whole-repo dead-code gate: ruff/pyright only catch unused imports and locals, while
199
+ # vulture also finds unused functions, methods, classes, attributes and unreachable code.
200
+ # tests/ is scanned as a usage source (code only tests call is not dead), and the
201
+ # whitelist marks names that are used dynamically and cannot be seen statically.
202
+ paths = ["src", "tests", "scripts", "vulture_whitelist.py"]
203
+ # Pydantic validators are called by the framework, pytest fixtures by the runner;
204
+ # neither shows up as a static reference.
205
+ ignore_decorators = ["@field_validator", "@model_validator", "@pytest.fixture"]
206
+ # `model_config` is read by pydantic; `pytestmark` is read by pytest.
207
+ ignore_names = ["model_config", "pytestmark"]
208
+ sort_by_size = true
209
+
210
+ [tool.deptry]
211
+ # Dependency-hygiene gate: catches unused declared dependencies (dead weight) and imports
212
+ # that only work through a transitive dependency (fragile). maturin's `python-source = "src"`
213
+ # layout is not auto-detected, so the package's own module must be declared first-party.
214
+ known_first_party = ["tablassert"]
215
+ # paper/ holds JOSS reproducibility scripts that run in their own environments (koza etc.),
216
+ # not in this project's; tests/ is already excluded by deptry's defaults.
217
+ extend_exclude = ["paper"]
218
+
219
+ [tool.deptry.package_module_name_map]
220
+ # PyPI distribution "aria2" exposes import module "aria2c"; the mapping silences deptry's
221
+ # name-assumption warning. The DEP002 ignore below still applies because the import is dynamic.
222
+ aria2 = "aria2c"
223
+
224
+ [tool.deptry.per_rule_ignores]
225
+ # Never imported by name at module scope, but required at runtime:
226
+ # - polars: loaded through the LazyModule("polars") mechanism (_lazy.py); every static
227
+ # `import polars` in src/ is TYPE_CHECKING-only by design.
228
+ # - aria2: cli.py resolves the bundled binary through import_module("aria2c"); the PyPI
229
+ # distribution "aria2" exposes import module "aria2c" (see the [aria2] extra comment).
230
+ # - sentence-transformers: loaded through LazyModule("sentence_transformers") in qc.py.
231
+ # - pyarrow/fastexcel: polars Excel/IPC engines (pl.read_excel engine="calamine" loads fastexcel).
232
+ DEP002 = ["polars", "aria2", "sentence-transformers", "pyarrow", "fastexcel"]
@@ -463,9 +463,9 @@ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
463
463
 
464
464
  [[package]]
465
465
  name = "rustix"
466
- version = "1.1.4"
466
+ version = "1.1.6"
467
467
  source = "registry+https://github.com/rust-lang/crates.io-index"
468
- checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
468
+ checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa"
469
469
  dependencies = [
470
470
  "bitflags",
471
471
  "errno",
@@ -626,9 +626,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
626
626
 
627
627
  [[package]]
628
628
  name = "uuid"
629
- version = "1.29.1"
629
+ version = "1.29.2"
630
630
  source = "registry+https://github.com/rust-lang/crates.io-index"
631
- checksum = "4a06f08a856b009bc7bbaf7d3e79808595d032dd19074c0c2ef26e49e03e6bdc"
631
+ checksum = "bed99139f7dac9b5cfe5f17be4d66dc7079a34dee1d74bcc77217119240e267c"
632
632
  dependencies = [
633
633
  "js-sys",
634
634
  "md-5",
@@ -713,9 +713,9 @@ dependencies = [
713
713
 
714
714
  [[package]]
715
715
  name = "xxhash-rust"
716
- version = "0.8.19"
716
+ version = "0.8.21"
717
717
  source = "registry+https://github.com/rust-lang/crates.io-index"
718
- checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7"
718
+ checksum = "40de123dbb523ebcc00d61ec01840ad192578afc378d859f3429caa38308fad5"
719
719
 
720
720
  [[package]]
721
721
  name = "zlib-rs"
@@ -731,27 +731,27 @@ checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
731
731
 
732
732
  [[package]]
733
733
  name = "zstd"
734
- version = "0.14.3"
734
+ version = "0.14.4"
735
735
  source = "registry+https://github.com/rust-lang/crates.io-index"
736
- checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70"
736
+ checksum = "ad5020ab8f27bfc0024ad0a07aed6bf022a16208f479686fbed4fdd0c85eb0b3"
737
737
  dependencies = [
738
738
  "zstd-safe",
739
739
  ]
740
740
 
741
741
  [[package]]
742
742
  name = "zstd-safe"
743
- version = "8.2.0"
743
+ version = "8.3.0"
744
744
  source = "registry+https://github.com/rust-lang/crates.io-index"
745
- checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b"
745
+ checksum = "5a9ea9cc2576c2fcc104e9cec04f46e3ede564f4fbc206831e4ef7e859a4140d"
746
746
  dependencies = [
747
747
  "zstd-sys",
748
748
  ]
749
749
 
750
750
  [[package]]
751
751
  name = "zstd-sys"
752
- version = "2.1.2+zstd.1.5.7"
752
+ version = "2.1.3+zstd.1.5.7"
753
753
  source = "registry+https://github.com/rust-lang/crates.io-index"
754
- checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3"
754
+ checksum = "f729a06d0257428fd9d997ac8db7e1407bd0262ab016b085b36bc52957c0052d"
755
755
  dependencies = [
756
756
  "cc",
757
757
  "pkg-config",
@@ -492,7 +492,6 @@ if TYPE_CHECKING:
492
492
  DISEASE: Categories
493
493
  GENE: Categories
494
494
  GENOMIC_ENTITY: Categories
495
- NAMED_THING: Categories
496
495
  PHENOTYPIC_FEATURE: Categories
497
496
  PROTEIN: Categories
498
497
 
@@ -5,7 +5,8 @@ import re
5
5
  import subprocess
6
6
  import sys
7
7
  import time
8
- from collections.abc import Callable
8
+ from collections.abc import Callable, Iterator
9
+ from contextlib import contextmanager
9
10
  from importlib import import_module
10
11
  from importlib.metadata import version as get_version
11
12
  from itertools import chain, pairwise
@@ -35,6 +36,25 @@ else:
35
36
  pl = LazyModule("polars")
36
37
  pydantic = LazyModule("pydantic")
37
38
 
39
+
40
+ @contextmanager
41
+ def _pipeline_pool() -> Iterator[Any]:
42
+ """Drain workers normally; terminate only when a pipeline stage fails.
43
+
44
+ Pool.__exit__ terminates even successful workers, bypassing cleanup of
45
+ worker-local logging queues and leaking named semaphores under forkserver.
46
+ """
47
+ pool = Pool()
48
+ try:
49
+ yield pool
50
+ pool.close()
51
+ pool.join()
52
+ except BaseException:
53
+ pool.terminate()
54
+ pool.join()
55
+ raise
56
+
57
+
38
58
  # Pipeline completion events (BUILD, VALIDATE).
39
59
  logger = cat("PIPELINE")
40
60
  # BABEL downloader events (reuse, restart, done, retry).
@@ -320,7 +340,7 @@ def build_graph_pipeline(
320
340
  # index and we reassemble by index to keep raw[i] aligned with g.tables[i].
321
341
  start, advance, _ = progress.section_loop(len(g.tables), "Load")
322
342
  raw: list[object] = [None for _ in g.tables]
323
- with Pool() as pool:
343
+ with _pipeline_pool() as pool:
324
344
  for idx, parsed in pool.imap_unordered(_load_table_indexed, enumerate(g.tables)):
325
345
  raw[idx] = parsed
326
346
  start(str(g.tables[idx]))
@@ -332,7 +352,7 @@ def build_graph_pipeline(
332
352
  # flattened sections order is byte-identical to the old starmap result.
333
353
  start, advance, _ = progress.section_loop(len(g.tables), "Extract")
334
354
  temp: list[list[dict[str, Any]]] = [[] for _ in g.tables]
335
- with Pool() as pool:
355
+ with _pipeline_pool() as pool:
336
356
  for idx, section_list in pool.imap_unordered(_extract_sections_indexed, zip(range(len(g.tables)), raw, g.tables, strict=True)):
337
357
  temp[idx] = section_list
338
358
  start(str(g.tables[idx]))
@@ -1287,7 +1287,6 @@ class RIGConfig(TablaBase):
1287
1287
  return text.rstrip("/")
1288
1288
 
1289
1289
 
1290
- DEFAULT_RIG_CONTRIBUTIONS: list[str] = ["Tablassert: KGX and RIG generation"]
1291
1290
  DEFAULT_RIG_UI_EXPLANATION: str = (
1292
1291
  "Source Tablassert data provides assertions derived from configured tabular records. "
1293
1292
  "The source record used to create this Translator edge was transformed into a "
@@ -1296,10 +1295,6 @@ DEFAULT_RIG_UI_EXPLANATION: str = (
1296
1295
  )
1297
1296
 
1298
1297
 
1299
- def default_rig_contributions() -> list[str]:
1300
- return DEFAULT_RIG_CONTRIBUTIONS.copy()
1301
-
1302
-
1303
1298
  #: Top-level graph keys that moved under `rig:`. Their presence is a hard error:
1304
1299
  #: silently mapping them would hide which of the two spellings a config means, and a
1305
1300
  #: half-migrated graph would build a RIG missing fields the author thinks they set.
@@ -68,17 +68,6 @@ DEFAULT_RETRY_AFTER_CAP: float = 60.0
68
68
  DEFAULT_TIMEOUT: int = 120
69
69
  """Per-request socket timeout in seconds for :func:`http_get_text`."""
70
70
 
71
- DEFAULT_FETCH_CONCURRENCY: int = 8
72
- """Worker-pool size for one article's file downloads.
73
-
74
- WHY 8: a PMC article ships ~10-15 supplementary files and the fleet's constraint was the LOCAL
75
- resolver, not the server. Eight in-flight requests collapse a 15-file article into two waves of
76
- round trips (instead of fifteen serial ones) without re-creating the DNS stampede that produced
77
- the ``EAI_NONAME`` flood this module exists to absorb: 16 fleet workers x 8 files = 128 lookups
78
- in the same instant is still comfortably below a healthy resolver's capacity, and each lookup is
79
- now retry-hardened anyway.
80
- """
81
-
82
71
  USER_AGENT: str = "tablassert"
83
72
  """``User-Agent`` sent by every request this module makes (unchanged from the pre-seam HTTP calls)."""
84
73
 
File without changes
File without changes
File without changes
File without changes