tablassert 16.0.0__tar.gz → 16.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-16.0.0 → tablassert-16.1.1}/PKG-INFO +2 -1
- {tablassert-16.0.0 → tablassert-16.1.1}/README.md +1 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/pyproject.toml +1 -1
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/cli.py +13 -3
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/lib.py +39 -2
- {tablassert-16.0.0 → tablassert-16.1.1}/LICENSE +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/rust/Cargo.lock +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/rust/Cargo.toml +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/rust/examples/count_tables.rs +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/rust/src/fullmap.rs +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/rust/src/json.rs +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/rust/src/lib.rs +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/rust/src/ndjson.rs +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/rust/src/uuid.rs +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/rust/tests/build_golden.rs +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/rust/tests/common/mod.rs +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/__init__.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/_lazy.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/agent.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/biolink.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/coerce.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/enums.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/errors.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/extras.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/fullmap.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/graph_target.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/ingests.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/log.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/models.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/nlp.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/progress.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/qc.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/rig.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/rs.pyi +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/study.py +0 -0
- {tablassert-16.0.0 → tablassert-16.1.1}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version: 16.
|
|
3
|
+
Version: 16.1.1
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -60,6 +60,7 @@ Project-URL: Source, https://github.com/SkyeAv/Tablassert
|
|
|
60
60
|
[](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
|
|
61
61
|
[](https://github.com/SkyeAv/Tablassert/blob/main/LICENSE)
|
|
62
62
|
[](https://skyeav.github.io/Tablassert/)
|
|
63
|
+
[](https://github.com/SkyeAv/Tablassert/stargazers)
|
|
63
64
|
|
|
64
65
|
> Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON,
|
|
65
66
|
> declaratively, with entity resolution built in and optional quality control.
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
[](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
|
|
6
6
|
[](https://github.com/SkyeAv/Tablassert/blob/main/LICENSE)
|
|
7
7
|
[](https://skyeav.github.io/Tablassert/)
|
|
8
|
+
[](https://github.com/SkyeAv/Tablassert/stargazers)
|
|
8
9
|
|
|
9
10
|
> Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON,
|
|
10
11
|
> declaratively, with entity resolution built in and optional quality control.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "16.
|
|
3
|
+
version = "16.1.1"
|
|
4
4
|
description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -119,11 +119,12 @@ def build_pipeline(
|
|
|
119
119
|
qc: bool = False,
|
|
120
120
|
log: bool = False,
|
|
121
121
|
head: bool = False,
|
|
122
|
+
no_original: bool = False,
|
|
122
123
|
threads: int | None = None,
|
|
123
124
|
) -> None:
|
|
124
125
|
"""Load a graph YAML and build it through the shared in-process core."""
|
|
125
126
|
graph: Graph = _load_graph(configuration_file)
|
|
126
|
-
build_graph_pipeline(graph, configuration_file, progress, release=release, qc=qc, log=log, head=head, threads=threads)
|
|
127
|
+
build_graph_pipeline(graph, configuration_file, progress, release=release, qc=qc, log=log, head=head, no_original=no_original, threads=threads)
|
|
127
128
|
|
|
128
129
|
|
|
129
130
|
def build_graph_pipeline(
|
|
@@ -134,6 +135,7 @@ def build_graph_pipeline(
|
|
|
134
135
|
qc: bool = False,
|
|
135
136
|
log: bool = False,
|
|
136
137
|
head: bool = False,
|
|
138
|
+
no_original: bool = False,
|
|
137
139
|
threads: int | None = None,
|
|
138
140
|
audit_sources: bool = True,
|
|
139
141
|
) -> None:
|
|
@@ -152,6 +154,8 @@ def build_graph_pipeline(
|
|
|
152
154
|
qc: When ``True``, run quality-control audits and final study assertions.
|
|
153
155
|
log: When ``True``, enable per-section verbose logging.
|
|
154
156
|
head: When ``True``, build a random sample of up to five rows per section.
|
|
157
|
+
no_original: When ``True``, omit the verbatim ``original_*`` source-cell
|
|
158
|
+
copies from the final edge NDJSON.
|
|
155
159
|
threads: Optional worker thread count for the parallel fullmap reads behind
|
|
156
160
|
entity resolution (auto when unset).
|
|
157
161
|
"""
|
|
@@ -258,6 +262,7 @@ def build_graph_pipeline(
|
|
|
258
262
|
section_sources if audit_sources else None,
|
|
259
263
|
on_phase=sub_step,
|
|
260
264
|
on_subgraph=advance,
|
|
265
|
+
no_original=no_original,
|
|
261
266
|
uuid_fields=g.uuid_fields,
|
|
262
267
|
uuid_domain=g.uuid_namespace,
|
|
263
268
|
)
|
|
@@ -663,6 +668,7 @@ def build_kg(
|
|
|
663
668
|
qc: Annotated[bool, cyclopts.Parameter(name=["--qc", "-q"], negative="")] = False,
|
|
664
669
|
log: Annotated[bool, cyclopts.Parameter(name=["--log", "-l"], negative="")] = False,
|
|
665
670
|
head: Annotated[bool, cyclopts.Parameter(name=["--head", "-hd"], negative="")] = False,
|
|
671
|
+
no_original: Annotated[bool, cyclopts.Parameter(name=["--no-original", "-no"], negative="")] = False,
|
|
666
672
|
threads: Annotated[int | None, cyclopts.Parameter(name=["--threads", "-t"])] = None,
|
|
667
673
|
) -> None:
|
|
668
674
|
"""Build a knowledge graph from a YAML configuration file.
|
|
@@ -670,7 +676,9 @@ def build_kg(
|
|
|
670
676
|
The positional config is a Graph YAML that orchestrates one or more table
|
|
671
677
|
configs into a single knowledge-graph build.
|
|
672
678
|
|
|
673
|
-
``--
|
|
679
|
+
``--no-original`` omits the verbatim source-cell copies (``original_subject``,
|
|
680
|
+
``original_object``, and any other ``original_*`` fields) from the final edge
|
|
681
|
+
NDJSON. ``--threads`` sets the worker count for the parallel fullmap reads behind entity
|
|
674
682
|
resolution (auto when unset). ``--qc`` requires the ``[qc]`` extra (``pip install
|
|
675
683
|
"tablassert[qc]"``); it is checked before the build starts, because the audit stage
|
|
676
684
|
runs LAST and a missing extra would otherwise surface only after entity resolution
|
|
@@ -687,7 +695,9 @@ def build_kg(
|
|
|
687
695
|
raise SystemExit(2)
|
|
688
696
|
if qc:
|
|
689
697
|
extras.require("qc", required_by="--qc")
|
|
690
|
-
run(
|
|
698
|
+
run(
|
|
699
|
+
7 if qc else 6, build_pipeline, graph_configuration_file, release=release, qc=qc, log=log, head=head, no_original=no_original, threads=threads
|
|
700
|
+
)
|
|
691
701
|
|
|
692
702
|
|
|
693
703
|
@APP.command(name="validate")
|
|
@@ -1050,6 +1050,29 @@ def drop_zero_effect_size(lf: pl.LazyFrame, col: str = "effect_size") -> pl.Lazy
|
|
|
1050
1050
|
return lf.filter(pl.col(col).is_null() | (pl.col(col).cast(pl.Float64) != 0.0))
|
|
1051
1051
|
|
|
1052
1052
|
|
|
1053
|
+
def drop_low_case_count(lf: pl.LazyFrame, col: str = "case_count", threshold: float = 25.0) -> pl.LazyFrame:
|
|
1054
|
+
"""Drop release-mode ``applied_to_treat`` edges whose case count is below the threshold.
|
|
1055
|
+
|
|
1056
|
+
Args:
|
|
1057
|
+
lf: Source LazyFrame.
|
|
1058
|
+
col: Case-count column name.
|
|
1059
|
+
threshold: Minimum case count required to keep the edge.
|
|
1060
|
+
|
|
1061
|
+
Returns:
|
|
1062
|
+
LazyFrame with low case-count edges removed.
|
|
1063
|
+
|
|
1064
|
+
Notes:
|
|
1065
|
+
Only filters when the case-count column exists; no-op for sections
|
|
1066
|
+
without a ``case_count`` column. Null case counts are kept (no count
|
|
1067
|
+
was detected for that row). Only wired in for ``applied_to_treat``
|
|
1068
|
+
sections at op-construction time, so it never touches other predicates.
|
|
1069
|
+
"""
|
|
1070
|
+
names: list[str] = lf.collect_schema().names()
|
|
1071
|
+
if col not in names:
|
|
1072
|
+
return lf
|
|
1073
|
+
return lf.filter(pl.col(col).is_null() | (pl.col(col).cast(pl.Float64) >= threshold))
|
|
1074
|
+
|
|
1075
|
+
|
|
1053
1076
|
def idxname(col: Any) -> str:
|
|
1054
1077
|
"""Convert Excel-style column letters (e.g. ``"AA"``) to a polars-style ``column_<n>`` name.
|
|
1055
1078
|
|
|
@@ -1215,6 +1238,7 @@ class Tcode(Section):
|
|
|
1215
1238
|
(sig, ()),
|
|
1216
1239
|
(drop_not_significant, ()) if self.release else None,
|
|
1217
1240
|
(drop_zero_effect_size, ()) if self.release else None,
|
|
1241
|
+
(drop_low_case_count, ()) if self.release and self.statement.predicate == "applied_to_treat" else None,
|
|
1218
1242
|
]
|
|
1219
1243
|
|
|
1220
1244
|
def _node_ops(self: Self, db: Path) -> list[Any]:
|
|
@@ -1401,6 +1425,7 @@ PHASE_OF: dict[Callable, str] = {
|
|
|
1401
1425
|
sig: "significance",
|
|
1402
1426
|
drop_not_significant: "significance",
|
|
1403
1427
|
drop_zero_effect_size: "significance",
|
|
1428
|
+
drop_low_case_count: "significance",
|
|
1404
1429
|
format_numeric: "finalize",
|
|
1405
1430
|
to_store: "write",
|
|
1406
1431
|
}
|
|
@@ -1670,7 +1695,11 @@ def fold_unknown_to_supporting_text(lf: pl.LazyFrame) -> pl.LazyFrame:
|
|
|
1670
1695
|
|
|
1671
1696
|
|
|
1672
1697
|
def _collect_subframes(
|
|
1673
|
-
subgraphs: list[Path],
|
|
1698
|
+
subgraphs: list[Path],
|
|
1699
|
+
on_phase: Callable[[str], None] | None = None,
|
|
1700
|
+
on_subgraph: Callable[[], None] | None = None,
|
|
1701
|
+
infores_id: str | None = None,
|
|
1702
|
+
no_original: bool = False,
|
|
1674
1703
|
) -> tuple[list[pl.LazyFrame], list[pl.LazyFrame]]:
|
|
1675
1704
|
"""Scan and normalize subgraph parquets into node/edge subframes.
|
|
1676
1705
|
|
|
@@ -1685,6 +1714,8 @@ def _collect_subframes(
|
|
|
1685
1714
|
on_subgraph: Optional callback fired once after each subgraph is
|
|
1686
1715
|
processed, used to tick the progress bar.
|
|
1687
1716
|
infores_id: Graph-level infores CURIE recorded as node ``provided_by``.
|
|
1717
|
+
no_original: When True, also drop the verbatim ``original_*``
|
|
1718
|
+
source-cell copies from the final edge frames.
|
|
1688
1719
|
|
|
1689
1720
|
Returns:
|
|
1690
1721
|
Tuple of ``(subnodes, subedges)``: per-section node and edge
|
|
@@ -1709,6 +1740,9 @@ def _collect_subframes(
|
|
|
1709
1740
|
subnodes.append(partial)
|
|
1710
1741
|
# Drop internal pre-resolution snapshot columns from final edges.
|
|
1711
1742
|
lf = lf.drop([c for c in lf.collect_schema().names() if c.endswith("_pre_resolution")])
|
|
1743
|
+
# --no-original: drop the verbatim source-cell copies from final edges too.
|
|
1744
|
+
if no_original:
|
|
1745
|
+
lf = lf.drop([c for c in lf.collect_schema().names() if c.startswith("original_")])
|
|
1712
1746
|
lf = fold_unknown_to_supporting_text(lf)
|
|
1713
1747
|
subedges.append(lf)
|
|
1714
1748
|
if on_subgraph is not None:
|
|
@@ -1797,6 +1831,7 @@ def compile_graph(
|
|
|
1797
1831
|
section_sources: list[dict[str, object]] | None = None,
|
|
1798
1832
|
on_phase: Callable[[str], None] | None = None,
|
|
1799
1833
|
on_subgraph: Callable[[], None] | None = None,
|
|
1834
|
+
no_original: bool = False,
|
|
1800
1835
|
uuid_fields: list[str] | None = None,
|
|
1801
1836
|
uuid_domain: str | None = None,
|
|
1802
1837
|
) -> None:
|
|
@@ -1817,6 +1852,8 @@ def compile_graph(
|
|
|
1817
1852
|
``write-edges`` / ``dedup`` / ``rig``), used to drive progress UX.
|
|
1818
1853
|
on_subgraph: Optional callback fired once per processed subgraph,
|
|
1819
1854
|
used to tick the progress bar.
|
|
1855
|
+
no_original: When ``True``, omit the verbatim ``original_*``
|
|
1856
|
+
source-cell copies from the final edge NDJSON.
|
|
1820
1857
|
uuid_fields: Optional edge fields that constitute edge identity
|
|
1821
1858
|
(``Graph.uuid_fields``). ``None`` hashes the whole edge record, so any
|
|
1822
1859
|
change to any field re-mints the id.
|
|
@@ -1853,7 +1890,7 @@ def compile_graph(
|
|
|
1853
1890
|
|
|
1854
1891
|
subnodes: list[pl.LazyFrame]
|
|
1855
1892
|
subedges: list[pl.LazyFrame]
|
|
1856
|
-
subnodes, subedges = _collect_subframes(subgraphs, on_phase, on_subgraph, rig_cfg.source_info.infores_id)
|
|
1893
|
+
subnodes, subedges = _collect_subframes(subgraphs, on_phase, on_subgraph, rig_cfg.source_info.infores_id, no_original)
|
|
1857
1894
|
_write_ndjson(subnodes, subedges, n, e, name, version, rig_cfg, section_sources, on_phase, domain, uuid_fields)
|
|
1858
1895
|
|
|
1859
1896
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|