tablassert 7.4.12__tar.gz → 7.4.14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-7.4.12 → tablassert-7.4.14}/AGENTS.md +1 -1
- {tablassert-7.4.12 → tablassert-7.4.14}/CHANGELOG.md +12 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/PKG-INFO +1 -1
- tablassert-7.4.14/docs/changelog.md +25 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/pyproject.toml +3 -3
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/cli.py +32 -39
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/lib.py +7 -3
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/models.py +0 -42
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/conftest.py +1 -14
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/test_lib.py +50 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/test_models.py +0 -192
- {tablassert-7.4.12 → tablassert-7.4.14}/uv.lock +153 -128
- tablassert-7.4.12/.github/workflows/autotag.yml +0 -26
- tablassert-7.4.12/docs/changelog.md +0 -11
- {tablassert-7.4.12 → tablassert-7.4.14}/.github/workflows/docker.yml +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/.github/workflows/docs.yml +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/.github/workflows/pipy.yml +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/.gitignore +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/.pre-commit-config.yaml +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/CITATION.cff +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/CONTRIBUTING.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/Dockerfile +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/LICENSE +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/README.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/api/fullmap.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/api/lib.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/api/qc.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/api/utils.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/cli.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/configuration/advanced-example.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/configuration/graph.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/configuration/table.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/datassert.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/docker.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/examples/tutorial-data.csv +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/examples/tutorial-graph.yaml +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/examples/tutorial-table.yaml +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/examples.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/index.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/installation.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/docs/tutorial.md +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/llms.txt +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/mkdocs.yml +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/__init__.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/downloader.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/enums.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/fullmap.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/ingests.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/log.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/nlp.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/progress.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/qc.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/src/tablassert/utils.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/__init__.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/fixtures/invalid_section_missing_source.yaml +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/fixtures/minimal_section.yaml +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/fixtures/minimal_section_with_sections.yaml +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/test_downloader.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/test_enums.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/test_fullmap.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/test_ingests.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/test_nlp.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/test_qc.py +0 -0
- {tablassert-7.4.12 → tablassert-7.4.14}/tests/test_utils.py +0 -0
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
- Install with `uv sync`. QC is not available unless you install an extra: `uv sync --extra qc` or `uv sync --extra qc-cuda`.
|
|
7
7
|
- CLI entrypoint is `tablassert.cli:APP`. Real user commands are:
|
|
8
8
|
- `uv run tablassert build <graph.yaml>`
|
|
9
|
-
- `uv run tablassert validate <table.yaml
|
|
9
|
+
- `uv run tablassert validate <table.yaml>`
|
|
10
10
|
|
|
11
11
|
## Verify Changes
|
|
12
12
|
|
|
@@ -2,6 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project are documented in this file.
|
|
4
4
|
|
|
5
|
+
## 7.4.14 - 2026-06-30
|
|
6
|
+
|
|
7
|
+
### Changes
|
|
8
|
+
- Extended `sig()` in `lib.py` to select a p-value column by fuzzy matching rather than requiring an exact `"p value"` name. All schema columns whose names contain the substring `"p value"` are now considered candidates; `fuzz.ratio` (rapidfuzz) scores each against the literal `"p value"` and the highest-scoring column is used to compute the `"significant"` output. An exact `"p value"` column scores 100 and is always preferred; columns like `"adjusted p value"` or `"log p value"` are used only when no exact match is present. If no column contains the substring the function continues to emit `"UNSURE"` for all rows.
|
|
9
|
+
- Added five regression tests in `test_lib.py` covering: exact-match preference, non-exact fallback, closest-match selection among multiple non-exact candidates, no-p-value column (UNSURE), and null value handling.
|
|
10
|
+
|
|
11
|
+
## 7.4.13 - 2026-06-30
|
|
12
|
+
|
|
13
|
+
### Changes
|
|
14
|
+
- Removed the datassert prevalidation failure for unresolved `statement.subject` / `statement.object` literal encodings under `method: value`. Graph builds no longer abort during `Tcode.model_validate(...)` for cases like `"Incertae Sedis XI"`; unresolved literal values are now allowed through config validation so downstream runtime handling can decide whether they map or get filtered.
|
|
15
|
+
- Added a regression test at the `Tcode.model_validate(...)` layer covering an unresolved `method: value` subject encoding, matching the build-time validation path reported in the field.
|
|
16
|
+
|
|
5
17
|
## 7.4.12 - 2026-06-29
|
|
6
18
|
|
|
7
19
|
### Changes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version: 7.4.
|
|
3
|
+
Version: 7.4.14
|
|
4
4
|
Summary: Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in.
|
|
5
5
|
Project-URL: Homepage, https://github.com/SkyeAv/Tablassert
|
|
6
6
|
Project-URL: Source, https://github.com/SkyeAv/Tablassert
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
The canonical release history lives in the repository root at [`CHANGELOG.md`](https://github.com/SkyeAv/Tablassert/blob/main/CHANGELOG.md).
|
|
4
|
+
|
|
5
|
+
## Current Release Notes
|
|
6
|
+
|
|
7
|
+
## 7.4.14 - 2026-06-30
|
|
8
|
+
|
|
9
|
+
### Changes
|
|
10
|
+
|
|
11
|
+
- Extended `sig()` in `lib.py` to select a p-value column by fuzzy matching rather than requiring an exact `"p value"` name. All schema columns whose names contain the substring `"p value"` are now considered candidates; `fuzz.ratio` (rapidfuzz) scores each against the literal `"p value"` and the highest-scoring column is used to compute the `"significant"` output. An exact `"p value"` column scores 100 and is always preferred; columns like `"adjusted p value"` or `"log p value"` are used only when no exact match is present. If no column contains the substring the function continues to emit `"UNSURE"` for all rows.
|
|
12
|
+
- Added five regression tests in `test_lib.py` covering: exact-match preference, non-exact fallback, closest-match selection among multiple non-exact candidates, no-p-value column (UNSURE), and null value handling.
|
|
13
|
+
|
|
14
|
+
## 7.4.13 - 2026-06-30
|
|
15
|
+
|
|
16
|
+
### Changes
|
|
17
|
+
|
|
18
|
+
- Removed the datassert prevalidation failure for unresolved `statement.subject` / `statement.object` literal encodings under `method: value`. Graph builds no longer abort during `Tcode.model_validate(...)` for cases like `"Incertae Sedis XI"`; unresolved literal values are now allowed through config validation so downstream runtime handling can decide whether they map or get filtered.
|
|
19
|
+
- Added a regression test at the `Tcode.model_validate(...)` layer covering an unresolved `method: value` subject encoding, matching the build-time validation path reported in the field.
|
|
20
|
+
|
|
21
|
+
## 7.4.12 - 2026-06-29
|
|
22
|
+
|
|
23
|
+
### Changes
|
|
24
|
+
|
|
25
|
+
- Expanded the placeholder-term filter regex in `distinct()` (`fullmap.py`) to drop additional non-informative terms during entity resolution. The `bad` pattern now also excludes `not applicable`, `p value`, `variable`, `result`, `exposure`, `expression`, and `symbol` alongside the existing `none`, `nan`, `na`, `null`, and `unknown`, preventing these generic column-header-like values from being sent through resolution and producing spurious CURIE mappings.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "7.4.
|
|
3
|
+
version = "7.4.14"
|
|
4
4
|
description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -90,17 +90,17 @@ qc-cuda = [
|
|
|
90
90
|
|
|
91
91
|
[dependency-groups]
|
|
92
92
|
dev = [
|
|
93
|
-
"grayskull>=3.1.0",
|
|
94
93
|
"mkdocs>=1.6.1",
|
|
95
94
|
"pre-commit>=4.5.1",
|
|
96
95
|
"pyright>=1.1.408",
|
|
97
96
|
"pytest>=9.0.2",
|
|
97
|
+
"pytest-cov>=7.1.0",
|
|
98
98
|
"ruff>=0.15.6",
|
|
99
99
|
]
|
|
100
100
|
|
|
101
101
|
[tool.pytest.ini_options]
|
|
102
102
|
testpaths = ["tests"]
|
|
103
|
-
markers = ["network: requires internet", "gpu: requires CUDAExecutionProvider"
|
|
103
|
+
markers = ["network: requires internet", "gpu: requires CUDAExecutionProvider"]
|
|
104
104
|
|
|
105
105
|
[tool.ruff]
|
|
106
106
|
line-length = 120
|
|
@@ -57,30 +57,29 @@ def build_pipeline(graph_configuration_file: Path, progress: "PipelineProgress")
|
|
|
57
57
|
sections: list[dict[str, Any]] = list(chain.from_iterable(temp))
|
|
58
58
|
n: int = len(sections)
|
|
59
59
|
|
|
60
|
+
# * Build TCode (3/6)
|
|
61
|
+
progress.stage(f"Building TCode | Sections: {n}")
|
|
62
|
+
advance = progress.section_loop(n, "TCode")
|
|
63
|
+
tcode: list[Tcode] = []
|
|
64
|
+
for idx, s in enumerate(sections, start=1):
|
|
65
|
+
try:
|
|
66
|
+
tcode.append(
|
|
67
|
+
Tcode.model_validate(
|
|
68
|
+
{**s, "number": idx, "store": (STORE / f"{mkhash(s)}.parquet"), "log": g.log, "qc": g.qc}
|
|
69
|
+
)
|
|
70
|
+
)
|
|
71
|
+
except pydantic.ValidationError as e:
|
|
72
|
+
raise RuntimeError(
|
|
73
|
+
f"02 | FAILED VALIDATION | CONFIG: {graph_configuration_file} | IDX: {idx} | HASH: {mkhash(s)} | PYDANTIC: {flatten_pydantic_error(e)}"
|
|
74
|
+
) from e
|
|
75
|
+
advance(format_section_oneline(tcode[-1]))
|
|
76
|
+
|
|
60
77
|
with ExitStack() as stack:
|
|
61
78
|
conns: list[object] = [
|
|
62
79
|
stack.enter_context(duckdb.connect(g.datassert / "data" / f"{x}.duckdb", read_only=True))
|
|
63
80
|
for x in range(SHARDS)
|
|
64
81
|
]
|
|
65
82
|
|
|
66
|
-
# * Build TCode (3/6)
|
|
67
|
-
progress.stage(f"Building TCode | Sections: {n}")
|
|
68
|
-
advance = progress.section_loop(n, "TCode")
|
|
69
|
-
tcode: list[Tcode] = []
|
|
70
|
-
for idx, s in enumerate(sections, start=1):
|
|
71
|
-
try:
|
|
72
|
-
tcode.append(
|
|
73
|
-
Tcode.model_validate(
|
|
74
|
-
{**s, "number": idx, "store": (STORE / f"{mkhash(s)}.parquet"), "log": g.log, "qc": g.qc},
|
|
75
|
-
context={"conns": conns},
|
|
76
|
-
)
|
|
77
|
-
)
|
|
78
|
-
except pydantic.ValidationError as e:
|
|
79
|
-
raise RuntimeError(
|
|
80
|
-
f"02 | FAILED VALIDATION | CONFIG: {graph_configuration_file} | IDX: {idx} | HASH: {mkhash(s)} | PYDANTIC: {flatten_pydantic_error(e)}"
|
|
81
|
-
) from e
|
|
82
|
-
advance(format_section_oneline(tcode[-1]))
|
|
83
|
-
|
|
84
83
|
# * Collect Instructions (4/6)
|
|
85
84
|
progress.stage(f"Collecting Instructions | Sections: {n}")
|
|
86
85
|
advance = progress.section_loop(n, "Collect")
|
|
@@ -106,9 +105,8 @@ def build_pipeline(graph_configuration_file: Path, progress: "PipelineProgress")
|
|
|
106
105
|
logger.info(f"BUILD DONE | SECTIONS: {n} | NAME: {g.name} | VERSION: {g.version}")
|
|
107
106
|
|
|
108
107
|
|
|
109
|
-
def validate_pipeline(table_configuration_file: Path,
|
|
108
|
+
def validate_pipeline(table_configuration_file: Path, progress: "PipelineProgress") -> None:
|
|
110
109
|
# ? Validate Section Syntax From A Configuration File
|
|
111
|
-
from tablassert.fullmap import SHARDS
|
|
112
110
|
from tablassert.ingests import from_yaml, to_sections
|
|
113
111
|
from tablassert.lib import Tcode
|
|
114
112
|
from tablassert.progress import flatten_pydantic_error
|
|
@@ -126,32 +124,27 @@ def validate_pipeline(table_configuration_file: Path, datassert: Path, progress:
|
|
|
126
124
|
# * Validate Section Syntax (3/3)
|
|
127
125
|
progress.stage(f"Validating Section Syntax | Sections: {n}")
|
|
128
126
|
advance = progress.section_loop(n, "Validate")
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
except pydantic.ValidationError as e:
|
|
139
|
-
raise RuntimeError(
|
|
140
|
-
f"02 | FAILED VALIDATION | CONFIG: {table_configuration_file} | IDX: {idx} | HASH: {h} | PYDANTIC: {flatten_pydantic_error(e)}"
|
|
141
|
-
) from e
|
|
142
|
-
advance(f"#{idx} | HASH: {h}")
|
|
127
|
+
for idx, s in enumerate(sections, start=1):
|
|
128
|
+
h: str = mkhash(s)
|
|
129
|
+
try:
|
|
130
|
+
Tcode.model_validate({**s, "number": idx, "store": (STORE / f"{h}.parquet")})
|
|
131
|
+
except pydantic.ValidationError as e:
|
|
132
|
+
raise RuntimeError(
|
|
133
|
+
f"02 | FAILED VALIDATION | CONFIG: {table_configuration_file} | IDX: {idx} | HASH: {h} | PYDANTIC: {flatten_pydantic_error(e)}"
|
|
134
|
+
) from e
|
|
135
|
+
advance(f"#{idx} | HASH: {h}")
|
|
143
136
|
|
|
144
137
|
logger.info(f"VALIDATE DONE | SECTIONS: {n} | CONFIG: {table_configuration_file.name}")
|
|
145
138
|
|
|
146
139
|
|
|
147
|
-
def run(stages: int, fn: Any,
|
|
140
|
+
def run(stages: int, fn: Any, arg: Path) -> None:
|
|
148
141
|
from tablassert.log import LOG_FORMAT, logger
|
|
149
142
|
from tablassert.progress import PipelineProgress
|
|
150
143
|
|
|
151
144
|
with PipelineProgress(total_stages=stages) as progress:
|
|
152
145
|
sink_id: int = logger.add(progress.log_sink, level="INFO", format=LOG_FORMAT)
|
|
153
146
|
try:
|
|
154
|
-
fn(
|
|
147
|
+
fn(arg, progress)
|
|
155
148
|
finally:
|
|
156
149
|
logger.remove(sink_id)
|
|
157
150
|
|
|
@@ -163,6 +156,6 @@ def build(graph_configuration_file: Path) -> None:
|
|
|
163
156
|
|
|
164
157
|
|
|
165
158
|
@APP.command
|
|
166
|
-
def validate(table_configuration_file: Path
|
|
167
|
-
"""Validate section syntax from a YAML configuration file
|
|
168
|
-
run(3, validate_pipeline, table_configuration_file
|
|
159
|
+
def validate(table_configuration_file: Path) -> None:
|
|
160
|
+
"""Validate section syntax from a YAML configuration file."""
|
|
161
|
+
run(3, validate_pipeline, table_configuration_file)
|
|
@@ -101,14 +101,18 @@ def sig(
|
|
|
101
101
|
out: str = "significant",
|
|
102
102
|
) -> pl.LazyFrame:
|
|
103
103
|
# ? Creates The "significant" Column
|
|
104
|
-
|
|
105
|
-
|
|
104
|
+
from rapidfuzz import fuzz
|
|
105
|
+
|
|
106
|
+
names: list[str] = lf.collect_schema().names()
|
|
107
|
+
candidates: list[str] = [c for c in names if col in c]
|
|
108
|
+
chosen: Optional[str] = max(candidates, key=lambda c: fuzz.ratio(c, col)) if candidates else None
|
|
109
|
+
if chosen is not None:
|
|
110
|
+
expr: pl.Expr = pl.col(chosen).cast(pl.Float64, strict=False)
|
|
106
111
|
cond: pl.Expr = le(expr, cutoff)
|
|
107
112
|
cutoff: pl.Expr = (
|
|
108
113
|
pl.when(expr.is_null()).then(pl.lit("UNSURE")).when(cond).then(pl.lit("YES")).otherwise(pl.lit("NO"))
|
|
109
114
|
)
|
|
110
115
|
return lf.with_columns(cutoff.alias(out))
|
|
111
|
-
|
|
112
116
|
else:
|
|
113
117
|
return lf.with_columns(pl.lit("UNSURE").alias(out))
|
|
114
118
|
|
|
@@ -25,9 +25,6 @@ from tablassert.enums import (
|
|
|
25
25
|
Tokens,
|
|
26
26
|
)
|
|
27
27
|
|
|
28
|
-
from tablassert.fullmap import resolve
|
|
29
|
-
from tablassert.nlp import level_one, level_two
|
|
30
|
-
|
|
31
28
|
if TYPE_CHECKING:
|
|
32
29
|
import httpx
|
|
33
30
|
import polars as pl
|
|
@@ -318,35 +315,6 @@ class Annotation(Encoding):
|
|
|
318
315
|
return annotation.replace("_", " ").strip()
|
|
319
316
|
|
|
320
317
|
|
|
321
|
-
def resolves_value_encodings(statement: Statement, conns: list[object]) -> None:
|
|
322
|
-
# ? Resolve Every Value-Method Literal Against The Shared Datassert Shards
|
|
323
|
-
nodes: list[tuple[str, NodeEncoding]] = [("subject", statement.subject), ("object", statement.object)]
|
|
324
|
-
if statement.qualifiers:
|
|
325
|
-
nodes += [(q.qualifier, q) for q in statement.qualifiers]
|
|
326
|
-
|
|
327
|
-
for label, node in nodes:
|
|
328
|
-
if not eq(node.method, EncodingMethods.VALUE):
|
|
329
|
-
continue
|
|
330
|
-
|
|
331
|
-
term: str = str(node.encoding)
|
|
332
|
-
lf: pl.LazyFrame = pl.DataFrame({"term": [term]}).lazy()
|
|
333
|
-
lf = level_one(lf, "term")
|
|
334
|
-
lf = level_two(lf, "term")
|
|
335
|
-
resolved: pl.DataFrame = resolve(
|
|
336
|
-
lf,
|
|
337
|
-
"term",
|
|
338
|
-
conns,
|
|
339
|
-
taxon=str(node.taxon) if node.taxon else None,
|
|
340
|
-
prioritize=node.prioritize,
|
|
341
|
-
avoid=node.avoid,
|
|
342
|
-
log=False,
|
|
343
|
-
column_context=False,
|
|
344
|
-
).collect()
|
|
345
|
-
if resolved.height == 0:
|
|
346
|
-
msg: str = f"21 | value encoding {term!r} in {label!r} did not resolve against datassert"
|
|
347
|
-
raise ValueError(msg)
|
|
348
|
-
|
|
349
|
-
|
|
350
318
|
class Section(TablaBase):
|
|
351
319
|
# ? Pydantic "Section" Model And Coercion
|
|
352
320
|
syntax: Syntaxes = Field(Syntaxes.TC3, description="Section configuration syntax version.")
|
|
@@ -358,16 +326,6 @@ class Section(TablaBase):
|
|
|
358
326
|
None, description="Optional extra encoded columns added to each row."
|
|
359
327
|
)
|
|
360
328
|
|
|
361
|
-
@field_validator("statement", mode="after")
|
|
362
|
-
@classmethod
|
|
363
|
-
def value_encodings_resolve(cls, statement: Statement, info: Any) -> Statement:
|
|
364
|
-
# ? Ensure Value-Method Encodings Resolve Against The Shared Datassert Shards
|
|
365
|
-
conns: Optional[list[object]] = info.context.get("conns") if info.context else None
|
|
366
|
-
if conns is None:
|
|
367
|
-
return statement # * skip without shared connections (contextless path)
|
|
368
|
-
resolves_value_encodings(statement, conns)
|
|
369
|
-
return statement
|
|
370
|
-
|
|
371
329
|
|
|
372
330
|
class Graph(TablaBase):
|
|
373
331
|
# ? Pydantic "Graph" Configuration
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import os
|
|
4
3
|
from pathlib import Path
|
|
5
|
-
from typing import Any
|
|
4
|
+
from typing import Any
|
|
6
5
|
|
|
7
6
|
import httpx
|
|
8
7
|
import pytest
|
|
@@ -27,15 +26,3 @@ def mockhttpxhead(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
|
27
26
|
@pytest.fixture
|
|
28
27
|
def fixtures_path() -> Path:
|
|
29
28
|
return Path(__file__).parent / "fixtures"
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
@pytest.fixture
|
|
33
|
-
def datassert_dir() -> Path:
|
|
34
|
-
# ? Shared Datassert Shard Directory (Skipped When Unavailable)
|
|
35
|
-
env: Optional[str] = os.environ.get("DATASSERT")
|
|
36
|
-
if not env:
|
|
37
|
-
pytest.skip("DATASSERT env var not set; skipping datassert-dependent test")
|
|
38
|
-
directory: Path = Path(env)
|
|
39
|
-
if not (directory / "data" / "0.duckdb").is_file():
|
|
40
|
-
pytest.skip(f"datassert shard data/0.duckdb not found under {directory}")
|
|
41
|
-
return directory
|
|
@@ -138,6 +138,20 @@ def test_tcode_collect_threads_download_context(fixtures_path: Path) -> None:
|
|
|
138
138
|
assert first_op[1] == ("https://example.com/test.tsv", Path("test.tsv"), "minimal_section.yaml", "sectionhash")
|
|
139
139
|
|
|
140
140
|
|
|
141
|
+
# ? Tcode Allows Unresolved Value Encodings During Validation
|
|
142
|
+
def test_tcode_model_allows_unresolved_value_encoding(fixtures_path: Path) -> None:
|
|
143
|
+
data: Any = from_yaml(fixtures_path / "minimal_section.yaml")
|
|
144
|
+
store: Path = Path("/tmp/sectionhash.parquet")
|
|
145
|
+
data["statement"]["subject"] = {"method": "value", "encoding": "Incertae Sedis XI"}
|
|
146
|
+
|
|
147
|
+
tcode_model: Tcode = Tcode.model_validate( # pyright: ignore
|
|
148
|
+
{**data, "number": 55, "config": fixtures_path / "minimal_section.yaml", "store": store}
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
assert tcode_model.statement.subject.method == "value"
|
|
152
|
+
assert tcode_model.statement.subject.encoding == "Incertae Sedis XI"
|
|
153
|
+
|
|
154
|
+
|
|
141
155
|
# ? Tcode collect Enables QC Logging By Default
|
|
142
156
|
def test_tcode_collect_skips_qc_by_default(fixtures_path: Path) -> None:
|
|
143
157
|
data: Any = from_yaml(fixtures_path / "minimal_section.yaml")
|
|
@@ -253,3 +267,39 @@ def test_resolve_many_runs_qc(monkeypatch: Any, tmp_path: Path) -> None:
|
|
|
253
267
|
|
|
254
268
|
assert result == [{"subject": "brca1", "original subject": "BRCA1", "subject two": "brca1", "passed": "YES"}]
|
|
255
269
|
assert ("qc", "subject", "", "", "passed", True, None) in calls
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
# ? sig Uses Exact "p value" Column When Present Alongside Other P-Value Columns
|
|
273
|
+
def test_sig_prefers_exact_p_value_column() -> None:
|
|
274
|
+
lf: pl.LazyFrame = pl.DataFrame({"p value": [0.01, 0.1], "adjusted p value": [0.5, 0.5]}).lazy()
|
|
275
|
+
result: pl.DataFrame = lib.sig(lf).collect()
|
|
276
|
+
assert list(result["significant"]) == ["YES", "NO"]
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
# ? sig Falls Back To Non-Exact P-Value Column When No Exact Match
|
|
280
|
+
def test_sig_uses_non_exact_p_value_column() -> None:
|
|
281
|
+
lf: pl.LazyFrame = pl.DataFrame({"adjusted p value": [0.01, 0.1]}).lazy()
|
|
282
|
+
result: pl.DataFrame = lib.sig(lf).collect()
|
|
283
|
+
assert list(result["significant"]) == ["YES", "NO"]
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
# ? sig Picks Closest Match When Multiple Non-Exact Columns Present
|
|
287
|
+
def test_sig_picks_closest_non_exact_match() -> None:
|
|
288
|
+
lf: pl.LazyFrame = pl.DataFrame({"log p value": [0.01], "adjusted p value corrected": [0.5]}).lazy()
|
|
289
|
+
result: pl.DataFrame = lib.sig(lf).collect()
|
|
290
|
+
# "log p value" has higher fuzz.ratio to "p value" than "adjusted p value corrected"
|
|
291
|
+
assert list(result["significant"]) == ["YES"]
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
# ? sig Returns UNSURE When No P-Value Column Exists
|
|
295
|
+
def test_sig_returns_unsure_with_no_p_value_column() -> None:
|
|
296
|
+
lf: pl.LazyFrame = pl.DataFrame({"gene": ["BRCA1"]}).lazy()
|
|
297
|
+
result: pl.DataFrame = lib.sig(lf).collect()
|
|
298
|
+
assert list(result["significant"]) == ["UNSURE"]
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
# ? sig Marks Null P-Values As UNSURE
|
|
302
|
+
def test_sig_marks_null_as_unsure() -> None:
|
|
303
|
+
lf: pl.LazyFrame = pl.DataFrame({"p value": [None, 0.01, 0.1]}).lazy()
|
|
304
|
+
result: pl.DataFrame = lib.sig(lf).collect()
|
|
305
|
+
assert list(result["significant"]) == ["UNSURE", "YES", "NO"]
|
|
@@ -3,11 +3,9 @@ from __future__ import annotations
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
from typing import Any
|
|
5
5
|
|
|
6
|
-
import polars as pl
|
|
7
6
|
import pytest
|
|
8
7
|
from pydantic import ValidationError
|
|
9
8
|
|
|
10
|
-
import tablassert.models as models
|
|
11
9
|
from tablassert.enums import Categories
|
|
12
10
|
from tablassert.ingests import from_yaml
|
|
13
11
|
from tablassert.models import (
|
|
@@ -282,193 +280,3 @@ def test_section_with_annotations() -> None:
|
|
|
282
280
|
],
|
|
283
281
|
)
|
|
284
282
|
assert len(section.annotations) == 2 # pyright: ignore
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
# ? Value Encoding Resolves Against Datassert (Context-Aware Pass)
|
|
288
|
-
def test_value_encoding_resolves_pass(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
289
|
-
def fake_resolve(_lf: Any, _col: str, _conns: list[object], **_kwargs: Any) -> Any:
|
|
290
|
-
return pl.DataFrame({"resolved": ["YES"]}).lazy()
|
|
291
|
-
|
|
292
|
-
monkeypatch.setattr(models, "resolve", fake_resolve)
|
|
293
|
-
section: Section = Section.model_validate(
|
|
294
|
-
{
|
|
295
|
-
"source": {"local": "./t.tsv", "url": "https://example.com/t.tsv", "kind": "text"},
|
|
296
|
-
"statement": {
|
|
297
|
-
"subject": {"method": "value", "encoding": "BRCA1"},
|
|
298
|
-
"object": {"method": "value", "encoding": "TP53"},
|
|
299
|
-
},
|
|
300
|
-
"provenance": {
|
|
301
|
-
"repo": "PMC",
|
|
302
|
-
"publication": "PMC000",
|
|
303
|
-
"contributors": [{"kind": "curation", "name": "T", "date": "2025"}],
|
|
304
|
-
},
|
|
305
|
-
},
|
|
306
|
-
context={"conns": [object()]},
|
|
307
|
-
)
|
|
308
|
-
assert section.statement.subject.encoding == "BRCA1"
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
# ? Value Encoding Fails To Resolve Raises Code 21
|
|
312
|
-
def test_value_encoding_resolves_fail(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
313
|
-
def fake_resolve_empty(_lf: Any, _col: str, _conns: list[object], **_kwargs: Any) -> Any:
|
|
314
|
-
return pl.DataFrame({"resolved": []}).lazy()
|
|
315
|
-
|
|
316
|
-
monkeypatch.setattr(models, "resolve", fake_resolve_empty)
|
|
317
|
-
with pytest.raises(ValidationError) as exc_info:
|
|
318
|
-
Section.model_validate(
|
|
319
|
-
{
|
|
320
|
-
"source": {"local": "./t.tsv", "url": "https://example.com/t.tsv", "kind": "text"},
|
|
321
|
-
"statement": {
|
|
322
|
-
"subject": {"method": "value", "encoding": "BRCA1"},
|
|
323
|
-
"object": {"method": "value", "encoding": "TP53"},
|
|
324
|
-
},
|
|
325
|
-
"provenance": {
|
|
326
|
-
"repo": "PMC",
|
|
327
|
-
"publication": "PMC000",
|
|
328
|
-
"contributors": [{"kind": "curation", "name": "T", "date": "2025"}],
|
|
329
|
-
},
|
|
330
|
-
},
|
|
331
|
-
context={"conns": [object()]},
|
|
332
|
-
)
|
|
333
|
-
assert "21 |" in str(exc_info.value)
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
# ? Value Encoding Validator Skips Without Context
|
|
337
|
-
def test_value_encoding_skips_without_context(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
338
|
-
def fake_resolve_empty(_lf: Any, _col: str, _conns: list[object], **_kwargs: Any) -> Any:
|
|
339
|
-
return pl.DataFrame({"resolved": []}).lazy()
|
|
340
|
-
|
|
341
|
-
monkeypatch.setattr(models, "resolve", fake_resolve_empty)
|
|
342
|
-
section: Section = Section( # pyright: ignore
|
|
343
|
-
source={"local": "./t.tsv", "url": "https://example.com/t.tsv", "kind": "text"},
|
|
344
|
-
statement={
|
|
345
|
-
"subject": {"method": "value", "encoding": "BRCA1"},
|
|
346
|
-
"object": {"method": "value", "encoding": "TP53"},
|
|
347
|
-
},
|
|
348
|
-
provenance={
|
|
349
|
-
"repo": "PMC",
|
|
350
|
-
"publication": "PMC000",
|
|
351
|
-
"contributors": [{"kind": "curation", "name": "T", "date": "2025"}],
|
|
352
|
-
},
|
|
353
|
-
)
|
|
354
|
-
assert section.statement.subject.encoding == "BRCA1"
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
# ? Column Method Encodings Are Not Checked
|
|
358
|
-
def test_column_encoding_not_checked(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
359
|
-
def boom_resolve(_lf: Any, _col: str, _conns: list[object], **_kwargs: Any) -> Any:
|
|
360
|
-
raise AssertionError("resolve must not be called for column-method encodings")
|
|
361
|
-
|
|
362
|
-
monkeypatch.setattr(models, "resolve", boom_resolve)
|
|
363
|
-
section: Section = Section.model_validate(
|
|
364
|
-
{
|
|
365
|
-
"source": {"local": "./t.tsv", "url": "https://example.com/t.tsv", "kind": "text"},
|
|
366
|
-
"statement": {
|
|
367
|
-
"subject": {"method": "column", "encoding": "A"},
|
|
368
|
-
"object": {"method": "column", "encoding": "B"},
|
|
369
|
-
},
|
|
370
|
-
"provenance": {
|
|
371
|
-
"repo": "PMC",
|
|
372
|
-
"publication": "PMC000",
|
|
373
|
-
"contributors": [{"kind": "curation", "name": "T", "date": "2025"}],
|
|
374
|
-
},
|
|
375
|
-
},
|
|
376
|
-
context={"conns": [object()]},
|
|
377
|
-
)
|
|
378
|
-
assert section.statement.subject.encoding == "A"
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
# ? Qualifier Value Encoding Is Checked Against Datassert
|
|
382
|
-
def test_qualifier_value_encoding_checked(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
383
|
-
def fake_resolve(lf: Any, col: str, _conns: list[object], **_kwargs: Any) -> Any:
|
|
384
|
-
term: str = str(lf.collect().get_column(col).to_list()[0])
|
|
385
|
-
if term in ("brca1", "tp53"):
|
|
386
|
-
return pl.DataFrame({"resolved": ["YES"]}).lazy()
|
|
387
|
-
return pl.DataFrame({"resolved": []}).lazy()
|
|
388
|
-
|
|
389
|
-
monkeypatch.setattr(models, "resolve", fake_resolve)
|
|
390
|
-
with pytest.raises(ValidationError) as exc_info:
|
|
391
|
-
Section.model_validate(
|
|
392
|
-
{
|
|
393
|
-
"source": {"local": "./t.tsv", "url": "https://example.com/t.tsv", "kind": "text"},
|
|
394
|
-
"statement": {
|
|
395
|
-
"subject": {"method": "value", "encoding": "BRCA1"},
|
|
396
|
-
"object": {"method": "value", "encoding": "TP53"},
|
|
397
|
-
"qualifiers": [
|
|
398
|
-
{"qualifier": "disease_context_qualifier", "method": "value", "encoding": "ZZZNOTAREALGENE123"}
|
|
399
|
-
],
|
|
400
|
-
},
|
|
401
|
-
"provenance": {
|
|
402
|
-
"repo": "PMC",
|
|
403
|
-
"publication": "PMC000",
|
|
404
|
-
"contributors": [{"kind": "curation", "name": "T", "date": "2025"}],
|
|
405
|
-
},
|
|
406
|
-
},
|
|
407
|
-
context={"conns": [object()]},
|
|
408
|
-
)
|
|
409
|
-
assert "21 |" in str(exc_info.value)
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
# ? Real Value Encoding Resolves Against The Datassert Shards
|
|
413
|
-
@pytest.mark.datassert
|
|
414
|
-
def test_real_value_encoding_resolves(datassert_dir: Path) -> None:
|
|
415
|
-
from contextlib import ExitStack
|
|
416
|
-
|
|
417
|
-
import duckdb
|
|
418
|
-
|
|
419
|
-
from tablassert.fullmap import SHARDS
|
|
420
|
-
|
|
421
|
-
with ExitStack() as stack:
|
|
422
|
-
conns: list[object] = [
|
|
423
|
-
stack.enter_context(duckdb.connect(datassert_dir / "data" / f"{x}.duckdb", read_only=True))
|
|
424
|
-
for x in range(SHARDS)
|
|
425
|
-
]
|
|
426
|
-
section: Section = Section.model_validate(
|
|
427
|
-
{
|
|
428
|
-
"source": {"local": "./t.tsv", "url": "https://example.com/t.tsv", "kind": "text"},
|
|
429
|
-
"statement": {
|
|
430
|
-
"subject": {"method": "value", "encoding": "BRCA1"},
|
|
431
|
-
"object": {"method": "value", "encoding": "TP53"},
|
|
432
|
-
},
|
|
433
|
-
"provenance": {
|
|
434
|
-
"repo": "PMC",
|
|
435
|
-
"publication": "PMC000",
|
|
436
|
-
"contributors": [{"kind": "curation", "name": "T", "date": "2025"}],
|
|
437
|
-
},
|
|
438
|
-
},
|
|
439
|
-
context={"conns": conns},
|
|
440
|
-
)
|
|
441
|
-
assert section.statement.subject.encoding == "BRCA1"
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
# ? Real Value Encoding Failure Raises Code 21
|
|
445
|
-
@pytest.mark.datassert
|
|
446
|
-
def test_real_value_encoding_fails(datassert_dir: Path) -> None:
|
|
447
|
-
from contextlib import ExitStack
|
|
448
|
-
|
|
449
|
-
import duckdb
|
|
450
|
-
|
|
451
|
-
from tablassert.fullmap import SHARDS
|
|
452
|
-
|
|
453
|
-
with ExitStack() as stack:
|
|
454
|
-
conns: list[object] = [
|
|
455
|
-
stack.enter_context(duckdb.connect(datassert_dir / "data" / f"{x}.duckdb", read_only=True))
|
|
456
|
-
for x in range(SHARDS)
|
|
457
|
-
]
|
|
458
|
-
with pytest.raises(ValidationError) as exc_info:
|
|
459
|
-
Section.model_validate(
|
|
460
|
-
{
|
|
461
|
-
"source": {"local": "./t.tsv", "url": "https://example.com/t.tsv", "kind": "text"},
|
|
462
|
-
"statement": {
|
|
463
|
-
"subject": {"method": "value", "encoding": "ZZZNOTAREALGENE123"},
|
|
464
|
-
"object": {"method": "value", "encoding": "TP53"},
|
|
465
|
-
},
|
|
466
|
-
"provenance": {
|
|
467
|
-
"repo": "PMC",
|
|
468
|
-
"publication": "PMC000",
|
|
469
|
-
"contributors": [{"kind": "curation", "name": "T", "date": "2025"}],
|
|
470
|
-
},
|
|
471
|
-
},
|
|
472
|
-
context={"conns": conns},
|
|
473
|
-
)
|
|
474
|
-
assert "21 |" in str(exc_info.value)
|