tablassert 10.0.0__tar.gz → 10.1.0__tar.gz

{tablassert-10.0.0 → tablassert-10.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tablassert
-Version: 10.0.0
+Version: 10.1.0
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Science/Research

{tablassert-10.0.0 → tablassert-10.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tablassert"
-version = "10.0.0"
+version = "10.1.0"
 description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
 authors = [
     { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }

{tablassert-10.0.0 → tablassert-10.1.0}/src/tablassert/cli.py

@@ -121,13 +121,15 @@ def build_pipeline(
     """Build a knowledge graph from a YAML configuration file.
 
     Runs the six-stage build pipeline: load tables → extract sections → build
-    Tcodes → collect instructions → build subgraphs → compile graph.
+    Tcodes → collect instructions → build subgraphs → compile graph. With ``qc``
+    enabled a seventh stage studies the final NDJSON files.
 
     Args:
         configuration_file: Path to the graph YAML file.
         progress: Pipeline progress reporter.
         release: When ``True``, emit release-mode artifacts.
-        qc: When ``True``, run quality-control audits on each section.
+        qc: When ``True``, run quality-control audits on each section and assert
+            over the final NDJSON files (failing the build on any violation).
         log: When ``True``, enable per-section verbose logging.
         head: When ``True``, preview a random sample of up to 5 rows per section (fast schema/shape check).
 
@@ -218,9 +220,35 @@ def build_pipeline(
         subgraphs, g.name, g.version, g.description, g.contributions, g.ui_explanation, g.tables, g.infores, on_phase=sub_step, on_subgraph=advance
     )
 
+    # Stage 7/7 (only with --qc): assert over the final NDJSON files.
+    if qc:
+        progress.stage("Studying Graph")
+        study_final_ndjson(g.name, g.version)
+
     logger.info("Built graph {name} v{version}: {n} sections", name=g.name, version=g.version, n=n)
 
 
+def study_final_ndjson(name: str, version: str) -> None:
+    """Run study assertions over a build's final NDJSON files (the ``--qc`` stage 7).
+
+    Args:
+        name: Graph name, used to locate ``./<name>_<version>.nodes.ndjson``.
+        version: Graph version, used to locate ``./<name>_<version>.edges.ndjson``.
+
+    Raises:
+        SystemExit: With status 1 when any study assertion is violated.
+    """
+    from tablassert.study import format_violations, study_kgx
+
+    violations = study_kgx(Path(f"./{name}_{version}.nodes.ndjson"), Path(f"./{name}_{version}.edges.ndjson"))
+    if violations:
+        summary: str = format_violations(violations)
+        print(summary, file=sys.stderr)
+        logger.warning("study assertions failed on final NDJSON:\n{summary}", summary=summary)
+        raise SystemExit(1)
+    logger.info("study assertions passed on final NDJSON")
+
+
 def validate_pipeline(table_configuration_file: Path, progress: PipelineProgress) -> None:
     """Validate section syntax from a YAML configuration file.
 
@@ -599,11 +627,14 @@ def build_kg(
 
     ``--qc`` requires the ``[qc]`` extra (``pip install "tablassert[qc]"``); it is
     checked before the build starts, because the audit stage runs LAST and a missing
-    extra would otherwise surface only after entity resolution has finished.
+    extra would otherwise surface only after entity resolution has finished. It also
+    runs a final study stage that asserts over the emitted NDJSON -- no duplicate node
+    ids, no undeclared or isolated nodes, no malformed lines or stray whitespace --
+    and fails the build (non-zero exit) when any assertion is violated.
     """
     if qc:
         extras.require("qc", required_by="--qc")
-    run(6, build_pipeline, graph_configuration_file, release=release, qc=qc, log=log, head=head)
+    run(7 if qc else 6, build_pipeline, graph_configuration_file, release=release, qc=qc, log=log, head=head)
 
 
 @APP.command(name="validate")
@@ -950,7 +981,7 @@ def _prebuilt_fullmap_urls(babel_version: str) -> tuple[str, str]:
 
     RENCI publishes a prebuilt ``fullmap.tar.zst`` (and a ``sha256sum.txt``) under
     ``{BABEL_BASE}/{babel_version}/fullmap/{tablassert_version}/``, where the version
-    directory is the INSTALLED Tablassert package version (e.g. ``10.0.0``) — resolved from
+    directory is the INSTALLED Tablassert package version (e.g. ``10.1.0``) — resolved from
     installed-package metadata, never hardcoded, so a new release looks itself up.
 
     Args:

{tablassert-10.0.0 → tablassert-10.1.0}/src/tablassert/lib.py

@@ -262,8 +262,10 @@ def prune_to_class(lf: pl.LazyFrame) -> pl.LazyFrame:
 
     Categories vary per row within a section, so this masks per row rather than
     dropping columns: values are nulled where the row's class rejects them, and the
-    Rust null-stripper then removes the key entirely. Scalars are wrapped where the
-    class declares the slot multivalued.
+    Rust null-stripper then removes the key entirely. Scalars are wrapped into
+    one-element lists when the slot is multivalued on every class that declares it
+    (a column has one dtype, so per-row wrapping is impossible; a slot Biolink
+    declares scalar on some classes and multivalued on others stays scalar).
 
     Args:
         lf: Edges LazyFrame carrying a resolved ``category`` column.
@@ -306,10 +308,18 @@ def prune_to_class(lf: pl.LazyFrame) -> pl.LazyFrame:
             # Preserve what the class refuses rather than deleting it outright; the
             # value is real evidence, it just has no slot on this association class.
             rescued.append(pl.when(ok | text.is_null()).then(None).otherwise(pl.concat_str([pl.lit(f"{col}="), text])))
-        # Biolink makes the same slot multivalued on some classes and scalar on others.
-        listed: dict[str, bool] = {cat: is_multivalued(association_class(cat), col) for cat in categories}
-        if any(listed.values()) and not isinstance(schema[col], pl.List):
-            keep = pl.when(first.replace_strict(listed, default=False)).then(pl.concat_list(keep)).otherwise(keep)
+        # A column has one dtype, so the multivalued wrap must be uniform across rows:
+        # wrap every value into a one-element list when every class declaring the slot
+        # types it multivalued. `is_multivalued` is False for classes that do not declare
+        # the slot at all, so the scan is restricted to declaring classes -- including
+        # them is what produced a spuriously mixed per-row wrap that died in
+        # strict_cast at collect. Rows whose class rejects the slot are already null and
+        # stay null. A hypothetically mixed slot keeps its scalar rather than crashing.
+        declaring: list[str] = [cat for cat in categories if accepts[cat]]
+        listed: dict[str, bool] = {cat: is_multivalued(association_class(cat), col) for cat in declaring}
+        if listed and all(listed.values()) and not isinstance(schema[col], pl.List):
+            # concat_list maps null -> [null]; the when preserves real nulls instead.
+            keep = pl.when(keep.is_null()).then(None).otherwise(pl.concat_list(keep))
         updates.append(keep.alias(col))
     if rescued:
         updates.append(pl.concat_list(rescued).list.drop_nulls().alias(PRUNED_COLUMN))
@@ -632,7 +642,7 @@ def split_list(lf: pl.LazyFrame, col: str, delimiter: str) -> pl.LazyFrame:
 
     A column encoding is scalar by construction, so a multivalued Biolink slot such as
     ``has_evidence`` fed from an aggregated cell would otherwise be emitted as a single
-    joined string -- and ``mask_illegal_edge_fields`` wraps that scalar into a
+    joined string -- and ``prune_to_class`` wraps that scalar into a
     one-element list, so the value survives Biolink validation while consumers iterate a
     single ``"a|b|c"`` blob instead of three ids.
 

{tablassert-10.0.0 → tablassert-10.1.0}/src/tablassert/models.py

@@ -19,6 +19,7 @@ from tablassert.biolink import (
     Predicates,
     Qualifiers,
 )
+from tablassert.coerce import effect_size_target, effect_type_target, pvalue_target, study_size_target
 from tablassert.enums import Comparisons, EncodingMethods, Files, FillMethods, Functions, Repositories, Tokens
 from tablassert.errors import BiolinkRelocationWarning, TablassertErrorCodes, TablassertValidationError
 
@@ -514,18 +515,24 @@ class Annotation(Encoding):
         # is the real problem -- an author asking for `supporting_study_size` has no way to discover
         # that Biolink attaches it to no class and the pipeline rerouted it.
         name: str = str(self.annotation)
-        if name in UNSATISFIABLE_EDGE_FIELDS:
+        # Judge the coerced target, not the raw alias: the clean-phase column coercions rename
+        # statistical aliases to their canonical slot before any relocation runs, so
+        # `adjusted p value` reaches the edge as `adjusted_p_value` and warning on the alias
+        # is a false positive.
+        target: str = pvalue_target(name) or study_size_target(name) or effect_size_target(name) or effect_type_target(name) or name
+        shown: str = f"`{name}` (coerced to `{target}`)" if target != name else f"`{name}`"
+        if target in UNSATISFIABLE_EDGE_FIELDS:
             warnings.warn(
-                f"`{name}` is declared in biolink-model {BIOLINK_VERSION} but attached to no association class, "
+                f"{shown} is declared in biolink-model {BIOLINK_VERSION} but attached to no association class, "
                 "so it cannot be emitted on an edge; its value is routed onto the inlined supporting study "
                 "instead. Use a slot a Biolink association declares (e.g. `p_value`, `adjusted_p_value`) if you "
                 "need it on the edge itself.",
                 BiolinkRelocationWarning,
                 stacklevel=2,
             )
-        elif name not in ALLOWED_EDGE_FIELDS:
+        elif target not in ALLOWED_EDGE_FIELDS:
             warnings.warn(
-                f"`{name}` is not a Biolink association slot, so it is folded into `supporting_text` as a "
+                f"{shown} is not a Biolink association slot, so it is folded into `supporting_text` as a "
                 f'"{name}: <value>" string rather than emitted as its own edge field.',
                 BiolinkRelocationWarning,
                 stacklevel=2,

tablassert-10.1.0/src/tablassert/study.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+import json
+from collections import Counter
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from tablassert.log import cat
+
+logger = cat("QC")
+
+# Human-readable phrasing for each assertion, used by format_violations.
+_MESSAGES: dict[str, str] = {
+    "file-missing": "file not found",
+    "malformed-lines": "empty or malformed JSON lines",
+    "whitespace-values": "values with leading/trailing whitespace",
+    "duplicate-node-ids": "duplicate node ids",
+    "undeclared-nodes": "nodes referenced by edges but not declared in the nodes file",
+    "isolated-nodes": "declared nodes participating in no edge",
+}
+
+
+@dataclass
+class StudyViolation:
+    """One failed study assertion over the final KGX NDJSON files.
+
+    Attributes:
+        check: Assertion key into ``_MESSAGES`` (e.g. ``"duplicate-node-ids"``).
+        label: Which file the violation belongs to (``"nodes"`` or ``"edges"``).
+        count: Exact number of offending records/values.
+        examples: Capped list of example offenders for the stderr summary.
+    """
+
+    check: str
+    label: str
+    count: int
+    examples: list[str] = field(default_factory=list)
+
+
+@dataclass
+class _FileScan:
+    """Accumulated facts from one streamed pass over an NDJSON file."""
+
+    ids: set[str]
+    duplicate_ids: Counter[str]
+    whitespace: Counter[str]
+    malformed: int
+    missing: bool
+    path: Path
+
+
+def _scan_ndjson(path: Path, *, edge: bool) -> _FileScan:
+    """Stream one NDJSON file, collecting the facts the study assertions need.
+
+    Args:
+        path: Path to a ``.nodes.ndjson`` or ``.edges.ndjson`` file.
+        edge: ``True`` to collect referenced ids from ``subject``/``object``;
+            ``False`` to collect declared node ``id``s and track duplicates.
+
+    Returns:
+        A :class:`_FileScan`; ``missing`` is set (and nothing else) when the
+        file does not exist, so a typo'd path can never read as a clean pass.
+    """
+    scan: _FileScan = _FileScan(set(), Counter(), Counter(), 0, not path.is_file(), path)
+    if scan.missing:
+        return scan
+    with path.open(encoding="utf-8") as handle:
+        for line in handle:
+            if not line.strip():
+                scan.malformed += 1
+                continue
+            try:
+                record: object = json.loads(line)
+            except json.JSONDecodeError:
+                scan.malformed += 1
+                continue
+            if not isinstance(record, dict):
+                scan.malformed += 1
+                continue
+            for key, value in record.items():
+                if isinstance(value, str) and value != value.strip():
+                    scan.whitespace[key] += 1
+            if edge:
+                for role in ("subject", "object"):
+                    ident: object = record.get(role)
+                    if isinstance(ident, str):
+                        scan.ids.add(ident)
+            else:
+                ident = record.get("id")
+                if isinstance(ident, str):
+                    # Rust dedup only removes byte-identical lines, so a repeated id
+                    # here means two nodes share an id with different content.
+                    if ident in scan.ids:
+                        scan.duplicate_ids[ident] += 1
+                    scan.ids.add(ident)
+    return scan
+
+
+def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10) -> list[StudyViolation]:
+    """Assert over the final KGX NDJSON files, in the spirit of studyKGtsvs.pl.
+
+    Streams both files once each and checks: duplicate node ids, nodes referenced
+    by edges but never declared (``undeclared``), declared nodes participating in
+    no edge (``isolated``), empty/malformed lines, and string values carrying
+    leading/trailing whitespace. Every check is an assertion -- the caller decides
+    whether violations fail the build.
+
+    Args:
+        nodes_path: Path to ``<name>_<version>.nodes.ndjson``.
+        edges_path: Path to ``<name>_<version>.edges.ndjson``.
+        example_limit: Maximum number of examples retained per violation.
+
+    Returns:
+        A list of :class:`StudyViolation`; empty when every assertion passes.
+    """
+    nodes: _FileScan = _scan_ndjson(nodes_path, edge=False)
+    edges: _FileScan = _scan_ndjson(edges_path, edge=True)
+
+    violations: list[StudyViolation] = []
+    for label, scan in (("nodes", nodes), ("edges", edges)):
+        if scan.missing:
+            violations.append(StudyViolation("file-missing", label, 1, [str(scan.path)]))
+            continue
+        if scan.malformed:
+            violations.append(StudyViolation("malformed-lines", label, scan.malformed))
+        if scan.whitespace:
+            examples: list[str] = [f"{field_name} ({n})" for field_name, n in scan.whitespace.most_common(example_limit)]
+            violations.append(StudyViolation("whitespace-values", label, sum(scan.whitespace.values()), examples))
+    if nodes.duplicate_ids:
+        examples = [ident for ident, _ in nodes.duplicate_ids.most_common(example_limit)]
+        violations.append(StudyViolation("duplicate-node-ids", "nodes", len(nodes.duplicate_ids), examples))
+    if not nodes.missing and not edges.missing:
+        undeclared: list[str] = sorted(edges.ids - nodes.ids)
+        if undeclared:
+            violations.append(StudyViolation("undeclared-nodes", "edges", len(undeclared), undeclared[:example_limit]))
+        isolated: list[str] = sorted(nodes.ids - edges.ids)
+        if isolated:
+            violations.append(StudyViolation("isolated-nodes", "nodes", len(isolated), isolated[:example_limit]))
+    return violations
+
+
+def format_violations(violations: list[StudyViolation]) -> str:
+    """Render violations as a human-readable, one-line-per-assertion summary.
+
+    Args:
+        violations: The failed assertions returned by :func:`study_kgx`.
+
+    Returns:
+        Newline-joined summary lines, e.g.
+        ``nodes: 3 duplicate node ids (e.g. HGNC:5, HGNC:6)``.
+    """
+    lines: list[str] = []
+    for violation in violations:
+        line: str = f"{violation.label}: {violation.count} {_MESSAGES[violation.check]}"
+        if violation.examples:
+            line += f" (e.g. {', '.join(violation.examples)})"
+        lines.append(line)
+    return "\n".join(lines)